An odd fix for GCC 7

Summary:
There are two statuses in DoCompactionWork could be hidden.
This path fix this bug.

This seems to be a bug since CompactionFilterV2 is introduced:
https://github.com/facebook/rocksdb/blame/3.6.fb/db/db_impl.cc#L3294

Test Plan: make

Reviewers: nkg-, igor, sdong

Reviewed By: igor, sdong

Subscribers: dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D28989

.gitignore

@@ -31,14 +31,8 @@ coverage/COVERAGE_REPORT
 package/
 .phutil_module_cache
 tags
-
-java/out
 java/*.log
 java/include/org_rocksdb_*.h
-
-.idea/
-*.iml
-
 unity.cc
 java/crossbuild/.vagrant
 .vagrant/

.travis.yml

@@ -14,6 +14,6 @@ before_install:
  - sudo dpkg -i libgflags-dev_2.0-1_amd64.deb
 # Lousy hack to disable use and testing of fallocate, which doesn't behave quite
 # as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
-script: OPT=-DTRAVIS make unity && make clean && OPT=-DTRAVIS make check
+script: OPT=-DTRAVIS make unity  && OPT=-DTRAVIS make check -j8
 notifications:
     email: false

HISTORY.md

@@ -1,38 +1,5 @@
 # Rocksdb Change Log
 
-### 3.9.0 (12/8/2014)
-
-### Public API changes
-* New API to create a checkpoint added. Given a directory name, creates a new
-  database which is an image of the existing database.
-* New API LinkFile added to Env. If you implement your own Env class, an
-  implementation of the API LinkFile will have to be provided.
-* MemTableRep takes MemTableAllocator instead of Arena
-
-### Improvements
-* RocksDBLite library now becomes smaller and will be compiled with -fno-exceptions flag.
-
-## 3.8.0 (11/14/2014)
-
-### Public API changes
-* BackupEngine::NewBackupEngine() was deprecated; please use BackupEngine::Open() from now on.
-* BackupableDB/RestoreBackupableDB have new GarbageCollect() methods, which will clean up files from corrupt and obsolete backups.
-* BackupableDB/RestoreBackupableDB have new GetCorruptedBackups() methods which list corrupt backups.
-
-### Cleanup
-* Bunch of code cleanup, some extra warnings turned on (-Wshadow, -Wshorten-64-to-32, -Wnon-virtual-dtor)
-
-### New features
-* CompactFiles and EventListener, although they are still in experimental state
-* Full ColumnFamily support in RocksJava.
-
-## 3.7.0 (11/6/2014)
-### Public API changes
-* Introduce SetOptions() API to allow adjusting a subset of options dynamically online
-* Introduce 4 new convenient functions for converting Options from string: GetColumnFamilyOptionsFromMap(), GetColumnFamilyOptionsFromString(), GetDBOptionsFromMap(), GetDBOptionsFromString()
-* Remove WriteBatchWithIndex.Delete() overloads using SliceParts
-* When opening a DB, if options.max_background_compactions is larger than the existing low pri pool of options.env, it will enlarge it. Similarly, options.max_background_flushes is larger than the existing high pri pool of options.env, it will enlarge it.
-
 ## 3.6.0 (10/7/2014)
 ### Disk format changes
 * If you're using RocksDB on ARM platforms and you're using default bloom filter, there is a disk format change you need to be aware of. There are three steps you need to do when you convert to new release: 1. turn off filter policy, 2. compact the whole database, 3. turn on filter policy

Makefile

@@ -6,16 +6,8 @@
 
 #-----------------------------------------------
 
-CFLAGS += ${EXTRA_CFLAGS}
-CXXFLAGS += ${EXTRA_CXXFLAGS}
-LDFLAGS += $(EXTRA_LDFLAGS)
-MACHINE ?= $(shell uname -m)
-
 ifneq ($(MAKECMDGOALS),dbg)
-OPT += -O2 -fno-omit-frame-pointer
-ifneq ($(MACHINE),ppc64) # ppc64 doesn't support -momit-leaf-frame-pointer
-OPT += -momit-leaf-frame-pointer
-endif
+OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
 else
 # intentionally left blank
 endif
@@ -43,12 +35,6 @@ else
 OPT += -DNDEBUG
 endif
 
-ifneq ($(filter -DROCKSDB_LITE,$(OPT)),) 
-	# found
-	CFLAGS += -fno-exceptions
-	CXXFLAGS += -fno-exceptions
-endif
-
 # ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc.
 ifdef COMPILE_WITH_ASAN
 	# ASAN compile flags
@@ -84,16 +70,16 @@ install:
 #-------------------------------------------------
 
 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
-CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor
+CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual
 
 LDFLAGS += $(PLATFORM_LDFLAGS)
 
 LIBOBJECTS = $(SOURCES:.cc=.o)
+LIBOBJECTS += $(SOURCESCPP:.cpp=.o)
 MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
-MOCKOBJECTS = $(MOCK_SOURCES:.cc=.o)
 
 TESTUTIL = ./util/testutil.o
-TESTHARNESS = ./util/testharness.o $(TESTUTIL) $(MOCKOBJECTS)
+TESTHARNESS = ./util/testharness.o $(TESTUTIL)
 BENCHHARNESS = ./util/benchharness.o
 VALGRIND_ERROR = 2
 VALGRIND_DIR = build_tools/VALGRIND_LOGS
@@ -129,13 +115,11 @@ TESTS = \
 	log_test \
 	manual_compaction_test \
 	memenv_test \
-	mock_env_test \
 	merge_test \
 	merger_test \
 	redis_test \
 	reduce_levels_test \
 	plain_table_db_test \
-	comparator_db_test \
 	prefix_test \
 	skiplist_test \
 	stringappend_test \
@@ -146,8 +130,6 @@ TESTS = \
 	spatial_db_test \
 	version_edit_test \
 	version_set_test \
-	compaction_picker_test \
-	version_builder_test \
 	file_indexer_test \
 	write_batch_test \
 	write_controller_test\
@@ -160,12 +142,7 @@ TESTS = \
 	cuckoo_table_builder_test \
 	cuckoo_table_reader_test \
 	cuckoo_table_db_test \
-	flush_job_test \
-	wal_manager_test \
-	listener_test \
-	compaction_job_test
-
-SUBSET :=  $(shell echo $(TESTS) |sed s/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/)
+	write_batch_with_index_test
 
 TOOLS = \
         sst_dump \
@@ -253,10 +230,6 @@ check: $(TESTS) ldb
 	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
 	python tools/ldb_test.py
 
-check_some: $(SUBSET) ldb
-	for t in $(SUBSET); do echo "***** Running $$t"; ./$$t || exit 1; done
-	python tools/ldb_test.py
-
 ldb_tests: ldb
 	python tools/ldb_test.py
 
@@ -408,9 +381,6 @@ log_write_bench: util/log_write_bench.o $(LIBOBJECTS) $(TESTHARNESS)
 plain_table_db_test: db/plain_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/plain_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-comparator_db_test: db/comparator_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/comparator_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
 table_reader_bench: table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) -pg
 
@@ -441,15 +411,6 @@ ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
 write_batch_with_index_test: utilities/write_batch_with_index/write_batch_with_index_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) utilities/write_batch_with_index/write_batch_with_index_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
-flush_job_test: db/flush_job_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/flush_job_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
-
-compaction_job_test: db/compaction_job_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/compaction_job_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
-
-wal_manager_test: db/wal_manager_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/wal_manager_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
-
 dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -486,12 +447,6 @@ version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
 version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-compaction_picker_test: db/compaction_picker_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/compaction_picker_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
-version_builder_test: db/version_builder_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/version_builder_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
 file_indexer_test : db/file_indexer_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/file_indexer_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -525,15 +480,6 @@ cuckoo_table_reader_test: table/cuckoo_table_reader_test.o $(LIBOBJECTS) $(TESTH
 cuckoo_table_db_test: db/cuckoo_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/cuckoo_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-listener_test: db/listener_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/listener_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
-thread_list_test: util/thread_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/thread_list_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
-compactor_test: utilities/compaction/compactor_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) utilities/compaction/compactor_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
 options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/options_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -544,9 +490,6 @@ $(MEMENVLIBRARY) : $(MEMENVOBJECTS)
 memenv_test : helpers/memenv/memenv_test.o $(MEMENVOBJECTS) $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) helpers/memenv/memenv_test.o $(MEMENVOBJECTS) $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-mock_env_test : util/mock_env_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/mock_env_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
 manual_compaction_test: util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -584,11 +527,7 @@ ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PAT
 ifeq ($(PLATFORM), OS_MACOSX)
 ROCKSDBJNILIB = librocksdbjni-osx.jnilib
 ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
-ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","")
-	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin
-else
-	JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
-endif
+JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
 endif
 
 libz.a:
@@ -616,10 +555,9 @@ libsnappy.a:
 
 rocksdbjavastatic: libz.a libbz2.a libsnappy.a
 	OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j
-	cd java;$(MAKE) javalib;
+	cd java;$(MAKE) java;
 	rm -f ./java/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(LIBOBJECTS) $(COVERAGEFLAGS) libz.a libbz2.a libsnappy.a
-	cd java;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class HISTORY*.md $(ROCKSDBJNILIB)
 	cd java/javadoc;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) *
 	cd java;jar -cf $(ROCKSDB_SOURCES_JAR) org
@@ -638,7 +576,7 @@ rocksdbjavastaticpublish: rocksdbjavastaticrelease
 
 rocksdbjava:
 	OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j32
-	cd java;$(MAKE) javalib;
+	cd java;$(MAKE) java;
 	rm -f ./java/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(LIBOBJECTS) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
 	cd java;jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class HISTORY*.md $(ROCKSDBJNILIB)

build_tools/build_detect_platform

@@ -48,7 +48,11 @@ COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX"
 # Default to fbcode gcc on internal fb machines
 if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
     FBCODE_BUILD="true"
-    source "$PWD/build_tools/fbcode_config.sh"
+    if [ -z "$USE_CLANG" ]; then
+        source $PWD/build_tools/fbcode_config.sh
+    else
+        source "$PWD/build_tools/fbcode.clang31.sh"
+    fi
 fi
 
 # Delete existing output, if it exists
@@ -163,15 +167,14 @@ DIRS="util db table utilities"
 set -f # temporarily disable globbing so that our patterns arent expanded
 PRUNE_TEST="-name *test*.cc -prune"
 PRUNE_BENCH="-name *bench*.cc -prune"
-PRUNE_MOCK="-name *mock*.cc -prune"
-PORTABLE_FILES=`cd "$ROCKSDB_ROOT"; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o $PRUNE_MOCK -o -name '*.cc' -print | sort | tr "\n" " "`
-MOCK_SOURCES=`cd "$ROCKSDB_ROOT"; find $DIRS -name '*mock*.cc' -print | grep -v "test" | sort | tr "\n" " "`
+PORTABLE_FILES=`cd "$ROCKSDB_ROOT"; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
+PORTABLE_CPP=`cd "$ROCKSDB_ROOT"; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
 set +f # re-enable globbing
 
 # The sources consist of the portable files, plus the platform-specific port
 # file.
 echo "SOURCES=$PORTABLE_FILES $GENERIC_PORT_FILES $PORT_FILES" >> "$OUTPUT"
-echo "MOCK_SOURCES=$MOCK_SOURCES" >> "$OUTPUT"
+echo "SOURCESCPP=$PORTABLE_CPP" >> "$OUTPUT"
 echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> "$OUTPUT"
 
 if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
@@ -179,6 +182,15 @@ if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
     # Also don't need any compilation tests if compiling on fbcode
     true
 else
+    # If -std=c++0x works, use <atomic>.  Otherwise use port_posix.h.
+    $CXX $CFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <atomic>
+      int main() {}
+EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_ATOMIC_PRESENT"
+    fi
+
     # Test whether fallocate is available
     $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
       #include <fcntl.h>
@@ -224,7 +236,7 @@ EOF
       int main() {}
 EOF
     if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
+        COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
     fi
 
@@ -284,14 +296,6 @@ EOF
     fi
 fi
 
-# Test whether -Wshorten-64-to-32 is available
-$CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
-  int main() {}
-EOF
-if [ "$?" = 0 ]; then
-    COMMON_FLAGS="$COMMON_FLAGS -Wshorten-64-to-32"
-fi
-
 # shall we use HDFS?
 
 if test "$USE_HDFS"; then

build_tools/fbcode.clang31.sh

@@ -0,0 +1,74 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile leveldb using
+# fbcode settings.  It uses the latest g++ compiler and also
+# uses jemalloc
+
+TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
+TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
+TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
+TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
+GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1
+
+# location of libgcc
+LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include"
+LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/libs"
+
+# location of glibc
+GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include"
+GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
+
+# location of snappy headers and libraries
+SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
+SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
+
+# location of zlib headers and libraries
+ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
+ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
+GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
+
+# location of bzip headers and libraries
+BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/include"
+BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/lib/libbz2.a"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
+GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=" -msse -msse4.2 "
+
+CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.2/0b7c69d/bin/clang $CLANG_INCLUDES"
+CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.2/0b7c69d/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $GFLAGS_INCLUDE"
+AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
+RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
+
+CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin -nostdlib "
+CFLAGS+=" -nostdinc -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1 "
+CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/x86_64-facebook-linux "
+CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include/c++/4.7.1/backward "
+CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include "
+CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include "
+CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/clang/clang-3.2/0b7c69d/lib/clang/3.2/include "
+CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include/linux "
+CFLAGS+=" -isystem $TOOLCHAIN_LIB_BASE/kernel-headers/kernel-headers-3.2.18_70_fbk11_00129_gc8882d0/da39a3e/include "
+CFLAGS+=" -Wall -Wno-sign-compare -Wno-unused-variable -Winvalid-pch -Wno-deprecated -Woverloaded-virtual"
+CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
+CXXFLAGS="$CFLAGS -nostdinc++"
+
+CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
+
+EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2"
+EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS "
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $GFLAGS_LIBS"
+
+export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED 

build_tools/fbcode.gcc471.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile leveldb using
+# fbcode settings.  It uses the latest g++ compiler and also
+# uses jemalloc
+
+TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
+TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
+TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
+TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
+
+# location of libhdfs libraries
+if test "$USE_HDFS"; then
+  JAVA_HOME="/usr/local/jdk-6u22-64"
+  JINCLUDE="-I$JAVA_HOME/include -I$JAVA_HOME/include/linux"
+  GLIBC_RUNTIME_PATH="/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1"
+  HDFSLIB=" -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64 "
+  HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib "
+  HDFSLIB+=" -ldl -lverify -ljava -ljvm "
+fi
+
+# location of libgcc
+LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/include"
+LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/libs"
+
+# location of glibc
+GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/include"
+GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
+
+# location of snappy headers and libraries
+SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
+SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
+
+# location of zlib headers and libraries
+ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
+ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"
+
+# location of bzip headers and libraries
+BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/include"
+BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/91ddd43/lib/libbz2.a"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
+GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=" -msse -msse4.2 "
+
+CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc"
+CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $GFLAGS_INCLUDE"
+AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
+RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
+
+CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
+CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
+CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
+CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2"
+
+EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS "
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS"
+
+VALGRIND_VER="$TOOLCHAIN_LIB_BASE/valgrind/valgrind-3.8.1/91ddd43/bin/"
+
+export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER

build_tools/fbcode.gcc481.sh

@@ -0,0 +1,86 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile rocksdb using
+# fbcode settings.  It uses the latest g++ compiler and also
+# uses jemalloc
+
+TOOLCHAIN_REV=53dc1fe83f84e9145b9ffb81b81aa7f6a49c87cc
+CENTOS_VERSION=`rpm -q --qf "%{VERSION}" $(rpm -q --whatprovides redhat-release)`
+if [ "$CENTOS_VERSION" = "6" ]; then
+  TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos6-native"
+else
+  TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
+fi
+TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.8.1-glibc-2.17"
+
+# location of libhdfs libraries
+if test "$USE_HDFS"; then
+  JAVA_HOME="/usr/local/jdk-6u22-64"
+  JINCLUDE="-I$JAVA_HOME/include -I$JAVA_HOME/include/linux"
+  GLIBC_RUNTIME_PATH="/usr/local/fbcode/gcc-4.8.1-glibc-2.17"
+  HDFSLIB=" -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64 "
+  HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib "
+  HDFSLIB+=" -ldl -lverify -ljava -ljvm "
+fi
+
+# location of libgcc
+LIBGCC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.8.1/8aac7fc/include"
+LIBGCC_LIBS=" -L $TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.8.1/8aac7fc/libs"
+
+# location of glibc
+GLIBC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/glibc/glibc-2.17/99df8fc/include"
+GLIBC_LIBS=" -L $TOOLCHAIN_LIB_BASE/glibc/glibc-2.17/99df8fc/lib"
+
+# location of snappy headers and libraries
+SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/43d84e2/include"
+SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/43d84e2/lib/libsnappy.a"
+
+# location of zlib headers and libraries
+ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/c3f970a/include"
+ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/c3f970a/lib/libz.a"
+
+# location of bzip headers and libraries
+BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/c3f970a/include"
+BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/c3f970a/lib/libbz2.a"
+
+LZ4_REV=065ec7e38fe83329031f6668c43bef83eff5808b
+LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/include"
+LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/c3f970a/include"
+GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/c3f970a/lib/libgflags.a"
+
+# location of jemalloc
+JEMALLOC_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-3.4.1/4d53c6f/include/"
+JEMALLOC_LIB=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-3.4.1/4d53c6f/lib/libjemalloc.a"
+
+# location of numa
+NUMA_REV=829d10dac0230f99cd7e1778869d2adf3da24b65
+NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/include/"
+NUMA_LIB=" /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libnuma.a"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=" -msse -msse4.2 "
+
+CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/gcc"
+CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
+AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
+RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
+
+CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
+CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
+CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DNUMA"
+
+EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
+EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/675d945/lib/libunwind.a"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS "
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
+
+VALGRIND_VER="$TOOLCHAIN_LIB_BASE/valgrind/valgrind-3.8.1/c3f970a/bin/"
+
+export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE

build_tools/fbcode_config.sh

@@ -35,18 +35,9 @@ if test -z $PIC_BUILD; then
   ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
   CFLAGS+=" -DZLIB"
 
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
   LZ4_INCLUDE=" -I $LZ4_BASE/include/"
   LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
   CFLAGS+=" -DLZ4"
-
-  ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-  CFLAGS+=" -DZSTD"
 fi
 
 # location of gflags headers and libraries
@@ -63,11 +54,6 @@ JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
 JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
 
 if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
   # location of libunwind
   LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
 fi
@@ -87,7 +73,7 @@ export USE_SSE=" -msse -msse4.2 "
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
 
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE"
 
 STDLIBS="-L $GCC_BASE/lib64"
 
@@ -131,7 +117,7 @@ CFLAGS+=" $DEPS_INCLUDE"
 CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"
 
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
 EXEC_LDFLAGS+=" -B$BINUTILS/gold"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
@@ -141,7 +127,7 @@ EXEC_LDFLAGS+=" -ldl"
 
 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
 
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
 
 VALGRIND_VER="$VALGRIND_BASE/bin/"
 

coverage/coverage_test.sh

@@ -11,8 +11,8 @@ fi
 ROOT=".."
 # Fetch right version of gcov
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
-  source $ROOT/build_tools/fbcode_config.sh
-  GCOV=$GCC_BASE/bin/gcov
+  source $ROOT/build_tools/fbcode.gcc471.sh
+  GCOV=$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1/cc6c9dc/bin/gcov
 else
   GCOV=$(which gcov)
 fi

db/builder.cc

@@ -72,13 +72,11 @@ Status BuildTable(const std::string& dbname, Env* env,
         ioptions, internal_comparator, file.get(),
         compression, compression_opts);
 
-    {
-      // the first key is the smallest key
-      Slice key = iter->key();
-      meta->smallest.DecodeFrom(key);
-      meta->smallest_seqno = GetInternalKeySeqno(key);
-      meta->largest_seqno = meta->smallest_seqno;
-    }
+    // the first key is the smallest key
+    Slice key = iter->key();
+    meta->smallest.DecodeFrom(key);
+    meta->smallest_seqno = GetInternalKeySeqno(key);
+    meta->largest_seqno = meta->smallest_seqno;
 
     MergeHelper merge(internal_comparator.user_comparator(),
                       ioptions.merge_operator, ioptions.info_log,

db/c.cc

@@ -56,7 +56,6 @@ using rocksdb::NewBloomFilterPolicy;
 using rocksdb::NewLRUCache;
 using rocksdb::Options;
 using rocksdb::BlockBasedTableOptions;
-using rocksdb::CuckooTableOptions;
 using rocksdb::RandomAccessFile;
 using rocksdb::Range;
 using rocksdb::ReadOptions;
@@ -80,14 +79,10 @@ struct rocksdb_writebatch_t      { WriteBatch        rep; };
 struct rocksdb_snapshot_t        { const Snapshot*   rep; };
 struct rocksdb_flushoptions_t    { FlushOptions      rep; };
 struct rocksdb_fifo_compaction_options_t { CompactionOptionsFIFO rep; };
-struct rocksdb_readoptions_t {
-   ReadOptions rep;
-   Slice upper_bound; // stack variable to set pointer to in ReadOptions
-};
+struct rocksdb_readoptions_t     { ReadOptions       rep; };
 struct rocksdb_writeoptions_t    { WriteOptions      rep; };
 struct rocksdb_options_t         { Options           rep; };
 struct rocksdb_block_based_table_options_t  { BlockBasedTableOptions rep; };
-struct rocksdb_cuckoo_table_options_t  { CuckooTableOptions rep; };
 struct rocksdb_seqfile_t         { SequentialFile*   rep; };
 struct rocksdb_randomfile_t      { RandomAccessFile* rep; };
 struct rocksdb_writablefile_t    { WritableFile*     rep; };
@@ -390,9 +385,11 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
     unsigned char success;
     size_t new_value_len;
     char* tmp_new_value = (*full_merge_)(
-        state_, key.data(), key.size(), existing_value_data, existing_value_len,
-        &operand_pointers[0], &operand_sizes[0], static_cast<int>(n), &success,
-        &new_value_len);
+        state_,
+        key.data(), key.size(),
+        existing_value_data, existing_value_len,
+        &operand_pointers[0], &operand_sizes[0], n,
+        &success, &new_value_len);
     new_value->assign(tmp_new_value, new_value_len);
 
     if (delete_value_ != nullptr) {
@@ -420,7 +417,7 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
     size_t new_value_len;
     char* tmp_new_value = (*partial_merge_)(
         state_, key.data(), key.size(), &operand_pointers[0], &operand_sizes[0],
-        static_cast<int>(operand_count), &success, &new_value_len);
+        operand_count, &success, &new_value_len);
     new_value->assign(tmp_new_value, new_value_len);
 
     if (delete_value_ != nullptr) {
@@ -1126,51 +1123,6 @@ void rocksdb_options_set_block_based_table_factory(
 }
 
 
-rocksdb_cuckoo_table_options_t*
-rocksdb_cuckoo_options_create() {
-  return new rocksdb_cuckoo_table_options_t;
-}
-
-void rocksdb_cuckoo_options_destroy(
-    rocksdb_cuckoo_table_options_t* options) {
-  delete options;
-}
-
-void rocksdb_cuckoo_options_set_hash_ratio(
-    rocksdb_cuckoo_table_options_t* options, double v) {
-  options->rep.hash_table_ratio = v;
-}
-
-void rocksdb_cuckoo_options_set_max_search_depth(
-    rocksdb_cuckoo_table_options_t* options, uint32_t v) {
-  options->rep.max_search_depth = v;
-}
-
-void rocksdb_cuckoo_options_set_cuckoo_block_size(
-    rocksdb_cuckoo_table_options_t* options, uint32_t v) {
-  options->rep.cuckoo_block_size = v;
-}
-
-void rocksdb_cuckoo_options_set_identity_as_first_hash(
-    rocksdb_cuckoo_table_options_t* options, unsigned char v) {
-  options->rep.identity_as_first_hash = v;
-}
-
-void rocksdb_cuckoo_options_set_use_module_hash(
-    rocksdb_cuckoo_table_options_t* options, unsigned char v) {
-  options->rep.use_module_hash = v;
-}
-
-void rocksdb_options_set_cuckoo_table_factory(
-    rocksdb_options_t *opt,
-    rocksdb_cuckoo_table_options_t* table_options) {
-  if (table_options) {
-    opt->rep.table_factory.reset(
-        rocksdb::NewCuckooTableFactory(table_options->rep));
-  }
-}
-
-
 rocksdb_options_t* rocksdb_options_create() {
   return new rocksdb_options_t;
 }
@@ -1264,11 +1216,6 @@ void rocksdb_options_set_info_log_level(
   opt->rep.info_log_level = static_cast<InfoLogLevel>(v);
 }
 
-void rocksdb_options_set_db_write_buffer_size(rocksdb_options_t* opt,
-                                              size_t s) {
-  opt->rep.db_write_buffer_size = s;
-}
-
 void rocksdb_options_set_write_buffer_size(rocksdb_options_t* opt, size_t s) {
   opt->rep.write_buffer_size = s;
 }
@@ -1277,10 +1224,6 @@ void rocksdb_options_set_max_open_files(rocksdb_options_t* opt, int n) {
   opt->rep.max_open_files = n;
 }
 
-void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n) {
-  opt->rep.max_total_wal_size = n;
-}
-
 void rocksdb_options_set_target_file_size_base(
     rocksdb_options_t* opt, uint64_t n) {
   opt->rep.target_file_size_base = n;
@@ -1899,14 +1842,8 @@ void rocksdb_readoptions_set_snapshot(
 void rocksdb_readoptions_set_iterate_upper_bound(
     rocksdb_readoptions_t* opt,
     const char* key, size_t keylen) {
-  if (key == nullptr) {
-    opt->upper_bound = Slice();
-    opt->rep.iterate_upper_bound = nullptr;
-
-  } else {
-    opt->upper_bound = Slice(key, keylen);
-    opt->rep.iterate_upper_bound = &opt->upper_bound;
-  }
+  Slice prefix = Slice(key, keylen);
+  opt->rep.iterate_upper_bound = &prefix;
 }
 
 void rocksdb_readoptions_set_read_tier(
@@ -2104,7 +2041,7 @@ void rocksdb_options_set_min_level_to_compress(rocksdb_options_t* opt, int level
 
 int rocksdb_livefiles_count(
   const rocksdb_livefiles_t* lf) {
-  return static_cast<int>(lf->rep.size());
+  return lf->rep.size();
 }
 
 const char* rocksdb_livefiles_name(

db/c_test.c

@@ -132,7 +132,7 @@ static void CmpDestroy(void* arg) { }
 
 static int CmpCompare(void* arg, const char* a, size_t alen,
                       const char* b, size_t blen) {
-  size_t n = (alen < blen) ? alen : blen;
+  int n = (alen < blen) ? alen : blen;
   int r = memcmp(a, b, n);
   if (r == 0) {
     if (alen < blen) r = -1;
@@ -576,39 +576,37 @@ int main(int argc, char** argv) {
 
   StartPhase("compaction_filter");
   {
-    rocksdb_options_t* options_with_filter = rocksdb_options_create();
-    rocksdb_options_set_create_if_missing(options_with_filter, 1);
+    rocksdb_options_t* options = rocksdb_options_create();
+    rocksdb_options_set_create_if_missing(options, 1);
     rocksdb_compactionfilter_t* cfilter;
     cfilter = rocksdb_compactionfilter_create(NULL, CFilterDestroy,
                                               CFilterFilter, CFilterName);
     // Create new database
     rocksdb_close(db);
-    rocksdb_destroy_db(options_with_filter, dbname, &err);
-    rocksdb_options_set_compaction_filter(options_with_filter, cfilter);
-    db = CheckCompaction(db, options_with_filter, roptions, woptions);
+    rocksdb_destroy_db(options, dbname, &err);
+    rocksdb_options_set_compaction_filter(options, cfilter);
+    db = CheckCompaction(db, options, roptions, woptions);
 
-    rocksdb_options_set_compaction_filter(options_with_filter, NULL);
+    rocksdb_options_set_compaction_filter(options, NULL);
     rocksdb_compactionfilter_destroy(cfilter);
-    rocksdb_options_destroy(options_with_filter);
+    rocksdb_options_destroy(options);
   }
 
   StartPhase("compaction_filter_factory");
   {
-    rocksdb_options_t* options_with_filter_factory = rocksdb_options_create();
-    rocksdb_options_set_create_if_missing(options_with_filter_factory, 1);
+    rocksdb_options_t* options = rocksdb_options_create();
+    rocksdb_options_set_create_if_missing(options, 1);
     rocksdb_compactionfilterfactory_t* factory;
     factory = rocksdb_compactionfilterfactory_create(
         NULL, CFilterFactoryDestroy, CFilterCreate, CFilterFactoryName);
     // Create new database
     rocksdb_close(db);
-    rocksdb_destroy_db(options_with_filter_factory, dbname, &err);
-    rocksdb_options_set_compaction_filter_factory(options_with_filter_factory,
-                                                  factory);
-    db = CheckCompaction(db, options_with_filter_factory, roptions, woptions);
+    rocksdb_destroy_db(options, dbname, &err);
+    rocksdb_options_set_compaction_filter_factory(options, factory);
+    db = CheckCompaction(db, options, roptions, woptions);
 
-    rocksdb_options_set_compaction_filter_factory(
-        options_with_filter_factory, NULL);
-    rocksdb_options_destroy(options_with_filter_factory);
+    rocksdb_options_set_compaction_filter_factory(options, NULL);
+    rocksdb_options_destroy(options);
   }
 
   StartPhase("compaction_filter_v2");
@@ -801,87 +799,8 @@ int main(int argc, char** argv) {
     rocksdb_iter_get_error(iter, &err);
     CheckNoError(err);
     rocksdb_iter_destroy(iter);
-
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
   }
 
-  StartPhase("cuckoo_options");
-  {
-    rocksdb_cuckoo_table_options_t* cuckoo_options;
-    cuckoo_options = rocksdb_cuckoo_options_create();
-    rocksdb_cuckoo_options_set_hash_ratio(cuckoo_options, 0.5);
-    rocksdb_cuckoo_options_set_max_search_depth(cuckoo_options, 200);
-    rocksdb_cuckoo_options_set_cuckoo_block_size(cuckoo_options, 10);
-    rocksdb_cuckoo_options_set_identity_as_first_hash(cuckoo_options, 1);
-    rocksdb_cuckoo_options_set_use_module_hash(cuckoo_options, 0);
-    rocksdb_options_set_cuckoo_table_factory(options, cuckoo_options);
-
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-
-    rocksdb_cuckoo_options_destroy(cuckoo_options);
-  }
-
-  StartPhase("iterate_upper_bound");
-  {
-    // Create new empty database
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
-
-    rocksdb_options_set_prefix_extractor(options, NULL);
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-
-    rocksdb_put(db, woptions, "a",    1, "0",    1, &err); CheckNoError(err);
-    rocksdb_put(db, woptions, "foo",  3, "bar",  3, &err); CheckNoError(err);
-    rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err); CheckNoError(err);
-    rocksdb_put(db, woptions, "g1",   2, "0",    1, &err); CheckNoError(err);
-
-    // testing basic case with no iterate_upper_bound and no prefix_extractor
-    {
-       rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
-       rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
-
-       rocksdb_iter_seek(iter, "foo", 3);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "foo", "bar");
-
-       rocksdb_iter_next(iter);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "foo1", "bar1");
-
-       rocksdb_iter_next(iter);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "g1", "0");
-
-       rocksdb_iter_destroy(iter);
-    }
-
-    // testing iterate_upper_bound and forward iterator
-    // to make sure it stops at bound
-    {
-       // iterate_upper_bound points beyond the last expected entry
-       rocksdb_readoptions_set_iterate_upper_bound(roptions, "foo2", 4);
-
-       rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
-
-       rocksdb_iter_seek(iter, "foo", 3);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "foo", "bar");
-
-       rocksdb_iter_next(iter);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "foo1", "bar1");
-
-       rocksdb_iter_next(iter);
-       // should stop here...
-       CheckCondition(!rocksdb_iter_valid(iter));
-
-       rocksdb_iter_destroy(iter);
-    }
-  }
 
   StartPhase("cleanup");
   rocksdb_close(db);

db/column_family.cc

@@ -19,15 +19,11 @@
 #include <algorithm>
 #include <limits>
 
-#include "db/compaction_picker.h"
 #include "db/db_impl.h"
-#include "db/job_context.h"
 #include "db/version_set.h"
-#include "db/writebuffer.h"
 #include "db/internal_stats.h"
-#include "db/job_context.h"
+#include "db/compaction_picker.h"
 #include "db/table_properties_collector.h"
-#include "db/version_set.h"
 #include "db/write_controller.h"
 #include "util/autovector.h"
 #include "util/hash_skiplist_rep.h"
@@ -65,9 +61,9 @@ uint64_t SlowdownAmount(int n, double bottom, double top) {
 }
 }  // namespace
 
-ColumnFamilyHandleImpl::ColumnFamilyHandleImpl(
-    ColumnFamilyData* column_family_data, DBImpl* db, port::Mutex* mutex)
-    : cfd_(column_family_data), db_(db), mutex_(mutex) {
+ColumnFamilyHandleImpl::ColumnFamilyHandleImpl(ColumnFamilyData* cfd,
+                                               DBImpl* db, port::Mutex* mutex)
+    : cfd_(cfd), db_(db), mutex_(mutex) {
   if (cfd_ != nullptr) {
     cfd_->Ref();
   }
@@ -75,25 +71,21 @@ ColumnFamilyHandleImpl::ColumnFamilyHandleImpl(
 
 ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
   if (cfd_ != nullptr) {
-    JobContext job_context;
+    DBImpl::DeletionState deletion_state;
     mutex_->Lock();
     if (cfd_->Unref()) {
       delete cfd_;
     }
-    db_->FindObsoleteFiles(&job_context, false, true);
+    db_->FindObsoleteFiles(deletion_state, false, true);
     mutex_->Unlock();
-    if (job_context.HaveSomethingToDelete()) {
-      db_->PurgeObsoleteFiles(job_context);
+    if (deletion_state.HaveSomethingToDelete()) {
+      db_->PurgeObsoleteFiles(deletion_state);
     }
   }
 }
 
 uint32_t ColumnFamilyHandleImpl::GetID() const { return cfd()->GetID(); }
 
-const std::string& ColumnFamilyHandleImpl::GetName() const {
-  return cfd()->GetName();
-}
-
 const Comparator* ColumnFamilyHandleImpl::user_comparator() const {
   return cfd()->user_comparator();
 }
@@ -224,16 +216,14 @@ void SuperVersionUnrefHandle(void* ptr) {
 }  // anonymous namespace
 
 ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
-                                   Version* _dummy_versions,
-                                   Cache* _table_cache,
-                                   WriteBuffer* write_buffer,
+                                   Version* dummy_versions, Cache* table_cache,
                                    const ColumnFamilyOptions& cf_options,
                                    const DBOptions* db_options,
                                    const EnvOptions& env_options,
                                    ColumnFamilySet* column_family_set)
     : id_(id),
       name_(name),
-      dummy_versions_(_dummy_versions),
+      dummy_versions_(dummy_versions),
       current_(nullptr),
       refs_(0),
       dropped_(false),
@@ -241,7 +231,6 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
       options_(*db_options, SanitizeOptions(&internal_comparator_, cf_options)),
       ioptions_(options_),
       mutable_cf_options_(options_, ioptions_),
-      write_buffer_(write_buffer),
       mem_(nullptr),
       imm_(options_.min_write_buffer_number_to_merge),
       super_version_(nullptr),
@@ -253,42 +242,27 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
       column_family_set_(column_family_set) {
   Ref();
 
-  // if _dummy_versions is nullptr, then this is a dummy column family.
-  if (_dummy_versions != nullptr) {
+  // if dummy_versions is nullptr, then this is a dummy column family.
+  if (dummy_versions != nullptr) {
     internal_stats_.reset(
         new InternalStats(ioptions_.num_levels, db_options->env, this));
-    table_cache_.reset(new TableCache(ioptions_, env_options, _table_cache));
-    if (ioptions_.compaction_style == kCompactionStyleLevel) {
-      compaction_picker_.reset(
-          new LevelCompactionPicker(ioptions_, &internal_comparator_));
-#ifndef ROCKSDB_LITE
-    } else if (ioptions_.compaction_style == kCompactionStyleUniversal) {
+    table_cache_.reset(new TableCache(ioptions_, env_options, table_cache));
+    if (ioptions_.compaction_style == kCompactionStyleUniversal) {
       compaction_picker_.reset(
           new UniversalCompactionPicker(ioptions_, &internal_comparator_));
-    } else if (ioptions_.compaction_style == kCompactionStyleFIFO) {
-      compaction_picker_.reset(
-          new FIFOCompactionPicker(ioptions_, &internal_comparator_));
-    } else if (ioptions_.compaction_style == kCompactionStyleNone) {
-      compaction_picker_.reset(new NullCompactionPicker(
-          ioptions_, &internal_comparator_));
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
-          "Column family %s does not use any background compaction. "
-          "Compactions can only be done via CompactFiles\n",
-          GetName().c_str());
-#endif  // !ROCKSDB_LITE
-    } else {
-      Log(InfoLogLevel::ERROR_LEVEL, ioptions_.info_log,
-          "Unable to recognize the specified compaction style %d. "
-          "Column family %s will use kCompactionStyleLevel.\n",
-          ioptions_.compaction_style, GetName().c_str());
+    } else if (ioptions_.compaction_style == kCompactionStyleLevel) {
       compaction_picker_.reset(
           new LevelCompactionPicker(ioptions_, &internal_comparator_));
+    } else {
+      assert(ioptions_.compaction_style == kCompactionStyleFIFO);
+      compaction_picker_.reset(
+          new FIFOCompactionPicker(ioptions_, &internal_comparator_));
     }
 
-    Log(InfoLogLevel::INFO_LEVEL,
-        ioptions_.info_log, "Options for column family \"%s\":\n",
+    Log(ioptions_.info_log, "Options for column family \"%s\":\n",
         name.c_str());
-    options_.Dump(ioptions_.info_log);
+    const ColumnFamilyOptions* cf_options = &options_;
+    cf_options->Dump(ioptions_.info_log);
   }
 
   RecalculateWriteStallConditions(mutable_cf_options_);
@@ -330,9 +304,8 @@ ColumnFamilyData::~ColumnFamilyData() {
 
   if (dummy_versions_ != nullptr) {
     // List must be empty
-    assert(dummy_versions_->TEST_Next() == dummy_versions_);
-    bool deleted __attribute__((unused)) = dummy_versions_->Unref();
-    assert(deleted);
+    assert(dummy_versions_->next_ == dummy_versions_);
+    delete dummy_versions_;
   }
 
   if (mem_ != nullptr) {
@@ -348,59 +321,56 @@ ColumnFamilyData::~ColumnFamilyData() {
 void ColumnFamilyData::RecalculateWriteStallConditions(
       const MutableCFOptions& mutable_cf_options) {
   if (current_ != nullptr) {
-    auto* vstorage = current_->storage_info();
-    const double score = vstorage->max_compaction_score();
-    const int max_level = vstorage->max_compaction_score_level();
+    const double score = current_->MaxCompactionScore();
+    const int max_level = current_->MaxCompactionScoreLevel();
 
     auto write_controller = column_family_set_->write_controller_;
 
-    if (imm()->size() >= mutable_cf_options.max_write_buffer_number) {
+    if (imm()->size() == options_.max_write_buffer_number) {
       write_controller_token_ = write_controller->GetStopToken();
       internal_stats_->AddCFStats(InternalStats::MEMTABLE_COMPACTION, 1);
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+      Log(ioptions_.info_log,
           "[%s] Stopping writes because we have %d immutable memtables "
-          "(waiting for flush), max_write_buffer_number is set to %d",
-          name_.c_str(), imm()->size(),
-          mutable_cf_options.max_write_buffer_number);
-    } else if (vstorage->NumLevelFiles(0) >=
+          "(waiting for flush)",
+          name_.c_str(), imm()->size());
+    } else if (current_->NumLevelFiles(0) >=
                mutable_cf_options.level0_stop_writes_trigger) {
       write_controller_token_ = write_controller->GetStopToken();
       internal_stats_->AddCFStats(InternalStats::LEVEL0_NUM_FILES, 1);
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+      Log(ioptions_.info_log,
           "[%s] Stopping writes because we have %d level-0 files",
-          name_.c_str(), vstorage->NumLevelFiles(0));
+          name_.c_str(), current_->NumLevelFiles(0));
     } else if (mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
-               vstorage->NumLevelFiles(0) >=
+               current_->NumLevelFiles(0) >=
                    mutable_cf_options.level0_slowdown_writes_trigger) {
-      uint64_t slowdown =
-          SlowdownAmount(vstorage->NumLevelFiles(0),
-                         mutable_cf_options.level0_slowdown_writes_trigger,
-                         mutable_cf_options.level0_stop_writes_trigger);
+      uint64_t slowdown = SlowdownAmount(
+          current_->NumLevelFiles(0),
+          mutable_cf_options.level0_slowdown_writes_trigger,
+          mutable_cf_options.level0_stop_writes_trigger);
       write_controller_token_ = write_controller->GetDelayToken(slowdown);
       internal_stats_->AddCFStats(InternalStats::LEVEL0_SLOWDOWN, slowdown);
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+      Log(ioptions_.info_log,
           "[%s] Stalling writes because we have %d level-0 files (%" PRIu64
           "us)",
-          name_.c_str(), vstorage->NumLevelFiles(0), slowdown);
-    } else if (mutable_cf_options.hard_rate_limit > 1.0 &&
-               score > mutable_cf_options.hard_rate_limit) {
+          name_.c_str(), current_->NumLevelFiles(0), slowdown);
+    } else if (options_.hard_rate_limit > 1.0 &&
+               score > options_.hard_rate_limit) {
       uint64_t kHardLimitSlowdown = 1000;
       write_controller_token_ =
           write_controller->GetDelayToken(kHardLimitSlowdown);
       internal_stats_->RecordLevelNSlowdown(max_level, kHardLimitSlowdown,
                                             false);
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+      Log(ioptions_.info_log,
           "[%s] Stalling writes because we hit hard limit on level %d. "
           "(%" PRIu64 "us)",
           name_.c_str(), max_level, kHardLimitSlowdown);
-    } else if (mutable_cf_options.soft_rate_limit > 0.0 &&
-               score > mutable_cf_options.soft_rate_limit) {
-      uint64_t slowdown = SlowdownAmount(score,
-          mutable_cf_options.soft_rate_limit,
-          mutable_cf_options.hard_rate_limit);
+    } else if (options_.soft_rate_limit > 0.0 &&
+               score > options_.soft_rate_limit) {
+      uint64_t slowdown = SlowdownAmount(score, options_.soft_rate_limit,
+                                         options_.hard_rate_limit);
       write_controller_token_ = write_controller->GetDelayToken(slowdown);
       internal_stats_->RecordLevelNSlowdown(max_level, slowdown, true);
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+      Log(ioptions_.info_log,
           "[%s] Stalling writes because we hit soft limit on level %d (%" PRIu64
           "us)",
           name_.c_str(), max_level, slowdown);
@@ -414,33 +384,21 @@ const EnvOptions* ColumnFamilyData::soptions() const {
   return &(column_family_set_->env_options_);
 }
 
-void ColumnFamilyData::SetCurrent(Version* current_version) {
-  current_ = current_version;
-}
+void ColumnFamilyData::SetCurrent(Version* current) { current_ = current; }
 
-MemTable* ColumnFamilyData::ConstructNewMemtable(
-    const MutableCFOptions& mutable_cf_options) {
+void ColumnFamilyData::CreateNewMemtable(const MemTableOptions& moptions) {
   assert(current_ != nullptr);
-  return new MemTable(internal_comparator_, ioptions_,
-                      mutable_cf_options, write_buffer_);
-}
-
-void ColumnFamilyData::CreateNewMemtable(
-    const MutableCFOptions& mutable_cf_options) {
   if (mem_ != nullptr) {
     delete mem_->Unref();
   }
-  SetMemtable(ConstructNewMemtable(mutable_cf_options));
+  mem_ = new MemTable(internal_comparator_, ioptions_, moptions);
   mem_->Ref();
 }
 
 Compaction* ColumnFamilyData::PickCompaction(
     const MutableCFOptions& mutable_options, LogBuffer* log_buffer) {
-  auto* result = compaction_picker_->PickCompaction(
-      GetName(), mutable_options, current_->storage_info(), log_buffer);
-  if (result != nullptr) {
-    result->SetInputVersion(current_);
-  }
+  auto result = compaction_picker_->PickCompaction(
+      mutable_options, current_, log_buffer);
   return result;
 }
 
@@ -449,13 +407,9 @@ Compaction* ColumnFamilyData::CompactRange(
     int input_level, int output_level, uint32_t output_path_id,
     const InternalKey* begin, const InternalKey* end,
     InternalKey** compaction_end) {
-  auto* result = compaction_picker_->CompactRange(
-      GetName(), mutable_cf_options, current_->storage_info(), input_level,
-      output_level, output_path_id, begin, end, compaction_end);
-  if (result != nullptr) {
-    result->SetInputVersion(current_);
-  }
-  return result;
+  return compaction_picker_->CompactRange(
+      mutable_cf_options, current_, input_level, output_level,
+      output_path_id, begin, end, compaction_end);
 }
 
 SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
@@ -533,22 +487,6 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
   return false;
 }
 
-void ColumnFamilyData::NotifyOnFlushCompleted(
-    DB* db, const std::string& file_path,
-    bool triggered_flush_slowdown,
-    bool triggered_flush_stop) {
-
-#ifndef ROCKSDB_LITE
-  auto listeners = ioptions()->listeners;
-  for (auto listener : listeners) {
-    listener->OnFlushCompleted(
-        db, GetName(), file_path,
-        // Use path 0 as fulled memtables are first flushed into path 0.
-        triggered_flush_slowdown, triggered_flush_stop);
-  }
-#endif  // ROCKSDB_LITE
-}
-
 SuperVersion* ColumnFamilyData::InstallSuperVersion(
     SuperVersion* new_superversion, port::Mutex* db_mutex) {
   db_mutex->AssertHeld();
@@ -593,28 +531,25 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
   }
 }
 
-#ifndef ROCKSDB_LITE
-Status ColumnFamilyData::SetOptions(
+bool ColumnFamilyData::SetOptions(
       const std::unordered_map<std::string, std::string>& options_map) {
   MutableCFOptions new_mutable_cf_options;
-  Status s = GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
-                                          &new_mutable_cf_options);
-  if (s.ok()) {
+  if (GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
+                                   &new_mutable_cf_options)) {
     mutable_cf_options_ = new_mutable_cf_options;
     mutable_cf_options_.RefreshDerivedOptions(ioptions_);
+    return true;
   }
-  return s;
+  return false;
 }
-#endif  // ROCKSDB_LITE
 
 ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                  const DBOptions* db_options,
                                  const EnvOptions& env_options,
                                  Cache* table_cache,
-                                 WriteBuffer* write_buffer,
                                  WriteController* write_controller)
     : max_column_family_(0),
-      dummy_cfd_(new ColumnFamilyData(0, "", nullptr, nullptr, nullptr,
+      dummy_cfd_(new ColumnFamilyData(0, "", nullptr, nullptr,
                                       ColumnFamilyOptions(), db_options,
                                       env_options, nullptr)),
       default_cfd_cache_(nullptr),
@@ -622,7 +557,6 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
       db_options_(db_options),
       env_options_(env_options),
       table_cache_(table_cache),
-      write_buffer_(write_buffer),
       write_controller_(write_controller),
       spin_lock_(ATOMIC_FLAG_INIT) {
   // initialize linked list
@@ -687,9 +621,8 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
     const ColumnFamilyOptions& options) {
   assert(column_families_.find(name) == column_families_.end());
   ColumnFamilyData* new_cfd =
-      new ColumnFamilyData(id, name, dummy_versions, table_cache_,
-                           write_buffer_, options, db_options_,
-                           env_options_, this);
+      new ColumnFamilyData(id, name, dummy_versions, table_cache_, options,
+                           db_options_, env_options_, this);
   Lock();
   column_families_.insert({name, id});
   column_family_data_.insert({id, new_cfd});
@@ -768,6 +701,11 @@ MemTable* ColumnFamilyMemTablesImpl::GetMemTable() const {
   return current_->mem();
 }
 
+const Options* ColumnFamilyMemTablesImpl::GetOptions() const {
+  assert(current_ != nullptr);
+  return current_->options();
+}
+
 ColumnFamilyHandle* ColumnFamilyMemTablesImpl::GetColumnFamilyHandle() {
   assert(current_ != nullptr);
   return &handle_;

db/column_family.h

@@ -52,7 +52,6 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
   virtual const Comparator* user_comparator() const;
 
   virtual uint32_t GetID() const;
-  virtual const std::string& GetName() const override;
 
  private:
   ColumnFamilyData* cfd_;
@@ -71,7 +70,7 @@ class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
   ColumnFamilyHandleInternal()
       : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr) {}
 
-  void SetCFD(ColumnFamilyData* _cfd) { internal_cfd_ = _cfd; }
+  void SetCFD(ColumnFamilyData* cfd) { internal_cfd_ = cfd; }
   virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
 
  private:
@@ -167,21 +166,19 @@ class ColumnFamilyData {
   bool IsDropped() const { return dropped_; }
 
   // thread-safe
-  int NumberLevels() const { return ioptions_.num_levels; }
+  int NumberLevels() const { return options_.num_levels; }
 
   void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
   uint64_t GetLogNumber() const { return log_number_; }
 
-  // !!! To be deprecated! Please don't not use this function anymore!
-  const Options* options() const { return &options_; }
-
   // thread-safe
+  const Options* options() const { return &options_; }
   const EnvOptions* soptions() const;
   const ImmutableCFOptions* ioptions() const { return &ioptions_; }
   // REQUIRES: DB mutex held
   // This returns the MutableCFOptions used by current SuperVersion
   // You shoul use this API to reference MutableCFOptions most of the time.
-  const MutableCFOptions* GetCurrentMutableCFOptions() const {
+  const MutableCFOptions* mutable_cf_options() const {
     return &(super_version_->mutable_cf_options);
   }
   // REQUIRES: DB mutex held
@@ -189,11 +186,9 @@ class ColumnFamilyData {
   const MutableCFOptions* GetLatestMutableCFOptions() const {
     return &mutable_cf_options_;
   }
-#ifndef ROCKSDB_LITE
   // REQUIRES: DB mutex held
-  Status SetOptions(
+  bool SetOptions(
       const std::unordered_map<std::string, std::string>& options_map);
-#endif  // ROCKSDB_LITE
 
   InternalStats* internal_stats() { return internal_stats_.get(); }
 
@@ -201,10 +196,9 @@ class ColumnFamilyData {
   MemTable* mem() { return mem_; }
   Version* current() { return current_; }
   Version* dummy_versions() { return dummy_versions_; }
-  void SetCurrent(Version* current);
-  MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options);
   void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
-  void CreateNewMemtable(const MutableCFOptions& mutable_cf_options);
+  void SetCurrent(Version* current);
+  void CreateNewMemtable(const MemTableOptions& moptions);
 
   TableCache* table_cache() const { return table_cache_.get(); }
 
@@ -256,16 +250,10 @@ class ColumnFamilyData {
 
   void ResetThreadLocalSuperVersions();
 
-  void NotifyOnFlushCompleted(
-      DB* db, const std::string& file_path,
-      bool triggered_flush_slowdown,
-      bool triggered_flush_stop);
-
  private:
   friend class ColumnFamilySet;
   ColumnFamilyData(uint32_t id, const std::string& name,
                    Version* dummy_versions, Cache* table_cache,
-                   WriteBuffer* write_buffer,
                    const ColumnFamilyOptions& options,
                    const DBOptions* db_options, const EnvOptions& env_options,
                    ColumnFamilySet* column_family_set);
@@ -296,8 +284,6 @@ class ColumnFamilyData {
 
   std::unique_ptr<InternalStats> internal_stats_;
 
-  WriteBuffer* write_buffer_;
-
   MemTable* mem_;
   MemTableList imm_;
   SuperVersion* super_version_;
@@ -370,7 +356,7 @@ class ColumnFamilySet {
 
   ColumnFamilySet(const std::string& dbname, const DBOptions* db_options,
                   const EnvOptions& env_options, Cache* table_cache,
-                  WriteBuffer* write_buffer, WriteController* write_controller);
+                  WriteController* write_controller);
   ~ColumnFamilySet();
 
   ColumnFamilyData* GetDefault() const;
@@ -425,7 +411,6 @@ class ColumnFamilySet {
   const DBOptions* const db_options_;
   const EnvOptions env_options_;
   Cache* table_cache_;
-  WriteBuffer* write_buffer_;
   WriteController* write_controller_;
   std::atomic_flag spin_lock_;
 };
@@ -450,6 +435,10 @@ class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
   // REQUIRES: Seek() called first
   virtual MemTable* GetMemTable() const override;
 
+  // Returns options for selected column family
+  // REQUIRES: Seek() called first
+  virtual const Options* GetOptions() const override;
+
   // Returns column family handle for the selected column family
   virtual ColumnFamilyHandle* GetColumnFamilyHandle() override;
 

db/column_family_test.cc

@@ -133,7 +133,7 @@ class ColumnFamilyTest {
   void CreateColumnFamilies(
       const std::vector<std::string>& cfs,
       const std::vector<ColumnFamilyOptions> options = {}) {
-    int cfi = static_cast<int>(handles_.size());
+    int cfi = handles_.size();
     handles_.resize(cfi + cfs.size());
     names_.resize(cfi + cfs.size());
     for (size_t i = 0; i < cfs.size(); ++i) {
@@ -218,7 +218,7 @@ class ColumnFamilyTest {
 
   int NumTableFilesAtLevel(int level, int cf) {
     return GetProperty(cf,
-                       "rocksdb.num-files-at-level" + ToString(level));
+                       "rocksdb.num-files-at-level" + std::to_string(level));
   }
 
   // Return spread of files per level
@@ -231,7 +231,7 @@ class ColumnFamilyTest {
       snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
       result += buf;
       if (f > 0) {
-        last_non_zero_offset = static_cast<int>(result.size());
+        last_non_zero_offset = result.size();
       }
     }
     result.resize(last_non_zero_offset);
@@ -287,8 +287,8 @@ class ColumnFamilyTest {
     assert(num_per_cf.size() == handles_.size());
 
     for (size_t i = 0; i < num_per_cf.size(); ++i) {
-      ASSERT_EQ(num_per_cf[i], GetProperty(static_cast<int>(i),
-                                           "rocksdb.num-immutable-mem-table"));
+      ASSERT_EQ(num_per_cf[i],
+                GetProperty(i, "rocksdb.num-immutable-mem-table"));
     }
   }
 
@@ -387,7 +387,7 @@ TEST(ColumnFamilyTest, DropTest) {
     Open({"default"});
     CreateColumnFamiliesAndReopen({"pikachu"});
     for (int i = 0; i < 100; ++i) {
-      ASSERT_OK(Put(1, ToString(i), "bar" + ToString(i)));
+      ASSERT_OK(Put(1, std::to_string(i), "bar" + std::to_string(i)));
     }
     ASSERT_OK(Flush(1));
 
@@ -529,28 +529,8 @@ TEST(ColumnFamilyTest, FlushTest) {
   ASSERT_OK(Put(1, "mirko", "v3"));
   ASSERT_OK(Put(0, "foo", "v2"));
   ASSERT_OK(Put(2, "fodor", "v5"));
-
-  for (int j = 0; j < 2; j++) {
-    ReadOptions ro;
-    std::vector<Iterator*> iterators;
-    // Hold super version.
-    if (j == 0) {
-      ASSERT_OK(db_->NewIterators(ro, handles_, &iterators));
-    }
-
-    for (int i = 0; i < 3; ++i) {
-      uint64_t max_total_in_memory_state =
-          dbfull()->TEST_max_total_in_memory_state();
-      Flush(i);
-      ASSERT_EQ(dbfull()->TEST_max_total_in_memory_state(),
-                max_total_in_memory_state);
-    }
-    ASSERT_OK(Put(1, "foofoo", "bar"));
-    ASSERT_OK(Put(0, "foofoo", "bar"));
-
-    for (auto* it : iterators) {
-      delete it;
-    }
+  for (int i = 0; i < 3; ++i) {
+    Flush(i);
   }
   Reopen();
 
@@ -794,14 +774,14 @@ TEST(ColumnFamilyTest, DifferentCompactionStyles) {
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 1; ++i) {
     PutRandomData(1, 11, 10000);
     WaitForFlush(1);
-    ASSERT_EQ(ToString(i + 1), FilesPerLevel(1));
+    ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(1));
   }
 
   // SETUP column family "two" -- level style with 4 levels
   for (int i = 0; i < two.level0_file_num_compaction_trigger - 1; ++i) {
     PutRandomData(2, 15, 10000);
     WaitForFlush(2);
-    ASSERT_EQ(ToString(i + 1), FilesPerLevel(2));
+    ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(2));
   }
 
   // TRIGGER compaction "one"
@@ -936,11 +916,11 @@ TEST(ColumnFamilyTest, DontRollEmptyLogs) {
   CreateColumnFamiliesAndReopen({"one", "two", "three", "four"});
 
   for (size_t i = 0; i < handles_.size(); ++i) {
-    PutRandomData(static_cast<int>(i), 10, 100);
+    PutRandomData(i, 10, 100);
   }
   int num_writable_file_start = env_->GetNumberOfNewWritableFileCalls();
   // this will trigger the flushes
-  for (int i = 0; i <= 4; ++i) {
+  for (size_t i = 0; i <= 4; ++i) {
     ASSERT_OK(Flush(i));
   }
 

db/compaction.cc

@@ -29,17 +29,7 @@ uint64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
   return sum;
 }
 
-void Compaction::SetInputVersion(Version* _input_version) {
-  input_version_ = _input_version;
-  cfd_ = input_version_->cfd();
-
-  cfd_->Ref();
-  input_version_->Ref();
-  edit_ = new VersionEdit();
-  edit_->SetColumnFamily(cfd_->GetID());
-}
-
-Compaction::Compaction(int number_levels, int start_level, int out_level,
+Compaction::Compaction(Version* input_version, int start_level, int out_level,
                        uint64_t target_file_size,
                        uint64_t max_grandparent_overlap_bytes,
                        uint32_t output_path_id,
@@ -49,10 +39,9 @@ Compaction::Compaction(int number_levels, int start_level, int out_level,
       output_level_(out_level),
       max_output_file_size_(target_file_size),
       max_grandparent_overlap_bytes_(max_grandparent_overlap_bytes),
-      input_version_(nullptr),
-      edit_(nullptr),
-      number_levels_(number_levels),
-      cfd_(nullptr),
+      input_version_(input_version),
+      number_levels_(input_version_->NumberLevels()),
+      cfd_(input_version_->cfd_),
       output_path_id_(output_path_id),
       output_compression_(output_compression),
       seek_compaction_(seek_compaction),
@@ -67,6 +56,10 @@ Compaction::Compaction(int number_levels, int start_level, int out_level,
       is_full_compaction_(false),
       is_manual_compaction_(false),
       level_ptrs_(std::vector<size_t>(number_levels_)) {
+  cfd_->Ref();
+  input_version_->Ref();
+  edit_ = new VersionEdit();
+  edit_->SetColumnFamily(cfd_->GetID());
   for (int i = 0; i < number_levels_; i++) {
     level_ptrs_[i] = 0;
   }
@@ -78,38 +71,6 @@ Compaction::Compaction(int number_levels, int start_level, int out_level,
   }
 }
 
-Compaction::Compaction(VersionStorageInfo* vstorage,
-    const autovector<CompactionInputFiles>& _inputs,
-    int _start_level, int _output_level,
-    uint64_t _max_grandparent_overlap_bytes,
-    const CompactionOptions& _options,
-    bool _deletion_compaction)
-    : start_level_(_start_level),
-      output_level_(_output_level),
-      max_output_file_size_(_options.output_file_size_limit),
-      max_grandparent_overlap_bytes_(_max_grandparent_overlap_bytes),
-      input_version_(nullptr),
-      number_levels_(vstorage->num_levels()),
-      cfd_(nullptr),
-      output_compression_(_options.compression),
-      seek_compaction_(false),
-      deletion_compaction_(_deletion_compaction),
-      inputs_(_inputs),
-      grandparent_index_(0),
-      seen_key_(false),
-      overlapped_bytes_(0),
-      base_index_(-1),
-      parent_index_(-1),
-      score_(0),
-      bottommost_level_(false),
-      is_full_compaction_(false),
-      is_manual_compaction_(false),
-      level_ptrs_(std::vector<size_t>(number_levels_)) {
-  for (int i = 0; i < number_levels_; i++) {
-    level_ptrs_[i] = 0;
-  }
-}
-
 Compaction::~Compaction() {
   delete edit_;
   if (input_version_ != nullptr) {
@@ -124,9 +85,8 @@ Compaction::~Compaction() {
 
 void Compaction::GenerateFileLevels() {
   input_levels_.resize(num_input_levels());
-  for (size_t which = 0; which < num_input_levels(); which++) {
-    DoGenerateLevelFilesBrief(&input_levels_[which], inputs_[which].files,
-                              &arena_);
+  for (int which = 0; which < num_input_levels(); which++) {
+    DoGenerateFileLevel(&input_levels_[which], inputs_[which].files, &arena_);
   }
 }
 
@@ -143,16 +103,15 @@ bool Compaction::IsTrivialMove() const {
           TotalFileSize(grandparents_) <= max_grandparent_overlap_bytes_);
 }
 
-void Compaction::AddInputDeletions(VersionEdit* out_edit) {
-  for (size_t which = 0; which < num_input_levels(); which++) {
+void Compaction::AddInputDeletions(VersionEdit* edit) {
+  for (int which = 0; which < num_input_levels(); which++) {
     for (size_t i = 0; i < inputs_[which].size(); i++) {
-      out_edit->DeleteFile(level(which), inputs_[which][i]->fd.GetNumber());
+      edit->DeleteFile(level(which), inputs_[which][i]->fd.GetNumber());
     }
   }
 }
 
 bool Compaction::KeyNotExistsBeyondOutputLevel(const Slice& user_key) {
-  assert(input_version_ != nullptr);
   assert(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO);
   if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
     return bottommost_level_;
@@ -160,8 +119,7 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(const Slice& user_key) {
   // Maybe use binary search to find right entry instead of linear search?
   const Comparator* user_cmp = cfd_->user_comparator();
   for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
-    const std::vector<FileMetaData*>& files =
-        input_version_->storage_info()->LevelFiles(lvl);
+    const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
     for (; level_ptrs_[lvl] < files.size(); ) {
       FileMetaData* f = files[level_ptrs_[lvl]];
       if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
@@ -207,7 +165,7 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
 
 // Mark (or clear) each file that is being compacted
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
-  for (size_t i = 0; i < num_input_levels(); i++) {
+  for (int i = 0; i < num_input_levels(); i++) {
     for (unsigned int j = 0; j < inputs_[i].size(); j++) {
       assert(mark_as_compacted ? !inputs_[i][j]->being_compacted :
                                   inputs_[i][j]->being_compacted);
@@ -217,9 +175,9 @@ void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
 }
 
 // Is this compaction producing files at the bottommost level?
-void Compaction::SetupBottomMostLevel(VersionStorageInfo* vstorage,
-                                      bool is_manual, bool level0_only) {
-  if (level0_only) {
+void Compaction::SetupBottomMostLevel(bool is_manual) {
+  assert(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO);
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
     // If universal compaction style is used and manual
     // compaction is occuring, then we are guaranteed that
     // all files will be picked in a single compaction
@@ -234,7 +192,7 @@ void Compaction::SetupBottomMostLevel(VersionStorageInfo* vstorage,
   bottommost_level_ = true;
   // checks whether there are files living beyond the output_level.
   for (int i = output_level_ + 1; i < number_levels_; i++) {
-    if (vstorage->NumLevelFiles(i) > 0) {
+    if (input_version_->NumLevelFiles(i) > 0) {
       bottommost_level_ = false;
       break;
     }
@@ -259,8 +217,7 @@ void Compaction::ReleaseCompactionFiles(Status status) {
 }
 
 void Compaction::ResetNextCompactionIndex() {
-  assert(input_version_ != nullptr);
-  input_version_->storage_info()->ResetNextCompactionIndex(start_level_);
+  input_version_->ResetNextCompactionIndex(start_level_);
 }
 
 namespace {
@@ -293,15 +250,14 @@ void Compaction::Summary(char* output, int len) {
     return;
   }
 
-  for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
-    if (level_iter > 0) {
+  for (int level = 0; level < num_input_levels(); ++level) {
+    if (level > 0) {
       write += snprintf(output + write, len - write, "], [");
       if (write < 0 || write >= len) {
         return;
       }
     }
-    write +=
-        InputSummary(inputs_[level_iter].files, output + write, len - write);
+    write += InputSummary(inputs_[level].files, output + write, len - write);
     if (write < 0 || write >= len) {
       return;
     }
@@ -317,8 +273,8 @@ uint64_t Compaction::OutputFilePreallocationSize(
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
     preallocation_size = mutable_options.MaxFileSizeForLevel(output_level());
   } else {
-    for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
-      for (const auto& f : inputs_[level_iter].files) {
+    for (int level = 0; level < num_input_levels(); ++level) {
+      for (const auto& f : inputs_[level].files) {
         preallocation_size += f->fd.GetFileSize();
       }
     }
@@ -328,15 +284,4 @@ uint64_t Compaction::OutputFilePreallocationSize(
   return preallocation_size * 1.1;
 }
 
-Compaction* Compaction::TEST_NewCompaction(
-    int num_levels, int start_level, int out_level, uint64_t target_file_size,
-    uint64_t max_grandparent_overlap_bytes, uint32_t output_path_id,
-    CompressionType output_compression, bool seek_compaction,
-    bool deletion_compaction) {
-  return new Compaction(num_levels, start_level, out_level, target_file_size,
-                        max_grandparent_overlap_bytes, output_path_id,
-                        output_compression, seek_compaction,
-                        deletion_compaction);
-}
-
 }  // namespace rocksdb

db/compaction.h

@@ -23,23 +23,15 @@ struct CompactionInputFiles {
   inline bool empty() const { return files.empty(); }
   inline size_t size() const { return files.size(); }
   inline void clear() { files.clear(); }
-  inline FileMetaData* operator[](size_t i) const { return files[i]; }
+  inline FileMetaData* operator[](int i) const { return files[i]; }
 };
 
 class Version;
 class ColumnFamilyData;
-class VersionStorageInfo;
 
 // A Compaction encapsulates information about a compaction.
 class Compaction {
  public:
-  Compaction(VersionStorageInfo* input_version,
-    const autovector<CompactionInputFiles>& inputs,
-    int start_level, int output_level,
-    uint64_t max_grandparent_overlap_bytes,
-    const CompactionOptions& options,
-    bool deletion_compaction);
-
   // No copying allowed
   Compaction(const Compaction&) = delete;
   void operator=(const Compaction&) = delete;
@@ -48,7 +40,7 @@ class Compaction {
 
   // Returns the level associated to the specified compaction input level.
   // If compaction_input_level is not specified, then input_level is set to 0.
-  int level(size_t compaction_input_level = 0) const {
+  int level(int compaction_input_level = 0) const {
     return inputs_[compaction_input_level].level;
   }
 
@@ -56,7 +48,7 @@ class Compaction {
   int output_level() const { return output_level_; }
 
   // Returns the number of input levels in this compaction.
-  size_t num_input_levels() const { return inputs_.size(); }
+  int num_input_levels() const { return inputs_.size(); }
 
   // Return the object that holds the edits to the descriptor done
   // by this compaction.
@@ -66,7 +58,7 @@ class Compaction {
   // compaction input level.
   // The function will return 0 if when "compaction_input_level" < 0
   // or "compaction_input_level" >= "num_input_levels()".
-  size_t num_input_files(size_t compaction_input_level) const {
+  int num_input_files(size_t compaction_input_level) const {
     if (compaction_input_level < inputs_.size()) {
       return inputs_[compaction_input_level].size();
     }
@@ -83,7 +75,7 @@ class Compaction {
   // specified compaction input level.
   // REQUIREMENT: "compaction_input_level" must be >= 0 and
   //              < "input_levels()"
-  FileMetaData* input(size_t compaction_input_level, size_t i) const {
+  FileMetaData* input(size_t compaction_input_level, int i) const {
     assert(compaction_input_level < inputs_.size());
     return inputs_[compaction_input_level][i];
   }
@@ -97,8 +89,8 @@ class Compaction {
     return &inputs_[compaction_input_level].files;
   }
 
-  // Returns the LevelFilesBrief of the specified compaction input level.
-  LevelFilesBrief* input_levels(size_t compaction_input_level) {
+  // Returns the FileLevel of the specified compaction input level.
+  FileLevel* input_levels(int compaction_input_level) {
     return &input_levels_[compaction_input_level];
   }
 
@@ -160,8 +152,6 @@ class Compaction {
   // Was this compaction triggered manually by the client?
   bool IsManualCompaction() { return is_manual_compaction_; }
 
-  void SetOutputPathId(uint32_t path_id) { output_path_id_ = path_id; }
-
   // Return the MutableCFOptions that should be used throughout the compaction
   // procedure
   const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
@@ -171,35 +161,13 @@ class Compaction {
   // is the sum of all input file sizes.
   uint64_t OutputFilePreallocationSize(const MutableCFOptions& mutable_options);
 
-  void SetInputVersion(Version* input_version);
-
-  // mark (or clear) all files that are being compacted
-  void MarkFilesBeingCompacted(bool mark_as_compacted);
-
-  // Initialize whether the compaction is producing files at the
-  // bottommost level.
-  //
-  // @see BottomMostLevel()
-  void SetupBottomMostLevel(VersionStorageInfo* vstorage, bool is_manual,
-                            bool level0_only);
-
-  static Compaction* TEST_NewCompaction(
-      int num_levels, int start_level, int out_level, uint64_t target_file_size,
-      uint64_t max_grandparent_overlap_bytes, uint32_t output_path_id,
-      CompressionType output_compression, bool seek_compaction = false,
-      bool deletion_compaction = false);
-
-  CompactionInputFiles* TEST_GetInputFiles(int l) {
-    return &inputs_[l];
-  }
-
  private:
   friend class CompactionPicker;
   friend class UniversalCompactionPicker;
   friend class FIFOCompactionPicker;
   friend class LevelCompactionPicker;
 
-  Compaction(int num_levels, int start_level, int out_level,
+  Compaction(Version* input_version, int start_level, int out_level,
              uint64_t target_file_size, uint64_t max_grandparent_overlap_bytes,
              uint32_t output_path_id, CompressionType output_compression,
              bool seek_compaction = false, bool deletion_compaction = false);
@@ -225,7 +193,7 @@ class Compaction {
   autovector<CompactionInputFiles> inputs_;
 
   // A copy of inputs_, organized more closely in memory
-  autovector<LevelFilesBrief, 2> input_levels_;
+  autovector<FileLevel, 2> input_levels_;
 
   // State used to check for number of of overlapping grandparent files
   // (grandparent == "output_level_ + 1")
@@ -255,6 +223,15 @@ class Compaction {
   // records indices for all levels beyond "output_level_".
   std::vector<size_t> level_ptrs_;
 
+  // mark (or clear) all files that are being compacted
+  void MarkFilesBeingCompacted(bool mark_as_compacted);
+
+  // Initialize whether the compaction is producing files at the
+  // bottommost level.
+  //
+  // @see BottomMostLevel()
+  void SetupBottomMostLevel(bool is_manual);
+
   // In case of compaction error, reset the nextIndex that is used
   // to pick up the next file to be compacted from files_by_size_
   void ResetNextCompactionIndex();

db/compaction_job.h

@@ -1,126 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#pragma once
-
-#include <atomic>
-#include <deque>
-#include <limits>
-#include <set>
-#include <utility>
-#include <vector>
-#include <string>
-#include <functional>
-
-#include "db/dbformat.h"
-#include "db/log_writer.h"
-#include "db/snapshot.h"
-#include "db/column_family.h"
-#include "db/version_edit.h"
-#include "db/memtable_list.h"
-#include "port/port.h"
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/memtablerep.h"
-#include "rocksdb/compaction_filter.h"
-#include "rocksdb/transaction_log.h"
-#include "util/autovector.h"
-#include "util/stop_watch.h"
-#include "util/thread_local.h"
-#include "util/scoped_arena_iterator.h"
-#include "db/internal_stats.h"
-#include "db/write_controller.h"
-#include "db/flush_scheduler.h"
-#include "db/write_thread.h"
-#include "db/job_context.h"
-
-namespace rocksdb {
-
-class MemTable;
-class TableCache;
-class Version;
-class VersionEdit;
-class VersionSet;
-class Arena;
-
-class CompactionJob {
- public:
-  // TODO(icanadi) make effort to reduce number of parameters here
-  // IMPORTANT: mutable_cf_options needs to be alive while CompactionJob is
-  // alive
-  CompactionJob(Compaction* compaction, const DBOptions& db_options,
-                const MutableCFOptions& mutable_cf_options,
-                const EnvOptions& env_options, VersionSet* versions,
-                std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
-                Directory* db_directory, Statistics* stats,
-                SnapshotList* snapshot_list, bool is_snapshot_supported,
-                std::shared_ptr<Cache> table_cache,
-                std::function<uint64_t()> yield_callback);
-
-  ~CompactionJob() { assert(compact_ == nullptr); }
-
-  // no copy/move
-  CompactionJob(CompactionJob&& job) = delete;
-  CompactionJob(const CompactionJob& job) = delete;
-  CompactionJob& operator=(const CompactionJob& job) = delete;
-
-  // REQUIRED: mutex held
-  void Prepare();
-  // REQUIRED mutex not held
-  Status Run();
-  // REQUIRED: mutex held
-  // status is the return of Run()
-  void Install(Status* status, port::Mutex* db_mutex);
-
- private:
-  void AllocateCompactionOutputFileNumbers();
-  // Call compaction filter if is_compaction_v2 is not true. Then iterate
-  // through input and compact the kv-pairs
-  Status ProcessKeyValueCompaction(int64_t* imm_micros, Iterator* input,
-                                   bool is_compaction_v2);
-  // Call compaction_filter_v2->Filter() on kv-pairs in compact
-  void CallCompactionFilterV2(CompactionFilterV2* compaction_filter_v2);
-  Status FinishCompactionOutputFile(Iterator* input);
-  Status InstallCompactionResults(port::Mutex* db_mutex);
-  SequenceNumber findEarliestVisibleSnapshot(
-      SequenceNumber in, const std::vector<SequenceNumber>& snapshots,
-      SequenceNumber* prev_snapshot);
-  void RecordCompactionIOStats();
-  Status OpenCompactionOutputFile();
-  void CleanupCompaction(const Status& status);
-
-  // CompactionJob state
-  struct CompactionState;
-  CompactionState* compact_;
-
-  bool bottommost_level_;
-  SequenceNumber earliest_snapshot_;
-  SequenceNumber visible_at_tip_;
-  SequenceNumber latest_snapshot_;
-
-  InternalStats::CompactionStats compaction_stats_;
-
-  // DBImpl state
-  const DBOptions& db_options_;
-  const MutableCFOptions& mutable_cf_options_;
-  const EnvOptions& env_options_;
-  Env* env_;
-  VersionSet* versions_;
-  std::atomic<bool>* shutting_down_;
-  LogBuffer* log_buffer_;
-  Directory* db_directory_;
-  Statistics* stats_;
-  SnapshotList* snapshots_;
-  bool is_snapshot_supported_;
-  std::shared_ptr<Cache> table_cache_;
-
-  // yield callback
-  std::function<uint64_t()> yield_callback_;
-};
-
-}  // namespace rocksdb

db/compaction_job_test.cc

@@ -1,182 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include <map>
-#include <string>
-
-#include "db/compaction_job.h"
-#include "db/column_family.h"
-#include "db/version_set.h"
-#include "db/writebuffer.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/options.h"
-#include "rocksdb/db.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-#include "table/mock_table.h"
-
-namespace rocksdb {
-
-// TODO(icanadi) Make it simpler once we mock out VersionSet
-class CompactionJobTest {
- public:
-  CompactionJobTest()
-      : env_(Env::Default()),
-        dbname_(test::TmpDir() + "/compaction_job_test"),
-        mutable_cf_options_(Options(), ImmutableCFOptions(Options())),
-        table_cache_(NewLRUCache(50000, 16, 8)),
-        write_buffer_(db_options_.db_write_buffer_size),
-        versions_(new VersionSet(dbname_, &db_options_, env_options_,
-                                 table_cache_.get(), &write_buffer_,
-                                 &write_controller_)),
-        shutting_down_(false),
-        mock_table_factory_(new mock::MockTableFactory()) {
-    ASSERT_OK(env_->CreateDirIfMissing(dbname_));
-    db_options_.db_paths.emplace_back(dbname_,
-                                      std::numeric_limits<uint64_t>::max());
-    NewDB();
-    std::vector<ColumnFamilyDescriptor> column_families;
-    cf_options_.table_factory = mock_table_factory_;
-    column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
-
-
-    ASSERT_OK(versions_->Recover(column_families, false));
-  }
-
-  std::string GenerateFileName(uint64_t file_number) {
-    FileMetaData meta;
-    std::vector<DbPath> db_paths;
-    db_paths.emplace_back(dbname_, std::numeric_limits<uint64_t>::max());
-    meta.fd = FileDescriptor(file_number, 0, 0);
-    return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
-  }
-
-  // returns expected result after compaction
-  mock::MockFileContents CreateTwoFiles() {
-    mock::MockFileContents expected_results;
-    const int kKeysPerFile = 10000;
-    SequenceNumber sequence_number = 0;
-    for (int i = 0; i < 2; ++i) {
-      mock::MockFileContents contents;
-      SequenceNumber smallest_seqno = 0, largest_seqno = 0;
-      InternalKey smallest, largest;
-      for (int k = 0; k < kKeysPerFile; ++k) {
-        auto key = ToString(i * (kKeysPerFile / 2) + k);
-        auto value = ToString(i * kKeysPerFile + k);
-        InternalKey internal_key(key, ++sequence_number, kTypeValue);
-        if (k == 0) {
-          smallest = internal_key;
-          smallest_seqno = sequence_number;
-        } else if (k == kKeysPerFile - 1) {
-          largest = internal_key;
-          largest_seqno = sequence_number;
-        }
-        std::pair<std::string, std::string> key_value(
-            {internal_key.Encode().ToString(), value});
-        contents.insert(key_value);
-        if (i == 1 || k < kKeysPerFile / 2) {
-          expected_results.insert(key_value);
-        }
-      }
-
-      uint64_t file_number = versions_->NewFileNumber();
-      ASSERT_OK(mock_table_factory_->CreateMockTable(
-          env_, GenerateFileName(file_number), std::move(contents)));
-
-      VersionEdit edit;
-      edit.AddFile(0, file_number, 0, 10, smallest, largest, smallest_seqno,
-                   largest_seqno);
-
-      mutex_.Lock();
-      versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
-                             mutable_cf_options_, &edit, &mutex_);
-      mutex_.Unlock();
-    }
-    versions_->SetLastSequence(sequence_number);
-    return expected_results;
-  }
-
-  void NewDB() {
-    VersionEdit new_db;
-    new_db.SetLogNumber(0);
-    new_db.SetNextFile(2);
-    new_db.SetLastSequence(0);
-
-    const std::string manifest = DescriptorFileName(dbname_, 1);
-    unique_ptr<WritableFile> file;
-    Status s = env_->NewWritableFile(
-        manifest, &file, env_->OptimizeForManifestWrite(env_options_));
-    ASSERT_OK(s);
-    {
-      log::Writer log(std::move(file));
-      std::string record;
-      new_db.EncodeTo(&record);
-      s = log.AddRecord(record);
-    }
-    ASSERT_OK(s);
-    // Make "CURRENT" file that points to the new manifest file.
-    s = SetCurrentFile(env_, dbname_, 1, nullptr);
-  }
-
-  Env* env_;
-  std::string dbname_;
-  EnvOptions env_options_;
-  MutableCFOptions mutable_cf_options_;
-  std::shared_ptr<Cache> table_cache_;
-  WriteController write_controller_;
-  DBOptions db_options_;
-  ColumnFamilyOptions cf_options_;
-  WriteBuffer write_buffer_;
-  std::unique_ptr<VersionSet> versions_;
-  port::Mutex mutex_;
-  std::atomic<bool> shutting_down_;
-  std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
-};
-
-TEST(CompactionJobTest, Simple) {
-  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-
-  auto expected_results = CreateTwoFiles();
-
-  auto files = cfd->current()->storage_info()->LevelFiles(0);
-  ASSERT_EQ(2U, files.size());
-
-  std::unique_ptr<Compaction> compaction(Compaction::TEST_NewCompaction(
-      7, 0, 1, 1024 * 1024, 10, 0, kNoCompression));
-  compaction->SetInputVersion(cfd->current());
-
-  auto compaction_input_files = compaction->TEST_GetInputFiles(0);
-  compaction_input_files->level = 0;
-  compaction_input_files->files.push_back(files[0]);
-  compaction_input_files->files.push_back(files[1]);
-
-  SnapshotList snapshots;
-  int yield_callback_called = 0;
-  std::function<uint64_t()> yield_callback = [&]() {
-    yield_callback_called++;
-    return 0;
-  };
-  LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
-  mutex_.Lock();
-  CompactionJob compaction_job(
-      compaction.get(), db_options_, *cfd->GetLatestMutableCFOptions(),
-      env_options_, versions_.get(), &shutting_down_, &log_buffer, nullptr,
-      nullptr, &snapshots, true, table_cache_, std::move(yield_callback));
-  compaction_job.Prepare();
-  mutex_.Unlock();
-  ASSERT_OK(compaction_job.Run());
-  mutex_.Lock();
-  Status s;
-  compaction_job.Install(&s, &mutex_);
-  ASSERT_OK(s);
-  mutex_.Unlock();
-
-  mock_table_factory_->AssertLatestFile(expected_results);
-  ASSERT_EQ(yield_callback_called, 20000);
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/compaction_picker.h

@@ -8,11 +8,6 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
-#include <vector>
-#include <memory>
-#include <set>
-#include <unordered_set>
-
 #include "db/version_set.h"
 #include "db/compaction.h"
 #include "rocksdb/status.h"
@@ -23,14 +18,12 @@
 #include <vector>
 #include <memory>
 #include <set>
-#include <string>
 
 namespace rocksdb {
 
 class LogBuffer;
 class Compaction;
-class VersionStorageInfo;
-struct CompactionInputFiles;
+class Version;
 
 class CompactionPicker {
  public:
@@ -42,10 +35,9 @@ class CompactionPicker {
   // Returns nullptr if there is no compaction to be done.
   // Otherwise returns a pointer to a heap-allocated object that
   // describes the compaction.  Caller should delete the result.
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) = 0;
+  virtual Compaction* PickCompaction(
+      const MutableCFOptions& mutable_cf_options,
+      Version* version, LogBuffer* log_buffer) = 0;
 
   // Return a compaction object for compacting the range [begin,end] in
   // the specified level.  Returns nullptr if there is nothing in that
@@ -59,35 +51,15 @@ class CompactionPicker {
   // Client is responsible for compaction_end storage -- when called,
   // *compaction_end should point to valid InternalKey!
   virtual Compaction* CompactRange(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, int input_level, int output_level,
-      uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
+      const MutableCFOptions& mutable_cf_options, Version* version,
+      int input_level, int output_level, uint32_t output_path_id,
+      const InternalKey* begin, const InternalKey* end,
       InternalKey** compaction_end);
 
   // Given the current number of levels, returns the lowest allowed level
   // for compaction input.
   virtual int MaxInputLevel(int current_num_levels) const = 0;
 
-  // The maximum allowed output level.  Default value is NumberLevels() - 1.
-  virtual int MaxOutputLevel() const {
-    return NumberLevels() - 1;
-  }
-
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const = 0;
-
-  // Sanitize the input set of compaction input files.
-  // When the input parameters do not describe a valid compaction, the
-  // function will try to fix the input_files by adding necessary
-  // files.  If it's not possible to conver an invalid input_files
-  // into a valid one by adding more files, the function will return a
-  // non-ok status with specific reason.
-#ifndef ROCKSDB_LITE
-  Status SanitizeCompactionInputFiles(
-      std::unordered_set<uint64_t>* input_files,
-      const ColumnFamilyMetaData& cf_meta,
-      const int output_level) const;
-#endif  // ROCKSDB_LITE
-
   // Free up the files that participated in a compaction
   void ReleaseCompactionFiles(Compaction* c, Status status);
 
@@ -95,24 +67,6 @@ class CompactionPicker {
   // compactions per level
   void SizeBeingCompacted(std::vector<uint64_t>& sizes);
 
-  // Returns true if any one of the specified files are being compacted
-  bool FilesInCompaction(const std::vector<FileMetaData*>& files);
-
-  // Takes a list of CompactionInputFiles and returns a Compaction object.
-  Compaction* FormCompaction(
-      const CompactionOptions& compact_options,
-      const autovector<CompactionInputFiles>& input_files,
-      int output_level, VersionStorageInfo* vstorage,
-      const MutableCFOptions& mutable_cf_options) const;
-
-  // Converts a set of compaction input file numbers into
-  // a list of CompactionInputFiles.
-  Status GetCompactionInputsFromFileNumbers(
-      autovector<CompactionInputFiles>* input_files,
-      std::unordered_set<uint64_t>* input_set,
-      const VersionStorageInfo* vstorage,
-      const CompactionOptions& compact_options) const;
-
  protected:
   int NumberLevels() const { return ioptions_.num_levels; }
 
@@ -139,45 +93,69 @@ class CompactionPicker {
   // populated.
   //
   // Will return false if it is impossible to apply this compaction.
-  bool ExpandWhileOverlapping(const std::string& cf_name,
-                              VersionStorageInfo* vstorage, Compaction* c);
+  bool ExpandWhileOverlapping(Compaction* c);
+
+  // Returns true if any one of the specified files are being compacted
+  bool FilesInCompaction(std::vector<FileMetaData*>& files);
 
   // Returns true if any one of the parent files are being compacted
-  bool ParentRangeInCompaction(VersionStorageInfo* vstorage,
-                               const InternalKey* smallest,
+  bool ParentRangeInCompaction(Version* version, const InternalKey* smallest,
                                const InternalKey* largest, int level,
                                int* index);
 
-  void SetupOtherInputs(const std::string& cf_name,
-                        const MutableCFOptions& mutable_cf_options,
-                        VersionStorageInfo* vstorage, Compaction* c);
+  void SetupOtherInputs(const MutableCFOptions& mutable_cf_options,
+                        Compaction* c);
 
   const ImmutableCFOptions& ioptions_;
 
-  // A helper function to SanitizeCompactionInputFiles() that
-  // sanitizes "input_files" by adding necessary files.
-#ifndef ROCKSDB_LITE
-  virtual Status SanitizeCompactionInputFilesForAllLevels(
-      std::unordered_set<uint64_t>* input_files,
-      const ColumnFamilyMetaData& cf_meta,
-      const int output_level) const;
-#endif  // ROCKSDB_LITE
-
   // record all the ongoing compactions for all levels
   std::vector<std::set<Compaction*>> compactions_in_progress_;
 
+
+ private:
   const InternalKeyComparator* const icmp_;
 };
 
+class UniversalCompactionPicker : public CompactionPicker {
+ public:
+  UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
+                            const InternalKeyComparator* icmp)
+      : CompactionPicker(ioptions, icmp) {}
+  virtual Compaction* PickCompaction(
+      const MutableCFOptions& mutable_cf_options,
+      Version* version, LogBuffer* log_buffer) override;
+
+  // The maxinum allowed input level.  Always return 0.
+  virtual int MaxInputLevel(int current_num_levels) const override {
+    return 0;
+  }
+
+ private:
+  // Pick Universal compaction to limit read amplification
+  Compaction* PickCompactionUniversalReadAmp(
+      const MutableCFOptions& mutable_cf_options,
+      Version* version, double score, unsigned int ratio,
+      unsigned int num_files, LogBuffer* log_buffer);
+
+  // Pick Universal compaction to limit space amplification.
+  Compaction* PickCompactionUniversalSizeAmp(
+      const MutableCFOptions& mutable_cf_options,
+      Version* version, double score, LogBuffer* log_buffer);
+
+  // Pick a path ID to place a newly generated file, with its estimated file
+  // size.
+  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
+                            uint64_t file_size);
+};
+
 class LevelCompactionPicker : public CompactionPicker {
  public:
   LevelCompactionPicker(const ImmutableCFOptions& ioptions,
                         const InternalKeyComparator* icmp)
       : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) override;
+  virtual Compaction* PickCompaction(
+      const MutableCFOptions& mutable_cf_options,
+      Version* version, LogBuffer* log_buffer) override;
 
   // Returns current_num_levels - 2, meaning the last level cannot be
   // compaction input level.
@@ -185,59 +163,13 @@ class LevelCompactionPicker : public CompactionPicker {
     return current_num_levels - 2;
   }
 
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
-      override;
-
  private:
   // For the specfied level, pick a compaction.
   // Returns nullptr if there is no compaction to be done.
   // If level is 0 and there is already a compaction on that level, this
   // function will return nullptr.
   Compaction* PickCompactionBySize(const MutableCFOptions& mutable_cf_options,
-                                   VersionStorageInfo* vstorage, int level,
-                                   double score);
-};
-
-#ifndef ROCKSDB_LITE
-class UniversalCompactionPicker : public CompactionPicker {
- public:
-  UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
-                            const InternalKeyComparator* icmp)
-      : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) override;
-
-  // The maxinum allowed input level.  Always returns 0.
-  virtual int MaxInputLevel(int current_num_levels) const override {
-    return 0;
-  }
-
-  // The maximum allowed output level.  Always returns 0.
-  virtual int MaxOutputLevel() const override {
-    return 0;
-  }
-
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
-      override;
-
- private:
-  // Pick Universal compaction to limit read amplification
-  Compaction* PickCompactionUniversalReadAmp(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, double score, unsigned int ratio,
-      unsigned int num_files, LogBuffer* log_buffer);
-
-  // Pick Universal compaction to limit space amplification.
-  Compaction* PickCompactionUniversalSizeAmp(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, double score, LogBuffer* log_buffer);
-
-  // Pick a path ID to place a newly generated file, with its estimated file
-  // size.
-  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
-                            uint64_t file_size);
+      Version* version, int level, double score);
 };
 
 class FIFOCompactionPicker : public CompactionPicker {
@@ -246,69 +178,22 @@ class FIFOCompactionPicker : public CompactionPicker {
                        const InternalKeyComparator* icmp)
       : CompactionPicker(ioptions, icmp) {}
 
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     VersionStorageInfo* version,
-                                     LogBuffer* log_buffer) override;
+  virtual Compaction* PickCompaction(
+      const MutableCFOptions& mutable_cf_options,
+      Version* version, LogBuffer* log_buffer) override;
 
   virtual Compaction* CompactRange(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, int input_level, int output_level,
-      uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
+      const MutableCFOptions& mutable_cf_options, Version* version,
+      int input_level, int output_level, uint32_t output_path_id,
+      const InternalKey* begin, const InternalKey* end,
       InternalKey** compaction_end) override;
 
-  // The maxinum allowed input level.  Always returns 0.
+  // The maxinum allowed input level.  Always return 0.
   virtual int MaxInputLevel(int current_num_levels) const override {
     return 0;
   }
-
-  // The maximum allowed output level.  Always returns 0.
-  virtual int MaxOutputLevel() const override {
-    return 0;
-  }
-
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
-      override;
 };
 
-class NullCompactionPicker : public CompactionPicker {
- public:
-  NullCompactionPicker(const ImmutableCFOptions& ioptions,
-                       const InternalKeyComparator* icmp) :
-      CompactionPicker(ioptions, icmp) {}
-  virtual ~NullCompactionPicker() {}
-
-  // Always return "nullptr"
-  Compaction* PickCompaction(const std::string& cf_name,
-                             const MutableCFOptions& mutable_cf_options,
-                             VersionStorageInfo* vstorage,
-                             LogBuffer* log_buffer) override {
-    return nullptr;
-  }
-
-  // Always return "nullptr"
-  Compaction* CompactRange(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, int input_level, int output_level,
-      uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
-      InternalKey** compaction_end) override {
-    return nullptr;
-  }
-
-  // Given the current number of levels, returns the highest allowed level
-  // for compaction input.
-  virtual int MaxInputLevel(int current_num_levels) const {
-    return current_num_levels - 2;
-  }
-
-  // Always returns false.
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
-      override {
-    return false;
-  }
-};
-#endif  // !ROCKSDB_LITE
-
 // Utility function
 extern uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files);
 

db/compaction_picker_test.cc

@@ -1,258 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include "db/compaction_picker.h"
-#include <string>
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace rocksdb {
-
-class CountingLogger : public Logger {
- public:
-  virtual void Logv(const char* format, va_list ap) override { log_count++; }
-  size_t log_count;
-};
-
-class CompactionPickerTest {
- public:
-  const Comparator* ucmp_;
-  InternalKeyComparator icmp_;
-  Options options_;
-  ImmutableCFOptions ioptions_;
-  MutableCFOptions mutable_cf_options_;
-  LevelCompactionPicker level_compaction_picker;
-  std::string cf_name_;
-  CountingLogger logger_;
-  LogBuffer log_buffer_;
-  uint32_t file_num_;
-  CompactionOptionsFIFO fifo_options_;
-  std::vector<uint64_t> size_being_compacted_;
-  std::unique_ptr<VersionStorageInfo> vstorage_;
-  std::vector<std::unique_ptr<FileMetaData>> files_;
-
-  CompactionPickerTest()
-      : ucmp_(BytewiseComparator()),
-        icmp_(ucmp_),
-        ioptions_(options_),
-        mutable_cf_options_(options_, ioptions_),
-        level_compaction_picker(ioptions_, &icmp_),
-        cf_name_("dummy"),
-        log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
-        file_num_(1),
-        vstorage_(nullptr) {
-    fifo_options_.max_table_files_size = 1;
-    mutable_cf_options_.RefreshDerivedOptions(ioptions_);
-    size_being_compacted_.resize(options_.num_levels);
-  }
-
-  ~CompactionPickerTest() {
-  }
-
-  void NewVersionStorage(int num_levels, CompactionStyle style) {
-    DeleteVersionStorage();
-    options_.num_levels = num_levels;
-    vstorage_.reset(new VersionStorageInfo(
-        &icmp_, ucmp_, options_.num_levels, style, nullptr));
-  }
-
-  void DeleteVersionStorage() {
-    vstorage_.reset();
-    files_.clear();
-  }
-
-  void Add(int level, uint32_t file_number, const char* smallest,
-           const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
-           SequenceNumber smallest_seq = 100,
-           SequenceNumber largest_seq = 100) {
-    assert(level < vstorage_->num_levels());
-    FileMetaData* f = new FileMetaData;
-    f->fd = FileDescriptor(file_number, path_id, file_size);
-    f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
-    f->largest = InternalKey(largest, largest_seq, kTypeValue);
-    f->compensated_file_size = file_size;
-    f->refs = 0;
-    vstorage_->AddFile(level, f);
-    files_.emplace_back(f);
-  }
-
-  void UpdateVersionStorageInfo() {
-    vstorage_->ComputeCompactionScore(mutable_cf_options_, fifo_options_,
-                                    size_being_compacted_);
-    vstorage_->UpdateFilesBySize();
-    vstorage_->UpdateNumNonEmptyLevels();
-    vstorage_->GenerateFileIndexer();
-    vstorage_->GenerateLevelFilesBrief();
-    vstorage_->SetFinalized();
-  }
-};
-
-TEST(CompactionPickerTest, Empty) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  UpdateVersionStorageInfo();
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() == nullptr);
-}
-
-TEST(CompactionPickerTest, Single) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
-  Add(0, 1U, "p", "q");
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() == nullptr);
-}
-
-TEST(CompactionPickerTest, Level0Trigger) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
-  Add(0, 1U, "150", "200");
-  Add(0, 2U, "200", "250");
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
-}
-
-TEST(CompactionPickerTest, Level1Trigger) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(1, 66U, "150", "200", 1000000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
-}
-
-TEST(CompactionPickerTest, Level1Trigger2) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  Add(1, 66U, "150", "200", 1000000001U);
-  Add(1, 88U, "201", "300", 1000000000U);
-  Add(2, 6U, "150", "179", 1000000000U);
-  Add(2, 7U, "180", "220", 1000000000U);
-  Add(2, 8U, "221", "300", 1000000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(2U, compaction->num_input_files(1));
-  ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
-}
-
-TEST(CompactionPickerTest, LevelMaxScore) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.target_file_size_base = 10000000;
-  mutable_cf_options_.target_file_size_multiplier = 10;
-  Add(0, 1U, "150", "200", 1000000000U);
-  // Level 1 score 1.2
-  Add(1, 66U, "150", "200", 6000000U);
-  Add(1, 88U, "201", "300", 6000000U);
-  // Level 2 score 1.8. File 7 is the largest. Should be picked
-  Add(2, 6U, "150", "179", 60000000U);
-  Add(2, 7U, "180", "220", 60000001U);
-  Add(2, 8U, "221", "300", 60000000U);
-  // Level 3 score slightly larger than 1
-  Add(3, 26U, "150", "170", 260000000U);
-  Add(3, 27U, "171", "179", 260000000U);
-  Add(3, 28U, "191", "220", 260000000U);
-  Add(3, 29U, "221", "300", 260000000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
-}
-
-TEST(CompactionPickerTest, NeedsCompactionLevel) {
-  const int kLevels = 6;
-  const int kFileCount = 20;
-  for (int level = 0; level < kLevels - 1; ++level) {
-    uint64_t file_size =
-        mutable_cf_options_.MaxBytesForLevel(level) * 2 / kFileCount;
-    for (int file_count = 1; file_count <= kFileCount; ++file_count) {
-      // start a brand new version in each test.
-      NewVersionStorage(kLevels, kCompactionStyleLevel);
-      for (int i = 0; i < file_count; ++i) {
-        Add(level, i, ToString((i + 100) * 1000).c_str(),
-            ToString((i + 100) * 1000 + 999).c_str(),
-            file_size, 0, i * 100, i * 100 + 99);
-      }
-      UpdateVersionStorageInfo();
-      ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level);
-      ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
-                vstorage_->CompactionScore(0) >= 1);
-      // release the version storage
-      DeleteVersionStorage();
-    }
-  }
-}
-
-TEST(CompactionPickerTest, NeedsCompactionUniversal) {
-  NewVersionStorage(1, kCompactionStyleUniversal);
-  UniversalCompactionPicker universal_compaction_picker(
-      ioptions_, &icmp_);
-  // must return false when there's no files.
-  ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
-            false);
-
-  // verify the trigger given different number of L0 files.
-  for (int i = 1;
-       i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
-    Add(0, i, ToString((i + 100) * 1000).c_str(),
-        ToString((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
-        i * 100 + 99);
-    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
-              vstorage_->CompactionScore(0) >= 1);
-  }
-}
-
-TEST(CompactionPickerTest, NeedsCompactionFIFO) {
-  NewVersionStorage(1, kCompactionStyleFIFO);
-  const int kFileCount =
-      mutable_cf_options_.level0_file_num_compaction_trigger * 3;
-  const uint64_t kFileSize = 100000;
-  const uint64_t kMaxSize = kFileSize * kFileCount / 2;
-
-  fifo_options_.max_table_files_size = kMaxSize;
-  ioptions_.compaction_options_fifo = fifo_options_;
-  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
-
-  // must return false when there's no files.
-  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false);
-
-  // verify whether compaction is needed based on the current
-  // size of L0 files.
-  uint64_t current_size = 0;
-  for (int i = 1; i <= kFileCount; ++i) {
-    Add(0, i, ToString((i + 100) * 1000).c_str(),
-        ToString((i + 100) * 1000 + 999).c_str(),
-        kFileSize, 0, i * 100, i * 100 + 99);
-    current_size += kFileSize;
-    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
-              vstorage_->CompactionScore(0) >= 1);
-  }
-}
-
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/comparator_db_test.cc

@@ -1,434 +0,0 @@
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-#include <map>
-#include <string>
-
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "util/hash.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-#include "utilities/merge_operators.h"
-
-using std::unique_ptr;
-
-namespace rocksdb {
-namespace {
-
-static const Comparator* comparator;
-
-// A comparator for std::map, using comparator
-struct MapComparator {
-  bool operator()(const std::string& a, const std::string& b) const {
-    return comparator->Compare(a, b) < 0;
-  }
-};
-
-typedef std::map<std::string, std::string, MapComparator> KVMap;
-
-class KVIter : public Iterator {
- public:
-  explicit KVIter(const KVMap* map) : map_(map), iter_(map_->end()) {}
-  virtual bool Valid() const { return iter_ != map_->end(); }
-  virtual void SeekToFirst() { iter_ = map_->begin(); }
-  virtual void SeekToLast() {
-    if (map_->empty()) {
-      iter_ = map_->end();
-    } else {
-      iter_ = map_->find(map_->rbegin()->first);
-    }
-  }
-  virtual void Seek(const Slice& k) { iter_ = map_->lower_bound(k.ToString()); }
-  virtual void Next() { ++iter_; }
-  virtual void Prev() {
-    if (iter_ == map_->begin()) {
-      iter_ = map_->end();
-      return;
-    }
-    --iter_;
-  }
-
-  virtual Slice key() const { return iter_->first; }
-  virtual Slice value() const { return iter_->second; }
-  virtual Status status() const { return Status::OK(); }
-
- private:
-  const KVMap* const map_;
-  KVMap::const_iterator iter_;
-};
-
-void AssertItersEqual(Iterator* iter1, Iterator* iter2) {
-  ASSERT_EQ(iter1->Valid(), iter2->Valid());
-  if (iter1->Valid()) {
-    ASSERT_EQ(iter1->key().ToString(), iter2->key().ToString());
-    ASSERT_EQ(iter1->value().ToString(), iter2->value().ToString());
-  }
-}
-
-// Measuring operations on DB (expect to be empty).
-// source_strings are candidate keys
-void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
-                            Random* rnd, int num_writes, int num_iter_ops,
-                            int num_trigger_flush) {
-  KVMap map;
-
-  for (int i = 0; i < num_writes; i++) {
-    if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) {
-      db->Flush(FlushOptions());
-    }
-
-    int type = rnd->Uniform(2);
-    int index = rnd->Uniform(static_cast<int>(source_strings.size()));
-    auto& key = source_strings[index];
-    switch (type) {
-      case 0:
-        // put
-        map[key] = key;
-        ASSERT_OK(db->Put(WriteOptions(), key, key));
-        break;
-      case 1:
-        // delete
-        if (map.find(key) != map.end()) {
-          map.erase(key);
-        }
-        ASSERT_OK(db->Delete(WriteOptions(), key));
-        break;
-      default:
-        assert(false);
-    }
-  }
-
-  std::unique_ptr<Iterator> iter(db->NewIterator(ReadOptions()));
-  std::unique_ptr<Iterator> result_iter(new KVIter(&map));
-
-  bool is_valid = false;
-  for (int i = 0; i < num_iter_ops; i++) {
-    // Random walk and make sure iter and result_iter returns the
-    // same key and value
-    int type = rnd->Uniform(6);
-    ASSERT_OK(iter->status());
-    switch (type) {
-      case 0:
-        // Seek to First
-        iter->SeekToFirst();
-        result_iter->SeekToFirst();
-        break;
-      case 1:
-        // Seek to last
-        iter->SeekToLast();
-        result_iter->SeekToLast();
-        break;
-      case 2: {
-        // Seek to random key
-        auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
-        auto key = source_strings[key_idx];
-        iter->Seek(key);
-        result_iter->Seek(key);
-        break;
-      }
-      case 3:
-        // Next
-        if (is_valid) {
-          iter->Next();
-          result_iter->Next();
-        } else {
-          continue;
-        }
-        break;
-      case 4:
-        // Prev
-        if (is_valid) {
-          iter->Prev();
-          result_iter->Prev();
-        } else {
-          continue;
-        }
-        break;
-      default: {
-        assert(type == 5);
-        auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
-        auto key = source_strings[key_idx];
-        std::string result;
-        auto status = db->Get(ReadOptions(), key, &result);
-        if (map.find(key) == map.end()) {
-          ASSERT_TRUE(status.IsNotFound());
-        } else {
-          ASSERT_EQ(map[key], result);
-        }
-        break;
-      }
-    }
-    AssertItersEqual(iter.get(), result_iter.get());
-    is_valid = iter->Valid();
-  }
-}
-
-class DoubleComparator : public Comparator {
- public:
-  DoubleComparator() {}
-
-  virtual const char* Name() const { return "DoubleComparator"; }
-
-  virtual int Compare(const Slice& a, const Slice& b) const {
-    double da = std::stod(a.ToString());
-    double db = std::stod(b.ToString());
-    if (da == db) {
-      return a.compare(b);
-    } else if (da > db) {
-      return 1;
-    } else {
-      return -1;
-    }
-  }
-  virtual void FindShortestSeparator(std::string* start,
-                                     const Slice& limit) const {}
-
-  virtual void FindShortSuccessor(std::string* key) const {}
-};
-
-class HashComparator : public Comparator {
- public:
-  HashComparator() {}
-
-  virtual const char* Name() const { return "HashComparator"; }
-
-  virtual int Compare(const Slice& a, const Slice& b) const {
-    uint32_t ha = Hash(a.data(), a.size(), 66);
-    uint32_t hb = Hash(b.data(), b.size(), 66);
-    if (ha == hb) {
-      return a.compare(b);
-    } else if (ha > hb) {
-      return 1;
-    } else {
-      return -1;
-    }
-  }
-  virtual void FindShortestSeparator(std::string* start,
-                                     const Slice& limit) const {}
-
-  virtual void FindShortSuccessor(std::string* key) const {}
-};
-
-class TwoStrComparator : public Comparator {
- public:
-  TwoStrComparator() {}
-
-  virtual const char* Name() const { return "TwoStrComparator"; }
-
-  virtual int Compare(const Slice& a, const Slice& b) const {
-    assert(a.size() >= 2);
-    assert(b.size() >= 2);
-    size_t size_a1 = static_cast<size_t>(a[0]);
-    size_t size_b1 = static_cast<size_t>(b[0]);
-    size_t size_a2 = static_cast<size_t>(a[1]);
-    size_t size_b2 = static_cast<size_t>(b[1]);
-    assert(size_a1 + size_a2 + 2 == a.size());
-    assert(size_b1 + size_b2 + 2 == b.size());
-
-    Slice a1 = Slice(a.data() + 2, size_a1);
-    Slice b1 = Slice(b.data() + 2, size_b1);
-    Slice a2 = Slice(a.data() + 2 + size_a1, size_a2);
-    Slice b2 = Slice(b.data() + 2 + size_b1, size_b2);
-
-    if (a1 != b1) {
-      return a1.compare(b1);
-    }
-    return a2.compare(b2);
-  }
-  virtual void FindShortestSeparator(std::string* start,
-                                     const Slice& limit) const {}
-
-  virtual void FindShortSuccessor(std::string* key) const {}
-};
-}  // namespace
-
-class ComparatorDBTest {
- private:
-  std::string dbname_;
-  Env* env_;
-  DB* db_;
-  Options last_options_;
-  std::unique_ptr<const Comparator> comparator_guard;
-
- public:
-  ComparatorDBTest() : env_(Env::Default()), db_(nullptr) {
-    comparator = BytewiseComparator();
-    dbname_ = test::TmpDir() + "/comparator_db_test";
-    ASSERT_OK(DestroyDB(dbname_, last_options_));
-  }
-
-  ~ComparatorDBTest() {
-    delete db_;
-    ASSERT_OK(DestroyDB(dbname_, last_options_));
-    comparator = BytewiseComparator();
-  }
-
-  DB* GetDB() { return db_; }
-
-  void SetOwnedComparator(const Comparator* cmp) {
-    comparator_guard.reset(cmp);
-    comparator = cmp;
-    last_options_.comparator = cmp;
-  }
-
-  // Return the current option configuration.
-  Options* GetOptions() { return &last_options_; }
-
-  void DestroyAndReopen() {
-    // Destroy using last options
-    Destroy();
-    ASSERT_OK(TryReopen());
-  }
-
-  void Destroy() {
-    delete db_;
-    db_ = nullptr;
-    ASSERT_OK(DestroyDB(dbname_, last_options_));
-  }
-
-  Status TryReopen() {
-    delete db_;
-    db_ = nullptr;
-    last_options_.create_if_missing = true;
-
-    return DB::Open(last_options_, dbname_, &db_);
-  }
-};
-
-TEST(ComparatorDBTest, Bytewise) {
-  for (int rand_seed = 301; rand_seed < 306; rand_seed++) {
-    DestroyAndReopen();
-    Random rnd(rand_seed);
-    DoRandomIteraratorTest(GetDB(),
-                           {"a", "b", "c", "d", "e", "f", "g", "h", "i"}, &rnd,
-                           8, 100, 3);
-  }
-}
-
-TEST(ComparatorDBTest, SimpleSuffixReverseComparator) {
-  SetOwnedComparator(new test::SimpleSuffixReverseComparator());
-
-  for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
-    Options* opt = GetOptions();
-    opt->comparator = comparator;
-    DestroyAndReopen();
-    Random rnd(rnd_seed);
-
-    std::vector<std::string> source_strings;
-    std::vector<std::string> source_prefixes;
-    // Randomly generate 5 prefixes
-    for (int i = 0; i < 5; i++) {
-      source_prefixes.push_back(test::RandomHumanReadableString(&rnd, 8));
-    }
-    for (int j = 0; j < 20; j++) {
-      int prefix_index = rnd.Uniform(static_cast<int>(source_prefixes.size()));
-      std::string key = source_prefixes[prefix_index] +
-                        test::RandomHumanReadableString(&rnd, rnd.Uniform(8));
-      source_strings.push_back(key);
-    }
-
-    DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 30, 600, 66);
-  }
-}
-
-TEST(ComparatorDBTest, Uint64Comparator) {
-  SetOwnedComparator(test::Uint64Comparator());
-
-  for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
-    Options* opt = GetOptions();
-    opt->comparator = comparator;
-    DestroyAndReopen();
-    Random rnd(rnd_seed);
-    Random64 rnd64(rnd_seed);
-
-    std::vector<std::string> source_strings;
-    // Randomly generate source keys
-    for (int i = 0; i < 100; i++) {
-      uint64_t r = rnd64.Next();
-      std::string str;
-      str.resize(8);
-      memcpy(&str[0], static_cast<void*>(&r), 8);
-      source_strings.push_back(str);
-    }
-
-    DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
-  }
-}
-
-TEST(ComparatorDBTest, DoubleComparator) {
-  SetOwnedComparator(new DoubleComparator());
-
-  for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
-    Options* opt = GetOptions();
-    opt->comparator = comparator;
-    DestroyAndReopen();
-    Random rnd(rnd_seed);
-
-    std::vector<std::string> source_strings;
-    // Randomly generate source keys
-    for (int i = 0; i < 100; i++) {
-      uint32_t r = rnd.Next();
-      uint32_t divide_order = rnd.Uniform(8);
-      double to_divide = 1.0;
-      for (uint32_t j = 0; j < divide_order; j++) {
-        to_divide *= 10.0;
-      }
-      source_strings.push_back(ToString(r / to_divide));
-    }
-
-    DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
-  }
-}
-
-TEST(ComparatorDBTest, HashComparator) {
-  SetOwnedComparator(new HashComparator());
-
-  for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
-    Options* opt = GetOptions();
-    opt->comparator = comparator;
-    DestroyAndReopen();
-    Random rnd(rnd_seed);
-
-    std::vector<std::string> source_strings;
-    // Randomly generate source keys
-    for (int i = 0; i < 100; i++) {
-      source_strings.push_back(test::RandomKey(&rnd, 8));
-    }
-
-    DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
-  }
-}
-
-TEST(ComparatorDBTest, TwoStrComparator) {
-  SetOwnedComparator(new TwoStrComparator());
-
-  for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
-    Options* opt = GetOptions();
-    opt->comparator = comparator;
-    DestroyAndReopen();
-    Random rnd(rnd_seed);
-
-    std::vector<std::string> source_strings;
-    // Randomly generate source keys
-    for (int i = 0; i < 100; i++) {
-      std::string str;
-      uint32_t size1 = rnd.Uniform(8);
-      uint32_t size2 = rnd.Uniform(8);
-      str.append(1, static_cast<char>(size1));
-      str.append(1, static_cast<char>(size2));
-      str.append(test::RandomKey(&rnd, size1));
-      str.append(test::RandomKey(&rnd, size2));
-      source_strings.push_back(str);
-    }
-
-    DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
-  }
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/corruption_test.cc

@@ -115,8 +115,8 @@ class CorruptionTest {
         continue;
       }
       missed += (key - next_expected);
-      next_expected = static_cast<unsigned int>(key + 1);
-      if (iter->value() != Value(static_cast<int>(key), &value_space)) {
+      next_expected = key + 1;
+      if (iter->value() != Value(key, &value_space)) {
         bad_values++;
       } else {
         correct++;
@@ -143,14 +143,14 @@ class CorruptionTest {
       if (-offset > sbuf.st_size) {
         offset = 0;
       } else {
-        offset = static_cast<int>(sbuf.st_size + offset);
+        offset = sbuf.st_size + offset;
       }
     }
     if (offset > sbuf.st_size) {
-      offset = static_cast<int>(sbuf.st_size);
+      offset = sbuf.st_size;
     }
     if (offset + bytes_to_corrupt > sbuf.st_size) {
-      bytes_to_corrupt = static_cast<int>(sbuf.st_size - offset);
+      bytes_to_corrupt = sbuf.st_size - offset;
     }
 
     // Do it
@@ -177,7 +177,7 @@ class CorruptionTest {
           type == filetype &&
           static_cast<int>(number) > picked_number) {  // Pick latest file
         fname = dbname_ + "/" + filenames[i];
-        picked_number = static_cast<int>(number);
+        picked_number = number;
       }
     }
     ASSERT_TRUE(!fname.empty()) << filetype;
@@ -246,8 +246,7 @@ TEST(CorruptionTest, RecoverWriteError) {
 TEST(CorruptionTest, NewFileErrorDuringWrite) {
   // Do enough writing to force minor compaction
   env_.writable_file_error_ = true;
-  const int num =
-      static_cast<int>(3 + (Options().write_buffer_size / kValueSize));
+  const int num = 3 + (Options().write_buffer_size / kValueSize);
   std::string value_storage;
   Status s;
   bool failed = false;

db/cuckoo_table_db_test.cc

@@ -92,7 +92,7 @@ class CuckooTableDBTest {
   // Return spread of files per level
   std::string FilesPerLevel() {
     std::string result;
-    size_t last_non_zero_offset = 0;
+    int last_non_zero_offset = 0;
     for (int level = 0; level < db_->NumberLevels(); level++) {
       int f = NumTableFilesAtLevel(level);
       char buf[100];
@@ -228,6 +228,33 @@ TEST(CuckooTableDBTest, Uint64Comparator) {
   ASSERT_EQ("v4", Get(Uint64Key(4)));
 }
 
+TEST(CuckooTableDBTest, CompactionTrigger) {
+  Options options = CurrentOptions();
+  options.write_buffer_size = 100 << 10;  // 100KB
+  options.level0_file_num_compaction_trigger = 2;
+  Reopen(&options);
+
+  // Write 11 values, each 10016 B
+  for (int idx = 0; idx < 11; ++idx) {
+    ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + idx)));
+  }
+  dbfull()->TEST_WaitForFlushMemTable();
+  ASSERT_EQ("1", FilesPerLevel());
+
+  // Generate one more file in level-0, and should trigger level-0 compaction
+  for (int idx = 11; idx < 22; ++idx) {
+    ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + idx)));
+  }
+  dbfull()->TEST_WaitForFlushMemTable();
+  ASSERT_EQ("2", FilesPerLevel());
+
+  dbfull()->TEST_CompactRange(0, nullptr, nullptr);
+  ASSERT_EQ("0,2", FilesPerLevel());
+  for (int idx = 0; idx < 22; ++idx) {
+    ASSERT_EQ(std::string(10000, 'a' + idx), Get(Key(idx)));
+  }
+}
+
 TEST(CuckooTableDBTest, CompactionIntoMultipleFiles) {
   // Create a big L0 file and check it compacts into multiple files in L1.
   Options options = CurrentOptions();

db/db_bench.cc

@@ -168,9 +168,8 @@ DEFINE_int32(duration, 0, "Time in seconds for the random-ops tests to run."
 
 DEFINE_int32(value_size, 100, "Size of each value");
 
-DEFINE_int32(seek_nexts, 0,
-             "How many times to call Next() after Seek() in "
-             "fillseekseq and seekrandom");
+DEFINE_int32(seekseq_next, 0, "How many times to call Next() after Seek() in "
+             "fillseekseq");
 
 DEFINE_bool(use_uint64_comparator, false, "use Uint64 user comparator");
 
@@ -198,9 +197,6 @@ DEFINE_bool(enable_numa, false,
             "CPU and memory of same node. Use \"$numactl --hardware\" command "
             "to see NUMA memory architecture.");
 
-DEFINE_int64(db_write_buffer_size, rocksdb::Options().db_write_buffer_size,
-             "Number of bytes to buffer in all memtables before compacting");
-
 DEFINE_int64(write_buffer_size, rocksdb::Options().write_buffer_size,
              "Number of bytes to buffer in memtable before compacting");
 
@@ -254,8 +250,7 @@ DEFINE_int32(universal_compression_size_percent, -1,
 DEFINE_int64(cache_size, -1, "Number of bytes to use as a cache of uncompressed"
              "data. Negative means use default settings.");
 
-DEFINE_int32(block_size,
-             static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size),
+DEFINE_int32(block_size, rocksdb::BlockBasedTableOptions().block_size,
              "Number of bytes in a block.");
 
 DEFINE_int32(block_restart_interval,
@@ -579,8 +574,6 @@ DEFINE_string(merge_operator, "", "The merge operator to use with the database."
 DEFINE_int32(skip_list_lookahead, 0, "Used with skip_list memtablerep; try "
              "linear search first for this many steps from the previous "
              "position");
-DEFINE_bool(report_file_operations, false, "if report number of file "
-            "operations");
 
 static const bool FLAGS_soft_rate_limit_dummy __attribute__((unused)) =
     RegisterFlagValidator(&FLAGS_soft_rate_limit, &ValidateRateLimit);
@@ -612,131 +605,6 @@ static const bool FLAGS_table_cache_numshardbits_dummy __attribute__((unused)) =
 
 namespace rocksdb {
 
-namespace {
-struct ReportFileOpCounters {
-  std::atomic<int> open_counter_;
-  std::atomic<int> read_counter_;
-  std::atomic<int> append_counter_;
-  std::atomic<uint64_t> bytes_read_;
-  std::atomic<uint64_t> bytes_written_;
-};
-
-// A special Env to records and report file operations in db_bench
-class ReportFileOpEnv : public EnvWrapper {
- public:
-  explicit ReportFileOpEnv(Env* base) : EnvWrapper(base) { reset(); }
-
-  void reset() {
-    counters_.open_counter_ = 0;
-    counters_.read_counter_ = 0;
-    counters_.append_counter_ = 0;
-    counters_.bytes_read_ = 0;
-    counters_.bytes_written_ = 0;
-  }
-
-  Status NewSequentialFile(const std::string& f, unique_ptr<SequentialFile>* r,
-                           const EnvOptions& soptions) {
-    class CountingFile : public SequentialFile {
-     private:
-      unique_ptr<SequentialFile> target_;
-      ReportFileOpCounters* counters_;
-
-     public:
-      CountingFile(unique_ptr<SequentialFile>&& target,
-                   ReportFileOpCounters* counters)
-          : target_(std::move(target)), counters_(counters) {}
-
-      virtual Status Read(size_t n, Slice* result, char* scratch) {
-        counters_->read_counter_.fetch_add(1, std::memory_order_relaxed);
-        Status rv = target_->Read(n, result, scratch);
-        counters_->bytes_read_.fetch_add(result->size(),
-                                         std::memory_order_relaxed);
-        return rv;
-      }
-
-      virtual Status Skip(uint64_t n) { return target_->Skip(n); }
-    };
-
-    Status s = target()->NewSequentialFile(f, r, soptions);
-    if (s.ok()) {
-      counters()->open_counter_.fetch_add(1, std::memory_order_relaxed);
-      r->reset(new CountingFile(std::move(*r), counters()));
-    }
-    return s;
-  }
-
-  Status NewRandomAccessFile(const std::string& f,
-                             unique_ptr<RandomAccessFile>* r,
-                             const EnvOptions& soptions) {
-    class CountingFile : public RandomAccessFile {
-     private:
-      unique_ptr<RandomAccessFile> target_;
-      ReportFileOpCounters* counters_;
-
-     public:
-      CountingFile(unique_ptr<RandomAccessFile>&& target,
-                   ReportFileOpCounters* counters)
-          : target_(std::move(target)), counters_(counters) {}
-      virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                          char* scratch) const {
-        counters_->read_counter_.fetch_add(1, std::memory_order_relaxed);
-        Status rv = target_->Read(offset, n, result, scratch);
-        counters_->bytes_read_.fetch_add(result->size(),
-                                         std::memory_order_relaxed);
-        return rv;
-      }
-    };
-
-    Status s = target()->NewRandomAccessFile(f, r, soptions);
-    if (s.ok()) {
-      counters()->open_counter_.fetch_add(1, std::memory_order_relaxed);
-      r->reset(new CountingFile(std::move(*r), counters()));
-    }
-    return s;
-  }
-
-  Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
-                         const EnvOptions& soptions) {
-    class CountingFile : public WritableFile {
-     private:
-      unique_ptr<WritableFile> target_;
-      ReportFileOpCounters* counters_;
-
-     public:
-      CountingFile(unique_ptr<WritableFile>&& target,
-                   ReportFileOpCounters* counters)
-          : target_(std::move(target)), counters_(counters) {}
-
-      Status Append(const Slice& data) {
-        counters_->append_counter_.fetch_add(1, std::memory_order_relaxed);
-        Status rv = target_->Append(data);
-        counters_->bytes_written_.fetch_add(data.size(),
-                                            std::memory_order_relaxed);
-        return rv;
-      }
-
-      Status Close() { return target_->Close(); }
-      Status Flush() { return target_->Flush(); }
-      Status Sync() { return target_->Sync(); }
-    };
-
-    Status s = target()->NewWritableFile(f, r, soptions);
-    if (s.ok()) {
-      counters()->open_counter_.fetch_add(1, std::memory_order_relaxed);
-      r->reset(new CountingFile(std::move(*r), counters()));
-    }
-    return s;
-  }
-
-  // getter
-  ReportFileOpCounters* counters() { return &counters_; }
-
- private:
-  ReportFileOpCounters counters_;
-};
-
-}  // namespace
-
 // Helper for quickly generating random data.
 class RandomGenerator {
  private:
@@ -941,21 +809,6 @@ class Stats {
     if (FLAGS_histogram) {
       fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
     }
-    if (FLAGS_report_file_operations) {
-      ReportFileOpEnv* env = static_cast<ReportFileOpEnv*>(FLAGS_env);
-      ReportFileOpCounters* counters = env->counters();
-      fprintf(stdout, "Num files opened: %d\n",
-              counters->open_counter_.load(std::memory_order_relaxed));
-      fprintf(stdout, "Num Read(): %d\n",
-              counters->read_counter_.load(std::memory_order_relaxed));
-      fprintf(stdout, "Num Append(): %d\n",
-              counters->append_counter_.load(std::memory_order_relaxed));
-      fprintf(stdout, "Num bytes read: %" PRIu64 "\n",
-              counters->bytes_read_.load(std::memory_order_relaxed));
-      fprintf(stdout, "Num bytes written: %" PRIu64 "\n",
-              counters->bytes_written_.load(std::memory_order_relaxed));
-      env->reset();
-    }
     fflush(stdout);
   }
 };
@@ -1045,7 +898,6 @@ class Benchmark {
   int64_t writes_;
   int64_t readwrites_;
   int64_t merge_keys_;
-  bool report_file_operations_;
 
   bool SanityCheck() {
     if (FLAGS_compression_ratio > 1) {
@@ -1196,7 +1048,7 @@ class Benchmark {
     while (start < s.size() && isspace(s[start])) {
       start++;
     }
-    unsigned int limit = static_cast<unsigned int>(s.size());
+    unsigned int limit = s.size();
     while (limit > start && isspace(s[limit-1])) {
       limit--;
     }
@@ -1265,18 +1117,7 @@ class Benchmark {
     readwrites_((FLAGS_writes < 0  && FLAGS_reads < 0)? FLAGS_num :
                 ((FLAGS_writes > FLAGS_reads) ? FLAGS_writes : FLAGS_reads)
                ),
-    merge_keys_(FLAGS_merge_keys < 0 ? FLAGS_num : FLAGS_merge_keys),
-    report_file_operations_(FLAGS_report_file_operations) {
-    if (report_file_operations_) {
-      if (!FLAGS_hdfs.empty()) {
-        fprintf(stderr,
-                "--hdfs and --report_file_operations cannot be enabled "
-                "at the same time");
-        exit(1);
-      }
-      FLAGS_env = new ReportFileOpEnv(rocksdb::Env::Default());
-    }
-
+    merge_keys_(FLAGS_merge_keys < 0 ? FLAGS_num : FLAGS_merge_keys) {
     if (FLAGS_prefix_size > FLAGS_key_size) {
       fprintf(stderr, "prefix size is larger than key size");
       exit(1);
@@ -1353,7 +1194,7 @@ class Benchmark {
   }
 
   std::string GetDbNameForMultiple(std::string base_name, size_t id) {
-    return base_name + ToString(id);
+    return base_name + std::to_string(id);
   }
 
   std::string ColumnFamilyName(int i) {
@@ -1367,13 +1208,11 @@ class Benchmark {
   }
 
   void Run() {
-    Options open_options;  // keep options around to properly destroy db later
-
     if (!SanityCheck()) {
       exit(1);
     }
     PrintHeader();
-    Open(&open_options);
+    Open();
     const char* benchmarks = FLAGS_benchmarks.c_str();
     while (benchmarks != nullptr) {
       const char* sep = strchr(benchmarks, ',');
@@ -1392,7 +1231,7 @@ class Benchmark {
       writes_ = (FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes);
       value_size_ = FLAGS_value_size;
       key_size_ = FLAGS_key_size;
-      entries_per_batch_ = FLAGS_batch_size;
+      entries_per_batch_ = 1;
       write_options_ = WriteOptions();
       if (FLAGS_sync) {
         write_options_.sync = true;
@@ -1447,6 +1286,7 @@ class Benchmark {
       } else if (name == Slice("readrandomfast")) {
         method = &Benchmark::ReadRandomFast;
       } else if (name == Slice("multireadrandom")) {
+        entries_per_batch_ = FLAGS_batch_size;
         fprintf(stderr, "entries_per_batch = %" PRIi64 "\n",
                 entries_per_batch_);
         method = &Benchmark::MultiReadRandom;
@@ -1534,15 +1374,15 @@ class Benchmark {
             delete db_.db;
             db_.db = nullptr;
             db_.cfh.clear();
-            DestroyDB(FLAGS_db, open_options);
+            DestroyDB(FLAGS_db, Options());
           }
           for (size_t i = 0; i < multi_dbs_.size(); i++) {
             delete multi_dbs_[i].db;
-            DestroyDB(GetDbNameForMultiple(FLAGS_db, i), open_options);
+            DestroyDB(GetDbNameForMultiple(FLAGS_db, i), Options());
           }
           multi_dbs_.clear();
         }
-        Open(&open_options);  // use open_options for the last accessed
+        Open();
       }
 
       if (method != nullptr) {
@@ -1693,13 +1533,13 @@ class Benchmark {
 
   void AcquireLoad(ThreadState* thread) {
     int dummy;
-    std::atomic<void*> ap(&dummy);
+    port::AtomicPointer ap(&dummy);
     int count = 0;
     void *ptr = nullptr;
     thread->stats.AddMessage("(each op is 1000 loads)");
     while (count < 100000) {
       for (int i = 0; i < 1000; i++) {
-        ptr = ap.load(std::memory_order_acquire);
+        ptr = ap.Acquire_Load();
       }
       count++;
       thread->stats.FinishedOps(nullptr, nullptr, 1);
@@ -1834,14 +1674,11 @@ class Benchmark {
     }
   }
 
-  void Open(Options* opts) {
-    Options& options = *opts;
-
+  void Open() {
     assert(db_.db == nullptr);
-
+    Options options;
     options.create_if_missing = !FLAGS_use_existing_db;
     options.create_missing_column_families = FLAGS_num_column_families > 1;
-    options.db_write_buffer_size = FLAGS_db_write_buffer_size;
     options.write_buffer_size = FLAGS_write_buffer_size;
     options.max_write_buffer_number = FLAGS_max_write_buffer_number;
     options.min_write_buffer_number_to_merge =
@@ -1886,15 +1723,14 @@ class Benchmark {
       exit(1);
     }
     switch (FLAGS_rep_factory) {
+      case kPrefixHash:
+        options.memtable_factory.reset(NewHashSkipListRepFactory(
+            FLAGS_hash_bucket_count));
+        break;
       case kSkipList:
         options.memtable_factory.reset(new SkipListFactory(
             FLAGS_skip_list_lookahead));
         break;
-#ifndef ROCKSDB_LITE
-      case kPrefixHash:
-        options.memtable_factory.reset(
-            NewHashSkipListRepFactory(FLAGS_hash_bucket_count));
-        break;
       case kHashLinkedList:
         options.memtable_factory.reset(NewHashLinkListRepFactory(
             FLAGS_hash_bucket_count));
@@ -1908,14 +1744,8 @@ class Benchmark {
         options.memtable_factory.reset(NewHashCuckooRepFactory(
             options.write_buffer_size, FLAGS_key_size + FLAGS_value_size));
         break;
-#else
-      default:
-        fprintf(stderr, "Only skip list is supported in lite mode\n");
-        exit(1);
-#endif  // ROCKSDB_LITE
     }
     if (FLAGS_use_plain_table) {
-#ifndef ROCKSDB_LITE
       if (FLAGS_rep_factory != kPrefixHash &&
           FLAGS_rep_factory != kHashLinkedList) {
         fprintf(stderr, "Waring: plain table is used with skipList\n");
@@ -1936,12 +1766,7 @@ class Benchmark {
       plain_table_options.hash_table_ratio = 0.75;
       options.table_factory = std::shared_ptr<TableFactory>(
           NewPlainTableFactory(plain_table_options));
-#else
-      fprintf(stderr, "Plain table is not supported in lite mode\n");
-      exit(1);
-#endif  // ROCKSDB_LITE
     } else if (FLAGS_use_cuckoo_table) {
-#ifndef ROCKSDB_LITE
       if (FLAGS_cuckoo_hash_ratio > 1 || FLAGS_cuckoo_hash_ratio < 0) {
         fprintf(stderr, "Invalid cuckoo_hash_ratio\n");
         exit(1);
@@ -1951,10 +1776,6 @@ class Benchmark {
       table_options.identity_as_first_hash = FLAGS_identity_as_first_hash;
       options.table_factory = std::shared_ptr<TableFactory>(
           NewCuckooTableFactory(table_options));
-#else
-      fprintf(stderr, "Cuckoo table is not supported in lite mode\n");
-      exit(1);
-#endif  // ROCKSDB_LITE
     } else {
       BlockBasedTableOptions block_based_options;
       if (FLAGS_use_hash_search) {
@@ -2136,9 +1957,8 @@ class Benchmark {
         for (uint64_t i = 0; i < num_; ++i) {
           values_[i] = i;
         }
-        std::shuffle(
-            values_.begin(), values_.end(),
-            std::default_random_engine(static_cast<unsigned int>(FLAGS_seed)));
+        std::shuffle(values_.begin(), values_.end(),
+            std::default_random_engine(FLAGS_seed));
       }
     }
 
@@ -2246,10 +2066,7 @@ class Benchmark {
   }
 
   void ReadSequential(ThreadState* thread, DB* db) {
-    ReadOptions options(FLAGS_verify_checksum, true);
-    options.tailing = FLAGS_use_tailing_iterator;
-
-    Iterator* iter = db->NewIterator(options);
+    Iterator* iter = db->NewIterator(ReadOptions(FLAGS_verify_checksum, true));
     int64_t i = 0;
     int64_t bytes = 0;
     for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
@@ -2448,7 +2265,6 @@ class Benchmark {
     std::unique_ptr<const char[]> key_guard(key.data());
 
     Duration duration(FLAGS_duration, reads_);
-    char value_buffer[256];
     while (!duration.Done(1)) {
       if (!FLAGS_use_tailing_iterator && FLAGS_iter_refresh_interval_us >= 0) {
         uint64_t now = FLAGS_env->NowMicros();
@@ -2480,16 +2296,6 @@ class Benchmark {
       if (iter_to_use->Valid() && iter_to_use->key().compare(key) == 0) {
         found++;
       }
-
-      for (int j = 0; j < FLAGS_seek_nexts && iter_to_use->Valid(); ++j) {
-        // Copy out iterator's value to make sure we read them.
-        Slice value = iter_to_use->value();
-        memcpy(value_buffer, value.data(),
-               std::min(value.size(), sizeof(value_buffer)));
-        iter_to_use->Next();
-        assert(iter_to_use->status().ok());
-      }
-
       thread->stats.FinishedOps(&db_, db_.db, 1);
     }
     delete single_iter;
@@ -3014,7 +2820,7 @@ class Benchmark {
       assert(iter->Valid() && iter->key() == key);
       thread->stats.FinishedOps(nullptr, db, 1);
 
-      for (int j = 0; j < FLAGS_seek_nexts && i + 1 < FLAGS_num; ++j) {
+      for (int j = 0; j < FLAGS_seekseq_next && i+1 < FLAGS_num; ++j) {
         iter->Next();
         GenerateKeyFromInt(++i, FLAGS_num, &key);
         assert(iter->Valid() && iter->key() == key);
@@ -3066,8 +2872,8 @@ int main(int argc, char** argv) {
     dbstats = rocksdb::CreateDBStatistics();
   }
 
-  std::vector<std::string> fanout = rocksdb::StringSplit(
-      FLAGS_max_bytes_for_level_multiplier_additional, ',');
+  std::vector<std::string> fanout =
+    rocksdb::stringSplit(FLAGS_max_bytes_for_level_multiplier_additional, ',');
   for (unsigned int j= 0; j < fanout.size(); j++) {
     FLAGS_max_bytes_for_level_multiplier_additional_v.push_back(
       std::stoi(fanout[j]));
@@ -3098,9 +2904,6 @@ int main(int argc, char** argv) {
   // The number of background threads should be at least as much the
   // max number of concurrent compactions.
   FLAGS_env->SetBackgroundThreads(FLAGS_max_background_compactions);
-  FLAGS_env->SetBackgroundThreads(FLAGS_max_background_flushes,
-                                  rocksdb::Env::Priority::HIGH);
-
   // Choose a location for the test database if none given with --db=<path>
   if (FLAGS_db.empty()) {
     std::string default_db_path;

db/db_filesnapshot.cc

@@ -19,14 +19,12 @@
 #include <stdint.h>
 #include "db/db_impl.h"
 #include "db/filename.h"
-#include "db/job_context.h"
 #include "db/version_set.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "port/port.h"
 #include "util/mutexlock.h"
 #include "util/sync_point.h"
-#include "util/file_util.h"
 
 namespace rocksdb {
 
@@ -34,10 +32,9 @@ Status DBImpl::DisableFileDeletions() {
   MutexLock l(&mutex_);
   ++disable_delete_obsolete_files_;
   if (disable_delete_obsolete_files_ == 1) {
-    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-        "File Deletions Disabled");
+    Log(db_options_.info_log, "File Deletions Disabled");
   } else {
-    Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
+    Log(db_options_.info_log,
         "File Deletions Disabled, but already disabled. Counter: %d",
         disable_delete_obsolete_files_);
   }
@@ -45,7 +42,7 @@ Status DBImpl::DisableFileDeletions() {
 }
 
 Status DBImpl::EnableFileDeletions(bool force) {
-  JobContext job_context;
+  DeletionState deletion_state;
   bool should_purge_files = false;
   {
     MutexLock l(&mutex_);
@@ -56,20 +53,18 @@ Status DBImpl::EnableFileDeletions(bool force) {
       --disable_delete_obsolete_files_;
     }
     if (disable_delete_obsolete_files_ == 0)  {
-      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-          "File Deletions Enabled");
+      Log(db_options_.info_log, "File Deletions Enabled");
       should_purge_files = true;
-      FindObsoleteFiles(&job_context, true);
+      FindObsoleteFiles(deletion_state, true);
     } else {
-      Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
+      Log(db_options_.info_log,
           "File Deletions Enable, but not really enabled. Counter: %d",
           disable_delete_obsolete_files_);
     }
   }
   if (should_purge_files)  {
-    PurgeObsoleteFiles(job_context);
+    PurgeObsoleteFiles(deletion_state);
   }
-  job_context.Clean();
   LogFlush(db_options_.info_log);
   return Status::OK();
 }
@@ -103,8 +98,8 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
 
     if (!status.ok()) {
       mutex_.Unlock();
-      Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
-          "Cannot Flush data %s\n", status.ToString().c_str());
+      Log(db_options_.info_log, "Cannot Flush data %s\n",
+          status.ToString().c_str());
       return status;
     }
   }
@@ -125,17 +120,65 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
   }
 
   ret.push_back(CurrentFileName(""));
-  ret.push_back(DescriptorFileName("", versions_->manifest_file_number()));
+  ret.push_back(DescriptorFileName("", versions_->ManifestFileNumber()));
 
   // find length of manifest file while holding the mutex lock
-  *manifest_file_size = versions_->manifest_file_size();
+  *manifest_file_size = versions_->ManifestFileSize();
 
   mutex_.Unlock();
   return Status::OK();
 }
 
 Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
-  return wal_manager_.GetSortedWalFiles(files);
+  // First get sorted files in db dir, then get sorted files from archived
+  // dir, to avoid a race condition where a log file is moved to archived
+  // dir in between.
+  Status s;
+  // list wal files in main db dir.
+  VectorLogPtr logs;
+  s = GetSortedWalsOfType(db_options_.wal_dir, logs, kAliveLogFile);
+  if (!s.ok()) {
+    return s;
+  }
+
+  // Reproduce the race condition where a log file is moved
+  // to archived dir, between these two sync points, used in
+  // (DBTest,TransactionLogIteratorRace)
+  TEST_SYNC_POINT("DBImpl::GetSortedWalFiles:1");
+  TEST_SYNC_POINT("DBImpl::GetSortedWalFiles:2");
+
+  files.clear();
+  // list wal files in archive dir.
+  std::string archivedir = ArchivalDirectory(db_options_.wal_dir);
+  if (env_->FileExists(archivedir)) {
+    s = GetSortedWalsOfType(archivedir, files, kArchivedLogFile);
+    if (!s.ok()) {
+      return s;
+    }
+  }
+
+  uint64_t latest_archived_log_number = 0;
+  if (!files.empty()) {
+    latest_archived_log_number = files.back()->LogNumber();
+    Log(db_options_.info_log, "Latest Archived log: %" PRIu64,
+        latest_archived_log_number);
+  }
+
+  files.reserve(files.size() + logs.size());
+  for (auto& log : logs) {
+    if (log->LogNumber() > latest_archived_log_number) {
+      files.push_back(std::move(log));
+    } else {
+      // When the race condition happens, we could see the
+      // same log in both db dir and archived dir. Simply
+      // ignore the one in db dir. Note that, if we read
+      // archived dir first, we would have missed the log file.
+      Log(db_options_.info_log, "%s already moved to archive",
+          log->PathName().c_str());
+    }
+  }
+
+  return s;
 }
 
 }

db/db_impl.h

@@ -12,9 +12,7 @@
 #include <deque>
 #include <limits>
 #include <set>
-#include <list>
 #include <utility>
-#include <list>
 #include <vector>
 #include <string>
 
@@ -23,8 +21,6 @@
 #include "db/snapshot.h"
 #include "db/column_family.h"
 #include "db/version_edit.h"
-#include "db/wal_manager.h"
-#include "db/writebuffer.h"
 #include "memtable_list.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
@@ -35,7 +31,6 @@
 #include "util/stop_watch.h"
 #include "util/thread_local.h"
 #include "util/scoped_arena_iterator.h"
-#include "util/hash.h"
 #include "db/internal_stats.h"
 #include "db/write_controller.h"
 #include "db/flush_scheduler.h"
@@ -50,7 +45,6 @@ class VersionEdit;
 class VersionSet;
 class CompactionFilterV2;
 class Arena;
-struct JobContext;
 
 class DBImpl : public DB {
  public:
@@ -118,15 +112,8 @@ class DBImpl : public DB {
                               bool reduce_level = false, int target_level = -1,
                               uint32_t target_path_id = 0);
 
-  using DB::CompactFiles;
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1);
-
   using DB::SetOptions;
-  Status SetOptions(ColumnFamilyHandle* column_family,
+  bool SetOptions(ColumnFamilyHandle* column_family,
       const std::unordered_map<std::string, std::string>& options_map);
 
   using DB::NumberLevels;
@@ -162,15 +149,6 @@ class DBImpl : public DB {
   virtual Status DeleteFile(std::string name);
 
   virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
-
-  // Obtains the meta data of the specified column family of the DB.
-  // Status::NotFound() will be returned if the current DB does not have
-  // any column family match the specified name.
-  // TODO(yhchiang): output parameter is placed in the end in this codebase.
-  virtual void GetColumnFamilyMetaData(
-      ColumnFamilyHandle* column_family,
-      ColumnFamilyMetaData* metadata) override;
-
 #endif  // ROCKSDB_LITE
 
   // checks if all live files exist on file system and that their file sizes
@@ -214,12 +192,25 @@ class DBImpl : public DB {
   // Return the current manifest file no.
   uint64_t TEST_Current_Manifest_FileNo();
 
+  // Trigger's a background call for testing.
+  void TEST_PurgeObsoleteteWAL();
+
   // get total level0 file size. Only for testing.
   uint64_t TEST_GetLevel0TotalSize();
 
+  void TEST_SetDefaultTimeToCheck(uint64_t default_interval_to_delete_obsolete_WAL)
+  {
+    default_interval_to_delete_obsolete_WAL_ = default_interval_to_delete_obsolete_WAL;
+  }
+
   void TEST_GetFilesMetaData(ColumnFamilyHandle* column_family,
                              std::vector<std::vector<FileMetaData>>* metadata);
 
+  Status TEST_ReadFirstRecord(const WalFileType type, const uint64_t number,
+                              SequenceNumber* sequence);
+
+  Status TEST_ReadFirstLine(const std::string& fname, SequenceNumber* sequence);
+
   void TEST_LockMutex();
 
   void TEST_UnlockMutex();
@@ -230,30 +221,93 @@ class DBImpl : public DB {
   // REQUIRES: mutex locked
   // pass the pointer that you got from TEST_BeginWrite()
   void TEST_EndWrite(void* w);
+#endif  // NDEBUG
 
-  uint64_t TEST_max_total_in_memory_state() {
-    return max_total_in_memory_state_;
-  }
-#endif  // ROCKSDB_LITE
+  // Structure to store information for candidate files to delete.
+  struct CandidateFileInfo {
+    std::string file_name;
+    uint32_t path_id;
+    CandidateFileInfo(std::string name, uint32_t path)
+        : file_name(name), path_id(path) {}
+    bool operator==(const CandidateFileInfo& other) const {
+      return file_name == other.file_name && path_id == other.path_id;
+    }
+  };
+
+  // needed for CleanupIteratorState
+  struct DeletionState {
+    inline bool HaveSomethingToDelete() const {
+      return  candidate_files.size() ||
+        sst_delete_files.size() ||
+        log_delete_files.size();
+    }
+
+    // a list of all files that we'll consider deleting
+    // (every once in a while this is filled up with all files
+    // in the DB directory)
+    std::vector<CandidateFileInfo> candidate_files;
+
+    // the list of all live sst files that cannot be deleted
+    std::vector<FileDescriptor> sst_live;
+
+    // a list of sst files that we need to delete
+    std::vector<FileMetaData*> sst_delete_files;
+
+    // a list of log files that we need to delete
+    std::vector<uint64_t> log_delete_files;
+
+    // a list of memtables to be free
+    autovector<MemTable*> memtables_to_free;
+
+    autovector<SuperVersion*> superversions_to_free;
+
+    SuperVersion* new_superversion;  // if nullptr no new superversion
+
+    // the current manifest_file_number, log_number and prev_log_number
+    // that corresponds to the set of files in 'live'.
+    uint64_t manifest_file_number, pending_manifest_file_number, log_number,
+        prev_log_number;
+
+    explicit DeletionState(bool create_superversion = false) {
+      manifest_file_number = 0;
+      pending_manifest_file_number = 0;
+      log_number = 0;
+      prev_log_number = 0;
+      new_superversion = create_superversion ? new SuperVersion() : nullptr;
+    }
+
+    ~DeletionState() {
+      // free pending memtables
+      for (auto m : memtables_to_free) {
+        delete m;
+      }
+      // free superversions
+      for (auto s : superversions_to_free) {
+        delete s;
+      }
+      // if new_superversion was not used, it will be non-nullptr and needs
+      // to be freed here
+      delete new_superversion;
+    }
+  };
 
   // Returns the list of live files in 'live' and the list
   // of all files in the filesystem in 'candidate_files'.
   // If force == false and the last call was less than
   // db_options_.delete_obsolete_files_period_micros microseconds ago,
-  // it will not fill up the job_context
-  void FindObsoleteFiles(JobContext* job_context, bool force,
+  // it will not fill up the deletion_state
+  void FindObsoleteFiles(DeletionState& deletion_state,
+                         bool force,
                          bool no_full_scan = false);
 
   // Diffs the files listed in filenames and those that do not
   // belong to live files are posibly removed. Also, removes all the
   // files in sst_delete_files and log_delete_files.
   // It is not necessary to hold the mutex when invoking this method.
-  void PurgeObsoleteFiles(const JobContext& background_contet);
+  void PurgeObsoleteFiles(DeletionState& deletion_state);
 
   ColumnFamilyHandle* DefaultColumnFamily() const;
 
-  const SnapshotList& snapshots() const { return snapshots_; }
-
  protected:
   Env* const env_;
   const std::string dbname_;
@@ -264,9 +318,6 @@ class DBImpl : public DB {
   Iterator* NewInternalIterator(const ReadOptions&, ColumnFamilyData* cfd,
                                 SuperVersion* super_version, Arena* arena);
 
-  void NotifyOnFlushCompleted(ColumnFamilyData* cfd, uint64_t file_number,
-                              const MutableCFOptions& mutable_cf_options);
-
  private:
   friend class DB;
   friend class InternalStats;
@@ -294,30 +345,11 @@ class DBImpl : public DB {
   // Delete any unneeded files and stale in-memory entries.
   void DeleteObsoleteFiles();
 
-  // Background process needs to call
-  //     auto x = CaptureCurrentFileNumberInPendingOutputs()
-  //     <do something>
-  //     ReleaseFileNumberFromPendingOutputs(x)
-  // This will protect any temporary files created while <do something> is
-  // executing from being deleted.
-  // -----------
-  // This function will capture current file number and append it to
-  // pending_outputs_. This will prevent any background process to delete any
-  // file created after this point.
-  std::list<uint64_t>::iterator CaptureCurrentFileNumberInPendingOutputs();
-  // This function should be called with the result of
-  // CaptureCurrentFileNumberInPendingOutputs(). It then marks that any file
-  // created between the calls CaptureCurrentFileNumberInPendingOutputs() and
-  // ReleaseFileNumberFromPendingOutputs() can now be deleted (if it's not live
-  // and blocked by any other pending_outputs_ calls)
-  void ReleaseFileNumberFromPendingOutputs(std::list<uint64_t>::iterator v);
-
   // Flush the in-memory write buffer to storage.  Switches to a new
   // log-file/memtable and writes a new descriptor iff successful.
-  Status FlushMemTableToOutputFile(ColumnFamilyData* cfd,
-                                   const MutableCFOptions& mutable_cf_options,
-                                   bool* madeProgress, JobContext* job_context,
-                                   LogBuffer* log_buffer);
+  Status FlushMemTableToOutputFile(
+      ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
+      bool* madeProgress, DeletionState& deletion_state, LogBuffer* log_buffer);
 
   // REQUIRES: log_numbers are sorted in ascending order
   Status RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
@@ -330,7 +362,12 @@ class DBImpl : public DB {
   // concurrent flush memtables to storage.
   Status WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
                                      VersionEdit* edit);
-  Status DelayWrite(uint64_t expiration_time);
+  Status WriteLevel0Table(ColumnFamilyData* cfd,
+      const MutableCFOptions& mutable_cf_options,
+      const autovector<MemTable*>& mems,
+      VersionEdit* edit, uint64_t* filenumber, LogBuffer* log_buffer);
+
+  void DelayWrite(uint64_t expiration_time);
 
   Status ScheduleFlushes(WriteContext* context);
 
@@ -346,31 +383,80 @@ class DBImpl : public DB {
   void RecordFlushIOStats();
   void RecordCompactionIOStats();
 
-#ifndef ROCKSDB_LITE
-  Status CompactFilesImpl(
-      const CompactionOptions& compact_options, ColumnFamilyData* cfd,
-      Version* version, const std::vector<std::string>& input_file_names,
-      const int output_level, int output_path_id);
-#endif  // ROCKSDB_LITE
-
-  ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name);
-
   void MaybeScheduleFlushOrCompaction();
   static void BGWorkCompaction(void* db);
   static void BGWorkFlush(void* db);
   void BackgroundCallCompaction();
   void BackgroundCallFlush();
-  Status BackgroundCompaction(bool* madeProgress, JobContext* job_context,
+  Status BackgroundCompaction(bool* madeProgress, DeletionState& deletion_state,
                               LogBuffer* log_buffer);
-  Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
+  Status BackgroundFlush(bool* madeProgress, DeletionState& deletion_state,
                          LogBuffer* log_buffer);
+  void CleanupCompaction(CompactionState* compact, Status status);
+  Status DoCompactionWork(CompactionState* compact,
+                          const MutableCFOptions& mutable_cf_options,
+                          DeletionState& deletion_state,
+                          LogBuffer* log_buffer);
 
   // This function is called as part of compaction. It enables Flush process to
   // preempt compaction, since it's higher prioirty
+  // Returns: micros spent executing
   uint64_t CallFlushDuringCompaction(ColumnFamilyData* cfd,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     JobContext* job_context,
-                                     LogBuffer* log_buffer);
+      const MutableCFOptions& mutable_cf_options, DeletionState& deletion_state,
+      LogBuffer* log_buffer);
+
+  // Call compaction filter if is_compaction_v2 is not true. Then iterate
+  // through input and compact the kv-pairs
+  Status ProcessKeyValueCompaction(
+    const MutableCFOptions& mutable_cf_options,
+    bool is_snapshot_supported,
+    SequenceNumber visible_at_tip,
+    SequenceNumber earliest_snapshot,
+    SequenceNumber latest_snapshot,
+    DeletionState& deletion_state,
+    bool bottommost_level,
+    int64_t& imm_micros,
+    Iterator* input,
+    CompactionState* compact,
+    bool is_compaction_v2,
+    int* num_output_records,
+    LogBuffer* log_buffer);
+
+  // Call compaction_filter_v2->Filter() on kv-pairs in compact
+  void CallCompactionFilterV2(CompactionState* compact,
+    CompactionFilterV2* compaction_filter_v2);
+
+  Status OpenCompactionOutputFile(CompactionState* compact,
+      const MutableCFOptions& mutable_cf_options);
+  Status FinishCompactionOutputFile(CompactionState* compact, Iterator* input);
+  Status InstallCompactionResults(CompactionState* compact,
+      const MutableCFOptions& mutable_cf_options, LogBuffer* log_buffer);
+  void AllocateCompactionOutputFileNumbers(CompactionState* compact);
+  void ReleaseCompactionUnusedFileNumbers(CompactionState* compact);
+
+#ifdef ROCKSDB_LITE
+  void PurgeObsoleteWALFiles() {
+    // this function is used for archiving WAL files. we don't need this in
+    // ROCKSDB_LITE
+  }
+#else
+  void PurgeObsoleteWALFiles();
+
+  Status GetSortedWalsOfType(const std::string& path,
+                             VectorLogPtr& log_files,
+                             WalFileType type);
+
+  // Requires: all_logs should be sorted with earliest log file first
+  // Retains all log files in all_logs which contain updates with seq no.
+  // Greater Than or Equal to the requested SequenceNumber.
+  Status RetainProbableWalFiles(VectorLogPtr& all_logs,
+                                const SequenceNumber target);
+
+  Status ReadFirstRecord(const WalFileType type, const uint64_t number,
+                         SequenceNumber* sequence);
+
+  Status ReadFirstLine(const std::string& fname, SequenceNumber* sequence);
+#endif  // ROCKSDB_LITE
 
   void PrintStatistics();
 
@@ -402,7 +488,7 @@ class DBImpl : public DB {
 
   // State below is protected by mutex_
   port::Mutex mutex_;
-  std::atomic<bool> shutting_down_;
+  port::AtomicPointer shutting_down_;
   // This condition variable is signaled on these conditions:
   // * whenever bg_compaction_scheduled_ goes down to 0
   // * if bg_manual_only_ > 0, whenever a compaction finishes, even if it hasn't
@@ -437,8 +523,6 @@ class DBImpl : public DB {
 
   std::unique_ptr<Directory> db_directory_;
 
-  WriteBuffer write_buffer_;
-
   WriteThread write_thread_;
 
   WriteBatch tmp_batch_;
@@ -448,16 +532,14 @@ class DBImpl : public DB {
 
   SnapshotList snapshots_;
 
-  // For each background job, pending_outputs_ keeps the current file number at
-  // the time that background job started.
-  // FindObsoleteFiles()/PurgeObsoleteFiles() never deletes any file that has
-  // number bigger than any of the file number in pending_outputs_. Since file
-  // numbers grow monotonically, this also means that pending_outputs_ is always
-  // sorted. After a background job is done executing, its file number is
-  // deleted from pending_outputs_, which allows PurgeObsoleteFiles() to clean
-  // it up.
-  // State is protected with db mutex.
-  std::list<uint64_t> pending_outputs_;
+  // cache for ReadFirstRecord() calls
+  std::unordered_map<uint64_t, SequenceNumber> read_first_record_cache_;
+  port::Mutex read_first_record_cache_mutex_;
+
+  // Set of table files to protect from deletion because they are
+  // part of ongoing compactions.
+  // map from pending file number ID to their path IDs.
+  FileNumToPathIdMap pending_outputs_;
 
   // At least one compaction or flush job is pending but not yet scheduled
   // because of the max background thread limit.
@@ -503,9 +585,16 @@ class DBImpl : public DB {
   // last time when DeleteObsoleteFiles was invoked
   uint64_t delete_obsolete_files_last_run_;
 
+  // last time when PurgeObsoleteWALFiles ran.
+  uint64_t purge_wal_files_last_run_;
+
   // last time stats were dumped to LOG
   std::atomic<uint64_t> last_stats_dump_time_microsec_;
 
+  // obsolete files will be deleted every this seconds if ttl deletion is
+  // enabled and archive size_limit is disabled.
+  uint64_t default_interval_to_delete_obsolete_WAL_;
+
   bool flush_on_destroy_; // Used when disableWAL is true.
 
   static const int KEEP_LOG_FILE_NUM = 1000;
@@ -514,10 +603,6 @@ class DBImpl : public DB {
   // The options to access storage files
   const EnvOptions env_options_;
 
-#ifndef ROCKSDB_LITE
-  WalManager wal_manager_;
-#endif  // ROCKSDB_LITE
-
   // A value of true temporarily disables scheduling of background work
   bool bg_work_gate_closed_;
 
@@ -527,12 +612,6 @@ class DBImpl : public DB {
   // Indicate DB was opened successfully
   bool opened_successfully_;
 
-  // The list of registered event listeners.
-  std::list<EventListener*> listeners_;
-
-  // count how many events are currently being notified.
-  int notifying_events_;
-
   // No copying allowed
   DBImpl(const DBImpl&);
   void operator=(const DBImpl&);
@@ -545,16 +624,11 @@ class DBImpl : public DB {
     SequenceNumber* prev_snapshot);
 
   // Background threads call this function, which is just a wrapper around
-  // the InstallSuperVersion() function. Background threads carry
-  // job_context which can have new_superversion already
-  // allocated.
-  void InstallSuperVersionBackground(
-      ColumnFamilyData* cfd, JobContext* job_context,
-      const MutableCFOptions& mutable_cf_options);
-
-  SuperVersion* InstallSuperVersion(
-    ColumnFamilyData* cfd, SuperVersion* new_sv,
-    const MutableCFOptions& mutable_cf_options);
+  // the cfd->InstallSuperVersion() function. Background threads carry
+  // deletion_state which can have new_superversion already allocated.
+  void InstallSuperVersion(ColumnFamilyData* cfd,
+                           DeletionState& deletion_state,
+                           const MutableCFOptions& mutable_cf_options);
 
   // Find Super version and reference it. Based on options, it might return
   // the thread local cached one.
@@ -598,4 +672,8 @@ static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
   if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
 }
 
+// Dump db file summary, implemented in util/
+extern void DumpDBFileSummary(const DBOptions& options,
+                              const std::string& dbname);
+
 }  // namespace rocksdb

db/db_impl_debug.cc

@@ -13,9 +13,11 @@
 
 namespace rocksdb {
 
+void DBImpl::TEST_PurgeObsoleteteWAL() { PurgeObsoleteWALFiles(); }
+
 uint64_t DBImpl::TEST_GetLevel0TotalSize() {
   MutexLock l(&mutex_);
-  return default_cf_handle_->cfd()->current()->storage_info()->NumLevelBytes(0);
+  return default_cf_handle_->cfd()->current()->NumLevelBytes(0);
 }
 
 Iterator* DBImpl::TEST_NewInternalIterator(Arena* arena,
@@ -45,7 +47,7 @@ int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
     cfd = cfh->cfd();
   }
   MutexLock l(&mutex_);
-  return cfd->current()->storage_info()->MaxNextLevelOverlappingBytes();
+  return cfd->current()->MaxNextLevelOverlappingBytes();
 }
 
 void DBImpl::TEST_GetFilesMetaData(
@@ -56,8 +58,7 @@ void DBImpl::TEST_GetFilesMetaData(
   MutexLock l(&mutex_);
   metadata->resize(NumberLevels());
   for (int level = 0; level < NumberLevels(); level++) {
-    const std::vector<FileMetaData*>& files =
-        cfd->current()->storage_info()->LevelFiles(level);
+    const std::vector<FileMetaData*>& files = cfd->current()->files_[level];
 
     (*metadata)[level].clear();
     for (const auto& f : files) {
@@ -67,7 +68,7 @@ void DBImpl::TEST_GetFilesMetaData(
 }
 
 uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
-  return versions_->manifest_file_number();
+  return versions_->ManifestFileNumber();
 }
 
 Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
@@ -81,8 +82,8 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
     cfd = cfh->cfd();
   }
   int output_level =
-      (cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
-       cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
+      (cfd->options()->compaction_style == kCompactionStyleUniversal ||
+       cfd->options()->compaction_style == kCompactionStyleFIFO)
           ? level
           : level + 1;
   return RunManualCompaction(cfd, level, output_level, 0, begin, end);
@@ -119,6 +120,17 @@ Status DBImpl::TEST_WaitForCompact() {
   return bg_error_;
 }
 
+Status DBImpl::TEST_ReadFirstRecord(const WalFileType type,
+                                    const uint64_t number,
+                                    SequenceNumber* sequence) {
+  return ReadFirstRecord(type, number, sequence);
+}
+
+Status DBImpl::TEST_ReadFirstLine(const std::string& fname,
+                                  SequenceNumber* sequence) {
+  return ReadFirstLine(fname, sequence);
+}
+
 void DBImpl::TEST_LockMutex() {
   mutex_.Lock();
 }

db/db_impl_readonly.cc

@@ -3,7 +3,6 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
-
 #include "db/db_impl_readonly.h"
 #include "utilities/compacted_db/compacted_db_impl.h"
 #include "db/db_impl.h"
@@ -13,8 +12,6 @@
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
-
 DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
                                const std::string& dbname)
     : DBImpl(db_options, dbname) {
@@ -56,7 +53,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
            ? reinterpret_cast<const SnapshotImpl*>(
                 read_options.snapshot)->number_
            : latest_snapshot),
-      super_version->mutable_cf_options.max_sequential_skip_in_iterations);
+      cfd->options()->max_sequential_skip_in_iterations);
   auto internal_iter = NewInternalIterator(
       read_options, cfd, super_version, db_iter->GetArena());
   db_iter->SetIterUnderDBIter(internal_iter);
@@ -75,17 +72,16 @@ Status DBImplReadOnly::NewIterators(
   SequenceNumber latest_snapshot = versions_->LastSequence();
 
   for (auto cfh : column_families) {
-    auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
-    auto* sv = cfd->GetSuperVersion()->Ref();
-    auto* db_iter = NewArenaWrappedDbIterator(
+    auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
+    auto db_iter = NewArenaWrappedDbIterator(
         env_, *cfd->ioptions(), cfd->user_comparator(),
         (read_options.snapshot != nullptr
             ? reinterpret_cast<const SnapshotImpl*>(
                   read_options.snapshot)->number_
             : latest_snapshot),
-        sv->mutable_cf_options.max_sequential_skip_in_iterations);
-    auto* internal_iter = NewInternalIterator(
-        read_options, cfd, sv, db_iter->GetArena());
+        cfd->options()->max_sequential_skip_in_iterations);
+    auto internal_iter = NewInternalIterator(
+        read_options, cfd, cfd->GetSuperVersion()->Ref(), db_iter->GetArena());
     db_iter->SetIterUnderDBIter(internal_iter);
     iterators->push_back(db_iter);
   }
@@ -99,10 +95,12 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
 
   // Try to first open DB as fully compacted DB
   Status s;
+#ifndef ROCKSDB_LITE
   s = CompactedDBImpl::Open(options, dbname, dbptr);
   if (s.ok()) {
     return s;
   }
+#endif
 
   DBOptions db_options(options);
   ColumnFamilyOptions cf_options(options);
@@ -163,20 +161,5 @@ Status DB::OpenForReadOnly(
   return s;
 }
 
-#else  // !ROCKSDB_LITE
-
-Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
-                           DB** dbptr, bool error_if_log_file_exist) {
-  return Status::NotSupported("Not supported in ROCKSDB_LITE.");
-}
-
-Status DB::OpenForReadOnly(
-    const DBOptions& db_options, const std::string& dbname,
-    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
-    bool error_if_log_file_exist) {
-  return Status::NotSupported("Not supported in ROCKSDB_LITE.");
-}
-#endif  // !ROCKSDB_LITE
 
 }   // namespace rocksdb

db/db_impl_readonly.h

@@ -4,9 +4,6 @@
 //  of patent rights can be found in the PATENTS file in the same directory.
 
 #pragma once
-
-#ifndef ROCKSDB_LITE
-
 #include "db/db_impl.h"
 #include <vector>
 #include <string>
@@ -65,20 +62,10 @@ class DBImplReadOnly : public DBImpl {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
-  using DBImpl::CompactFiles;
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1) override {
-    return Status::NotSupported("Not supported operation in read only mode.");
-  }
-
 #ifndef ROCKSDB_LITE
   virtual Status DisableFileDeletions() override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
-
   virtual Status EnableFileDeletions(bool force) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
@@ -103,5 +90,3 @@ class DBImplReadOnly : public DBImpl {
   void operator=(const DBImplReadOnly&);
 };
 }
-
-#endif  // !ROCKSDB_LITE

db/db_iter.cc

@@ -162,8 +162,7 @@ class DBIter: public Iterator {
 inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
   if (!ParseInternalKey(iter_->key(), ikey)) {
     status_ = Status::Corruption("corrupted internal key in DBIter");
-    Log(InfoLogLevel::ERROR_LEVEL,
-        logger_, "corrupted internal key in DBIter: %s",
+    Log(logger_, "corrupted internal key in DBIter: %s",
         iter_->key().ToString(true).c_str());
     return false;
   } else {
@@ -279,11 +278,9 @@ void DBIter::FindNextUserEntryInternal(bool skipping) {
 //       iter_ points to the next entry (or invalid)
 void DBIter::MergeValuesNewToOld() {
   if (!user_merge_operator_) {
-    Log(InfoLogLevel::ERROR_LEVEL,
-        logger_, "Options::merge_operator is null.");
-    status_ = Status::InvalidArgument("user_merge_operator_ must be set.");
-    valid_ = false;
-    return;
+    Log(logger_, "Options::merge_operator is null.");
+    throw std::logic_error("DBIter::MergeValuesNewToOld() with"
+                           " Options::merge_operator null");
   }
 
   // Start the merge process by pushing the first operand
@@ -313,8 +310,8 @@ void DBIter::MergeValuesNewToOld() {
       // hit a put, merge the put value with operands and store the
       // final result in saved_value_. We are done!
       // ignore corruption if there is any.
-      const Slice val = iter_->value();
-      user_merge_operator_->FullMerge(ikey.user_key, &val, operands,
+      const Slice value = iter_->value();
+      user_merge_operator_->FullMerge(ikey.user_key, &value, operands,
                                       &saved_value_, logger_);
       // iter_ is positioned after put
       iter_->Next();
@@ -324,8 +321,8 @@ void DBIter::MergeValuesNewToOld() {
     if (kTypeMerge == ikey.type) {
       // hit a merge, add the value as an operand and run associative merge.
       // when complete, add result to operands and continue.
-      const Slice& val = iter_->value();
-      operands.push_front(val.ToString());
+      const Slice& value = iter_->value();
+      operands.push_front(value.ToString());
     }
   }
 
@@ -506,8 +503,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     return true;
   }
 
-  const Slice& val = iter_->value();
-  user_merge_operator_->FullMerge(saved_key_.GetKey(), &val, operands,
+  const Slice& value = iter_->value();
+  user_merge_operator_->FullMerge(saved_key_.GetKey(), &value, operands,
                                   &saved_value_, logger_);
   valid_ = true;
   return true;

db/db_iter_test.cc

@@ -19,7 +19,7 @@
 
 namespace rocksdb {
 
-static uint64_t TestGetTickerCount(const Options& options,
+static uint32_t TestGetTickerCount(const Options& options,
                                    Tickers ticker_type) {
   return options.statistics->getTickerCount(ticker_type);
 }
@@ -33,23 +33,20 @@ class TestIterator : public Iterator {
         iter_(0),
         cmp(comparator) {}
 
-  void AddMerge(std::string argkey, std::string argvalue) {
-    Add(argkey, kTypeMerge, argvalue);
+  void AddMerge(std::string key, std::string value) {
+    Add(key, kTypeMerge, value);
   }
 
-  void AddDeletion(std::string argkey) {
-    Add(argkey, kTypeDeletion, std::string());
+  void AddDeletion(std::string key) { Add(key, kTypeDeletion, std::string()); }
+
+  void AddPut(std::string key, std::string value) {
+    Add(key, kTypeValue, value);
   }
 
-  void AddPut(std::string argkey, std::string argvalue) {
-    Add(argkey, kTypeValue, argvalue);
-  }
-
-  void Add(std::string argkey, ValueType type, std::string argvalue) {
+  void Add(std::string key, ValueType type, std::string value) {
     valid_ = true;
-    ParsedInternalKey internal_key(argkey, sequence_number_++, type);
-    data_.push_back(
-        std::pair<std::string, std::string>(std::string(), argvalue));
+    ParsedInternalKey internal_key(key, sequence_number_++, type);
+    data_.push_back(std::pair<std::string, std::string>(std::string(), value));
     AppendInternalKey(&data_.back().first, internal_key);
   }
 
@@ -220,6 +217,7 @@ TEST(DBIteratorTest, DBIteratorPrevNext) {
   }
 
   {
+    Options options;
     TestIterator* internal_iter = new TestIterator(BytewiseComparator());
     internal_iter->AddPut("a", "val_a");
     internal_iter->AddPut("b", "val_b");
@@ -256,6 +254,7 @@ TEST(DBIteratorTest, DBIteratorPrevNext) {
   }
 
   {
+    Options options;
     TestIterator* internal_iter = new TestIterator(BytewiseComparator());
     internal_iter->AddPut("a", "val_a");
     internal_iter->AddPut("a", "val_a");
@@ -365,8 +364,8 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       TestIterator* internal_iter = new TestIterator(BytewiseComparator());
       internal_iter->AddMerge("b", "merge_1");
       internal_iter->AddMerge("a", "merge_2");
-      for (size_t k = 0; k < 200; ++k) {
-        internal_iter->AddPut("c", ToString(k));
+      for (size_t i = 0; i < 200; ++i) {
+        internal_iter->AddPut("c", std::to_string(i));
       }
       internal_iter->Finish();
 
@@ -379,7 +378,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       ASSERT_TRUE(db_iter->Valid());
 
       ASSERT_EQ(db_iter->key().ToString(), "c");
-      ASSERT_EQ(db_iter->value().ToString(), ToString(i));
+      ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
       db_iter->Prev();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -401,7 +400,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       TestIterator* internal_iter = new TestIterator(BytewiseComparator());
       internal_iter->AddMerge("b", "merge_1");
       internal_iter->AddMerge("a", "merge_2");
-      for (size_t k = 0; k < 200; ++k) {
+      for (size_t i = 0; i < 200; ++i) {
         internal_iter->AddDeletion("c");
       }
       internal_iter->AddPut("c", "200");
@@ -464,7 +463,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
   {
     for (size_t i = 0; i < 200; ++i) {
       TestIterator* internal_iter = new TestIterator(BytewiseComparator());
-      for (size_t k = 0; k < 200; ++k) {
+      for (size_t i = 0; i < 200; ++i) {
         internal_iter->AddDeletion("c");
       }
       internal_iter->AddPut("c", "200");
@@ -512,12 +511,12 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       TestIterator* internal_iter = new TestIterator(BytewiseComparator());
       internal_iter->AddMerge("b", "merge_1");
       internal_iter->AddMerge("a", "merge_2");
-      for (size_t k = 0; k < 200; ++k) {
-        internal_iter->AddPut("d", ToString(k));
+      for (size_t i = 0; i < 200; ++i) {
+        internal_iter->AddPut("d", std::to_string(i));
       }
 
-      for (size_t k = 0; k < 200; ++k) {
-        internal_iter->AddPut("c", ToString(k));
+      for (size_t i = 0; i < 200; ++i) {
+        internal_iter->AddPut("c", std::to_string(i));
       }
       internal_iter->Finish();
 
@@ -529,7 +528,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       ASSERT_TRUE(db_iter->Valid());
 
       ASSERT_EQ(db_iter->key().ToString(), "d");
-      ASSERT_EQ(db_iter->value().ToString(), ToString(i));
+      ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
       db_iter->Prev();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -551,8 +550,8 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       TestIterator* internal_iter = new TestIterator(BytewiseComparator());
       internal_iter->AddMerge("b", "b");
       internal_iter->AddMerge("a", "a");
-      for (size_t k = 0; k < 200; ++k) {
-        internal_iter->AddMerge("c", ToString(k));
+      for (size_t i = 0; i < 200; ++i) {
+        internal_iter->AddMerge("c", std::to_string(i));
       }
       internal_iter->Finish();
 
@@ -566,7 +565,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       ASSERT_EQ(db_iter->key().ToString(), "c");
       std::string merge_result = "0";
       for (size_t j = 1; j <= i; ++j) {
-        merge_result += "," + ToString(j);
+        merge_result += "," + std::to_string(j);
       }
       ASSERT_EQ(db_iter->value().ToString(), merge_result);
 
@@ -1391,6 +1390,7 @@ TEST(DBIteratorTest, DBIterator) {
   }
 
   {
+    Options options;
     TestIterator* internal_iter = new TestIterator(BytewiseComparator());
     internal_iter->AddDeletion("a");
     internal_iter->AddPut("a", "0");

db/dbformat.cc

@@ -127,8 +127,8 @@ void InternalKeyComparator::FindShortSuccessor(std::string* key) const {
   }
 }
 
-LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s) {
-  size_t usize = _user_key.size();
+LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
+  size_t usize = user_key.size();
   size_t needed = usize + 13;  // A conservative estimate
   char* dst;
   if (needed <= sizeof(space_)) {
@@ -137,10 +137,9 @@ LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s) {
     dst = new char[needed];
   }
   start_ = dst;
-  // NOTE: We don't support users keys of more than 2GB :)
-  dst = EncodeVarint32(dst, static_cast<uint32_t>(usize + 8));
+  dst = EncodeVarint32(dst, usize + 8);
   kstart_ = dst;
-  memcpy(dst, _user_key.data(), usize);
+  memcpy(dst, user_key.data(), usize);
   dst += usize;
   EncodeFixed64(dst, PackSequenceAndType(s, kValueTypeForSeek));
   dst += 8;

db/dbformat.h

@@ -132,8 +132,8 @@ class InternalKey {
   std::string rep_;
  public:
   InternalKey() { }   // Leave rep_ as empty to indicate it is invalid
-  InternalKey(const Slice& _user_key, SequenceNumber s, ValueType t) {
-    AppendInternalKey(&rep_, ParsedInternalKey(_user_key, s, t));
+  InternalKey(const Slice& user_key, SequenceNumber s, ValueType t) {
+    AppendInternalKey(&rep_, ParsedInternalKey(user_key, s, t));
   }
 
   bool Valid() const {
@@ -201,24 +201,18 @@ class LookupKey {
  public:
   // Initialize *this for looking up user_key at a snapshot with
   // the specified sequence number.
-  LookupKey(const Slice& _user_key, SequenceNumber sequence);
+  LookupKey(const Slice& user_key, SequenceNumber sequence);
 
   ~LookupKey();
 
   // Return a key suitable for lookup in a MemTable.
-  Slice memtable_key() const {
-    return Slice(start_, static_cast<size_t>(end_ - start_));
-  }
+  Slice memtable_key() const { return Slice(start_, end_ - start_); }
 
   // Return an internal key (suitable for passing to an internal iterator)
-  Slice internal_key() const {
-    return Slice(kstart_, static_cast<size_t>(end_ - kstart_));
-  }
+  Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
 
   // Return the user key
-  Slice user_key() const {
-    return Slice(kstart_, static_cast<size_t>(end_ - kstart_ - 8));
-  }
+  Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
 
  private:
   // We construct a char array of the form:
@@ -325,8 +319,8 @@ class IterKey {
 
   void EncodeLengthPrefixedKey(const Slice& key) {
     auto size = key.size();
-    EnlargeBufferIfNeeded(size + static_cast<size_t>(VarintLength(size)));
-    char* ptr = EncodeVarint32(key_, static_cast<uint32_t>(size));
+    EnlargeBufferIfNeeded(size + VarintLength(size));
+    char* ptr = EncodeVarint32(key_, size);
     memcpy(ptr, key.data(), size);
   }
 

db/deletefile_test.cc

@@ -34,7 +34,6 @@ class DeleteFileTest {
   DeleteFileTest() {
     db_ = nullptr;
     env_ = Env::Default();
-    options_.enable_thread_tracking = true;
     options_.max_background_flushes = 0;
     options_.write_buffer_size = 1024*1024*1000;
     options_.target_file_size_base = 1024*1024*1000;
@@ -79,7 +78,7 @@ class DeleteFileTest {
     options.sync = false;
     ReadOptions roptions;
     for (int i = startkey; i < (numkeys + startkey) ; i++) {
-      std::string temp = ToString(i);
+      std::string temp = std::to_string(i);
       Slice key(temp);
       Slice value(temp);
       ASSERT_OK(db_->Put(options, key, value));
@@ -288,75 +287,6 @@ TEST(DeleteFileTest, DeleteLogFiles) {
   CloseDB();
 }
 
-TEST(DeleteFileTest, DeleteNonDefaultColumnFamily) {
-  CloseDB();
-  DBOptions db_options;
-  db_options.create_if_missing = true;
-  db_options.create_missing_column_families = true;
-  std::vector<ColumnFamilyDescriptor> column_families;
-  column_families.emplace_back();
-  column_families.emplace_back("new_cf", ColumnFamilyOptions());
-
-  std::vector<rocksdb::ColumnFamilyHandle*> handles;
-  rocksdb::DB* db;
-  ASSERT_OK(DB::Open(db_options, dbname_, column_families, &handles, &db));
-
-  Random rnd(5);
-  for (int i = 0; i < 1000; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), handles[1], test::RandomKey(&rnd, 10),
-                      test::RandomKey(&rnd, 10)));
-  }
-  ASSERT_OK(db->Flush(FlushOptions(), handles[1]));
-  for (int i = 0; i < 1000; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), handles[1], test::RandomKey(&rnd, 10),
-                      test::RandomKey(&rnd, 10)));
-  }
-  ASSERT_OK(db->Flush(FlushOptions(), handles[1]));
-
-  std::vector<LiveFileMetaData> metadata;
-  db->GetLiveFilesMetaData(&metadata);
-  ASSERT_EQ(2U, metadata.size());
-  ASSERT_EQ("new_cf", metadata[0].column_family_name);
-  ASSERT_EQ("new_cf", metadata[1].column_family_name);
-  auto old_file = metadata[0].smallest_seqno < metadata[1].smallest_seqno
-                      ? metadata[0].name
-                      : metadata[1].name;
-  auto new_file = metadata[0].smallest_seqno > metadata[1].smallest_seqno
-                      ? metadata[0].name
-                      : metadata[1].name;
-  ASSERT_TRUE(db->DeleteFile(new_file).IsInvalidArgument());
-  ASSERT_OK(db->DeleteFile(old_file));
-
-  {
-    std::unique_ptr<Iterator> itr(db->NewIterator(ReadOptions(), handles[1]));
-    int count = 0;
-    for (itr->SeekToFirst(); itr->Valid(); itr->Next()) {
-      ASSERT_OK(itr->status());
-      ++count;
-    }
-    ASSERT_EQ(count, 1000);
-  }
-
-  delete handles[0];
-  delete handles[1];
-  delete db;
-
-  ASSERT_OK(DB::Open(db_options, dbname_, column_families, &handles, &db));
-  {
-    std::unique_ptr<Iterator> itr(db->NewIterator(ReadOptions(), handles[1]));
-    int count = 0;
-    for (itr->SeekToFirst(); itr->Valid(); itr->Next()) {
-      ASSERT_OK(itr->status());
-      ++count;
-    }
-    ASSERT_EQ(count, 1000);
-  }
-
-  delete handles[0];
-  delete handles[1];
-  delete db;
-}
-
 } //namespace rocksdb
 
 int main(int argc, char** argv) {

db/file_indexer.cc

@@ -17,16 +17,17 @@ namespace rocksdb {
 FileIndexer::FileIndexer(const Comparator* ucmp)
     : num_levels_(0), ucmp_(ucmp), level_rb_(nullptr) {}
 
-size_t FileIndexer::NumLevelIndex() const { return next_level_index_.size(); }
+uint32_t FileIndexer::NumLevelIndex() {
+  return next_level_index_.size();
+}
 
-size_t FileIndexer::LevelIndexSize(size_t level) const {
+uint32_t FileIndexer::LevelIndexSize(uint32_t level) {
   return next_level_index_[level].num_index;
 }
 
-void FileIndexer::GetNextLevelIndex(const size_t level, const size_t file_index,
-                                    const int cmp_smallest,
-                                    const int cmp_largest, int32_t* left_bound,
-                                    int32_t* right_bound) const {
+void FileIndexer::GetNextLevelIndex(
+    const uint32_t level, const uint32_t file_index, const int cmp_smallest,
+    const int cmp_largest, int32_t* left_bound, int32_t* right_bound) {
   assert(level > 0);
 
   // Last level, no hint
@@ -68,7 +69,7 @@ void FileIndexer::GetNextLevelIndex(const size_t level, const size_t file_index,
   assert(*right_bound <= level_rb_[level + 1]);
 }
 
-void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels,
+void FileIndexer::UpdateIndex(Arena* arena, const uint32_t num_levels,
                               std::vector<FileMetaData*>* const files) {
   if (files == nullptr) {
     return;
@@ -89,17 +90,17 @@ void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels,
   }
 
   // L1 - Ln-1
-  for (size_t level = 1; level < num_levels_ - 1; ++level) {
+  for (uint32_t level = 1; level < num_levels_ - 1; ++level) {
     const auto& upper_files = files[level];
-    const int32_t upper_size = static_cast<int32_t>(upper_files.size());
+    const int32_t upper_size = upper_files.size();
     const auto& lower_files = files[level + 1];
-    level_rb_[level] = static_cast<int32_t>(upper_files.size()) - 1;
+    level_rb_[level] = upper_files.size() - 1;
     if (upper_size == 0) {
       continue;
     }
     IndexLevel& index_level = next_level_index_[level];
     index_level.num_index = upper_size;
-    mem = arena->AllocateAligned(upper_size * sizeof(IndexUnit));
+    char* mem = arena->AllocateAligned(upper_size * sizeof(IndexUnit));
     index_level.index_units = new (mem) IndexUnit[upper_size];
 
     CalculateLB(
@@ -128,8 +129,7 @@ void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels,
         [](IndexUnit* index, int32_t f_idx) { index->largest_rb = f_idx; });
   }
 
-  level_rb_[num_levels_ - 1] =
-      static_cast<int32_t>(files[num_levels_ - 1].size()) - 1;
+  level_rb_[num_levels_ - 1] = files[num_levels_ - 1].size() - 1;
 }
 
 void FileIndexer::CalculateLB(
@@ -137,8 +137,8 @@ void FileIndexer::CalculateLB(
     const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
     std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
     std::function<void(IndexUnit*, int32_t)> set_index) {
-  const int32_t upper_size = static_cast<int32_t>(upper_files.size());
-  const int32_t lower_size = static_cast<int32_t>(lower_files.size());
+  const int32_t upper_size = upper_files.size();
+  const int32_t lower_size = lower_files.size();
   int32_t upper_idx = 0;
   int32_t lower_idx = 0;
 
@@ -175,8 +175,8 @@ void FileIndexer::CalculateRB(
     const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
     std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
     std::function<void(IndexUnit*, int32_t)> set_index) {
-  const int32_t upper_size = static_cast<int32_t>(upper_files.size());
-  const int32_t lower_size = static_cast<int32_t>(lower_files.size());
+  const int32_t upper_size = upper_files.size();
+  const int32_t lower_size = lower_files.size();
   int32_t upper_idx = upper_size - 1;
   int32_t lower_idx = lower_size - 1;
 

db/file_indexer.h

@@ -42,19 +42,19 @@ class FileIndexer {
  public:
   explicit FileIndexer(const Comparator* ucmp);
 
-  size_t NumLevelIndex() const;
+  uint32_t NumLevelIndex();
 
-  size_t LevelIndexSize(size_t level) const;
+  uint32_t LevelIndexSize(uint32_t level);
 
   // Return a file index range in the next level to search for a key based on
   // smallest and largest key comparision for the current file specified by
   // level and file_index. When *left_index < *right_index, both index should
   // be valid and fit in the vector size.
-  void GetNextLevelIndex(const size_t level, const size_t file_index,
-                         const int cmp_smallest, const int cmp_largest,
-                         int32_t* left_bound, int32_t* right_bound) const;
+  void GetNextLevelIndex(
+    const uint32_t level, const uint32_t file_index, const int cmp_smallest,
+    const int cmp_largest, int32_t* left_bound, int32_t* right_bound);
 
-  void UpdateIndex(Arena* arena, const size_t num_levels,
+  void UpdateIndex(Arena* arena, const uint32_t num_levels,
                    std::vector<FileMetaData*>* const files);
 
   enum {
@@ -62,7 +62,7 @@ class FileIndexer {
   };
 
  private:
-  size_t num_levels_;
+  uint32_t num_levels_;
   const Comparator* ucmp_;
 
   struct IndexUnit {

db/file_indexer_test.cc

@@ -22,15 +22,8 @@ class IntComparator : public Comparator {
   int Compare(const Slice& a, const Slice& b) const {
     assert(a.size() == 8);
     assert(b.size() == 8);
-    int64_t diff = *reinterpret_cast<const int64_t*>(a.data()) -
-                   *reinterpret_cast<const int64_t*>(b.data());
-    if (diff < 0) {
-      return -1;
-    } else if (diff == 0) {
-      return 0;
-    } else {
-      return 1;
-    }
+    return *reinterpret_cast<const int64_t*>(a.data()) -
+      *reinterpret_cast<const int64_t*>(b.data());
   }
 
   const char* Name() const {
@@ -101,6 +94,7 @@ TEST(FileIndexerTest, Empty) {
 // Case 1: no overlap, files are on the left of next level files
 TEST(FileIndexerTest, no_overlap_left) {
   Arena arena;
+  uint32_t kNumLevels = 4;
   indexer = new FileIndexer(&ucmp);
   // level 1
   AddFile(1, 100, 200);
@@ -141,6 +135,7 @@ TEST(FileIndexerTest, no_overlap_left) {
 // Case 2: no overlap, files are on the right of next level files
 TEST(FileIndexerTest, no_overlap_right) {
   Arena arena;
+  uint32_t kNumLevels = 4;
   indexer = new FileIndexer(&ucmp);
   // level 1
   AddFile(1, 2100, 2200);
@@ -183,6 +178,7 @@ TEST(FileIndexerTest, no_overlap_right) {
 // Case 3: empty L2
 TEST(FileIndexerTest, empty_L2) {
   Arena arena;
+  uint32_t kNumLevels = 4;
   indexer = new FileIndexer(&ucmp);
   for (uint32_t i = 1; i < kNumLevels; ++i) {
     ASSERT_EQ(0U, indexer->LevelIndexSize(i));

db/filename.cc

@@ -79,17 +79,6 @@ std::string MakeTableFileName(const std::string& path, uint64_t number) {
   return MakeFileName(path, number, "sst");
 }
 
-uint64_t TableFileNameToNumber(const std::string& name) {
-  uint64_t number = 0;
-  uint64_t base = 1;
-  int pos = static_cast<int>(name.find_last_of('.'));
-  while (--pos >= 0 && name[pos] >= '0' && name[pos] <= '9') {
-    number += (name[pos] - '0') * base;
-    base *= 10;
-  }
-  return number;
-}
-
 std::string TableFileName(const std::vector<DbPath>& db_paths, uint64_t number,
                           uint32_t path_id) {
   assert(number > 0);

db/filename.h

@@ -36,6 +36,9 @@ enum FileType {
   kIdentityFile
 };
 
+// map from file number to path ID.
+typedef std::unordered_map<uint64_t, uint32_t> FileNumToPathIdMap;
+
 // Return the name of the log file with the specified number
 // in the db named by "dbname".  The result will be prefixed with
 // "dbname".
@@ -52,10 +55,6 @@ extern std::string ArchivedLogFileName(const std::string& dbname,
 
 extern std::string MakeTableFileName(const std::string& name, uint64_t number);
 
-// the reverse function of MakeTableFileName
-// TODO(yhchiang): could merge this function with ParseFileName()
-extern uint64_t TableFileNameToNumber(const std::string& name);
-
 // Return the name of the sstable with the specified number
 // in the db named by "dbname".  The result will be prefixed with
 // "dbname".

db/flush_job.cc

@@ -1,213 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/flush_job.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
-#include <algorithm>
-#include <vector>
-
-#include "db/builder.h"
-#include "db/db_iter.h"
-#include "db/dbformat.h"
-#include "db/filename.h"
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "db/memtable.h"
-#include "db/memtable_list.h"
-#include "db/merge_context.h"
-#include "db/version_set.h"
-#include "port/port.h"
-#include "port/likely.h"
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/statistics.h"
-#include "rocksdb/status.h"
-#include "rocksdb/table.h"
-#include "table/block.h"
-#include "table/block_based_table_factory.h"
-#include "table/merger.h"
-#include "table/table_builder.h"
-#include "table/two_level_iterator.h"
-#include "util/coding.h"
-#include "util/logging.h"
-#include "util/log_buffer.h"
-#include "util/mutexlock.h"
-#include "util/perf_context_imp.h"
-#include "util/iostats_context_imp.h"
-#include "util/stop_watch.h"
-#include "util/sync_point.h"
-
-namespace rocksdb {
-
-FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
-                   const DBOptions& db_options,
-                   const MutableCFOptions& mutable_cf_options,
-                   const EnvOptions& env_options, VersionSet* versions,
-                   port::Mutex* db_mutex, std::atomic<bool>* shutting_down,
-                   SequenceNumber newest_snapshot, JobContext* job_context,
-                   LogBuffer* log_buffer, Directory* db_directory,
-                   CompressionType output_compression, Statistics* stats)
-    : dbname_(dbname),
-      cfd_(cfd),
-      db_options_(db_options),
-      mutable_cf_options_(mutable_cf_options),
-      env_options_(env_options),
-      versions_(versions),
-      db_mutex_(db_mutex),
-      shutting_down_(shutting_down),
-      newest_snapshot_(newest_snapshot),
-      job_context_(job_context),
-      log_buffer_(log_buffer),
-      db_directory_(db_directory),
-      output_compression_(output_compression),
-      stats_(stats) {}
-
-Status FlushJob::Run(uint64_t* file_number) {
-  // Save the contents of the earliest memtable as a new Table
-  uint64_t fn;
-  autovector<MemTable*> mems;
-  cfd_->imm()->PickMemtablesToFlush(&mems);
-  if (mems.empty()) {
-    LogToBuffer(log_buffer_, "[%s] Nothing in memtable to flush",
-                cfd_->GetName().c_str());
-    return Status::OK();
-  }
-
-  // entries mems are (implicitly) sorted in ascending order by their created
-  // time. We will use the first memtable's `edit` to keep the meta info for
-  // this flush.
-  MemTable* m = mems[0];
-  VersionEdit* edit = m->GetEdits();
-  edit->SetPrevLogNumber(0);
-  // SetLogNumber(log_num) indicates logs with number smaller than log_num
-  // will no longer be picked up for recovery.
-  edit->SetLogNumber(mems.back()->GetNextLogNumber());
-  edit->SetColumnFamily(cfd_->GetID());
-
-  // This will release and re-acquire the mutex.
-  Status s = WriteLevel0Table(mems, edit, &fn);
-
-  if (s.ok() &&
-      (shutting_down_->load(std::memory_order_acquire) || cfd_->IsDropped())) {
-    s = Status::ShutdownInProgress(
-        "Database shutdown or Column family drop during flush");
-  }
-
-  if (!s.ok()) {
-    cfd_->imm()->RollbackMemtableFlush(mems, fn);
-  } else {
-    // Replace immutable memtable with the generated Table
-    s = cfd_->imm()->InstallMemtableFlushResults(
-        cfd_, mutable_cf_options_, mems, versions_, db_mutex_, fn,
-        &job_context_->memtables_to_free, db_directory_, log_buffer_);
-  }
-
-  if (s.ok() && file_number != nullptr) {
-    *file_number = fn;
-  }
-  return s;
-}
-
-Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
-                                  VersionEdit* edit, uint64_t* filenumber) {
-  db_mutex_->AssertHeld();
-  const uint64_t start_micros = db_options_.env->NowMicros();
-  FileMetaData meta;
-
-  meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
-  *filenumber = meta.fd.GetNumber();
-  // path 0 for level 0 file.
-
-  const SequenceNumber earliest_seqno_in_memtable =
-      mems[0]->GetFirstSequenceNumber();
-  Version* base = cfd_->current();
-  base->Ref();  // it is likely that we do not need this reference
-  Status s;
-  {
-    db_mutex_->Unlock();
-    if (log_buffer_) {
-      log_buffer_->FlushBufferToLog();
-    }
-    std::vector<Iterator*> memtables;
-    ReadOptions ro;
-    ro.total_order_seek = true;
-    Arena arena;
-    for (MemTable* m : mems) {
-      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-          "[%s] Flushing memtable with next log file: %" PRIu64 "\n",
-          cfd_->GetName().c_str(), m->GetNextLogNumber());
-      memtables.push_back(m->NewIterator(ro, &arena));
-    }
-    {
-      ScopedArenaIterator iter(
-          NewMergingIterator(&cfd_->internal_comparator(), &memtables[0],
-                             static_cast<int>(memtables.size()), &arena));
-      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-          "[%s] Level-0 flush table #%" PRIu64 ": started",
-          cfd_->GetName().c_str(), meta.fd.GetNumber());
-
-      s = BuildTable(dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
-                     cfd_->table_cache(), iter.get(), &meta,
-                     cfd_->internal_comparator(), newest_snapshot_,
-                     earliest_seqno_in_memtable, output_compression_,
-                     cfd_->ioptions()->compression_opts, Env::IO_HIGH);
-      LogFlush(db_options_.info_log);
-    }
-    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-        "[%s] Level-0 flush table #%" PRIu64 ": %" PRIu64 " bytes %s",
-        cfd_->GetName().c_str(), meta.fd.GetNumber(), meta.fd.GetFileSize(),
-        s.ToString().c_str());
-
-    if (!db_options_.disableDataSync && db_directory_ != nullptr) {
-      db_directory_->Fsync();
-    }
-    db_mutex_->Lock();
-  }
-  base->Unref();
-
-  // re-acquire the most current version
-  base = cfd_->current();
-
-  // Note that if file_size is zero, the file has been deleted and
-  // should not be added to the manifest.
-  int level = 0;
-  if (s.ok() && meta.fd.GetFileSize() > 0) {
-    const Slice min_user_key = meta.smallest.user_key();
-    const Slice max_user_key = meta.largest.user_key();
-    // if we have more than 1 background thread, then we cannot
-    // insert files directly into higher levels because some other
-    // threads could be concurrently producing compacted files for
-    // that key range.
-    if (base != nullptr && db_options_.max_background_compactions <= 1 &&
-        db_options_.max_background_flushes == 0 &&
-        cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
-      level = base->storage_info()->PickLevelForMemTableOutput(
-          mutable_cf_options_, min_user_key, max_user_key);
-    }
-    edit->AddFile(level, meta.fd.GetNumber(), meta.fd.GetPathId(),
-                  meta.fd.GetFileSize(), meta.smallest, meta.largest,
-                  meta.smallest_seqno, meta.largest_seqno);
-  }
-
-  InternalStats::CompactionStats stats(1);
-  stats.micros = db_options_.env->NowMicros() - start_micros;
-  stats.bytes_written = meta.fd.GetFileSize();
-  cfd_->internal_stats()->AddCompactionStats(level, stats);
-  cfd_->internal_stats()->AddCFStats(InternalStats::BYTES_FLUSHED,
-                                     meta.fd.GetFileSize());
-  RecordTick(stats_, COMPACT_WRITE_BYTES, meta.fd.GetFileSize());
-  return s;
-}
-
-}  // namespace rocksdb

db/flush_job.h

@@ -1,84 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#pragma once
-
-#include <atomic>
-#include <deque>
-#include <limits>
-#include <set>
-#include <utility>
-#include <vector>
-#include <string>
-
-#include "db/dbformat.h"
-#include "db/log_writer.h"
-#include "db/snapshot.h"
-#include "db/column_family.h"
-#include "db/version_edit.h"
-#include "db/memtable_list.h"
-#include "port/port.h"
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/memtablerep.h"
-#include "rocksdb/transaction_log.h"
-#include "util/autovector.h"
-#include "util/stop_watch.h"
-#include "util/thread_local.h"
-#include "util/scoped_arena_iterator.h"
-#include "db/internal_stats.h"
-#include "db/write_controller.h"
-#include "db/flush_scheduler.h"
-#include "db/write_thread.h"
-#include "db/job_context.h"
-
-namespace rocksdb {
-
-class MemTable;
-class TableCache;
-class Version;
-class VersionEdit;
-class VersionSet;
-class Arena;
-
-class FlushJob {
- public:
-  // TODO(icanadi) make effort to reduce number of parameters here
-  // IMPORTANT: mutable_cf_options needs to be alive while FlushJob is alive
-  FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
-           const DBOptions& db_options,
-           const MutableCFOptions& mutable_cf_options,
-           const EnvOptions& env_options, VersionSet* versions,
-           port::Mutex* db_mutex, std::atomic<bool>* shutting_down,
-           SequenceNumber newest_snapshot, JobContext* job_context,
-           LogBuffer* log_buffer, Directory* db_directory,
-           CompressionType output_compression, Statistics* stats);
-  ~FlushJob() {}
-
-  Status Run(uint64_t* file_number = nullptr);
-
- private:
-  Status WriteLevel0Table(const autovector<MemTable*>& mems, VersionEdit* edit,
-                          uint64_t* filenumber);
-  const std::string& dbname_;
-  ColumnFamilyData* cfd_;
-  const DBOptions& db_options_;
-  const MutableCFOptions& mutable_cf_options_;
-  const EnvOptions& env_options_;
-  VersionSet* versions_;
-  port::Mutex* db_mutex_;
-  std::atomic<bool>* shutting_down_;
-  SequenceNumber newest_snapshot_;
-  JobContext* job_context_;
-  LogBuffer* log_buffer_;
-  Directory* db_directory_;
-  CompressionType output_compression_;
-  Statistics* stats_;
-};
-
-}  // namespace rocksdb

db/flush_job_test.cc

@@ -1,124 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include <map>
-#include <string>
-
-#include "db/flush_job.h"
-#include "db/column_family.h"
-#include "db/version_set.h"
-#include "db/writebuffer.h"
-#include "rocksdb/cache.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-#include "table/mock_table.h"
-
-namespace rocksdb {
-
-// TODO(icanadi) Mock out everything else:
-// 1. VersionSet
-// 2. Memtable
-class FlushJobTest {
- public:
-  FlushJobTest()
-      : env_(Env::Default()),
-        dbname_(test::TmpDir() + "/flush_job_test"),
-        table_cache_(NewLRUCache(50000, 16, 8)),
-        write_buffer_(db_options_.db_write_buffer_size),
-        versions_(new VersionSet(dbname_, &db_options_, env_options_,
-                                 table_cache_.get(), &write_buffer_,
-                                 &write_controller_)),
-        shutting_down_(false),
-        mock_table_factory_(new mock::MockTableFactory()) {
-    ASSERT_OK(env_->CreateDirIfMissing(dbname_));
-    db_options_.db_paths.emplace_back(dbname_,
-                                      std::numeric_limits<uint64_t>::max());
-    // TODO(icanadi) Remove this once we mock out VersionSet
-    NewDB();
-    std::vector<ColumnFamilyDescriptor> column_families;
-    cf_options_.table_factory = mock_table_factory_;
-    column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
-
-    ASSERT_OK(versions_->Recover(column_families, false));
-  }
-
-  void NewDB() {
-    VersionEdit new_db;
-    new_db.SetLogNumber(0);
-    new_db.SetNextFile(2);
-    new_db.SetLastSequence(0);
-
-    const std::string manifest = DescriptorFileName(dbname_, 1);
-    unique_ptr<WritableFile> file;
-    Status s = env_->NewWritableFile(
-        manifest, &file, env_->OptimizeForManifestWrite(env_options_));
-    ASSERT_OK(s);
-    {
-      log::Writer log(std::move(file));
-      std::string record;
-      new_db.EncodeTo(&record);
-      s = log.AddRecord(record);
-    }
-    ASSERT_OK(s);
-    // Make "CURRENT" file that points to the new manifest file.
-    s = SetCurrentFile(env_, dbname_, 1, nullptr);
-  }
-
-  Env* env_;
-  std::string dbname_;
-  EnvOptions env_options_;
-  std::shared_ptr<Cache> table_cache_;
-  WriteController write_controller_;
-  DBOptions db_options_;
-  WriteBuffer write_buffer_;
-  ColumnFamilyOptions cf_options_;
-  std::unique_ptr<VersionSet> versions_;
-  port::Mutex mutex_;
-  std::atomic<bool> shutting_down_;
-  std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
-};
-
-TEST(FlushJobTest, Empty) {
-  JobContext job_context;
-  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
-                     db_options_, *cfd->GetLatestMutableCFOptions(),
-                     env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     SequenceNumber(), &job_context, nullptr, nullptr,
-                     kNoCompression, nullptr);
-  ASSERT_OK(flush_job.Run());
-  job_context.Clean();
-}
-
-TEST(FlushJobTest, NonEmpty) {
-  JobContext job_context;
-  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions());
-  new_mem->Ref();
-  std::map<std::string, std::string> inserted_keys;
-  for (int i = 1; i < 10000; ++i) {
-    std::string key(ToString(i));
-    std::string value("value" + ToString(i));
-    new_mem->Add(SequenceNumber(i), kTypeValue, key, value);
-    InternalKey internal_key(key, SequenceNumber(i), kTypeValue);
-    inserted_keys.insert({internal_key.Encode().ToString(), value});
-  }
-  cfd->imm()->Add(new_mem);
-
-  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
-                     db_options_, *cfd->GetLatestMutableCFOptions(),
-                     env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     SequenceNumber(), &job_context, nullptr, nullptr,
-                     kNoCompression, nullptr);
-  mutex_.Lock();
-  ASSERT_OK(flush_job.Run());
-  mutex_.Unlock();
-  mock_table_factory_->AssertSingleFile(inserted_keys);
-  job_context.Clean();
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/forward_iterator.cc

@@ -10,7 +10,6 @@
 #include <string>
 #include <utility>
 
-#include "db/job_context.h"
 #include "db/db_impl.h"
 #include "db/db_iter.h"
 #include "db/column_family.h"
@@ -115,31 +114,25 @@ class LevelIterator : public Iterator {
 };
 
 ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
-    ColumnFamilyData* cfd, SuperVersion* current_sv)
+                                 ColumnFamilyData* cfd)
     : db_(db),
       read_options_(read_options),
       cfd_(cfd),
-      prefix_extractor_(cfd->ioptions()->prefix_extractor),
+      prefix_extractor_(cfd->options()->prefix_extractor.get()),
       user_comparator_(cfd->user_comparator()),
       immutable_min_heap_(MinIterComparator(&cfd_->internal_comparator())),
-      sv_(current_sv),
+      sv_(nullptr),
       mutable_iter_(nullptr),
       current_(nullptr),
-      status_(Status::OK()),
-      immutable_status_(Status::OK()),
       valid_(false),
       is_prev_set_(false),
-      is_prev_inclusive_(false) {
-  if (sv_) {
-    RebuildIterators(false);
-  }
-}
+      is_prev_inclusive_(false) {}
 
 ForwardIterator::~ForwardIterator() {
-  Cleanup(true);
+  Cleanup();
 }
 
-void ForwardIterator::Cleanup(bool release_sv) {
+void ForwardIterator::Cleanup() {
   if (mutable_iter_ != nullptr) {
     mutable_iter_->~Iterator();
   }
@@ -156,17 +149,15 @@ void ForwardIterator::Cleanup(bool release_sv) {
   }
   level_iters_.clear();
 
-  if (release_sv) {
-    if (sv_ != nullptr && sv_->Unref()) {
-      JobContext job_context;
-      db_->mutex_.Lock();
-      sv_->Cleanup();
-      db_->FindObsoleteFiles(&job_context, false, true);
-      db_->mutex_.Unlock();
-      delete sv_;
-      if (job_context.HaveSomethingToDelete()) {
-        db_->PurgeObsoleteFiles(job_context);
-      }
+  if (sv_ != nullptr && sv_->Unref()) {
+    DBImpl::DeletionState deletion_state;
+    db_->mutex_.Lock();
+    sv_->Cleanup();
+    db_->FindObsoleteFiles(deletion_state, false, true);
+    db_->mutex_.Unlock();
+    delete sv_;
+    if (deletion_state.HaveSomethingToDelete()) {
+      db_->PurgeObsoleteFiles(deletion_state);
     }
   }
 }
@@ -178,8 +169,8 @@ bool ForwardIterator::Valid() const {
 void ForwardIterator::SeekToFirst() {
   if (sv_ == nullptr ||
       sv_ ->version_number != cfd_->GetSuperVersionNumber()) {
-    RebuildIterators(true);
-  } else if (immutable_status_.IsIncomplete()) {
+    RebuildIterators();
+  } else if (status_.IsIncomplete()) {
     ResetIncompleteIterators();
   }
   SeekInternal(Slice(), true);
@@ -188,8 +179,8 @@ void ForwardIterator::SeekToFirst() {
 void ForwardIterator::Seek(const Slice& internal_key) {
   if (sv_ == nullptr ||
       sv_ ->version_number != cfd_->GetSuperVersionNumber()) {
-    RebuildIterators(true);
-  } else if (immutable_status_.IsIncomplete()) {
+    RebuildIterators();
+  } else if (status_.IsIncomplete()) {
     ResetIncompleteIterators();
   }
   SeekInternal(internal_key, false);
@@ -197,7 +188,6 @@ void ForwardIterator::Seek(const Slice& internal_key) {
 
 void ForwardIterator::SeekInternal(const Slice& internal_key,
                                    bool seek_to_first) {
-  assert(mutable_iter_);
   // mutable
   seek_to_first ? mutable_iter_->SeekToFirst() :
                   mutable_iter_->Seek(internal_key);
@@ -207,16 +197,13 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
   // if it turns to need to seek immutable often. We probably want to have
   // an option to turn it off.
   if (seek_to_first || NeedToSeekImmutable(internal_key)) {
-    immutable_status_ = Status::OK();
     {
       auto tmp = MinIterHeap(MinIterComparator(&cfd_->internal_comparator()));
       immutable_min_heap_.swap(tmp);
     }
     for (auto* m : imm_iters_) {
       seek_to_first ? m->SeekToFirst() : m->Seek(internal_key);
-      if (!m->status().ok()) {
-        immutable_status_ = m->status();
-      } else if (m->Valid()) {
+      if (m->Valid()) {
         immutable_min_heap_.push(m);
       }
     }
@@ -225,23 +212,27 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
     if (!seek_to_first) {
       user_key = ExtractUserKey(internal_key);
     }
-    const VersionStorageInfo* vstorage = sv_->current->storage_info();
-    const std::vector<FileMetaData*>& l0 = vstorage->LevelFiles(0);
-    for (uint32_t i = 0; i < l0.size(); ++i) {
+    auto* files = sv_->current->files_;
+    for (uint32_t i = 0; i < files[0].size(); ++i) {
       if (seek_to_first) {
         l0_iters_[i]->SeekToFirst();
       } else {
         // If the target key passes over the larget key, we are sure Next()
         // won't go over this file.
         if (user_comparator_->Compare(user_key,
-              l0[i]->largest.user_key()) > 0) {
+              files[0][i]->largest.user_key()) > 0) {
           continue;
         }
         l0_iters_[i]->Seek(internal_key);
       }
 
-      if (!l0_iters_[i]->status().ok()) {
-        immutable_status_ = l0_iters_[i]->status();
+      if (l0_iters_[i]->status().IsIncomplete()) {
+        // if any of the immutable iterators is incomplete (no-io option was
+        // used), we are unable to reliably find the smallest key
+        assert(read_options_.read_tier == kBlockCacheTier);
+        status_ = l0_iters_[i]->status();
+        valid_ = false;
+        return;
       } else if (l0_iters_[i]->Valid()) {
         immutable_min_heap_.push(l0_iters_[i]);
       }
@@ -249,71 +240,75 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
 
     int32_t search_left_bound = 0;
     int32_t search_right_bound = FileIndexer::kLevelMaxIndex;
-    for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
-      const std::vector<FileMetaData*>& level_files =
-          vstorage->LevelFiles(level);
-      if (level_files.empty()) {
+    for (int32_t level = 1; level < sv_->current->NumberLevels(); ++level) {
+      if (files[level].empty()) {
         search_left_bound = 0;
         search_right_bound = FileIndexer::kLevelMaxIndex;
         continue;
       }
       assert(level_iters_[level - 1] != nullptr);
       uint32_t f_idx = 0;
-      const auto& indexer = vstorage->file_indexer();
       if (!seek_to_first) {
+        // TODO(ljin): remove before committing
+        // f_idx = FindFileInRange(
+        //    files[level], internal_key, 0, files[level].size());
+
         if (search_left_bound == search_right_bound) {
           f_idx = search_left_bound;
         } else if (search_left_bound < search_right_bound) {
-          f_idx =
-              FindFileInRange(level_files, internal_key, search_left_bound,
-                              search_right_bound == FileIndexer::kLevelMaxIndex
-                                  ? static_cast<uint32_t>(level_files.size())
-                                  : search_right_bound);
+          f_idx = FindFileInRange(
+              files[level], internal_key, search_left_bound,
+              search_right_bound == FileIndexer::kLevelMaxIndex ?
+                files[level].size() : search_right_bound);
         } else {
           // search_left_bound > search_right_bound
           // There are only 2 cases this can happen:
           // (1) target key is smaller than left most file
           // (2) target key is larger than right most file
-          assert(search_left_bound == (int32_t)level_files.size() ||
+          assert(search_left_bound == (int32_t)files[level].size() ||
                  search_right_bound == -1);
           if (search_right_bound == -1) {
             assert(search_left_bound == 0);
             f_idx = 0;
           } else {
-            indexer.GetNextLevelIndex(
-                level, level_files.size() - 1,
+            sv_->current->file_indexer_.GetNextLevelIndex(
+                level, files[level].size() - 1,
                 1, 1, &search_left_bound, &search_right_bound);
             continue;
           }
         }
 
         // Prepare hints for the next level
-        if (f_idx < level_files.size()) {
+        if (f_idx < files[level].size()) {
           int cmp_smallest = user_comparator_->Compare(
-              user_key, level_files[f_idx]->smallest.user_key());
+              user_key, files[level][f_idx]->smallest.user_key());
           int cmp_largest = -1;
           if (cmp_smallest >= 0) {
             cmp_smallest = user_comparator_->Compare(
-                user_key, level_files[f_idx]->smallest.user_key());
+                user_key, files[level][f_idx]->smallest.user_key());
           }
-          indexer.GetNextLevelIndex(level, f_idx,
+          sv_->current->file_indexer_.GetNextLevelIndex(level, f_idx,
               cmp_smallest, cmp_largest,
               &search_left_bound, &search_right_bound);
         } else {
-          indexer.GetNextLevelIndex(
-              level, level_files.size() - 1,
+          sv_->current->file_indexer_.GetNextLevelIndex(
+              level, files[level].size() - 1,
               1, 1, &search_left_bound, &search_right_bound);
         }
       }
 
       // Seek
-      if (f_idx < level_files.size()) {
+      if (f_idx < files[level].size()) {
         level_iters_[level - 1]->SetFileIndex(f_idx);
         seek_to_first ? level_iters_[level - 1]->SeekToFirst() :
                         level_iters_[level - 1]->Seek(internal_key);
 
-        if (!level_iters_[level - 1]->status().ok()) {
-          immutable_status_ = level_iters_[level - 1]->status();
+        if (level_iters_[level - 1]->status().IsIncomplete()) {
+          // see above
+          assert(read_options_.read_tier == kBlockCacheTier);
+          status_ = level_iters_[level - 1]->status();
+          valid_ = false;
+          return;
         } else if (level_iters_[level - 1]->Valid()) {
           immutable_min_heap_.push(level_iters_[level - 1]);
         }
@@ -343,7 +338,7 @@ void ForwardIterator::Next() {
     std::string current_key = key().ToString();
     Slice old_key(current_key.data(), current_key.size());
 
-    RebuildIterators(true);
+    RebuildIterators();
     SeekInternal(old_key, false);
     if (!valid_ || key().compare(old_key) != 0) {
       return;
@@ -368,8 +363,11 @@ void ForwardIterator::Next() {
 
   current_->Next();
   if (current_ != mutable_iter_) {
-    if (!current_->status().ok()) {
-      immutable_status_ = current_->status();
+    if (current_->status().IsIncomplete()) {
+      assert(read_options_.read_tier == kBlockCacheTier);
+      status_ = current_->status();
+      valid_ = false;
+      return;
     } else if (current_->Valid()) {
       immutable_min_heap_.push(current_);
     }
@@ -395,35 +393,45 @@ Status ForwardIterator::status() const {
     return mutable_iter_->status();
   }
 
-  return immutable_status_;
+  for (auto *it : imm_iters_) {
+    if (it && !it->status().ok()) {
+      return it->status();
+    }
+  }
+  for (auto *it : l0_iters_) {
+    if (it && !it->status().ok()) {
+      return it->status();
+    }
+  }
+  for (auto *it : level_iters_) {
+    if (it && !it->status().ok()) {
+      return it->status();
+    }
+  }
+
+  return Status::OK();
 }
 
-void ForwardIterator::RebuildIterators(bool refresh_sv) {
+void ForwardIterator::RebuildIterators() {
   // Clean up
-  Cleanup(refresh_sv);
-  if (refresh_sv) {
-    // New
-    sv_ = cfd_->GetReferencedSuperVersion(&(db_->mutex_));
-  }
+  Cleanup();
+  // New
+  sv_ = cfd_->GetReferencedSuperVersion(&(db_->mutex_));
   mutable_iter_ = sv_->mem->NewIterator(read_options_, &arena_);
   sv_->imm->AddIterators(read_options_, &imm_iters_, &arena_);
-
-  const auto* vstorage = sv_->current->storage_info();
-  const auto& l0_files = vstorage->LevelFiles(0);
+  const auto& l0_files = sv_->current->files_[0];
   l0_iters_.reserve(l0_files.size());
   for (const auto* l0 : l0_files) {
     l0_iters_.push_back(cfd_->table_cache()->NewIterator(
         read_options_, *cfd_->soptions(), cfd_->internal_comparator(), l0->fd));
   }
-  level_iters_.reserve(vstorage->num_levels() - 1);
-  for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
-    const auto& level_files = vstorage->LevelFiles(level);
-
-    if (level_files.empty()) {
+  level_iters_.reserve(sv_->current->NumberLevels() - 1);
+  for (int32_t level = 1; level < sv_->current->NumberLevels(); ++level) {
+    if (sv_->current->files_[level].empty()) {
       level_iters_.push_back(nullptr);
     } else {
-      level_iters_.push_back(
-          new LevelIterator(cfd_, read_options_, level_files));
+      level_iters_.push_back(new LevelIterator(cfd_, read_options_,
+          sv_->current->files_[level]));
     }
   }
 
@@ -432,7 +440,7 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
 }
 
 void ForwardIterator::ResetIncompleteIterators() {
-  const auto& l0_files = sv_->current->storage_info()->LevelFiles(0);
+  const auto& l0_files = sv_->current->files_[0];
   for (uint32_t i = 0; i < l0_iters_.size(); ++i) {
     assert(i < l0_files.size());
     if (!l0_iters_[i]->status().IsIncomplete()) {
@@ -489,7 +497,7 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
   // 'target' belongs to that interval (immutable_min_heap_.top() is already
   // at the correct position).
 
-  if (!valid_ || !current_ || !is_prev_set_ || !immutable_status_.ok()) {
+  if (!valid_ || !current_ || !is_prev_set_) {
     return true;
   }
   Slice prev_key = prev_key_.GetKey();

db/forward_iterator.h

@@ -51,7 +51,7 @@ typedef std::priority_queue<Iterator*,
 class ForwardIterator : public Iterator {
  public:
   ForwardIterator(DBImpl* db, const ReadOptions& read_options,
-                  ColumnFamilyData* cfd, SuperVersion* current_sv = nullptr);
+                  ColumnFamilyData* cfd);
   virtual ~ForwardIterator();
 
   void SeekToLast() override {
@@ -72,8 +72,8 @@ class ForwardIterator : public Iterator {
   virtual Status status() const override;
 
  private:
-  void Cleanup(bool release_sv);
-  void RebuildIterators(bool refresh_sv);
+  void Cleanup();
+  void RebuildIterators();
   void ResetIncompleteIterators();
   void SeekInternal(const Slice& internal_key, bool seek_to_first);
   void UpdateCurrent();
@@ -97,7 +97,6 @@ class ForwardIterator : public Iterator {
   Iterator* current_;
   // internal iterator status
   Status status_;
-  Status immutable_status_;
   bool valid_;
 
   IterKey prev_key_;

db/internal_stats.cc

@@ -15,13 +15,10 @@
 #include <inttypes.h>
 #include <vector>
 #include "db/column_family.h"
-
 #include "db/db_impl.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
 namespace {
 const double kMB = 1048576.0;
 const double kGB = kMB * 1024;
@@ -31,55 +28,68 @@ void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
       buf, len,
       "\n** Compaction Stats [%s] **\n"
       "Level   Files   Size(MB) Score Read(GB)  Rn(GB) Rnp1(GB) "
-      "Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) "
-      "Comp(sec) Comp(cnt) Avg(sec) "
-      "Stall(sec) Stall(cnt) Avg(ms)     RecordIn   RecordDrop\n"
+      "Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s)  Rn(cnt) "
+      "Rnp1(cnt) Wnp1(cnt) Wnew(cnt)  Comp(sec) Comp(cnt) Avg(sec) "
+      "Stall(sec) Stall(cnt) Avg(ms) RecordIn RecordDrop\n"
       "--------------------------------------------------------------------"
       "--------------------------------------------------------------------"
-      "----------------------------------------------------------\n",
+      "--------------------------------------------------------------------\n",
       cf_name.c_str());
 }
 
 void PrintLevelStats(char* buf, size_t len, const std::string& name,
     int num_files, int being_compacted, double total_file_size, double score,
-    double w_amp, double stall_us, uint64_t stalls,
+    double rw_amp, double w_amp, double stall_us, uint64_t stalls,
     const InternalStats::CompactionStats& stats) {
   uint64_t bytes_read = stats.bytes_readn + stats.bytes_readnp1;
   uint64_t bytes_new = stats.bytes_written - stats.bytes_readnp1;
   double elapsed = (stats.micros + 1) / 1000000.0;
 
   snprintf(buf, len,
-           "%4s %5d/%-3d %8.0f %5.1f " /* Level, Files, Size(MB), Score */
-           "%8.1f "                    /* Read(GB) */
-           "%7.1f "                    /* Rn(GB) */
-           "%8.1f "                    /* Rnp1(GB) */
-           "%9.1f "                    /* Write(GB) */
-           "%8.1f "                    /* Wnew(GB) */
-           "%9.1f "                    /* Moved(GB) */
-           "%5.1f "                    /* W-Amp */
-           "%8.1f "                    /* Rd(MB/s) */
-           "%8.1f "                    /* Wr(MB/s) */
-           "%9.0f "                   /* Comp(sec) */
-           "%9d "                      /* Comp(cnt) */
-           "%8.3f "                    /* Avg(sec) */
-           "%10.2f "                   /* Stall(sec) */
-           "%10" PRIu64
-           " "      /* Stall(cnt) */
-           "%7.2f " /* Avg(ms) */
-           "%12" PRIu64
-           " " /* input entries */
-           "%12" PRIu64 "\n" /* number of records reduced */,
-           name.c_str(), num_files, being_compacted, total_file_size / kMB,
-           score, bytes_read / kGB, stats.bytes_readn / kGB,
-           stats.bytes_readnp1 / kGB, stats.bytes_written / kGB,
-           bytes_new / kGB, stats.bytes_moved / kGB,
-           w_amp, bytes_read / kMB / elapsed,
-           stats.bytes_written / kMB / elapsed,
-           stats.micros / 1000000.0, stats.count,
-           stats.count == 0 ? 0 : stats.micros / 1000000.0 / stats.count,
-           stall_us / 1000000.0, stalls,
-           stalls == 0 ? 0 : stall_us / 1000.0 / stalls,
-           stats.num_input_records, stats.num_dropped_records);
+    "%4s %5d/%-3d %8.0f %5.1f " /* Level, Files, Size(MB), Score */
+    "%8.1f " /* Read(GB) */
+    "%7.1f " /* Rn(GB) */
+    "%8.1f " /* Rnp1(GB) */
+    "%9.1f " /* Write(GB) */
+    "%8.1f " /* Wnew(GB) */
+    "%6.1f " /* RW-Amp */
+    "%5.1f " /* W-Amp */
+    "%8.1f " /* Rd(MB/s) */
+    "%8.1f " /* Wr(MB/s) */
+    "%8d " /* Rn(cnt) */
+    "%9d " /* Rnp1(cnt) */
+    "%9d " /* Wnp1(cnt) */
+    "%9d " /* Wnew(cnt) */
+    "%10.0f " /* Comp(sec) */
+    "%9d " /* Comp(cnt) */
+    "%8.3f " /* Avg(sec) */
+    "%10.2f " /* Stall(sec) */
+    "%10" PRIu64 " " /* Stall(cnt) */
+    "%7.2f" /* Avg(ms) */
+    "%8d " /* input entries */
+    "%10d\n" /* number of records reduced */,
+    name.c_str(), num_files, being_compacted, total_file_size / kMB, score,
+    bytes_read / kGB,
+    stats.bytes_readn / kGB,
+    stats.bytes_readnp1 / kGB,
+    stats.bytes_written / kGB,
+    bytes_new / kGB,
+    rw_amp,
+    w_amp,
+    bytes_read / kMB / elapsed,
+    stats.bytes_written / kMB / elapsed,
+    stats.files_in_leveln,
+    stats.files_in_levelnp1,
+    stats.files_out_levelnp1,
+    stats.files_out_levelnp1 - stats.files_in_levelnp1,
+    stats.micros / 1000000.0,
+    stats.count,
+    stats.count == 0 ? 0 : stats.micros / 1000000.0 / stats.count,
+    stall_us / 1000000.0,
+    stalls,
+    stalls == 0 ? 0 : stall_us / 1000.0 / stalls,
+    stats.num_input_records,
+    stats.num_dropped_records);
 }
 
 
@@ -123,8 +133,6 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
     return kBackgroundErrors;
   } else if (in == "cur-size-active-mem-table") {
     return kCurSizeActiveMemTable;
-  } else if (in == "cur-size-all-mem-tables") {
-    return kCurSizeAllMemTables;
   } else if (in == "num-entries-active-mem-table") {
     return kNumEntriesInMutableMemtable;
   } else if (in == "num-entries-imm-mem-tables") {
@@ -136,10 +144,6 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
     return kEstimatedUsageByTableReaders;
   } else if (in == "is-file-deletions-enabled") {
     return kIsFileDeletionEnabled;
-  } else if (in == "num-snapshots") {
-    return kNumSnapshots;
-  } else if (in == "oldest-snapshot-time") {
-    return kOldestSnapshotTime;
   }
   return kUnknown;
 }
@@ -163,8 +167,7 @@ bool InternalStats::GetStringProperty(DBPropertyType property_type,
                                       const Slice& property,
                                       std::string* value) {
   assert(value != nullptr);
-  auto* current = cfd_->current();
-  const auto* vstorage = current->storage_info();
+  Version* current = cfd_->current();
   Slice in = property;
 
   switch (property_type) {
@@ -177,7 +180,7 @@ bool InternalStats::GetStringProperty(DBPropertyType property_type,
       } else {
         char buf[100];
         snprintf(buf, sizeof(buf), "%d",
-                 vstorage->NumLevelFiles(static_cast<int>(level)));
+                 current->NumLevelFiles(static_cast<int>(level)));
         *value = buf;
         return true;
       }
@@ -191,8 +194,8 @@ bool InternalStats::GetStringProperty(DBPropertyType property_type,
 
       for (int level = 0; level < number_levels_; level++) {
         snprintf(buf, sizeof(buf), "%3d %8d %8.0f\n", level,
-                 vstorage->NumLevelFiles(level),
-                 vstorage->NumLevelBytes(level) / kMB);
+                 current->NumLevelFiles(level),
+                 current->NumLevelBytes(level) / kMB);
         value->append(buf);
       }
       return true;
@@ -224,7 +227,7 @@ bool InternalStats::GetStringProperty(DBPropertyType property_type,
 
 bool InternalStats::GetIntProperty(DBPropertyType property_type,
                                    uint64_t* value, DBImpl* db) const {
-  const auto* vstorage = cfd_->current()->storage_info();
+  Version* current = cfd_->current();
 
   switch (property_type) {
     case kNumImmutableMemTable:
@@ -237,7 +240,7 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
     case kCompactionPending:
       // 1 if the system already determines at least one compacdtion is needed.
       // 0 otherwise,
-      *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
+      *value = (current->NeedsCompaction() ? 1 : 0);
       return true;
     case kBackgroundErrors:
       // Accumulated number of  errors in background flushes or compactions.
@@ -247,17 +250,12 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
       // Current size of the active memtable
       *value = cfd_->mem()->ApproximateMemoryUsage();
       return true;
-    case kCurSizeAllMemTables:
-      // Current size of the active memtable + immutable memtables
-      *value = cfd_->mem()->ApproximateMemoryUsage() +
-               cfd_->imm()->ApproximateMemoryUsage();
-      return true;
     case kNumEntriesInMutableMemtable:
-      // Current number of entires in the active memtable
+      // Current size of the active memtable
       *value = cfd_->mem()->GetNumEntries();
       return true;
     case kNumEntriesInImmutableMemtable:
-      // Current number of entries in the immutable memtables
+      // Current size of the active memtable
       *value = cfd_->imm()->current()->GetTotalNumEntries();
       return true;
     case kEstimatedNumKeys:
@@ -265,13 +263,7 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
       // Use estimated entries in tables + total entries in memtables.
       *value = cfd_->mem()->GetNumEntries() +
                cfd_->imm()->current()->GetTotalNumEntries() +
-               vstorage->GetEstimatedActiveKeys();
-      return true;
-    case kNumSnapshots:
-      *value = db->snapshots().count();
-      return true;
-    case kOldestSnapshotTime:
-      *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
+               current->GetEstimatedActiveKeys();
       return true;
 #ifndef ROCKSDB_LITE
     case kIsFileDeletionEnabled:
@@ -294,25 +286,16 @@ void InternalStats::DumpDBStats(std::string* value) {
   value->append(buf);
   // Cumulative
   uint64_t user_bytes_written = db_stats_[InternalStats::BYTES_WRITTEN];
-  uint64_t num_keys_written = db_stats_[InternalStats::NUMBER_KEYS_WRITTEN];
   uint64_t write_other = db_stats_[InternalStats::WRITE_DONE_BY_OTHER];
   uint64_t write_self = db_stats_[InternalStats::WRITE_DONE_BY_SELF];
   uint64_t wal_bytes = db_stats_[InternalStats::WAL_FILE_BYTES];
   uint64_t wal_synced = db_stats_[InternalStats::WAL_FILE_SYNCED];
   uint64_t write_with_wal = db_stats_[InternalStats::WRITE_WITH_WAL];
   // Data
-  // writes: total number of write requests.
-  // keys: total number of key updates issued by all the write requests
-  // batches: number of group commits issued to the DB. Each group can contain
-  //          one or more writes.
-  // so writes/keys is the average number of put in multi-put or put
-  // writes/batches is the average group commit size.
-  //
-  // The format is the same for interval stats.
   snprintf(buf, sizeof(buf),
-           "Cumulative writes: %" PRIu64 " writes, %" PRIu64 " keys, %" PRIu64
-           " batches, %.1f writes per batch, %.2f GB user ingest\n",
-           write_other + write_self, num_keys_written, write_self,
+           "Cumulative writes: %" PRIu64 " writes, %" PRIu64 " batches, "
+           "%.1f writes per batch, %.2f GB user ingest\n",
+           write_other + write_self, write_self,
            (write_other + write_self) / static_cast<double>(write_self + 1),
            user_bytes_written / kGB);
   value->append(buf);
@@ -328,13 +311,11 @@ void InternalStats::DumpDBStats(std::string* value) {
   // Interval
   uint64_t interval_write_other = write_other - db_stats_snapshot_.write_other;
   uint64_t interval_write_self = write_self - db_stats_snapshot_.write_self;
-  uint64_t interval_num_keys_written =
-      num_keys_written - db_stats_snapshot_.num_keys_written;
   snprintf(buf, sizeof(buf),
-           "Interval writes: %" PRIu64 " writes, %" PRIu64 " keys, %" PRIu64
-           " batches, %.1f writes per batch, %.1f MB user ingest\n",
+           "Interval writes: %" PRIu64 " writes, %" PRIu64 " batches, "
+           "%.1f writes per batch, %.1f MB user ingest\n",
            interval_write_other + interval_write_self,
-           interval_num_keys_written, interval_write_self,
+           interval_write_self,
            static_cast<double>(interval_write_other + interval_write_self) /
                (interval_write_self + 1),
            (user_bytes_written - db_stats_snapshot_.ingest_bytes) / kMB);
@@ -359,32 +340,30 @@ void InternalStats::DumpDBStats(std::string* value) {
   db_stats_snapshot_.ingest_bytes = user_bytes_written;
   db_stats_snapshot_.write_other = write_other;
   db_stats_snapshot_.write_self = write_self;
-  db_stats_snapshot_.num_keys_written = num_keys_written;
   db_stats_snapshot_.wal_bytes = wal_bytes;
   db_stats_snapshot_.wal_synced = wal_synced;
   db_stats_snapshot_.write_with_wal = write_with_wal;
 }
 
 void InternalStats::DumpCFStats(std::string* value) {
-  const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
+  Version* current = cfd_->current();
 
   int num_levels_to_check =
-      (cfd_->ioptions()->compaction_style != kCompactionStyleUniversal &&
-       cfd_->ioptions()->compaction_style != kCompactionStyleFIFO)
-          ? vstorage->num_levels() - 1
+      (cfd_->options()->compaction_style != kCompactionStyleUniversal &&
+       cfd_->options()->compaction_style != kCompactionStyleFIFO)
+          ? current->NumberLevels() - 1
           : 1;
-
   // Compaction scores are sorted base on its value. Restore them to the
   // level order
   std::vector<double> compaction_score(number_levels_, 0);
   for (int i = 0; i < num_levels_to_check; ++i) {
-    compaction_score[vstorage->CompactionScoreLevel(i)] =
-        vstorage->CompactionScore(i);
+    compaction_score[current->compaction_level_[i]] =
+      current->compaction_score_[i];
   }
   // Count # of files being compacted for each level
   std::vector<int> files_being_compacted(number_levels_, 0);
   for (int level = 0; level < num_levels_to_check; ++level) {
-    for (auto* f : vstorage->LevelFiles(level)) {
+    for (auto* f : current->files_[level]) {
       if (f->being_compacted) {
         ++files_being_compacted[level];
       }
@@ -407,7 +386,7 @@ void InternalStats::DumpCFStats(std::string* value) {
   uint64_t total_stall_count = 0;
   double total_stall_us = 0;
   for (int level = 0; level < number_levels_; level++) {
-    int files = vstorage->NumLevelFiles(level);
+    int files = current->NumLevelFiles(level);
     total_files += files;
     total_files_being_compacted += files_being_compacted[level];
     if (comp_stats_[level].micros > 0 || files > 0) {
@@ -426,29 +405,36 @@ void InternalStats::DumpCFStats(std::string* value) {
             stall_leveln_slowdown_hard_[level]);
 
       stats_sum.Add(comp_stats_[level]);
-      total_file_size += vstorage->NumLevelBytes(level);
+      total_file_size += current->NumLevelBytes(level);
       total_stall_us += stall_us;
       total_stall_count += stalls;
       total_slowdown_soft += stall_leveln_slowdown_soft_[level];
       total_slowdown_count_soft += stall_leveln_slowdown_count_soft_[level];
       total_slowdown_hard += stall_leveln_slowdown_hard_[level];
       total_slowdown_count_hard += stall_leveln_slowdown_count_hard_[level];
+      int64_t bytes_read = comp_stats_[level].bytes_readn +
+                           comp_stats_[level].bytes_readnp1;
+      double rw_amp = (comp_stats_[level].bytes_readn == 0) ? 0.0
+          : (comp_stats_[level].bytes_written + bytes_read) /
+            static_cast<double>(comp_stats_[level].bytes_readn);
       double w_amp = (comp_stats_[level].bytes_readn == 0) ? 0.0
           : comp_stats_[level].bytes_written /
             static_cast<double>(comp_stats_[level].bytes_readn);
-      PrintLevelStats(buf, sizeof(buf), "L" + ToString(level), files,
-                      files_being_compacted[level],
-                      vstorage->NumLevelBytes(level), compaction_score[level],
-                      w_amp, stall_us, stalls, comp_stats_[level]);
+      PrintLevelStats(buf, sizeof(buf), "L" + std::to_string(level),
+          files, files_being_compacted[level], current->NumLevelBytes(level),
+          compaction_score[level], rw_amp, w_amp, stall_us, stalls,
+          comp_stats_[level]);
       value->append(buf);
     }
   }
   uint64_t curr_ingest = cf_stats_value_[BYTES_FLUSHED];
   // Cumulative summary
+  double rw_amp = (stats_sum.bytes_written + stats_sum.bytes_readn +
+      stats_sum.bytes_readnp1) / static_cast<double>(curr_ingest + 1);
   double w_amp = stats_sum.bytes_written / static_cast<double>(curr_ingest + 1);
   // Stats summary across levels
   PrintLevelStats(buf, sizeof(buf), "Sum", total_files,
-      total_files_being_compacted, total_file_size, 0, w_amp,
+      total_files_being_compacted, total_file_size, 0, rw_amp, w_amp,
       total_stall_us, total_stall_count, stats_sum);
   value->append(buf);
   // Interval summary
@@ -456,9 +442,12 @@ void InternalStats::DumpCFStats(std::string* value) {
       curr_ingest - cf_stats_snapshot_.ingest_bytes + 1;
   CompactionStats interval_stats(stats_sum);
   interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
+  rw_amp = (interval_stats.bytes_written +
+      interval_stats.bytes_readn + interval_stats.bytes_readnp1) /
+      static_cast<double>(interval_ingest);
   w_amp = interval_stats.bytes_written / static_cast<double>(interval_ingest);
   PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0,
-      w_amp, total_stall_us - cf_stats_snapshot_.stall_us,
+      rw_amp, w_amp, total_stall_us - cf_stats_snapshot_.stall_us,
       total_stall_count - cf_stats_snapshot_.stall_count, interval_stats);
   value->append(buf);
 
@@ -494,14 +483,4 @@ void InternalStats::DumpCFStats(std::string* value) {
   cf_stats_snapshot_.stall_count = total_stall_count;
 }
 
-
-#else
-
-DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
-                               bool* need_out_of_mutex) {
-  return kUnknown;
-}
-
-#endif  // !ROCKSDB_LITE
-
 }  // namespace rocksdb

db/internal_stats.h

@@ -36,8 +36,6 @@ enum DBPropertyType : uint32_t {
   kCompactionPending,      // Return 1 if a compaction is pending. Otherwise 0.
   kBackgroundErrors,       // Return accumulated background errors encountered.
   kCurSizeActiveMemTable,  // Return current size of the active memtable
-  kCurSizeAllMemTables,    // Return current size of all (active + immutable)
-                           // memtables
   kNumEntriesInMutableMemtable,    // Return number of entries in the mutable
                                    // memtable.
   kNumEntriesInImmutableMemtable,  // Return sum of number of entries in all
@@ -46,16 +44,12 @@ enum DBPropertyType : uint32_t {
   kEstimatedUsageByTableReaders,  // Estimated memory by table readers.
   kIsFileDeletionEnabled,         // Equals disable_delete_obsolete_files_,
                                   // 0 means file deletions enabled
-  kNumSnapshots,                  // Number of snapshots in the system
-  kOldestSnapshotTime,            // Unix timestamp of the first snapshot
 };
 
 extern DBPropertyType GetPropertyType(const Slice& property,
                                       bool* is_int_property,
                                       bool* need_out_of_mutex);
 
-
-#ifndef ROCKSDB_LITE
 class InternalStats {
  public:
   enum InternalCFStatsType {
@@ -71,7 +65,6 @@ class InternalStats {
     WAL_FILE_BYTES,
     WAL_FILE_SYNCED,
     BYTES_WRITTEN,
-    NUMBER_KEYS_WRITTEN,
     WRITE_DONE_BY_OTHER,
     WRITE_DONE_BY_SELF,
     WRITE_WITH_WAL,
@@ -121,9 +114,6 @@ class InternalStats {
     // Total bytes written during compaction between levels N and N+1
     uint64_t bytes_written;
 
-    // Total bytes moved to this level
-    uint64_t bytes_moved;
-
     // Files read from level N during compaction between levels N and N+1
     int files_in_leveln;
 
@@ -134,34 +124,32 @@ class InternalStats {
     int files_out_levelnp1;
 
     // Total incoming entries during compaction between levels N and N+1
-    uint64_t num_input_records;
+    int num_input_records;
 
     // Accumulated diff number of entries
     // (num input entries - num output entires) for compaction  levels N and N+1
-    uint64_t num_dropped_records;
+    int num_dropped_records;
 
     // Number of compactions done
     int count;
 
-    explicit CompactionStats(int _count = 0)
+    explicit CompactionStats(int count = 0)
         : micros(0),
           bytes_readn(0),
           bytes_readnp1(0),
           bytes_written(0),
-          bytes_moved(0),
           files_in_leveln(0),
           files_in_levelnp1(0),
           files_out_levelnp1(0),
           num_input_records(0),
           num_dropped_records(0),
-          count(_count) {}
+          count(count) {}
 
     explicit CompactionStats(const CompactionStats& c)
         : micros(c.micros),
           bytes_readn(c.bytes_readn),
           bytes_readnp1(c.bytes_readnp1),
           bytes_written(c.bytes_written),
-          bytes_moved(c.bytes_moved),
           files_in_leveln(c.files_in_leveln),
           files_in_levelnp1(c.files_in_levelnp1),
           files_out_levelnp1(c.files_out_levelnp1),
@@ -174,7 +162,6 @@ class InternalStats {
       this->bytes_readn += c.bytes_readn;
       this->bytes_readnp1 += c.bytes_readnp1;
       this->bytes_written += c.bytes_written;
-      this->bytes_moved += c.bytes_moved;
       this->files_in_leveln += c.files_in_leveln;
       this->files_in_levelnp1 += c.files_in_levelnp1;
       this->files_out_levelnp1 += c.files_out_levelnp1;
@@ -188,7 +175,6 @@ class InternalStats {
       this->bytes_readn -= c.bytes_readn;
       this->bytes_readnp1 -= c.bytes_readnp1;
       this->bytes_written -= c.bytes_written;
-      this->bytes_moved -= c.bytes_moved;
       this->files_in_leveln -= c.files_in_leveln;
       this->files_in_levelnp1 -= c.files_in_levelnp1;
       this->files_out_levelnp1 -= c.files_out_levelnp1;
@@ -202,10 +188,6 @@ class InternalStats {
     comp_stats_[level].Add(stats);
   }
 
-  void IncBytesMoved(int level, uint64_t amount) {
-    comp_stats_[level].bytes_moved += amount;
-  }
-
   void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {
     if (soft) {
       stall_leveln_slowdown_soft_[level] += micros;
@@ -280,11 +262,6 @@ class InternalStats {
     // another thread.
     uint64_t write_other;
     uint64_t write_self;
-    // Total number of keys written. write_self and write_other measure number
-    // of write requests written, Each of the write request can contain updates
-    // to multiple keys. num_keys_written is total number of keys updated by all
-    // those writes.
-    uint64_t num_keys_written;
     double seconds_up;
 
     DBStatsSnapshot()
@@ -294,7 +271,6 @@ class InternalStats {
           write_with_wal(0),
           write_other(0),
           write_self(0),
-          num_keys_written(0),
           seconds_up(0) {}
   } db_stats_snapshot_;
 
@@ -311,77 +287,4 @@ class InternalStats {
   const uint64_t started_at_;
 };
 
-#else
-
-class InternalStats {
- public:
-  enum InternalCFStatsType {
-    LEVEL0_SLOWDOWN,
-    MEMTABLE_COMPACTION,
-    LEVEL0_NUM_FILES,
-    WRITE_STALLS_ENUM_MAX,
-    BYTES_FLUSHED,
-    INTERNAL_CF_STATS_ENUM_MAX,
-  };
-
-  enum InternalDBStatsType {
-    WAL_FILE_BYTES,
-    WAL_FILE_SYNCED,
-    BYTES_WRITTEN,
-    NUMBER_KEYS_WRITTEN,
-    WRITE_DONE_BY_OTHER,
-    WRITE_DONE_BY_SELF,
-    WRITE_WITH_WAL,
-    INTERNAL_DB_STATS_ENUM_MAX,
-  };
-
-  InternalStats(int num_levels, Env* env, ColumnFamilyData* cfd) {}
-
-  struct CompactionStats {
-    uint64_t micros;
-    uint64_t bytes_readn;
-    uint64_t bytes_readnp1;
-    uint64_t bytes_written;
-    uint64_t bytes_moved;
-    int files_in_leveln;
-    int files_in_levelnp1;
-    int files_out_levelnp1;
-    uint64_t num_input_records;
-    uint64_t num_dropped_records;
-    int count;
-
-    explicit CompactionStats(int _count = 0) {}
-
-    explicit CompactionStats(const CompactionStats& c) {}
-
-    void Add(const CompactionStats& c) {}
-
-    void Subtract(const CompactionStats& c) {}
-  };
-
-  void AddCompactionStats(int level, const CompactionStats& stats) {}
-
-  void IncBytesMoved(int level, uint64_t amount) {}
-
-  void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {}
-
-  void AddCFStats(InternalCFStatsType type, uint64_t value) {}
-
-  void AddDBStats(InternalDBStatsType type, uint64_t value) {}
-
-  uint64_t GetBackgroundErrorCount() const { return 0; }
-
-  uint64_t BumpAndGetBackgroundErrorCount() { return 0; }
-
-  bool GetStringProperty(DBPropertyType property_type, const Slice& property,
-                         std::string* value) { return false; }
-
-  bool GetIntProperty(DBPropertyType property_type, uint64_t* value,
-                      DBImpl* db) const { return false; }
-
-  bool GetIntPropertyOutOfMutex(DBPropertyType property_type, Version* version,
-                                uint64_t* value) const { return false; }
-};
-#endif  // !ROCKSDB_LITE
-
 }  // namespace rocksdb

db/job_context.h

@@ -1,102 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-
-#include <string>
-#include <vector>
-
-#include "db/column_family.h"
-
-namespace rocksdb {
-
-class MemTable;
-
-struct JobContext {
-  inline bool HaveSomethingToDelete() const {
-    return candidate_files.size() || sst_delete_files.size() ||
-           log_delete_files.size() || new_superversion != nullptr ||
-           superversions_to_free.size() > 0 || memtables_to_free.size() > 0;
-  }
-
-  // Structure to store information for candidate files to delete.
-  struct CandidateFileInfo {
-    std::string file_name;
-    uint32_t path_id;
-    CandidateFileInfo(std::string name, uint32_t path)
-        : file_name(std::move(name)), path_id(path) {}
-    bool operator==(const CandidateFileInfo& other) const {
-      return file_name == other.file_name && path_id == other.path_id;
-    }
-  };
-
-  // a list of all files that we'll consider deleting
-  // (every once in a while this is filled up with all files
-  // in the DB directory)
-  std::vector<CandidateFileInfo> candidate_files;
-
-  // the list of all live sst files that cannot be deleted
-  std::vector<FileDescriptor> sst_live;
-
-  // a list of sst files that we need to delete
-  std::vector<FileMetaData*> sst_delete_files;
-
-  // a list of log files that we need to delete
-  std::vector<uint64_t> log_delete_files;
-
-  // a list of memtables to be free
-  autovector<MemTable*> memtables_to_free;
-
-  autovector<SuperVersion*> superversions_to_free;
-
-  SuperVersion* new_superversion;  // if nullptr no new superversion
-
-  // the current manifest_file_number, log_number and prev_log_number
-  // that corresponds to the set of files in 'live'.
-  uint64_t manifest_file_number;
-  uint64_t pending_manifest_file_number;
-  uint64_t log_number;
-  uint64_t prev_log_number;
-
-  uint64_t min_pending_output = 0;
-
-  explicit JobContext(bool create_superversion = false) {
-    manifest_file_number = 0;
-    pending_manifest_file_number = 0;
-    log_number = 0;
-    prev_log_number = 0;
-    new_superversion = create_superversion ? new SuperVersion() : nullptr;
-  }
-
-  void Clean() {
-    // free pending memtables
-    for (auto m : memtables_to_free) {
-      delete m;
-    }
-    // free superversions
-    for (auto s : superversions_to_free) {
-      delete s;
-    }
-    // if new_superversion was not used, it will be non-nullptr and needs
-    // to be freed here
-    delete new_superversion;
-
-    memtables_to_free.clear();
-    superversions_to_free.clear();
-    new_superversion = nullptr;
-  }
-
-  ~JobContext() {
-    assert(memtables_to_free.size() == 0);
-    assert(superversions_to_free.size() == 0);
-    assert(new_superversion == nullptr);
-  }
-};
-
-}  // namespace rocksdb

db/listener_test.cc

@@ -1,348 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include "db/dbformat.h"
-#include "db/db_impl.h"
-#include "db/filename.h"
-#include "db/version_set.h"
-#include "db/write_batch_internal.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/compaction_filter.h"
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/filter_policy.h"
-#include "rocksdb/perf_context.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/slice_transform.h"
-#include "rocksdb/table.h"
-#include "rocksdb/options.h"
-#include "rocksdb/table_properties.h"
-#include "table/block_based_table_factory.h"
-#include "table/plain_table_factory.h"
-#include "util/hash.h"
-#include "util/hash_linklist_rep.h"
-#include "utilities/merge_operators.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/rate_limiter.h"
-#include "util/statistics.h"
-#include "util/testharness.h"
-#include "util/sync_point.h"
-#include "util/testutil.h"
-
-#ifndef ROCKSDB_LITE
-
-namespace rocksdb {
-
-class EventListenerTest {
- public:
-  EventListenerTest() {
-    dbname_ = test::TmpDir() + "/listener_test";
-    ASSERT_OK(DestroyDB(dbname_, Options()));
-    db_ = nullptr;
-    Reopen();
-  }
-
-  ~EventListenerTest() {
-    Close();
-    Options options;
-    options.db_paths.emplace_back(dbname_, 0);
-    options.db_paths.emplace_back(dbname_ + "_2", 0);
-    options.db_paths.emplace_back(dbname_ + "_3", 0);
-    options.db_paths.emplace_back(dbname_ + "_4", 0);
-    ASSERT_OK(DestroyDB(dbname_, options));
-  }
-
-  void CreateColumnFamilies(const std::vector<std::string>& cfs,
-                            const ColumnFamilyOptions* options = nullptr) {
-    ColumnFamilyOptions cf_opts;
-    cf_opts = ColumnFamilyOptions(Options());
-    size_t cfi = handles_.size();
-    handles_.resize(cfi + cfs.size());
-    for (auto cf : cfs) {
-      ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]));
-    }
-  }
-
-  void Close() {
-    for (auto h : handles_) {
-      delete h;
-    }
-    handles_.clear();
-    delete db_;
-    db_ = nullptr;
-  }
-
-  void ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
-                                const Options* options = nullptr) {
-    ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
-  }
-
-  Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs,
-                                     const Options* options = nullptr) {
-    Close();
-    Options opts = (options == nullptr) ? Options() : *options;
-    std::vector<const Options*> v_opts(cfs.size(), &opts);
-    return TryReopenWithColumnFamilies(cfs, v_opts);
-  }
-
-  Status TryReopenWithColumnFamilies(
-      const std::vector<std::string>& cfs,
-      const std::vector<const Options*>& options) {
-    Close();
-    ASSERT_EQ(cfs.size(), options.size());
-    std::vector<ColumnFamilyDescriptor> column_families;
-    for (size_t i = 0; i < cfs.size(); ++i) {
-      column_families.push_back(ColumnFamilyDescriptor(cfs[i], *options[i]));
-    }
-    DBOptions db_opts = DBOptions(*options[0]);
-    return DB::Open(db_opts, dbname_, column_families, &handles_, &db_);
-  }
-
-  Status TryReopen(Options* options = nullptr) {
-    Close();
-    Options opts;
-    if (options != nullptr) {
-      opts = *options;
-    } else {
-      opts.create_if_missing = true;
-    }
-
-    return DB::Open(opts, dbname_, &db_);
-  }
-
-  void Reopen(Options* options = nullptr) {
-    ASSERT_OK(TryReopen(options));
-  }
-
-  void CreateAndReopenWithCF(const std::vector<std::string>& cfs,
-                             const Options* options = nullptr) {
-    CreateColumnFamilies(cfs, options);
-    std::vector<std::string> cfs_plus_default = cfs;
-    cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName);
-    ReopenWithColumnFamilies(cfs_plus_default, options);
-  }
-
-  DBImpl* dbfull() {
-    return reinterpret_cast<DBImpl*>(db_);
-  }
-
-  Status Put(int cf, const Slice& k, const Slice& v,
-             WriteOptions wo = WriteOptions()) {
-    return db_->Put(wo, handles_[cf], k, v);
-  }
-
-  Status Flush(int cf = 0) {
-    FlushOptions opt = FlushOptions();
-    opt.wait = true;
-    if (cf == 0) {
-      return db_->Flush(opt);
-    } else {
-      return db_->Flush(opt, handles_[cf]);
-    }
-  }
-
-
-  DB* db_;
-  std::string dbname_;
-  std::vector<ColumnFamilyHandle*> handles_;
-};
-
-class TestFlushListener : public EventListener {
- public:
-  void OnFlushCompleted(
-      DB* db, const std::string& name,
-      const std::string& file_path,
-      bool triggered_writes_slowdown,
-      bool triggered_writes_stop) override {
-    flushed_dbs_.push_back(db);
-    flushed_column_family_names_.push_back(name);
-    if (triggered_writes_slowdown) {
-      slowdown_count++;
-    }
-    if (triggered_writes_stop) {
-      stop_count++;
-    }
-  }
-
-  std::vector<std::string> flushed_column_family_names_;
-  std::vector<DB*> flushed_dbs_;
-  int slowdown_count;
-  int stop_count;
-};
-
-TEST(EventListenerTest, OnSingleDBFlushTest) {
-  Options options;
-  options.write_buffer_size = 100000;
-  TestFlushListener* listener = new TestFlushListener();
-  options.listeners.emplace_back(listener);
-  std::vector<std::string> cf_names = {
-      "pikachu", "ilya", "muromec", "dobrynia",
-      "nikitich", "alyosha", "popovich"};
-  CreateAndReopenWithCF(cf_names, &options);
-
-  ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
-  ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
-  ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
-  ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
-  ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
-  ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
-  ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
-  for (size_t i = 1; i < 8; ++i) {
-    ASSERT_OK(Flush(static_cast<int>(i)));
-    dbfull()->TEST_WaitForFlushMemTable();
-    ASSERT_EQ(listener->flushed_dbs_.size(), i);
-    ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
-  }
-
-  // make sure call-back functions are called in the right order
-  for (size_t i = 0; i < cf_names.size(); ++i) {
-    ASSERT_EQ(listener->flushed_dbs_[i], db_);
-    ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]);
-  }
-}
-
-TEST(EventListenerTest, MultiCF) {
-  Options options;
-  options.write_buffer_size = 100000;
-  TestFlushListener* listener = new TestFlushListener();
-  options.listeners.emplace_back(listener);
-  std::vector<std::string> cf_names = {
-      "pikachu", "ilya", "muromec", "dobrynia",
-      "nikitich", "alyosha", "popovich"};
-  CreateAndReopenWithCF(cf_names, &options);
-
-  ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
-  ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
-  ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
-  ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
-  ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
-  ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
-  ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
-  for (size_t i = 1; i < 8; ++i) {
-    ASSERT_OK(Flush(static_cast<int>(i)));
-    ASSERT_EQ(listener->flushed_dbs_.size(), i);
-    ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
-  }
-
-  // make sure call-back functions are called in the right order
-  for (size_t i = 0; i < cf_names.size(); i++) {
-    ASSERT_EQ(listener->flushed_dbs_[i], db_);
-    ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]);
-  }
-}
-
-TEST(EventListenerTest, MultiDBMultiListeners) {
-  std::vector<TestFlushListener*> listeners;
-  const int kNumDBs = 5;
-  const int kNumListeners = 10;
-  for (int i = 0; i < kNumListeners; ++i) {
-    listeners.emplace_back(new TestFlushListener());
-  }
-
-  std::vector<std::string> cf_names = {
-      "pikachu", "ilya", "muromec", "dobrynia",
-      "nikitich", "alyosha", "popovich"};
-
-  Options options;
-  options.create_if_missing = true;
-  for (int i = 0; i < kNumListeners; ++i) {
-    options.listeners.emplace_back(listeners[i]);
-  }
-  DBOptions db_opts(options);
-  ColumnFamilyOptions cf_opts(options);
-
-  std::vector<DB*> dbs;
-  std::vector<std::vector<ColumnFamilyHandle *>> vec_handles;
-
-  for (int d = 0; d < kNumDBs; ++d) {
-    ASSERT_OK(DestroyDB(dbname_ + ToString(d), options));
-    DB* db;
-    std::vector<ColumnFamilyHandle*> handles;
-    ASSERT_OK(DB::Open(options, dbname_ + ToString(d), &db));
-    for (size_t c = 0; c < cf_names.size(); ++c) {
-      ColumnFamilyHandle* handle;
-      db->CreateColumnFamily(cf_opts, cf_names[c], &handle);
-      handles.push_back(handle);
-    }
-
-    vec_handles.push_back(std::move(handles));
-    dbs.push_back(db);
-  }
-
-  for (int d = 0; d < kNumDBs; ++d) {
-    for (size_t c = 0; c < cf_names.size(); ++c) {
-      ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c],
-                cf_names[c], cf_names[c]));
-    }
-  }
-
-  for (size_t c = 0; c < cf_names.size(); ++c) {
-    for (int d = 0; d < kNumDBs; ++d) {
-      ASSERT_OK(dbs[d]->Flush(FlushOptions(), vec_handles[d][c]));
-      reinterpret_cast<DBImpl*>(dbs[d])->TEST_WaitForFlushMemTable();
-    }
-  }
-
-  for (auto* listener : listeners) {
-    int pos = 0;
-    for (size_t c = 0; c < cf_names.size(); ++c) {
-      for (int d = 0; d < kNumDBs; ++d) {
-        ASSERT_EQ(listener->flushed_dbs_[pos], dbs[d]);
-        ASSERT_EQ(listener->flushed_column_family_names_[pos], cf_names[c]);
-        pos++;
-      }
-    }
-  }
-
-  for (auto handles : vec_handles) {
-    for (auto h : handles) {
-      delete h;
-    }
-    handles.clear();
-  }
-  vec_handles.clear();
-
-  for (auto db : dbs) {
-    delete db;
-  }
-}
-
-TEST(EventListenerTest, DisableBGCompaction) {
-  Options options;
-  TestFlushListener* listener = new TestFlushListener();
-  const int kSlowdownTrigger = 5;
-  const int kStopTrigger = 10;
-  options.level0_slowdown_writes_trigger = kSlowdownTrigger;
-  options.level0_stop_writes_trigger = kStopTrigger;
-  options.listeners.emplace_back(listener);
-  // BG compaction is disabled.  Number of L0 files will simply keeps
-  // increasing in this test.
-  options.compaction_style = kCompactionStyleNone;
-  options.compression = kNoCompression;
-  options.write_buffer_size = 100000;  // Small write buffer
-
-  CreateAndReopenWithCF({"pikachu"}, &options);
-  WriteOptions wopts;
-  wopts.timeout_hint_us = 100000;
-  ColumnFamilyMetaData cf_meta;
-  db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
-  // keep writing until writes are forced to stop.
-  for (int i = 0; static_cast<int>(cf_meta.file_count) < kStopTrigger; ++i) {
-    Put(1, ToString(i), std::string(100000, 'x'), wopts);
-    db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
-  }
-  ASSERT_GE(listener->slowdown_count, kStopTrigger - kSlowdownTrigger);
-  ASSERT_GE(listener->stop_count, 1);
-}
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE
-
-int main(int argc, char** argv) {
-  return rocksdb::test::RunAllTests();
-}
-

db/log_and_apply_bench.cc

@@ -6,23 +6,17 @@
 
 #include <vector>
 
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
 #include "util/testharness.h"
 #include "util/benchharness.h"
 #include "db/version_set.h"
 #include "db/write_controller.h"
-#include "db/writebuffer.h"
 #include "util/mutexlock.h"
 
 namespace rocksdb {
 
-std::string MakeKey(uint64_t num) {
+std::string MakeKey(unsigned int num) {
   char buf[30];
-  snprintf(buf, sizeof(buf), "%016" PRIu64, num);
+  snprintf(buf, sizeof(buf), "%016u", num);
   return std::string(buf);
 }
 
@@ -53,10 +47,9 @@ void BM_LogAndApply(int iters, int num_base_files) {
     // Notice we are using the default options not through SanitizeOptions().
     // We might want to initialize some options manually if needed.
     options.db_paths.emplace_back(dbname, 0);
-    WriteBuffer wb(options.db_write_buffer_size);
     // The parameter of table cache is passed in as null, so any file I/O
     // operation is likely to fail.
-    vset = new VersionSet(dbname, &options, sopt, nullptr, &wb, &wc);
+    vset = new VersionSet(dbname, &options, sopt, nullptr, &wc);
     std::vector<ColumnFamilyDescriptor> dummy;
     dummy.push_back(ColumnFamilyDescriptor());
     ASSERT_OK(vset->Recover(dummy));

db/log_reader.cc

@@ -20,9 +20,9 @@ namespace log {
 Reader::Reporter::~Reporter() {
 }
 
-Reader::Reader(unique_ptr<SequentialFile>&& _file, Reporter* reporter,
+Reader::Reader(unique_ptr<SequentialFile>&& file, Reporter* reporter,
                bool checksum, uint64_t initial_offset)
-    : file_(std::move(_file)),
+    : file_(std::move(file)),
       reporter_(reporter),
       checksum_(checksum),
       backing_store_(new char[kBlockSize]),
@@ -32,7 +32,8 @@ Reader::Reader(unique_ptr<SequentialFile>&& _file, Reporter* reporter,
       eof_offset_(0),
       last_record_offset_(0),
       end_of_buffer_offset_(0),
-      initial_offset_(initial_offset) {}
+      initial_offset_(initial_offset) {
+}
 
 Reader::~Reader() {
   delete[] backing_store_;
@@ -54,7 +55,7 @@ bool Reader::SkipToInitialBlock() {
   if (block_start_location > 0) {
     Status skip_status = file_->Skip(block_start_location);
     if (!skip_status.ok()) {
-      ReportDrop(static_cast<size_t>(block_start_location), skip_status);
+      ReportDrop(block_start_location, skip_status);
       return false;
     }
   }

db/log_test.cc

@@ -558,9 +558,9 @@ TEST(LogTest, ErrorJoinsRecords) {
 
   ASSERT_EQ("correct", Read());
   ASSERT_EQ("EOF", Read());
-  size_t dropped = DroppedBytes();
-  ASSERT_LE(dropped, 2 * kBlockSize + 100);
-  ASSERT_GE(dropped, 2 * kBlockSize);
+  const unsigned int dropped = DroppedBytes();
+  ASSERT_LE(dropped, 2*kBlockSize + 100);
+  ASSERT_GE(dropped, 2*kBlockSize);
 }
 
 TEST(LogTest, ReadStart) {

db/memtable.cc

@@ -15,7 +15,6 @@
 
 #include "db/dbformat.h"
 #include "db/merge_context.h"
-#include "db/writebuffer.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
@@ -33,8 +32,7 @@
 namespace rocksdb {
 
 MemTableOptions::MemTableOptions(
-    const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options)
+    const MutableCFOptions& mutable_cf_options, const Options& options)
   : write_buffer_size(mutable_cf_options.write_buffer_size),
     arena_block_size(mutable_cf_options.arena_block_size),
     memtable_prefix_bloom_bits(mutable_cf_options.memtable_prefix_bloom_bits),
@@ -42,48 +40,43 @@ MemTableOptions::MemTableOptions(
         mutable_cf_options.memtable_prefix_bloom_probes),
     memtable_prefix_bloom_huge_page_tlb_size(
         mutable_cf_options.memtable_prefix_bloom_huge_page_tlb_size),
-    inplace_update_support(ioptions.inplace_update_support),
-    inplace_update_num_locks(mutable_cf_options.inplace_update_num_locks),
-    inplace_callback(ioptions.inplace_callback),
+    inplace_update_support(options.inplace_update_support),
+    inplace_update_num_locks(options.inplace_update_num_locks),
+    inplace_callback(options.inplace_callback),
     max_successive_merges(mutable_cf_options.max_successive_merges),
-    filter_deletes(mutable_cf_options.filter_deletes),
-    statistics(ioptions.statistics),
-    merge_operator(ioptions.merge_operator),
-    info_log(ioptions.info_log) {}
+    filter_deletes(mutable_cf_options.filter_deletes) {}
 
 MemTable::MemTable(const InternalKeyComparator& cmp,
                    const ImmutableCFOptions& ioptions,
-                   const MutableCFOptions& mutable_cf_options,
-                   WriteBuffer* write_buffer)
+                   const MemTableOptions& moptions)
     : comparator_(cmp),
-      moptions_(ioptions, mutable_cf_options),
+      ioptions_(ioptions),
+      moptions_(moptions),
       refs_(0),
-      kArenaBlockSize(OptimizeBlockSize(moptions_.arena_block_size)),
-      arena_(moptions_.arena_block_size),
-      allocator_(&arena_, write_buffer),
+      kArenaBlockSize(OptimizeBlockSize(moptions.arena_block_size)),
+      arena_(moptions.arena_block_size),
       table_(ioptions.memtable_factory->CreateMemTableRep(
-          comparator_, &allocator_, ioptions.prefix_extractor,
-          ioptions.info_log)),
+          comparator_, &arena_, ioptions.prefix_extractor, ioptions.info_log)),
       num_entries_(0),
       flush_in_progress_(false),
       flush_completed_(false),
       file_number_(0),
       first_seqno_(0),
       mem_next_logfile_number_(0),
-      locks_(moptions_.inplace_update_support ?
-             moptions_.inplace_update_num_locks : 0),
+      locks_(moptions.inplace_update_support ? moptions.inplace_update_num_locks
+                                             : 0),
       prefix_extractor_(ioptions.prefix_extractor),
       should_flush_(ShouldFlushNow()),
       flush_scheduled_(false) {
   // if should_flush_ == true without an entry inserted, something must have
   // gone wrong already.
   assert(!should_flush_);
-  if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) {
+  if (prefix_extractor_ && moptions.memtable_prefix_bloom_bits > 0) {
     prefix_bloom_.reset(new DynamicBloom(
-        &allocator_,
-        moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality,
-        moptions_.memtable_prefix_bloom_probes, nullptr,
-        moptions_.memtable_prefix_bloom_huge_page_tlb_size,
+        &arena_,
+        moptions.memtable_prefix_bloom_bits, ioptions.bloom_locality,
+        moptions.memtable_prefix_bloom_probes, nullptr,
+        moptions.memtable_prefix_bloom_huge_page_tlb_size,
         ioptions.info_log));
   }
 }
@@ -183,7 +176,7 @@ Slice MemTableRep::UserKey(const char* key) const {
 }
 
 KeyHandle MemTableRep::Allocate(const size_t len, char** buf) {
-  *buf = allocator_->Allocate(len);
+  *buf = arena_->Allocate(len);
   return static_cast<KeyHandle>(*buf);
 }
 
@@ -192,7 +185,7 @@ KeyHandle MemTableRep::Allocate(const size_t len, char** buf) {
 // into this scratch space.
 const char* EncodeKey(std::string* scratch, const Slice& target) {
   scratch->clear();
-  PutVarint32(scratch, static_cast<uint32_t>(target.size()));
+  PutVarint32(scratch, target.size());
   scratch->append(target.data(), target.size());
   return scratch->data();
 }
@@ -292,12 +285,12 @@ void MemTable::Add(SequenceNumber s, ValueType type,
   //  key bytes    : char[internal_key.size()]
   //  value_size   : varint32 of value.size()
   //  value bytes  : char[value.size()]
-  uint32_t key_size = static_cast<uint32_t>(key.size());
-  uint32_t val_size = static_cast<uint32_t>(value.size());
-  uint32_t internal_key_size = key_size + 8;
-  const uint32_t encoded_len = VarintLength(internal_key_size) +
-                               internal_key_size + VarintLength(val_size) +
-                               val_size;
+  size_t key_size = key.size();
+  size_t val_size = value.size();
+  size_t internal_key_size = key_size + 8;
+  const size_t encoded_len =
+      VarintLength(internal_key_size) + internal_key_size +
+      VarintLength(val_size) + val_size;
   char* buf = nullptr;
   KeyHandle handle = table_->Allocate(encoded_len, &buf);
   assert(buf != nullptr);
@@ -461,10 +454,10 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
     saver.status = s;
     saver.mem = this;
     saver.merge_context = merge_context;
-    saver.merge_operator = moptions_.merge_operator;
-    saver.logger = moptions_.info_log;
+    saver.merge_operator = ioptions_.merge_operator;
+    saver.logger = ioptions_.info_log;
     saver.inplace_update_support = moptions_.inplace_update_support;
-    saver.statistics = moptions_.statistics;
+    saver.statistics = ioptions_.statistics;
     table_->Get(key, &saver, SaveValue);
   }
 
@@ -506,8 +499,8 @@ void MemTable::Update(SequenceNumber seq,
       switch (static_cast<ValueType>(tag & 0xff)) {
         case kTypeValue: {
           Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length);
-          uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
-          uint32_t new_size = static_cast<uint32_t>(value.size());
+          uint32_t prev_size = prev_value.size();
+          uint32_t new_size = value.size();
 
           // Update value, if new value size  <= previous value size
           if (new_size <= prev_size ) {
@@ -564,10 +557,10 @@ bool MemTable::UpdateCallback(SequenceNumber seq,
       switch (static_cast<ValueType>(tag & 0xff)) {
         case kTypeValue: {
           Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length);
-          uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
+          uint32_t  prev_size = prev_value.size();
 
           char* prev_buffer = const_cast<char*>(prev_value.data());
-          uint32_t new_prev_size = prev_size;
+          uint32_t  new_prev_size = prev_size;
 
           std::string str_value;
           WriteLock wl(GetLock(lkey.user_key()));
@@ -585,12 +578,12 @@ bool MemTable::UpdateCallback(SequenceNumber seq,
                 memcpy(p, prev_buffer, new_prev_size);
               }
             }
-            RecordTick(moptions_.statistics, NUMBER_KEYS_UPDATED);
+            RecordTick(ioptions_.statistics, NUMBER_KEYS_UPDATED);
             should_flush_ = ShouldFlushNow();
             return true;
           } else if (status == UpdateStatus::UPDATED) {
             Add(seq, kTypeValue, key, Slice(str_value));
-            RecordTick(moptions_.statistics, NUMBER_KEYS_WRITTEN);
+            RecordTick(ioptions_.statistics, NUMBER_KEYS_WRITTEN);
             should_flush_ = ShouldFlushNow();
             return true;
           } else if (status == UpdateStatus::UPDATE_FAILED) {

db/memtable.h

@@ -19,22 +19,21 @@
 #include "rocksdb/db.h"
 #include "rocksdb/memtablerep.h"
 #include "rocksdb/immutable_options.h"
-#include "db/memtable_allocator.h"
 #include "util/arena.h"
 #include "util/dynamic_bloom.h"
 #include "util/mutable_cf_options.h"
 
 namespace rocksdb {
 
+class Arena;
 class Mutex;
 class MemTableIterator;
 class MergeContext;
-class WriteBuffer;
 
 struct MemTableOptions {
   explicit MemTableOptions(
-      const ImmutableCFOptions& ioptions,
-      const MutableCFOptions& mutable_cf_options);
+      const MutableCFOptions& mutable_cf_options,
+      const Options& options);
   size_t write_buffer_size;
   size_t arena_block_size;
   uint32_t memtable_prefix_bloom_bits;
@@ -48,9 +47,6 @@ struct MemTableOptions {
                                    std::string* merged_value);
   size_t max_successive_merges;
   bool filter_deletes;
-  Statistics* statistics;
-  MergeOperator* merge_operator;
-  Logger* info_log;
 };
 
 class MemTable {
@@ -68,8 +64,7 @@ class MemTable {
   // is zero and the caller must call Ref() at least once.
   explicit MemTable(const InternalKeyComparator& comparator,
                     const ImmutableCFOptions& ioptions,
-                    const MutableCFOptions& mutable_cf_options,
-                    WriteBuffer* write_buffer);
+                    const MemTableOptions& moptions);
 
   ~MemTable();
 
@@ -185,10 +180,7 @@ class MemTable {
   void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
 
   // Notify the underlying storage that no more items will be added
-  void MarkImmutable() {
-    table_->MarkReadOnly();
-    allocator_.DoneAllocating();
-  }
+  void MarkImmutable() { table_->MarkReadOnly(); }
 
   // return true if the current MemTableRep supports merge operator.
   bool IsMergeOperatorSupported() const {
@@ -205,6 +197,9 @@ class MemTable {
     return comparator_.comparator;
   }
 
+  const Arena& TEST_GetArena() const { return arena_; }
+
+  const ImmutableCFOptions* GetImmutableOptions() const { return &ioptions_; }
   const MemTableOptions* GetMemTableOptions() const { return &moptions_; }
 
  private:
@@ -216,11 +211,11 @@ class MemTable {
   friend class MemTableList;
 
   KeyComparator comparator_;
+  const ImmutableCFOptions& ioptions_;
   const MemTableOptions moptions_;
   int refs_;
   const size_t kArenaBlockSize;
   Arena arena_;
-  MemTableAllocator allocator_;
   unique_ptr<MemTableRep> table_;
 
   uint64_t num_entries_;

db/memtable_allocator.cc

@@ -1,52 +0,0 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <assert.h>
-
-#include "db/memtable_allocator.h"
-#include "db/writebuffer.h"
-#include "util/arena.h"
-
-namespace rocksdb {
-
-MemTableAllocator::MemTableAllocator(Arena* arena, WriteBuffer* write_buffer)
-    : arena_(arena), write_buffer_(write_buffer), bytes_allocated_(0) {
-}
-
-MemTableAllocator::~MemTableAllocator() {
-  DoneAllocating();
-}
-
-char* MemTableAllocator::Allocate(size_t bytes) {
-  assert(write_buffer_ != nullptr);
-  bytes_allocated_ += bytes;
-  write_buffer_->ReserveMem(bytes);
-  return arena_->Allocate(bytes);
-}
-
-char* MemTableAllocator::AllocateAligned(size_t bytes, size_t huge_page_size,
-                                         Logger* logger) {
-  assert(write_buffer_ != nullptr);
-  bytes_allocated_ += bytes;
-  write_buffer_->ReserveMem(bytes);
-  return arena_->AllocateAligned(bytes, huge_page_size, logger);
-}
-
-void MemTableAllocator::DoneAllocating() {
-  if (write_buffer_ != nullptr) {
-    write_buffer_->FreeMem(bytes_allocated_);
-    write_buffer_ = nullptr;
-  }
-}
-
-size_t MemTableAllocator::BlockSize() const {
-  return arena_->BlockSize();
-}
-
-}  // namespace rocksdb

db/memtable_allocator.h

@@ -1,47 +0,0 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// This is used by the MemTable to allocate write buffer memory. It connects
-// to WriteBuffer so we can track and enforce overall write buffer limits.
-
-#pragma once
-#include "util/allocator.h"
-
-namespace rocksdb {
-
-class Arena;
-class Logger;
-class WriteBuffer;
-
-class MemTableAllocator : public Allocator {
- public:
-  explicit MemTableAllocator(Arena* arena, WriteBuffer* write_buffer);
-  ~MemTableAllocator();
-
-  // Allocator interface
-  char* Allocate(size_t bytes) override;
-  char* AllocateAligned(size_t bytes, size_t huge_page_size = 0,
-                        Logger* logger = nullptr) override;
-  size_t BlockSize() const override;
-
-  // Call when we're finished allocating memory so we can free it from
-  // the write buffer's limit.
-  void DoneAllocating();
-
- private:
-  Arena* arena_;
-  WriteBuffer* write_buffer_;
-  size_t bytes_allocated_;
-
-  // No copying allowed
-  MemTableAllocator(const MemTableAllocator&);
-  void operator=(const MemTableAllocator&);
-};
-
-}  // namespace rocksdb

db/memtable_list.cc

@@ -5,11 +5,6 @@
 //
 #include "db/memtable_list.h"
 
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
 #include <string>
 #include "rocksdb/db.h"
 #include "db/memtable.h"
@@ -119,7 +114,7 @@ void MemTableListVersion::Remove(MemTable* m) {
 bool MemTableList::IsFlushPending() const {
   if ((flush_requested_ && num_flush_not_started_ >= 1) ||
       (num_flush_not_started_ >= min_write_buffer_number_to_merge_)) {
-    assert(imm_flush_needed.load(std::memory_order_relaxed));
+    assert(imm_flush_needed.NoBarrier_Load() != nullptr);
     return true;
   }
   return false;
@@ -134,7 +129,7 @@ void MemTableList::PickMemtablesToFlush(autovector<MemTable*>* ret) {
       assert(!m->flush_completed_);
       num_flush_not_started_--;
       if (num_flush_not_started_ == 0) {
-        imm_flush_needed.store(false, std::memory_order_release);
+        imm_flush_needed.Release_Store(nullptr);
       }
       m->flush_in_progress_ = true;  // flushing will start very soon
       ret->push_back(m);
@@ -144,7 +139,8 @@ void MemTableList::PickMemtablesToFlush(autovector<MemTable*>* ret) {
 }
 
 void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
-                                         uint64_t file_number) {
+                                         uint64_t file_number,
+                                         FileNumToPathIdMap* pending_outputs) {
   assert(!mems.empty());
 
   // If the flush was not successful, then just reset state.
@@ -158,14 +154,16 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
     m->edit_.Clear();
     num_flush_not_started_++;
   }
-  imm_flush_needed.store(true, std::memory_order_release);
+  pending_outputs->erase(file_number);
+  imm_flush_needed.Release_Store(reinterpret_cast<void *>(1));
 }
 
 // Record a successful flush in the manifest file
 Status MemTableList::InstallMemtableFlushResults(
     ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
-    const autovector<MemTable*>& mems, VersionSet* vset, port::Mutex* mu,
-    uint64_t file_number, autovector<MemTable*>* to_delete,
+    const autovector<MemTable*>& mems, VersionSet* vset,
+    port::Mutex* mu, Logger* info_log, uint64_t file_number,
+    FileNumToPathIdMap* pending_outputs, autovector<MemTable*>* to_delete,
     Directory* db_directory, LogBuffer* log_buffer) {
   mu->AssertHeld();
 
@@ -196,8 +194,8 @@ Status MemTableList::InstallMemtableFlushResults(
       break;
     }
 
-    LogToBuffer(log_buffer, "[%s] Level-0 commit table #%" PRIu64 " started",
-                cfd->GetName().c_str(), m->file_number_);
+    LogToBuffer(log_buffer, "[%s] Level-0 commit table #%lu started",
+                cfd->GetName().c_str(), (unsigned long)m->file_number_);
 
     // this can release and reacquire the mutex.
     s = vset->LogAndApply(cfd, mutable_cf_options, &m->edit_, mu, db_directory);
@@ -211,26 +209,34 @@ Status MemTableList::InstallMemtableFlushResults(
     uint64_t mem_id = 1;  // how many memtables has been flushed.
     do {
       if (s.ok()) { // commit new state
-        LogToBuffer(log_buffer, "[%s] Level-0 commit table #%" PRIu64
-                                ": memtable #%" PRIu64 " done",
-                    cfd->GetName().c_str(), m->file_number_, mem_id);
+        LogToBuffer(log_buffer,
+                    "[%s] Level-0 commit table #%lu: memtable #%lu done",
+                    cfd->GetName().c_str(), (unsigned long)m->file_number_,
+                    (unsigned long)mem_id);
         current_->Remove(m);
         assert(m->file_number_ > 0);
 
+        // pending_outputs can be cleared only after the newly created file
+        // has been written to a committed version so that other concurrently
+        // executing compaction threads do not mistakenly assume that this
+        // file is not live.
+        pending_outputs->erase(m->file_number_);
         if (m->Unref() != nullptr) {
           to_delete->push_back(m);
         }
       } else {
         //commit failed. setup state so that we can flush again.
-        LogToBuffer(log_buffer, "Level-0 commit table #%" PRIu64
-                                ": memtable #%" PRIu64 " failed",
-                    m->file_number_, mem_id);
+        Log(info_log,
+            "Level-0 commit table #%lu: memtable #%lu failed",
+            (unsigned long)m->file_number_,
+            (unsigned long)mem_id);
         m->flush_completed_ = false;
         m->flush_in_progress_ = false;
         m->edit_.Clear();
         num_flush_not_started_++;
+        pending_outputs->erase(m->file_number_);
         m->file_number_ = 0;
-        imm_flush_needed.store(true, std::memory_order_release);
+        imm_flush_needed.Release_Store((void *)1);
       }
       ++mem_id;
     } while (!current_->memlist_.empty() && (m = current_->memlist_.back()) &&
@@ -253,17 +259,17 @@ void MemTableList::Add(MemTable* m) {
   m->MarkImmutable();
   num_flush_not_started_++;
   if (num_flush_not_started_ == 1) {
-    imm_flush_needed.store(true, std::memory_order_release);
+    imm_flush_needed.Release_Store((void *)1);
   }
 }
 
 // Returns an estimate of the number of bytes of data in use.
 size_t MemTableList::ApproximateMemoryUsage() {
-  size_t total_size = 0;
+  size_t size = 0;
   for (auto& memtable : current_->memlist_) {
-    total_size += memtable->ApproximateMemoryUsage();
+    size += memtable->ApproximateMemoryUsage();
   }
-  return total_size;
+  return size;
 }
 
 void MemTableList::InstallNewVersion() {

db/memtable_list.h

@@ -78,12 +78,12 @@ class MemTableList {
  public:
   // A list of memtables.
   explicit MemTableList(int min_write_buffer_number_to_merge)
-      : imm_flush_needed(false),
-        min_write_buffer_number_to_merge_(min_write_buffer_number_to_merge),
+      : min_write_buffer_number_to_merge_(min_write_buffer_number_to_merge),
         current_(new MemTableListVersion()),
         num_flush_not_started_(0),
         commit_in_progress_(false),
         flush_requested_(false) {
+    imm_flush_needed.Release_Store(nullptr);
     current_->Ref();
   }
   ~MemTableList() {}
@@ -92,7 +92,7 @@ class MemTableList {
 
   // so that background threads can detect non-nullptr pointer to
   // determine whether there is anything more to start flushing.
-  std::atomic<bool> imm_flush_needed;
+  port::AtomicPointer imm_flush_needed;
 
   // Returns the total number of memtables in the list
   int size() const;
@@ -108,13 +108,15 @@ class MemTableList {
   // Reset status of the given memtable list back to pending state so that
   // they can get picked up again on the next round of flush.
   void RollbackMemtableFlush(const autovector<MemTable*>& mems,
-                             uint64_t file_number);
+                             uint64_t file_number,
+                             FileNumToPathIdMap* pending_outputs);
 
   // Commit a successful flush in the manifest file
   Status InstallMemtableFlushResults(
       ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
-      const autovector<MemTable*>& m, VersionSet* vset, port::Mutex* mu,
-      uint64_t file_number, autovector<MemTable*>* to_delete,
+      const autovector<MemTable*>& m, VersionSet* vset,
+      port::Mutex* mu, Logger* info_log, uint64_t file_number,
+      FileNumToPathIdMap* pending_outputs, autovector<MemTable*>* to_delete,
       Directory* db_directory, LogBuffer* log_buffer);
 
   // New memtables are inserted at the front of the list.

db/merge_helper.cc

@@ -85,10 +85,9 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
       // We store the result in keys_.back() and operands_.back()
       // if nothing went wrong (i.e.: no operand corruption on disk)
       if (success_) {
-        std::string& original_key =
-            keys_.back();  // The original key encountered
+        std::string& key = keys_.back();  // The original key encountered
         orig_ikey.type = kTypeValue;
-        UpdateInternalKey(&original_key[0], original_key.size(),
+        UpdateInternalKey(&key[0], key.size(),
                           orig_ikey.sequence, orig_ikey.type);
         swap(operands_.back(), merge_result);
       } else {
@@ -109,17 +108,17 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
       //   => store result in operands_.back() (and update keys_.back())
       //   => change the entry type to kTypeValue for keys_.back()
       // We are done! Success!
-      const Slice val = iter->value();
-      success_ = user_merge_operator_->FullMerge(ikey.user_key, &val, operands_,
-                                                 &merge_result, logger_);
+      const Slice value = iter->value();
+      success_ = user_merge_operator_->FullMerge(ikey.user_key, &value,
+                                                 operands_, &merge_result,
+                                                 logger_);
 
       // We store the result in keys_.back() and operands_.back()
       // if nothing went wrong (i.e.: no operand corruption on disk)
       if (success_) {
-        std::string& original_key =
-            keys_.back();  // The original key encountered
+        std::string& key = keys_.back();  // The original key encountered
         orig_ikey.type = kTypeValue;
-        UpdateInternalKey(&original_key[0], original_key.size(),
+        UpdateInternalKey(&key[0], key.size(),
                           orig_ikey.sequence, orig_ikey.type);
         swap(operands_.back(), merge_result);
       } else {
@@ -178,9 +177,9 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
                                                logger_);
 
     if (success_) {
-      std::string& original_key = keys_.back();  // The original key encountered
+      std::string& key = keys_.back();  // The original key encountered
       orig_ikey.type = kTypeValue;
-      UpdateInternalKey(&original_key[0], original_key.size(),
+      UpdateInternalKey(&key[0], key.size(),
                         orig_ikey.sequence, orig_ikey.type);
 
       // The final value() is always stored in operands_.back()

db/merge_test.cc

@@ -23,11 +23,15 @@ using namespace std;
 using namespace rocksdb;
 
 namespace {
-size_t num_merge_operator_calls;
-void resetNumMergeOperatorCalls() { num_merge_operator_calls = 0; }
+  int numMergeOperatorCalls;
+  void resetNumMergeOperatorCalls() {
+    numMergeOperatorCalls = 0;
+  }
 
-size_t num_partial_merge_calls;
-void resetNumPartialMergeCalls() { num_partial_merge_calls = 0; }
+  int num_partial_merge_calls;
+  void resetNumPartialMergeCalls() {
+    num_partial_merge_calls = 0;
+  }
 }
 
 class CountMergeOperator : public AssociativeMergeOperator {
@@ -41,7 +45,7 @@ class CountMergeOperator : public AssociativeMergeOperator {
                      const Slice& value,
                      std::string* new_value,
                      Logger* logger) const override {
-    ++num_merge_operator_calls;
+    ++numMergeOperatorCalls;
     if (existing_value == nullptr) {
       new_value->assign(value.data(), value.size());
       return true;
@@ -303,31 +307,31 @@ void testCounters(Counters& counters, DB* db, bool test_compaction) {
   }
 }
 
-void testSuccessiveMerge(Counters& counters, size_t max_num_merges,
-                         size_t num_merges) {
+void testSuccessiveMerge(
+    Counters& counters, int max_num_merges, int num_merges) {
 
   counters.assert_remove("z");
   uint64_t sum = 0;
 
-  for (size_t i = 1; i <= num_merges; ++i) {
+  for (int i = 1; i <= num_merges; ++i) {
     resetNumMergeOperatorCalls();
     counters.assert_add("z", i);
     sum += i;
 
     if (i % (max_num_merges + 1) == 0) {
-      assert(num_merge_operator_calls == max_num_merges + 1);
+      assert(numMergeOperatorCalls == max_num_merges + 1);
     } else {
-      assert(num_merge_operator_calls == 0);
+      assert(numMergeOperatorCalls == 0);
     }
 
     resetNumMergeOperatorCalls();
     assert(counters.assert_get("z") == sum);
-    assert(num_merge_operator_calls == i % (max_num_merges + 1));
+    assert(numMergeOperatorCalls == i % (max_num_merges + 1));
   }
 }
 
-void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
-                      size_t min_merge, size_t count) {
+void testPartialMerge(Counters* counters, DB* db, int max_merge, int min_merge,
+                      int count) {
   FlushOptions o;
   o.wait = true;
 
@@ -335,7 +339,7 @@ void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
   //              operands exceeds the threshold.
   uint64_t tmp_sum = 0;
   resetNumPartialMergeCalls();
-  for (size_t i = 1; i <= count; i++) {
+  for (int i = 1; i <= count; i++) {
     counters->assert_add("b", i);
     tmp_sum += i;
   }
@@ -344,7 +348,7 @@ void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
   ASSERT_EQ(tmp_sum, counters->assert_get("b"));
   if (count > max_merge) {
     // in this case, FullMerge should be called instead.
-    ASSERT_EQ(num_partial_merge_calls, 0U);
+    ASSERT_EQ(num_partial_merge_calls, 0);
   } else {
     // if count >= min_merge, then partial merge should be called once.
     ASSERT_EQ((count >= min_merge), (num_partial_merge_calls == 1));
@@ -354,18 +358,20 @@ void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
   resetNumPartialMergeCalls();
   tmp_sum = 0;
   db->Put(rocksdb::WriteOptions(), "c", "10");
-  for (size_t i = 1; i <= count; i++) {
+  for (int i = 1; i <= count; i++) {
     counters->assert_add("c", i);
     tmp_sum += i;
   }
   db->Flush(o);
   db->CompactRange(nullptr, nullptr);
   ASSERT_EQ(tmp_sum, counters->assert_get("c"));
-  ASSERT_EQ(num_partial_merge_calls, 0U);
+  ASSERT_EQ(num_partial_merge_calls, 0);
 }
 
-void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
-                                    size_t num_merges) {
+void testSingleBatchSuccessiveMerge(
+    DB* db,
+    int max_num_merges,
+    int num_merges) {
   assert(num_merges > max_num_merges);
 
   Slice key("BatchSuccessiveMerge");
@@ -374,7 +380,7 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
 
   // Create the batch
   WriteBatch batch;
-  for (size_t i = 0; i < num_merges; ++i) {
+  for (int i = 0; i < num_merges; ++i) {
     batch.Merge(key, merge_value_slice);
   }
 
@@ -384,9 +390,8 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
     Status s = db->Write(WriteOptions(), &batch);
     assert(s.ok());
   }
-  ASSERT_EQ(
-      num_merge_operator_calls,
-      static_cast<size_t>(num_merges - (num_merges % (max_num_merges + 1))));
+  assert(numMergeOperatorCalls ==
+      num_merges - (num_merges % (max_num_merges + 1)));
 
   // Get the value
   resetNumMergeOperatorCalls();
@@ -398,11 +403,18 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
   assert(get_value_str.size() == sizeof(uint64_t));
   uint64_t get_value = DecodeFixed64(&get_value_str[0]);
   ASSERT_EQ(get_value, num_merges * merge_value);
-  ASSERT_EQ(num_merge_operator_calls,
-            static_cast<size_t>((num_merges % (max_num_merges + 1))));
+  ASSERT_EQ(numMergeOperatorCalls, (num_merges % (max_num_merges + 1)));
 }
 
 void runTest(int argc, const string& dbname, const bool use_ttl = false) {
+  auto db = OpenDb(dbname, use_ttl);
+
+  {
+    cout << "Test read-modify-write counters... \n";
+    Counters counters(db, 0);
+    testCounters(counters, db.get(), true);
+  }
+
   bool compact = false;
   if (argc > 1) {
     compact = true;
@@ -410,22 +422,13 @@ void runTest(int argc, const string& dbname, const bool use_ttl = false) {
   }
 
   {
-    auto db = OpenDb(dbname, use_ttl);
-
-    {
-      cout << "Test read-modify-write counters... \n";
-      Counters counters(db, 0);
-      testCounters(counters, db.get(), true);
-    }
-
-    {
-      cout << "Test merge-based counters... \n";
-      MergeBasedCounters counters(db, 0);
-      testCounters(counters, db.get(), compact);
-    }
+    cout << "Test merge-based counters... \n";
+    MergeBasedCounters counters(db, 0);
+    testCounters(counters, db.get(), compact);
   }
 
   DestroyDB(dbname, Options());
+  db.reset();
 
   {
     cout << "Test merge in memtable... \n";

db/perf_context_test.cc

@@ -14,7 +14,6 @@
 #include "util/histogram.h"
 #include "util/stop_watch.h"
 #include "util/testharness.h"
-#include "util/string_util.h"
 
 
 bool FLAGS_random_key = false;
@@ -39,13 +38,8 @@ std::shared_ptr<DB> OpenDb(bool read_only = false) {
       FLAGS_min_write_buffer_number_to_merge;
 
     if (FLAGS_use_set_based_memetable) {
-#ifndef ROCKSDB_LITE
       options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(0));
       options.memtable_factory.reset(NewHashSkipListRepFactory());
-#else
-      fprintf(stderr, "Prefix hash is not supported in lite mode\n");
-      exit(1);
-#endif  // ROCKSDB_LITE
     }
 
     Status s;
@@ -67,21 +61,21 @@ TEST(PerfContextTest, SeekIntoDeletion) {
   ReadOptions read_options;
 
   for (int i = 0; i < FLAGS_total_keys; ++i) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
     db->Put(write_options, key, value);
   }
 
   for (int i = 0; i < FLAGS_total_keys -1 ; ++i) {
-    std::string key = "k" + ToString(i);
+    std::string key = "k" + std::to_string(i);
     db->Delete(write_options, key);
   }
 
   HistogramImpl hist_get;
   HistogramImpl hist_get_time;
   for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
-    std::string key = "k" + ToString(i);
+    std::string key = "k" + std::to_string(i);
     std::string value;
 
     perf_context.Reset();
@@ -94,32 +88,27 @@ TEST(PerfContextTest, SeekIntoDeletion) {
     hist_get_time.Add(elapsed_nanos);
   }
 
-  std::cout << "Get user key comparison: \n" << hist_get.ToString()
+  std::cout << "Get uesr key comparison: \n" << hist_get.ToString()
             << "Get time: \n" << hist_get_time.ToString();
 
-  {
-    HistogramImpl hist_seek_to_first;
-    std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
+  HistogramImpl hist_seek_to_first;
+  std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
 
-    perf_context.Reset();
-    StopWatchNano timer(Env::Default(), true);
-    iter->SeekToFirst();
-    hist_seek_to_first.Add(perf_context.user_key_comparison_count);
-    auto elapsed_nanos = timer.ElapsedNanos();
+  perf_context.Reset();
+  StopWatchNano timer(Env::Default(), true);
+  iter->SeekToFirst();
+  hist_seek_to_first.Add(perf_context.user_key_comparison_count);
+  auto elapsed_nanos = timer.ElapsedNanos();
 
-    std::cout << "SeekToFirst uesr key comparison: \n"
-              << hist_seek_to_first.ToString()
-              << "ikey skipped: " << perf_context.internal_key_skipped_count
-              << "\n"
-              << "idelete skipped: "
-              << perf_context.internal_delete_skipped_count << "\n"
-              << "elapsed: " << elapsed_nanos << "\n";
-  }
+  std::cout << "SeekToFirst uesr key comparison: \n" << hist_seek_to_first.ToString()
+            << "ikey skipped: " << perf_context.internal_key_skipped_count << "\n"
+            << "idelete skipped: " << perf_context.internal_delete_skipped_count << "\n"
+            << "elapsed: " << elapsed_nanos << "\n";
 
   HistogramImpl hist_seek;
   for (int i = 0; i < FLAGS_total_keys; ++i) {
     std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
-    std::string key = "k" + ToString(i);
+    std::string key = "k" + std::to_string(i);
 
     perf_context.Reset();
     StopWatchNano timer(Env::Default(), true);
@@ -232,9 +221,10 @@ void ProfileQueries(bool enabled_time = false) {
       db->Flush(fo);
       continue;
     }
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
+    std::vector<Slice> keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -246,10 +236,10 @@ void ProfileQueries(bool enabled_time = false) {
   }
 
   for (const int i : keys) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
-    std::vector<Slice> multiget_keys = {Slice(key)};
+    std::vector<Slice> keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -262,7 +252,7 @@ void ProfileQueries(bool enabled_time = false) {
     hist_get.Add(perf_context.user_key_comparison_count);
 
     perf_context.Reset();
-    db->MultiGet(read_options, multiget_keys, &values);
+    db->MultiGet(read_options, keys, &values);
     hist_mget_snapshot.Add(perf_context.get_snapshot_time);
     hist_mget_memtable.Add(perf_context.get_from_memtable_time);
     hist_mget_files.Add(perf_context.get_from_output_files_time);
@@ -336,10 +326,10 @@ void ProfileQueries(bool enabled_time = false) {
   hist_mget_num_memtable_checked.Clear();
 
   for (const int i : keys) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
-    std::vector<Slice> multiget_keys = {Slice(key)};
+    std::vector<Slice> keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -352,7 +342,7 @@ void ProfileQueries(bool enabled_time = false) {
     hist_get.Add(perf_context.user_key_comparison_count);
 
     perf_context.Reset();
-    db->MultiGet(read_options, multiget_keys, &values);
+    db->MultiGet(read_options, keys, &values);
     hist_mget_snapshot.Add(perf_context.get_snapshot_time);
     hist_mget_memtable.Add(perf_context.get_from_memtable_time);
     hist_mget_files.Add(perf_context.get_from_output_files_time);
@@ -452,8 +442,8 @@ TEST(PerfContextTest, SeekKeyComparison) {
   SetPerfLevel(kEnableTime);
   StopWatchNano timer(Env::Default());
   for (const int i : keys) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
     perf_context.Reset();
     timer.Start();
@@ -472,8 +462,8 @@ TEST(PerfContextTest, SeekKeyComparison) {
   HistogramImpl hist_next;
 
   for (int i = 0; i < FLAGS_total_keys; ++i) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
     std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
     perf_context.Reset();

db/plain_table_db_test.cc

@@ -158,7 +158,7 @@ class PlainTableDBTest {
   // Return spread of files per level
   std::string FilesPerLevel() {
     std::string result;
-    size_t last_non_zero_offset = 0;
+    int last_non_zero_offset = 0;
     for (int level = 0; level < db_->NumberLevels(); level++) {
       int f = NumTableFilesAtLevel(level);
       char buf[100];
@@ -628,7 +628,7 @@ TEST(PlainTableDBTest, IteratorLargeKeys) {
   };
 
   for (size_t i = 0; i < 7; i++) {
-    ASSERT_OK(Put(key_list[i], ToString(i)));
+    ASSERT_OK(Put(key_list[i], std::to_string(i)));
   }
 
   dbfull()->TEST_FlushMemTable();
@@ -639,7 +639,7 @@ TEST(PlainTableDBTest, IteratorLargeKeys) {
   for (size_t i = 0; i < 7; i++) {
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(key_list[i], iter->key().ToString());
-    ASSERT_EQ(ToString(i), iter->value().ToString());
+    ASSERT_EQ(std::to_string(i), iter->value().ToString());
     iter->Next();
   }
 
@@ -676,7 +676,7 @@ TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
       MakeLongKeyWithPrefix(26, '6')};
 
   for (size_t i = 0; i < 7; i++) {
-    ASSERT_OK(Put(key_list[i], ToString(i)));
+    ASSERT_OK(Put(key_list[i], std::to_string(i)));
   }
 
   dbfull()->TEST_FlushMemTable();
@@ -687,7 +687,7 @@ TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
   for (size_t i = 0; i < 7; i++) {
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(key_list[i], iter->key().ToString());
-    ASSERT_EQ(ToString(i), iter->value().ToString());
+    ASSERT_EQ(std::to_string(i), iter->value().ToString());
     iter->Next();
   }
 
@@ -696,12 +696,40 @@ TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
   delete iter;
 }
 
+// A test comparator which compare two strings in this way:
+// (1) first compare prefix of 8 bytes in alphabet order,
+// (2) if two strings share the same prefix, sort the other part of the string
+//     in the reverse alphabet order.
+class SimpleSuffixReverseComparator : public Comparator {
+ public:
+  SimpleSuffixReverseComparator() {}
+
+  virtual const char* Name() const { return "SimpleSuffixReverseComparator"; }
+
+  virtual int Compare(const Slice& a, const Slice& b) const {
+    Slice prefix_a = Slice(a.data(), 8);
+    Slice prefix_b = Slice(b.data(), 8);
+    int prefix_comp = prefix_a.compare(prefix_b);
+    if (prefix_comp != 0) {
+      return prefix_comp;
+    } else {
+      Slice suffix_a = Slice(a.data() + 8, a.size() - 8);
+      Slice suffix_b = Slice(b.data() + 8, b.size() - 8);
+      return -(suffix_a.compare(suffix_b));
+    }
+  }
+  virtual void FindShortestSeparator(std::string* start,
+                                     const Slice& limit) const {}
+
+  virtual void FindShortSuccessor(std::string* key) const {}
+};
+
 TEST(PlainTableDBTest, IteratorReverseSuffixComparator) {
   Options options = CurrentOptions();
   options.create_if_missing = true;
   // Set only one bucket to force bucket conflict.
   // Test index interval for the same prefix to be 1, 2 and 4
-  test::SimpleSuffixReverseComparator comp;
+  SimpleSuffixReverseComparator comp;
   options.comparator = &comp;
   DestroyAndReopen(&options);
 
@@ -864,7 +892,7 @@ TEST(PlainTableDBTest, HashBucketConflictReverseSuffixComparator) {
     for (unsigned char i = 1; i <= 3; i++) {
       Options options = CurrentOptions();
       options.create_if_missing = true;
-      test::SimpleSuffixReverseComparator comp;
+      SimpleSuffixReverseComparator comp;
       options.comparator = &comp;
       // Set only one bucket to force bucket conflict.
       // Test index interval for the same prefix to be 1, 2 and 4

db/prefix_test.cc

@@ -29,14 +29,14 @@ using GFLAGS::ParseCommandLineFlags;
 
 DEFINE_bool(trigger_deadlock, false,
             "issue delete in range scan to trigger PrefixHashMap deadlock");
-DEFINE_int32(bucket_count, 100000, "number of buckets");
+DEFINE_uint64(bucket_count, 100000, "number of buckets");
 DEFINE_uint64(num_locks, 10001, "number of locks");
 DEFINE_bool(random_prefix, false, "randomize prefix");
 DEFINE_uint64(total_prefixes, 100000, "total number of prefixes");
 DEFINE_uint64(items_per_prefix, 1, "total number of values per prefix");
 DEFINE_int64(write_buffer_size, 33554432, "");
-DEFINE_int32(max_write_buffer_number, 2, "");
-DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
+DEFINE_int64(max_write_buffer_number, 2, "");
+DEFINE_int64(min_write_buffer_number_to_merge, 1, "");
 DEFINE_int32(skiplist_height, 4, "");
 DEFINE_int32(memtable_prefix_bloom_bits, 10000000, "");
 DEFINE_int32(memtable_prefix_bloom_probes, 10, "");
@@ -52,8 +52,7 @@ struct TestKey {
   uint64_t prefix;
   uint64_t sorted;
 
-  TestKey(uint64_t _prefix, uint64_t _sorted)
-      : prefix(_prefix), sorted(_sorted) {}
+  TestKey(uint64_t prefix, uint64_t sorted) : prefix(prefix), sorted(sorted) {}
 };
 
 // return a slice backed by test_key
@@ -442,7 +441,7 @@ TEST(PrefixTest, DynamicPrefixIterator) {
     for (auto prefix : prefixes) {
       TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
       Slice key = TestKeyToSlice(test_key);
-      std::string value = "v" + ToString(0);
+      std::string value = "v" + std::to_string(0);
 
       perf_context.Reset();
       StopWatchNano timer(Env::Default(), true);

db/repair.cc

@@ -45,7 +45,6 @@
 #include "db/memtable.h"
 #include "db/table_cache.h"
 #include "db/version_edit.h"
-#include "db/writebuffer.h"
 #include "db/write_batch_internal.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/db.h"
@@ -95,7 +94,7 @@ class Repairer {
       for (size_t i = 0; i < tables_.size(); i++) {
         bytes += tables_[i].meta.fd.GetFileSize();
       }
-      Log(InfoLogLevel::WARN_LEVEL, options_.info_log,
+      Log(options_.info_log,
           "**** Repaired rocksdb %s; "
           "recovered %zu files; %" PRIu64
           "bytes. "
@@ -176,7 +175,7 @@ class Repairer {
       std::string logname = LogFileName(dbname_, logs_[i]);
       Status status = ConvertLogToTable(logs_[i]);
       if (!status.ok()) {
-        Log(InfoLogLevel::WARN_LEVEL, options_.info_log,
+        Log(options_.info_log,
             "Log #%" PRIu64 ": ignoring conversion error: %s", logs_[i],
             status.ToString().c_str());
       }
@@ -191,8 +190,7 @@ class Repairer {
       uint64_t lognum;
       virtual void Corruption(size_t bytes, const Status& s) {
         // We print error messages for corruption, but continue repairing.
-        Log(InfoLogLevel::ERROR_LEVEL, info_log,
-            "Log #%" PRIu64 ": dropping %d bytes; %s", lognum,
+        Log(info_log, "Log #%" PRIu64 ": dropping %d bytes; %s", lognum,
             static_cast<int>(bytes), s.ToString().c_str());
       }
     };
@@ -221,10 +219,9 @@ class Repairer {
     std::string scratch;
     Slice record;
     WriteBatch batch;
-    WriteBuffer wb(options_.db_write_buffer_size);
     MemTable* mem = new MemTable(icmp_, ioptions_,
-                                 MutableCFOptions(options_, ioptions_), &wb);
-    auto cf_mems_default = new ColumnFamilyMemTablesDefault(mem);
+        MemTableOptions(MutableCFOptions(options_, ioptions_), options_));
+    auto cf_mems_default = new ColumnFamilyMemTablesDefault(mem, &options_);
     mem->Ref();
     int counter = 0;
     while (reader.ReadRecord(&record, &scratch)) {
@@ -238,8 +235,7 @@ class Repairer {
       if (status.ok()) {
         counter += WriteBatchInternal::Count(&batch);
       } else {
-        Log(InfoLogLevel::WARN_LEVEL,
-            options_.info_log, "Log #%" PRIu64 ": ignoring %s", log,
+        Log(options_.info_log, "Log #%" PRIu64 ": ignoring %s", log,
             status.ToString().c_str());
         status = Status::OK();  // Keep going with rest of file
       }
@@ -266,9 +262,9 @@ class Repairer {
         table_fds_.push_back(meta.fd);
       }
     }
-    Log(InfoLogLevel::INFO_LEVEL, options_.info_log,
-        "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s",
-        log, counter, meta.fd.GetNumber(), status.ToString().c_str());
+    Log(options_.info_log,
+        "Log #%" PRIu64 ": %d ops saved to Table #%" PRIu64 " %s", log, counter,
+        meta.fd.GetNumber(), status.ToString().c_str());
     return status;
   }
 
@@ -283,8 +279,7 @@ class Repairer {
         char file_num_buf[kFormatFileNumberBufSize];
         FormatFileNumber(t.meta.fd.GetNumber(), t.meta.fd.GetPathId(),
                          file_num_buf, sizeof(file_num_buf));
-        Log(InfoLogLevel::WARN_LEVEL, options_.info_log,
-            "Table #%s: ignoring %s", file_num_buf,
+        Log(options_.info_log, "Table #%s: ignoring %s", file_num_buf,
             status.ToString().c_str());
         ArchiveFile(fname);
       } else {
@@ -311,8 +306,7 @@ class Repairer {
       for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
         Slice key = iter->key();
         if (!ParseInternalKey(key, &parsed)) {
-          Log(InfoLogLevel::ERROR_LEVEL,
-              options_.info_log, "Table #%" PRIu64 ": unparsable key %s",
+          Log(options_.info_log, "Table #%" PRIu64 ": unparsable key %s",
               t->meta.fd.GetNumber(), EscapeString(key).c_str());
           continue;
         }
@@ -335,8 +329,7 @@ class Repairer {
       }
       delete iter;
     }
-    Log(InfoLogLevel::INFO_LEVEL,
-        options_.info_log, "Table #%" PRIu64 ": %d entries %s",
+    Log(options_.info_log, "Table #%" PRIu64 ": %d entries %s",
         t->meta.fd.GetNumber(), counter, status.ToString().c_str());
     return status;
   }
@@ -413,8 +406,7 @@ class Repairer {
     new_file.append("/");
     new_file.append((slash == nullptr) ? fname.c_str() : slash + 1);
     Status s = env_->RenameFile(fname, new_file);
-    Log(InfoLogLevel::INFO_LEVEL,
-        options_.info_log, "Archiving %s: %s\n",
+    Log(options_.info_log, "Archiving %s: %s\n",
         fname.c_str(), s.ToString().c_str());
   }
 };

db/skiplist.h

@@ -32,10 +32,10 @@
 
 #pragma once
 #include <assert.h>
-#include <atomic>
 #include <stdlib.h>
+#include "util/arena.h"
 #include "port/port.h"
-#include "util/allocator.h"
+#include "util/arena.h"
 #include "util/random.h"
 
 namespace rocksdb {
@@ -47,9 +47,9 @@ class SkipList {
 
  public:
   // Create a new SkipList object that will use "cmp" for comparing keys,
-  // and will allocate memory using "*allocator".  Objects allocated in the
-  // allocator must remain allocated for the lifetime of the skiplist object.
-  explicit SkipList(Comparator cmp, Allocator* allocator,
+  // and will allocate memory using "*arena".  Objects allocated in the arena
+  // must remain allocated for the lifetime of the skiplist object.
+  explicit SkipList(Comparator cmp, Arena* arena,
                     int32_t max_height = 12, int32_t branching_factor = 4);
 
   // Insert key into the list.
@@ -109,20 +109,21 @@ class SkipList {
 
   // Immutable after construction
   Comparator const compare_;
-  Allocator* const allocator_;    // Allocator used for allocations of nodes
+  Arena* const arena_;    // Arena used for allocations of nodes
 
   Node* const head_;
 
   // Modified only by Insert().  Read racily by readers, but stale
   // values are ok.
-  std::atomic<int> max_height_;  // Height of the entire list
+  port::AtomicPointer max_height_;   // Height of the entire list
 
   // Used for optimizing sequential insert patterns
   Node** prev_;
   int32_t prev_height_;
 
   inline int GetMaxHeight() const {
-    return max_height_.load(std::memory_order_relaxed);
+    return static_cast<int>(
+        reinterpret_cast<intptr_t>(max_height_.NoBarrier_Load()));
   }
 
   // Read/written only by Insert().
@@ -168,35 +169,35 @@ struct SkipList<Key, Comparator>::Node {
     assert(n >= 0);
     // Use an 'acquire load' so that we observe a fully initialized
     // version of the returned Node.
-    return (next_[n].load(std::memory_order_acquire));
+    return reinterpret_cast<Node*>(next_[n].Acquire_Load());
   }
   void SetNext(int n, Node* x) {
     assert(n >= 0);
     // Use a 'release store' so that anybody who reads through this
     // pointer observes a fully initialized version of the inserted node.
-    next_[n].store(x, std::memory_order_release);
+    next_[n].Release_Store(x);
   }
 
   // No-barrier variants that can be safely used in a few locations.
   Node* NoBarrier_Next(int n) {
     assert(n >= 0);
-    return next_[n].load(std::memory_order_relaxed);
+    return reinterpret_cast<Node*>(next_[n].NoBarrier_Load());
   }
   void NoBarrier_SetNext(int n, Node* x) {
     assert(n >= 0);
-    next_[n].store(x, std::memory_order_relaxed);
+    next_[n].NoBarrier_Store(x);
   }
 
  private:
   // Array of length equal to the node height.  next_[0] is lowest level link.
-  std::atomic<Node*> next_[1];
+  port::AtomicPointer next_[1];
 };
 
 template<typename Key, class Comparator>
 typename SkipList<Key, Comparator>::Node*
 SkipList<Key, Comparator>::NewNode(const Key& key, int height) {
-  char* mem = allocator_->AllocateAligned(
-      sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1));
+  char* mem = arena_->AllocateAligned(
+      sizeof(Node) + sizeof(port::AtomicPointer) * (height - 1));
   return new (mem) Node(key);
 }
 
@@ -355,24 +356,23 @@ typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast()
 }
 
 template<typename Key, class Comparator>
-SkipList<Key, Comparator>::SkipList(const Comparator cmp, Allocator* allocator,
+SkipList<Key, Comparator>::SkipList(const Comparator cmp, Arena* arena,
                                    int32_t max_height,
                                    int32_t branching_factor)
     : kMaxHeight_(max_height),
       kBranching_(branching_factor),
       compare_(cmp),
-      allocator_(allocator),
+      arena_(arena),
       head_(NewNode(0 /* any key will do */, max_height)),
-      max_height_(1),
+      max_height_(reinterpret_cast<void*>(1)),
       prev_height_(1),
       rnd_(0xdeadbeef) {
   assert(kMaxHeight_ > 0);
   assert(kBranching_ > 0);
-  // Allocate the prev_ Node* array, directly from the passed-in allocator.
+  // Allocate the prev_ Node* array, directly from the passed-in arena.
   // prev_ does not need to be freed, as its life cycle is tied up with
-  // the allocator as a whole.
-  prev_ = reinterpret_cast<Node**>(
-            allocator_->AllocateAligned(sizeof(Node*) * kMaxHeight_));
+  // the arena as a whole.
+  prev_ = (Node**) arena_->AllocateAligned(sizeof(Node*) * kMaxHeight_);
   for (int i = 0; i < kMaxHeight_; i++) {
     head_->SetNext(i, nullptr);
     prev_[i] = head_;
@@ -402,7 +402,7 @@ void SkipList<Key, Comparator>::Insert(const Key& key) {
     // the loop below.  In the former case the reader will
     // immediately drop to the next level since nullptr sorts after all
     // keys.  In the latter case the reader will use the new node.
-    max_height_.store(height, std::memory_order_relaxed);
+    max_height_.NoBarrier_Store(reinterpret_cast<void*>(height));
   }
 
   x = NewNode(key, height);

db/skiplist_test.cc

@@ -191,11 +191,13 @@ class ConcurrentTest {
 
   // Per-key generation
   struct State {
-    std::atomic<int> generation[K];
-    void Set(int k, int v) {
-      generation[k].store(v, std::memory_order_release);
+    port::AtomicPointer generation[K];
+    void Set(int k, intptr_t v) {
+      generation[k].Release_Store(reinterpret_cast<void*>(v));
+    }
+    intptr_t Get(int k) {
+      return reinterpret_cast<intptr_t>(generation[k].Acquire_Load());
     }
-    int Get(int k) { return generation[k].load(std::memory_order_acquire); }
 
     State() {
       for (unsigned int k = 0; k < K; k++) {
@@ -219,9 +221,9 @@ class ConcurrentTest {
   // REQUIRES: External synchronization
   void WriteStep(Random* rnd) {
     const uint32_t k = rnd->Next() % K;
-    const int g = current_.Get(k) + 1;
-    const Key new_key = MakeKey(k, g);
-    list_.Insert(new_key);
+    const intptr_t g = current_.Get(k) + 1;
+    const Key key = MakeKey(k, g);
+    list_.Insert(key);
     current_.Set(k, g);
   }
 
@@ -253,10 +255,11 @@ class ConcurrentTest {
         // Note that generation 0 is never inserted, so it is ok if
         // <*,0,*> is missing.
         ASSERT_TRUE((gen(pos) == 0U) ||
-                    (gen(pos) > static_cast<uint64_t>(initial_state.Get(
-                                    static_cast<int>(key(pos))))))
-            << "key: " << key(pos) << "; gen: " << gen(pos)
-            << "; initgen: " << initial_state.Get(static_cast<int>(key(pos)));
+                    (gen(pos) > (uint64_t)initial_state.Get(key(pos)))
+                    ) << "key: " << key(pos)
+                      << "; gen: " << gen(pos)
+                      << "; initgen: "
+                      << initial_state.Get(key(pos));
 
         // Advance to next key in the valid key space
         if (key(pos) < key(current)) {
@@ -300,7 +303,7 @@ class TestState {
  public:
   ConcurrentTest t_;
   int seed_;
-  std::atomic<bool> quit_flag_;
+  port::AtomicPointer quit_flag_;
 
   enum ReaderState {
     STARTING,
@@ -309,7 +312,10 @@ class TestState {
   };
 
   explicit TestState(int s)
-      : seed_(s), quit_flag_(false), state_(STARTING), state_cv_(&mu_) {}
+      : seed_(s),
+        quit_flag_(nullptr),
+        state_(STARTING),
+        state_cv_(&mu_) {}
 
   void Wait(ReaderState s) {
     mu_.Lock();
@@ -337,7 +343,7 @@ static void ConcurrentReader(void* arg) {
   Random rnd(state->seed_);
   int64_t reads = 0;
   state->Change(TestState::RUNNING);
-  while (!state->quit_flag_.load(std::memory_order_acquire)) {
+  while (!state->quit_flag_.Acquire_Load()) {
     state->t_.ReadStep(&rnd);
     ++reads;
   }
@@ -356,10 +362,10 @@ static void RunConcurrent(int run) {
     TestState state(seed + 1);
     Env::Default()->Schedule(ConcurrentReader, &state);
     state.Wait(TestState::RUNNING);
-    for (int k = 0; k < kSize; k++) {
+    for (int i = 0; i < kSize; i++) {
       state.t_.WriteStep(&rnd);
     }
-    state.quit_flag_.store(true, std::memory_order_release);
+    state.quit_flag_.Release_Store(&state);  // Any non-nullptr arg will do
     state.Wait(TestState::DONE);
   }
 }

db/snapshot.h

@@ -28,8 +28,6 @@ class SnapshotImpl : public Snapshot {
   SnapshotImpl* next_;
 
   SnapshotList* list_;                 // just for sanity checks
-
-  int64_t unix_time_;
 };
 
 class SnapshotList {
@@ -38,23 +36,20 @@ class SnapshotList {
     list_.prev_ = &list_;
     list_.next_ = &list_;
     list_.number_ = 0xFFFFFFFFL;      // placeholder marker, for debugging
-    count_ = 0;
   }
 
   bool empty() const { return list_.next_ == &list_; }
   SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
   SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
 
-  const SnapshotImpl* New(SequenceNumber seq, uint64_t unix_time) {
+  const SnapshotImpl* New(SequenceNumber seq) {
     SnapshotImpl* s = new SnapshotImpl;
     s->number_ = seq;
-    s->unix_time_ = unix_time;
     s->list_ = this;
     s->next_ = &list_;
     s->prev_ = list_.prev_;
     s->prev_->next_ = s;
     s->next_->prev_ = s;
-    count_++;
     return s;
   }
 
@@ -62,7 +57,6 @@ class SnapshotList {
     assert(s->list_ == this);
     s->prev_->next_ = s->next_;
     s->next_->prev_ = s->prev_;
-    count_--;
     delete s;
   }
 
@@ -84,20 +78,9 @@ class SnapshotList {
     return newest()->number_;
   }
 
-  int64_t GetOldestSnapshotTime() const {
-    if (empty()) {
-      return 0;
-    } else {
-      return oldest()->unix_time_;
-    }
-  }
-
-  uint64_t count() const { return count_; }
-
  private:
   // Dummy head of doubly-linked list of snapshots
   SnapshotImpl list_;
-  uint64_t count_;
 };
 
 }  // namespace rocksdb

db/table_cache.cc

@@ -88,6 +88,7 @@ Status TableCache::FindTable(const EnvOptions& env_options,
       // We do not cache error results so that if the error is transient,
       // or somebody repairs the file, we recover automatically.
     } else {
+      assert(file.get() == nullptr);
       *handle = cache_->Insert(key, table_reader.release(), 1, &DeleteEntry);
     }
   }

db/table_properties_collector.cc

@@ -7,7 +7,6 @@
 
 #include "db/dbformat.h"
 #include "util/coding.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
@@ -41,7 +40,7 @@ Status InternalKeyPropertiesCollector::Finish(
 UserCollectedProperties
 InternalKeyPropertiesCollector::GetReadableProperties() const {
   return {
-    { "kDeletedKeys", ToString(deleted_keys_) }
+    { "kDeletedKeys", std::to_string(deleted_keys_) }
   };
 }
 

db/transaction_log_impl.cc

@@ -4,11 +4,6 @@
 //  of patent rights can be found in the PATENTS file in the same directory.
 
 #ifndef ROCKSDB_LITE
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
 #include "db/transaction_log_impl.h"
 #include "db/write_batch_internal.h"
 
@@ -18,7 +13,7 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
     const std::string& dir, const DBOptions* options,
     const TransactionLogIterator::ReadOptions& read_options,
     const EnvOptions& soptions, const SequenceNumber seq,
-    std::unique_ptr<VectorLogPtr> files, VersionSet const* const versions)
+    std::unique_ptr<VectorLogPtr> files, DBImpl const* const dbimpl)
     : dir_(dir),
       options_(options),
       read_options_(read_options),
@@ -30,9 +25,9 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
       currentFileIndex_(0),
       currentBatchSeq_(0),
       currentLastSeq_(0),
-      versions_(versions) {
+      dbimpl_(dbimpl) {
   assert(files_ != nullptr);
-  assert(versions_ != nullptr);
+  assert(dbimpl_ != nullptr);
 
   reporter_.env = options_->env;
   reporter_.info_log = options_->info_log.get();
@@ -48,14 +43,14 @@ Status TransactionLogIteratorImpl::OpenLogFile(
     return env->NewSequentialFile(fname, file, soptions_);
   } else {
     std::string fname = LogFileName(dir_, logFile->LogNumber());
-    Status s = env->NewSequentialFile(fname, file, soptions_);
-    if (!s.ok()) {
+    Status status = env->NewSequentialFile(fname, file, soptions_);
+    if (!status.ok()) {
       //  If cannot open file in DB directory.
       //  Try the archive dir, as it could have moved in the meanwhile.
       fname = ArchivedLogFileName(dir_, logFile->LogNumber());
-      s = env->NewSequentialFile(fname, file, soptions_);
+      status = env->NewSequentialFile(fname, file, soptions_);
     }
-    return s;
+    return status;
   }
 }
 
@@ -79,7 +74,7 @@ bool TransactionLogIteratorImpl::RestrictedRead(
     Slice* record,
     std::string* scratch) {
   // Don't read if no more complete entries to read from logs
-  if (currentLastSeq_ >= versions_->LastSequence()) {
+  if (currentLastSeq_ >= dbimpl_->GetLatestSequenceNumber()) {
     return false;
   }
   return currentLogReader_->ReadRecord(record, scratch);
@@ -182,15 +177,15 @@ void TransactionLogIteratorImpl::NextImpl(bool internal) {
     // Open the next file
     if (currentFileIndex_ < files_->size() - 1) {
       ++currentFileIndex_;
-      Status s = OpenLogReader(files_->at(currentFileIndex_).get());
-      if (!s.ok()) {
+      Status status =OpenLogReader(files_->at(currentFileIndex_).get());
+      if (!status.ok()) {
         isValid_ = false;
-        currentStatus_ = s;
+        currentStatus_ = status;
         return;
       }
     } else {
       isValid_ = false;
-      if (currentLastSeq_ == versions_->LastSequence()) {
+      if (currentLastSeq_ == dbimpl_->GetLatestSequenceNumber()) {
         currentStatus_ = Status::OK();
       } else {
         currentStatus_ = Status::Corruption("NO MORE DATA LEFT");
@@ -208,10 +203,12 @@ bool TransactionLogIteratorImpl::IsBatchExpected(
   if (batchSeq != expectedSeq) {
     char buf[200];
     snprintf(buf, sizeof(buf),
-             "Discontinuity in log records. Got seq=%" PRIu64
-             ", Expected seq=%" PRIu64 ", Last flushed seq=%" PRIu64
-             ".Log iterator will reseek the correct batch.",
-             batchSeq, expectedSeq, versions_->LastSequence());
+             "Discontinuity in log records. Got seq=%lu, Expected seq=%lu, "
+             "Last flushed seq=%lu.Log iterator will reseek the correct "
+             "batch.",
+             (unsigned long)batchSeq,
+             (unsigned long)expectedSeq,
+             (unsigned long)dbimpl_->GetLatestSequenceNumber());
     reporter_.Info(buf);
     return false;
   }
@@ -243,7 +240,7 @@ void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
   currentLastSeq_ = currentBatchSeq_ +
                     WriteBatchInternal::Count(batch.get()) - 1;
   // currentBatchSeq_ can only change here
-  assert(currentLastSeq_ <= versions_->LastSequence());
+  assert(currentLastSeq_ <= dbimpl_->GetLatestSequenceNumber());
 
   currentBatch_ = move(batch);
   isValid_ = true;
@@ -252,9 +249,9 @@ void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
 
 Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) {
   unique_ptr<SequentialFile> file;
-  Status s = OpenLogFile(logFile, &file);
-  if (!s.ok()) {
-    return s;
+  Status status = OpenLogFile(logFile, &file);
+  if (!status.ok()) {
+    return status;
   }
   assert(file);
   currentLogReader_.reset(new log::Reader(std::move(file), &reporter_,

db/transaction_log_impl.h

@@ -11,12 +11,23 @@
 #include "rocksdb/options.h"
 #include "rocksdb/types.h"
 #include "rocksdb/transaction_log.h"
-#include "db/version_set.h"
+#include "db/db_impl.h"
 #include "db/log_reader.h"
 #include "db/filename.h"
 
 namespace rocksdb {
 
+struct LogReporter : public log::Reader::Reporter {
+  Env* env;
+  Logger* info_log;
+  virtual void Corruption(size_t bytes, const Status& s) {
+    Log(info_log, "dropping %zu bytes; %s", bytes, s.ToString().c_str());
+  }
+  virtual void Info(const char* s) {
+    Log(info_log, "%s", s);
+  }
+};
+
 class LogFileImpl : public LogFile {
  public:
   LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
@@ -60,7 +71,7 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
       const std::string& dir, const DBOptions* options,
       const TransactionLogIterator::ReadOptions& read_options,
       const EnvOptions& soptions, const SequenceNumber seqNum,
-      std::unique_ptr<VectorLogPtr> files, VersionSet const* const versions);
+      std::unique_ptr<VectorLogPtr> files, DBImpl const* const dbimpl);
 
   virtual bool Valid();
 
@@ -84,24 +95,10 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
   std::unique_ptr<WriteBatch> currentBatch_;
   unique_ptr<log::Reader> currentLogReader_;
   Status OpenLogFile(const LogFile* logFile, unique_ptr<SequentialFile>* file);
-
-  struct LogReporter : public log::Reader::Reporter {
-    Env* env;
-    Logger* info_log;
-    virtual void Corruption(size_t bytes, const Status& s) {
-      Log(InfoLogLevel::ERROR_LEVEL, info_log, "dropping %zu bytes; %s", bytes,
-          s.ToString().c_str());
-    }
-    virtual void Info(const char* s) {
-      Log(InfoLogLevel::INFO_LEVEL, info_log, "%s", s);
-    }
-  } reporter_;
-
+  LogReporter reporter_;
   SequenceNumber currentBatchSeq_; // sequence number at start of current batch
   SequenceNumber currentLastSeq_; // last sequence in the current batch
-  // Used only to get latest seq. num
-  // TODO(icanadi) can this be just a callback?
-  VersionSet const* const versions_;
+  DBImpl const * const dbimpl_; // The db on whose log files this iterates
 
   // Reads from transaction log only if the writebatch record has been written
   bool RestrictedRead(Slice* record, std::string* scratch);

db/version_builder.cc

@@ -1,314 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/version_builder.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
-#include <algorithm>
-#include <set>
-#include <vector>
-
-#include "db/dbformat.h"
-#include "db/table_cache.h"
-#include "db/version_set.h"
-#include "table/table_reader.h"
-
-namespace rocksdb {
-
-bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b) {
-  if (a->smallest_seqno != b->smallest_seqno) {
-    return a->smallest_seqno > b->smallest_seqno;
-  }
-  if (a->largest_seqno != b->largest_seqno) {
-    return a->largest_seqno > b->largest_seqno;
-  }
-  // Break ties by file number
-  return a->fd.GetNumber() > b->fd.GetNumber();
-}
-
-namespace {
-bool BySmallestKey(FileMetaData* a, FileMetaData* b,
-                   const InternalKeyComparator* cmp) {
-  int r = cmp->Compare(a->smallest, b->smallest);
-  if (r != 0) {
-    return (r < 0);
-  }
-  // Break ties by file number
-  return (a->fd.GetNumber() < b->fd.GetNumber());
-}
-}  // namespace
-
-class VersionBuilder::Rep {
- private:
-  // Helper to sort files_ in v
-  // kLevel0 -- NewestFirstBySeqNo
-  // kLevelNon0 -- BySmallestKey
-  struct FileComparator {
-    enum SortMethod { kLevel0 = 0, kLevelNon0 = 1, } sort_method;
-    const InternalKeyComparator* internal_comparator;
-
-    bool operator()(FileMetaData* f1, FileMetaData* f2) const {
-      switch (sort_method) {
-        case kLevel0:
-          return NewestFirstBySeqNo(f1, f2);
-        case kLevelNon0:
-          return BySmallestKey(f1, f2, internal_comparator);
-      }
-      assert(false);
-      return false;
-    }
-  };
-
-  typedef std::set<FileMetaData*, FileComparator> FileSet;
-  struct LevelState {
-    std::set<uint64_t> deleted_files;
-    FileSet* added_files;
-  };
-
-  const EnvOptions& env_options_;
-  TableCache* table_cache_;
-  VersionStorageInfo* base_vstorage_;
-  LevelState* levels_;
-  FileComparator level_zero_cmp_;
-  FileComparator level_nonzero_cmp_;
-
- public:
-  Rep(const EnvOptions& env_options, TableCache* table_cache,
-      VersionStorageInfo* base_vstorage)
-      : env_options_(env_options),
-        table_cache_(table_cache),
-        base_vstorage_(base_vstorage) {
-    levels_ = new LevelState[base_vstorage_->num_levels()];
-    level_zero_cmp_.sort_method = FileComparator::kLevel0;
-    level_nonzero_cmp_.sort_method = FileComparator::kLevelNon0;
-    level_nonzero_cmp_.internal_comparator =
-        base_vstorage_->InternalComparator();
-
-    levels_[0].added_files = new FileSet(level_zero_cmp_);
-    for (int level = 1; level < base_vstorage_->num_levels(); level++) {
-      levels_[level].added_files = new FileSet(level_nonzero_cmp_);
-    }
-  }
-
-  ~Rep() {
-    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
-      const FileSet* added = levels_[level].added_files;
-      std::vector<FileMetaData*> to_unref;
-      to_unref.reserve(added->size());
-      for (FileSet::const_iterator it = added->begin(); it != added->end();
-           ++it) {
-        to_unref.push_back(*it);
-      }
-      delete added;
-      for (uint32_t i = 0; i < to_unref.size(); i++) {
-        FileMetaData* f = to_unref[i];
-        f->refs--;
-        if (f->refs <= 0) {
-          if (f->table_reader_handle) {
-            assert(table_cache_ != nullptr);
-            table_cache_->ReleaseHandle(f->table_reader_handle);
-            f->table_reader_handle = nullptr;
-          }
-          delete f;
-        }
-      }
-    }
-
-    delete[] levels_;
-  }
-
-  void CheckConsistency(VersionStorageInfo* vstorage) {
-#ifndef NDEBUG
-    // make sure the files are sorted correctly
-    for (int level = 0; level < vstorage->num_levels(); level++) {
-      auto& level_files = vstorage->LevelFiles(level);
-      for (size_t i = 1; i < level_files.size(); i++) {
-        auto f1 = level_files[i - 1];
-        auto f2 = level_files[i];
-        if (level == 0) {
-          assert(level_zero_cmp_(f1, f2));
-          assert(f1->largest_seqno > f2->largest_seqno);
-        } else {
-          assert(level_nonzero_cmp_(f1, f2));
-
-          // Make sure there is no overlap in levels > 0
-          if (vstorage->InternalComparator()->Compare(f1->largest,
-                                                      f2->smallest) >= 0) {
-            fprintf(stderr, "overlapping ranges in same level %s vs. %s\n",
-                    (f1->largest).DebugString().c_str(),
-                    (f2->smallest).DebugString().c_str());
-            abort();
-          }
-        }
-      }
-    }
-#endif
-  }
-
-  void CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number,
-                                  int level) {
-#ifndef NDEBUG
-    // a file to be deleted better exist in the previous version
-    bool found = false;
-    for (int l = 0; !found && l < base_vstorage_->num_levels(); l++) {
-      const std::vector<FileMetaData*>& base_files =
-          base_vstorage_->LevelFiles(l);
-      for (unsigned int i = 0; i < base_files.size(); i++) {
-        FileMetaData* f = base_files[i];
-        if (f->fd.GetNumber() == number) {
-          found = true;
-          break;
-        }
-      }
-    }
-    // if the file did not exist in the previous version, then it
-    // is possibly moved from lower level to higher level in current
-    // version
-    for (int l = level + 1; !found && l < base_vstorage_->num_levels(); l++) {
-      const FileSet* added = levels_[l].added_files;
-      for (FileSet::const_iterator added_iter = added->begin();
-           added_iter != added->end(); ++added_iter) {
-        FileMetaData* f = *added_iter;
-        if (f->fd.GetNumber() == number) {
-          found = true;
-          break;
-        }
-      }
-    }
-
-    // maybe this file was added in a previous edit that was Applied
-    if (!found) {
-      const FileSet* added = levels_[level].added_files;
-      for (FileSet::const_iterator added_iter = added->begin();
-           added_iter != added->end(); ++added_iter) {
-        FileMetaData* f = *added_iter;
-        if (f->fd.GetNumber() == number) {
-          found = true;
-          break;
-        }
-      }
-    }
-    if (!found) {
-      fprintf(stderr, "not found %" PRIu64 "\n", number);
-    }
-    assert(found);
-#endif
-  }
-
-  // Apply all of the edits in *edit to the current state.
-  void Apply(VersionEdit* edit) {
-    CheckConsistency(base_vstorage_);
-
-    // Delete files
-    const VersionEdit::DeletedFileSet& del = edit->GetDeletedFiles();
-    for (const auto& del_file : del) {
-      const auto level = del_file.first;
-      const auto number = del_file.second;
-      levels_[level].deleted_files.insert(number);
-      CheckConsistencyForDeletes(edit, number, level);
-    }
-
-    // Add new files
-    for (const auto& new_file : edit->GetNewFiles()) {
-      const int level = new_file.first;
-      FileMetaData* f = new FileMetaData(new_file.second);
-      f->refs = 1;
-
-      levels_[level].deleted_files.erase(f->fd.GetNumber());
-      levels_[level].added_files->insert(f);
-    }
-  }
-
-  // Save the current state in *v.
-  void SaveTo(VersionStorageInfo* vstorage) {
-    CheckConsistency(base_vstorage_);
-    CheckConsistency(vstorage);
-
-    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
-      const auto& cmp = (level == 0) ? level_zero_cmp_ : level_nonzero_cmp_;
-      // Merge the set of added files with the set of pre-existing files.
-      // Drop any deleted files.  Store the result in *v.
-      const auto& base_files = base_vstorage_->LevelFiles(level);
-      auto base_iter = base_files.begin();
-      auto base_end = base_files.end();
-      const auto& added_files = *levels_[level].added_files;
-      vstorage->Reserve(level, base_files.size() + added_files.size());
-
-      for (const auto& added : added_files) {
-        // Add all smaller files listed in base_
-        for (auto bpos = std::upper_bound(base_iter, base_end, added, cmp);
-             base_iter != bpos; ++base_iter) {
-          MaybeAddFile(vstorage, level, *base_iter);
-        }
-
-        MaybeAddFile(vstorage, level, added);
-      }
-
-      // Add remaining base files
-      for (; base_iter != base_end; ++base_iter) {
-        MaybeAddFile(vstorage, level, *base_iter);
-      }
-    }
-
-    CheckConsistency(vstorage);
-  }
-
-  void LoadTableHandlers() {
-    assert(table_cache_ != nullptr);
-    for (int level = 0; level < base_vstorage_->num_levels(); level++) {
-      for (auto& file_meta : *(levels_[level].added_files)) {
-        assert(!file_meta->table_reader_handle);
-        table_cache_->FindTable(
-            env_options_, *(base_vstorage_->InternalComparator()),
-            file_meta->fd, &file_meta->table_reader_handle, false);
-        if (file_meta->table_reader_handle != nullptr) {
-          // Load table_reader
-          file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle(
-              file_meta->table_reader_handle);
-        }
-      }
-    }
-  }
-
-  void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f) {
-    if (levels_[level].deleted_files.count(f->fd.GetNumber()) > 0) {
-      // File is deleted: do nothing
-    } else {
-      vstorage->AddFile(level, f);
-    }
-  }
-};
-
-VersionBuilder::VersionBuilder(const EnvOptions& env_options,
-                               TableCache* table_cache,
-                               VersionStorageInfo* base_vstorage)
-    : rep_(new Rep(env_options, table_cache, base_vstorage)) {}
-VersionBuilder::~VersionBuilder() { delete rep_; }
-void VersionBuilder::CheckConsistency(VersionStorageInfo* vstorage) {
-  rep_->CheckConsistency(vstorage);
-}
-void VersionBuilder::CheckConsistencyForDeletes(VersionEdit* edit,
-                                                uint64_t number, int level) {
-  rep_->CheckConsistencyForDeletes(edit, number, level);
-}
-void VersionBuilder::Apply(VersionEdit* edit) { rep_->Apply(edit); }
-void VersionBuilder::SaveTo(VersionStorageInfo* vstorage) {
-  rep_->SaveTo(vstorage);
-}
-void VersionBuilder::LoadTableHandlers() { rep_->LoadTableHandlers(); }
-void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level,
-                                  FileMetaData* f) {
-  rep_->MaybeAddFile(vstorage, level, f);
-}
-
-}  // namespace rocksdb

db/version_builder.h

@@ -1,42 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-#pragma once
-#include "rocksdb/env.h"
-
-namespace rocksdb {
-
-class TableCache;
-class VersionStorageInfo;
-class VersionEdit;
-struct FileMetaData;
-
-// A helper class so we can efficiently apply a whole sequence
-// of edits to a particular state without creating intermediate
-// Versions that contain full copies of the intermediate state.
-class VersionBuilder {
- public:
-  VersionBuilder(const EnvOptions& env_options, TableCache* table_cache,
-                 VersionStorageInfo* base_vstorage);
-  ~VersionBuilder();
-  void CheckConsistency(VersionStorageInfo* vstorage);
-  void CheckConsistencyForDeletes(VersionEdit* edit, uint64_t number,
-                                  int level);
-  void Apply(VersionEdit* edit);
-  void SaveTo(VersionStorageInfo* vstorage);
-  void LoadTableHandlers();
-  void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f);
-
- private:
-  class Rep;
-  Rep* rep_;
-};
-
-extern bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b);
-}  // namespace rocksdb

db/version_builder_test.cc

@@ -1,153 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include <string>
-#include "db/version_edit.h"
-#include "db/version_set.h"
-#include "util/logging.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-
-namespace rocksdb {
-
-class VersionBuilderTest {
- public:
-  const Comparator* ucmp_;
-  InternalKeyComparator icmp_;
-  Options options_;
-  ImmutableCFOptions ioptions_;
-  MutableCFOptions mutable_cf_options_;
-  VersionStorageInfo vstorage_;
-  uint32_t file_num_;
-  CompactionOptionsFIFO fifo_options_;
-  std::vector<uint64_t> size_being_compacted_;
-
-  VersionBuilderTest()
-      : ucmp_(BytewiseComparator()),
-        icmp_(ucmp_),
-        ioptions_(options_),
-        mutable_cf_options_(options_, ioptions_),
-        vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
-                 nullptr),
-        file_num_(1) {
-    mutable_cf_options_.RefreshDerivedOptions(ioptions_);
-    size_being_compacted_.resize(options_.num_levels);
-  }
-
-  ~VersionBuilderTest() {
-    for (int i = 0; i < vstorage_.num_levels(); i++) {
-      for (auto* f : vstorage_.LevelFiles(i)) {
-        if (--f->refs == 0) {
-          delete f;
-        }
-      }
-    }
-  }
-
-  InternalKey GetInternalKey(const char* ukey,
-                             SequenceNumber smallest_seq = 100) {
-    return InternalKey(ukey, smallest_seq, kTypeValue);
-  }
-
-  void Add(int level, uint32_t file_number, const char* smallest,
-           const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
-           SequenceNumber smallest_seq = 100,
-           SequenceNumber largest_seq = 100,
-           uint64_t num_entries = 0, uint64_t num_deletions = 0,
-           bool sampled = false) {
-    assert(level < vstorage_.num_levels());
-    FileMetaData* f = new FileMetaData;
-    f->fd = FileDescriptor(file_number, path_id, file_size);
-    f->smallest = GetInternalKey(smallest, smallest_seq);
-    f->largest = GetInternalKey(largest, largest_seq);
-    f->compensated_file_size = file_size;
-    f->refs = 0;
-    f->num_entries = num_entries;
-    f->num_deletions = num_deletions;
-    vstorage_.AddFile(level, f);
-    if (sampled) {
-      f->init_stats_from_file = true;
-      vstorage_.UpdateAccumulatedStats(f);
-    }
-  }
-
-  void UpdateVersionStorageInfo() {
-    vstorage_.ComputeCompactionScore(mutable_cf_options_, fifo_options_,
-                                    size_being_compacted_);
-    vstorage_.UpdateFilesBySize();
-    vstorage_.UpdateNumNonEmptyLevels();
-    vstorage_.GenerateFileIndexer();
-    vstorage_.GenerateLevelFilesBrief();
-    vstorage_.SetFinalized();
-  }
-};
-
-TEST(VersionBuilderTest, ApplyAndSaveTo) {
-  Add(0, 1U, "150", "200", 100U);
-  // Level 1 score 1.2
-  Add(1, 66U, "150", "200", 100U);
-  Add(1, 88U, "201", "300", 100U);
-  // Level 2 score 1.8. File 7 is the largest. Should be picked
-  Add(2, 6U, "150", "179", 100U);
-  Add(2, 7U, "180", "220", 100U);
-  Add(2, 8U, "221", "300", 100U);
-  // Level 3 score slightly larger than 1
-  Add(3, 26U, "150", "170", 100U);
-  Add(3, 27U, "171", "179", 100U);
-  Add(3, 28U, "191", "220", 100U);
-  Add(3, 29U, "221", "300", 100U);
-  UpdateVersionStorageInfo();
-
-  VersionEdit version_edit;
-  version_edit.AddFile(2, 666, 0, 100U, GetInternalKey("301"),
-                       GetInternalKey("350"), 200, 200);
-  version_edit.DeleteFile(3, 27U);
-
-  EnvOptions env_options;
-
-  VersionBuilder version_builder(env_options, nullptr, &vstorage_);
-
-  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
-                                  kCompactionStyleLevel, nullptr);
-  version_builder.Apply(&version_edit);
-  version_builder.SaveTo(&new_vstorage);
-
-  ASSERT_EQ(400U, new_vstorage.NumLevelBytes(2));
-  ASSERT_EQ(300U, new_vstorage.NumLevelBytes(3));
-
-  for (int i = 0; i < new_vstorage.num_levels(); i++) {
-    for (auto* f : new_vstorage.LevelFiles(i)) {
-      if (--f->refs == 0) {
-        delete f;
-      }
-    }
-  }
-}
-
-TEST(VersionBuilderTest, EstimatedActiveKeys) {
-  const uint32_t kTotalSamples = 20;
-  const uint32_t kNumLevels = 5;
-  const uint32_t kFilesPerLevel = 8;
-  const uint32_t kNumFiles = kNumLevels * kFilesPerLevel;
-  const uint32_t kEntriesPerFile = 1000;
-  const uint32_t kDeletionsPerFile = 100;
-  for (uint32_t i = 0; i < kNumFiles; ++i) {
-    Add(static_cast<int>(i / kFilesPerLevel), i + 1,
-        ToString((i + 100) * 1000).c_str(),
-        ToString((i + 100) * 1000 + 999).c_str(),
-        100U,  0, 100, 100,
-        kEntriesPerFile, kDeletionsPerFile,
-        (i < kTotalSamples));
-  }
-  // minus 2X for the number of deletion entries because:
-  // 1x for deletion entry does not count as a data entry.
-  // 1x for each deletion entry will actually remove one data entry.
-  ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(),
-            (kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles);
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/version_edit.cc

@@ -168,6 +168,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
 
   // Temporary storage for parsing
   int level;
+  uint64_t number;
   FileMetaData f;
   Slice str;
   InternalKey key;
@@ -236,9 +237,9 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         }
         break;
 
-      case kDeletedFile: {
-        uint64_t number;
-        if (GetLevel(&input, &level, &msg) && GetVarint64(&input, &number)) {
+      case kDeletedFile:
+        if (GetLevel(&input, &level, &msg) &&
+            GetVarint64(&input, &number)) {
           deleted_files_.insert(std::make_pair(level, number));
         } else {
           if (!msg) {
@@ -246,7 +247,6 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
           }
         }
         break;
-      }
 
       case kNewFile: {
         uint64_t number;
@@ -297,7 +297,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
           new_files_.push_back(std::make_pair(level, f));
         } else {
           if (!msg) {
-            msg = "new-file3 entry";
+            msg = "new-file2 entry";
           }
         }
         break;

db/version_edit.h

@@ -8,7 +8,6 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
-#include <algorithm>
 #include <set>
 #include <utility>
 #include <vector>
@@ -39,10 +38,10 @@ struct FileDescriptor {
 
   FileDescriptor() : FileDescriptor(0, 0, 0) {}
 
-  FileDescriptor(uint64_t number, uint32_t path_id, uint64_t _file_size)
+  FileDescriptor(uint64_t number, uint32_t path_id, uint64_t file_size)
       : table_reader(nullptr),
         packed_number_and_path_id(PackFileNumberAndPathId(number, path_id)),
-        file_size(_file_size) {}
+        file_size(file_size) {}
 
   FileDescriptor& operator=(const FileDescriptor& fd) {
     table_reader = fd.table_reader;
@@ -75,7 +74,7 @@ struct FileMetaData {
   // Stats for compensating deletion entries during compaction
 
   // File size compensated by deletion entry.
-  // This is updated in Version::UpdateAccumulatedStats() first time when the
+  // This is updated in Version::UpdateTemporaryStats() first time when the
   // file is created or loaded.  After it is updated, it is immutable.
   uint64_t compensated_file_size;
   uint64_t num_entries;            // the number of entries.
@@ -110,16 +109,20 @@ struct FdWithKeyRange {
         largest_key() {
   }
 
-  FdWithKeyRange(FileDescriptor _fd, Slice _smallest_key, Slice _largest_key)
-      : fd(_fd), smallest_key(_smallest_key), largest_key(_largest_key) {}
+  FdWithKeyRange(FileDescriptor fd,
+      Slice smallest_key, Slice largest_key)
+      : fd(fd),
+        smallest_key(smallest_key),
+        largest_key(largest_key) {
+  }
 };
 
 // Data structure to store an array of FdWithKeyRange in one level
 // Actual data is guaranteed to be stored closely
-struct LevelFilesBrief {
+struct FileLevel {
   size_t num_files;
   FdWithKeyRange* files;
-  LevelFilesBrief() {
+  FileLevel() {
     num_files = 0;
     files = nullptr;
   }
@@ -160,7 +163,7 @@ class VersionEdit {
   // Add the specified file at the specified number.
   // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
   // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
-  void AddFile(int level, uint64_t file, uint32_t file_path_id,
+  void AddFile(int level, uint64_t file, uint64_t file_path_id,
                uint64_t file_size, const InternalKey& smallest,
                const InternalKey& largest, const SequenceNumber& smallest_seqno,
                const SequenceNumber& largest_seqno) {
@@ -180,7 +183,9 @@ class VersionEdit {
   }
 
   // Number of edits
-  size_t NumEntries() { return new_files_.size() + deleted_files_.size(); }
+  int NumEntries() {
+    return new_files_.size() + deleted_files_.size();
+  }
 
   bool IsColumnFamilyManipulation() {
     return is_column_family_add_ || is_column_family_drop_;
@@ -211,19 +216,14 @@ class VersionEdit {
   bool EncodeTo(std::string* dst) const;
   Status DecodeFrom(const Slice& src);
 
-  typedef std::set<std::pair<int, uint64_t>> DeletedFileSet;
-
-  const DeletedFileSet& GetDeletedFiles() { return deleted_files_; }
-  const std::vector<std::pair<int, FileMetaData>>& GetNewFiles() {
-    return new_files_;
-  }
-
   std::string DebugString(bool hex_key = false) const;
 
  private:
   friend class VersionSet;
   friend class Version;
 
+  typedef std::set< std::pair<int, uint64_t>> DeletedFileSet;
+
   bool GetLevel(Slice* input, int* level, const char** msg);
 
   int max_level_;

db/version_edit_test.cc

@@ -26,12 +26,11 @@ class VersionEditTest { };
 
 TEST(VersionEditTest, EncodeDecode) {
   static const uint64_t kBig = 1ull << 50;
-  static const uint32_t kBig32Bit = 1ull << 30;
 
   VersionEdit edit;
   for (int i = 0; i < 4; i++) {
     TestEncodeDecode(edit);
-    edit.AddFile(3, kBig + 300 + i, kBig32Bit + 400 + i, 0,
+    edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, 0,
                  InternalKey("foo", kBig + 500 + i, kTypeValue),
                  InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
                  kBig + 500 + i, kBig + 600 + i);

db/version_set.h

@@ -26,7 +26,6 @@
 #include <atomic>
 #include <limits>
 #include "db/dbformat.h"
-#include "db/version_builder.h"
 #include "db/version_edit.h"
 #include "port/port.h"
 #include "db/table_cache.h"
@@ -39,18 +38,16 @@
 
 namespace rocksdb {
 
-namespace log {
-class Writer;
-}
+namespace log { class Writer; }
 
 class Compaction;
+class CompactionPicker;
 class Iterator;
 class LogBuffer;
 class LookupKey;
 class MemTable;
 class Version;
 class VersionSet;
-class WriteBuffer;
 class MergeContext;
 class ColumnFamilyData;
 class ColumnFamilySet;
@@ -62,7 +59,7 @@ class MergeIteratorBuilder;
 // REQUIRES: "file_level.files" contains a sorted list of
 // non-overlapping files.
 extern int FindFile(const InternalKeyComparator& icmp,
-                    const LevelFilesBrief& file_level,
+                    const FileLevel& file_level,
                     const Slice& key);
 
 // Returns true iff some file in "files" overlaps the user key range
@@ -74,98 +71,96 @@ extern int FindFile(const InternalKeyComparator& icmp,
 extern bool SomeFileOverlapsRange(
     const InternalKeyComparator& icmp,
     bool disjoint_sorted_files,
-    const LevelFilesBrief& file_level,
+    const FileLevel& file_level,
     const Slice* smallest_user_key,
     const Slice* largest_user_key);
 
-// Generate LevelFilesBrief from vector<FdWithKeyRange*>
+// Generate FileLevel from vector<FdWithKeyRange*>
 // Would copy smallest_key and largest_key data to sequential memory
 // arena: Arena used to allocate the memory
-extern void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
+extern void DoGenerateFileLevel(FileLevel* file_level,
         const std::vector<FileMetaData*>& files,
         Arena* arena);
 
-class VersionStorageInfo {
+class Version {
  public:
-  VersionStorageInfo(const InternalKeyComparator* internal_comparator,
-                     const Comparator* user_comparator, int num_levels,
-                     CompactionStyle compaction_style,
-                     VersionStorageInfo* src_vstorage);
-  ~VersionStorageInfo();
+  // Append to *iters a sequence of iterators that will
+  // yield the contents of this Version when merged together.
+  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
 
-  void Reserve(int level, size_t size) { files_[level].reserve(size); }
+  void AddIterators(const ReadOptions&, const EnvOptions& soptions,
+                    MergeIteratorBuilder* merger_iter_builder);
 
-  void AddFile(int level, FileMetaData* f);
-
-  void SetFinalized() { finalized_ = true; }
-
-  // Update num_non_empty_levels_.
-  void UpdateNumNonEmptyLevels();
-
-  void GenerateFileIndexer() {
-    file_indexer_.UpdateIndex(&arena_, num_non_empty_levels_, files_);
-  }
-
-  // Update the accumulated stats from a file-meta.
-  void UpdateAccumulatedStats(FileMetaData* file_meta);
-
-  void ComputeCompensatedSizes();
+  // Lookup the value for key.  If found, store it in *val and
+  // return OK.  Else return a non-OK status.
+  // Uses *operands to store merge_operator operations to apply later
+  // REQUIRES: lock is not held
+  void Get(const ReadOptions&, const LookupKey& key, std::string* val,
+           Status* status, MergeContext* merge_context,
+           bool* value_found = nullptr);
 
   // Updates internal structures that keep track of compaction scores
   // We use compaction scores to figure out which compaction to do next
   // REQUIRES: If Version is not yet saved to current_, it can be called without
   // a lock. Once a version is saved to current_, call only with mutex held
-  // TODO find a better way to pass compaction_options_fifo.
   void ComputeCompactionScore(
       const MutableCFOptions& mutable_cf_options,
-      const CompactionOptionsFIFO& compaction_options_fifo,
       std::vector<uint64_t>& size_being_compacted);
 
-  // Generate level_files_brief_ from files_
-  void GenerateLevelFilesBrief();
-  // Sort all files for this version based on their file size and
-  // record results in files_by_size_. The largest files are listed first.
-  void UpdateFilesBySize();
+  // Generate file_levels_ from files_
+  void GenerateFileLevels();
 
-  int MaxInputLevel() const;
+  // Update scores, pre-calculated variables. It needs to be called before
+  // applying the version to the version set.
+  void PrepareApply(
+      const MutableCFOptions& mutable_cf_options,
+      std::vector<uint64_t>& size_being_compacted);
+
+  // Reference count management (so Versions do not disappear out from
+  // under live iterators)
+  void Ref();
+  // Decrease reference count. Delete the object if no reference left
+  // and return true. Otherwise, return false.
+  bool Unref();
+
+  // Returns true iff some level needs a compaction.
+  bool NeedsCompaction() const;
 
   // Returns the maxmimum compaction score for levels 1 to max
-  double max_compaction_score() const { return max_compaction_score_; }
+  double MaxCompactionScore() const { return max_compaction_score_; }
 
   // See field declaration
-  int max_compaction_score_level() const { return max_compaction_score_level_; }
-
-  // Return level number that has idx'th highest score
-  int CompactionScoreLevel(int idx) const { return compaction_level_[idx]; }
-
-  // Return idx'th highest score
-  double CompactionScore(int idx) const { return compaction_score_[idx]; }
+  int MaxCompactionScoreLevel() const { return max_compaction_score_level_; }
 
   void GetOverlappingInputs(
-      int level, const InternalKey* begin,  // nullptr means before all keys
-      const InternalKey* end,               // nullptr means after all keys
+      int level,
+      const InternalKey* begin,         // nullptr means before all keys
+      const InternalKey* end,           // nullptr means after all keys
       std::vector<FileMetaData*>* inputs,
-      int hint_index = -1,         // index of overlap file
-      int* file_index = nullptr);  // return index of overlap file
+      int hint_index = -1,              // index of overlap file
+      int* file_index = nullptr);          // return index of overlap file
 
   void GetOverlappingInputsBinarySearch(
-      int level, const Slice& begin,  // nullptr means before all keys
-      const Slice& end,               // nullptr means after all keys
+      int level,
+      const Slice& begin,         // nullptr means before all keys
+      const Slice& end,           // nullptr means after all keys
       std::vector<FileMetaData*>* inputs,
-      int hint_index,    // index of overlap file
-      int* file_index);  // return index of overlap file
+      int hint_index,             // index of overlap file
+      int* file_index);           // return index of overlap file
 
   void ExtendOverlappingInputs(
-      int level, const Slice& begin,  // nullptr means before all keys
-      const Slice& end,               // nullptr means after all keys
+      int level,
+      const Slice& begin,         // nullptr means before all keys
+      const Slice& end,           // nullptr means after all keys
       std::vector<FileMetaData*>* inputs,
-      unsigned int index);  // start extending from this index
+      unsigned int index);                 // start extending from this index
 
   // Returns true iff some file in the specified level overlaps
   // some part of [*smallest_user_key,*largest_user_key].
   // smallest_user_key==NULL represents a key smaller than all keys in the DB.
   // largest_user_key==NULL represents a key largest than all keys in the DB.
-  bool OverlapInLevel(int level, const Slice* smallest_user_key,
+  bool OverlapInLevel(int level,
+                      const Slice* smallest_user_key,
                       const Slice* largest_user_key);
 
   // Returns true iff the first or last file in inputs contains
@@ -175,70 +170,21 @@ class VersionStorageInfo {
   bool HasOverlappingUserKey(const std::vector<FileMetaData*>* inputs,
                              int level);
 
+
   // Return the level at which we should place a new memtable compaction
   // result that covers the range [smallest_user_key,largest_user_key].
   int PickLevelForMemTableOutput(const MutableCFOptions& mutable_cf_options,
                                  const Slice& smallest_user_key,
                                  const Slice& largest_user_key);
 
-  int num_levels() const { return num_levels_; }
+  int NumberLevels() const { return num_levels_; }
 
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  int num_non_empty_levels() const {
-    assert(finalized_);
-    return num_non_empty_levels_;
-  }
-
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  int NumLevelFiles(int level) const {
-    assert(finalized_);
-    return static_cast<int>(files_[level].size());
-  }
+  // REQUIRES: lock is held
+  int NumLevelFiles(int level) const { return files_[level].size(); }
 
   // Return the combined file size of all files at the specified level.
   uint64_t NumLevelBytes(int level) const;
 
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  const std::vector<FileMetaData*>& LevelFiles(int level) const {
-    return files_[level];
-  }
-
-  const rocksdb::LevelFilesBrief& LevelFilesBrief(int level) const {
-    assert(level < static_cast<int>(level_files_brief_.size()));
-    return level_files_brief_[level];
-  }
-
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  const std::vector<int>& FilesBySize(int level) const {
-    assert(finalized_);
-    return files_by_size_[level];
-  }
-
-  // REQUIRES: lock is held
-  // Set the index that is used to offset into files_by_size_ to find
-  // the next compaction candidate file.
-  void SetNextCompactionIndex(int level, int index) {
-    next_file_to_compact_by_size_[level] = index;
-  }
-
-  // REQUIRES: lock is held
-  int NextCompactionIndex(int level) const {
-    return next_file_to_compact_by_size_[level];
-  }
-
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  const FileIndexer& file_indexer() const {
-    assert(finalized_);
-    return file_indexer_;
-  }
-
-  // Only the first few entries of files_by_size_ are sorted.
-  // There is no need to sort all the files because it is likely
-  // that on a running system, we need to look at only the first
-  // few largest files because a new version is created every few
-  // seconds/minutes (because of concurrent compactions).
-  static const size_t kNumberFilesToSort = 50;
-
   // Return a human-readable short (single-line) summary of the number
   // of files per level.  Uses *scratch as backing store.
   struct LevelSummaryStorage {
@@ -256,44 +202,103 @@ class VersionStorageInfo {
   // file at a level >= 1.
   int64_t MaxNextLevelOverlappingBytes();
 
+  // Add all files listed in the current version to *live.
+  void AddLiveFiles(std::vector<FileDescriptor>* live);
+
   // Return a human readable string that describes this version's contents.
   std::string DebugString(bool hex = false) const;
 
+  // Returns the version nuber of this version
+  uint64_t GetVersionNumber() const { return version_number_; }
+
   uint64_t GetAverageValueSize() const {
-    if (accumulated_num_non_deletions_ == 0) {
+    if (num_non_deletions_ == 0) {
       return 0;
     }
-    assert(accumulated_raw_key_size_ + accumulated_raw_value_size_ > 0);
-    assert(accumulated_file_size_ > 0);
-    return accumulated_raw_value_size_ / accumulated_num_non_deletions_ *
-           accumulated_file_size_ /
-           (accumulated_raw_key_size_ + accumulated_raw_value_size_);
+    assert(total_raw_key_size_ + total_raw_value_size_ > 0);
+    assert(total_file_size_ > 0);
+    return total_raw_value_size_ / num_non_deletions_ * total_file_size_ /
+           (total_raw_key_size_ + total_raw_value_size_);
   }
 
-  uint64_t GetEstimatedActiveKeys() const;
+  // REQUIRES: lock is held
+  // On success, "tp" will contains the table properties of the file
+  // specified in "file_meta".  If the file name of "file_meta" is
+  // known ahread, passing it by a non-null "fname" can save a
+  // file-name conversion.
+  Status GetTableProperties(std::shared_ptr<const TableProperties>* tp,
+                            const FileMetaData* file_meta,
+                            const std::string* fname = nullptr);
 
-  // re-initializes the index that is used to offset into files_by_size_
-  // to find the next compaction candidate file.
-  void ResetNextCompactionIndex(int level) {
-    next_file_to_compact_by_size_[level] = 0;
-  }
+  // REQUIRES: lock is held
+  // On success, *props will be populated with all SSTables' table properties.
+  // The keys of `props` are the sst file name, the values of `props` are the
+  // tables' propertis, represented as shared_ptr.
+  Status GetPropertiesOfAllTables(TablePropertiesCollection* props);
 
-  const InternalKeyComparator* InternalComparator() {
-    return internal_comparator_;
-  }
+  uint64_t GetEstimatedActiveKeys();
+
+  size_t GetMemoryUsageByTableReaders();
+
+  // used to sort files by size
+  struct Fsize {
+    int index;
+    FileMetaData* file;
+  };
 
  private:
+  friend class Compaction;
+  friend class VersionSet;
+  friend class DBImpl;
+  friend class CompactedDBImpl;
+  friend class ColumnFamilyData;
+  friend class CompactionPicker;
+  friend class LevelCompactionPicker;
+  friend class UniversalCompactionPicker;
+  friend class FIFOCompactionPicker;
+  friend class ForwardIterator;
+  friend class InternalStats;
+
+  class LevelFileNumIterator;
+  class LevelFileIteratorState;
+
+  bool PrefixMayMatch(const ReadOptions& read_options, Iterator* level_iter,
+                      const Slice& internal_prefix) const;
+
+  // Update num_non_empty_levels_.
+  void UpdateNumNonEmptyLevels();
+
+  // The helper function of UpdateTemporaryStats, which may fill the missing
+  // fields of file_mata from its associated TableProperties.
+  // Returns true if it does initialize FileMetaData.
+  bool MaybeInitializeFileMetaData(FileMetaData* file_meta);
+
+  // Update the temporary stats associated with the current version.
+  // This temporary stats will be used in compaction.
+  void UpdateTemporaryStats();
+
+  // Sort all files for this version based on their file size and
+  // record results in files_by_size_. The largest files are listed first.
+  void UpdateFilesBySize();
+
+  ColumnFamilyData* cfd_;  // ColumnFamilyData to which this Version belongs
   const InternalKeyComparator* internal_comparator_;
   const Comparator* user_comparator_;
-  int num_levels_;            // Number of levels
-  int num_non_empty_levels_;  // Number of levels. Any level larger than it
-                              // is guaranteed to be empty.
-  // A short brief metadata of files per level
-  autovector<rocksdb::LevelFilesBrief> level_files_brief_;
-  FileIndexer file_indexer_;
-  Arena arena_;  // Used to allocate space for file_levels_
+  TableCache* table_cache_;
+  const MergeOperator* merge_operator_;
 
-  CompactionStyle compaction_style_;
+  autovector<FileLevel> file_levels_;   // A copy of list of files per level
+  Logger* info_log_;
+  Statistics* db_statistics_;
+  int num_levels_;              // Number of levels
+  int num_non_empty_levels_;    // Number of levels. Any level larger than it
+                                // is guaranteed to be empty.
+  FileIndexer file_indexer_;
+  VersionSet* vset_;            // VersionSet to which this Version belongs
+  Arena arena_;                 // Used to allocate space for file_levels_
+  Version* next_;               // Next version in linked list
+  Version* prev_;               // Previous version in linked list
+  int refs_;                    // Number of live refs to this version
 
   // List of files per level, files in each level are arranged
   // in increasing order of keys
@@ -326,147 +331,31 @@ class VersionStorageInfo {
   double max_compaction_score_ = 0.0;   // max score in l1 to ln-1
   int max_compaction_score_level_ = 0;  // level on which max score occurs
 
-  // the following are the sampled temporary stats.
-  // the current accumulated size of sampled files.
-  uint64_t accumulated_file_size_;
-  // the current accumulated size of all raw keys based on the sampled files.
-  uint64_t accumulated_raw_key_size_;
-  // the current accumulated size of all raw keys based on the sampled files.
-  uint64_t accumulated_raw_value_size_;
-  // total number of non-deletion entries
-  uint64_t accumulated_num_non_deletions_;
-  // total number of deletion entries
-  uint64_t accumulated_num_deletions_;
-  // the number of samples
-  uint64_t num_samples_;
-
-  bool finalized_;
-
-  friend class Version;
-  friend class VersionSet;
-  // No copying allowed
-  VersionStorageInfo(const VersionStorageInfo&) = delete;
-  void operator=(const VersionStorageInfo&) = delete;
-};
-
-class Version {
- public:
-  // Append to *iters a sequence of iterators that will
-  // yield the contents of this Version when merged together.
-  // REQUIRES: This version has been saved (see VersionSet::SaveTo)
-  void AddIterators(const ReadOptions&, const EnvOptions& soptions,
-                    MergeIteratorBuilder* merger_iter_builder);
-
-  // Lookup the value for key.  If found, store it in *val and
-  // return OK.  Else return a non-OK status.
-  // Uses *operands to store merge_operator operations to apply later
-  // REQUIRES: lock is not held
-  void Get(const ReadOptions&, const LookupKey& key, std::string* val,
-           Status* status, MergeContext* merge_context,
-           bool* value_found = nullptr);
-
-  // Update scores, pre-calculated variables. It needs to be called before
-  // applying the version to the version set.
-  void PrepareApply(const MutableCFOptions& mutable_cf_options,
-                    std::vector<uint64_t>& size_being_compacted);
-
-  // Reference count management (so Versions do not disappear out from
-  // under live iterators)
-  void Ref();
-  // Decrease reference count. Delete the object if no reference left
-  // and return true. Otherwise, return false.
-  bool Unref();
-
-  // Add all files listed in the current version to *live.
-  void AddLiveFiles(std::vector<FileDescriptor>* live);
-
-  // Return a human readable string that describes this version's contents.
-  std::string DebugString(bool hex = false) const;
-
-  // Returns the version nuber of this version
-  uint64_t GetVersionNumber() const { return version_number_; }
-
-  // REQUIRES: lock is held
-  // On success, "tp" will contains the table properties of the file
-  // specified in "file_meta".  If the file name of "file_meta" is
-  // known ahread, passing it by a non-null "fname" can save a
-  // file-name conversion.
-  Status GetTableProperties(std::shared_ptr<const TableProperties>* tp,
-                            const FileMetaData* file_meta,
-                            const std::string* fname = nullptr);
-
-  // REQUIRES: lock is held
-  // On success, *props will be populated with all SSTables' table properties.
-  // The keys of `props` are the sst file name, the values of `props` are the
-  // tables' propertis, represented as shared_ptr.
-  Status GetPropertiesOfAllTables(TablePropertiesCollection* props);
-
-  uint64_t GetEstimatedActiveKeys() {
-    return storage_info_.GetEstimatedActiveKeys();
-  }
-
-  size_t GetMemoryUsageByTableReaders();
-
-  ColumnFamilyData* cfd() const { return cfd_; }
-
-
-  // Return the next Version in the linked list. Used for debug only
-  Version* TEST_Next() const {
-    return next_;
-  }
-
-  VersionStorageInfo* storage_info() { return &storage_info_; }
-
-  VersionSet* version_set() { return vset_; }
-
-  void GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta);
-
- private:
-  friend class VersionSet;
-
-  const InternalKeyComparator* internal_comparator() const {
-    return storage_info_.internal_comparator_;
-  }
-  const Comparator* user_comparator() const {
-    return storage_info_.user_comparator_;
-  }
-
-  bool PrefixMayMatch(const ReadOptions& read_options, Iterator* level_iter,
-                      const Slice& internal_prefix) const;
-
-  // The helper function of UpdateAccumulatedStats, which may fill the missing
-  // fields of file_mata from its associated TableProperties.
-  // Returns true if it does initialize FileMetaData.
-  bool MaybeInitializeFileMetaData(FileMetaData* file_meta);
-
-  // Update the accumulated stats associated with the current version.
-  // This accumulated stats will be used in compaction.
-  void UpdateAccumulatedStats();
-
-  // Sort all files for this version based on their file size and
-  // record results in files_by_size_. The largest files are listed first.
-  void UpdateFilesBySize();
-
-  ColumnFamilyData* cfd_;  // ColumnFamilyData to which this Version belongs
-  Logger* info_log_;
-  Statistics* db_statistics_;
-  TableCache* table_cache_;
-  const MergeOperator* merge_operator_;
-
-  VersionStorageInfo storage_info_;
-  VersionSet* vset_;            // VersionSet to which this Version belongs
-  Version* next_;               // Next version in linked list
-  Version* prev_;               // Previous version in linked list
-  int refs_;                    // Number of live refs to this version
-
   // A version number that uniquely represents this version. This is
   // used for debugging and logging purposes only.
   uint64_t version_number_;
 
   Version(ColumnFamilyData* cfd, VersionSet* vset, uint64_t version_number = 0);
 
+  // total file size
+  uint64_t total_file_size_;
+  // the total size of all raw keys.
+  uint64_t total_raw_key_size_;
+  // the total size of all raw values.
+  uint64_t total_raw_value_size_;
+  // total number of non-deletion entries
+  uint64_t num_non_deletions_;
+  // total number of deletion entries
+  uint64_t num_deletions_;
+
   ~Version();
 
+  // re-initializes the index that is used to offset into files_by_size_
+  // to find the next compaction candidate file.
+  void ResetNextCompactionIndex(int level) {
+    next_file_to_compact_by_size_[level] = 0;
+  }
+
   // No copying allowed
   Version(const Version&);
   void operator=(const Version&);
@@ -476,7 +365,7 @@ class VersionSet {
  public:
   VersionSet(const std::string& dbname, const DBOptions* db_options,
              const EnvOptions& env_options, Cache* table_cache,
-             WriteBuffer* write_buffer, WriteController* write_controller);
+             WriteController* write_controller);
   ~VersionSet();
 
   // Apply *edit to the current version to form a new descriptor that
@@ -485,12 +374,13 @@ class VersionSet {
   // column_family_options has to be set if edit is column family add
   // REQUIRES: *mu is held on entry.
   // REQUIRES: no other thread concurrently calls LogAndApply()
-  Status LogAndApply(
-      ColumnFamilyData* column_family_data,
-      const MutableCFOptions& mutable_cf_options, VersionEdit* edit,
-      port::Mutex* mu, Directory* db_directory = nullptr,
-      bool new_descriptor_log = false,
-      const ColumnFamilyOptions* column_family_options = nullptr);
+  Status LogAndApply(ColumnFamilyData* column_family_data,
+                     const MutableCFOptions& mutable_cf_options,
+                     VersionEdit* edit,
+                     port::Mutex* mu, Directory* db_directory = nullptr,
+                     bool new_descriptor_log = false,
+                     const ColumnFamilyOptions* column_family_options =
+                         nullptr);
 
   // Recover the last saved descriptor from persistent storage.
   // If read_only == true, Recover() will not complain if some column families
@@ -525,16 +415,23 @@ class VersionSet {
 #endif  // ROCKSDB_LITE
 
   // Return the current manifest file number
-  uint64_t manifest_file_number() const { return manifest_file_number_; }
+  uint64_t ManifestFileNumber() const { return manifest_file_number_; }
 
-  uint64_t pending_manifest_file_number() const {
+  uint64_t PendingManifestFileNumber() const {
     return pending_manifest_file_number_;
   }
 
-  uint64_t current_next_file_number() const { return next_file_number_.load(); }
-
   // Allocate and return a new file number
-  uint64_t NewFileNumber() { return next_file_number_.fetch_add(1); }
+  uint64_t NewFileNumber() { return next_file_number_++; }
+
+  // Arrange to reuse "file_number" unless a newer file number has
+  // already been allocated.
+  // REQUIRES: "file_number" was returned by a call to NewFileNumber().
+  void ReuseLogFileNumber(uint64_t file_number) {
+    if (next_file_number_ == file_number + 1) {
+      next_file_number_ = file_number;
+    }
+  }
 
   // Return the last sequence number.
   uint64_t LastSequence() const {
@@ -548,12 +445,11 @@ class VersionSet {
   }
 
   // Mark the specified file number as used.
-  // REQUIRED: this is only called during single-threaded recovery
-  void MarkFileNumberUsedDuringRecovery(uint64_t number);
+  void MarkFileNumberUsed(uint64_t number);
 
   // Return the log file number for the log file that is currently
   // being compacted, or zero if there is no such log file.
-  uint64_t prev_log_number() const { return prev_log_number_; }
+  uint64_t PrevLogNumber() const { return prev_log_number_; }
 
   // Returns the minimum log number such that all
   // log numbers less than or equal to it can be deleted
@@ -579,7 +475,7 @@ class VersionSet {
   uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key);
 
   // Return the size of the current manifest file
-  uint64_t manifest_file_size() const { return manifest_file_size_; }
+  uint64_t ManifestFileSize() const { return manifest_file_size_; }
 
   // verify that the files that we started with for a compaction
   // still exist in the current version and in the same original level.
@@ -590,18 +486,18 @@ class VersionSet {
   Status GetMetadataForFile(uint64_t number, int* filelevel,
                             FileMetaData** metadata, ColumnFamilyData** cfd);
 
-  void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata);
+  void GetLiveFilesMetaData(
+    std::vector<LiveFileMetaData> *metadata);
 
   void GetObsoleteFiles(std::vector<FileMetaData*>* files);
 
   ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); }
-  const EnvOptions& env_options() { return env_options_; }
 
  private:
+  class Builder;
   struct ManifestWriter;
 
   friend class Version;
-  friend class DBImpl;
 
   struct LogReporter : public log::Reader::Reporter {
     Status* status;
@@ -626,7 +522,7 @@ class VersionSet {
   Env* const env_;
   const std::string dbname_;
   const DBOptions* const db_options_;
-  std::atomic<uint64_t> next_file_number_;
+  uint64_t next_file_number_;
   uint64_t manifest_file_number_;
   uint64_t pending_manifest_file_number_;
   std::atomic<uint64_t> last_sequence_;
@@ -658,7 +554,7 @@ class VersionSet {
   void operator=(const VersionSet&);
 
   void LogAndApplyCFHelper(VersionEdit* edit);
-  void LogAndApplyHelper(ColumnFamilyData* cfd, VersionBuilder* b, Version* v,
+  void LogAndApplyHelper(ColumnFamilyData* cfd, Builder* b, Version* v,
                          VersionEdit* edit, port::Mutex* mu);
 };
 

db/version_set_test.cc

@@ -14,15 +14,15 @@
 
 namespace rocksdb {
 
-class GenerateLevelFilesBriefTest {
+class GenerateFileLevelTest {
  public:
   std::vector<FileMetaData*> files_;
-  LevelFilesBrief file_level_;
+  FileLevel file_level_;
   Arena arena_;
 
-  GenerateLevelFilesBriefTest() { }
+  GenerateFileLevelTest() { }
 
-  ~GenerateLevelFilesBriefTest() {
+  ~GenerateFileLevelTest() {
     for (unsigned int i = 0; i < files_.size(); i++) {
       delete files_[i];
     }
@@ -49,33 +49,33 @@ class GenerateLevelFilesBriefTest {
   }
 };
 
-TEST(GenerateLevelFilesBriefTest, Empty) {
-  DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
+TEST(GenerateFileLevelTest, Empty) {
+  DoGenerateFileLevel(&file_level_, files_, &arena_);
   ASSERT_EQ(0u, file_level_.num_files);
   ASSERT_EQ(0, Compare());
 }
 
-TEST(GenerateLevelFilesBriefTest, Single) {
+TEST(GenerateFileLevelTest, Single) {
   Add("p", "q");
-  DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
+  DoGenerateFileLevel(&file_level_, files_, &arena_);
   ASSERT_EQ(1u, file_level_.num_files);
   ASSERT_EQ(0, Compare());
 }
 
 
-TEST(GenerateLevelFilesBriefTest, Multiple) {
+TEST(GenerateFileLevelTest, Multiple) {
   Add("150", "200");
   Add("200", "250");
   Add("300", "350");
   Add("400", "450");
-  DoGenerateLevelFilesBrief(&file_level_, files_, &arena_);
+  DoGenerateFileLevel(&file_level_, files_, &arena_);
   ASSERT_EQ(4u, file_level_.num_files);
   ASSERT_EQ(0, Compare());
 }
 
 class FindLevelFileTest {
  public:
-  LevelFilesBrief file_level_;
+  FileLevel file_level_;
   bool disjoint_sorted_files_;
   Arena arena_;
 

db/wal_manager.cc

@@ -1,457 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/wal_manager.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
-#include <algorithm>
-#include <vector>
-#include <memory>
-
-#include "db/filename.h"
-#include "db/transaction_log_impl.h"
-#include "db/log_reader.h"
-#include "db/log_writer.h"
-#include "db/write_batch_internal.h"
-#include "port/port.h"
-#include "rocksdb/env.h"
-#include "rocksdb/options.h"
-#include "rocksdb/write_batch.h"
-#include "util/coding.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/sync_point.h"
-#include "util/string_util.h"
-
-namespace rocksdb {
-
-#ifndef ROCKSDB_LITE
-
-Status WalManager::GetSortedWalFiles(VectorLogPtr& files) {
-  // First get sorted files in db dir, then get sorted files from archived
-  // dir, to avoid a race condition where a log file is moved to archived
-  // dir in between.
-  Status s;
-  // list wal files in main db dir.
-  VectorLogPtr logs;
-  s = GetSortedWalsOfType(db_options_.wal_dir, logs, kAliveLogFile);
-  if (!s.ok()) {
-    return s;
-  }
-
-  // Reproduce the race condition where a log file is moved
-  // to archived dir, between these two sync points, used in
-  // (DBTest,TransactionLogIteratorRace)
-  TEST_SYNC_POINT("WalManager::GetSortedWalFiles:1");
-  TEST_SYNC_POINT("WalManager::GetSortedWalFiles:2");
-
-  files.clear();
-  // list wal files in archive dir.
-  std::string archivedir = ArchivalDirectory(db_options_.wal_dir);
-  if (env_->FileExists(archivedir)) {
-    s = GetSortedWalsOfType(archivedir, files, kArchivedLogFile);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  uint64_t latest_archived_log_number = 0;
-  if (!files.empty()) {
-    latest_archived_log_number = files.back()->LogNumber();
-    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-        "Latest Archived log: %" PRIu64,
-        latest_archived_log_number);
-  }
-
-  files.reserve(files.size() + logs.size());
-  for (auto& log : logs) {
-    if (log->LogNumber() > latest_archived_log_number) {
-      files.push_back(std::move(log));
-    } else {
-      // When the race condition happens, we could see the
-      // same log in both db dir and archived dir. Simply
-      // ignore the one in db dir. Note that, if we read
-      // archived dir first, we would have missed the log file.
-      Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
-          "%s already moved to archive", log->PathName().c_str());
-    }
-  }
-
-  return s;
-}
-
-Status WalManager::GetUpdatesSince(
-    SequenceNumber seq, std::unique_ptr<TransactionLogIterator>* iter,
-    const TransactionLogIterator::ReadOptions& read_options,
-    VersionSet* version_set) {
-
-  //  Get all sorted Wal Files.
-  //  Do binary search and open files and find the seq number.
-
-  std::unique_ptr<VectorLogPtr> wal_files(new VectorLogPtr);
-  Status s = GetSortedWalFiles(*wal_files);
-  if (!s.ok()) {
-    return s;
-  }
-
-  s = RetainProbableWalFiles(*wal_files, seq);
-  if (!s.ok()) {
-    return s;
-  }
-  iter->reset(new TransactionLogIteratorImpl(
-      db_options_.wal_dir, &db_options_, read_options, env_options_, seq,
-      std::move(wal_files), version_set));
-  return (*iter)->status();
-}
-
-// 1. Go through all archived files and
-//    a. if ttl is enabled, delete outdated files
-//    b. if archive size limit is enabled, delete empty files,
-//        compute file number and size.
-// 2. If size limit is enabled:
-//    a. compute how many files should be deleted
-//    b. get sorted non-empty archived logs
-//    c. delete what should be deleted
-void WalManager::PurgeObsoleteWALFiles() {
-  bool const ttl_enabled = db_options_.WAL_ttl_seconds > 0;
-  bool const size_limit_enabled = db_options_.WAL_size_limit_MB > 0;
-  if (!ttl_enabled && !size_limit_enabled) {
-    return;
-  }
-
-  int64_t current_time;
-  Status s = env_->GetCurrentTime(&current_time);
-  if (!s.ok()) {
-    Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
-        "Can't get current time: %s", s.ToString().c_str());
-    assert(false);
-    return;
-  }
-  uint64_t const now_seconds = static_cast<uint64_t>(current_time);
-  uint64_t const time_to_check = (ttl_enabled && !size_limit_enabled)
-                                     ? db_options_.WAL_ttl_seconds / 2
-                                     : kDefaultIntervalToDeleteObsoleteWAL;
-
-  if (purge_wal_files_last_run_ + time_to_check > now_seconds) {
-    return;
-  }
-
-  purge_wal_files_last_run_ = now_seconds;
-
-  std::string archival_dir = ArchivalDirectory(db_options_.wal_dir);
-  std::vector<std::string> files;
-  s = env_->GetChildren(archival_dir, &files);
-  if (!s.ok()) {
-    Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
-        "Can't get archive files: %s", s.ToString().c_str());
-    assert(false);
-    return;
-  }
-
-  size_t log_files_num = 0;
-  uint64_t log_file_size = 0;
-
-  for (auto& f : files) {
-    uint64_t number;
-    FileType type;
-    if (ParseFileName(f, &number, &type) && type == kLogFile) {
-      std::string const file_path = archival_dir + "/" + f;
-      if (ttl_enabled) {
-        uint64_t file_m_time;
-        s = env_->GetFileModificationTime(file_path, &file_m_time);
-        if (!s.ok()) {
-          Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
-              "Can't get file mod time: %s: %s",
-              file_path.c_str(), s.ToString().c_str());
-          continue;
-        }
-        if (now_seconds - file_m_time > db_options_.WAL_ttl_seconds) {
-          s = env_->DeleteFile(file_path);
-          if (!s.ok()) {
-            Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
-                "Can't delete file: %s: %s",
-                file_path.c_str(), s.ToString().c_str());
-            continue;
-          } else {
-            MutexLock l(&read_first_record_cache_mutex_);
-            read_first_record_cache_.erase(number);
-          }
-          continue;
-        }
-      }
-
-      if (size_limit_enabled) {
-        uint64_t file_size;
-        s = env_->GetFileSize(file_path, &file_size);
-        if (!s.ok()) {
-          Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
-              "Unable to get file size: %s: %s",
-              file_path.c_str(), s.ToString().c_str());
-          return;
-        } else {
-          if (file_size > 0) {
-            log_file_size = std::max(log_file_size, file_size);
-            ++log_files_num;
-          } else {
-            s = env_->DeleteFile(file_path);
-            if (!s.ok()) {
-              Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
-                  "Unable to delete file: %s: %s",
-                  file_path.c_str(), s.ToString().c_str());
-              continue;
-            } else {
-              MutexLock l(&read_first_record_cache_mutex_);
-              read_first_record_cache_.erase(number);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  if (0 == log_files_num || !size_limit_enabled) {
-    return;
-  }
-
-  size_t const files_keep_num =
-      db_options_.WAL_size_limit_MB * 1024 * 1024 / log_file_size;
-  if (log_files_num <= files_keep_num) {
-    return;
-  }
-
-  size_t files_del_num = log_files_num - files_keep_num;
-  VectorLogPtr archived_logs;
-  GetSortedWalsOfType(archival_dir, archived_logs, kArchivedLogFile);
-
-  if (files_del_num > archived_logs.size()) {
-    Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
-        "Trying to delete more archived log files than "
-        "exist. Deleting all");
-    files_del_num = archived_logs.size();
-  }
-
-  for (size_t i = 0; i < files_del_num; ++i) {
-    std::string const file_path = archived_logs[i]->PathName();
-    s = env_->DeleteFile(db_options_.wal_dir + "/" + file_path);
-    if (!s.ok()) {
-      Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
-          "Unable to delete file: %s: %s", file_path.c_str(),
-          s.ToString().c_str());
-      continue;
-    } else {
-      MutexLock l(&read_first_record_cache_mutex_);
-      read_first_record_cache_.erase(archived_logs[i]->LogNumber());
-    }
-  }
-}
-
-void WalManager::ArchiveWALFile(const std::string& fname, uint64_t number) {
-  auto archived_log_name = ArchivedLogFileName(db_options_.wal_dir, number);
-  // The sync point below is used in (DBTest,TransactionLogIteratorRace)
-  TEST_SYNC_POINT("WalManager::PurgeObsoleteFiles:1");
-  Status s = env_->RenameFile(fname, archived_log_name);
-  // The sync point below is used in (DBTest,TransactionLogIteratorRace)
-  TEST_SYNC_POINT("WalManager::PurgeObsoleteFiles:2");
-  Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-      "Move log file %s to %s -- %s\n", fname.c_str(),
-      archived_log_name.c_str(), s.ToString().c_str());
-}
-
-namespace {
-struct CompareLogByPointer {
-  bool operator()(const std::unique_ptr<LogFile>& a,
-                  const std::unique_ptr<LogFile>& b) {
-    LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get());
-    LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get());
-    return *a_impl < *b_impl;
-  }
-};
-}
-
-Status WalManager::GetSortedWalsOfType(const std::string& path,
-                                       VectorLogPtr& log_files,
-                                       WalFileType log_type) {
-  std::vector<std::string> all_files;
-  const Status status = env_->GetChildren(path, &all_files);
-  if (!status.ok()) {
-    return status;
-  }
-  log_files.reserve(all_files.size());
-  for (const auto& f : all_files) {
-    uint64_t number;
-    FileType type;
-    if (ParseFileName(f, &number, &type) && type == kLogFile) {
-      SequenceNumber sequence;
-      Status s = ReadFirstRecord(log_type, number, &sequence);
-      if (!s.ok()) {
-        return s;
-      }
-      if (sequence == 0) {
-        // empty file
-        continue;
-      }
-
-      // Reproduce the race condition where a log file is moved
-      // to archived dir, between these two sync points, used in
-      // (DBTest,TransactionLogIteratorRace)
-      TEST_SYNC_POINT("WalManager::GetSortedWalsOfType:1");
-      TEST_SYNC_POINT("WalManager::GetSortedWalsOfType:2");
-
-      uint64_t size_bytes;
-      s = env_->GetFileSize(LogFileName(path, number), &size_bytes);
-      // re-try in case the alive log file has been moved to archive.
-      if (!s.ok() && log_type == kAliveLogFile &&
-          env_->FileExists(ArchivedLogFileName(path, number))) {
-        s = env_->GetFileSize(ArchivedLogFileName(path, number), &size_bytes);
-      }
-      if (!s.ok()) {
-        return s;
-      }
-
-      log_files.push_back(std::move(std::unique_ptr<LogFile>(
-          new LogFileImpl(number, log_type, sequence, size_bytes))));
-    }
-  }
-  CompareLogByPointer compare_log_files;
-  std::sort(log_files.begin(), log_files.end(), compare_log_files);
-  return status;
-}
-
-Status WalManager::RetainProbableWalFiles(VectorLogPtr& all_logs,
-                                          const SequenceNumber target) {
-  int64_t start = 0;  // signed to avoid overflow when target is < first file.
-  int64_t end = static_cast<int64_t>(all_logs.size()) - 1;
-  // Binary Search. avoid opening all files.
-  while (end >= start) {
-    int64_t mid = start + (end - start) / 2;  // Avoid overflow.
-    SequenceNumber current_seq_num = all_logs.at(mid)->StartSequence();
-    if (current_seq_num == target) {
-      end = mid;
-      break;
-    } else if (current_seq_num < target) {
-      start = mid + 1;
-    } else {
-      end = mid - 1;
-    }
-  }
-  // end could be -ve.
-  size_t start_index = std::max(static_cast<int64_t>(0), end);
-  // The last wal file is always included
-  all_logs.erase(all_logs.begin(), all_logs.begin() + start_index);
-  return Status::OK();
-}
-
-Status WalManager::ReadFirstRecord(const WalFileType type,
-                                   const uint64_t number,
-                                   SequenceNumber* sequence) {
-  if (type != kAliveLogFile && type != kArchivedLogFile) {
-    Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
-        "[WalManger] Unknown file type %s", ToString(type).c_str());
-    return Status::NotSupported(
-        "File Type Not Known " + ToString(type));
-  }
-  {
-    MutexLock l(&read_first_record_cache_mutex_);
-    auto itr = read_first_record_cache_.find(number);
-    if (itr != read_first_record_cache_.end()) {
-      *sequence = itr->second;
-      return Status::OK();
-    }
-  }
-  Status s;
-  if (type == kAliveLogFile) {
-    std::string fname = LogFileName(db_options_.wal_dir, number);
-    s = ReadFirstLine(fname, sequence);
-    if (env_->FileExists(fname) && !s.ok()) {
-      // return any error that is not caused by non-existing file
-      return s;
-    }
-  }
-
-  if (type == kArchivedLogFile || !s.ok()) {
-    //  check if the file got moved to archive.
-    std::string archived_file =
-        ArchivedLogFileName(db_options_.wal_dir, number);
-    s = ReadFirstLine(archived_file, sequence);
-  }
-
-  if (s.ok() && *sequence != 0) {
-    MutexLock l(&read_first_record_cache_mutex_);
-    read_first_record_cache_.insert({number, *sequence});
-  }
-  return s;
-}
-
-// the function returns status.ok() and sequence == 0 if the file exists, but is
-// empty
-Status WalManager::ReadFirstLine(const std::string& fname,
-                                 SequenceNumber* sequence) {
-  struct LogReporter : public log::Reader::Reporter {
-    Env* env;
-    Logger* info_log;
-    const char* fname;
-
-    Status* status;
-    bool ignore_error;  // true if db_options_.paranoid_checks==false
-    virtual void Corruption(size_t bytes, const Status& s) {
-      Log(InfoLogLevel::WARN_LEVEL, info_log,
-          "[WalManager] %s%s: dropping %d bytes; %s",
-          (this->ignore_error ? "(ignoring error) " : ""), fname,
-          static_cast<int>(bytes), s.ToString().c_str());
-      if (this->status->ok()) {
-        // only keep the first error
-        *this->status = s;
-      }
-    }
-  };
-
-  std::unique_ptr<SequentialFile> file;
-  Status status = env_->NewSequentialFile(fname, &file, env_options_);
-
-  if (!status.ok()) {
-    return status;
-  }
-
-  LogReporter reporter;
-  reporter.env = env_;
-  reporter.info_log = db_options_.info_log.get();
-  reporter.fname = fname.c_str();
-  reporter.status = &status;
-  reporter.ignore_error = !db_options_.paranoid_checks;
-  log::Reader reader(std::move(file), &reporter, true /*checksum*/,
-                     0 /*initial_offset*/);
-  std::string scratch;
-  Slice record;
-
-  if (reader.ReadRecord(&record, &scratch) &&
-      (status.ok() || !db_options_.paranoid_checks)) {
-    if (record.size() < 12) {
-      reporter.Corruption(record.size(),
-                          Status::Corruption("log record too small"));
-      // TODO read record's till the first no corrupt entry?
-    } else {
-      WriteBatch batch;
-      WriteBatchInternal::SetContents(&batch, record);
-      *sequence = WriteBatchInternal::Sequence(&batch);
-      return Status::OK();
-    }
-  }
-
-  // ReadRecord returns false on EOF, which means that the log file is empty. we
-  // return status.ok() in that case and set sequence number to 0
-  *sequence = 0;
-  return status;
-}
-
-#endif  // ROCKSDB_LITE
-}  // namespace rocksdb

db/wal_manager.h

@@ -1,95 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#pragma once
-
-#include <atomic>
-#include <deque>
-#include <limits>
-#include <set>
-#include <utility>
-#include <vector>
-#include <string>
-#include <memory>
-
-#include "port/port.h"
-
-#include "rocksdb/env.h"
-#include "rocksdb/options.h"
-#include "rocksdb/types.h"
-#include "rocksdb/transaction_log.h"
-#include "rocksdb/status.h"
-
-#include "db/version_set.h"
-
-namespace rocksdb {
-
-#ifndef ROCKSDB_LITE
-class WalManager {
- public:
-  WalManager(const DBOptions& db_options, const EnvOptions& env_options)
-      : db_options_(db_options),
-        env_options_(env_options),
-        env_(db_options.env),
-        purge_wal_files_last_run_(0) {}
-
-  Status GetSortedWalFiles(VectorLogPtr& files);
-
-  Status GetUpdatesSince(
-      SequenceNumber seq_number, std::unique_ptr<TransactionLogIterator>* iter,
-      const TransactionLogIterator::ReadOptions& read_options,
-      VersionSet* version_set);
-
-  void PurgeObsoleteWALFiles();
-
-  void ArchiveWALFile(const std::string& fname, uint64_t number);
-
-  Status TEST_ReadFirstRecord(const WalFileType type, const uint64_t number,
-                              SequenceNumber* sequence) {
-    return ReadFirstRecord(type, number, sequence);
-  }
-
-  Status TEST_ReadFirstLine(const std::string& fname,
-                            SequenceNumber* sequence) {
-    return ReadFirstLine(fname, sequence);
-  }
-
- private:
-  Status GetSortedWalsOfType(const std::string& path, VectorLogPtr& log_files,
-                             WalFileType type);
-  // Requires: all_logs should be sorted with earliest log file first
-  // Retains all log files in all_logs which contain updates with seq no.
-  // Greater Than or Equal to the requested SequenceNumber.
-  Status RetainProbableWalFiles(VectorLogPtr& all_logs,
-                                const SequenceNumber target);
-
-  Status ReadFirstRecord(const WalFileType type, const uint64_t number,
-                         SequenceNumber* sequence);
-
-  Status ReadFirstLine(const std::string& fname, SequenceNumber* sequence);
-
-  // ------- state from DBImpl ------
-  const DBOptions& db_options_;
-  const EnvOptions& env_options_;
-  Env* env_;
-
-  // ------- WalManager state -------
-  // cache for ReadFirstRecord() calls
-  std::unordered_map<uint64_t, SequenceNumber> read_first_record_cache_;
-  port::Mutex read_first_record_cache_mutex_;
-
-  // last time when PurgeObsoleteWALFiles ran.
-  uint64_t purge_wal_files_last_run_;
-
-  // obsolete files will be deleted every this seconds if ttl deletion is
-  // enabled and archive size_limit is disabled.
-  static const uint64_t kDefaultIntervalToDeleteObsoleteWAL = 600;
-};
-
-#endif  // ROCKSDB_LITE
-}  // namespace rocksdb

db/wal_manager_test.cc

@@ -1,283 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include <map>
-#include <string>
-
-#include "rocksdb/cache.h"
-#include "rocksdb/write_batch.h"
-
-#include "db/wal_manager.h"
-#include "db/log_writer.h"
-#include "db/column_family.h"
-#include "db/version_set.h"
-#include "db/writebuffer.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-#include "table/mock_table.h"
-#include "db/db_impl.h"
-
-namespace rocksdb {
-
-// TODO(icanadi) mock out VersionSet
-// TODO(icanadi) move other WalManager-specific tests from db_test here
-class WalManagerTest {
- public:
-  WalManagerTest()
-      : env_(Env::Default()),
-        dbname_(test::TmpDir() + "/wal_manager_test"),
-        table_cache_(NewLRUCache(50000, 16, 8)),
-        write_buffer_(db_options_.db_write_buffer_size),
-        current_log_number_(0) {
-    DestroyDB(dbname_, Options());
-  }
-
-  void Init() {
-    ASSERT_OK(env_->CreateDirIfMissing(dbname_));
-    ASSERT_OK(env_->CreateDirIfMissing(ArchivalDirectory(dbname_)));
-    db_options_.db_paths.emplace_back(dbname_,
-                                      std::numeric_limits<uint64_t>::max());
-    db_options_.wal_dir = dbname_;
-
-    versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
-                                   table_cache_.get(), &write_buffer_,
-                                   &write_controller_));
-
-    wal_manager_.reset(new WalManager(db_options_, env_options_));
-  }
-
-  void Reopen() {
-    wal_manager_.reset(new WalManager(db_options_, env_options_));
-  }
-
-  // NOT thread safe
-  void Put(const std::string& key, const std::string& value) {
-    assert(current_log_writer_.get() != nullptr);
-    uint64_t seq =  versions_->LastSequence() + 1;
-    WriteBatch batch;
-    batch.Put(key, value);
-    WriteBatchInternal::SetSequence(&batch, seq);
-    current_log_writer_->AddRecord(WriteBatchInternal::Contents(&batch));
-    versions_->SetLastSequence(seq);
-  }
-
-  // NOT thread safe
-  void RollTheLog(bool archived) {
-    current_log_number_++;
-    std::string fname = ArchivedLogFileName(dbname_, current_log_number_);
-    unique_ptr<WritableFile> file;
-    ASSERT_OK(env_->NewWritableFile(fname, &file, env_options_));
-    current_log_writer_.reset(new log::Writer(std::move(file)));
-  }
-
-  void CreateArchiveLogs(int num_logs, int entries_per_log) {
-    for (int i = 1; i <= num_logs; ++i) {
-      RollTheLog(true);
-      for (int k = 0; k < entries_per_log; ++k) {
-        Put(ToString(k), std::string(1024, 'a'));
-      }
-    }
-  }
-
-  std::unique_ptr<TransactionLogIterator> OpenTransactionLogIter(
-      const SequenceNumber seq) {
-    unique_ptr<TransactionLogIterator> iter;
-    Status status = wal_manager_->GetUpdatesSince(
-        seq, &iter, TransactionLogIterator::ReadOptions(), versions_.get());
-    ASSERT_OK(status);
-    return std::move(iter);
-  }
-
-  Env* env_;
-  std::string dbname_;
-  WriteController write_controller_;
-  EnvOptions env_options_;
-  std::shared_ptr<Cache> table_cache_;
-  DBOptions db_options_;
-  WriteBuffer write_buffer_;
-  std::unique_ptr<VersionSet> versions_;
-  std::unique_ptr<WalManager> wal_manager_;
-
-  std::unique_ptr<log::Writer> current_log_writer_;
-  uint64_t current_log_number_;
-};
-
-TEST(WalManagerTest, ReadFirstRecordCache) {
-  Init();
-  std::string path = dbname_ + "/000001.log";
-  unique_ptr<WritableFile> file;
-  ASSERT_OK(env_->NewWritableFile(path, &file, EnvOptions()));
-
-  SequenceNumber s;
-  ASSERT_OK(wal_manager_->TEST_ReadFirstLine(path, &s));
-  ASSERT_EQ(s, 0U);
-
-  ASSERT_OK(wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1, &s));
-  ASSERT_EQ(s, 0U);
-
-  log::Writer writer(std::move(file));
-  WriteBatch batch;
-  batch.Put("foo", "bar");
-  WriteBatchInternal::SetSequence(&batch, 10);
-  writer.AddRecord(WriteBatchInternal::Contents(&batch));
-
-  // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here.
-  // Waiting for lei to finish with db_test
-  // env_->count_sequential_reads_ = true;
-  // sequential_read_counter_ sanity test
-  // ASSERT_EQ(env_->sequential_read_counter_.Read(), 0);
-
-  ASSERT_OK(wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1, &s));
-  ASSERT_EQ(s, 10U);
-  // did a read
-  // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here
-  // ASSERT_EQ(env_->sequential_read_counter_.Read(), 1);
-
-  ASSERT_OK(wal_manager_->TEST_ReadFirstRecord(kAliveLogFile, 1, &s));
-  ASSERT_EQ(s, 10U);
-  // no new reads since the value is cached
-  // TODO(icanadi) move SpecialEnv outside of db_test, so we can reuse it here
-  // ASSERT_EQ(env_->sequential_read_counter_.Read(), 1);
-}
-
-namespace {
-uint64_t GetLogDirSize(std::string dir_path, Env* env) {
-  uint64_t dir_size = 0;
-  std::vector<std::string> files;
-  env->GetChildren(dir_path, &files);
-  for (auto& f : files) {
-    uint64_t number;
-    FileType type;
-    if (ParseFileName(f, &number, &type) && type == kLogFile) {
-      std::string const file_path = dir_path + "/" + f;
-      uint64_t file_size;
-      env->GetFileSize(file_path, &file_size);
-      dir_size += file_size;
-    }
-  }
-  return dir_size;
-}
-std::vector<std::uint64_t> ListSpecificFiles(
-    Env* env, const std::string& path, const FileType expected_file_type) {
-  std::vector<std::string> files;
-  std::vector<uint64_t> file_numbers;
-  env->GetChildren(path, &files);
-  uint64_t number;
-  FileType type;
-  for (size_t i = 0; i < files.size(); ++i) {
-    if (ParseFileName(files[i], &number, &type)) {
-      if (type == expected_file_type) {
-        file_numbers.push_back(number);
-      }
-    }
-  }
-  return std::move(file_numbers);
-}
-
-int CountRecords(TransactionLogIterator* iter) {
-  int count = 0;
-  SequenceNumber lastSequence = 0;
-  BatchResult res;
-  while (iter->Valid()) {
-    res = iter->GetBatch();
-    ASSERT_TRUE(res.sequence > lastSequence);
-    ++count;
-    lastSequence = res.sequence;
-    ASSERT_OK(iter->status());
-    iter->Next();
-  }
-  return count;
-}
-}  // namespace
-
-TEST(WalManagerTest, WALArchivalSizeLimit) {
-  db_options_.WAL_ttl_seconds = 0;
-  db_options_.WAL_size_limit_MB = 1000;
-  Init();
-
-  // TEST : Create WalManager with huge size limit and no ttl.
-  // Create some archived files and call PurgeObsoleteWALFiles().
-  // Count the archived log files that survived.
-  // Assert that all of them did.
-  // Change size limit. Re-open WalManager.
-  // Assert that archive is not greater than WAL_size_limit_MB after
-  // PurgeObsoleteWALFiles()
-  // Set ttl and time_to_check_ to small values. Re-open db.
-  // Assert that there are no archived logs left.
-
-  std::string archive_dir = ArchivalDirectory(dbname_);
-  CreateArchiveLogs(20, 5000);
-
-  std::vector<std::uint64_t> log_files =
-      ListSpecificFiles(env_, archive_dir, kLogFile);
-  ASSERT_EQ(log_files.size(), 20U);
-
-  db_options_.WAL_size_limit_MB = 8;
-  Reopen();
-  wal_manager_->PurgeObsoleteWALFiles();
-
-  uint64_t archive_size = GetLogDirSize(archive_dir, env_);
-  ASSERT_TRUE(archive_size <= db_options_.WAL_size_limit_MB * 1024 * 1024);
-
-  db_options_.WAL_ttl_seconds = 1;
-  env_->SleepForMicroseconds(2 * 1000 * 1000);
-  Reopen();
-  wal_manager_->PurgeObsoleteWALFiles();
-
-  log_files = ListSpecificFiles(env_, archive_dir, kLogFile);
-  ASSERT_TRUE(log_files.empty());
-}
-
-TEST(WalManagerTest, WALArchivalTtl) {
-  db_options_.WAL_ttl_seconds = 1000;
-  Init();
-
-  // TEST : Create WalManager with a ttl and no size limit.
-  // Create some archived log files and call PurgeObsoleteWALFiles().
-  // Assert that files are not deleted
-  // Reopen db with small ttl.
-  // Assert that all archived logs was removed.
-
-  std::string archive_dir = ArchivalDirectory(dbname_);
-  CreateArchiveLogs(20, 5000);
-
-  std::vector<uint64_t> log_files =
-      ListSpecificFiles(env_, archive_dir, kLogFile);
-  ASSERT_GT(log_files.size(), 0U);
-
-  db_options_.WAL_ttl_seconds = 1;
-  env_->SleepForMicroseconds(3 * 1000 * 1000);
-  Reopen();
-  wal_manager_->PurgeObsoleteWALFiles();
-
-  log_files = ListSpecificFiles(env_, archive_dir, kLogFile);
-  ASSERT_TRUE(log_files.empty());
-}
-
-TEST(WalManagerTest, TransactionLogIteratorMoveOverZeroFiles) {
-  Init();
-  RollTheLog(false);
-  Put("key1", std::string(1024, 'a'));
-  // Create a zero record WAL file.
-  RollTheLog(false);
-  RollTheLog(false);
-
-  Put("key2", std::string(1024, 'a'));
-
-  auto iter = OpenTransactionLogIter(0);
-  ASSERT_EQ(2, CountRecords(iter.get()));
-}
-
-TEST(WalManagerTest, TransactionLogIteratorJustEmptyFile) {
-  Init();
-  RollTheLog(false);
-  auto iter = OpenTransactionLogIter(0);
-  // Check that an empty iterator is returned
-  ASSERT_TRUE(!iter->Valid());
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/write_batch.cc

@@ -48,6 +48,20 @@ WriteBatch::~WriteBatch() { }
 
 WriteBatch::Handler::~Handler() { }
 
+void WriteBatch::Handler::Put(const Slice& key, const Slice& value) {
+  // you need to either implement Put or PutCF
+  throw std::runtime_error("Handler::Put not implemented!");
+}
+
+void WriteBatch::Handler::Merge(const Slice& key, const Slice& value) {
+  throw std::runtime_error("Handler::Merge not implemented!");
+}
+
+void WriteBatch::Handler::Delete(const Slice& key) {
+  // you need to either implement Delete or DeleteCF
+  throw std::runtime_error("Handler::Delete not implemented!");
+}
+
 void WriteBatch::Handler::LogData(const Slice& blob) {
   // If the user has not specified something to do with blobs, then we ignore
   // them.
@@ -335,12 +349,13 @@ class MemTableInserter : public WriteBatch::Handler {
       return seek_status;
     }
     MemTable* mem = cf_mems_->GetMemTable();
+    auto* ioptions = mem->GetImmutableOptions();
     auto* moptions = mem->GetMemTableOptions();
     if (!moptions->inplace_update_support) {
       mem->Add(sequence_, kTypeValue, key, value);
     } else if (moptions->inplace_callback == nullptr) {
       mem->Update(sequence_, key, value);
-      RecordTick(moptions->statistics, NUMBER_KEYS_UPDATED);
+      RecordTick(ioptions->statistics, NUMBER_KEYS_UPDATED);
     } else {
       if (mem->UpdateCallback(sequence_, key, value)) {
       } else {
@@ -360,18 +375,18 @@ class MemTableInserter : public WriteBatch::Handler {
         Status s = db_->Get(ropts, cf_handle, key, &prev_value);
 
         char* prev_buffer = const_cast<char*>(prev_value.c_str());
-        uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
+        uint32_t prev_size = prev_value.size();
         auto status = moptions->inplace_callback(s.ok() ? prev_buffer : nullptr,
                                                  s.ok() ? &prev_size : nullptr,
                                                  value, &merged_value);
         if (status == UpdateStatus::UPDATED_INPLACE) {
           // prev_value is updated in-place with final value.
           mem->Add(sequence_, kTypeValue, key, Slice(prev_buffer, prev_size));
-          RecordTick(moptions->statistics, NUMBER_KEYS_WRITTEN);
+          RecordTick(ioptions->statistics, NUMBER_KEYS_WRITTEN);
         } else if (status == UpdateStatus::UPDATED) {
           // merged_value contains the final value.
           mem->Add(sequence_, kTypeValue, key, Slice(merged_value));
-          RecordTick(moptions->statistics, NUMBER_KEYS_WRITTEN);
+          RecordTick(ioptions->statistics, NUMBER_KEYS_WRITTEN);
         }
       }
     }
@@ -391,6 +406,7 @@ class MemTableInserter : public WriteBatch::Handler {
       return seek_status;
     }
     MemTable* mem = cf_mems_->GetMemTable();
+    auto* ioptions = mem->GetImmutableOptions();
     auto* moptions = mem->GetMemTableOptions();
     bool perform_merge = false;
 
@@ -425,16 +441,16 @@ class MemTableInserter : public WriteBatch::Handler {
       Slice get_value_slice = Slice(get_value);
 
       // 2) Apply this merge
-      auto merge_operator = moptions->merge_operator;
+      auto merge_operator = ioptions->merge_operator;
       assert(merge_operator);
 
       std::deque<std::string> operands;
       operands.push_front(value.ToString());
       std::string new_value;
       if (!merge_operator->FullMerge(key, &get_value_slice, operands,
-                                     &new_value, moptions->info_log)) {
+                                     &new_value, ioptions->info_log)) {
           // Failed to merge!
-        RecordTick(moptions->statistics, NUMBER_MERGE_FAILURES);
+        RecordTick(ioptions->statistics, NUMBER_MERGE_FAILURES);
 
         // Store the delta in memtable
         perform_merge = false;
@@ -461,6 +477,7 @@ class MemTableInserter : public WriteBatch::Handler {
       return seek_status;
     }
     MemTable* mem = cf_mems_->GetMemTable();
+    auto* ioptions = mem->GetImmutableOptions();
     auto* moptions = mem->GetMemTableOptions();
     if (!dont_filter_deletes_ && moptions->filter_deletes) {
       SnapshotImpl read_from_snapshot;
@@ -473,7 +490,7 @@ class MemTableInserter : public WriteBatch::Handler {
         cf_handle = db_->DefaultColumnFamily();
       }
       if (!db_->KeyMayExist(ropts, cf_handle, key, &value)) {
-        RecordTick(moptions->statistics, NUMBER_FILTERED_DELETES);
+        RecordTick(ioptions->statistics, NUMBER_FILTERED_DELETES);
         return Status::OK();
       }
     }

db/write_batch_internal.h

@@ -26,14 +26,15 @@ class ColumnFamilyMemTables {
   // been processed)
   virtual uint64_t GetLogNumber() const = 0;
   virtual MemTable* GetMemTable() const = 0;
+  virtual const Options* GetOptions() const = 0;
   virtual ColumnFamilyHandle* GetColumnFamilyHandle() = 0;
   virtual void CheckMemtableFull() = 0;
 };
 
 class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
  public:
-  explicit ColumnFamilyMemTablesDefault(MemTable* mem)
-      : ok_(false), mem_(mem) {}
+  ColumnFamilyMemTablesDefault(MemTable* mem, const Options* options)
+      : ok_(false), mem_(mem), options_(options) {}
 
   bool Seek(uint32_t column_family_id) override {
     ok_ = (column_family_id == 0);
@@ -47,6 +48,11 @@ class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
     return mem_;
   }
 
+  const Options* GetOptions() const override {
+    assert(ok_);
+    return options_;
+  }
+
   ColumnFamilyHandle* GetColumnFamilyHandle() override { return nullptr; }
 
   void CheckMemtableFull() override {}
@@ -54,6 +60,7 @@ class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
  private:
   bool ok_;
   MemTable* mem_;
+  const Options* const options_;
 };
 
 // WriteBatchInternal provides static methods for manipulating a

db/write_batch_test.cc

@@ -13,7 +13,6 @@
 #include "db/memtable.h"
 #include "db/column_family.h"
 #include "db/write_batch_internal.h"
-#include "db/writebuffer.h"
 #include "rocksdb/env.h"
 #include "rocksdb/memtablerep.h"
 #include "rocksdb/utilities/write_batch_with_index.h"
@@ -29,12 +28,11 @@ static std::string PrintContents(WriteBatch* b) {
   Options options;
   options.memtable_factory = factory;
   ImmutableCFOptions ioptions(options);
-  WriteBuffer wb(options.db_write_buffer_size);
   MemTable* mem = new MemTable(cmp, ioptions,
-                               MutableCFOptions(options, ioptions), &wb);
+      MemTableOptions(MutableCFOptions(options, ioptions), options));
   mem->Ref();
   std::string state;
-  ColumnFamilyMemTablesDefault cf_mems_default(mem);
+  ColumnFamilyMemTablesDefault cf_mems_default(mem, &options);
   Status s = WriteBatchInternal::InsertInto(b, &cf_mems_default);
   int count = 0;
   Arena arena;
@@ -157,7 +155,7 @@ namespace {
       if (column_family_id == 0) {
         seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
       } else {
-        seen += "PutCF(" + ToString(column_family_id) + ", " +
+        seen += "PutCF(" + std::to_string(column_family_id) + ", " +
                 key.ToString() + ", " + value.ToString() + ")";
       }
       return Status::OK();
@@ -167,7 +165,7 @@ namespace {
       if (column_family_id == 0) {
         seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
       } else {
-        seen += "MergeCF(" + ToString(column_family_id) + ", " +
+        seen += "MergeCF(" + std::to_string(column_family_id) + ", " +
                 key.ToString() + ", " + value.ToString() + ")";
       }
       return Status::OK();
@@ -179,7 +177,7 @@ namespace {
       if (column_family_id == 0) {
         seen += "Delete(" + key.ToString() + ")";
       } else {
-        seen += "DeleteCF(" + ToString(column_family_id) + ", " +
+        seen += "DeleteCF(" + std::to_string(column_family_id) + ", " +
                 key.ToString() + ")";
       }
       return Status::OK();
@@ -187,39 +185,6 @@ namespace {
   };
 }
 
-TEST(WriteBatchTest, MergeNotImplemented) {
-  WriteBatch batch;
-  batch.Merge(Slice("foo"), Slice("bar"));
-  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Merge(foo, bar)@0",
-            PrintContents(&batch));
-
-  WriteBatch::Handler handler;
-  ASSERT_OK(batch.Iterate(&handler));
-}
-
-TEST(WriteBatchTest, PutNotImplemented) {
-  WriteBatch batch;
-  batch.Put(Slice("k1"), Slice("v1"));
-  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Put(k1, v1)@0",
-            PrintContents(&batch));
-
-  WriteBatch::Handler handler;
-  ASSERT_OK(batch.Iterate(&handler));
-}
-
-TEST(WriteBatchTest, DeleteNotImplemented) {
-  WriteBatch batch;
-  batch.Delete(Slice("k2"));
-  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Delete(k2)@0",
-            PrintContents(&batch));
-
-  WriteBatch::Handler handler;
-  ASSERT_OK(batch.Iterate(&handler));
-}
-
 TEST(WriteBatchTest, Blob) {
   WriteBatch batch;
   batch.Put(Slice("k1"), Slice("v1"));

db/writebuffer.h

@@ -1,44 +0,0 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// WriteBuffer is for managing memory allocation for one or more MemTables.
-
-#pragma once
-
-namespace rocksdb {
-
-class WriteBuffer {
- public:
-  explicit WriteBuffer(size_t _buffer_size)
-    : buffer_size_(_buffer_size), memory_used_(0) {}
-
-  ~WriteBuffer() {}
-
-  size_t memory_usage() const { return memory_used_; }
-  size_t buffer_size() const { return buffer_size_; }
-
-  // Should only be called from write thread
-  bool ShouldFlush() const {
-    return buffer_size() > 0 && memory_usage() >= buffer_size();
-  }
-
-  // Should only be called from write thread
-  void ReserveMem(size_t mem) { memory_used_ += mem; }
-  void FreeMem(size_t mem) { memory_used_ -= mem; }
-
- private:
-  const size_t buffer_size_;
-  size_t memory_used_;
-
-  // No copying allowed
-  WriteBuffer(const WriteBuffer&);
-  void operator=(const WriteBuffer&);
-};
-
-}  // namespace rocksdb

examples/Makefile

@@ -1,17 +1,9 @@
 include ../build_config.mk
 
-.PHONY: clean
-
-all: simple_example column_families_example compact_files_example
+all: simple_example column_families_example
 
 simple_example: simple_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
 column_families_example: column_families_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
-
-compact_files_example: compact_files_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
-
-clean: simple_example column_families_example compact_files_example
-	rm -rf ./simple_example ./column_families_example ./compact_files_example

examples/compact_files_example.cc

@@ -1,175 +0,0 @@
-// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-//
-// An example code demonstrating how to use CompactFiles, EventListener,
-// and GetColumnFamilyMetaData APIs to implement custom compaction algorithm.
-
-#include <mutex>
-#include <string>
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/options.h"
-
-using namespace rocksdb;
-std::string kDBPath = "/tmp/rocksdb_compact_files_example";
-class CompactionTask;
-
-// This is an example interface of external-compaction algorithm.
-// Compaction algorithm can be implemented outside the core-RocksDB
-// code by using the pluggable compaction APIs that RocksDb provides.
-class Compactor : public EventListener {
- public:
-  // Picks and returns a compaction task given the specified DB
-  // and column family.  It is the caller's responsibility to
-  // destroy the returned CompactionTask.  Returns "nullptr"
-  // if it cannot find a proper compaction task.
-  virtual CompactionTask* PickCompaction(
-      DB* db, const std::string& cf_name) = 0;
-
-  // Schedule and run the specified compaction task in background.
-  virtual void ScheduleCompaction(CompactionTask *task) = 0;
-};
-
-// Example structure that describes a compaction task.
-struct CompactionTask {
-  CompactionTask(
-      DB* db, Compactor* compactor,
-      const std::string& column_family_name,
-      const std::vector<std::string>& input_file_names,
-      const int output_level,
-      const CompactionOptions& compact_options,
-      bool retry_on_fail)
-          : db(db),
-            compactor(compactor),
-            column_family_name(column_family_name),
-            input_file_names(input_file_names),
-            output_level(output_level),
-            compact_options(compact_options),
-            retry_on_fail(false) {}
-  DB* db;
-  Compactor* compactor;
-  const std::string& column_family_name;
-  std::vector<std::string> input_file_names;
-  int output_level;
-  CompactionOptions compact_options;
-  bool retry_on_fail;
-};
-
-// A simple compaction algorithm that always compacts everything
-// to the highest level whenever possible.
-class FullCompactor : public Compactor {
- public:
-  explicit FullCompactor(const Options options) : options_(options) {
-    compact_options_.compression = options_.compression;
-    compact_options_.output_file_size_limit =
-        options_.target_file_size_base;
-  }
-
-  // When flush happens, it determins whether to trigger compaction.
-  // If triggered_writes_stop is true, it will also set the retry
-  // flag of compaction-task to true.
-  void OnFlushCompleted(
-      DB* db, const std::string& cf_name,
-      const std::string& file_path,
-      bool triggered_writes_slowdown,
-      bool triggered_writes_stop) override {
-    CompactionTask* task = PickCompaction(db, cf_name);
-    if (task != nullptr) {
-      if (triggered_writes_stop) {
-        task->retry_on_fail = true;
-      }
-      // Schedule compaction in a different thread.
-      ScheduleCompaction(task);
-    }
-  }
-
-  // Always pick a compaction which includes all files whenever possible.
-  CompactionTask* PickCompaction(
-      DB* db, const std::string& cf_name) override {
-    ColumnFamilyMetaData cf_meta;
-    db->GetColumnFamilyMetaData(&cf_meta);
-
-    std::vector<std::string> input_file_names;
-    for (auto level : cf_meta.levels) {
-      for (auto file : level.files) {
-        if (file.being_compacted) {
-          return nullptr;
-        }
-        input_file_names.push_back(file.name);
-      }
-    }
-    return new CompactionTask(
-        db, this, cf_name, input_file_names,
-        options_.num_levels - 1, compact_options_, false);
-  }
-
-  // Schedule the specified compaction task in background.
-  void ScheduleCompaction(CompactionTask* task) override {
-    options_.env->Schedule(&FullCompactor::CompactFiles, task);
-  }
-
-  static void CompactFiles(void* arg) {
-    CompactionTask* task = reinterpret_cast<CompactionTask*>(arg);
-    assert(task);
-    assert(task->db);
-    Status s = task->db->CompactFiles(
-        task->compact_options,
-        task->input_file_names,
-        task->output_level);
-    printf("CompactFiles() finished with status %s\n", s.ToString().c_str());
-    if (!s.ok() && !s.IsIOError() && task->retry_on_fail) {
-      // If a compaction task with its retry_on_fail=true failed,
-      // try to schedule another compaction in case the reason
-      // is not an IO error.
-      CompactionTask* new_task = task->compactor->PickCompaction(
-          task->db, task->column_family_name);
-      task->compactor->ScheduleCompaction(new_task);
-    }
-    // release the task
-    delete task;
-  }
-
- private:
-  Options options_;
-  CompactionOptions compact_options_;
-};
-
-int main() {
-  Options options;
-  options.create_if_missing = true;
-  // Disable RocksDB background compaction.
-  options.compaction_style = kCompactionStyleNone;
-  // Small slowdown and stop trigger for experimental purpose.
-  options.level0_slowdown_writes_trigger = 3;
-  options.level0_stop_writes_trigger = 5;
-  options.IncreaseParallelism(5);
-  options.listeners.emplace_back(new FullCompactor(options));
-
-  DB* db = nullptr;
-  DestroyDB(kDBPath, options);
-  Status s = DB::Open(options, kDBPath, &db);
-  assert(s.ok());
-  assert(db);
-
-  // if background compaction is not working, write will stall
-  // because of options.level0_stop_writes_trigger
-  for (int i = 1000; i < 99999; ++i) {
-    db->Put(WriteOptions(), ToString(i),
-                            std::string(500, 'a' + (i % 26)));
-  }
-
-  // verify the values are still there
-  std::string value;
-  for (int i = 1000; i < 99999; ++i) {
-    db->Get(ReadOptions(), ToString(i),
-                           &value);
-    assert(value == std::string(500, 'a' + (i % 26)));
-  }
-
-  // close the db.
-  delete db;
-
-  return 0;
-}

hdfs/env_hdfs.h

@@ -93,8 +93,6 @@ class HdfsEnv : public Env {
 
   virtual Status RenameFile(const std::string& src, const std::string& target);
 
-  virtual Status LinkFile(const std::string& src, const std::string& target);
-
   virtual Status LockFile(const std::string& fname, FileLock** lock);
 
   virtual Status UnlockFile(FileLock* lock);
@@ -147,10 +145,6 @@ class HdfsEnv : public Env {
     posixEnv->SetBackgroundThreads(number, pri);
   }
 
-  virtual void IncBackgroundThreadsIfNeeded(int number, Priority pri) override {
-    posixEnv->IncBackgroundThreadsIfNeeded(number, pri);
-  }
-
   virtual std::string TimeToString(uint64_t number) {
     return posixEnv->TimeToString(number);
   }
@@ -238,7 +232,7 @@ class HdfsEnv : public Env {
   explicit HdfsEnv(const std::string& fsname) {
     fprintf(stderr, "You have not build rocksdb with HDFS support\n");
     fprintf(stderr, "Please see hdfs/README for details\n");
-    abort();
+    throw std::exception();
   }
 
   virtual ~HdfsEnv() {
@@ -293,10 +287,6 @@ class HdfsEnv : public Env {
 
   virtual Status RenameFile(const std::string& src, const std::string& target){return notsup;}
 
-  virtual Status LinkFile(const std::string& src, const std::string& target) {
-    return notsup;
-  }
-
   virtual Status LockFile(const std::string& fname, FileLock** lock){return notsup;}
 
   virtual Status UnlockFile(FileLock* lock){return notsup;}
@@ -329,7 +319,7 @@ class HdfsEnv : public Env {
       std::string* outputpath) {return notsup;}
 
   virtual void SetBackgroundThreads(int number, Priority pri = LOW) {}
-  virtual void IncBackgroundThreadsIfNeeded(int number, Priority pri) {}
+
   virtual std::string TimeToString(uint64_t number) { return "";}
 };
 }

helpers/memenv/memenv.cc

@@ -351,14 +351,15 @@ class InMemoryEnv : public EnvWrapper {
     return Status::NotSupported("getFileMTime", "Not supported in MemEnv");
   }
 
-  virtual Status RenameFile(const std::string& src, const std::string& dest) {
+  virtual Status RenameFile(const std::string& src,
+                            const std::string& target) {
     MutexLock lock(&mutex_);
     if (file_map_.find(src) == file_map_.end()) {
       return Status::IOError(src, "File not found");
     }
 
-    DeleteFileInternal(dest);
-    file_map_[dest] = file_map_[src];
+    DeleteFileInternal(target);
+    file_map_[target] = file_map_[src];
     file_map_.erase(src);
     return Status::OK();
   }

include/rocksdb/c.h

@@ -77,8 +77,6 @@ typedef struct rocksdb_mergeoperator_t   rocksdb_mergeoperator_t;
 typedef struct rocksdb_options_t         rocksdb_options_t;
 typedef struct rocksdb_block_based_table_options_t
     rocksdb_block_based_table_options_t;
-typedef struct rocksdb_cuckoo_table_options_t
-    rocksdb_cuckoo_table_options_t;
 typedef struct rocksdb_randomfile_t      rocksdb_randomfile_t;
 typedef struct rocksdb_readoptions_t     rocksdb_readoptions_t;
 typedef struct rocksdb_seqfile_t         rocksdb_seqfile_t;
@@ -378,25 +376,6 @@ extern void rocksdb_block_based_options_set_whole_key_filtering(
 extern void rocksdb_options_set_block_based_table_factory(
     rocksdb_options_t *opt, rocksdb_block_based_table_options_t* table_options);
 
-/* Cuckoo table options */
-
-extern rocksdb_cuckoo_table_options_t*
-    rocksdb_cuckoo_options_create();
-extern void rocksdb_cuckoo_options_destroy(
-    rocksdb_cuckoo_table_options_t* options);
-extern void rocksdb_cuckoo_options_set_hash_ratio(
-    rocksdb_cuckoo_table_options_t* options, double v);
-extern void rocksdb_cuckoo_options_set_max_search_depth(
-    rocksdb_cuckoo_table_options_t* options, uint32_t v);
-extern void rocksdb_cuckoo_options_set_cuckoo_block_size(
-    rocksdb_cuckoo_table_options_t* options, uint32_t v);
-extern void rocksdb_cuckoo_options_set_identity_as_first_hash(
-    rocksdb_cuckoo_table_options_t* options, unsigned char v);
-extern void rocksdb_cuckoo_options_set_use_module_hash(
-    rocksdb_cuckoo_table_options_t* options, unsigned char v);
-extern void rocksdb_options_set_cuckoo_table_factory(
-    rocksdb_options_t *opt, rocksdb_cuckoo_table_options_t* table_options);
-
 /* Options */
 
 extern rocksdb_options_t* rocksdb_options_create();
@@ -440,7 +419,6 @@ extern void rocksdb_options_set_info_log(rocksdb_options_t*, rocksdb_logger_t*);
 extern void rocksdb_options_set_info_log_level(rocksdb_options_t*, int);
 extern void rocksdb_options_set_write_buffer_size(rocksdb_options_t*, size_t);
 extern void rocksdb_options_set_max_open_files(rocksdb_options_t*, int);
-extern void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n);
 extern void rocksdb_options_set_compression_options(
     rocksdb_options_t*, int, int, int);
 extern void rocksdb_options_set_prefix_extractor(

include/rocksdb/comparator.h

@@ -63,7 +63,7 @@ class Comparator {
 extern const Comparator* BytewiseComparator();
 
 // Return a builtin comparator that uses reverse lexicographic byte-wise
-// ordering.
+// ordering. 
 extern const Comparator* ReverseBytewiseComparator();
 
 }  // namespace rocksdb

include/rocksdb/db.h

@@ -15,34 +15,19 @@
 #include <vector>
 #include <string>
 #include <unordered_map>
-#include "rocksdb/metadata.h"
 #include "rocksdb/version.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
 #include "rocksdb/types.h"
 #include "rocksdb/transaction_log.h"
-#include "rocksdb/listener.h"
 
 namespace rocksdb {
 
-struct Options;
-struct DBOptions;
-struct ColumnFamilyOptions;
-struct ReadOptions;
-struct WriteOptions;
-struct FlushOptions;
-struct CompactionOptions;
-struct TableProperties;
-class WriteBatch;
-class Env;
-class EventListener;
-
 using std::unique_ptr;
 
 class ColumnFamilyHandle {
  public:
   virtual ~ColumnFamilyHandle() {}
-  virtual const std::string& GetName() const = 0;
 };
 extern const std::string kDefaultColumnFamilyName;
 
@@ -59,6 +44,27 @@ struct ColumnFamilyDescriptor {
 static const int kMajorVersion = __ROCKSDB_MAJOR__;
 static const int kMinorVersion = __ROCKSDB_MINOR__;
 
+struct Options;
+struct ReadOptions;
+struct WriteOptions;
+struct FlushOptions;
+struct TableProperties;
+class WriteBatch;
+class Env;
+
+// Metadata associated with each SST file.
+struct LiveFileMetaData {
+  std::string column_family_name;  // Name of the column family
+  std::string db_path;
+  std::string name;                // Name of the file
+  int level;               // Level at which this file resides.
+  size_t size;             // File size in bytes.
+  std::string smallestkey; // Smallest user defined key in the file.
+  std::string largestkey;  // Largest user defined key in the file.
+  SequenceNumber smallest_seqno; // smallest seqno in file
+  SequenceNumber largest_seqno;  // largest seqno in file
+};
+
 // Abstract handle to particular state of a DB.
 // A Snapshot is an immutable object and can therefore be safely
 // accessed from multiple threads without any external synchronization.
@@ -100,9 +106,6 @@ class DB {
   // that modify data, like put/delete, will return error.
   // If the db is opened in read only mode, then no compactions
   // will happen.
-  //
-  // Not supported in ROCKSDB_LITE, in which case the function will
-  // return Status::NotSupported.
   static Status OpenForReadOnly(const Options& options,
       const std::string& name, DB** dbptr,
       bool error_if_log_file_exist = false);
@@ -112,9 +115,6 @@ class DB {
   // database that should be opened. However, you always need to specify default
   // column family. The default column family name is 'default' and it's stored
   // in rocksdb::kDefaultColumnFamilyName
-  //
-  // Not supported in ROCKSDB_LITE, in which case the function will
-  // return Status::NotSupported.
   static Status OpenForReadOnly(
       const DBOptions& db_options, const std::string& name,
       const std::vector<ColumnFamilyDescriptor>& column_families,
@@ -195,8 +195,6 @@ class DB {
   }
 
   // Apply the specified updates to the database.
-  // If `updates` contains no update, WAL will still be synced if
-  // options.sync=true.
   // Returns OK on success, non-OK on failure.
   // Note: consider setting options.sync = true.
   virtual Status Write(const WriteOptions& options, WriteBatch* updates) = 0;
@@ -361,35 +359,15 @@ class DB {
     return CompactRange(DefaultColumnFamily(), begin, end, reduce_level,
                         target_level, target_path_id);
   }
-  virtual Status SetOptions(ColumnFamilyHandle* column_family,
+  virtual bool SetOptions(ColumnFamilyHandle* column_family,
       const std::unordered_map<std::string, std::string>& new_options) {
-    return Status::NotSupported("Not implemented");
+    return true;
   }
-  virtual Status SetOptions(
+  virtual bool SetOptions(
       const std::unordered_map<std::string, std::string>& new_options) {
     return SetOptions(DefaultColumnFamily(), new_options);
   }
 
-  // CompactFiles() inputs a list of files specified by file numbers
-  // and compacts them to the specified level.  Note that the behavior
-  // is different from CompactRange in that CompactFiles() will
-  // perform the compaction job using the CURRENT thread.
-  //
-  // @see GetDataBaseMetaData
-  // @see GetColumnFamilyMetaData
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1) = 0;
-
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1) {
-    return CompactFiles(compact_options, DefaultColumnFamily(),
-                        input_file_names, output_level, output_path_id);
-  }
   // Number of levels used for this DB.
   virtual int NumberLevels(ColumnFamilyHandle* column_family) = 0;
   virtual int NumberLevels() { return NumberLevels(DefaultColumnFamily()); }
@@ -496,21 +474,6 @@ class DB {
   // and end key
   virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {}
 
-  // Obtains the meta data of the specified column family of the DB.
-  // Status::NotFound() will be returned if the current DB does not have
-  // any column family match the specified name.
-  //
-  // If cf_name is not specified, then the metadata of the default
-  // column family will be returned.
-  virtual void GetColumnFamilyMetaData(
-      ColumnFamilyHandle* column_family,
-      ColumnFamilyMetaData* metadata) {}
-
-  // Get the metadata of the default column family.
-  void GetColumnFamilyMetaData(
-      ColumnFamilyMetaData* metadata) {
-    GetColumnFamilyMetaData(DefaultColumnFamily(), metadata);
-  }
 #endif  // ROCKSDB_LITE
 
   // Sets the globally unique ID created at database creation time by invoking
@@ -547,7 +510,6 @@ Status DestroyDB(const std::string& name, const Options& options);
 Status RepairDB(const std::string& dbname, const Options& options);
 #endif
 
-
 }  // namespace rocksdb
 
 #endif  // STORAGE_ROCKSDB_INCLUDE_DB_H_

include/rocksdb/env.h

@@ -20,7 +20,6 @@
 #include <cstdarg>
 #include <string>
 #include <memory>
-#include <limits>
 #include <vector>
 #include <stdint.h>
 #include "rocksdb/status.h"
@@ -178,11 +177,6 @@ class Env {
   virtual Status RenameFile(const std::string& src,
                             const std::string& target) = 0;
 
-  // Hard Link file src to target.
-  virtual Status LinkFile(const std::string& src, const std::string& target) {
-    return Status::NotSupported("LinkFile is not supported for this Env");
-  }
-
   // Lock the specified file.  Used to prevent concurrent access to
   // the same db by multiple processes.  On failure, stores nullptr in
   // *lock and returns non-OK.
@@ -278,11 +272,6 @@ class Env {
   // default number: 1
   virtual void SetBackgroundThreads(int number, Priority pri = LOW) = 0;
 
-  // Enlarge number of background worker threads of a specific thread pool
-  // for this environment if it is smaller than specified. 'LOW' is the default
-  // pool.
-  virtual void IncBackgroundThreadsIfNeeded(int number, Priority pri) = 0;
-
   // Lower IO priority for threads from the specified pool.
   virtual void LowerThreadPoolIOPriority(Priority pool = LOW) {}
 
@@ -482,8 +471,8 @@ class WritableFile {
     if (new_last_preallocated_block > last_preallocated_block_) {
       size_t num_spanned_blocks =
         new_last_preallocated_block - last_preallocated_block_;
-      Allocate(static_cast<off_t>(block_size * last_preallocated_block_),
-               static_cast<off_t>(block_size * num_spanned_blocks));
+      Allocate(block_size * last_preallocated_block_,
+               block_size * num_spanned_blocks);
       last_preallocated_block_ = new_last_preallocated_block;
     }
   }
@@ -586,8 +575,7 @@ enum InfoLogLevel : unsigned char {
 // An interface for writing log messages.
 class Logger {
  public:
-  size_t kDoNotSupportGetLogFileSize = std::numeric_limits<size_t>::max();
-
+  enum { DO_NOT_SUPPORT_GET_LOG_FILE_SIZE = -1 };
   explicit Logger(const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL)
       : log_level_(log_level) {}
   virtual ~Logger();
@@ -620,7 +608,9 @@ class Logger {
       Logv(new_format, ap);
     }
   }
-  virtual size_t GetLogFileSize() const { return kDoNotSupportGetLogFileSize; }
+  virtual size_t GetLogFileSize() const {
+    return DO_NOT_SUPPORT_GET_LOG_FILE_SIZE;
+  }
   // Flush to the OS buffers
   virtual void Flush() {}
   virtual InfoLogLevel GetInfoLogLevel() const { return log_level_; }
@@ -752,11 +742,6 @@ class EnvWrapper : public Env {
   Status RenameFile(const std::string& s, const std::string& t) {
     return target_->RenameFile(s, t);
   }
-
-  Status LinkFile(const std::string& s, const std::string& t) {
-    return target_->LinkFile(s, t);
-  }
-
   Status LockFile(const std::string& f, FileLock** l) {
     return target_->LockFile(f, l);
   }
@@ -797,11 +782,6 @@ class EnvWrapper : public Env {
   void SetBackgroundThreads(int num, Priority pri) {
     return target_->SetBackgroundThreads(num, pri);
   }
-
-  void IncBackgroundThreadsIfNeeded(int num, Priority pri) {
-    return target_->IncBackgroundThreadsIfNeeded(num, pri);
-  }
-
   void LowerThreadPoolIOPriority(Priority pool = LOW) override {
     target_->LowerThreadPoolIOPriority(pool);
   }