One more include<functional>

.travis.yml

@@ -14,6 +14,6 @@ before_install:
  - sudo dpkg -i libgflags-dev_2.0-1_amd64.deb
 # Lousy hack to disable use and testing of fallocate, which doesn't behave quite
 # as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
-script: OPT=-DTRAVIS make unity && make clean && OPT=-DTRAVIS make check
+script: OPT=-DTRAVIS make unity && make clean && OPT="-DTRAVIS -Wshadow" make check -j8
 notifications:
     email: false

HISTORY.md

@@ -1,30 +1,7 @@
 # Rocksdb Change Log
 
-### 3.9.0 (12/8/2014)
+## Unreleased
 
-### Public API changes
-* New API to create a checkpoint added. Given a directory name, creates a new
-  database which is an image of the existing database.
-* New API LinkFile added to Env. If you implement your own Env class, an
-  implementation of the API LinkFile will have to be provided.
-* MemTableRep takes MemTableAllocator instead of Arena
-
-### Improvements
-* RocksDBLite library now becomes smaller and will be compiled with -fno-exceptions flag.
-
-## 3.8.0 (11/14/2014)
-
-### Public API changes
-* BackupEngine::NewBackupEngine() was deprecated; please use BackupEngine::Open() from now on.
-* BackupableDB/RestoreBackupableDB have new GarbageCollect() methods, which will clean up files from corrupt and obsolete backups.
-* BackupableDB/RestoreBackupableDB have new GetCorruptedBackups() methods which list corrupt backups.
-
-### Cleanup
-* Bunch of code cleanup, some extra warnings turned on (-Wshadow, -Wshorten-64-to-32, -Wnon-virtual-dtor)
-
-### New features
-* CompactFiles and EventListener, although they are still in experimental state
-* Full ColumnFamily support in RocksJava.
 
 ## 3.7.0 (11/6/2014)
 ### Public API changes

Makefile

@@ -6,16 +6,8 @@
 
 #-----------------------------------------------
 
-CFLAGS += ${EXTRA_CFLAGS}
-CXXFLAGS += ${EXTRA_CXXFLAGS}
-LDFLAGS += $(EXTRA_LDFLAGS)
-MACHINE ?= $(shell uname -m)
-
 ifneq ($(MAKECMDGOALS),dbg)
-OPT += -O2 -fno-omit-frame-pointer
-ifneq ($(MACHINE),ppc64) # ppc64 doesn't support -momit-leaf-frame-pointer
-OPT += -momit-leaf-frame-pointer
-endif
+OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
 else
 # intentionally left blank
 endif
@@ -43,12 +35,6 @@ else
 OPT += -DNDEBUG
 endif
 
-ifneq ($(filter -DROCKSDB_LITE,$(OPT)),) 
-	# found
-	CFLAGS += -fno-exceptions
-	CXXFLAGS += -fno-exceptions
-endif
-
 # ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc.
 ifdef COMPILE_WITH_ASAN
 	# ASAN compile flags
@@ -84,7 +70,7 @@ install:
 #-------------------------------------------------
 
 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
-CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor
+CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual
 
 LDFLAGS += $(PLATFORM_LDFLAGS)
 
@@ -160,12 +146,9 @@ TESTS = \
 	cuckoo_table_builder_test \
 	cuckoo_table_reader_test \
 	cuckoo_table_db_test \
+	write_batch_with_index_test \
 	flush_job_test \
-	wal_manager_test \
-	listener_test \
-	compaction_job_test
-
-SUBSET :=  $(shell echo $(TESTS) |sed s/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/)
+	wal_manager_test
 
 TOOLS = \
         sst_dump \
@@ -253,10 +236,6 @@ check: $(TESTS) ldb
 	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
 	python tools/ldb_test.py
 
-check_some: $(SUBSET) ldb
-	for t in $(SUBSET); do echo "***** Running $$t"; ./$$t || exit 1; done
-	python tools/ldb_test.py
-
 ldb_tests: ldb
 	python tools/ldb_test.py
 
@@ -444,9 +423,6 @@ write_batch_with_index_test: utilities/write_batch_with_index/write_batch_with_i
 flush_job_test: db/flush_job_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/flush_job_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
-compaction_job_test: db/compaction_job_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/compaction_job_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
-
 wal_manager_test: db/wal_manager_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/wal_manager_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -525,15 +501,6 @@ cuckoo_table_reader_test: table/cuckoo_table_reader_test.o $(LIBOBJECTS) $(TESTH
 cuckoo_table_db_test: db/cuckoo_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/cuckoo_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
-listener_test: db/listener_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/listener_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
-thread_list_test: util/thread_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/thread_list_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
-compactor_test: utilities/compaction/compactor_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) utilities/compaction/compactor_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
-
 options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/options_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -616,10 +583,9 @@ libsnappy.a:
 
 rocksdbjavastatic: libz.a libbz2.a libsnappy.a
 	OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j
-	cd java;$(MAKE) javalib;
+	cd java;$(MAKE) java;
 	rm -f ./java/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(LIBOBJECTS) $(COVERAGEFLAGS) libz.a libbz2.a libsnappy.a
-	cd java;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class HISTORY*.md $(ROCKSDBJNILIB)
 	cd java/javadoc;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) *
 	cd java;jar -cf $(ROCKSDB_SOURCES_JAR) org
@@ -638,7 +604,7 @@ rocksdbjavastaticpublish: rocksdbjavastaticrelease
 
 rocksdbjava:
 	OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j32
-	cd java;$(MAKE) javalib;
+	cd java;$(MAKE) java;
 	rm -f ./java/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(LIBOBJECTS) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
 	cd java;jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class HISTORY*.md $(ROCKSDBJNILIB)

build_tools/build_detect_platform

@@ -284,14 +284,6 @@ EOF
     fi
 fi
 
-# Test whether -Wshorten-64-to-32 is available
-$CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
-  int main() {}
-EOF
-if [ "$?" = 0 ]; then
-    COMMON_FLAGS="$COMMON_FLAGS -Wshorten-64-to-32"
-fi
-
 # shall we use HDFS?
 
 if test "$USE_HDFS"; then

db/c.cc

@@ -56,7 +56,6 @@ using rocksdb::NewBloomFilterPolicy;
 using rocksdb::NewLRUCache;
 using rocksdb::Options;
 using rocksdb::BlockBasedTableOptions;
-using rocksdb::CuckooTableOptions;
 using rocksdb::RandomAccessFile;
 using rocksdb::Range;
 using rocksdb::ReadOptions;
@@ -80,14 +79,10 @@ struct rocksdb_writebatch_t      { WriteBatch        rep; };
 struct rocksdb_snapshot_t        { const Snapshot*   rep; };
 struct rocksdb_flushoptions_t    { FlushOptions      rep; };
 struct rocksdb_fifo_compaction_options_t { CompactionOptionsFIFO rep; };
-struct rocksdb_readoptions_t {
-   ReadOptions rep;
-   Slice upper_bound; // stack variable to set pointer to in ReadOptions
-};
+struct rocksdb_readoptions_t     { ReadOptions       rep; };
 struct rocksdb_writeoptions_t    { WriteOptions      rep; };
 struct rocksdb_options_t         { Options           rep; };
 struct rocksdb_block_based_table_options_t  { BlockBasedTableOptions rep; };
-struct rocksdb_cuckoo_table_options_t  { CuckooTableOptions rep; };
 struct rocksdb_seqfile_t         { SequentialFile*   rep; };
 struct rocksdb_randomfile_t      { RandomAccessFile* rep; };
 struct rocksdb_writablefile_t    { WritableFile*     rep; };
@@ -390,9 +385,11 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
     unsigned char success;
     size_t new_value_len;
     char* tmp_new_value = (*full_merge_)(
-        state_, key.data(), key.size(), existing_value_data, existing_value_len,
-        &operand_pointers[0], &operand_sizes[0], static_cast<int>(n), &success,
-        &new_value_len);
+        state_,
+        key.data(), key.size(),
+        existing_value_data, existing_value_len,
+        &operand_pointers[0], &operand_sizes[0], n,
+        &success, &new_value_len);
     new_value->assign(tmp_new_value, new_value_len);
 
     if (delete_value_ != nullptr) {
@@ -420,7 +417,7 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
     size_t new_value_len;
     char* tmp_new_value = (*partial_merge_)(
         state_, key.data(), key.size(), &operand_pointers[0], &operand_sizes[0],
-        static_cast<int>(operand_count), &success, &new_value_len);
+        operand_count, &success, &new_value_len);
     new_value->assign(tmp_new_value, new_value_len);
 
     if (delete_value_ != nullptr) {
@@ -1126,51 +1123,6 @@ void rocksdb_options_set_block_based_table_factory(
 }
 
 
-rocksdb_cuckoo_table_options_t*
-rocksdb_cuckoo_options_create() {
-  return new rocksdb_cuckoo_table_options_t;
-}
-
-void rocksdb_cuckoo_options_destroy(
-    rocksdb_cuckoo_table_options_t* options) {
-  delete options;
-}
-
-void rocksdb_cuckoo_options_set_hash_ratio(
-    rocksdb_cuckoo_table_options_t* options, double v) {
-  options->rep.hash_table_ratio = v;
-}
-
-void rocksdb_cuckoo_options_set_max_search_depth(
-    rocksdb_cuckoo_table_options_t* options, uint32_t v) {
-  options->rep.max_search_depth = v;
-}
-
-void rocksdb_cuckoo_options_set_cuckoo_block_size(
-    rocksdb_cuckoo_table_options_t* options, uint32_t v) {
-  options->rep.cuckoo_block_size = v;
-}
-
-void rocksdb_cuckoo_options_set_identity_as_first_hash(
-    rocksdb_cuckoo_table_options_t* options, unsigned char v) {
-  options->rep.identity_as_first_hash = v;
-}
-
-void rocksdb_cuckoo_options_set_use_module_hash(
-    rocksdb_cuckoo_table_options_t* options, unsigned char v) {
-  options->rep.use_module_hash = v;
-}
-
-void rocksdb_options_set_cuckoo_table_factory(
-    rocksdb_options_t *opt,
-    rocksdb_cuckoo_table_options_t* table_options) {
-  if (table_options) {
-    opt->rep.table_factory.reset(
-        rocksdb::NewCuckooTableFactory(table_options->rep));
-  }
-}
-
-
 rocksdb_options_t* rocksdb_options_create() {
   return new rocksdb_options_t;
 }
@@ -1264,11 +1216,6 @@ void rocksdb_options_set_info_log_level(
   opt->rep.info_log_level = static_cast<InfoLogLevel>(v);
 }
 
-void rocksdb_options_set_db_write_buffer_size(rocksdb_options_t* opt,
-                                              size_t s) {
-  opt->rep.db_write_buffer_size = s;
-}
-
 void rocksdb_options_set_write_buffer_size(rocksdb_options_t* opt, size_t s) {
   opt->rep.write_buffer_size = s;
 }
@@ -1277,10 +1224,6 @@ void rocksdb_options_set_max_open_files(rocksdb_options_t* opt, int n) {
   opt->rep.max_open_files = n;
 }
 
-void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n) {
-  opt->rep.max_total_wal_size = n;
-}
-
 void rocksdb_options_set_target_file_size_base(
     rocksdb_options_t* opt, uint64_t n) {
   opt->rep.target_file_size_base = n;
@@ -1899,14 +1842,8 @@ void rocksdb_readoptions_set_snapshot(
 void rocksdb_readoptions_set_iterate_upper_bound(
     rocksdb_readoptions_t* opt,
     const char* key, size_t keylen) {
-  if (key == nullptr) {
-    opt->upper_bound = Slice();
-    opt->rep.iterate_upper_bound = nullptr;
-
-  } else {
-    opt->upper_bound = Slice(key, keylen);
-    opt->rep.iterate_upper_bound = &opt->upper_bound;
-  }
+  Slice prefix = Slice(key, keylen);
+  opt->rep.iterate_upper_bound = &prefix;
 }
 
 void rocksdb_readoptions_set_read_tier(
@@ -2104,7 +2041,7 @@ void rocksdb_options_set_min_level_to_compress(rocksdb_options_t* opt, int level
 
 int rocksdb_livefiles_count(
   const rocksdb_livefiles_t* lf) {
-  return static_cast<int>(lf->rep.size());
+  return lf->rep.size();
 }
 
 const char* rocksdb_livefiles_name(

db/c_test.c

@@ -132,7 +132,7 @@ static void CmpDestroy(void* arg) { }
 
 static int CmpCompare(void* arg, const char* a, size_t alen,
                       const char* b, size_t blen) {
-  size_t n = (alen < blen) ? alen : blen;
+  int n = (alen < blen) ? alen : blen;
   int r = memcmp(a, b, n);
   if (r == 0) {
     if (alen < blen) r = -1;
@@ -801,87 +801,8 @@ int main(int argc, char** argv) {
     rocksdb_iter_get_error(iter, &err);
     CheckNoError(err);
     rocksdb_iter_destroy(iter);
-
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
   }
 
-  StartPhase("cuckoo_options");
-  {
-    rocksdb_cuckoo_table_options_t* cuckoo_options;
-    cuckoo_options = rocksdb_cuckoo_options_create();
-    rocksdb_cuckoo_options_set_hash_ratio(cuckoo_options, 0.5);
-    rocksdb_cuckoo_options_set_max_search_depth(cuckoo_options, 200);
-    rocksdb_cuckoo_options_set_cuckoo_block_size(cuckoo_options, 10);
-    rocksdb_cuckoo_options_set_identity_as_first_hash(cuckoo_options, 1);
-    rocksdb_cuckoo_options_set_use_module_hash(cuckoo_options, 0);
-    rocksdb_options_set_cuckoo_table_factory(options, cuckoo_options);
-
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-
-    rocksdb_cuckoo_options_destroy(cuckoo_options);
-  }
-
-  StartPhase("iterate_upper_bound");
-  {
-    // Create new empty database
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
-
-    rocksdb_options_set_prefix_extractor(options, NULL);
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-
-    rocksdb_put(db, woptions, "a",    1, "0",    1, &err); CheckNoError(err);
-    rocksdb_put(db, woptions, "foo",  3, "bar",  3, &err); CheckNoError(err);
-    rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err); CheckNoError(err);
-    rocksdb_put(db, woptions, "g1",   2, "0",    1, &err); CheckNoError(err);
-
-    // testing basic case with no iterate_upper_bound and no prefix_extractor
-    {
-       rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
-       rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
-
-       rocksdb_iter_seek(iter, "foo", 3);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "foo", "bar");
-
-       rocksdb_iter_next(iter);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "foo1", "bar1");
-
-       rocksdb_iter_next(iter);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "g1", "0");
-
-       rocksdb_iter_destroy(iter);
-    }
-
-    // testing iterate_upper_bound and forward iterator
-    // to make sure it stops at bound
-    {
-       // iterate_upper_bound points beyond the last expected entry
-       rocksdb_readoptions_set_iterate_upper_bound(roptions, "foo2", 4);
-
-       rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
-
-       rocksdb_iter_seek(iter, "foo", 3);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "foo", "bar");
-
-       rocksdb_iter_next(iter);
-       CheckCondition(rocksdb_iter_valid(iter));
-       CheckIter(iter, "foo1", "bar1");
-
-       rocksdb_iter_next(iter);
-       // should stop here...
-       CheckCondition(!rocksdb_iter_valid(iter));
-
-       rocksdb_iter_destroy(iter);
-    }
-  }
 
   StartPhase("cleanup");
   rocksdb_close(db);

db/column_family.cc

@@ -19,15 +19,12 @@
 #include <algorithm>
 #include <limits>
 
-#include "db/compaction_picker.h"
 #include "db/db_impl.h"
 #include "db/job_context.h"
 #include "db/version_set.h"
-#include "db/writebuffer.h"
 #include "db/internal_stats.h"
-#include "db/job_context.h"
+#include "db/compaction_picker.h"
 #include "db/table_properties_collector.h"
-#include "db/version_set.h"
 #include "db/write_controller.h"
 #include "util/autovector.h"
 #include "util/hash_skiplist_rep.h"
@@ -90,10 +87,6 @@ ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
 
 uint32_t ColumnFamilyHandleImpl::GetID() const { return cfd()->GetID(); }
 
-const std::string& ColumnFamilyHandleImpl::GetName() const {
-  return cfd()->GetName();
-}
-
 const Comparator* ColumnFamilyHandleImpl::user_comparator() const {
   return cfd()->user_comparator();
 }
@@ -226,7 +219,6 @@ void SuperVersionUnrefHandle(void* ptr) {
 ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
                                    Version* _dummy_versions,
                                    Cache* _table_cache,
-                                   WriteBuffer* write_buffer,
                                    const ColumnFamilyOptions& cf_options,
                                    const DBOptions* db_options,
                                    const EnvOptions& env_options,
@@ -241,7 +233,6 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
       options_(*db_options, SanitizeOptions(&internal_comparator_, cf_options)),
       ioptions_(options_),
       mutable_cf_options_(options_, ioptions_),
-      write_buffer_(write_buffer),
       mem_(nullptr),
       imm_(options_.min_write_buffer_number_to_merge),
       super_version_(nullptr),
@@ -258,31 +249,16 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
     internal_stats_.reset(
         new InternalStats(ioptions_.num_levels, db_options->env, this));
     table_cache_.reset(new TableCache(ioptions_, env_options, _table_cache));
-    if (ioptions_.compaction_style == kCompactionStyleLevel) {
-      compaction_picker_.reset(
-          new LevelCompactionPicker(ioptions_, &internal_comparator_));
-#ifndef ROCKSDB_LITE
-    } else if (ioptions_.compaction_style == kCompactionStyleUniversal) {
+    if (ioptions_.compaction_style == kCompactionStyleUniversal) {
       compaction_picker_.reset(
           new UniversalCompactionPicker(ioptions_, &internal_comparator_));
-    } else if (ioptions_.compaction_style == kCompactionStyleFIFO) {
-      compaction_picker_.reset(
-          new FIFOCompactionPicker(ioptions_, &internal_comparator_));
-    } else if (ioptions_.compaction_style == kCompactionStyleNone) {
-      compaction_picker_.reset(new NullCompactionPicker(
-          ioptions_, &internal_comparator_));
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
-          "Column family %s does not use any background compaction. "
-          "Compactions can only be done via CompactFiles\n",
-          GetName().c_str());
-#endif  // !ROCKSDB_LITE
-    } else {
-      Log(InfoLogLevel::ERROR_LEVEL, ioptions_.info_log,
-          "Unable to recognize the specified compaction style %d. "
-          "Column family %s will use kCompactionStyleLevel.\n",
-          ioptions_.compaction_style, GetName().c_str());
+    } else if (ioptions_.compaction_style == kCompactionStyleLevel) {
       compaction_picker_.reset(
           new LevelCompactionPicker(ioptions_, &internal_comparator_));
+    } else {
+      assert(ioptions_.compaction_style == kCompactionStyleFIFO);
+      compaction_picker_.reset(
+          new FIFOCompactionPicker(ioptions_, &internal_comparator_));
     }
 
     Log(InfoLogLevel::INFO_LEVEL,
@@ -418,19 +394,13 @@ void ColumnFamilyData::SetCurrent(Version* current_version) {
   current_ = current_version;
 }
 
-MemTable* ColumnFamilyData::ConstructNewMemtable(
-    const MutableCFOptions& mutable_cf_options) {
-  assert(current_ != nullptr);
-  return new MemTable(internal_comparator_, ioptions_,
-                      mutable_cf_options, write_buffer_);
-}
-
 void ColumnFamilyData::CreateNewMemtable(
     const MutableCFOptions& mutable_cf_options) {
+  assert(current_ != nullptr);
   if (mem_ != nullptr) {
     delete mem_->Unref();
   }
-  SetMemtable(ConstructNewMemtable(mutable_cf_options));
+  mem_ = new MemTable(internal_comparator_, ioptions_, mutable_cf_options);
   mem_->Ref();
 }
 
@@ -533,22 +503,6 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
   return false;
 }
 
-void ColumnFamilyData::NotifyOnFlushCompleted(
-    DB* db, const std::string& file_path,
-    bool triggered_flush_slowdown,
-    bool triggered_flush_stop) {
-
-#ifndef ROCKSDB_LITE
-  auto listeners = ioptions()->listeners;
-  for (auto listener : listeners) {
-    listener->OnFlushCompleted(
-        db, GetName(), file_path,
-        // Use path 0 as fulled memtables are first flushed into path 0.
-        triggered_flush_slowdown, triggered_flush_stop);
-  }
-#endif  // ROCKSDB_LITE
-}
-
 SuperVersion* ColumnFamilyData::InstallSuperVersion(
     SuperVersion* new_superversion, port::Mutex* db_mutex) {
   db_mutex->AssertHeld();
@@ -593,7 +547,6 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
   }
 }
 
-#ifndef ROCKSDB_LITE
 Status ColumnFamilyData::SetOptions(
       const std::unordered_map<std::string, std::string>& options_map) {
   MutableCFOptions new_mutable_cf_options;
@@ -605,16 +558,14 @@ Status ColumnFamilyData::SetOptions(
   }
   return s;
 }
-#endif  // ROCKSDB_LITE
 
 ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                  const DBOptions* db_options,
                                  const EnvOptions& env_options,
                                  Cache* table_cache,
-                                 WriteBuffer* write_buffer,
                                  WriteController* write_controller)
     : max_column_family_(0),
-      dummy_cfd_(new ColumnFamilyData(0, "", nullptr, nullptr, nullptr,
+      dummy_cfd_(new ColumnFamilyData(0, "", nullptr, nullptr,
                                       ColumnFamilyOptions(), db_options,
                                       env_options, nullptr)),
       default_cfd_cache_(nullptr),
@@ -622,7 +573,6 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
       db_options_(db_options),
       env_options_(env_options),
       table_cache_(table_cache),
-      write_buffer_(write_buffer),
       write_controller_(write_controller),
       spin_lock_(ATOMIC_FLAG_INIT) {
   // initialize linked list
@@ -687,9 +637,8 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
     const ColumnFamilyOptions& options) {
   assert(column_families_.find(name) == column_families_.end());
   ColumnFamilyData* new_cfd =
-      new ColumnFamilyData(id, name, dummy_versions, table_cache_,
-                           write_buffer_, options, db_options_,
-                           env_options_, this);
+      new ColumnFamilyData(id, name, dummy_versions, table_cache_, options,
+                           db_options_, env_options_, this);
   Lock();
   column_families_.insert({name, id});
   column_family_data_.insert({id, new_cfd});
@@ -768,6 +717,11 @@ MemTable* ColumnFamilyMemTablesImpl::GetMemTable() const {
   return current_->mem();
 }
 
+const Options* ColumnFamilyMemTablesImpl::GetOptions() const {
+  assert(current_ != nullptr);
+  return current_->options();
+}
+
 ColumnFamilyHandle* ColumnFamilyMemTablesImpl::GetColumnFamilyHandle() {
   assert(current_ != nullptr);
   return &handle_;

db/column_family.h

@@ -52,7 +52,6 @@ class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
   virtual const Comparator* user_comparator() const;
 
   virtual uint32_t GetID() const;
-  virtual const std::string& GetName() const override;
 
  private:
   ColumnFamilyData* cfd_;
@@ -172,10 +171,8 @@ class ColumnFamilyData {
   void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
   uint64_t GetLogNumber() const { return log_number_; }
 
-  // !!! To be deprecated! Please don't not use this function anymore!
-  const Options* options() const { return &options_; }
-
   // thread-safe
+  const Options* options() const { return &options_; }
   const EnvOptions* soptions() const;
   const ImmutableCFOptions* ioptions() const { return &ioptions_; }
   // REQUIRES: DB mutex held
@@ -189,11 +186,9 @@ class ColumnFamilyData {
   const MutableCFOptions* GetLatestMutableCFOptions() const {
     return &mutable_cf_options_;
   }
-#ifndef ROCKSDB_LITE
   // REQUIRES: DB mutex held
   Status SetOptions(
       const std::unordered_map<std::string, std::string>& options_map);
-#endif  // ROCKSDB_LITE
 
   InternalStats* internal_stats() { return internal_stats_.get(); }
 
@@ -201,9 +196,8 @@ class ColumnFamilyData {
   MemTable* mem() { return mem_; }
   Version* current() { return current_; }
   Version* dummy_versions() { return dummy_versions_; }
-  void SetCurrent(Version* current);
-  MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options);
   void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
+  void SetCurrent(Version* current);
   void CreateNewMemtable(const MutableCFOptions& mutable_cf_options);
 
   TableCache* table_cache() const { return table_cache_.get(); }
@@ -256,16 +250,10 @@ class ColumnFamilyData {
 
   void ResetThreadLocalSuperVersions();
 
-  void NotifyOnFlushCompleted(
-      DB* db, const std::string& file_path,
-      bool triggered_flush_slowdown,
-      bool triggered_flush_stop);
-
  private:
   friend class ColumnFamilySet;
   ColumnFamilyData(uint32_t id, const std::string& name,
                    Version* dummy_versions, Cache* table_cache,
-                   WriteBuffer* write_buffer,
                    const ColumnFamilyOptions& options,
                    const DBOptions* db_options, const EnvOptions& env_options,
                    ColumnFamilySet* column_family_set);
@@ -296,8 +284,6 @@ class ColumnFamilyData {
 
   std::unique_ptr<InternalStats> internal_stats_;
 
-  WriteBuffer* write_buffer_;
-
   MemTable* mem_;
   MemTableList imm_;
   SuperVersion* super_version_;
@@ -370,7 +356,7 @@ class ColumnFamilySet {
 
   ColumnFamilySet(const std::string& dbname, const DBOptions* db_options,
                   const EnvOptions& env_options, Cache* table_cache,
-                  WriteBuffer* write_buffer, WriteController* write_controller);
+                  WriteController* write_controller);
   ~ColumnFamilySet();
 
   ColumnFamilyData* GetDefault() const;
@@ -425,7 +411,6 @@ class ColumnFamilySet {
   const DBOptions* const db_options_;
   const EnvOptions env_options_;
   Cache* table_cache_;
-  WriteBuffer* write_buffer_;
   WriteController* write_controller_;
   std::atomic_flag spin_lock_;
 };
@@ -450,6 +435,10 @@ class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
   // REQUIRES: Seek() called first
   virtual MemTable* GetMemTable() const override;
 
+  // Returns options for selected column family
+  // REQUIRES: Seek() called first
+  virtual const Options* GetOptions() const override;
+
   // Returns column family handle for the selected column family
   virtual ColumnFamilyHandle* GetColumnFamilyHandle() override;
 

db/column_family_test.cc

@@ -133,7 +133,7 @@ class ColumnFamilyTest {
   void CreateColumnFamilies(
       const std::vector<std::string>& cfs,
       const std::vector<ColumnFamilyOptions> options = {}) {
-    int cfi = static_cast<int>(handles_.size());
+    int cfi = handles_.size();
     handles_.resize(cfi + cfs.size());
     names_.resize(cfi + cfs.size());
     for (size_t i = 0; i < cfs.size(); ++i) {
@@ -218,7 +218,7 @@ class ColumnFamilyTest {
 
   int NumTableFilesAtLevel(int level, int cf) {
     return GetProperty(cf,
-                       "rocksdb.num-files-at-level" + ToString(level));
+                       "rocksdb.num-files-at-level" + std::to_string(level));
   }
 
   // Return spread of files per level
@@ -231,7 +231,7 @@ class ColumnFamilyTest {
       snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
       result += buf;
       if (f > 0) {
-        last_non_zero_offset = static_cast<int>(result.size());
+        last_non_zero_offset = result.size();
       }
     }
     result.resize(last_non_zero_offset);
@@ -287,8 +287,8 @@ class ColumnFamilyTest {
     assert(num_per_cf.size() == handles_.size());
 
     for (size_t i = 0; i < num_per_cf.size(); ++i) {
-      ASSERT_EQ(num_per_cf[i], GetProperty(static_cast<int>(i),
-                                           "rocksdb.num-immutable-mem-table"));
+      ASSERT_EQ(num_per_cf[i],
+                GetProperty(i, "rocksdb.num-immutable-mem-table"));
     }
   }
 
@@ -387,7 +387,7 @@ TEST(ColumnFamilyTest, DropTest) {
     Open({"default"});
     CreateColumnFamiliesAndReopen({"pikachu"});
     for (int i = 0; i < 100; ++i) {
-      ASSERT_OK(Put(1, ToString(i), "bar" + ToString(i)));
+      ASSERT_OK(Put(1, std::to_string(i), "bar" + std::to_string(i)));
     }
     ASSERT_OK(Flush(1));
 
@@ -529,28 +529,8 @@ TEST(ColumnFamilyTest, FlushTest) {
   ASSERT_OK(Put(1, "mirko", "v3"));
   ASSERT_OK(Put(0, "foo", "v2"));
   ASSERT_OK(Put(2, "fodor", "v5"));
-
-  for (int j = 0; j < 2; j++) {
-    ReadOptions ro;
-    std::vector<Iterator*> iterators;
-    // Hold super version.
-    if (j == 0) {
-      ASSERT_OK(db_->NewIterators(ro, handles_, &iterators));
-    }
-
-    for (int i = 0; i < 3; ++i) {
-      uint64_t max_total_in_memory_state =
-          dbfull()->TEST_max_total_in_memory_state();
-      Flush(i);
-      ASSERT_EQ(dbfull()->TEST_max_total_in_memory_state(),
-                max_total_in_memory_state);
-    }
-    ASSERT_OK(Put(1, "foofoo", "bar"));
-    ASSERT_OK(Put(0, "foofoo", "bar"));
-
-    for (auto* it : iterators) {
-      delete it;
-    }
+  for (int i = 0; i < 3; ++i) {
+    Flush(i);
   }
   Reopen();
 
@@ -794,14 +774,14 @@ TEST(ColumnFamilyTest, DifferentCompactionStyles) {
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 1; ++i) {
     PutRandomData(1, 11, 10000);
     WaitForFlush(1);
-    ASSERT_EQ(ToString(i + 1), FilesPerLevel(1));
+    ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(1));
   }
 
   // SETUP column family "two" -- level style with 4 levels
   for (int i = 0; i < two.level0_file_num_compaction_trigger - 1; ++i) {
     PutRandomData(2, 15, 10000);
     WaitForFlush(2);
-    ASSERT_EQ(ToString(i + 1), FilesPerLevel(2));
+    ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(2));
   }
 
   // TRIGGER compaction "one"
@@ -936,11 +916,11 @@ TEST(ColumnFamilyTest, DontRollEmptyLogs) {
   CreateColumnFamiliesAndReopen({"one", "two", "three", "four"});
 
   for (size_t i = 0; i < handles_.size(); ++i) {
-    PutRandomData(static_cast<int>(i), 10, 100);
+    PutRandomData(i, 10, 100);
   }
   int num_writable_file_start = env_->GetNumberOfNewWritableFileCalls();
   // this will trigger the flushes
-  for (int i = 0; i <= 4; ++i) {
+  for (size_t i = 0; i <= 4; ++i) {
     ASSERT_OK(Flush(i));
   }
 

db/compaction.cc

@@ -78,38 +78,6 @@ Compaction::Compaction(int number_levels, int start_level, int out_level,
   }
 }
 
-Compaction::Compaction(VersionStorageInfo* vstorage,
-    const autovector<CompactionInputFiles>& _inputs,
-    int _start_level, int _output_level,
-    uint64_t _max_grandparent_overlap_bytes,
-    const CompactionOptions& _options,
-    bool _deletion_compaction)
-    : start_level_(_start_level),
-      output_level_(_output_level),
-      max_output_file_size_(_options.output_file_size_limit),
-      max_grandparent_overlap_bytes_(_max_grandparent_overlap_bytes),
-      input_version_(nullptr),
-      number_levels_(vstorage->num_levels()),
-      cfd_(nullptr),
-      output_compression_(_options.compression),
-      seek_compaction_(false),
-      deletion_compaction_(_deletion_compaction),
-      inputs_(_inputs),
-      grandparent_index_(0),
-      seen_key_(false),
-      overlapped_bytes_(0),
-      base_index_(-1),
-      parent_index_(-1),
-      score_(0),
-      bottommost_level_(false),
-      is_full_compaction_(false),
-      is_manual_compaction_(false),
-      level_ptrs_(std::vector<size_t>(number_levels_)) {
-  for (int i = 0; i < number_levels_; i++) {
-    level_ptrs_[i] = 0;
-  }
-}
-
 Compaction::~Compaction() {
   delete edit_;
   if (input_version_ != nullptr) {
@@ -124,9 +92,9 @@ Compaction::~Compaction() {
 
 void Compaction::GenerateFileLevels() {
   input_levels_.resize(num_input_levels());
-  for (size_t which = 0; which < num_input_levels(); which++) {
-    DoGenerateLevelFilesBrief(&input_levels_[which], inputs_[which].files,
-                              &arena_);
+  for (int which = 0; which < num_input_levels(); which++) {
+    DoGenerateLevelFilesBrief(
+        &input_levels_[which], inputs_[which].files, &arena_);
   }
 }
 
@@ -144,7 +112,7 @@ bool Compaction::IsTrivialMove() const {
 }
 
 void Compaction::AddInputDeletions(VersionEdit* out_edit) {
-  for (size_t which = 0; which < num_input_levels(); which++) {
+  for (int which = 0; which < num_input_levels(); which++) {
     for (size_t i = 0; i < inputs_[which].size(); i++) {
       out_edit->DeleteFile(level(which), inputs_[which][i]->fd.GetNumber());
     }
@@ -207,7 +175,7 @@ bool Compaction::ShouldStopBefore(const Slice& internal_key) {
 
 // Mark (or clear) each file that is being compacted
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
-  for (size_t i = 0; i < num_input_levels(); i++) {
+  for (int i = 0; i < num_input_levels(); i++) {
     for (unsigned int j = 0; j < inputs_[i].size(); j++) {
       assert(mark_as_compacted ? !inputs_[i][j]->being_compacted :
                                   inputs_[i][j]->being_compacted);
@@ -293,7 +261,7 @@ void Compaction::Summary(char* output, int len) {
     return;
   }
 
-  for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
+  for (int level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
     if (level_iter > 0) {
       write += snprintf(output + write, len - write, "], [");
       if (write < 0 || write >= len) {
@@ -317,7 +285,7 @@ uint64_t Compaction::OutputFilePreallocationSize(
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
     preallocation_size = mutable_options.MaxFileSizeForLevel(output_level());
   } else {
-    for (size_t level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
+    for (int level_iter = 0; level_iter < num_input_levels(); ++level_iter) {
       for (const auto& f : inputs_[level_iter].files) {
         preallocation_size += f->fd.GetFileSize();
       }
@@ -328,15 +296,4 @@ uint64_t Compaction::OutputFilePreallocationSize(
   return preallocation_size * 1.1;
 }
 
-Compaction* Compaction::TEST_NewCompaction(
-    int num_levels, int start_level, int out_level, uint64_t target_file_size,
-    uint64_t max_grandparent_overlap_bytes, uint32_t output_path_id,
-    CompressionType output_compression, bool seek_compaction,
-    bool deletion_compaction) {
-  return new Compaction(num_levels, start_level, out_level, target_file_size,
-                        max_grandparent_overlap_bytes, output_path_id,
-                        output_compression, seek_compaction,
-                        deletion_compaction);
-}
-
 }  // namespace rocksdb

db/compaction.h

@@ -23,7 +23,7 @@ struct CompactionInputFiles {
   inline bool empty() const { return files.empty(); }
   inline size_t size() const { return files.size(); }
   inline void clear() { files.clear(); }
-  inline FileMetaData* operator[](size_t i) const { return files[i]; }
+  inline FileMetaData* operator[](int i) const { return files[i]; }
 };
 
 class Version;
@@ -33,13 +33,6 @@ class VersionStorageInfo;
 // A Compaction encapsulates information about a compaction.
 class Compaction {
  public:
-  Compaction(VersionStorageInfo* input_version,
-    const autovector<CompactionInputFiles>& inputs,
-    int start_level, int output_level,
-    uint64_t max_grandparent_overlap_bytes,
-    const CompactionOptions& options,
-    bool deletion_compaction);
-
   // No copying allowed
   Compaction(const Compaction&) = delete;
   void operator=(const Compaction&) = delete;
@@ -48,7 +41,7 @@ class Compaction {
 
   // Returns the level associated to the specified compaction input level.
   // If compaction_input_level is not specified, then input_level is set to 0.
-  int level(size_t compaction_input_level = 0) const {
+  int level(int compaction_input_level = 0) const {
     return inputs_[compaction_input_level].level;
   }
 
@@ -56,7 +49,7 @@ class Compaction {
   int output_level() const { return output_level_; }
 
   // Returns the number of input levels in this compaction.
-  size_t num_input_levels() const { return inputs_.size(); }
+  int num_input_levels() const { return inputs_.size(); }
 
   // Return the object that holds the edits to the descriptor done
   // by this compaction.
@@ -66,7 +59,7 @@ class Compaction {
   // compaction input level.
   // The function will return 0 if when "compaction_input_level" < 0
   // or "compaction_input_level" >= "num_input_levels()".
-  size_t num_input_files(size_t compaction_input_level) const {
+  int num_input_files(size_t compaction_input_level) const {
     if (compaction_input_level < inputs_.size()) {
       return inputs_[compaction_input_level].size();
     }
@@ -83,7 +76,7 @@ class Compaction {
   // specified compaction input level.
   // REQUIREMENT: "compaction_input_level" must be >= 0 and
   //              < "input_levels()"
-  FileMetaData* input(size_t compaction_input_level, size_t i) const {
+  FileMetaData* input(size_t compaction_input_level, int i) const {
     assert(compaction_input_level < inputs_.size());
     return inputs_[compaction_input_level][i];
   }
@@ -98,7 +91,7 @@ class Compaction {
   }
 
   // Returns the LevelFilesBrief of the specified compaction input level.
-  LevelFilesBrief* input_levels(size_t compaction_input_level) {
+  LevelFilesBrief* input_levels(int compaction_input_level) {
     return &input_levels_[compaction_input_level];
   }
 
@@ -160,8 +153,6 @@ class Compaction {
   // Was this compaction triggered manually by the client?
   bool IsManualCompaction() { return is_manual_compaction_; }
 
-  void SetOutputPathId(uint32_t path_id) { output_path_id_ = path_id; }
-
   // Return the MutableCFOptions that should be used throughout the compaction
   // procedure
   const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
@@ -173,26 +164,6 @@ class Compaction {
 
   void SetInputVersion(Version* input_version);
 
-  // mark (or clear) all files that are being compacted
-  void MarkFilesBeingCompacted(bool mark_as_compacted);
-
-  // Initialize whether the compaction is producing files at the
-  // bottommost level.
-  //
-  // @see BottomMostLevel()
-  void SetupBottomMostLevel(VersionStorageInfo* vstorage, bool is_manual,
-                            bool level0_only);
-
-  static Compaction* TEST_NewCompaction(
-      int num_levels, int start_level, int out_level, uint64_t target_file_size,
-      uint64_t max_grandparent_overlap_bytes, uint32_t output_path_id,
-      CompressionType output_compression, bool seek_compaction = false,
-      bool deletion_compaction = false);
-
-  CompactionInputFiles* TEST_GetInputFiles(int l) {
-    return &inputs_[l];
-  }
-
  private:
   friend class CompactionPicker;
   friend class UniversalCompactionPicker;
@@ -255,6 +226,16 @@ class Compaction {
   // records indices for all levels beyond "output_level_".
   std::vector<size_t> level_ptrs_;
 
+  // mark (or clear) all files that are being compacted
+  void MarkFilesBeingCompacted(bool mark_as_compacted);
+
+  // Initialize whether the compaction is producing files at the
+  // bottommost level.
+  //
+  // @see BottomMostLevel()
+  void SetupBottomMostLevel(VersionStorageInfo* vstorage, bool is_manual,
+                            bool level0_only);
+
   // In case of compaction error, reset the nextIndex that is used
   // to pick up the next file to be compacted from files_by_size_
   void ResetNextCompactionIndex();

db/compaction_job.cc

@@ -71,6 +71,7 @@ struct CompactionJob::CompactionState {
     SequenceNumber smallest_seqno, largest_seqno;
   };
   std::vector<Output> outputs;
+  std::list<uint64_t> allocated_file_numbers;
 
   // State kept for output being generated
   std::unique_ptr<WritableFile> outfile;
@@ -203,7 +204,8 @@ struct CompactionJob::CompactionState {
 CompactionJob::CompactionJob(
     Compaction* compaction, const DBOptions& db_options,
     const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
-    VersionSet* versions, std::atomic<bool>* shutting_down,
+    VersionSet* versions, port::Mutex* db_mutex,
+    std::atomic<bool>* shutting_down, FileNumToPathIdMap* pending_outputs,
     LogBuffer* log_buffer, Directory* db_directory, Statistics* stats,
     SnapshotList* snapshots, bool is_snapshot_supported,
     std::shared_ptr<Cache> table_cache,
@@ -215,7 +217,9 @@ CompactionJob::CompactionJob(
       env_options_(env_options),
       env_(db_options.env),
       versions_(versions),
+      db_mutex_(db_mutex),
       shutting_down_(shutting_down),
+      pending_outputs_(pending_outputs),
       log_buffer_(log_buffer),
       db_directory_(db_directory),
       stats_(stats),
@@ -225,13 +229,13 @@ CompactionJob::CompactionJob(
       yield_callback_(std::move(yield_callback)) {}
 
 void CompactionJob::Prepare() {
+  db_mutex_->AssertHeld();
   compact_->CleanupBatchBuffer();
   compact_->CleanupMergedBuffer();
 
   // Generate file_levels_ for compaction berfore making Iterator
   compact_->compaction->GenerateFileLevels();
   ColumnFamilyData* cfd = compact_->compaction->column_family_data();
-  assert(cfd != nullptr);
   LogToBuffer(
       log_buffer_, "[%s] Compacting %d@%d + %d@%d files, score %.2f",
       cfd->GetName().c_str(), compact_->compaction->num_input_files(0),
@@ -265,6 +269,9 @@ void CompactionJob::Prepare() {
 
   // Is this compaction producing files at the bottommost level?
   bottommost_level_ = compact_->compaction->BottomMostLevel();
+
+  // Allocate the output file numbers before we release the lock
+  AllocateCompactionOutputFileNumbers();
 }
 
 Status CompactionJob::Run() {
@@ -385,22 +392,17 @@ Status CompactionJob::Run() {
         compact_->MergeKeyValueSliceBuffer(&cfd->internal_comparator());
 
         status = ProcessKeyValueCompaction(&imm_micros, input.get(), true);
-        if (!status.ok()) {
-          break;
-        }
 
         compact_->CleanupBatchBuffer();
         compact_->CleanupMergedBuffer();
       }
     }  // done processing all prefix batches
     // finish the last batch
-    if (status.ok()) {
-      if (compact_->key_str_buf_.size() > 0) {
-        CallCompactionFilterV2(compaction_filter_v2);
-      }
-      compact_->MergeKeyValueSliceBuffer(&cfd->internal_comparator());
-      status = ProcessKeyValueCompaction(&imm_micros, input.get(), true);
+    if (compact_->key_str_buf_.size() > 0) {
+      CallCompactionFilterV2(compaction_filter_v2);
     }
+    compact_->MergeKeyValueSliceBuffer(&cfd->internal_comparator());
+    status = ProcessKeyValueCompaction(&imm_micros, input.get(), true);
   }  // checking for compaction filter v2
 
   if (status.ok() &&
@@ -421,33 +423,32 @@ Status CompactionJob::Run() {
   }
 
   compaction_stats_.micros = env_->NowMicros() - start_micros - imm_micros;
-  compaction_stats_.files_in_leveln =
-      static_cast<int>(compact_->compaction->num_input_files(0));
+  compaction_stats_.files_in_leveln = compact_->compaction->num_input_files(0);
   compaction_stats_.files_in_levelnp1 =
-      static_cast<int>(compact_->compaction->num_input_files(1));
+      compact_->compaction->num_input_files(1);
   MeasureTime(stats_, COMPACTION_TIME, compaction_stats_.micros);
 
-  size_t num_output_files = compact_->outputs.size();
+  int num_output_files = compact_->outputs.size();
   if (compact_->builder != nullptr) {
     // An error occurred so ignore the last output.
     assert(num_output_files > 0);
     --num_output_files;
   }
-  compaction_stats_.files_out_levelnp1 = static_cast<int>(num_output_files);
+  compaction_stats_.files_out_levelnp1 = num_output_files;
 
-  for (size_t i = 0; i < compact_->compaction->num_input_files(0); i++) {
+  for (int i = 0; i < compact_->compaction->num_input_files(0); i++) {
     compaction_stats_.bytes_readn +=
         compact_->compaction->input(0, i)->fd.GetFileSize();
     compaction_stats_.num_input_records +=
         static_cast<uint64_t>(compact_->compaction->input(0, i)->num_entries);
   }
 
-  for (size_t i = 0; i < compact_->compaction->num_input_files(1); i++) {
+  for (int i = 0; i < compact_->compaction->num_input_files(1); i++) {
     compaction_stats_.bytes_readnp1 +=
         compact_->compaction->input(1, i)->fd.GetFileSize();
   }
 
-  for (size_t i = 0; i < num_output_files; i++) {
+  for (int i = 0; i < num_output_files; i++) {
     compaction_stats_.bytes_written += compact_->outputs[i].file_size;
   }
   if (compact_->num_input_records > compact_->num_output_records) {
@@ -462,14 +463,18 @@ Status CompactionJob::Run() {
   return status;
 }
 
-void CompactionJob::Install(Status* status, port::Mutex* db_mutex) {
-  db_mutex->AssertHeld();
+Status CompactionJob::Install(Status status) {
+  db_mutex_->AssertHeld();
   ColumnFamilyData* cfd = compact_->compaction->column_family_data();
   cfd->internal_stats()->AddCompactionStats(
       compact_->compaction->output_level(), compaction_stats_);
 
-  if (status->ok()) {
-    *status = InstallCompactionResults(db_mutex);
+  // if there were any unused file number (mostly in case of
+  // compaction error), free up the entry from pending_putputs
+  ReleaseCompactionUnusedFileNumbers();
+
+  if (status.ok()) {
+    status = InstallCompactionResults();
   }
   VersionStorageInfo::LevelSummaryStorage tmp;
   const auto& stats = compaction_stats_;
@@ -490,10 +495,26 @@ void CompactionJob::Install(Status* status, port::Mutex* db_mutex) {
               (stats.bytes_written + stats.bytes_readnp1 + stats.bytes_readn) /
                   static_cast<double>(stats.bytes_readn),
               stats.bytes_written / static_cast<double>(stats.bytes_readn),
-              status->ToString().c_str(), stats.num_input_records,
+              status.ToString().c_str(), stats.num_input_records,
               stats.num_dropped_records);
 
-  CleanupCompaction(*status);
+  CleanupCompaction(status);
+  return status;
+}
+
+// Allocate the file numbers for the output file. We allocate as
+// many output file numbers as there are files in level+1 (at least one)
+// Insert them into pending_outputs so that they do not get deleted.
+void CompactionJob::AllocateCompactionOutputFileNumbers() {
+  db_mutex_->AssertHeld();
+  assert(compact_->builder == nullptr);
+  int filesNeeded = compact_->compaction->num_input_files(1);
+  for (int i = 0; i < std::max(filesNeeded, 1); i++) {
+    uint64_t file_number = versions_->NewFileNumber();
+    pending_outputs_->insert(
+        {file_number, compact_->compaction->GetOutputPathId()});
+    compact_->allocated_file_numbers.push_back(file_number);
+  }
 }
 
 Status CompactionJob::ProcessKeyValueCompaction(int64_t* imm_micros,
@@ -868,8 +889,7 @@ void CompactionJob::CallCompactionFilterV2(
   assert(compact_->to_delete_buf_.size() == compact_->key_str_buf_.size());
   assert(compact_->to_delete_buf_.size() ==
          compact_->existing_value_str_buf_.size());
-  assert(compact_->value_changed_buf_.empty() ||
-         compact_->to_delete_buf_.size() ==
+  assert(compact_->to_delete_buf_.size() ==
          compact_->value_changed_buf_.size());
 
   int new_value_idx = 0;
@@ -884,8 +904,7 @@ void CompactionJob::CallCompactionFilterV2(
       // no value associated with delete
       compact_->existing_value_str_buf_[i].clear();
       RecordTick(stats_, COMPACTION_KEY_DROP_USER);
-    } else if (!compact_->value_changed_buf_.empty() &&
-        compact_->value_changed_buf_[i]) {
+    } else if (compact_->value_changed_buf_[i]) {
       compact_->existing_value_str_buf_[i] =
           compact_->new_value_buf_[new_value_idx++];
     }
@@ -947,8 +966,8 @@ Status CompactionJob::FinishCompactionOutputFile(Iterator* input) {
   return s;
 }
 
-Status CompactionJob::InstallCompactionResults(port::Mutex* db_mutex) {
-  db_mutex->AssertHeld();
+Status CompactionJob::InstallCompactionResults() {
+  db_mutex_->AssertHeld();
 
   // paranoia: verify that the files that we started with
   // still exist in the current version and in the same original level.
@@ -984,7 +1003,7 @@ Status CompactionJob::InstallCompactionResults(port::Mutex* db_mutex) {
   }
   return versions_->LogAndApply(
       compact_->compaction->column_family_data(), mutable_cf_options_,
-      compact_->compaction->edit(), db_mutex, db_directory_);
+      compact_->compaction->edit(), db_mutex_, db_directory_);
 }
 
 // Given a sequence number, return the sequence number of the
@@ -996,7 +1015,6 @@ Status CompactionJob::InstallCompactionResults(port::Mutex* db_mutex) {
 inline SequenceNumber CompactionJob::findEarliestVisibleSnapshot(
     SequenceNumber in, const std::vector<SequenceNumber>& snapshots,
     SequenceNumber* prev_snapshot) {
-  assert(snapshots.size());
   SequenceNumber prev __attribute__((unused)) = 0;
   for (const auto cur : snapshots) {
     assert(prev <= cur);
@@ -1023,11 +1041,31 @@ void CompactionJob::RecordCompactionIOStats() {
   IOSTATS_RESET(bytes_written);
 }
 
+// Frees up unused file number.
+void CompactionJob::ReleaseCompactionUnusedFileNumbers() {
+  db_mutex_->AssertHeld();
+  for (const auto file_number : compact_->allocated_file_numbers) {
+    pending_outputs_->erase(file_number);
+  }
+}
+
 Status CompactionJob::OpenCompactionOutputFile() {
   assert(compact_ != nullptr);
   assert(compact_->builder == nullptr);
-  // no need to lock because VersionSet::next_file_number_ is atomic
-  uint64_t file_number = versions_->NewFileNumber();
+  uint64_t file_number;
+  // If we have not yet exhausted the pre-allocated file numbers,
+  // then use the one from the front. Otherwise, we have to acquire
+  // the heavyweight lock and allocate a new file number.
+  if (!compact_->allocated_file_numbers.empty()) {
+    file_number = compact_->allocated_file_numbers.front();
+    compact_->allocated_file_numbers.pop_front();
+  } else {
+    db_mutex_->Lock();
+    file_number = versions_->NewFileNumber();
+    pending_outputs_->insert(
+        {file_number, compact_->compaction->GetOutputPathId()});
+    db_mutex_->Unlock();
+  }
   // Make the output file
   std::string fname = TableFileName(db_options_.db_paths, file_number,
                                     compact_->compaction->GetOutputPathId());
@@ -1051,8 +1089,8 @@ Status CompactionJob::OpenCompactionOutputFile() {
 
   compact_->outputs.push_back(out);
   compact_->outfile->SetIOPriority(Env::IO_LOW);
-  compact_->outfile->SetPreallocationBlockSize(static_cast<size_t>(
-      compact_->compaction->OutputFilePreallocationSize(mutable_cf_options_)));
+  compact_->outfile->SetPreallocationBlockSize(
+      compact_->compaction->OutputFilePreallocationSize(mutable_cf_options_));
 
   ColumnFamilyData* cfd = compact_->compaction->column_family_data();
   compact_->builder.reset(NewTableBuilder(
@@ -1063,7 +1101,8 @@ Status CompactionJob::OpenCompactionOutputFile() {
   return s;
 }
 
-void CompactionJob::CleanupCompaction(const Status& status) {
+void CompactionJob::CleanupCompaction(Status status) {
+  db_mutex_->AssertHeld();
   if (compact_->builder != nullptr) {
     // May happen if we get a shutdown call in the middle of compaction
     compact_->builder->Abandon();
@@ -1073,6 +1112,7 @@ void CompactionJob::CleanupCompaction(const Status& status) {
   }
   for (size_t i = 0; i < compact_->outputs.size(); i++) {
     const CompactionState::Output& out = compact_->outputs[i];
+    pending_outputs_->erase(out.number);
 
     // If this file was inserted into the table cache then remove
     // them here because this compaction was not committed.

db/compaction_job.h

@@ -56,7 +56,8 @@ class CompactionJob {
   CompactionJob(Compaction* compaction, const DBOptions& db_options,
                 const MutableCFOptions& mutable_cf_options,
                 const EnvOptions& env_options, VersionSet* versions,
-                std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
+                port::Mutex* db_mutex, std::atomic<bool>* shutting_down,
+                FileNumToPathIdMap* pending_outputs, LogBuffer* log_buffer,
                 Directory* db_directory, Statistics* stats,
                 SnapshotList* snapshot_list, bool is_snapshot_supported,
                 std::shared_ptr<Cache> table_cache,
@@ -75,7 +76,7 @@ class CompactionJob {
   Status Run();
   // REQUIRED: mutex held
   // status is the return of Run()
-  void Install(Status* status, port::Mutex* db_mutex);
+  Status Install(Status status);
 
  private:
   void AllocateCompactionOutputFileNumbers();
@@ -86,13 +87,14 @@ class CompactionJob {
   // Call compaction_filter_v2->Filter() on kv-pairs in compact
   void CallCompactionFilterV2(CompactionFilterV2* compaction_filter_v2);
   Status FinishCompactionOutputFile(Iterator* input);
-  Status InstallCompactionResults(port::Mutex* db_mutex);
+  Status InstallCompactionResults();
   SequenceNumber findEarliestVisibleSnapshot(
       SequenceNumber in, const std::vector<SequenceNumber>& snapshots,
       SequenceNumber* prev_snapshot);
   void RecordCompactionIOStats();
+  void ReleaseCompactionUnusedFileNumbers();
   Status OpenCompactionOutputFile();
-  void CleanupCompaction(const Status& status);
+  void CleanupCompaction(Status status);
 
   // CompactionJob state
   struct CompactionState;
@@ -111,7 +113,9 @@ class CompactionJob {
   const EnvOptions& env_options_;
   Env* env_;
   VersionSet* versions_;
+  port::Mutex* db_mutex_;
   std::atomic<bool>* shutting_down_;
+  FileNumToPathIdMap* pending_outputs_;
   LogBuffer* log_buffer_;
   Directory* db_directory_;
   Statistics* stats_;

db/compaction_job_test.cc

@@ -1,182 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include <map>
-#include <string>
-
-#include "db/compaction_job.h"
-#include "db/column_family.h"
-#include "db/version_set.h"
-#include "db/writebuffer.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/options.h"
-#include "rocksdb/db.h"
-#include "util/testharness.h"
-#include "util/testutil.h"
-#include "table/mock_table.h"
-
-namespace rocksdb {
-
-// TODO(icanadi) Make it simpler once we mock out VersionSet
-class CompactionJobTest {
- public:
-  CompactionJobTest()
-      : env_(Env::Default()),
-        dbname_(test::TmpDir() + "/compaction_job_test"),
-        mutable_cf_options_(Options(), ImmutableCFOptions(Options())),
-        table_cache_(NewLRUCache(50000, 16, 8)),
-        write_buffer_(db_options_.db_write_buffer_size),
-        versions_(new VersionSet(dbname_, &db_options_, env_options_,
-                                 table_cache_.get(), &write_buffer_,
-                                 &write_controller_)),
-        shutting_down_(false),
-        mock_table_factory_(new mock::MockTableFactory()) {
-    ASSERT_OK(env_->CreateDirIfMissing(dbname_));
-    db_options_.db_paths.emplace_back(dbname_,
-                                      std::numeric_limits<uint64_t>::max());
-    NewDB();
-    std::vector<ColumnFamilyDescriptor> column_families;
-    cf_options_.table_factory = mock_table_factory_;
-    column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
-
-
-    ASSERT_OK(versions_->Recover(column_families, false));
-  }
-
-  std::string GenerateFileName(uint64_t file_number) {
-    FileMetaData meta;
-    std::vector<DbPath> db_paths;
-    db_paths.emplace_back(dbname_, std::numeric_limits<uint64_t>::max());
-    meta.fd = FileDescriptor(file_number, 0, 0);
-    return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
-  }
-
-  // returns expected result after compaction
-  mock::MockFileContents CreateTwoFiles() {
-    mock::MockFileContents expected_results;
-    const int kKeysPerFile = 10000;
-    SequenceNumber sequence_number = 0;
-    for (int i = 0; i < 2; ++i) {
-      mock::MockFileContents contents;
-      SequenceNumber smallest_seqno = 0, largest_seqno = 0;
-      InternalKey smallest, largest;
-      for (int k = 0; k < kKeysPerFile; ++k) {
-        auto key = ToString(i * (kKeysPerFile / 2) + k);
-        auto value = ToString(i * kKeysPerFile + k);
-        InternalKey internal_key(key, ++sequence_number, kTypeValue);
-        if (k == 0) {
-          smallest = internal_key;
-          smallest_seqno = sequence_number;
-        } else if (k == kKeysPerFile - 1) {
-          largest = internal_key;
-          largest_seqno = sequence_number;
-        }
-        std::pair<std::string, std::string> key_value(
-            {internal_key.Encode().ToString(), value});
-        contents.insert(key_value);
-        if (i == 1 || k < kKeysPerFile / 2) {
-          expected_results.insert(key_value);
-        }
-      }
-
-      uint64_t file_number = versions_->NewFileNumber();
-      ASSERT_OK(mock_table_factory_->CreateMockTable(
-          env_, GenerateFileName(file_number), std::move(contents)));
-
-      VersionEdit edit;
-      edit.AddFile(0, file_number, 0, 10, smallest, largest, smallest_seqno,
-                   largest_seqno);
-
-      mutex_.Lock();
-      versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
-                             mutable_cf_options_, &edit, &mutex_);
-      mutex_.Unlock();
-    }
-    versions_->SetLastSequence(sequence_number);
-    return expected_results;
-  }
-
-  void NewDB() {
-    VersionEdit new_db;
-    new_db.SetLogNumber(0);
-    new_db.SetNextFile(2);
-    new_db.SetLastSequence(0);
-
-    const std::string manifest = DescriptorFileName(dbname_, 1);
-    unique_ptr<WritableFile> file;
-    Status s = env_->NewWritableFile(
-        manifest, &file, env_->OptimizeForManifestWrite(env_options_));
-    ASSERT_OK(s);
-    {
-      log::Writer log(std::move(file));
-      std::string record;
-      new_db.EncodeTo(&record);
-      s = log.AddRecord(record);
-    }
-    ASSERT_OK(s);
-    // Make "CURRENT" file that points to the new manifest file.
-    s = SetCurrentFile(env_, dbname_, 1, nullptr);
-  }
-
-  Env* env_;
-  std::string dbname_;
-  EnvOptions env_options_;
-  MutableCFOptions mutable_cf_options_;
-  std::shared_ptr<Cache> table_cache_;
-  WriteController write_controller_;
-  DBOptions db_options_;
-  ColumnFamilyOptions cf_options_;
-  WriteBuffer write_buffer_;
-  std::unique_ptr<VersionSet> versions_;
-  port::Mutex mutex_;
-  std::atomic<bool> shutting_down_;
-  std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
-};
-
-TEST(CompactionJobTest, Simple) {
-  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-
-  auto expected_results = CreateTwoFiles();
-
-  auto files = cfd->current()->storage_info()->LevelFiles(0);
-  ASSERT_EQ(2U, files.size());
-
-  std::unique_ptr<Compaction> compaction(Compaction::TEST_NewCompaction(
-      7, 0, 1, 1024 * 1024, 10, 0, kNoCompression));
-  compaction->SetInputVersion(cfd->current());
-
-  auto compaction_input_files = compaction->TEST_GetInputFiles(0);
-  compaction_input_files->level = 0;
-  compaction_input_files->files.push_back(files[0]);
-  compaction_input_files->files.push_back(files[1]);
-
-  SnapshotList snapshots;
-  int yield_callback_called = 0;
-  std::function<uint64_t()> yield_callback = [&]() {
-    yield_callback_called++;
-    return 0;
-  };
-  LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
-  mutex_.Lock();
-  CompactionJob compaction_job(
-      compaction.get(), db_options_, *cfd->GetLatestMutableCFOptions(),
-      env_options_, versions_.get(), &shutting_down_, &log_buffer, nullptr,
-      nullptr, &snapshots, true, table_cache_, std::move(yield_callback));
-  compaction_job.Prepare();
-  mutex_.Unlock();
-  ASSERT_OK(compaction_job.Run());
-  mutex_.Lock();
-  Status s;
-  compaction_job.Install(&s, &mutex_);
-  ASSERT_OK(s);
-  mutex_.Unlock();
-
-  mock_table_factory_->AssertLatestFile(expected_results);
-  ASSERT_EQ(yield_callback_called, 20000);
-}
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/compaction_picker.cc

@@ -19,7 +19,6 @@
 #include "db/filename.h"
 #include "util/log_buffer.h"
 #include "util/statistics.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
@@ -47,7 +46,7 @@ CompressionType GetCompressionType(
   // If the use has specified a different compression level for each level,
   // then pick the compression for that level.
   if (!ioptions.compression_per_level.empty()) {
-    const int n = static_cast<int>(ioptions.compression_per_level.size()) - 1;
+    const int n = ioptions.compression_per_level.size() - 1;
     // It is possible for level_ to be -1; in that case, we use level
     // 0's compression.  This occurs mostly in backwards compatibility
     // situations when the builder doesn't know what level the file
@@ -76,7 +75,7 @@ void CompactionPicker::SizeBeingCompacted(std::vector<uint64_t>& sizes) {
     uint64_t total = 0;
     for (auto c : compactions_in_progress_[level]) {
       assert(c->level() == level);
-      for (size_t i = 0; i < c->num_input_files(0); i++) {
+      for (int i = 0; i < c->num_input_files(0); i++) {
         total += c->input(0, i)->compensated_file_size;
       }
     }
@@ -186,8 +185,7 @@ bool CompactionPicker::ExpandWhileOverlapping(const std::string& cf_name,
 }
 
 // Returns true if any one of specified files are being compacted
-bool CompactionPicker::FilesInCompaction(
-    const std::vector<FileMetaData*>& files) {
+bool CompactionPicker::FilesInCompaction(std::vector<FileMetaData*>& files) {
   for (unsigned int i = 0; i < files.size(); i++) {
     if (files[i]->being_compacted) {
       return true;
@@ -196,89 +194,6 @@ bool CompactionPicker::FilesInCompaction(
   return false;
 }
 
-Compaction* CompactionPicker::FormCompaction(
-      const CompactionOptions& compact_options,
-      const autovector<CompactionInputFiles>& input_files,
-      int output_level, VersionStorageInfo* vstorage,
-      const MutableCFOptions& mutable_cf_options) const {
-  uint64_t max_grandparent_overlap_bytes =
-      output_level + 1 < vstorage->num_levels() ?
-          mutable_cf_options.MaxGrandParentOverlapBytes(output_level + 1) :
-          std::numeric_limits<uint64_t>::max();
-  assert(input_files.size());
-  auto c = new Compaction(vstorage, input_files,
-      input_files[0].level, output_level,
-      max_grandparent_overlap_bytes,
-      compact_options, false);
-  c->mutable_cf_options_ = mutable_cf_options;
-  c->MarkFilesBeingCompacted(true);
-
-  // TODO(yhchiang): complete the SetBottomMostLevel as follows
-  // If there is no any key of the range in DB that is older than the
-  // range to compact, it is bottom most.  For leveled compaction,
-  // if number-of_level-1 is empty, and output is going to number-of_level-2,
-  // it is also bottom-most.  On the other hand, if number of level=1 (
-  // something like universal), the compaction is only "bottom-most" if
-  // the oldest file is involved.
-  c->SetupBottomMostLevel(
-      vstorage,
-      (output_level == vstorage->num_levels() - 1),
-      (output_level == 0));
-  return c;
-}
-
-Status CompactionPicker::GetCompactionInputsFromFileNumbers(
-    autovector<CompactionInputFiles>* input_files,
-    std::unordered_set<uint64_t>* input_set,
-    const VersionStorageInfo* vstorage,
-    const CompactionOptions& compact_options) const {
-  if (input_set->size() == 0U) {
-    return Status::InvalidArgument(
-        "Compaction must include at least one file.");
-  }
-  assert(input_files);
-
-  autovector<CompactionInputFiles> matched_input_files;
-  matched_input_files.resize(vstorage->num_levels());
-  int first_non_empty_level = -1;
-  int last_non_empty_level = -1;
-  // TODO(yhchiang): use a lazy-initialized mapping from
-  //                 file_number to FileMetaData in Version.
-  for (int level = 0; level < vstorage->num_levels(); ++level) {
-    for (auto file : vstorage->LevelFiles(level)) {
-      auto iter = input_set->find(file->fd.GetNumber());
-      if (iter != input_set->end()) {
-        matched_input_files[level].files.push_back(file);
-        input_set->erase(iter);
-        last_non_empty_level = level;
-        if (first_non_empty_level == -1) {
-          first_non_empty_level = level;
-        }
-      }
-    }
-  }
-
-  if (!input_set->empty()) {
-    std::string message(
-        "Cannot find matched SST files for the following file numbers:");
-    for (auto fn : *input_set) {
-      message += " ";
-      message += ToString(fn);
-    }
-    return Status::InvalidArgument(message);
-  }
-
-  for (int level = first_non_empty_level;
-       level <= last_non_empty_level; ++level) {
-    matched_input_files[level].level = level;
-    input_files->emplace_back(std::move(matched_input_files[level]));
-  }
-
-  return Status::OK();
-}
-
-
-
 // Returns true if any one of the parent files are being compacted
 bool CompactionPicker::ParentRangeInCompaction(VersionStorageInfo* vstorage,
                                                const InternalKey* smallest,
@@ -387,7 +302,6 @@ Compaction* CompactionPicker::CompactRange(
     begin = nullptr;
     end = nullptr;
   }
-
   vstorage->GetOverlappingInputs(input_level, begin, end, &inputs);
   if (inputs.empty()) {
     return nullptr;
@@ -448,247 +362,6 @@ Compaction* CompactionPicker::CompactRange(
   return c;
 }
 
-#ifndef ROCKSDB_LITE
-namespace {
-// Test whether two files have overlapping key-ranges.
-bool HaveOverlappingKeyRanges(
-    const Comparator* c,
-    const SstFileMetaData& a, const SstFileMetaData& b) {
-  if (c->Compare(a.smallestkey, b.smallestkey) >= 0) {
-    if (c->Compare(a.smallestkey, b.largestkey) <= 0) {
-      // b.smallestkey <= a.smallestkey <= b.largestkey
-      return true;
-    }
-  } else if (c->Compare(a.largestkey, b.smallestkey) >= 0) {
-    // a.smallestkey < b.smallestkey <= a.largestkey
-    return true;
-  }
-  if (c->Compare(a.largestkey, b.largestkey) <= 0) {
-    if (c->Compare(a.largestkey, b.smallestkey) >= 0) {
-      // b.smallestkey <= a.largestkey <= b.largestkey
-      return true;
-    }
-  } else if (c->Compare(a.smallestkey, b.largestkey) <= 0) {
-    // a.smallestkey <= b.largestkey < a.largestkey
-    return true;
-  }
-  return false;
-}
-}  // namespace
-
-Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels(
-      std::unordered_set<uint64_t>* input_files,
-      const ColumnFamilyMetaData& cf_meta,
-      const int output_level) const {
-  auto& levels = cf_meta.levels;
-  auto comparator = icmp_->user_comparator();
-
-  // TODO(yhchiang): If there is any input files of L1 or up and there
-  // is at least one L0 files. All L0 files older than the L0 file needs
-  // to be included. Otherwise, it is a false conditoin
-
-  // TODO(yhchiang): add is_adjustable to CompactionOptions
-
-  // the smallest and largest key of the current compaction input
-  std::string smallestkey;
-  std::string largestkey;
-  // a flag for initializing smallest and largest key
-  bool is_first = false;
-  const int kNotFound = -1;
-
-  // For each level, it does the following things:
-  // 1. Find the first and the last compaction input files
-  //    in the current level.
-  // 2. Include all files between the first and the last
-  //    compaction input files.
-  // 3. Update the compaction key-range.
-  // 4. For all remaining levels, include files that have
-  //    overlapping key-range with the compaction key-range.
-  for (int l = 0; l <= output_level; ++l) {
-    auto& current_files = levels[l].files;
-    int first_included = static_cast<int>(current_files.size());
-    int last_included = kNotFound;
-
-    // identify the first and the last compaction input files
-    // in the current level.
-    for (size_t f = 0; f < current_files.size(); ++f) {
-      if (input_files->find(TableFileNameToNumber(current_files[f].name)) !=
-          input_files->end()) {
-        first_included = std::min(first_included, static_cast<int>(f));
-        last_included = std::max(last_included, static_cast<int>(f));
-        if (is_first == false) {
-          smallestkey = current_files[f].smallestkey;
-          largestkey = current_files[f].largestkey;
-          is_first = true;
-        }
-      }
-    }
-    if (last_included == kNotFound) {
-      continue;
-    }
-
-    if (l != 0) {
-      // expend the compaction input of the current level if it
-      // has overlapping key-range with other non-compaction input
-      // files in the same level.
-      while (first_included > 0) {
-        if (comparator->Compare(
-                current_files[first_included - 1].largestkey,
-                current_files[first_included].smallestkey) < 0) {
-          break;
-        }
-        first_included--;
-      }
-
-      while (last_included < static_cast<int>(current_files.size()) - 1) {
-        if (comparator->Compare(
-                current_files[last_included + 1].smallestkey,
-                current_files[last_included].largestkey) > 0) {
-          break;
-        }
-        last_included++;
-      }
-    }
-
-    // include all files between the first and the last compaction input files.
-    for (int f = first_included; f <= last_included; ++f) {
-      if (current_files[f].being_compacted) {
-        return Status::Aborted(
-            "Necessary compaction input file " + current_files[f].name +
-            " is currently being compacted.");
-      }
-      input_files->insert(
-          TableFileNameToNumber(current_files[f].name));
-    }
-
-    // update smallest and largest key
-    if (l == 0) {
-      for (int f = first_included; f <= last_included; ++f) {
-        if (comparator->Compare(
-            smallestkey, current_files[f].smallestkey) > 0) {
-          smallestkey = current_files[f].smallestkey;
-        }
-        if (comparator->Compare(
-            largestkey, current_files[f].largestkey) < 0) {
-          largestkey = current_files[f].largestkey;
-        }
-      }
-    } else {
-      if (comparator->Compare(
-          smallestkey, current_files[first_included].smallestkey) > 0) {
-        smallestkey = current_files[first_included].smallestkey;
-      }
-      if (comparator->Compare(
-          largestkey, current_files[last_included].largestkey) < 0) {
-        largestkey = current_files[last_included].largestkey;
-      }
-    }
-
-    SstFileMetaData aggregated_file_meta;
-    aggregated_file_meta.smallestkey = smallestkey;
-    aggregated_file_meta.largestkey = largestkey;
-
-    // For all lower levels, include all overlapping files.
-    for (int m = l + 1; m <= output_level; ++m) {
-      for (auto& next_lv_file : levels[m].files) {
-        if (HaveOverlappingKeyRanges(
-            comparator, aggregated_file_meta, next_lv_file)) {
-          if (next_lv_file.being_compacted) {
-            return Status::Aborted(
-                "File " + next_lv_file.name +
-                " that has overlapping key range with one of the compaction "
-                " input file is currently being compacted.");
-          }
-          input_files->insert(
-              TableFileNameToNumber(next_lv_file.name));
-        }
-      }
-    }
-  }
-  return Status::OK();
-}
-
-Status CompactionPicker::SanitizeCompactionInputFiles(
-    std::unordered_set<uint64_t>* input_files,
-    const ColumnFamilyMetaData& cf_meta,
-    const int output_level) const {
-  assert(static_cast<int>(cf_meta.levels.size()) - 1 ==
-         cf_meta.levels[cf_meta.levels.size() - 1].level);
-  if (output_level >= static_cast<int>(cf_meta.levels.size())) {
-    return Status::InvalidArgument(
-        "Output level for column family " + cf_meta.name +
-        " must between [0, " +
-        ToString(cf_meta.levels[cf_meta.levels.size() - 1].level) +
-        "].");
-  }
-
-  if (output_level > MaxOutputLevel()) {
-    return Status::InvalidArgument(
-        "Exceed the maximum output level defined by "
-        "the current compaction algorithm --- " +
-            ToString(MaxOutputLevel()));
-  }
-
-  if (output_level < 0) {
-    return Status::InvalidArgument(
-        "Output level cannot be negative.");
-  }
-
-  if (input_files->size() == 0) {
-    return Status::InvalidArgument(
-        "A compaction must contain at least one file.");
-  }
-
-  Status s = SanitizeCompactionInputFilesForAllLevels(
-      input_files, cf_meta, output_level);
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  // for all input files, check whether the file number matches
-  // any currently-existing files.
-  for (auto file_num : *input_files) {
-    bool found = false;
-    for (auto level_meta : cf_meta.levels) {
-      for (auto file_meta : level_meta.files) {
-        if (file_num == TableFileNameToNumber(file_meta.name)) {
-          if (file_meta.being_compacted) {
-            return Status::Aborted(
-                "Specified compaction input file " +
-                MakeTableFileName("", file_num) +
-                " is already being compacted.");
-          }
-          found = true;
-          break;
-        }
-      }
-      if (found) {
-        break;
-      }
-    }
-    if (!found) {
-      return Status::InvalidArgument(
-          "Specified compaction input file " +
-          MakeTableFileName("", file_num) +
-          " does not exist in column family " + cf_meta.name + ".");
-    }
-  }
-
-  return Status::OK();
-}
-#endif  // !ROCKSDB_LITE
-
-bool LevelCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage)
-    const {
-  for (int i = 0; i <= vstorage->MaxInputLevel(); i++) {
-    if (vstorage->CompactionScore(i) >= 1) {
-      return true;
-    }
-  }
-  return false;
-}
-
 Compaction* LevelCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
@@ -843,13 +516,6 @@ Compaction* LevelCompactionPicker::PickCompactionBySize(
   return c;
 }
 
-#ifndef ROCKSDB_LITE
-bool UniversalCompactionPicker::NeedsCompaction(
-    const VersionStorageInfo* vstorage) const {
-  const int kLevel0 = 0;
-  return vstorage->CompactionScore(kLevel0) >= 1;
-}
-
 // Universal style of compaction. Pick files that are contiguous in
 // time-range to compact.
 //
@@ -891,8 +557,7 @@ Compaction* UniversalCompactionPicker::PickCompaction(
       // If max read amplification is exceeding configured limits, then force
       // compaction without looking at filesize ratios and try to reduce
       // the number of files to fewer than level0_file_num_compaction_trigger.
-      unsigned int num_files =
-          static_cast<unsigned int>(level_files.size()) -
+      unsigned int num_files = level_files.size() -
           mutable_cf_options.level0_file_num_compaction_trigger;
       if ((c = PickCompactionUniversalReadAmp(
                cf_name, mutable_cf_options, vstorage, score, UINT_MAX,
@@ -1096,7 +761,8 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
   if (ratio_to_compress >= 0) {
     uint64_t total_size = vstorage->NumLevelBytes(kLevel0);
     uint64_t older_file_size = 0;
-    for (size_t i = files.size() - 1; i >= first_index_after; i--) {
+    for (unsigned int i = files.size() - 1;
+         i >= first_index_after; i--) {
       older_file_size += files[i]->fd.GetFileSize();
       if (older_file_size * 100L >= total_size * (long) ratio_to_compress) {
         enable_compression = false;
@@ -1249,12 +915,6 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
   return c;
 }
 
-bool FIFOCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage)
-    const {
-  const int kLevel0 = 0;
-  return vstorage->CompactionScore(kLevel0) >= 1;
-}
-
 Compaction* FIFOCompactionPicker::PickCompaction(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
@@ -1263,7 +923,7 @@ Compaction* FIFOCompactionPicker::PickCompaction(
   const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
   uint64_t total_size = 0;
   for (const auto& file : level_files) {
-    total_size += file->fd.file_size;
+    total_size += file->compensated_file_size;
   }
 
   if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size ||
@@ -1327,6 +987,4 @@ Compaction* FIFOCompactionPicker::CompactRange(
   return c;
 }
 
-#endif  // !ROCKSDB_LITE
-
 }  // namespace rocksdb

db/compaction_picker.h

@@ -8,11 +8,6 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
-#include <vector>
-#include <memory>
-#include <set>
-#include <unordered_set>
-
 #include "db/version_set.h"
 #include "db/compaction.h"
 #include "rocksdb/status.h"
@@ -30,7 +25,6 @@ namespace rocksdb {
 class LogBuffer;
 class Compaction;
 class VersionStorageInfo;
-struct CompactionInputFiles;
 
 class CompactionPicker {
  public:
@@ -68,26 +62,6 @@ class CompactionPicker {
   // for compaction input.
   virtual int MaxInputLevel(int current_num_levels) const = 0;
 
-  // The maximum allowed output level.  Default value is NumberLevels() - 1.
-  virtual int MaxOutputLevel() const {
-    return NumberLevels() - 1;
-  }
-
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const = 0;
-
-  // Sanitize the input set of compaction input files.
-  // When the input parameters do not describe a valid compaction, the
-  // function will try to fix the input_files by adding necessary
-  // files.  If it's not possible to conver an invalid input_files
-  // into a valid one by adding more files, the function will return a
-  // non-ok status with specific reason.
-#ifndef ROCKSDB_LITE
-  Status SanitizeCompactionInputFiles(
-      std::unordered_set<uint64_t>* input_files,
-      const ColumnFamilyMetaData& cf_meta,
-      const int output_level) const;
-#endif  // ROCKSDB_LITE
-
   // Free up the files that participated in a compaction
   void ReleaseCompactionFiles(Compaction* c, Status status);
 
@@ -95,24 +69,6 @@ class CompactionPicker {
   // compactions per level
   void SizeBeingCompacted(std::vector<uint64_t>& sizes);
 
-  // Returns true if any one of the specified files are being compacted
-  bool FilesInCompaction(const std::vector<FileMetaData*>& files);
-
-  // Takes a list of CompactionInputFiles and returns a Compaction object.
-  Compaction* FormCompaction(
-      const CompactionOptions& compact_options,
-      const autovector<CompactionInputFiles>& input_files,
-      int output_level, VersionStorageInfo* vstorage,
-      const MutableCFOptions& mutable_cf_options) const;
-
-  // Converts a set of compaction input file numbers into
-  // a list of CompactionInputFiles.
-  Status GetCompactionInputsFromFileNumbers(
-      autovector<CompactionInputFiles>* input_files,
-      std::unordered_set<uint64_t>* input_set,
-      const VersionStorageInfo* vstorage,
-      const CompactionOptions& compact_options) const;
-
  protected:
   int NumberLevels() const { return ioptions_.num_levels; }
 
@@ -142,6 +98,9 @@ class CompactionPicker {
   bool ExpandWhileOverlapping(const std::string& cf_name,
                               VersionStorageInfo* vstorage, Compaction* c);
 
+  // Returns true if any one of the specified files are being compacted
+  bool FilesInCompaction(std::vector<FileMetaData*>& files);
+
   // Returns true if any one of the parent files are being compacted
   bool ParentRangeInCompaction(VersionStorageInfo* vstorage,
                                const InternalKey* smallest,
@@ -154,51 +113,14 @@ class CompactionPicker {
 
   const ImmutableCFOptions& ioptions_;
 
-  // A helper function to SanitizeCompactionInputFiles() that
-  // sanitizes "input_files" by adding necessary files.
-#ifndef ROCKSDB_LITE
-  virtual Status SanitizeCompactionInputFilesForAllLevels(
-      std::unordered_set<uint64_t>* input_files,
-      const ColumnFamilyMetaData& cf_meta,
-      const int output_level) const;
-#endif  // ROCKSDB_LITE
-
   // record all the ongoing compactions for all levels
   std::vector<std::set<Compaction*>> compactions_in_progress_;
 
+
+ private:
   const InternalKeyComparator* const icmp_;
 };
 
-class LevelCompactionPicker : public CompactionPicker {
- public:
-  LevelCompactionPicker(const ImmutableCFOptions& ioptions,
-                        const InternalKeyComparator* icmp)
-      : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) override;
-
-  // Returns current_num_levels - 2, meaning the last level cannot be
-  // compaction input level.
-  virtual int MaxInputLevel(int current_num_levels) const override {
-    return current_num_levels - 2;
-  }
-
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
-      override;
-
- private:
-  // For the specfied level, pick a compaction.
-  // Returns nullptr if there is no compaction to be done.
-  // If level is 0 and there is already a compaction on that level, this
-  // function will return nullptr.
-  Compaction* PickCompactionBySize(const MutableCFOptions& mutable_cf_options,
-                                   VersionStorageInfo* vstorage, int level,
-                                   double score);
-};
-
-#ifndef ROCKSDB_LITE
 class UniversalCompactionPicker : public CompactionPicker {
  public:
   UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
@@ -209,19 +131,11 @@ class UniversalCompactionPicker : public CompactionPicker {
                                      VersionStorageInfo* vstorage,
                                      LogBuffer* log_buffer) override;
 
-  // The maxinum allowed input level.  Always returns 0.
+  // The maxinum allowed input level.  Always return 0.
   virtual int MaxInputLevel(int current_num_levels) const override {
     return 0;
   }
 
-  // The maximum allowed output level.  Always returns 0.
-  virtual int MaxOutputLevel() const override {
-    return 0;
-  }
-
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
-      override;
-
  private:
   // Pick Universal compaction to limit read amplification
   Compaction* PickCompactionUniversalReadAmp(
@@ -240,6 +154,32 @@ class UniversalCompactionPicker : public CompactionPicker {
                             uint64_t file_size);
 };
 
+class LevelCompactionPicker : public CompactionPicker {
+ public:
+  LevelCompactionPicker(const ImmutableCFOptions& ioptions,
+                        const InternalKeyComparator* icmp)
+      : CompactionPicker(ioptions, icmp) {}
+  virtual Compaction* PickCompaction(const std::string& cf_name,
+                                     const MutableCFOptions& mutable_cf_options,
+                                     VersionStorageInfo* vstorage,
+                                     LogBuffer* log_buffer) override;
+
+  // Returns current_num_levels - 2, meaning the last level cannot be
+  // compaction input level.
+  virtual int MaxInputLevel(int current_num_levels) const override {
+    return current_num_levels - 2;
+  }
+
+ private:
+  // For the specfied level, pick a compaction.
+  // Returns nullptr if there is no compaction to be done.
+  // If level is 0 and there is already a compaction on that level, this
+  // function will return nullptr.
+  Compaction* PickCompactionBySize(const MutableCFOptions& mutable_cf_options,
+                                   VersionStorageInfo* vstorage, int level,
+                                   double score);
+};
+
 class FIFOCompactionPicker : public CompactionPicker {
  public:
   FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
@@ -257,58 +197,12 @@ class FIFOCompactionPicker : public CompactionPicker {
       uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
       InternalKey** compaction_end) override;
 
-  // The maxinum allowed input level.  Always returns 0.
+  // The maxinum allowed input level.  Always return 0.
   virtual int MaxInputLevel(int current_num_levels) const override {
     return 0;
   }
-
-  // The maximum allowed output level.  Always returns 0.
-  virtual int MaxOutputLevel() const override {
-    return 0;
-  }
-
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
-      override;
 };
 
-class NullCompactionPicker : public CompactionPicker {
- public:
-  NullCompactionPicker(const ImmutableCFOptions& ioptions,
-                       const InternalKeyComparator* icmp) :
-      CompactionPicker(ioptions, icmp) {}
-  virtual ~NullCompactionPicker() {}
-
-  // Always return "nullptr"
-  Compaction* PickCompaction(const std::string& cf_name,
-                             const MutableCFOptions& mutable_cf_options,
-                             VersionStorageInfo* vstorage,
-                             LogBuffer* log_buffer) override {
-    return nullptr;
-  }
-
-  // Always return "nullptr"
-  Compaction* CompactRange(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, int input_level, int output_level,
-      uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
-      InternalKey** compaction_end) override {
-    return nullptr;
-  }
-
-  // Given the current number of levels, returns the highest allowed level
-  // for compaction input.
-  virtual int MaxInputLevel(int current_num_levels) const {
-    return current_num_levels - 2;
-  }
-
-  // Always returns false.
-  virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
-      override {
-    return false;
-  }
-};
-#endif  // !ROCKSDB_LITE
-
 // Utility function
 extern uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files);
 

db/compaction_picker_test.cc

@@ -19,126 +19,113 @@ class CountingLogger : public Logger {
 
 class CompactionPickerTest {
  public:
-  const Comparator* ucmp_;
-  InternalKeyComparator icmp_;
-  Options options_;
-  ImmutableCFOptions ioptions_;
-  MutableCFOptions mutable_cf_options_;
+  const Comparator* ucmp;
+  InternalKeyComparator icmp;
+  Options options;
+  ImmutableCFOptions ioptions;
+  MutableCFOptions mutable_cf_options;
   LevelCompactionPicker level_compaction_picker;
-  std::string cf_name_;
-  CountingLogger logger_;
-  LogBuffer log_buffer_;
-  uint32_t file_num_;
-  CompactionOptionsFIFO fifo_options_;
-  std::vector<uint64_t> size_being_compacted_;
-  std::unique_ptr<VersionStorageInfo> vstorage_;
-  std::vector<std::unique_ptr<FileMetaData>> files_;
+  std::string cf_name;
+  CountingLogger logger;
+  LogBuffer log_buffer;
+  VersionStorageInfo vstorage;
+  uint32_t file_num;
+  CompactionOptionsFIFO fifo_options;
+  std::vector<uint64_t> size_being_compacted;
 
   CompactionPickerTest()
-      : ucmp_(BytewiseComparator()),
-        icmp_(ucmp_),
-        ioptions_(options_),
-        mutable_cf_options_(options_, ioptions_),
-        level_compaction_picker(ioptions_, &icmp_),
-        cf_name_("dummy"),
-        log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
-        file_num_(1),
-        vstorage_(nullptr) {
-    fifo_options_.max_table_files_size = 1;
-    mutable_cf_options_.RefreshDerivedOptions(ioptions_);
-    size_being_compacted_.resize(options_.num_levels);
+      : ucmp(BytewiseComparator()),
+        icmp(ucmp),
+        ioptions(options),
+        mutable_cf_options(options, ioptions),
+        level_compaction_picker(ioptions, &icmp),
+        cf_name("dummy"),
+        log_buffer(InfoLogLevel::INFO_LEVEL, &logger),
+        vstorage(&icmp, ucmp, options.num_levels, kCompactionStyleLevel,
+                 nullptr),
+        file_num(1) {
+    fifo_options.max_table_files_size = 1;
+    mutable_cf_options.RefreshDerivedOptions(ioptions);
+    size_being_compacted.resize(options.num_levels);
   }
 
   ~CompactionPickerTest() {
-  }
-
-  void NewVersionStorage(int num_levels, CompactionStyle style) {
-    DeleteVersionStorage();
-    options_.num_levels = num_levels;
-    vstorage_.reset(new VersionStorageInfo(
-        &icmp_, ucmp_, options_.num_levels, style, nullptr));
-  }
-
-  void DeleteVersionStorage() {
-    vstorage_.reset();
-    files_.clear();
+    for (int i = 0; i < vstorage.num_levels(); i++) {
+      for (auto* f : vstorage.LevelFiles(i)) {
+        delete f;
+      }
+    }
   }
 
   void Add(int level, uint32_t file_number, const char* smallest,
            const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
            SequenceNumber smallest_seq = 100,
            SequenceNumber largest_seq = 100) {
-    assert(level < vstorage_->num_levels());
+    assert(level < vstorage.num_levels());
     FileMetaData* f = new FileMetaData;
     f->fd = FileDescriptor(file_number, path_id, file_size);
     f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
     f->largest = InternalKey(largest, largest_seq, kTypeValue);
     f->compensated_file_size = file_size;
     f->refs = 0;
-    vstorage_->AddFile(level, f);
-    files_.emplace_back(f);
+    vstorage.MaybeAddFile(level, f);
   }
 
   void UpdateVersionStorageInfo() {
-    vstorage_->ComputeCompactionScore(mutable_cf_options_, fifo_options_,
-                                    size_being_compacted_);
-    vstorage_->UpdateFilesBySize();
-    vstorage_->UpdateNumNonEmptyLevels();
-    vstorage_->GenerateFileIndexer();
-    vstorage_->GenerateLevelFilesBrief();
-    vstorage_->SetFinalized();
+    vstorage.ComputeCompactionScore(mutable_cf_options, fifo_options,
+                                    size_being_compacted);
+    vstorage.UpdateFilesBySize();
+    vstorage.UpdateNumNonEmptyLevels();
+    vstorage.GenerateFileIndexer();
+    vstorage.GenerateLevelFilesBrief();
+    vstorage.SetFinalized();
   }
 };
 
 TEST(CompactionPickerTest, Empty) {
-  NewVersionStorage(6, kCompactionStyleLevel);
   UpdateVersionStorageInfo();
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+      cf_name, mutable_cf_options, &vstorage, &log_buffer));
   ASSERT_TRUE(compaction.get() == nullptr);
 }
 
 TEST(CompactionPickerTest, Single) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
+  mutable_cf_options.level0_file_num_compaction_trigger = 2;
   Add(0, 1U, "p", "q");
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+      cf_name, mutable_cf_options, &vstorage, &log_buffer));
   ASSERT_TRUE(compaction.get() == nullptr);
 }
 
 TEST(CompactionPickerTest, Level0Trigger) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
+  mutable_cf_options.level0_file_num_compaction_trigger = 2;
   Add(0, 1U, "150", "200");
   Add(0, 2U, "200", "250");
 
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+      cf_name, mutable_cf_options, &vstorage, &log_buffer));
   ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_files(0));
+  ASSERT_EQ(2, compaction->num_input_files(0));
   ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
   ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
 }
 
 TEST(CompactionPickerTest, Level1Trigger) {
-  NewVersionStorage(6, kCompactionStyleLevel);
   Add(1, 66U, "150", "200", 1000000000U);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+      cf_name, mutable_cf_options, &vstorage, &log_buffer));
   ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
+  ASSERT_EQ(1, compaction->num_input_files(0));
   ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
 }
 
 TEST(CompactionPickerTest, Level1Trigger2) {
-  NewVersionStorage(6, kCompactionStyleLevel);
   Add(1, 66U, "150", "200", 1000000001U);
   Add(1, 88U, "201", "300", 1000000000U);
   Add(2, 6U, "150", "179", 1000000000U);
@@ -147,19 +134,18 @@ TEST(CompactionPickerTest, Level1Trigger2) {
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+      cf_name, mutable_cf_options, &vstorage, &log_buffer));
   ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(2U, compaction->num_input_files(1));
+  ASSERT_EQ(1, compaction->num_input_files(0));
+  ASSERT_EQ(2, compaction->num_input_files(1));
   ASSERT_EQ(66U, compaction->input(0, 0)->fd.GetNumber());
   ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
   ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
 }
 
 TEST(CompactionPickerTest, LevelMaxScore) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.target_file_size_base = 10000000;
-  mutable_cf_options_.target_file_size_multiplier = 10;
+  mutable_cf_options.target_file_size_base = 10000000;
+  mutable_cf_options.target_file_size_multiplier = 10;
   Add(0, 1U, "150", "200", 1000000000U);
   // Level 1 score 1.2
   Add(1, 66U, "150", "200", 6000000U);
@@ -176,83 +162,12 @@ TEST(CompactionPickerTest, LevelMaxScore) {
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+      cf_name, mutable_cf_options, &vstorage, &log_buffer));
   ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_files(0));
+  ASSERT_EQ(1, compaction->num_input_files(0));
   ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
 }
 
-TEST(CompactionPickerTest, NeedsCompactionLevel) {
-  const int kLevels = 6;
-  const int kFileCount = 20;
-  for (int level = 0; level < kLevels - 1; ++level) {
-    uint64_t file_size =
-        mutable_cf_options_.MaxBytesForLevel(level) * 2 / kFileCount;
-    for (int file_count = 1; file_count <= kFileCount; ++file_count) {
-      // start a brand new version in each test.
-      NewVersionStorage(kLevels, kCompactionStyleLevel);
-      for (int i = 0; i < file_count; ++i) {
-        Add(level, i, ToString((i + 100) * 1000).c_str(),
-            ToString((i + 100) * 1000 + 999).c_str(),
-            file_size, 0, i * 100, i * 100 + 99);
-      }
-      UpdateVersionStorageInfo();
-      ASSERT_EQ(vstorage_->CompactionScoreLevel(0), level);
-      ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
-                vstorage_->CompactionScore(0) >= 1);
-      // release the version storage
-      DeleteVersionStorage();
-    }
-  }
-}
-
-TEST(CompactionPickerTest, NeedsCompactionUniversal) {
-  NewVersionStorage(1, kCompactionStyleUniversal);
-  UniversalCompactionPicker universal_compaction_picker(
-      ioptions_, &icmp_);
-  // must return false when there's no files.
-  ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
-            false);
-
-  // verify the trigger given different number of L0 files.
-  for (int i = 1;
-       i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
-    Add(0, i, ToString((i + 100) * 1000).c_str(),
-        ToString((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
-        i * 100 + 99);
-    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
-              vstorage_->CompactionScore(0) >= 1);
-  }
-}
-
-TEST(CompactionPickerTest, NeedsCompactionFIFO) {
-  NewVersionStorage(1, kCompactionStyleFIFO);
-  const int kFileCount =
-      mutable_cf_options_.level0_file_num_compaction_trigger * 3;
-  const uint64_t kFileSize = 100000;
-  const uint64_t kMaxSize = kFileSize * kFileCount / 2;
-
-  fifo_options_.max_table_files_size = kMaxSize;
-  ioptions_.compaction_options_fifo = fifo_options_;
-  FIFOCompactionPicker fifo_compaction_picker(ioptions_, &icmp_);
-
-  // must return false when there's no files.
-  ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()), false);
-
-  // verify whether compaction is needed based on the current
-  // size of L0 files.
-  uint64_t current_size = 0;
-  for (int i = 1; i <= kFileCount; ++i) {
-    Add(0, i, ToString((i + 100) * 1000).c_str(),
-        ToString((i + 100) * 1000 + 999).c_str(),
-        kFileSize, 0, i * 100, i * 100 + 99);
-    current_size += kFileSize;
-    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
-              vstorage_->CompactionScore(0) >= 1);
-  }
-}
-
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/comparator_db_test.cc

@@ -82,7 +82,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
     }
 
     int type = rnd->Uniform(2);
-    int index = rnd->Uniform(static_cast<int>(source_strings.size()));
+    int index = rnd->Uniform(source_strings.size());
     auto& key = source_strings[index];
     switch (type) {
       case 0:
@@ -124,7 +124,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
         break;
       case 2: {
         // Seek to random key
-        auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
+        auto key_idx = rnd->Uniform(source_strings.size());
         auto key = source_strings[key_idx];
         iter->Seek(key);
         result_iter->Seek(key);
@@ -150,7 +150,7 @@ void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
         break;
       default: {
         assert(type == 5);
-        auto key_idx = rnd->Uniform(static_cast<int>(source_strings.size()));
+        auto key_idx = rnd->Uniform(source_strings.size());
         auto key = source_strings[key_idx];
         std::string result;
         auto status = db->Get(ReadOptions(), key, &result);
@@ -325,7 +325,7 @@ TEST(ComparatorDBTest, SimpleSuffixReverseComparator) {
       source_prefixes.push_back(test::RandomHumanReadableString(&rnd, 8));
     }
     for (int j = 0; j < 20; j++) {
-      int prefix_index = rnd.Uniform(static_cast<int>(source_prefixes.size()));
+      int prefix_index = rnd.Uniform(source_prefixes.size());
       std::string key = source_prefixes[prefix_index] +
                         test::RandomHumanReadableString(&rnd, rnd.Uniform(8));
       source_strings.push_back(key);
@@ -377,7 +377,7 @@ TEST(ComparatorDBTest, DoubleComparator) {
       for (uint32_t j = 0; j < divide_order; j++) {
         to_divide *= 10.0;
       }
-      source_strings.push_back(ToString(r / to_divide));
+      source_strings.push_back(std::to_string(r / to_divide));
     }
 
     DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);

db/corruption_test.cc

@@ -115,8 +115,8 @@ class CorruptionTest {
         continue;
       }
       missed += (key - next_expected);
-      next_expected = static_cast<unsigned int>(key + 1);
-      if (iter->value() != Value(static_cast<int>(key), &value_space)) {
+      next_expected = key + 1;
+      if (iter->value() != Value(key, &value_space)) {
         bad_values++;
       } else {
         correct++;
@@ -143,14 +143,14 @@ class CorruptionTest {
       if (-offset > sbuf.st_size) {
         offset = 0;
       } else {
-        offset = static_cast<int>(sbuf.st_size + offset);
+        offset = sbuf.st_size + offset;
       }
     }
     if (offset > sbuf.st_size) {
-      offset = static_cast<int>(sbuf.st_size);
+      offset = sbuf.st_size;
     }
     if (offset + bytes_to_corrupt > sbuf.st_size) {
-      bytes_to_corrupt = static_cast<int>(sbuf.st_size - offset);
+      bytes_to_corrupt = sbuf.st_size - offset;
     }
 
     // Do it
@@ -177,7 +177,7 @@ class CorruptionTest {
           type == filetype &&
           static_cast<int>(number) > picked_number) {  // Pick latest file
         fname = dbname_ + "/" + filenames[i];
-        picked_number = static_cast<int>(number);
+        picked_number = number;
       }
     }
     ASSERT_TRUE(!fname.empty()) << filetype;
@@ -246,8 +246,7 @@ TEST(CorruptionTest, RecoverWriteError) {
 TEST(CorruptionTest, NewFileErrorDuringWrite) {
   // Do enough writing to force minor compaction
   env_.writable_file_error_ = true;
-  const int num =
-      static_cast<int>(3 + (Options().write_buffer_size / kValueSize));
+  const int num = 3 + (Options().write_buffer_size / kValueSize);
   std::string value_storage;
   Status s;
   bool failed = false;

db/cuckoo_table_db_test.cc

@@ -92,7 +92,7 @@ class CuckooTableDBTest {
   // Return spread of files per level
   std::string FilesPerLevel() {
     std::string result;
-    size_t last_non_zero_offset = 0;
+    int last_non_zero_offset = 0;
     for (int level = 0; level < db_->NumberLevels(); level++) {
       int f = NumTableFilesAtLevel(level);
       char buf[100];
@@ -228,6 +228,33 @@ TEST(CuckooTableDBTest, Uint64Comparator) {
   ASSERT_EQ("v4", Get(Uint64Key(4)));
 }
 
+TEST(CuckooTableDBTest, CompactionTrigger) {
+  Options options = CurrentOptions();
+  options.write_buffer_size = 100 << 10;  // 100KB
+  options.level0_file_num_compaction_trigger = 2;
+  Reopen(&options);
+
+  // Write 11 values, each 10016 B
+  for (int idx = 0; idx < 11; ++idx) {
+    ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + idx)));
+  }
+  dbfull()->TEST_WaitForFlushMemTable();
+  ASSERT_EQ("1", FilesPerLevel());
+
+  // Generate one more file in level-0, and should trigger level-0 compaction
+  for (int idx = 11; idx < 22; ++idx) {
+    ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + idx)));
+  }
+  dbfull()->TEST_WaitForFlushMemTable();
+  ASSERT_EQ("2", FilesPerLevel());
+
+  dbfull()->TEST_CompactRange(0, nullptr, nullptr);
+  ASSERT_EQ("0,2", FilesPerLevel());
+  for (int idx = 0; idx < 22; ++idx) {
+    ASSERT_EQ(std::string(10000, 'a' + idx), Get(Key(idx)));
+  }
+}
+
 TEST(CuckooTableDBTest, CompactionIntoMultipleFiles) {
   // Create a big L0 file and check it compacts into multiple files in L1.
   Options options = CurrentOptions();

db/db_bench.cc

@@ -198,9 +198,6 @@ DEFINE_bool(enable_numa, false,
             "CPU and memory of same node. Use \"$numactl --hardware\" command "
             "to see NUMA memory architecture.");
 
-DEFINE_int64(db_write_buffer_size, rocksdb::Options().db_write_buffer_size,
-             "Number of bytes to buffer in all memtables before compacting");
-
 DEFINE_int64(write_buffer_size, rocksdb::Options().write_buffer_size,
              "Number of bytes to buffer in memtable before compacting");
 
@@ -254,8 +251,7 @@ DEFINE_int32(universal_compression_size_percent, -1,
 DEFINE_int64(cache_size, -1, "Number of bytes to use as a cache of uncompressed"
              "data. Negative means use default settings.");
 
-DEFINE_int32(block_size,
-             static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size),
+DEFINE_int32(block_size, rocksdb::BlockBasedTableOptions().block_size,
              "Number of bytes in a block.");
 
 DEFINE_int32(block_restart_interval,
@@ -1196,7 +1192,7 @@ class Benchmark {
     while (start < s.size() && isspace(s[start])) {
       start++;
     }
-    unsigned int limit = static_cast<unsigned int>(s.size());
+    unsigned int limit = s.size();
     while (limit > start && isspace(s[limit-1])) {
       limit--;
     }
@@ -1353,7 +1349,7 @@ class Benchmark {
   }
 
   std::string GetDbNameForMultiple(std::string base_name, size_t id) {
-    return base_name + ToString(id);
+    return base_name + std::to_string(id);
   }
 
   std::string ColumnFamilyName(int i) {
@@ -1367,13 +1363,11 @@ class Benchmark {
   }
 
   void Run() {
-    Options open_options;  // keep options around to properly destroy db later
-
     if (!SanityCheck()) {
       exit(1);
     }
     PrintHeader();
-    Open(&open_options);
+    Open();
     const char* benchmarks = FLAGS_benchmarks.c_str();
     while (benchmarks != nullptr) {
       const char* sep = strchr(benchmarks, ',');
@@ -1534,15 +1528,15 @@ class Benchmark {
             delete db_.db;
             db_.db = nullptr;
             db_.cfh.clear();
-            DestroyDB(FLAGS_db, open_options);
+            DestroyDB(FLAGS_db, Options());
           }
           for (size_t i = 0; i < multi_dbs_.size(); i++) {
             delete multi_dbs_[i].db;
-            DestroyDB(GetDbNameForMultiple(FLAGS_db, i), open_options);
+            DestroyDB(GetDbNameForMultiple(FLAGS_db, i), Options());
           }
           multi_dbs_.clear();
         }
-        Open(&open_options);  // use open_options for the last accessed
+        Open();
       }
 
       if (method != nullptr) {
@@ -1834,14 +1828,11 @@ class Benchmark {
     }
   }
 
-  void Open(Options* opts) {
-    Options& options = *opts;
-
+  void Open() {
     assert(db_.db == nullptr);
-
+    Options options;
     options.create_if_missing = !FLAGS_use_existing_db;
     options.create_missing_column_families = FLAGS_num_column_families > 1;
-    options.db_write_buffer_size = FLAGS_db_write_buffer_size;
     options.write_buffer_size = FLAGS_write_buffer_size;
     options.max_write_buffer_number = FLAGS_max_write_buffer_number;
     options.min_write_buffer_number_to_merge =
@@ -1886,15 +1877,14 @@ class Benchmark {
       exit(1);
     }
     switch (FLAGS_rep_factory) {
+      case kPrefixHash:
+        options.memtable_factory.reset(NewHashSkipListRepFactory(
+            FLAGS_hash_bucket_count));
+        break;
       case kSkipList:
         options.memtable_factory.reset(new SkipListFactory(
             FLAGS_skip_list_lookahead));
         break;
-#ifndef ROCKSDB_LITE
-      case kPrefixHash:
-        options.memtable_factory.reset(
-            NewHashSkipListRepFactory(FLAGS_hash_bucket_count));
-        break;
       case kHashLinkedList:
         options.memtable_factory.reset(NewHashLinkListRepFactory(
             FLAGS_hash_bucket_count));
@@ -1908,14 +1898,8 @@ class Benchmark {
         options.memtable_factory.reset(NewHashCuckooRepFactory(
             options.write_buffer_size, FLAGS_key_size + FLAGS_value_size));
         break;
-#else
-      default:
-        fprintf(stderr, "Only skip list is supported in lite mode\n");
-        exit(1);
-#endif  // ROCKSDB_LITE
     }
     if (FLAGS_use_plain_table) {
-#ifndef ROCKSDB_LITE
       if (FLAGS_rep_factory != kPrefixHash &&
           FLAGS_rep_factory != kHashLinkedList) {
         fprintf(stderr, "Waring: plain table is used with skipList\n");
@@ -1936,12 +1920,7 @@ class Benchmark {
       plain_table_options.hash_table_ratio = 0.75;
       options.table_factory = std::shared_ptr<TableFactory>(
           NewPlainTableFactory(plain_table_options));
-#else
-      fprintf(stderr, "Plain table is not supported in lite mode\n");
-      exit(1);
-#endif  // ROCKSDB_LITE
     } else if (FLAGS_use_cuckoo_table) {
-#ifndef ROCKSDB_LITE
       if (FLAGS_cuckoo_hash_ratio > 1 || FLAGS_cuckoo_hash_ratio < 0) {
         fprintf(stderr, "Invalid cuckoo_hash_ratio\n");
         exit(1);
@@ -1951,10 +1930,6 @@ class Benchmark {
       table_options.identity_as_first_hash = FLAGS_identity_as_first_hash;
       options.table_factory = std::shared_ptr<TableFactory>(
           NewCuckooTableFactory(table_options));
-#else
-      fprintf(stderr, "Cuckoo table is not supported in lite mode\n");
-      exit(1);
-#endif  // ROCKSDB_LITE
     } else {
       BlockBasedTableOptions block_based_options;
       if (FLAGS_use_hash_search) {
@@ -2136,9 +2111,8 @@ class Benchmark {
         for (uint64_t i = 0; i < num_; ++i) {
           values_[i] = i;
         }
-        std::shuffle(
-            values_.begin(), values_.end(),
-            std::default_random_engine(static_cast<unsigned int>(FLAGS_seed)));
+        std::shuffle(values_.begin(), values_.end(),
+            std::default_random_engine(FLAGS_seed));
       }
     }
 
@@ -2246,10 +2220,7 @@ class Benchmark {
   }
 
   void ReadSequential(ThreadState* thread, DB* db) {
-    ReadOptions options(FLAGS_verify_checksum, true);
-    options.tailing = FLAGS_use_tailing_iterator;
-
-    Iterator* iter = db->NewIterator(options);
+    Iterator* iter = db->NewIterator(ReadOptions(FLAGS_verify_checksum, true));
     int64_t i = 0;
     int64_t bytes = 0;
     for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
@@ -3066,8 +3037,8 @@ int main(int argc, char** argv) {
     dbstats = rocksdb::CreateDBStatistics();
   }
 
-  std::vector<std::string> fanout = rocksdb::StringSplit(
-      FLAGS_max_bytes_for_level_multiplier_additional, ',');
+  std::vector<std::string> fanout =
+    rocksdb::stringSplit(FLAGS_max_bytes_for_level_multiplier_additional, ',');
   for (unsigned int j= 0; j < fanout.size(); j++) {
     FLAGS_max_bytes_for_level_multiplier_additional_v.push_back(
       std::stoi(fanout[j]));

db/db_filesnapshot.cc

@@ -19,14 +19,12 @@
 #include <stdint.h>
 #include "db/db_impl.h"
 #include "db/filename.h"
-#include "db/job_context.h"
 #include "db/version_set.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "port/port.h"
 #include "util/mutexlock.h"
 #include "util/sync_point.h"
-#include "util/file_util.h"
 
 namespace rocksdb {
 
@@ -69,7 +67,6 @@ Status DBImpl::EnableFileDeletions(bool force) {
   if (should_purge_files)  {
     PurgeObsoleteFiles(job_context);
   }
-  job_context.Clean();
   LogFlush(db_options_.info_log);
   return Status::OK();
 }
@@ -137,7 +134,6 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
 Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
   return wal_manager_.GetSortedWalFiles(files);
 }
-
 }
 
 #endif  // ROCKSDB_LITE

db/db_impl.cc

@@ -32,7 +32,6 @@
 #include "db/db_iter.h"
 #include "db/dbformat.h"
 #include "db/filename.h"
-#include "db/job_context.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
 #include "db/memtable.h"
@@ -44,7 +43,6 @@
 #include "db/forward_iterator.h"
 #include "db/transaction_log_impl.h"
 #include "db/version_set.h"
-#include "db/writebuffer.h"
 #include "db/write_batch_internal.h"
 #include "port/port.h"
 #include "rocksdb/cache.h"
@@ -66,7 +64,6 @@
 #include "util/autovector.h"
 #include "util/build_version.h"
 #include "util/coding.h"
-#include "util/db_info_dumper.h"
 #include "util/hash_skiplist_rep.h"
 #include "util/hash_linklist_rep.h"
 #include "util/logging.h"
@@ -76,7 +73,6 @@
 #include "util/iostats_context_imp.h"
 #include "util/stop_watch.h"
 #include "util/sync_point.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
@@ -201,7 +197,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
       default_cf_handle_(nullptr),
       total_log_size_(0),
       max_total_in_memory_state_(0),
-      write_buffer_(options.db_write_buffer_size),
       tmp_batch_(),
       bg_schedule_needed_(false),
       bg_compaction_scheduled_(0),
@@ -218,8 +213,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
 #endif  // ROCKSDB_LITE
       bg_work_gate_closed_(false),
       refitting_level_(false),
-      opened_successfully_(false),
-      notifying_events_(0) {
+      opened_successfully_(false) {
   env_->GetAbsolutePath(dbname, &db_absolute_path_);
 
   // Reserve ten files or so for other uses and give the rest to TableCache.
@@ -232,8 +226,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
                   db_options_.table_cache_remove_scan_count_limit);
 
   versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
-                                 table_cache_.get(), &write_buffer_,
-                                 &write_controller_));
+                                 table_cache_.get(), &write_controller_));
   column_family_memtables_.reset(new ColumnFamilyMemTablesImpl(
       versions_->GetColumnFamilySet(), &flush_scheduler_));
 
@@ -246,7 +239,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
 
 DBImpl::~DBImpl() {
   mutex_.Lock();
-
   if (flush_on_destroy_) {
     for (auto cfd : *versions_->GetColumnFamilySet()) {
       if (!cfd->mem()->IsEmpty()) {
@@ -262,10 +254,10 @@ DBImpl::~DBImpl() {
 
   // Wait for background work to finish
   shutting_down_.store(true, std::memory_order_release);
-  while (bg_compaction_scheduled_ || bg_flush_scheduled_ || notifying_events_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_) {
     bg_cv_.Wait();
   }
-  listeners_.clear();
+
   flush_scheduler_.Clear();
 
   if (default_cf_handle_ != nullptr) {
@@ -292,7 +284,6 @@ DBImpl::~DBImpl() {
     if (job_context.HaveSomethingToDelete()) {
       PurgeObsoleteFiles(job_context);
     }
-    job_context.Clean();
   }
 
   // versions need to be destroyed before table_cache since it can hold
@@ -452,11 +443,8 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   }
 
   // don't delete live files
-  if (pending_outputs_.size()) {
-    job_context->min_pending_output = *pending_outputs_.begin();
-  } else {
-    // delete all of them
-    job_context->min_pending_output = std::numeric_limits<uint64_t>::max();
+  for (auto pair : pending_outputs_) {
+    job_context->sst_live.emplace_back(pair.first, pair.second, 0);
   }
   versions_->AddLiveFiles(&job_context->sst_live);
 
@@ -579,10 +567,7 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
         keep = (number >= state.manifest_file_number);
         break;
       case kTableFile:
-        // If the second condition is not there, this makes
-        // DontDeletePendingOutputs fail
-        keep = (sst_live_map.find(number) != sst_live_map.end()) ||
-               number >= state.min_pending_output;
+        keep = (sst_live_map.find(number) != sst_live_map.end());
         break;
       case kTempFile:
         // Any temp files that are currently being written to must
@@ -672,7 +657,6 @@ void DBImpl::DeleteObsoleteFiles() {
   if (job_context.HaveSomethingToDelete()) {
     PurgeObsoleteFiles(job_context);
   }
-  job_context.Clean();
 }
 
 Status DBImpl::Recover(
@@ -844,7 +828,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     // The previous incarnation may not have written any MANIFEST
     // records after allocating this log number.  So we manually
     // update the file number allocation counter in VersionSet.
-    versions_->MarkFileNumberUsedDuringRecovery(log_number);
+    versions_->MarkFileNumberUsed(log_number);
     // Open the log file
     std::string fname = LogFileName(db_options_.wal_dir, log_number);
     unique_ptr<SequentialFile> file;
@@ -978,7 +962,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
       // not actually used. that is because VersionSet assumes
       // VersionSet::next_file_number_ always to be strictly greater than any
       // log number
-      versions_->MarkFileNumberUsedDuringRecovery(max_log_number + 1);
+      versions_->MarkFileNumberUsed(max_log_number + 1);
       status = versions_->LogAndApply(
           cfd, *cfd->GetLatestMutableCFOptions(), edit, &mutex_);
       if (!status.ok()) {
@@ -997,8 +981,7 @@ Status DBImpl::WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
   const uint64_t start_micros = env_->NowMicros();
   FileMetaData meta;
   meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
-  auto pending_outputs_inserted_elem =
-      CaptureCurrentFileNumberInPendingOutputs();
+  pending_outputs_[meta.fd.GetNumber()] = 0;  // path 0 for level 0 file.
   ReadOptions ro;
   ro.total_order_seek = true;
   Arena arena;
@@ -1030,7 +1013,7 @@ Status DBImpl::WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
         cfd->GetName().c_str(), meta.fd.GetNumber(), meta.fd.GetFileSize(),
         s.ToString().c_str());
   }
-  ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
+  pending_outputs_.erase(meta.fd.GetNumber());
 
   // Note that if file_size is zero, the file has been deleted and
   // should not be added to the manifest.
@@ -1061,12 +1044,11 @@ Status DBImpl::FlushMemTableToOutputFile(
 
   FlushJob flush_job(dbname_, cfd, db_options_, mutable_cf_options,
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     snapshots_.GetNewest(), job_context, log_buffer,
-                     db_directory_.get(), GetCompressionFlush(*cfd->ioptions()),
-                     stats_);
+                     &pending_outputs_, snapshots_.GetNewest(), job_context,
+                     log_buffer, db_directory_.get(),
+                     GetCompressionFlush(*cfd->ioptions()), stats_);
 
-  uint64_t file_number;
-  Status s = flush_job.Run(&file_number);
+  Status s = flush_job.Run();
 
   if (s.ok()) {
     InstallSuperVersionBackground(cfd, job_context, mutable_cf_options);
@@ -1096,43 +1078,9 @@ Status DBImpl::FlushMemTableToOutputFile(
     bg_error_ = s;
   }
   RecordFlushIOStats();
-#ifndef ROCKSDB_LITE
-  if (s.ok()) {
-    // may temporarily unlock and lock the mutex.
-    NotifyOnFlushCompleted(cfd, file_number, mutable_cf_options);
-  }
-#endif  // ROCKSDB_LITE
   return s;
 }
 
-void DBImpl::NotifyOnFlushCompleted(
-    ColumnFamilyData* cfd, uint64_t file_number,
-    const MutableCFOptions& mutable_cf_options) {
-  mutex_.AssertHeld();
-  if (shutting_down_.load(std::memory_order_acquire)) {
-    return;
-  }
-  bool triggered_flush_slowdown =
-      (cfd->current()->storage_info()->NumLevelFiles(0) >=
-       mutable_cf_options.level0_slowdown_writes_trigger);
-  bool triggered_flush_stop =
-      (cfd->current()->storage_info()->NumLevelFiles(0) >=
-       mutable_cf_options.level0_stop_writes_trigger);
-  notifying_events_++;
-  // release lock while notifying events
-  mutex_.Unlock();
-  // TODO(yhchiang): make db_paths dynamic.
-  cfd->NotifyOnFlushCompleted(
-        this, MakeTableFileName(db_options_.db_paths[0].path, file_number),
-        triggered_flush_slowdown,
-        triggered_flush_stop);
-  mutex_.Lock();
-  notifying_events_--;
-  assert(notifying_events_ >= 0);
-  // no need to signal bg_cv_ as it will be signaled at the end of the
-  // flush process.
-}
-
 Status DBImpl::CompactRange(ColumnFamilyHandle* column_family,
                             const Slice* begin, const Slice* end,
                             bool reduce_level, int target_level,
@@ -1194,178 +1142,8 @@ Status DBImpl::CompactRange(ColumnFamilyHandle* column_family,
   return s;
 }
 
-Status DBImpl::CompactFiles(
-    const CompactionOptions& compact_options,
-    ColumnFamilyHandle* column_family,
-    const std::vector<std::string>& input_file_names,
-    const int output_level, const int output_path_id) {
-#ifdef ROCKSDB_LITE
-    // not supported in lite version
-  return Status::NotSupported("Not supported in ROCKSDB LITE");
-#else
-  MutexLock l(&mutex_);
-  if (column_family == nullptr) {
-    return Status::InvalidArgument("ColumnFamilyHandle must be non-null.");
-  }
-
-  auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
-  assert(cfd);
-  // TODO(yhchiang): use superversion
-  cfd->Ref();
-  auto version = cfd->current();
-  version->Ref();
-  auto s = CompactFilesImpl(compact_options, cfd, version,
-                            input_file_names, output_level, output_path_id);
-  // TODO(yhchiang): unref could move into CompactFilesImpl().  Otherwise,
-  // FindObsoleteFiles might never able to find any file to delete.
-  version->Unref();
-  // TODO(yhchiang): cfd should be deleted after its last reference.
-  cfd->Unref();
-  return s;
-#endif  // ROCKSDB_LITE
-}
-
-#ifndef ROCKSDB_LITE
-Status DBImpl::CompactFilesImpl(
-    const CompactionOptions& compact_options, ColumnFamilyData* cfd,
-    Version* version, const std::vector<std::string>& input_file_names,
-    const int output_level, int output_path_id) {
-  mutex_.AssertHeld();
-
-  LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
-
-  if (shutting_down_.load(std::memory_order_acquire)) {
-    return Status::ShutdownInProgress();
-  }
-
-  std::unordered_set<uint64_t> input_set;
-  for (auto file_name : input_file_names) {
-    input_set.insert(TableFileNameToNumber(file_name));
-  }
-
-  ColumnFamilyMetaData cf_meta;
-  // TODO(yhchiang): can directly use version here if none of the
-  // following functions call is pluggable to external developers.
-  version->GetColumnFamilyMetaData(&cf_meta);
-
-  if (output_path_id < 0) {
-    if (db_options_.db_paths.size() == 1U) {
-      output_path_id = 0;
-    } else {
-      return Status::NotSupported(
-          "Automatic output path selection is not "
-          "yet supported in CompactFiles()");
-    }
-  }
-
-  Status s = cfd->compaction_picker()->SanitizeCompactionInputFiles(
-      &input_set, cf_meta, output_level);
-  if (!s.ok()) {
-    return s;
-  }
-
-  autovector<CompactionInputFiles> input_files;
-  s = cfd->compaction_picker()->GetCompactionInputsFromFileNumbers(
-      &input_files, &input_set, version->storage_info(), compact_options);
-  if (!s.ok()) {
-    return s;
-  }
-
-  for (auto inputs : input_files) {
-    if (cfd->compaction_picker()->FilesInCompaction(inputs.files)) {
-      return Status::Aborted(
-          "Some of the necessary compaction input "
-          "files are already being compacted");
-    }
-  }
-
-  // At this point, CompactFiles will be run.
-  bg_compaction_scheduled_++;
-
-  unique_ptr<Compaction> c;
-  assert(cfd->compaction_picker());
-  c.reset(cfd->compaction_picker()->FormCompaction(
-        compact_options, input_files,
-        output_level, version->storage_info(),
-        *cfd->GetLatestMutableCFOptions()));
-  assert(c);
-  c->SetInputVersion(version);
-  c->SetOutputPathId(static_cast<uint32_t>(output_path_id));
-  // deletion compaction currently not allowed in CompactFiles.
-  assert(!c->IsDeletionCompaction());
-
-  JobContext job_context(true);
-  auto yield_callback = [&]() {
-    return CallFlushDuringCompaction(c->column_family_data(),
-                                     *c->mutable_cf_options(), &job_context,
-                                     &log_buffer);
-  };
-  CompactionJob compaction_job(c.get(), db_options_, *c->mutable_cf_options(),
-                               env_options_, versions_.get(), &shutting_down_,
-                               &log_buffer, db_directory_.get(), stats_,
-                               &snapshots_, IsSnapshotSupported(), table_cache_,
-                               std::move(yield_callback));
-  compaction_job.Prepare();
-
-  mutex_.Unlock();
-  Status status = compaction_job.Run();
-  mutex_.Lock();
-  compaction_job.Install(&status, &mutex_);
-  if (status.ok()) {
-    InstallSuperVersionBackground(c->column_family_data(), &job_context,
-                                  *c->mutable_cf_options());
-  }
-  c->ReleaseCompactionFiles(s);
-  c->ReleaseInputs();
-  c.reset();
-
-  if (status.ok()) {
-    // Done
-  } else if (status.IsShutdownInProgress()) {
-    // Ignore compaction errors found during shutting down
-  } else {
-    Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log, "Compaction error: %s",
-        status.ToString().c_str());
-    if (db_options_.paranoid_checks && bg_error_.ok()) {
-      bg_error_ = status;
-    }
-  }
-
-  // If !s.ok(), this means that Compaction failed. In that case, we want
-  // to delete all obsolete files we might have created and we force
-  // FindObsoleteFiles(). This is because job_context does not
-  // catch all created files if compaction failed.
-  // TODO(yhchiang): write an unit-test to make sure files are actually
-  //                 deleted after CompactFiles.
-  FindObsoleteFiles(&job_context, !s.ok());
-
-  // delete unnecessary files if any, this is done outside the mutex
-  if (job_context.HaveSomethingToDelete() || !log_buffer.IsEmpty()) {
-    mutex_.Unlock();
-    // Have to flush the info logs before bg_compaction_scheduled_--
-    // because if bg_flush_scheduled_ becomes 0 and the lock is
-    // released, the deconstructor of DB can kick in and destroy all the
-    // states of DB so info_log might not be available after that point.
-    // It also applies to access other states that DB owns.
-    log_buffer.FlushBufferToLog();
-    if (job_context.HaveSomethingToDelete()) {
-      PurgeObsoleteFiles(job_context);
-    }
-    job_context.Clean();
-    mutex_.Lock();
-  }
-
-  bg_compaction_scheduled_--;
-
-  return status;
-}
-#endif  // ROCKSDB_LITE
-
 Status DBImpl::SetOptions(ColumnFamilyHandle* column_family,
     const std::unordered_map<std::string, std::string>& options_map) {
-#ifdef ROCKSDB_LITE
-  return Status::NotSupported("Not supported in ROCKSDB LITE");
-#else
   auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
   if (options_map.empty()) {
     Log(InfoLogLevel::WARN_LEVEL,
@@ -1401,7 +1179,6 @@ Status DBImpl::SetOptions(ColumnFamilyHandle* column_family,
         "[%s] SetOptions failed", cfd->GetName().c_str());
   }
   return s;
-#endif  // ROCKSDB_LITE
 }
 
 // return the same level if it cannot be moved
@@ -1689,8 +1466,7 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
     bool is_compaction_needed = false;
     // no need to refcount since we're under a mutex
     for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (cfd->compaction_picker()->NeedsCompaction(
-              cfd->current()->storage_info())) {
+      if (cfd->current()->storage_info()->NeedsCompaction()) {
         is_compaction_needed = true;
         break;
       }
@@ -1774,9 +1550,6 @@ void DBImpl::BackgroundCallFlush() {
   {
     MutexLock l(&mutex_);
 
-    auto pending_outputs_inserted_elem =
-        CaptureCurrentFileNumberInPendingOutputs();
-
     Status s;
     if (!shutting_down_.load(std::memory_order_acquire)) {
       s = BackgroundFlush(&madeProgress, &job_context, &log_buffer);
@@ -1800,8 +1573,6 @@ void DBImpl::BackgroundCallFlush() {
       }
     }
 
-    ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
-
     // If !s.ok(), this means that Flush failed. In that case, we want
     // to delete all obsolete files and we force FindObsoleteFiles()
     FindObsoleteFiles(&job_context, !s.ok());
@@ -1817,7 +1588,6 @@ void DBImpl::BackgroundCallFlush() {
       if (job_context.HaveSomethingToDelete()) {
         PurgeObsoleteFiles(job_context);
       }
-      job_context.Clean();
       mutex_.Lock();
     }
 
@@ -1846,10 +1616,6 @@ void DBImpl::BackgroundCallCompaction() {
   LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
   {
     MutexLock l(&mutex_);
-
-    auto pending_outputs_inserted_elem =
-        CaptureCurrentFileNumberInPendingOutputs();
-
     assert(bg_compaction_scheduled_);
     Status s;
     if (!shutting_down_.load(std::memory_order_acquire)) {
@@ -1874,8 +1640,6 @@ void DBImpl::BackgroundCallCompaction() {
       }
     }
 
-    ReleaseFileNumberFromPendingOutputs(pending_outputs_inserted_elem);
-
     // If !s.ok(), this means that Compaction failed. In that case, we want
     // to delete all obsolete files we might have created and we force
     // FindObsoleteFiles(). This is because job_context does not
@@ -1894,7 +1658,6 @@ void DBImpl::BackgroundCallCompaction() {
       if (job_context.HaveSomethingToDelete()) {
         PurgeObsoleteFiles(job_context);
       }
-      job_context.Clean();
       mutex_.Lock();
     }
 
@@ -2068,8 +1831,6 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
                                   *c->mutable_cf_options());
 
     VersionStorageInfo::LevelSummaryStorage tmp;
-    c->column_family_data()->internal_stats()->IncBytesMoved(
-        c->level() + 1, f->fd.GetFileSize());
     LogToBuffer(log_buffer, "[%s] Moved #%" PRIu64 " to level-%d %" PRIu64
                             " bytes %s: %s\n",
                 c->column_family_data()->GetName().c_str(), f->fd.GetNumber(),
@@ -2085,16 +1846,16 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
                                        *c->mutable_cf_options(), job_context,
                                        log_buffer);
     };
-    CompactionJob compaction_job(c.get(), db_options_, *c->mutable_cf_options(),
-                                 env_options_, versions_.get(), &shutting_down_,
-                                 log_buffer, db_directory_.get(), stats_,
-                                 &snapshots_, IsSnapshotSupported(),
-                                 table_cache_, std::move(yield_callback));
+    CompactionJob compaction_job(
+        c.get(), db_options_, *c->mutable_cf_options(), env_options_,
+        versions_.get(), &mutex_, &shutting_down_, &pending_outputs_,
+        log_buffer, db_directory_.get(), stats_, &snapshots_,
+        IsSnapshotSupported(), table_cache_, std::move(yield_callback));
     compaction_job.Prepare();
     mutex_.Unlock();
     status = compaction_job.Run();
     mutex_.Lock();
-    compaction_job.Install(&status, &mutex_);
+    status = compaction_job.Install(status);
     if (status.ok()) {
       InstallSuperVersionBackground(c->column_family_data(), job_context,
                                     *c->mutable_cf_options());
@@ -2203,7 +1964,6 @@ static void CleanupIteratorState(void* arg1, void* arg2) {
     if (job_context.HaveSomethingToDelete()) {
       state->db->PurgeObsoleteFiles(job_context);
     }
-    job_context.Clean();
   }
 
   delete state;
@@ -2270,15 +2030,6 @@ SuperVersion* DBImpl::InstallSuperVersion(
     ColumnFamilyData* cfd, SuperVersion* new_sv,
     const MutableCFOptions& mutable_cf_options) {
   mutex_.AssertHeld();
-
-  // Update max_total_in_memory_state_
-  size_t old_memtable_size = 0;
-  auto* old_sv = cfd->GetSuperVersion();
-  if (old_sv) {
-    old_memtable_size = old_sv->mutable_cf_options.write_buffer_size *
-                        old_sv->mutable_cf_options.max_write_buffer_number;
-  }
-
   auto* old = cfd->InstallSuperVersion(
       new_sv ? new_sv : new SuperVersion(), &mutex_, mutable_cf_options);
 
@@ -2288,6 +2039,11 @@ SuperVersion* DBImpl::InstallSuperVersion(
   MaybeScheduleFlushOrCompaction();
 
   // Update max_total_in_memory_state_
+  auto old_memtable_size = 0;
+  if (old) {
+    old_memtable_size = old->mutable_cf_options.write_buffer_size *
+                        old->mutable_cf_options.max_write_buffer_number;
+  }
   max_total_in_memory_state_ =
       max_total_in_memory_state_ - old_memtable_size +
       mutable_cf_options.write_buffer_size *
@@ -2463,46 +2219,41 @@ std::vector<Status> DBImpl::MultiGet(
 Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
                                   const std::string& column_family_name,
                                   ColumnFamilyHandle** handle) {
-  Status s;
   *handle = nullptr;
-  {
-    MutexLock l(&mutex_);
+  MutexLock l(&mutex_);
 
-    if (versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name) !=
-        nullptr) {
-      return Status::InvalidArgument("Column family already exists");
-    }
-    VersionEdit edit;
-    edit.AddColumnFamily(column_family_name);
-    uint32_t new_id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID();
-    edit.SetColumnFamily(new_id);
-    edit.SetLogNumber(logfile_number_);
-    edit.SetComparatorName(cf_options.comparator->Name());
-
-    // LogAndApply will both write the creation in MANIFEST and create
-    // ColumnFamilyData object
-    Options opt(db_options_, cf_options);
-    s = versions_->LogAndApply(nullptr,
-        MutableCFOptions(opt, ImmutableCFOptions(opt)),
-        &edit, &mutex_, db_directory_.get(), false, &cf_options);
-    if (s.ok()) {
-      single_column_family_mode_ = false;
-      auto* cfd =
-          versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
-      assert(cfd != nullptr);
-      delete InstallSuperVersion(
-          cfd, nullptr, *cfd->GetLatestMutableCFOptions());
-      *handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
-      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-          "Created column family [%s] (ID %u)",
-          column_family_name.c_str(), (unsigned)cfd->GetID());
-    } else {
-      Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
-          "Creating column family [%s] FAILED -- %s",
-          column_family_name.c_str(), s.ToString().c_str());
-    }
-  }  // MutexLock l(&mutex_)
+  if (versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name) !=
+      nullptr) {
+    return Status::InvalidArgument("Column family already exists");
+  }
+  VersionEdit edit;
+  edit.AddColumnFamily(column_family_name);
+  uint32_t new_id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID();
+  edit.SetColumnFamily(new_id);
+  edit.SetLogNumber(logfile_number_);
+  edit.SetComparatorName(cf_options.comparator->Name());
 
+  // LogAndApply will both write the creation in MANIFEST and create
+  // ColumnFamilyData object
+  Options opt(db_options_, cf_options);
+  Status s = versions_->LogAndApply(nullptr,
+      MutableCFOptions(opt, ImmutableCFOptions(opt)),
+      &edit, &mutex_, db_directory_.get(), false, &cf_options);
+  if (s.ok()) {
+    single_column_family_mode_ = false;
+    auto cfd =
+        versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
+    assert(cfd != nullptr);
+    delete InstallSuperVersion(cfd, nullptr, *cfd->GetLatestMutableCFOptions());
+    *handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "Created column family [%s] (ID %u)",
+        column_family_name.c_str(), (unsigned)cfd->GetID());
+  } else {
+    Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
+        "Creating column family [%s] FAILED -- %s",
+        column_family_name.c_str(), s.ToString().c_str());
+  }
   return s;
 }
 
@@ -2535,9 +2286,6 @@ Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) {
   }
 
   if (s.ok()) {
-    // Note that here we erase the associated cf_info of the to-be-dropped
-    // cfd before its ref-count goes to zero to avoid having to erase cf_info
-    // later inside db_mutex.
     assert(cfd->IsDropped());
     auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
     max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
@@ -2714,13 +2462,10 @@ bool DBImpl::IsSnapshotSupported() const {
 }
 
 const Snapshot* DBImpl::GetSnapshot() {
-  int64_t unix_time = 0;
-  env_->GetCurrentTime(&unix_time);  // Ignore error
-
   MutexLock l(&mutex_);
   // returns null if the underlying memtable does not support snapshot.
   if (!IsSnapshotSupported()) return nullptr;
-  return snapshots_.New(versions_->LastSequence(), unix_time);
+  return snapshots_.New(versions_->LastSequence());
 }
 
 void DBImpl::ReleaseSnapshot(const Snapshot* s) {
@@ -2827,23 +2572,6 @@ Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
       }
     }
     MaybeScheduleFlushOrCompaction();
-  } else if (UNLIKELY(write_buffer_.ShouldFlush())) {
-    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
-        "Flushing all column families. Write buffer is using %" PRIu64
-        " bytes out of a total of %" PRIu64 ".",
-        write_buffer_.memory_usage(), write_buffer_.buffer_size());
-    // no need to refcount because drop is happening in write thread, so can't
-    // happen while we're in the write thread
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (!cfd->mem()->IsEmpty()) {
-        status = SetNewMemtableAndNewLogFile(cfd, &context);
-        if (!status.ok()) {
-          break;
-        }
-        cfd->imm()->FlushRequested();
-      }
-    }
-    MaybeScheduleFlushOrCompaction();
   }
 
   if (UNLIKELY(status.ok() && !bg_error_.ok())) {
@@ -2979,8 +2707,7 @@ Status DBImpl::DelayWrite(uint64_t expiration_time) {
   auto delay = write_controller_.GetDelay();
   if (write_controller_.IsStopped() == false && delay > 0) {
     mutex_.Unlock();
-    // hopefully we don't have to sleep more than 2 billion microseconds
-    env_->SleepForMicroseconds(static_cast<int>(delay));
+    env_->SleepForMicroseconds(delay);
     mutex_.Lock();
   }
 
@@ -3051,7 +2778,8 @@ Status DBImpl::SetNewMemtableAndNewLogFile(ColumnFamilyData* cfd,
     }
 
     if (s.ok()) {
-      new_mem = cfd->ConstructNewMemtable(mutable_cf_options);
+      new_mem = new MemTable(cfd->internal_comparator(), *cfd->ioptions(),
+                             mutable_cf_options);
       new_superversion = new SuperVersion();
     }
   }
@@ -3062,6 +2790,8 @@ Status DBImpl::SetNewMemtableAndNewLogFile(ColumnFamilyData* cfd,
   if (!s.ok()) {
     // how do we fail if we're not creating new log?
     assert(creating_new_log);
+    // Avoid chewing through file number space in a tight loop.
+    versions_->ReuseLogFileNumber(new_log_number);
     assert(!new_mem);
     assert(!new_log);
     return s;
@@ -3142,7 +2872,7 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
     bool ret_value = GetIntPropertyInternal(column_family, property_type,
                                             need_out_of_mutex, &int_value);
     if (ret_value) {
-      *value = ToString(int_value);
+      *value = std::to_string(int_value);
     }
     return ret_value;
   } else {
@@ -3238,22 +2968,6 @@ void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
   }
 }
 
-std::list<uint64_t>::iterator
-DBImpl::CaptureCurrentFileNumberInPendingOutputs() {
-  // We need to remember the iterator of our insert, because after the
-  // background job is done, we need to remove that element from
-  // pending_outputs_.
-  pending_outputs_.push_back(versions_->current_next_file_number());
-  auto pending_outputs_inserted_elem = pending_outputs_.end();
-  --pending_outputs_inserted_elem;
-  return pending_outputs_inserted_elem;
-}
-
-void DBImpl::ReleaseFileNumberFromPendingOutputs(
-    std::list<uint64_t>::iterator v) {
-  pending_outputs_.erase(v);
-}
-
 #ifndef ROCKSDB_LITE
 Status DBImpl::GetUpdatesSince(
     SequenceNumber seq, unique_ptr<TransactionLogIterator>* iter,
@@ -3306,7 +3020,6 @@ Status DBImpl::DeleteFile(std::string name) {
     if (!status.ok()) {
       Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
           "DeleteFile %s failed. File not found\n", name.c_str());
-      job_context.Clean();
       return Status::InvalidArgument("File not found");
     }
     assert(level < cfd->NumberLevels());
@@ -3315,7 +3028,6 @@ Status DBImpl::DeleteFile(std::string name) {
     if (metadata->being_compacted) {
       Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
           "DeleteFile %s Skipped. File about to be compacted\n", name.c_str());
-      job_context.Clean();
       return Status::OK();
     }
 
@@ -3327,7 +3039,6 @@ Status DBImpl::DeleteFile(std::string name) {
       if (vstoreage->NumLevelFiles(i) != 0) {
         Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
             "DeleteFile %s FAILED. File not in last level\n", name.c_str());
-        job_context.Clean();
         return Status::InvalidArgument("File not in last level");
       }
     }
@@ -3337,7 +3048,6 @@ Status DBImpl::DeleteFile(std::string name) {
       Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
           "DeleteFile %s failed ---"
           " target file in level 0 must be the oldest.", name.c_str());
-      job_context.Clean();
       return Status::InvalidArgument("File in level 0, but not oldest");
     }
     edit.SetColumnFamily(cfd->GetID());
@@ -3355,7 +3065,6 @@ Status DBImpl::DeleteFile(std::string name) {
   if (job_context.HaveSomethingToDelete()) {
     PurgeObsoleteFiles(job_context);
   }
-  job_context.Clean();
   {
     MutexLock l(&mutex_);
     // schedule flush if file deletion means we freed the space for flushes to
@@ -3369,17 +3078,6 @@ void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
   MutexLock l(&mutex_);
   versions_->GetLiveFilesMetaData(metadata);
 }
-
-void DBImpl::GetColumnFamilyMetaData(
-    ColumnFamilyHandle* column_family,
-    ColumnFamilyMetaData* cf_meta) {
-  assert(column_family);
-  auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family)->cfd();
-  auto* sv = GetAndRefSuperVersion(cfd);
-  sv->current->GetColumnFamilyMetaData(cf_meta);
-  ReturnAndCleanupSuperVersion(cfd, sv);
-}
-
 #endif  // ROCKSDB_LITE
 
 Status DBImpl::CheckConsistency() {
@@ -3399,8 +3097,8 @@ Status DBImpl::CheckConsistency() {
     } else if (fsize != md.size) {
       corruption_messages += "Sst file size mismatch: " + file_path +
                              ". Size recorded in manifest " +
-                             ToString(md.size) + ", actual size " +
-                             ToString(fsize) + "\n";
+                             std::to_string(md.size) + ", actual size " +
+                             std::to_string(fsize) + "\n";
     }
   }
   if (corruption_messages.size() == 0) {
@@ -3425,7 +3123,7 @@ Status DBImpl::GetDbIdentity(std::string& identity) {
   }
   char buffer[file_size];
   Slice id;
-  s = idfile->Read(static_cast<size_t>(file_size), &id, buffer);
+  s = idfile->Read(file_size, &id, buffer);
   if (!s.ok()) {
     return s;
   }
@@ -3631,7 +3329,7 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
     impl->opened_successfully_ = true;
     *dbptr = impl;
   } else {
-    for (auto* h : *handles) {
+    for (auto h : *handles) {
       delete h;
     }
     handles->clear();

db/db_impl.h

@@ -12,9 +12,7 @@
 #include <deque>
 #include <limits>
 #include <set>
-#include <list>
 #include <utility>
-#include <list>
 #include <vector>
 #include <string>
 
@@ -24,7 +22,6 @@
 #include "db/column_family.h"
 #include "db/version_edit.h"
 #include "db/wal_manager.h"
-#include "db/writebuffer.h"
 #include "memtable_list.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
@@ -35,11 +32,11 @@
 #include "util/stop_watch.h"
 #include "util/thread_local.h"
 #include "util/scoped_arena_iterator.h"
-#include "util/hash.h"
 #include "db/internal_stats.h"
 #include "db/write_controller.h"
 #include "db/flush_scheduler.h"
 #include "db/write_thread.h"
+#include "db/job_context.h"
 
 namespace rocksdb {
 
@@ -50,7 +47,6 @@ class VersionEdit;
 class VersionSet;
 class CompactionFilterV2;
 class Arena;
-struct JobContext;
 
 class DBImpl : public DB {
  public:
@@ -118,13 +114,6 @@ class DBImpl : public DB {
                               bool reduce_level = false, int target_level = -1,
                               uint32_t target_path_id = 0);
 
-  using DB::CompactFiles;
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1);
-
   using DB::SetOptions;
   Status SetOptions(ColumnFamilyHandle* column_family,
       const std::unordered_map<std::string, std::string>& options_map);
@@ -162,15 +151,6 @@ class DBImpl : public DB {
   virtual Status DeleteFile(std::string name);
 
   virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
-
-  // Obtains the meta data of the specified column family of the DB.
-  // Status::NotFound() will be returned if the current DB does not have
-  // any column family match the specified name.
-  // TODO(yhchiang): output parameter is placed in the end in this codebase.
-  virtual void GetColumnFamilyMetaData(
-      ColumnFamilyHandle* column_family,
-      ColumnFamilyMetaData* metadata) override;
-
 #endif  // ROCKSDB_LITE
 
   // checks if all live files exist on file system and that their file sizes
@@ -230,11 +210,7 @@ class DBImpl : public DB {
   // REQUIRES: mutex locked
   // pass the pointer that you got from TEST_BeginWrite()
   void TEST_EndWrite(void* w);
-
-  uint64_t TEST_max_total_in_memory_state() {
-    return max_total_in_memory_state_;
-  }
-#endif  // ROCKSDB_LITE
+#endif  // NDEBUG
 
   // Returns the list of live files in 'live' and the list
   // of all files in the filesystem in 'candidate_files'.
@@ -252,8 +228,6 @@ class DBImpl : public DB {
 
   ColumnFamilyHandle* DefaultColumnFamily() const;
 
-  const SnapshotList& snapshots() const { return snapshots_; }
-
  protected:
   Env* const env_;
   const std::string dbname_;
@@ -264,9 +238,6 @@ class DBImpl : public DB {
   Iterator* NewInternalIterator(const ReadOptions&, ColumnFamilyData* cfd,
                                 SuperVersion* super_version, Arena* arena);
 
-  void NotifyOnFlushCompleted(ColumnFamilyData* cfd, uint64_t file_number,
-                              const MutableCFOptions& mutable_cf_options);
-
  private:
   friend class DB;
   friend class InternalStats;
@@ -294,24 +265,6 @@ class DBImpl : public DB {
   // Delete any unneeded files and stale in-memory entries.
   void DeleteObsoleteFiles();
 
-  // Background process needs to call
-  //     auto x = CaptureCurrentFileNumberInPendingOutputs()
-  //     <do something>
-  //     ReleaseFileNumberFromPendingOutputs(x)
-  // This will protect any temporary files created while <do something> is
-  // executing from being deleted.
-  // -----------
-  // This function will capture current file number and append it to
-  // pending_outputs_. This will prevent any background process to delete any
-  // file created after this point.
-  std::list<uint64_t>::iterator CaptureCurrentFileNumberInPendingOutputs();
-  // This function should be called with the result of
-  // CaptureCurrentFileNumberInPendingOutputs(). It then marks that any file
-  // created between the calls CaptureCurrentFileNumberInPendingOutputs() and
-  // ReleaseFileNumberFromPendingOutputs() can now be deleted (if it's not live
-  // and blocked by any other pending_outputs_ calls)
-  void ReleaseFileNumberFromPendingOutputs(std::list<uint64_t>::iterator v);
-
   // Flush the in-memory write buffer to storage.  Switches to a new
   // log-file/memtable and writes a new descriptor iff successful.
   Status FlushMemTableToOutputFile(ColumnFamilyData* cfd,
@@ -346,15 +299,6 @@ class DBImpl : public DB {
   void RecordFlushIOStats();
   void RecordCompactionIOStats();
 
-#ifndef ROCKSDB_LITE
-  Status CompactFilesImpl(
-      const CompactionOptions& compact_options, ColumnFamilyData* cfd,
-      Version* version, const std::vector<std::string>& input_file_names,
-      const int output_level, int output_path_id);
-#endif  // ROCKSDB_LITE
-
-  ColumnFamilyData* GetColumnFamilyDataByName(const std::string& cf_name);
-
   void MaybeScheduleFlushOrCompaction();
   static void BGWorkCompaction(void* db);
   static void BGWorkFlush(void* db);
@@ -437,8 +381,6 @@ class DBImpl : public DB {
 
   std::unique_ptr<Directory> db_directory_;
 
-  WriteBuffer write_buffer_;
-
   WriteThread write_thread_;
 
   WriteBatch tmp_batch_;
@@ -448,16 +390,10 @@ class DBImpl : public DB {
 
   SnapshotList snapshots_;
 
-  // For each background job, pending_outputs_ keeps the current file number at
-  // the time that background job started.
-  // FindObsoleteFiles()/PurgeObsoleteFiles() never deletes any file that has
-  // number bigger than any of the file number in pending_outputs_. Since file
-  // numbers grow monotonically, this also means that pending_outputs_ is always
-  // sorted. After a background job is done executing, its file number is
-  // deleted from pending_outputs_, which allows PurgeObsoleteFiles() to clean
-  // it up.
-  // State is protected with db mutex.
-  std::list<uint64_t> pending_outputs_;
+  // Set of table files to protect from deletion because they are
+  // part of ongoing compactions.
+  // map from pending file number ID to their path IDs.
+  FileNumToPathIdMap pending_outputs_;
 
   // At least one compaction or flush job is pending but not yet scheduled
   // because of the max background thread limit.
@@ -527,12 +463,6 @@ class DBImpl : public DB {
   // Indicate DB was opened successfully
   bool opened_successfully_;
 
-  // The list of registered event listeners.
-  std::list<EventListener*> listeners_;
-
-  // count how many events are currently being notified.
-  int notifying_events_;
-
   // No copying allowed
   DBImpl(const DBImpl&);
   void operator=(const DBImpl&);
@@ -598,4 +528,8 @@ static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
   if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
 }
 
+// Dump db file summary, implemented in util/
+extern void DumpDBFileSummary(const DBOptions& options,
+                              const std::string& dbname);
+
 }  // namespace rocksdb

db/db_impl_debug.cc

@@ -81,8 +81,8 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
     cfd = cfh->cfd();
   }
   int output_level =
-      (cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
-       cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
+      (cfd->options()->compaction_style == kCompactionStyleUniversal ||
+       cfd->options()->compaction_style == kCompactionStyleFIFO)
           ? level
           : level + 1;
   return RunManualCompaction(cfd, level, output_level, 0, begin, end);

db/db_impl_readonly.cc

@@ -3,7 +3,6 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
-
 #include "db/db_impl_readonly.h"
 #include "utilities/compacted_db/compacted_db_impl.h"
 #include "db/db_impl.h"
@@ -13,8 +12,6 @@
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
-
 DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
                                const std::string& dbname)
     : DBImpl(db_options, dbname) {
@@ -99,10 +96,12 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
 
   // Try to first open DB as fully compacted DB
   Status s;
+#ifndef ROCKSDB_LITE
   s = CompactedDBImpl::Open(options, dbname, dbptr);
   if (s.ok()) {
     return s;
   }
+#endif
 
   DBOptions db_options(options);
   ColumnFamilyOptions cf_options(options);
@@ -163,20 +162,5 @@ Status DB::OpenForReadOnly(
   return s;
 }
 
-#else  // !ROCKSDB_LITE
-
-Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
-                           DB** dbptr, bool error_if_log_file_exist) {
-  return Status::NotSupported("Not supported in ROCKSDB_LITE.");
-}
-
-Status DB::OpenForReadOnly(
-    const DBOptions& db_options, const std::string& dbname,
-    const std::vector<ColumnFamilyDescriptor>& column_families,
-    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
-    bool error_if_log_file_exist) {
-  return Status::NotSupported("Not supported in ROCKSDB_LITE.");
-}
-#endif  // !ROCKSDB_LITE
 
 }   // namespace rocksdb

db/db_impl_readonly.h

@@ -4,9 +4,6 @@
 //  of patent rights can be found in the PATENTS file in the same directory.
 
 #pragma once
-
-#ifndef ROCKSDB_LITE
-
 #include "db/db_impl.h"
 #include <vector>
 #include <string>
@@ -65,20 +62,10 @@ class DBImplReadOnly : public DBImpl {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
-  using DBImpl::CompactFiles;
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1) override {
-    return Status::NotSupported("Not supported operation in read only mode.");
-  }
-
 #ifndef ROCKSDB_LITE
   virtual Status DisableFileDeletions() override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
-
   virtual Status EnableFileDeletions(bool force) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
@@ -103,5 +90,3 @@ class DBImplReadOnly : public DBImpl {
   void operator=(const DBImplReadOnly&);
 };
 }
-
-#endif  // !ROCKSDB_LITE

db/db_iter.cc

@@ -281,9 +281,8 @@ void DBIter::MergeValuesNewToOld() {
   if (!user_merge_operator_) {
     Log(InfoLogLevel::ERROR_LEVEL,
         logger_, "Options::merge_operator is null.");
-    status_ = Status::InvalidArgument("user_merge_operator_ must be set.");
-    valid_ = false;
-    return;
+    throw std::logic_error("DBIter::MergeValuesNewToOld() with"
+                           " Options::merge_operator null");
   }
 
   // Start the merge process by pushing the first operand

db/db_iter_test.cc

@@ -19,7 +19,7 @@
 
 namespace rocksdb {
 
-static uint64_t TestGetTickerCount(const Options& options,
+static uint32_t TestGetTickerCount(const Options& options,
                                    Tickers ticker_type) {
   return options.statistics->getTickerCount(ticker_type);
 }
@@ -366,7 +366,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->AddMerge("b", "merge_1");
       internal_iter->AddMerge("a", "merge_2");
       for (size_t k = 0; k < 200; ++k) {
-        internal_iter->AddPut("c", ToString(k));
+        internal_iter->AddPut("c", std::to_string(k));
       }
       internal_iter->Finish();
 
@@ -379,7 +379,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       ASSERT_TRUE(db_iter->Valid());
 
       ASSERT_EQ(db_iter->key().ToString(), "c");
-      ASSERT_EQ(db_iter->value().ToString(), ToString(i));
+      ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
       db_iter->Prev();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -513,11 +513,11 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->AddMerge("b", "merge_1");
       internal_iter->AddMerge("a", "merge_2");
       for (size_t k = 0; k < 200; ++k) {
-        internal_iter->AddPut("d", ToString(k));
+        internal_iter->AddPut("d", std::to_string(k));
       }
 
       for (size_t k = 0; k < 200; ++k) {
-        internal_iter->AddPut("c", ToString(k));
+        internal_iter->AddPut("c", std::to_string(k));
       }
       internal_iter->Finish();
 
@@ -529,7 +529,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       ASSERT_TRUE(db_iter->Valid());
 
       ASSERT_EQ(db_iter->key().ToString(), "d");
-      ASSERT_EQ(db_iter->value().ToString(), ToString(i));
+      ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
       db_iter->Prev();
       ASSERT_TRUE(db_iter->Valid());
 
@@ -552,7 +552,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       internal_iter->AddMerge("b", "b");
       internal_iter->AddMerge("a", "a");
       for (size_t k = 0; k < 200; ++k) {
-        internal_iter->AddMerge("c", ToString(k));
+        internal_iter->AddMerge("c", std::to_string(k));
       }
       internal_iter->Finish();
 
@@ -566,7 +566,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
       ASSERT_EQ(db_iter->key().ToString(), "c");
       std::string merge_result = "0";
       for (size_t j = 1; j <= i; ++j) {
-        merge_result += "," + ToString(j);
+        merge_result += "," + std::to_string(j);
       }
       ASSERT_EQ(db_iter->value().ToString(), merge_result);
 

db/dbformat.cc

@@ -137,8 +137,7 @@ LookupKey::LookupKey(const Slice& _user_key, SequenceNumber s) {
     dst = new char[needed];
   }
   start_ = dst;
-  // NOTE: We don't support users keys of more than 2GB :)
-  dst = EncodeVarint32(dst, static_cast<uint32_t>(usize + 8));
+  dst = EncodeVarint32(dst, usize + 8);
   kstart_ = dst;
   memcpy(dst, _user_key.data(), usize);
   dst += usize;

db/dbformat.h

@@ -206,19 +206,13 @@ class LookupKey {
   ~LookupKey();
 
   // Return a key suitable for lookup in a MemTable.
-  Slice memtable_key() const {
-    return Slice(start_, static_cast<size_t>(end_ - start_));
-  }
+  Slice memtable_key() const { return Slice(start_, end_ - start_); }
 
   // Return an internal key (suitable for passing to an internal iterator)
-  Slice internal_key() const {
-    return Slice(kstart_, static_cast<size_t>(end_ - kstart_));
-  }
+  Slice internal_key() const { return Slice(kstart_, end_ - kstart_); }
 
   // Return the user key
-  Slice user_key() const {
-    return Slice(kstart_, static_cast<size_t>(end_ - kstart_ - 8));
-  }
+  Slice user_key() const { return Slice(kstart_, end_ - kstart_ - 8); }
 
  private:
   // We construct a char array of the form:
@@ -325,8 +319,8 @@ class IterKey {
 
   void EncodeLengthPrefixedKey(const Slice& key) {
     auto size = key.size();
-    EnlargeBufferIfNeeded(size + static_cast<size_t>(VarintLength(size)));
-    char* ptr = EncodeVarint32(key_, static_cast<uint32_t>(size));
+    EnlargeBufferIfNeeded(size + VarintLength(size));
+    char* ptr = EncodeVarint32(key_, size);
     memcpy(ptr, key.data(), size);
   }
 

db/deletefile_test.cc

@@ -34,7 +34,6 @@ class DeleteFileTest {
   DeleteFileTest() {
     db_ = nullptr;
     env_ = Env::Default();
-    options_.enable_thread_tracking = true;
     options_.max_background_flushes = 0;
     options_.write_buffer_size = 1024*1024*1000;
     options_.target_file_size_base = 1024*1024*1000;
@@ -79,7 +78,7 @@ class DeleteFileTest {
     options.sync = false;
     ReadOptions roptions;
     for (int i = startkey; i < (numkeys + startkey) ; i++) {
-      std::string temp = ToString(i);
+      std::string temp = std::to_string(i);
       Slice key(temp);
       Slice value(temp);
       ASSERT_OK(db_->Put(options, key, value));

db/file_indexer.cc

@@ -17,16 +17,17 @@ namespace rocksdb {
 FileIndexer::FileIndexer(const Comparator* ucmp)
     : num_levels_(0), ucmp_(ucmp), level_rb_(nullptr) {}
 
-size_t FileIndexer::NumLevelIndex() const { return next_level_index_.size(); }
+uint32_t FileIndexer::NumLevelIndex() const {
+  return next_level_index_.size();
+}
 
-size_t FileIndexer::LevelIndexSize(size_t level) const {
+uint32_t FileIndexer::LevelIndexSize(uint32_t level) const {
   return next_level_index_[level].num_index;
 }
 
-void FileIndexer::GetNextLevelIndex(const size_t level, const size_t file_index,
-                                    const int cmp_smallest,
-                                    const int cmp_largest, int32_t* left_bound,
-                                    int32_t* right_bound) const {
+void FileIndexer::GetNextLevelIndex(
+    const uint32_t level, const uint32_t file_index, const int cmp_smallest,
+    const int cmp_largest, int32_t* left_bound, int32_t* right_bound) const {
   assert(level > 0);
 
   // Last level, no hint
@@ -68,7 +69,7 @@ void FileIndexer::GetNextLevelIndex(const size_t level, const size_t file_index,
   assert(*right_bound <= level_rb_[level + 1]);
 }
 
-void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels,
+void FileIndexer::UpdateIndex(Arena* arena, const uint32_t num_levels,
                               std::vector<FileMetaData*>* const files) {
   if (files == nullptr) {
     return;
@@ -89,11 +90,11 @@ void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels,
   }
 
   // L1 - Ln-1
-  for (size_t level = 1; level < num_levels_ - 1; ++level) {
+  for (uint32_t level = 1; level < num_levels_ - 1; ++level) {
     const auto& upper_files = files[level];
-    const int32_t upper_size = static_cast<int32_t>(upper_files.size());
+    const int32_t upper_size = upper_files.size();
     const auto& lower_files = files[level + 1];
-    level_rb_[level] = static_cast<int32_t>(upper_files.size()) - 1;
+    level_rb_[level] = upper_files.size() - 1;
     if (upper_size == 0) {
       continue;
     }
@@ -128,8 +129,7 @@ void FileIndexer::UpdateIndex(Arena* arena, const size_t num_levels,
         [](IndexUnit* index, int32_t f_idx) { index->largest_rb = f_idx; });
   }
 
-  level_rb_[num_levels_ - 1] =
-      static_cast<int32_t>(files[num_levels_ - 1].size()) - 1;
+  level_rb_[num_levels_ - 1] = files[num_levels_ - 1].size() - 1;
 }
 
 void FileIndexer::CalculateLB(
@@ -137,8 +137,8 @@ void FileIndexer::CalculateLB(
     const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
     std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
     std::function<void(IndexUnit*, int32_t)> set_index) {
-  const int32_t upper_size = static_cast<int32_t>(upper_files.size());
-  const int32_t lower_size = static_cast<int32_t>(lower_files.size());
+  const int32_t upper_size = upper_files.size();
+  const int32_t lower_size = lower_files.size();
   int32_t upper_idx = 0;
   int32_t lower_idx = 0;
 
@@ -175,8 +175,8 @@ void FileIndexer::CalculateRB(
     const std::vector<FileMetaData*>& lower_files, IndexLevel* index_level,
     std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
     std::function<void(IndexUnit*, int32_t)> set_index) {
-  const int32_t upper_size = static_cast<int32_t>(upper_files.size());
-  const int32_t lower_size = static_cast<int32_t>(lower_files.size());
+  const int32_t upper_size = upper_files.size();
+  const int32_t lower_size = lower_files.size();
   int32_t upper_idx = upper_size - 1;
   int32_t lower_idx = lower_size - 1;
 

db/file_indexer.h

@@ -42,19 +42,19 @@ class FileIndexer {
  public:
   explicit FileIndexer(const Comparator* ucmp);
 
-  size_t NumLevelIndex() const;
+  uint32_t NumLevelIndex() const;
 
-  size_t LevelIndexSize(size_t level) const;
+  uint32_t LevelIndexSize(uint32_t level) const;
 
   // Return a file index range in the next level to search for a key based on
   // smallest and largest key comparision for the current file specified by
   // level and file_index. When *left_index < *right_index, both index should
   // be valid and fit in the vector size.
-  void GetNextLevelIndex(const size_t level, const size_t file_index,
-                         const int cmp_smallest, const int cmp_largest,
-                         int32_t* left_bound, int32_t* right_bound) const;
+  void GetNextLevelIndex(
+    const uint32_t level, const uint32_t file_index, const int cmp_smallest,
+    const int cmp_largest, int32_t* left_bound, int32_t* right_bound) const;
 
-  void UpdateIndex(Arena* arena, const size_t num_levels,
+  void UpdateIndex(Arena* arena, const uint32_t num_levels,
                    std::vector<FileMetaData*>* const files);
 
   enum {
@@ -62,7 +62,7 @@ class FileIndexer {
   };
 
  private:
-  size_t num_levels_;
+  uint32_t num_levels_;
   const Comparator* ucmp_;
 
   struct IndexUnit {

db/file_indexer_test.cc

@@ -22,15 +22,8 @@ class IntComparator : public Comparator {
   int Compare(const Slice& a, const Slice& b) const {
     assert(a.size() == 8);
     assert(b.size() == 8);
-    int64_t diff = *reinterpret_cast<const int64_t*>(a.data()) -
-                   *reinterpret_cast<const int64_t*>(b.data());
-    if (diff < 0) {
-      return -1;
-    } else if (diff == 0) {
-      return 0;
-    } else {
-      return 1;
-    }
+    return *reinterpret_cast<const int64_t*>(a.data()) -
+      *reinterpret_cast<const int64_t*>(b.data());
   }
 
   const char* Name() const {

db/filename.cc

@@ -79,17 +79,6 @@ std::string MakeTableFileName(const std::string& path, uint64_t number) {
   return MakeFileName(path, number, "sst");
 }
 
-uint64_t TableFileNameToNumber(const std::string& name) {
-  uint64_t number = 0;
-  uint64_t base = 1;
-  int pos = static_cast<int>(name.find_last_of('.'));
-  while (--pos >= 0 && name[pos] >= '0' && name[pos] <= '9') {
-    number += (name[pos] - '0') * base;
-    base *= 10;
-  }
-  return number;
-}
-
 std::string TableFileName(const std::vector<DbPath>& db_paths, uint64_t number,
                           uint32_t path_id) {
   assert(number > 0);

db/filename.h

@@ -36,6 +36,9 @@ enum FileType {
   kIdentityFile
 };
 
+// map from file number to path ID.
+typedef std::unordered_map<uint64_t, uint32_t> FileNumToPathIdMap;
+
 // Return the name of the log file with the specified number
 // in the db named by "dbname".  The result will be prefixed with
 // "dbname".
@@ -52,10 +55,6 @@ extern std::string ArchivedLogFileName(const std::string& dbname,
 
 extern std::string MakeTableFileName(const std::string& name, uint64_t number);
 
-// the reverse function of MakeTableFileName
-// TODO(yhchiang): could merge this function with ParseFileName()
-extern uint64_t TableFileNameToNumber(const std::string& name);
-
 // Return the name of the sstable with the specified number
 // in the db named by "dbname".  The result will be prefixed with
 // "dbname".

db/flush_job.cc

@@ -55,6 +55,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                    const MutableCFOptions& mutable_cf_options,
                    const EnvOptions& env_options, VersionSet* versions,
                    port::Mutex* db_mutex, std::atomic<bool>* shutting_down,
+                   FileNumToPathIdMap* pending_outputs,
                    SequenceNumber newest_snapshot, JobContext* job_context,
                    LogBuffer* log_buffer, Directory* db_directory,
                    CompressionType output_compression, Statistics* stats)
@@ -66,6 +67,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
       versions_(versions),
       db_mutex_(db_mutex),
       shutting_down_(shutting_down),
+      pending_outputs_(pending_outputs),
       newest_snapshot_(newest_snapshot),
       job_context_(job_context),
       log_buffer_(log_buffer),
@@ -73,9 +75,9 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
       output_compression_(output_compression),
       stats_(stats) {}
 
-Status FlushJob::Run(uint64_t* file_number) {
+Status FlushJob::Run() {
   // Save the contents of the earliest memtable as a new Table
-  uint64_t fn;
+  uint64_t file_number;
   autovector<MemTable*> mems;
   cfd_->imm()->PickMemtablesToFlush(&mems);
   if (mems.empty()) {
@@ -96,7 +98,7 @@ Status FlushJob::Run(uint64_t* file_number) {
   edit->SetColumnFamily(cfd_->GetID());
 
   // This will release and re-acquire the mutex.
-  Status s = WriteLevel0Table(mems, edit, &fn);
+  Status s = WriteLevel0Table(mems, edit, &file_number);
 
   if (s.ok() &&
       (shutting_down_->load(std::memory_order_acquire) || cfd_->IsDropped())) {
@@ -105,17 +107,15 @@ Status FlushJob::Run(uint64_t* file_number) {
   }
 
   if (!s.ok()) {
-    cfd_->imm()->RollbackMemtableFlush(mems, fn);
+    cfd_->imm()->RollbackMemtableFlush(mems, file_number, pending_outputs_);
   } else {
     // Replace immutable memtable with the generated Table
     s = cfd_->imm()->InstallMemtableFlushResults(
-        cfd_, mutable_cf_options_, mems, versions_, db_mutex_, fn,
-        &job_context_->memtables_to_free, db_directory_, log_buffer_);
+        cfd_, mutable_cf_options_, mems, versions_, db_mutex_, file_number,
+        pending_outputs_, &job_context_->memtables_to_free, db_directory_,
+        log_buffer_);
   }
 
-  if (s.ok() && file_number != nullptr) {
-    *file_number = fn;
-  }
   return s;
 }
 
@@ -128,6 +128,7 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
   meta.fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
   *filenumber = meta.fd.GetNumber();
   // path 0 for level 0 file.
+  pending_outputs_->insert({meta.fd.GetNumber(), 0});
 
   const SequenceNumber earliest_seqno_in_memtable =
       mems[0]->GetFirstSequenceNumber();
@@ -150,9 +151,9 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
       memtables.push_back(m->NewIterator(ro, &arena));
     }
     {
-      ScopedArenaIterator iter(
-          NewMergingIterator(&cfd_->internal_comparator(), &memtables[0],
-                             static_cast<int>(memtables.size()), &arena));
+      ScopedArenaIterator iter(NewMergingIterator(&cfd_->internal_comparator(),
+                                                  &memtables[0],
+                                                  memtables.size(), &arena));
       Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
           "[%s] Level-0 flush table #%" PRIu64 ": started",
           cfd_->GetName().c_str(), meta.fd.GetNumber());
@@ -179,6 +180,15 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
   // re-acquire the most current version
   base = cfd_->current();
 
+  // There could be multiple threads writing to its own level-0 file.
+  // The pending_outputs cannot be cleared here, otherwise this newly
+  // created file might not be considered as a live-file by another
+  // compaction thread that is concurrently deleting obselete files.
+  // The pending_outputs can be cleared only after the new version is
+  // committed so that other threads can recognize this file as a
+  // valid one.
+  // pending_outputs_.erase(meta.number);
+
   // Note that if file_size is zero, the file has been deleted and
   // should not be added to the manifest.
   int level = 0;

db/flush_job.h

@@ -55,12 +55,13 @@ class FlushJob {
            const MutableCFOptions& mutable_cf_options,
            const EnvOptions& env_options, VersionSet* versions,
            port::Mutex* db_mutex, std::atomic<bool>* shutting_down,
-           SequenceNumber newest_snapshot, JobContext* job_context,
-           LogBuffer* log_buffer, Directory* db_directory,
-           CompressionType output_compression, Statistics* stats);
+           FileNumToPathIdMap* pending_outputs, SequenceNumber newest_snapshot,
+           JobContext* job_context, LogBuffer* log_buffer,
+           Directory* db_directory, CompressionType output_compression,
+           Statistics* stats);
   ~FlushJob() {}
 
-  Status Run(uint64_t* file_number = nullptr);
+  Status Run();
 
  private:
   Status WriteLevel0Table(const autovector<MemTable*>& mems, VersionEdit* edit,
@@ -73,6 +74,7 @@ class FlushJob {
   VersionSet* versions_;
   port::Mutex* db_mutex_;
   std::atomic<bool>* shutting_down_;
+  FileNumToPathIdMap* pending_outputs_;
   SequenceNumber newest_snapshot_;
   JobContext* job_context_;
   LogBuffer* log_buffer_;

db/flush_job_test.cc

@@ -9,7 +9,6 @@
 #include "db/flush_job.h"
 #include "db/column_family.h"
 #include "db/version_set.h"
-#include "db/writebuffer.h"
 #include "rocksdb/cache.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
@@ -26,12 +25,10 @@ class FlushJobTest {
       : env_(Env::Default()),
         dbname_(test::TmpDir() + "/flush_job_test"),
         table_cache_(NewLRUCache(50000, 16, 8)),
-        write_buffer_(db_options_.db_write_buffer_size),
         versions_(new VersionSet(dbname_, &db_options_, env_options_,
-                                 table_cache_.get(), &write_buffer_,
-                                 &write_controller_)),
+                                 table_cache_.get(), &write_controller_)),
         shutting_down_(false),
-        mock_table_factory_(new mock::MockTableFactory()) {
+        mock_table_factory_(new MockTableFactory()) {
     ASSERT_OK(env_->CreateDirIfMissing(dbname_));
     db_options_.db_paths.emplace_back(dbname_,
                                       std::numeric_limits<uint64_t>::max());
@@ -72,12 +69,12 @@ class FlushJobTest {
   std::shared_ptr<Cache> table_cache_;
   WriteController write_controller_;
   DBOptions db_options_;
-  WriteBuffer write_buffer_;
   ColumnFamilyOptions cf_options_;
   std::unique_ptr<VersionSet> versions_;
   port::Mutex mutex_;
   std::atomic<bool> shutting_down_;
-  std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
+  FileNumToPathIdMap pending_outputs_;
+  std::shared_ptr<MockTableFactory> mock_table_factory_;
 };
 
 TEST(FlushJobTest, Empty) {
@@ -86,21 +83,22 @@ TEST(FlushJobTest, Empty) {
   FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     SequenceNumber(), &job_context, nullptr, nullptr,
-                     kNoCompression, nullptr);
+                     &pending_outputs_, SequenceNumber(), &job_context, nullptr,
+                     nullptr, kNoCompression, nullptr);
   ASSERT_OK(flush_job.Run());
-  job_context.Clean();
 }
 
 TEST(FlushJobTest, NonEmpty) {
   JobContext job_context;
   auto cfd = versions_->GetColumnFamilySet()->GetDefault();
-  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions());
+
+  auto new_mem = new MemTable(cfd->internal_comparator(), *cfd->ioptions(),
+                              *cfd->GetLatestMutableCFOptions());
   new_mem->Ref();
   std::map<std::string, std::string> inserted_keys;
   for (int i = 1; i < 10000; ++i) {
-    std::string key(ToString(i));
-    std::string value("value" + ToString(i));
+    std::string key(std::to_string(i));
+    std::string value("value" + std::to_string(i));
     new_mem->Add(SequenceNumber(i), kTypeValue, key, value);
     InternalKey internal_key(key, SequenceNumber(i), kTypeValue);
     inserted_keys.insert({internal_key.Encode().ToString(), value});
@@ -110,13 +108,12 @@ TEST(FlushJobTest, NonEmpty) {
   FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     SequenceNumber(), &job_context, nullptr, nullptr,
-                     kNoCompression, nullptr);
+                     &pending_outputs_, SequenceNumber(), &job_context, nullptr,
+                     nullptr, kNoCompression, nullptr);
   mutex_.Lock();
   ASSERT_OK(flush_job.Run());
   mutex_.Unlock();
   mock_table_factory_->AssertSingleFile(inserted_keys);
-  job_context.Clean();
 }
 
 }  // namespace rocksdb

db/forward_iterator.cc

@@ -119,14 +119,12 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
     : db_(db),
       read_options_(read_options),
       cfd_(cfd),
-      prefix_extractor_(cfd->ioptions()->prefix_extractor),
+      prefix_extractor_(cfd->options()->prefix_extractor.get()),
       user_comparator_(cfd->user_comparator()),
       immutable_min_heap_(MinIterComparator(&cfd_->internal_comparator())),
       sv_(current_sv),
       mutable_iter_(nullptr),
       current_(nullptr),
-      status_(Status::OK()),
-      immutable_status_(Status::OK()),
       valid_(false),
       is_prev_set_(false),
       is_prev_inclusive_(false) {
@@ -179,7 +177,7 @@ void ForwardIterator::SeekToFirst() {
   if (sv_ == nullptr ||
       sv_ ->version_number != cfd_->GetSuperVersionNumber()) {
     RebuildIterators(true);
-  } else if (immutable_status_.IsIncomplete()) {
+  } else if (status_.IsIncomplete()) {
     ResetIncompleteIterators();
   }
   SeekInternal(Slice(), true);
@@ -189,7 +187,7 @@ void ForwardIterator::Seek(const Slice& internal_key) {
   if (sv_ == nullptr ||
       sv_ ->version_number != cfd_->GetSuperVersionNumber()) {
     RebuildIterators(true);
-  } else if (immutable_status_.IsIncomplete()) {
+  } else if (status_.IsIncomplete()) {
     ResetIncompleteIterators();
   }
   SeekInternal(internal_key, false);
@@ -207,16 +205,13 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
   // if it turns to need to seek immutable often. We probably want to have
   // an option to turn it off.
   if (seek_to_first || NeedToSeekImmutable(internal_key)) {
-    immutable_status_ = Status::OK();
     {
       auto tmp = MinIterHeap(MinIterComparator(&cfd_->internal_comparator()));
       immutable_min_heap_.swap(tmp);
     }
     for (auto* m : imm_iters_) {
       seek_to_first ? m->SeekToFirst() : m->Seek(internal_key);
-      if (!m->status().ok()) {
-        immutable_status_ = m->status();
-      } else if (m->Valid()) {
+      if (m->Valid()) {
         immutable_min_heap_.push(m);
       }
     }
@@ -240,8 +235,13 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         l0_iters_[i]->Seek(internal_key);
       }
 
-      if (!l0_iters_[i]->status().ok()) {
-        immutable_status_ = l0_iters_[i]->status();
+      if (l0_iters_[i]->status().IsIncomplete()) {
+        // if any of the immutable iterators is incomplete (no-io option was
+        // used), we are unable to reliably find the smallest key
+        assert(read_options_.read_tier == kBlockCacheTier);
+        status_ = l0_iters_[i]->status();
+        valid_ = false;
+        return;
       } else if (l0_iters_[i]->Valid()) {
         immutable_min_heap_.push(l0_iters_[i]);
       }
@@ -264,11 +264,10 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         if (search_left_bound == search_right_bound) {
           f_idx = search_left_bound;
         } else if (search_left_bound < search_right_bound) {
-          f_idx =
-              FindFileInRange(level_files, internal_key, search_left_bound,
-                              search_right_bound == FileIndexer::kLevelMaxIndex
-                                  ? static_cast<uint32_t>(level_files.size())
-                                  : search_right_bound);
+          f_idx = FindFileInRange(
+              level_files, internal_key, search_left_bound,
+              search_right_bound == FileIndexer::kLevelMaxIndex ?
+                level_files.size() : search_right_bound);
         } else {
           // search_left_bound > search_right_bound
           // There are only 2 cases this can happen:
@@ -312,8 +311,12 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         seek_to_first ? level_iters_[level - 1]->SeekToFirst() :
                         level_iters_[level - 1]->Seek(internal_key);
 
-        if (!level_iters_[level - 1]->status().ok()) {
-          immutable_status_ = level_iters_[level - 1]->status();
+        if (level_iters_[level - 1]->status().IsIncomplete()) {
+          // see above
+          assert(read_options_.read_tier == kBlockCacheTier);
+          status_ = level_iters_[level - 1]->status();
+          valid_ = false;
+          return;
         } else if (level_iters_[level - 1]->Valid()) {
           immutable_min_heap_.push(level_iters_[level - 1]);
         }
@@ -368,8 +371,11 @@ void ForwardIterator::Next() {
 
   current_->Next();
   if (current_ != mutable_iter_) {
-    if (!current_->status().ok()) {
-      immutable_status_ = current_->status();
+    if (current_->status().IsIncomplete()) {
+      assert(read_options_.read_tier == kBlockCacheTier);
+      status_ = current_->status();
+      valid_ = false;
+      return;
     } else if (current_->Valid()) {
       immutable_min_heap_.push(current_);
     }
@@ -395,7 +401,23 @@ Status ForwardIterator::status() const {
     return mutable_iter_->status();
   }
 
-  return immutable_status_;
+  for (auto *it : imm_iters_) {
+    if (it && !it->status().ok()) {
+      return it->status();
+    }
+  }
+  for (auto *it : l0_iters_) {
+    if (it && !it->status().ok()) {
+      return it->status();
+    }
+  }
+  for (auto *it : level_iters_) {
+    if (it && !it->status().ok()) {
+      return it->status();
+    }
+  }
+
+  return Status::OK();
 }
 
 void ForwardIterator::RebuildIterators(bool refresh_sv) {
@@ -489,7 +511,7 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
   // 'target' belongs to that interval (immutable_min_heap_.top() is already
   // at the correct position).
 
-  if (!valid_ || !current_ || !is_prev_set_ || !immutable_status_.ok()) {
+  if (!valid_ || !current_ || !is_prev_set_) {
     return true;
   }
   Slice prev_key = prev_key_.GetKey();

db/forward_iterator.h

@@ -97,7 +97,6 @@ class ForwardIterator : public Iterator {
   Iterator* current_;
   // internal iterator status
   Status status_;
-  Status immutable_status_;
   bool valid_;
 
   IterKey prev_key_;

db/internal_stats.cc

@@ -15,13 +15,10 @@
 #include <inttypes.h>
 #include <vector>
 #include "db/column_family.h"
-
 #include "db/db_impl.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
 namespace {
 const double kMB = 1048576.0;
 const double kGB = kMB * 1024;
@@ -31,18 +28,18 @@ void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
       buf, len,
       "\n** Compaction Stats [%s] **\n"
       "Level   Files   Size(MB) Score Read(GB)  Rn(GB) Rnp1(GB) "
-      "Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) "
-      "Comp(sec) Comp(cnt) Avg(sec) "
+      "Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s)  Rn(cnt) "
+      "Rnp1(cnt) Wnp1(cnt) Wnew(cnt)  Comp(sec) Comp(cnt) Avg(sec) "
       "Stall(sec) Stall(cnt) Avg(ms)     RecordIn   RecordDrop\n"
       "--------------------------------------------------------------------"
       "--------------------------------------------------------------------"
-      "----------------------------------------------------------\n",
+      "--------------------------------------------------------------------\n",
       cf_name.c_str());
 }
 
 void PrintLevelStats(char* buf, size_t len, const std::string& name,
     int num_files, int being_compacted, double total_file_size, double score,
-    double w_amp, double stall_us, uint64_t stalls,
+    double rw_amp, double w_amp, double stall_us, uint64_t stalls,
     const InternalStats::CompactionStats& stats) {
   uint64_t bytes_read = stats.bytes_readn + stats.bytes_readnp1;
   uint64_t bytes_new = stats.bytes_written - stats.bytes_readnp1;
@@ -55,11 +52,15 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
            "%8.1f "                    /* Rnp1(GB) */
            "%9.1f "                    /* Write(GB) */
            "%8.1f "                    /* Wnew(GB) */
-           "%9.1f "                    /* Moved(GB) */
+           "%6.1f "                    /* RW-Amp */
            "%5.1f "                    /* W-Amp */
            "%8.1f "                    /* Rd(MB/s) */
            "%8.1f "                    /* Wr(MB/s) */
-           "%9.0f "                   /* Comp(sec) */
+           "%8d "                      /* Rn(cnt) */
+           "%9d "                      /* Rnp1(cnt) */
+           "%9d "                      /* Wnp1(cnt) */
+           "%9d "                      /* Wnew(cnt) */
+           "%10.0f "                   /* Comp(sec) */
            "%9d "                      /* Comp(cnt) */
            "%8.3f "                    /* Avg(sec) */
            "%10.2f "                   /* Stall(sec) */
@@ -72,9 +73,10 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
            name.c_str(), num_files, being_compacted, total_file_size / kMB,
            score, bytes_read / kGB, stats.bytes_readn / kGB,
            stats.bytes_readnp1 / kGB, stats.bytes_written / kGB,
-           bytes_new / kGB, stats.bytes_moved / kGB,
-           w_amp, bytes_read / kMB / elapsed,
-           stats.bytes_written / kMB / elapsed,
+           bytes_new / kGB, rw_amp, w_amp, bytes_read / kMB / elapsed,
+           stats.bytes_written / kMB / elapsed, stats.files_in_leveln,
+           stats.files_in_levelnp1, stats.files_out_levelnp1,
+           stats.files_out_levelnp1 - stats.files_in_levelnp1,
            stats.micros / 1000000.0, stats.count,
            stats.count == 0 ? 0 : stats.micros / 1000000.0 / stats.count,
            stall_us / 1000000.0, stalls,
@@ -136,10 +138,6 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
     return kEstimatedUsageByTableReaders;
   } else if (in == "is-file-deletions-enabled") {
     return kIsFileDeletionEnabled;
-  } else if (in == "num-snapshots") {
-    return kNumSnapshots;
-  } else if (in == "oldest-snapshot-time") {
-    return kOldestSnapshotTime;
   }
   return kUnknown;
 }
@@ -237,7 +235,7 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
     case kCompactionPending:
       // 1 if the system already determines at least one compacdtion is needed.
       // 0 otherwise,
-      *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
+      *value = (vstorage->NeedsCompaction() ? 1 : 0);
       return true;
     case kBackgroundErrors:
       // Accumulated number of  errors in background flushes or compactions.
@@ -267,12 +265,6 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
                cfd_->imm()->current()->GetTotalNumEntries() +
                vstorage->GetEstimatedActiveKeys();
       return true;
-    case kNumSnapshots:
-      *value = db->snapshots().count();
-      return true;
-    case kOldestSnapshotTime:
-      *value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
-      return true;
 #ifndef ROCKSDB_LITE
     case kIsFileDeletionEnabled:
       *value = db->IsFileDeletionsEnabled();
@@ -369,8 +361,8 @@ void InternalStats::DumpCFStats(std::string* value) {
   const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
 
   int num_levels_to_check =
-      (cfd_->ioptions()->compaction_style != kCompactionStyleUniversal &&
-       cfd_->ioptions()->compaction_style != kCompactionStyleFIFO)
+      (cfd_->options()->compaction_style != kCompactionStyleUniversal &&
+       cfd_->options()->compaction_style != kCompactionStyleFIFO)
           ? vstorage->num_levels() - 1
           : 1;
 
@@ -433,22 +425,29 @@ void InternalStats::DumpCFStats(std::string* value) {
       total_slowdown_count_soft += stall_leveln_slowdown_count_soft_[level];
       total_slowdown_hard += stall_leveln_slowdown_hard_[level];
       total_slowdown_count_hard += stall_leveln_slowdown_count_hard_[level];
+      int64_t bytes_read = comp_stats_[level].bytes_readn +
+                           comp_stats_[level].bytes_readnp1;
+      double rw_amp = (comp_stats_[level].bytes_readn == 0) ? 0.0
+          : (comp_stats_[level].bytes_written + bytes_read) /
+            static_cast<double>(comp_stats_[level].bytes_readn);
       double w_amp = (comp_stats_[level].bytes_readn == 0) ? 0.0
           : comp_stats_[level].bytes_written /
             static_cast<double>(comp_stats_[level].bytes_readn);
-      PrintLevelStats(buf, sizeof(buf), "L" + ToString(level), files,
+      PrintLevelStats(buf, sizeof(buf), "L" + std::to_string(level), files,
                       files_being_compacted[level],
                       vstorage->NumLevelBytes(level), compaction_score[level],
-                      w_amp, stall_us, stalls, comp_stats_[level]);
+                      rw_amp, w_amp, stall_us, stalls, comp_stats_[level]);
       value->append(buf);
     }
   }
   uint64_t curr_ingest = cf_stats_value_[BYTES_FLUSHED];
   // Cumulative summary
+  double rw_amp = (stats_sum.bytes_written + stats_sum.bytes_readn +
+      stats_sum.bytes_readnp1) / static_cast<double>(curr_ingest + 1);
   double w_amp = stats_sum.bytes_written / static_cast<double>(curr_ingest + 1);
   // Stats summary across levels
   PrintLevelStats(buf, sizeof(buf), "Sum", total_files,
-      total_files_being_compacted, total_file_size, 0, w_amp,
+      total_files_being_compacted, total_file_size, 0, rw_amp, w_amp,
       total_stall_us, total_stall_count, stats_sum);
   value->append(buf);
   // Interval summary
@@ -456,9 +455,12 @@ void InternalStats::DumpCFStats(std::string* value) {
       curr_ingest - cf_stats_snapshot_.ingest_bytes + 1;
   CompactionStats interval_stats(stats_sum);
   interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
+  rw_amp = (interval_stats.bytes_written +
+      interval_stats.bytes_readn + interval_stats.bytes_readnp1) /
+      static_cast<double>(interval_ingest);
   w_amp = interval_stats.bytes_written / static_cast<double>(interval_ingest);
   PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0,
-      w_amp, total_stall_us - cf_stats_snapshot_.stall_us,
+      rw_amp, w_amp, total_stall_us - cf_stats_snapshot_.stall_us,
       total_stall_count - cf_stats_snapshot_.stall_count, interval_stats);
   value->append(buf);
 
@@ -494,14 +496,4 @@ void InternalStats::DumpCFStats(std::string* value) {
   cf_stats_snapshot_.stall_count = total_stall_count;
 }
 
-
-#else
-
-DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
-                               bool* need_out_of_mutex) {
-  return kUnknown;
-}
-
-#endif  // !ROCKSDB_LITE
-
 }  // namespace rocksdb

db/internal_stats.h

@@ -46,16 +46,12 @@ enum DBPropertyType : uint32_t {
   kEstimatedUsageByTableReaders,  // Estimated memory by table readers.
   kIsFileDeletionEnabled,         // Equals disable_delete_obsolete_files_,
                                   // 0 means file deletions enabled
-  kNumSnapshots,                  // Number of snapshots in the system
-  kOldestSnapshotTime,            // Unix timestamp of the first snapshot
 };
 
 extern DBPropertyType GetPropertyType(const Slice& property,
                                       bool* is_int_property,
                                       bool* need_out_of_mutex);
 
-
-#ifndef ROCKSDB_LITE
 class InternalStats {
  public:
   enum InternalCFStatsType {
@@ -121,9 +117,6 @@ class InternalStats {
     // Total bytes written during compaction between levels N and N+1
     uint64_t bytes_written;
 
-    // Total bytes moved to this level
-    uint64_t bytes_moved;
-
     // Files read from level N during compaction between levels N and N+1
     int files_in_leveln;
 
@@ -148,7 +141,6 @@ class InternalStats {
           bytes_readn(0),
           bytes_readnp1(0),
           bytes_written(0),
-          bytes_moved(0),
           files_in_leveln(0),
           files_in_levelnp1(0),
           files_out_levelnp1(0),
@@ -161,7 +153,6 @@ class InternalStats {
           bytes_readn(c.bytes_readn),
           bytes_readnp1(c.bytes_readnp1),
           bytes_written(c.bytes_written),
-          bytes_moved(c.bytes_moved),
           files_in_leveln(c.files_in_leveln),
           files_in_levelnp1(c.files_in_levelnp1),
           files_out_levelnp1(c.files_out_levelnp1),
@@ -174,7 +165,6 @@ class InternalStats {
       this->bytes_readn += c.bytes_readn;
       this->bytes_readnp1 += c.bytes_readnp1;
       this->bytes_written += c.bytes_written;
-      this->bytes_moved += c.bytes_moved;
       this->files_in_leveln += c.files_in_leveln;
       this->files_in_levelnp1 += c.files_in_levelnp1;
       this->files_out_levelnp1 += c.files_out_levelnp1;
@@ -188,7 +178,6 @@ class InternalStats {
       this->bytes_readn -= c.bytes_readn;
       this->bytes_readnp1 -= c.bytes_readnp1;
       this->bytes_written -= c.bytes_written;
-      this->bytes_moved -= c.bytes_moved;
       this->files_in_leveln -= c.files_in_leveln;
       this->files_in_levelnp1 -= c.files_in_levelnp1;
       this->files_out_levelnp1 -= c.files_out_levelnp1;
@@ -202,10 +191,6 @@ class InternalStats {
     comp_stats_[level].Add(stats);
   }
 
-  void IncBytesMoved(int level, uint64_t amount) {
-    comp_stats_[level].bytes_moved += amount;
-  }
-
   void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {
     if (soft) {
       stall_leveln_slowdown_soft_[level] += micros;
@@ -311,77 +296,4 @@ class InternalStats {
   const uint64_t started_at_;
 };
 
-#else
-
-class InternalStats {
- public:
-  enum InternalCFStatsType {
-    LEVEL0_SLOWDOWN,
-    MEMTABLE_COMPACTION,
-    LEVEL0_NUM_FILES,
-    WRITE_STALLS_ENUM_MAX,
-    BYTES_FLUSHED,
-    INTERNAL_CF_STATS_ENUM_MAX,
-  };
-
-  enum InternalDBStatsType {
-    WAL_FILE_BYTES,
-    WAL_FILE_SYNCED,
-    BYTES_WRITTEN,
-    NUMBER_KEYS_WRITTEN,
-    WRITE_DONE_BY_OTHER,
-    WRITE_DONE_BY_SELF,
-    WRITE_WITH_WAL,
-    INTERNAL_DB_STATS_ENUM_MAX,
-  };
-
-  InternalStats(int num_levels, Env* env, ColumnFamilyData* cfd) {}
-
-  struct CompactionStats {
-    uint64_t micros;
-    uint64_t bytes_readn;
-    uint64_t bytes_readnp1;
-    uint64_t bytes_written;
-    uint64_t bytes_moved;
-    int files_in_leveln;
-    int files_in_levelnp1;
-    int files_out_levelnp1;
-    uint64_t num_input_records;
-    uint64_t num_dropped_records;
-    int count;
-
-    explicit CompactionStats(int _count = 0) {}
-
-    explicit CompactionStats(const CompactionStats& c) {}
-
-    void Add(const CompactionStats& c) {}
-
-    void Subtract(const CompactionStats& c) {}
-  };
-
-  void AddCompactionStats(int level, const CompactionStats& stats) {}
-
-  void IncBytesMoved(int level, uint64_t amount) {}
-
-  void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {}
-
-  void AddCFStats(InternalCFStatsType type, uint64_t value) {}
-
-  void AddDBStats(InternalDBStatsType type, uint64_t value) {}
-
-  uint64_t GetBackgroundErrorCount() const { return 0; }
-
-  uint64_t BumpAndGetBackgroundErrorCount() { return 0; }
-
-  bool GetStringProperty(DBPropertyType property_type, const Slice& property,
-                         std::string* value) { return false; }
-
-  bool GetIntProperty(DBPropertyType property_type, uint64_t* value,
-                      DBImpl* db) const { return false; }
-
-  bool GetIntPropertyOutOfMutex(DBPropertyType property_type, Version* version,
-                                uint64_t* value) const { return false; }
-};
-#endif  // !ROCKSDB_LITE
-
 }  // namespace rocksdb

db/job_context.h

@@ -21,8 +21,7 @@ class MemTable;
 struct JobContext {
   inline bool HaveSomethingToDelete() const {
     return candidate_files.size() || sst_delete_files.size() ||
-           log_delete_files.size() || new_superversion != nullptr ||
-           superversions_to_free.size() > 0 || memtables_to_free.size() > 0;
+           log_delete_files.size();
   }
 
   // Structure to store information for candidate files to delete.
@@ -59,12 +58,8 @@ struct JobContext {
 
   // the current manifest_file_number, log_number and prev_log_number
   // that corresponds to the set of files in 'live'.
-  uint64_t manifest_file_number;
-  uint64_t pending_manifest_file_number;
-  uint64_t log_number;
-  uint64_t prev_log_number;
-
-  uint64_t min_pending_output = 0;
+  uint64_t manifest_file_number, pending_manifest_file_number, log_number,
+      prev_log_number;
 
   explicit JobContext(bool create_superversion = false) {
     manifest_file_number = 0;
@@ -74,7 +69,7 @@ struct JobContext {
     new_superversion = create_superversion ? new SuperVersion() : nullptr;
   }
 
-  void Clean() {
+  ~JobContext() {
     // free pending memtables
     for (auto m : memtables_to_free) {
       delete m;
@@ -86,16 +81,6 @@ struct JobContext {
     // if new_superversion was not used, it will be non-nullptr and needs
     // to be freed here
     delete new_superversion;
-
-    memtables_to_free.clear();
-    superversions_to_free.clear();
-    new_superversion = nullptr;
-  }
-
-  ~JobContext() {
-    assert(memtables_to_free.size() == 0);
-    assert(superversions_to_free.size() == 0);
-    assert(new_superversion == nullptr);
   }
 };
 

db/listener_test.cc

@@ -1,348 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-
-#include "db/dbformat.h"
-#include "db/db_impl.h"
-#include "db/filename.h"
-#include "db/version_set.h"
-#include "db/write_batch_internal.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/compaction_filter.h"
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/filter_policy.h"
-#include "rocksdb/perf_context.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/slice_transform.h"
-#include "rocksdb/table.h"
-#include "rocksdb/options.h"
-#include "rocksdb/table_properties.h"
-#include "table/block_based_table_factory.h"
-#include "table/plain_table_factory.h"
-#include "util/hash.h"
-#include "util/hash_linklist_rep.h"
-#include "utilities/merge_operators.h"
-#include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/rate_limiter.h"
-#include "util/statistics.h"
-#include "util/testharness.h"
-#include "util/sync_point.h"
-#include "util/testutil.h"
-
-#ifndef ROCKSDB_LITE
-
-namespace rocksdb {
-
-class EventListenerTest {
- public:
-  EventListenerTest() {
-    dbname_ = test::TmpDir() + "/listener_test";
-    ASSERT_OK(DestroyDB(dbname_, Options()));
-    db_ = nullptr;
-    Reopen();
-  }
-
-  ~EventListenerTest() {
-    Close();
-    Options options;
-    options.db_paths.emplace_back(dbname_, 0);
-    options.db_paths.emplace_back(dbname_ + "_2", 0);
-    options.db_paths.emplace_back(dbname_ + "_3", 0);
-    options.db_paths.emplace_back(dbname_ + "_4", 0);
-    ASSERT_OK(DestroyDB(dbname_, options));
-  }
-
-  void CreateColumnFamilies(const std::vector<std::string>& cfs,
-                            const ColumnFamilyOptions* options = nullptr) {
-    ColumnFamilyOptions cf_opts;
-    cf_opts = ColumnFamilyOptions(Options());
-    size_t cfi = handles_.size();
-    handles_.resize(cfi + cfs.size());
-    for (auto cf : cfs) {
-      ASSERT_OK(db_->CreateColumnFamily(cf_opts, cf, &handles_[cfi++]));
-    }
-  }
-
-  void Close() {
-    for (auto h : handles_) {
-      delete h;
-    }
-    handles_.clear();
-    delete db_;
-    db_ = nullptr;
-  }
-
-  void ReopenWithColumnFamilies(const std::vector<std::string>& cfs,
-                                const Options* options = nullptr) {
-    ASSERT_OK(TryReopenWithColumnFamilies(cfs, options));
-  }
-
-  Status TryReopenWithColumnFamilies(const std::vector<std::string>& cfs,
-                                     const Options* options = nullptr) {
-    Close();
-    Options opts = (options == nullptr) ? Options() : *options;
-    std::vector<const Options*> v_opts(cfs.size(), &opts);
-    return TryReopenWithColumnFamilies(cfs, v_opts);
-  }
-
-  Status TryReopenWithColumnFamilies(
-      const std::vector<std::string>& cfs,
-      const std::vector<const Options*>& options) {
-    Close();
-    ASSERT_EQ(cfs.size(), options.size());
-    std::vector<ColumnFamilyDescriptor> column_families;
-    for (size_t i = 0; i < cfs.size(); ++i) {
-      column_families.push_back(ColumnFamilyDescriptor(cfs[i], *options[i]));
-    }
-    DBOptions db_opts = DBOptions(*options[0]);
-    return DB::Open(db_opts, dbname_, column_families, &handles_, &db_);
-  }
-
-  Status TryReopen(Options* options = nullptr) {
-    Close();
-    Options opts;
-    if (options != nullptr) {
-      opts = *options;
-    } else {
-      opts.create_if_missing = true;
-    }
-
-    return DB::Open(opts, dbname_, &db_);
-  }
-
-  void Reopen(Options* options = nullptr) {
-    ASSERT_OK(TryReopen(options));
-  }
-
-  void CreateAndReopenWithCF(const std::vector<std::string>& cfs,
-                             const Options* options = nullptr) {
-    CreateColumnFamilies(cfs, options);
-    std::vector<std::string> cfs_plus_default = cfs;
-    cfs_plus_default.insert(cfs_plus_default.begin(), kDefaultColumnFamilyName);
-    ReopenWithColumnFamilies(cfs_plus_default, options);
-  }
-
-  DBImpl* dbfull() {
-    return reinterpret_cast<DBImpl*>(db_);
-  }
-
-  Status Put(int cf, const Slice& k, const Slice& v,
-             WriteOptions wo = WriteOptions()) {
-    return db_->Put(wo, handles_[cf], k, v);
-  }
-
-  Status Flush(int cf = 0) {
-    FlushOptions opt = FlushOptions();
-    opt.wait = true;
-    if (cf == 0) {
-      return db_->Flush(opt);
-    } else {
-      return db_->Flush(opt, handles_[cf]);
-    }
-  }
-
-
-  DB* db_;
-  std::string dbname_;
-  std::vector<ColumnFamilyHandle*> handles_;
-};
-
-class TestFlushListener : public EventListener {
- public:
-  void OnFlushCompleted(
-      DB* db, const std::string& name,
-      const std::string& file_path,
-      bool triggered_writes_slowdown,
-      bool triggered_writes_stop) override {
-    flushed_dbs_.push_back(db);
-    flushed_column_family_names_.push_back(name);
-    if (triggered_writes_slowdown) {
-      slowdown_count++;
-    }
-    if (triggered_writes_stop) {
-      stop_count++;
-    }
-  }
-
-  std::vector<std::string> flushed_column_family_names_;
-  std::vector<DB*> flushed_dbs_;
-  int slowdown_count;
-  int stop_count;
-};
-
-TEST(EventListenerTest, OnSingleDBFlushTest) {
-  Options options;
-  options.write_buffer_size = 100000;
-  TestFlushListener* listener = new TestFlushListener();
-  options.listeners.emplace_back(listener);
-  std::vector<std::string> cf_names = {
-      "pikachu", "ilya", "muromec", "dobrynia",
-      "nikitich", "alyosha", "popovich"};
-  CreateAndReopenWithCF(cf_names, &options);
-
-  ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
-  ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
-  ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
-  ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
-  ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
-  ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
-  ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
-  for (size_t i = 1; i < 8; ++i) {
-    ASSERT_OK(Flush(static_cast<int>(i)));
-    dbfull()->TEST_WaitForFlushMemTable();
-    ASSERT_EQ(listener->flushed_dbs_.size(), i);
-    ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
-  }
-
-  // make sure call-back functions are called in the right order
-  for (size_t i = 0; i < cf_names.size(); ++i) {
-    ASSERT_EQ(listener->flushed_dbs_[i], db_);
-    ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]);
-  }
-}
-
-TEST(EventListenerTest, MultiCF) {
-  Options options;
-  options.write_buffer_size = 100000;
-  TestFlushListener* listener = new TestFlushListener();
-  options.listeners.emplace_back(listener);
-  std::vector<std::string> cf_names = {
-      "pikachu", "ilya", "muromec", "dobrynia",
-      "nikitich", "alyosha", "popovich"};
-  CreateAndReopenWithCF(cf_names, &options);
-
-  ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
-  ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
-  ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
-  ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
-  ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
-  ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
-  ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
-  for (size_t i = 1; i < 8; ++i) {
-    ASSERT_OK(Flush(static_cast<int>(i)));
-    ASSERT_EQ(listener->flushed_dbs_.size(), i);
-    ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
-  }
-
-  // make sure call-back functions are called in the right order
-  for (size_t i = 0; i < cf_names.size(); i++) {
-    ASSERT_EQ(listener->flushed_dbs_[i], db_);
-    ASSERT_EQ(listener->flushed_column_family_names_[i], cf_names[i]);
-  }
-}
-
-TEST(EventListenerTest, MultiDBMultiListeners) {
-  std::vector<TestFlushListener*> listeners;
-  const int kNumDBs = 5;
-  const int kNumListeners = 10;
-  for (int i = 0; i < kNumListeners; ++i) {
-    listeners.emplace_back(new TestFlushListener());
-  }
-
-  std::vector<std::string> cf_names = {
-      "pikachu", "ilya", "muromec", "dobrynia",
-      "nikitich", "alyosha", "popovich"};
-
-  Options options;
-  options.create_if_missing = true;
-  for (int i = 0; i < kNumListeners; ++i) {
-    options.listeners.emplace_back(listeners[i]);
-  }
-  DBOptions db_opts(options);
-  ColumnFamilyOptions cf_opts(options);
-
-  std::vector<DB*> dbs;
-  std::vector<std::vector<ColumnFamilyHandle *>> vec_handles;
-
-  for (int d = 0; d < kNumDBs; ++d) {
-    ASSERT_OK(DestroyDB(dbname_ + ToString(d), options));
-    DB* db;
-    std::vector<ColumnFamilyHandle*> handles;
-    ASSERT_OK(DB::Open(options, dbname_ + ToString(d), &db));
-    for (size_t c = 0; c < cf_names.size(); ++c) {
-      ColumnFamilyHandle* handle;
-      db->CreateColumnFamily(cf_opts, cf_names[c], &handle);
-      handles.push_back(handle);
-    }
-
-    vec_handles.push_back(std::move(handles));
-    dbs.push_back(db);
-  }
-
-  for (int d = 0; d < kNumDBs; ++d) {
-    for (size_t c = 0; c < cf_names.size(); ++c) {
-      ASSERT_OK(dbs[d]->Put(WriteOptions(), vec_handles[d][c],
-                cf_names[c], cf_names[c]));
-    }
-  }
-
-  for (size_t c = 0; c < cf_names.size(); ++c) {
-    for (int d = 0; d < kNumDBs; ++d) {
-      ASSERT_OK(dbs[d]->Flush(FlushOptions(), vec_handles[d][c]));
-      reinterpret_cast<DBImpl*>(dbs[d])->TEST_WaitForFlushMemTable();
-    }
-  }
-
-  for (auto* listener : listeners) {
-    int pos = 0;
-    for (size_t c = 0; c < cf_names.size(); ++c) {
-      for (int d = 0; d < kNumDBs; ++d) {
-        ASSERT_EQ(listener->flushed_dbs_[pos], dbs[d]);
-        ASSERT_EQ(listener->flushed_column_family_names_[pos], cf_names[c]);
-        pos++;
-      }
-    }
-  }
-
-  for (auto handles : vec_handles) {
-    for (auto h : handles) {
-      delete h;
-    }
-    handles.clear();
-  }
-  vec_handles.clear();
-
-  for (auto db : dbs) {
-    delete db;
-  }
-}
-
-TEST(EventListenerTest, DisableBGCompaction) {
-  Options options;
-  TestFlushListener* listener = new TestFlushListener();
-  const int kSlowdownTrigger = 5;
-  const int kStopTrigger = 10;
-  options.level0_slowdown_writes_trigger = kSlowdownTrigger;
-  options.level0_stop_writes_trigger = kStopTrigger;
-  options.listeners.emplace_back(listener);
-  // BG compaction is disabled.  Number of L0 files will simply keeps
-  // increasing in this test.
-  options.compaction_style = kCompactionStyleNone;
-  options.compression = kNoCompression;
-  options.write_buffer_size = 100000;  // Small write buffer
-
-  CreateAndReopenWithCF({"pikachu"}, &options);
-  WriteOptions wopts;
-  wopts.timeout_hint_us = 100000;
-  ColumnFamilyMetaData cf_meta;
-  db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
-  // keep writing until writes are forced to stop.
-  for (int i = 0; static_cast<int>(cf_meta.file_count) < kStopTrigger; ++i) {
-    Put(1, ToString(i), std::string(100000, 'x'), wopts);
-    db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
-  }
-  ASSERT_GE(listener->slowdown_count, kStopTrigger - kSlowdownTrigger);
-  ASSERT_GE(listener->stop_count, 1);
-}
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE
-
-int main(int argc, char** argv) {
-  return rocksdb::test::RunAllTests();
-}
-

db/log_and_apply_bench.cc

@@ -6,23 +6,17 @@
 
 #include <vector>
 
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
 #include "util/testharness.h"
 #include "util/benchharness.h"
 #include "db/version_set.h"
 #include "db/write_controller.h"
-#include "db/writebuffer.h"
 #include "util/mutexlock.h"
 
 namespace rocksdb {
 
-std::string MakeKey(uint64_t num) {
+std::string MakeKey(unsigned int num) {
   char buf[30];
-  snprintf(buf, sizeof(buf), "%016" PRIu64, num);
+  snprintf(buf, sizeof(buf), "%016u", num);
   return std::string(buf);
 }
 
@@ -53,10 +47,9 @@ void BM_LogAndApply(int iters, int num_base_files) {
     // Notice we are using the default options not through SanitizeOptions().
     // We might want to initialize some options manually if needed.
     options.db_paths.emplace_back(dbname, 0);
-    WriteBuffer wb(options.db_write_buffer_size);
     // The parameter of table cache is passed in as null, so any file I/O
     // operation is likely to fail.
-    vset = new VersionSet(dbname, &options, sopt, nullptr, &wb, &wc);
+    vset = new VersionSet(dbname, &options, sopt, nullptr, &wc);
     std::vector<ColumnFamilyDescriptor> dummy;
     dummy.push_back(ColumnFamilyDescriptor());
     ASSERT_OK(vset->Recover(dummy));

db/log_reader.cc

@@ -54,7 +54,7 @@ bool Reader::SkipToInitialBlock() {
   if (block_start_location > 0) {
     Status skip_status = file_->Skip(block_start_location);
     if (!skip_status.ok()) {
-      ReportDrop(static_cast<size_t>(block_start_location), skip_status);
+      ReportDrop(block_start_location, skip_status);
       return false;
     }
   }

db/log_test.cc

@@ -558,9 +558,9 @@ TEST(LogTest, ErrorJoinsRecords) {
 
   ASSERT_EQ("correct", Read());
   ASSERT_EQ("EOF", Read());
-  size_t dropped = DroppedBytes();
-  ASSERT_LE(dropped, 2 * kBlockSize + 100);
-  ASSERT_GE(dropped, 2 * kBlockSize);
+  const unsigned int dropped = DroppedBytes();
+  ASSERT_LE(dropped, 2*kBlockSize + 100);
+  ASSERT_GE(dropped, 2*kBlockSize);
 }
 
 TEST(LogTest, ReadStart) {

db/memtable.cc

@@ -15,7 +15,6 @@
 
 #include "db/dbformat.h"
 #include "db/merge_context.h"
-#include "db/writebuffer.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
@@ -53,17 +52,14 @@ MemTableOptions::MemTableOptions(
 
 MemTable::MemTable(const InternalKeyComparator& cmp,
                    const ImmutableCFOptions& ioptions,
-                   const MutableCFOptions& mutable_cf_options,
-                   WriteBuffer* write_buffer)
+                   const MutableCFOptions& mutable_cf_options)
     : comparator_(cmp),
       moptions_(ioptions, mutable_cf_options),
       refs_(0),
       kArenaBlockSize(OptimizeBlockSize(moptions_.arena_block_size)),
       arena_(moptions_.arena_block_size),
-      allocator_(&arena_, write_buffer),
       table_(ioptions.memtable_factory->CreateMemTableRep(
-          comparator_, &allocator_, ioptions.prefix_extractor,
-          ioptions.info_log)),
+          comparator_, &arena_, ioptions.prefix_extractor, ioptions.info_log)),
       num_entries_(0),
       flush_in_progress_(false),
       flush_completed_(false),
@@ -80,7 +76,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
   assert(!should_flush_);
   if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) {
     prefix_bloom_.reset(new DynamicBloom(
-        &allocator_,
+        &arena_,
         moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality,
         moptions_.memtable_prefix_bloom_probes, nullptr,
         moptions_.memtable_prefix_bloom_huge_page_tlb_size,
@@ -183,7 +179,7 @@ Slice MemTableRep::UserKey(const char* key) const {
 }
 
 KeyHandle MemTableRep::Allocate(const size_t len, char** buf) {
-  *buf = allocator_->Allocate(len);
+  *buf = arena_->Allocate(len);
   return static_cast<KeyHandle>(*buf);
 }
 
@@ -192,7 +188,7 @@ KeyHandle MemTableRep::Allocate(const size_t len, char** buf) {
 // into this scratch space.
 const char* EncodeKey(std::string* scratch, const Slice& target) {
   scratch->clear();
-  PutVarint32(scratch, static_cast<uint32_t>(target.size()));
+  PutVarint32(scratch, target.size());
   scratch->append(target.data(), target.size());
   return scratch->data();
 }
@@ -292,12 +288,12 @@ void MemTable::Add(SequenceNumber s, ValueType type,
   //  key bytes    : char[internal_key.size()]
   //  value_size   : varint32 of value.size()
   //  value bytes  : char[value.size()]
-  uint32_t key_size = static_cast<uint32_t>(key.size());
-  uint32_t val_size = static_cast<uint32_t>(value.size());
-  uint32_t internal_key_size = key_size + 8;
-  const uint32_t encoded_len = VarintLength(internal_key_size) +
-                               internal_key_size + VarintLength(val_size) +
-                               val_size;
+  size_t key_size = key.size();
+  size_t val_size = value.size();
+  size_t internal_key_size = key_size + 8;
+  const size_t encoded_len =
+      VarintLength(internal_key_size) + internal_key_size +
+      VarintLength(val_size) + val_size;
   char* buf = nullptr;
   KeyHandle handle = table_->Allocate(encoded_len, &buf);
   assert(buf != nullptr);
@@ -506,8 +502,8 @@ void MemTable::Update(SequenceNumber seq,
       switch (static_cast<ValueType>(tag & 0xff)) {
         case kTypeValue: {
           Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length);
-          uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
-          uint32_t new_size = static_cast<uint32_t>(value.size());
+          uint32_t prev_size = prev_value.size();
+          uint32_t new_size = value.size();
 
           // Update value, if new value size  <= previous value size
           if (new_size <= prev_size ) {
@@ -564,10 +560,10 @@ bool MemTable::UpdateCallback(SequenceNumber seq,
       switch (static_cast<ValueType>(tag & 0xff)) {
         case kTypeValue: {
           Slice prev_value = GetLengthPrefixedSlice(key_ptr + key_length);
-          uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
+          uint32_t  prev_size = prev_value.size();
 
           char* prev_buffer = const_cast<char*>(prev_value.data());
-          uint32_t new_prev_size = prev_size;
+          uint32_t  new_prev_size = prev_size;
 
           std::string str_value;
           WriteLock wl(GetLock(lkey.user_key()));

db/memtable.h

@@ -19,17 +19,16 @@
 #include "rocksdb/db.h"
 #include "rocksdb/memtablerep.h"
 #include "rocksdb/immutable_options.h"
-#include "db/memtable_allocator.h"
 #include "util/arena.h"
 #include "util/dynamic_bloom.h"
 #include "util/mutable_cf_options.h"
 
 namespace rocksdb {
 
+class Arena;
 class Mutex;
 class MemTableIterator;
 class MergeContext;
-class WriteBuffer;
 
 struct MemTableOptions {
   explicit MemTableOptions(
@@ -68,8 +67,7 @@ class MemTable {
   // is zero and the caller must call Ref() at least once.
   explicit MemTable(const InternalKeyComparator& comparator,
                     const ImmutableCFOptions& ioptions,
-                    const MutableCFOptions& mutable_cf_options,
-                    WriteBuffer* write_buffer);
+                    const MutableCFOptions& mutable_cf_options);
 
   ~MemTable();
 
@@ -185,10 +183,7 @@ class MemTable {
   void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
 
   // Notify the underlying storage that no more items will be added
-  void MarkImmutable() {
-    table_->MarkReadOnly();
-    allocator_.DoneAllocating();
-  }
+  void MarkImmutable() { table_->MarkReadOnly(); }
 
   // return true if the current MemTableRep supports merge operator.
   bool IsMergeOperatorSupported() const {
@@ -205,6 +200,8 @@ class MemTable {
     return comparator_.comparator;
   }
 
+  const Arena& TEST_GetArena() const { return arena_; }
+
   const MemTableOptions* GetMemTableOptions() const { return &moptions_; }
 
  private:
@@ -220,7 +217,6 @@ class MemTable {
   int refs_;
   const size_t kArenaBlockSize;
   Arena arena_;
-  MemTableAllocator allocator_;
   unique_ptr<MemTableRep> table_;
 
   uint64_t num_entries_;

db/memtable_allocator.cc

@@ -1,52 +0,0 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include <assert.h>
-
-#include "db/memtable_allocator.h"
-#include "db/writebuffer.h"
-#include "util/arena.h"
-
-namespace rocksdb {
-
-MemTableAllocator::MemTableAllocator(Arena* arena, WriteBuffer* write_buffer)
-    : arena_(arena), write_buffer_(write_buffer), bytes_allocated_(0) {
-}
-
-MemTableAllocator::~MemTableAllocator() {
-  DoneAllocating();
-}
-
-char* MemTableAllocator::Allocate(size_t bytes) {
-  assert(write_buffer_ != nullptr);
-  bytes_allocated_ += bytes;
-  write_buffer_->ReserveMem(bytes);
-  return arena_->Allocate(bytes);
-}
-
-char* MemTableAllocator::AllocateAligned(size_t bytes, size_t huge_page_size,
-                                         Logger* logger) {
-  assert(write_buffer_ != nullptr);
-  bytes_allocated_ += bytes;
-  write_buffer_->ReserveMem(bytes);
-  return arena_->AllocateAligned(bytes, huge_page_size, logger);
-}
-
-void MemTableAllocator::DoneAllocating() {
-  if (write_buffer_ != nullptr) {
-    write_buffer_->FreeMem(bytes_allocated_);
-    write_buffer_ = nullptr;
-  }
-}
-
-size_t MemTableAllocator::BlockSize() const {
-  return arena_->BlockSize();
-}
-
-}  // namespace rocksdb

db/memtable_allocator.h

@@ -1,47 +0,0 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// This is used by the MemTable to allocate write buffer memory. It connects
-// to WriteBuffer so we can track and enforce overall write buffer limits.
-
-#pragma once
-#include "util/allocator.h"
-
-namespace rocksdb {
-
-class Arena;
-class Logger;
-class WriteBuffer;
-
-class MemTableAllocator : public Allocator {
- public:
-  explicit MemTableAllocator(Arena* arena, WriteBuffer* write_buffer);
-  ~MemTableAllocator();
-
-  // Allocator interface
-  char* Allocate(size_t bytes) override;
-  char* AllocateAligned(size_t bytes, size_t huge_page_size = 0,
-                        Logger* logger = nullptr) override;
-  size_t BlockSize() const override;
-
-  // Call when we're finished allocating memory so we can free it from
-  // the write buffer's limit.
-  void DoneAllocating();
-
- private:
-  Arena* arena_;
-  WriteBuffer* write_buffer_;
-  size_t bytes_allocated_;
-
-  // No copying allowed
-  MemTableAllocator(const MemTableAllocator&);
-  void operator=(const MemTableAllocator&);
-};
-
-}  // namespace rocksdb

db/memtable_list.cc

@@ -144,7 +144,8 @@ void MemTableList::PickMemtablesToFlush(autovector<MemTable*>* ret) {
 }
 
 void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
-                                         uint64_t file_number) {
+                                         uint64_t file_number,
+                                         FileNumToPathIdMap* pending_outputs) {
   assert(!mems.empty());
 
   // If the flush was not successful, then just reset state.
@@ -158,6 +159,7 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
     m->edit_.Clear();
     num_flush_not_started_++;
   }
+  pending_outputs->erase(file_number);
   imm_flush_needed.store(true, std::memory_order_release);
 }
 
@@ -165,8 +167,9 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
 Status MemTableList::InstallMemtableFlushResults(
     ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
     const autovector<MemTable*>& mems, VersionSet* vset, port::Mutex* mu,
-    uint64_t file_number, autovector<MemTable*>* to_delete,
-    Directory* db_directory, LogBuffer* log_buffer) {
+    uint64_t file_number, FileNumToPathIdMap* pending_outputs,
+    autovector<MemTable*>* to_delete, Directory* db_directory,
+    LogBuffer* log_buffer) {
   mu->AssertHeld();
 
   // flush was sucessful
@@ -217,6 +220,11 @@ Status MemTableList::InstallMemtableFlushResults(
         current_->Remove(m);
         assert(m->file_number_ > 0);
 
+        // pending_outputs can be cleared only after the newly created file
+        // has been written to a committed version so that other concurrently
+        // executing compaction threads do not mistakenly assume that this
+        // file is not live.
+        pending_outputs->erase(m->file_number_);
         if (m->Unref() != nullptr) {
           to_delete->push_back(m);
         }
@@ -229,6 +237,7 @@ Status MemTableList::InstallMemtableFlushResults(
         m->flush_in_progress_ = false;
         m->edit_.Clear();
         num_flush_not_started_++;
+        pending_outputs->erase(m->file_number_);
         m->file_number_ = 0;
         imm_flush_needed.store(true, std::memory_order_release);
       }

db/memtable_list.h

@@ -108,14 +108,16 @@ class MemTableList {
   // Reset status of the given memtable list back to pending state so that
   // they can get picked up again on the next round of flush.
   void RollbackMemtableFlush(const autovector<MemTable*>& mems,
-                             uint64_t file_number);
+                             uint64_t file_number,
+                             FileNumToPathIdMap* pending_outputs);
 
   // Commit a successful flush in the manifest file
   Status InstallMemtableFlushResults(
       ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
       const autovector<MemTable*>& m, VersionSet* vset, port::Mutex* mu,
-      uint64_t file_number, autovector<MemTable*>* to_delete,
-      Directory* db_directory, LogBuffer* log_buffer);
+      uint64_t file_number, FileNumToPathIdMap* pending_outputs,
+      autovector<MemTable*>* to_delete, Directory* db_directory,
+      LogBuffer* log_buffer);
 
   // New memtables are inserted at the front of the list.
   // Takes ownership of the referenced held on *m by the caller of Add().

db/merge_test.cc

@@ -23,11 +23,15 @@ using namespace std;
 using namespace rocksdb;
 
 namespace {
-size_t num_merge_operator_calls;
-void resetNumMergeOperatorCalls() { num_merge_operator_calls = 0; }
+  int numMergeOperatorCalls;
+  void resetNumMergeOperatorCalls() {
+    numMergeOperatorCalls = 0;
+  }
 
-size_t num_partial_merge_calls;
-void resetNumPartialMergeCalls() { num_partial_merge_calls = 0; }
+  int num_partial_merge_calls;
+  void resetNumPartialMergeCalls() {
+    num_partial_merge_calls = 0;
+  }
 }
 
 class CountMergeOperator : public AssociativeMergeOperator {
@@ -41,7 +45,7 @@ class CountMergeOperator : public AssociativeMergeOperator {
                      const Slice& value,
                      std::string* new_value,
                      Logger* logger) const override {
-    ++num_merge_operator_calls;
+    ++numMergeOperatorCalls;
     if (existing_value == nullptr) {
       new_value->assign(value.data(), value.size());
       return true;
@@ -303,31 +307,31 @@ void testCounters(Counters& counters, DB* db, bool test_compaction) {
   }
 }
 
-void testSuccessiveMerge(Counters& counters, size_t max_num_merges,
-                         size_t num_merges) {
+void testSuccessiveMerge(
+    Counters& counters, int max_num_merges, int num_merges) {
 
   counters.assert_remove("z");
   uint64_t sum = 0;
 
-  for (size_t i = 1; i <= num_merges; ++i) {
+  for (int i = 1; i <= num_merges; ++i) {
     resetNumMergeOperatorCalls();
     counters.assert_add("z", i);
     sum += i;
 
     if (i % (max_num_merges + 1) == 0) {
-      assert(num_merge_operator_calls == max_num_merges + 1);
+      assert(numMergeOperatorCalls == max_num_merges + 1);
     } else {
-      assert(num_merge_operator_calls == 0);
+      assert(numMergeOperatorCalls == 0);
     }
 
     resetNumMergeOperatorCalls();
     assert(counters.assert_get("z") == sum);
-    assert(num_merge_operator_calls == i % (max_num_merges + 1));
+    assert(numMergeOperatorCalls == i % (max_num_merges + 1));
   }
 }
 
-void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
-                      size_t min_merge, size_t count) {
+void testPartialMerge(Counters* counters, DB* db, int max_merge, int min_merge,
+                      int count) {
   FlushOptions o;
   o.wait = true;
 
@@ -335,7 +339,7 @@ void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
   //              operands exceeds the threshold.
   uint64_t tmp_sum = 0;
   resetNumPartialMergeCalls();
-  for (size_t i = 1; i <= count; i++) {
+  for (int i = 1; i <= count; i++) {
     counters->assert_add("b", i);
     tmp_sum += i;
   }
@@ -344,7 +348,7 @@ void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
   ASSERT_EQ(tmp_sum, counters->assert_get("b"));
   if (count > max_merge) {
     // in this case, FullMerge should be called instead.
-    ASSERT_EQ(num_partial_merge_calls, 0U);
+    ASSERT_EQ(num_partial_merge_calls, 0);
   } else {
     // if count >= min_merge, then partial merge should be called once.
     ASSERT_EQ((count >= min_merge), (num_partial_merge_calls == 1));
@@ -354,18 +358,20 @@ void testPartialMerge(Counters* counters, DB* db, size_t max_merge,
   resetNumPartialMergeCalls();
   tmp_sum = 0;
   db->Put(rocksdb::WriteOptions(), "c", "10");
-  for (size_t i = 1; i <= count; i++) {
+  for (int i = 1; i <= count; i++) {
     counters->assert_add("c", i);
     tmp_sum += i;
   }
   db->Flush(o);
   db->CompactRange(nullptr, nullptr);
   ASSERT_EQ(tmp_sum, counters->assert_get("c"));
-  ASSERT_EQ(num_partial_merge_calls, 0U);
+  ASSERT_EQ(num_partial_merge_calls, 0);
 }
 
-void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
-                                    size_t num_merges) {
+void testSingleBatchSuccessiveMerge(
+    DB* db,
+    int max_num_merges,
+    int num_merges) {
   assert(num_merges > max_num_merges);
 
   Slice key("BatchSuccessiveMerge");
@@ -374,7 +380,7 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
 
   // Create the batch
   WriteBatch batch;
-  for (size_t i = 0; i < num_merges; ++i) {
+  for (int i = 0; i < num_merges; ++i) {
     batch.Merge(key, merge_value_slice);
   }
 
@@ -384,9 +390,8 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
     Status s = db->Write(WriteOptions(), &batch);
     assert(s.ok());
   }
-  ASSERT_EQ(
-      num_merge_operator_calls,
-      static_cast<size_t>(num_merges - (num_merges % (max_num_merges + 1))));
+  assert(numMergeOperatorCalls ==
+      num_merges - (num_merges % (max_num_merges + 1)));
 
   // Get the value
   resetNumMergeOperatorCalls();
@@ -398,8 +403,7 @@ void testSingleBatchSuccessiveMerge(DB* db, size_t max_num_merges,
   assert(get_value_str.size() == sizeof(uint64_t));
   uint64_t get_value = DecodeFixed64(&get_value_str[0]);
   ASSERT_EQ(get_value, num_merges * merge_value);
-  ASSERT_EQ(num_merge_operator_calls,
-            static_cast<size_t>((num_merges % (max_num_merges + 1))));
+  ASSERT_EQ(numMergeOperatorCalls, (num_merges % (max_num_merges + 1)));
 }
 
 void runTest(int argc, const string& dbname, const bool use_ttl = false) {

db/perf_context_test.cc

@@ -14,7 +14,6 @@
 #include "util/histogram.h"
 #include "util/stop_watch.h"
 #include "util/testharness.h"
-#include "util/string_util.h"
 
 
 bool FLAGS_random_key = false;
@@ -39,13 +38,8 @@ std::shared_ptr<DB> OpenDb(bool read_only = false) {
       FLAGS_min_write_buffer_number_to_merge;
 
     if (FLAGS_use_set_based_memetable) {
-#ifndef ROCKSDB_LITE
       options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(0));
       options.memtable_factory.reset(NewHashSkipListRepFactory());
-#else
-      fprintf(stderr, "Prefix hash is not supported in lite mode\n");
-      exit(1);
-#endif  // ROCKSDB_LITE
     }
 
     Status s;
@@ -67,21 +61,21 @@ TEST(PerfContextTest, SeekIntoDeletion) {
   ReadOptions read_options;
 
   for (int i = 0; i < FLAGS_total_keys; ++i) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
     db->Put(write_options, key, value);
   }
 
   for (int i = 0; i < FLAGS_total_keys -1 ; ++i) {
-    std::string key = "k" + ToString(i);
+    std::string key = "k" + std::to_string(i);
     db->Delete(write_options, key);
   }
 
   HistogramImpl hist_get;
   HistogramImpl hist_get_time;
   for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
-    std::string key = "k" + ToString(i);
+    std::string key = "k" + std::to_string(i);
     std::string value;
 
     perf_context.Reset();
@@ -94,32 +88,27 @@ TEST(PerfContextTest, SeekIntoDeletion) {
     hist_get_time.Add(elapsed_nanos);
   }
 
-  std::cout << "Get user key comparison: \n" << hist_get.ToString()
+  std::cout << "Get uesr key comparison: \n" << hist_get.ToString()
             << "Get time: \n" << hist_get_time.ToString();
 
-  {
-    HistogramImpl hist_seek_to_first;
-    std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
+  HistogramImpl hist_seek_to_first;
+  std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
 
-    perf_context.Reset();
-    StopWatchNano timer(Env::Default(), true);
-    iter->SeekToFirst();
-    hist_seek_to_first.Add(perf_context.user_key_comparison_count);
-    auto elapsed_nanos = timer.ElapsedNanos();
+  perf_context.Reset();
+  StopWatchNano timer(Env::Default(), true);
+  iter->SeekToFirst();
+  hist_seek_to_first.Add(perf_context.user_key_comparison_count);
+  auto elapsed_nanos = timer.ElapsedNanos();
 
-    std::cout << "SeekToFirst uesr key comparison: \n"
-              << hist_seek_to_first.ToString()
-              << "ikey skipped: " << perf_context.internal_key_skipped_count
-              << "\n"
-              << "idelete skipped: "
-              << perf_context.internal_delete_skipped_count << "\n"
-              << "elapsed: " << elapsed_nanos << "\n";
-  }
+  std::cout << "SeekToFirst uesr key comparison: \n" << hist_seek_to_first.ToString()
+            << "ikey skipped: " << perf_context.internal_key_skipped_count << "\n"
+            << "idelete skipped: " << perf_context.internal_delete_skipped_count << "\n"
+            << "elapsed: " << elapsed_nanos << "\n";
 
   HistogramImpl hist_seek;
   for (int i = 0; i < FLAGS_total_keys; ++i) {
     std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
-    std::string key = "k" + ToString(i);
+    std::string key = "k" + std::to_string(i);
 
     perf_context.Reset();
     StopWatchNano timer(Env::Default(), true);
@@ -232,9 +221,10 @@ void ProfileQueries(bool enabled_time = false) {
       db->Flush(fo);
       continue;
     }
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
+    std::vector<Slice> keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -246,10 +236,10 @@ void ProfileQueries(bool enabled_time = false) {
   }
 
   for (const int i : keys) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
-    std::vector<Slice> multiget_keys = {Slice(key)};
+    std::vector<Slice> keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -262,7 +252,7 @@ void ProfileQueries(bool enabled_time = false) {
     hist_get.Add(perf_context.user_key_comparison_count);
 
     perf_context.Reset();
-    db->MultiGet(read_options, multiget_keys, &values);
+    db->MultiGet(read_options, keys, &values);
     hist_mget_snapshot.Add(perf_context.get_snapshot_time);
     hist_mget_memtable.Add(perf_context.get_from_memtable_time);
     hist_mget_files.Add(perf_context.get_from_output_files_time);
@@ -336,10 +326,10 @@ void ProfileQueries(bool enabled_time = false) {
   hist_mget_num_memtable_checked.Clear();
 
   for (const int i : keys) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
-    std::vector<Slice> multiget_keys = {Slice(key)};
+    std::vector<Slice> keys = {Slice(key)};
     std::vector<std::string> values;
 
     perf_context.Reset();
@@ -352,7 +342,7 @@ void ProfileQueries(bool enabled_time = false) {
     hist_get.Add(perf_context.user_key_comparison_count);
 
     perf_context.Reset();
-    db->MultiGet(read_options, multiget_keys, &values);
+    db->MultiGet(read_options, keys, &values);
     hist_mget_snapshot.Add(perf_context.get_snapshot_time);
     hist_mget_memtable.Add(perf_context.get_from_memtable_time);
     hist_mget_files.Add(perf_context.get_from_output_files_time);
@@ -452,8 +442,8 @@ TEST(PerfContextTest, SeekKeyComparison) {
   SetPerfLevel(kEnableTime);
   StopWatchNano timer(Env::Default());
   for (const int i : keys) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
     perf_context.Reset();
     timer.Start();
@@ -472,8 +462,8 @@ TEST(PerfContextTest, SeekKeyComparison) {
   HistogramImpl hist_next;
 
   for (int i = 0; i < FLAGS_total_keys; ++i) {
-    std::string key = "k" + ToString(i);
-    std::string value = "v" + ToString(i);
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
 
     std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
     perf_context.Reset();

db/plain_table_db_test.cc

@@ -158,7 +158,7 @@ class PlainTableDBTest {
   // Return spread of files per level
   std::string FilesPerLevel() {
     std::string result;
-    size_t last_non_zero_offset = 0;
+    int last_non_zero_offset = 0;
     for (int level = 0; level < db_->NumberLevels(); level++) {
       int f = NumTableFilesAtLevel(level);
       char buf[100];
@@ -628,7 +628,7 @@ TEST(PlainTableDBTest, IteratorLargeKeys) {
   };
 
   for (size_t i = 0; i < 7; i++) {
-    ASSERT_OK(Put(key_list[i], ToString(i)));
+    ASSERT_OK(Put(key_list[i], std::to_string(i)));
   }
 
   dbfull()->TEST_FlushMemTable();
@@ -639,7 +639,7 @@ TEST(PlainTableDBTest, IteratorLargeKeys) {
   for (size_t i = 0; i < 7; i++) {
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(key_list[i], iter->key().ToString());
-    ASSERT_EQ(ToString(i), iter->value().ToString());
+    ASSERT_EQ(std::to_string(i), iter->value().ToString());
     iter->Next();
   }
 
@@ -676,7 +676,7 @@ TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
       MakeLongKeyWithPrefix(26, '6')};
 
   for (size_t i = 0; i < 7; i++) {
-    ASSERT_OK(Put(key_list[i], ToString(i)));
+    ASSERT_OK(Put(key_list[i], std::to_string(i)));
   }
 
   dbfull()->TEST_FlushMemTable();
@@ -687,7 +687,7 @@ TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
   for (size_t i = 0; i < 7; i++) {
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(key_list[i], iter->key().ToString());
-    ASSERT_EQ(ToString(i), iter->value().ToString());
+    ASSERT_EQ(std::to_string(i), iter->value().ToString());
     iter->Next();
   }
 

db/prefix_test.cc

@@ -29,14 +29,14 @@ using GFLAGS::ParseCommandLineFlags;
 
 DEFINE_bool(trigger_deadlock, false,
             "issue delete in range scan to trigger PrefixHashMap deadlock");
-DEFINE_int32(bucket_count, 100000, "number of buckets");
+DEFINE_uint64(bucket_count, 100000, "number of buckets");
 DEFINE_uint64(num_locks, 10001, "number of locks");
 DEFINE_bool(random_prefix, false, "randomize prefix");
 DEFINE_uint64(total_prefixes, 100000, "total number of prefixes");
 DEFINE_uint64(items_per_prefix, 1, "total number of values per prefix");
 DEFINE_int64(write_buffer_size, 33554432, "");
-DEFINE_int32(max_write_buffer_number, 2, "");
-DEFINE_int32(min_write_buffer_number_to_merge, 1, "");
+DEFINE_int64(max_write_buffer_number, 2, "");
+DEFINE_int64(min_write_buffer_number_to_merge, 1, "");
 DEFINE_int32(skiplist_height, 4, "");
 DEFINE_int32(memtable_prefix_bloom_bits, 10000000, "");
 DEFINE_int32(memtable_prefix_bloom_probes, 10, "");
@@ -442,7 +442,7 @@ TEST(PrefixTest, DynamicPrefixIterator) {
     for (auto prefix : prefixes) {
       TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
       Slice key = TestKeyToSlice(test_key);
-      std::string value = "v" + ToString(0);
+      std::string value = "v" + std::to_string(0);
 
       perf_context.Reset();
       StopWatchNano timer(Env::Default(), true);

db/repair.cc

@@ -45,7 +45,6 @@
 #include "db/memtable.h"
 #include "db/table_cache.h"
 #include "db/version_edit.h"
-#include "db/writebuffer.h"
 #include "db/write_batch_internal.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/db.h"
@@ -221,10 +220,9 @@ class Repairer {
     std::string scratch;
     Slice record;
     WriteBatch batch;
-    WriteBuffer wb(options_.db_write_buffer_size);
     MemTable* mem = new MemTable(icmp_, ioptions_,
-                                 MutableCFOptions(options_, ioptions_), &wb);
-    auto cf_mems_default = new ColumnFamilyMemTablesDefault(mem);
+                                 MutableCFOptions(options_, ioptions_));
+    auto cf_mems_default = new ColumnFamilyMemTablesDefault(mem, &options_);
     mem->Ref();
     int counter = 0;
     while (reader.ReadRecord(&record, &scratch)) {

db/skiplist.h

@@ -34,8 +34,9 @@
 #include <assert.h>
 #include <atomic>
 #include <stdlib.h>
+#include "util/arena.h"
 #include "port/port.h"
-#include "util/allocator.h"
+#include "util/arena.h"
 #include "util/random.h"
 
 namespace rocksdb {
@@ -47,9 +48,9 @@ class SkipList {
 
  public:
   // Create a new SkipList object that will use "cmp" for comparing keys,
-  // and will allocate memory using "*allocator".  Objects allocated in the
-  // allocator must remain allocated for the lifetime of the skiplist object.
-  explicit SkipList(Comparator cmp, Allocator* allocator,
+  // and will allocate memory using "*arena".  Objects allocated in the arena
+  // must remain allocated for the lifetime of the skiplist object.
+  explicit SkipList(Comparator cmp, Arena* arena,
                     int32_t max_height = 12, int32_t branching_factor = 4);
 
   // Insert key into the list.
@@ -109,7 +110,7 @@ class SkipList {
 
   // Immutable after construction
   Comparator const compare_;
-  Allocator* const allocator_;    // Allocator used for allocations of nodes
+  Arena* const arena_;    // Arena used for allocations of nodes
 
   Node* const head_;
 
@@ -195,7 +196,7 @@ struct SkipList<Key, Comparator>::Node {
 template<typename Key, class Comparator>
 typename SkipList<Key, Comparator>::Node*
 SkipList<Key, Comparator>::NewNode(const Key& key, int height) {
-  char* mem = allocator_->AllocateAligned(
+  char* mem = arena_->AllocateAligned(
       sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1));
   return new (mem) Node(key);
 }
@@ -355,24 +356,23 @@ typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast()
 }
 
 template<typename Key, class Comparator>
-SkipList<Key, Comparator>::SkipList(const Comparator cmp, Allocator* allocator,
+SkipList<Key, Comparator>::SkipList(const Comparator cmp, Arena* arena,
                                    int32_t max_height,
                                    int32_t branching_factor)
     : kMaxHeight_(max_height),
       kBranching_(branching_factor),
       compare_(cmp),
-      allocator_(allocator),
+      arena_(arena),
       head_(NewNode(0 /* any key will do */, max_height)),
       max_height_(1),
       prev_height_(1),
       rnd_(0xdeadbeef) {
   assert(kMaxHeight_ > 0);
   assert(kBranching_ > 0);
-  // Allocate the prev_ Node* array, directly from the passed-in allocator.
+  // Allocate the prev_ Node* array, directly from the passed-in arena.
   // prev_ does not need to be freed, as its life cycle is tied up with
-  // the allocator as a whole.
-  prev_ = reinterpret_cast<Node**>(
-            allocator_->AllocateAligned(sizeof(Node*) * kMaxHeight_));
+  // the arena as a whole.
+  prev_ = (Node**) arena_->AllocateAligned(sizeof(Node*) * kMaxHeight_);
   for (int i = 0; i < kMaxHeight_; i++) {
     head_->SetNext(i, nullptr);
     prev_[i] = head_;

db/skiplist_test.cc

@@ -253,10 +253,11 @@ class ConcurrentTest {
         // Note that generation 0 is never inserted, so it is ok if
         // <*,0,*> is missing.
         ASSERT_TRUE((gen(pos) == 0U) ||
-                    (gen(pos) > static_cast<uint64_t>(initial_state.Get(
-                                    static_cast<int>(key(pos))))))
-            << "key: " << key(pos) << "; gen: " << gen(pos)
-            << "; initgen: " << initial_state.Get(static_cast<int>(key(pos)));
+                    (gen(pos) > (uint64_t)initial_state.Get(key(pos)))
+                    ) << "key: " << key(pos)
+                      << "; gen: " << gen(pos)
+                      << "; initgen: "
+                      << initial_state.Get(key(pos));
 
         // Advance to next key in the valid key space
         if (key(pos) < key(current)) {

db/snapshot.h

@@ -28,8 +28,6 @@ class SnapshotImpl : public Snapshot {
   SnapshotImpl* next_;
 
   SnapshotList* list_;                 // just for sanity checks
-
-  int64_t unix_time_;
 };
 
 class SnapshotList {
@@ -38,23 +36,20 @@ class SnapshotList {
     list_.prev_ = &list_;
     list_.next_ = &list_;
     list_.number_ = 0xFFFFFFFFL;      // placeholder marker, for debugging
-    count_ = 0;
   }
 
   bool empty() const { return list_.next_ == &list_; }
   SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
   SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
 
-  const SnapshotImpl* New(SequenceNumber seq, uint64_t unix_time) {
+  const SnapshotImpl* New(SequenceNumber seq) {
     SnapshotImpl* s = new SnapshotImpl;
     s->number_ = seq;
-    s->unix_time_ = unix_time;
     s->list_ = this;
     s->next_ = &list_;
     s->prev_ = list_.prev_;
     s->prev_->next_ = s;
     s->next_->prev_ = s;
-    count_++;
     return s;
   }
 
@@ -62,7 +57,6 @@ class SnapshotList {
     assert(s->list_ == this);
     s->prev_->next_ = s->next_;
     s->next_->prev_ = s->prev_;
-    count_--;
     delete s;
   }
 
@@ -84,20 +78,9 @@ class SnapshotList {
     return newest()->number_;
   }
 
-  int64_t GetOldestSnapshotTime() const {
-    if (empty()) {
-      return 0;
-    } else {
-      return oldest()->unix_time_;
-    }
-  }
-
-  uint64_t count() const { return count_; }
-
  private:
   // Dummy head of doubly-linked list of snapshots
   SnapshotImpl list_;
-  uint64_t count_;
 };
 
 }  // namespace rocksdb

db/table_properties_collector.cc

@@ -7,7 +7,6 @@
 
 #include "db/dbformat.h"
 #include "util/coding.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
@@ -41,7 +40,7 @@ Status InternalKeyPropertiesCollector::Finish(
 UserCollectedProperties
 InternalKeyPropertiesCollector::GetReadableProperties() const {
   return {
-    { "kDeletedKeys", ToString(deleted_keys_) }
+    { "kDeletedKeys", std::to_string(deleted_keys_) }
   };
 }
 

db/version_builder.cc

@@ -284,7 +284,7 @@ class VersionBuilder::Rep {
     if (levels_[level].deleted_files.count(f->fd.GetNumber()) > 0) {
       // File is deleted: do nothing
     } else {
-      vstorage->AddFile(level, f);
+      vstorage->MaybeAddFile(level, f);
     }
   }
 };

db/version_builder_test.cc

@@ -14,31 +14,31 @@ namespace rocksdb {
 
 class VersionBuilderTest {
  public:
-  const Comparator* ucmp_;
-  InternalKeyComparator icmp_;
-  Options options_;
-  ImmutableCFOptions ioptions_;
-  MutableCFOptions mutable_cf_options_;
-  VersionStorageInfo vstorage_;
-  uint32_t file_num_;
-  CompactionOptionsFIFO fifo_options_;
-  std::vector<uint64_t> size_being_compacted_;
+  const Comparator* ucmp;
+  InternalKeyComparator icmp;
+  Options options;
+  ImmutableCFOptions ioptions;
+  MutableCFOptions mutable_cf_options;
+  VersionStorageInfo vstorage;
+  uint32_t file_num;
+  CompactionOptionsFIFO fifo_options;
+  std::vector<uint64_t> size_being_compacted;
 
   VersionBuilderTest()
-      : ucmp_(BytewiseComparator()),
-        icmp_(ucmp_),
-        ioptions_(options_),
-        mutable_cf_options_(options_, ioptions_),
-        vstorage_(&icmp_, ucmp_, options_.num_levels, kCompactionStyleLevel,
+      : ucmp(BytewiseComparator()),
+        icmp(ucmp),
+        ioptions(options),
+        mutable_cf_options(options, ioptions),
+        vstorage(&icmp, ucmp, options.num_levels, kCompactionStyleLevel,
                  nullptr),
-        file_num_(1) {
-    mutable_cf_options_.RefreshDerivedOptions(ioptions_);
-    size_being_compacted_.resize(options_.num_levels);
+        file_num(1) {
+    mutable_cf_options.RefreshDerivedOptions(ioptions);
+    size_being_compacted.resize(options.num_levels);
   }
 
   ~VersionBuilderTest() {
-    for (int i = 0; i < vstorage_.num_levels(); i++) {
-      for (auto* f : vstorage_.LevelFiles(i)) {
+    for (int i = 0; i < vstorage.num_levels(); i++) {
+      for (auto* f : vstorage.LevelFiles(i)) {
         if (--f->refs == 0) {
           delete f;
         }
@@ -54,33 +54,25 @@ class VersionBuilderTest {
   void Add(int level, uint32_t file_number, const char* smallest,
            const char* largest, uint64_t file_size = 0, uint32_t path_id = 0,
            SequenceNumber smallest_seq = 100,
-           SequenceNumber largest_seq = 100,
-           uint64_t num_entries = 0, uint64_t num_deletions = 0,
-           bool sampled = false) {
-    assert(level < vstorage_.num_levels());
+           SequenceNumber largest_seq = 100) {
+    assert(level < vstorage.num_levels());
     FileMetaData* f = new FileMetaData;
     f->fd = FileDescriptor(file_number, path_id, file_size);
     f->smallest = GetInternalKey(smallest, smallest_seq);
     f->largest = GetInternalKey(largest, largest_seq);
     f->compensated_file_size = file_size;
     f->refs = 0;
-    f->num_entries = num_entries;
-    f->num_deletions = num_deletions;
-    vstorage_.AddFile(level, f);
-    if (sampled) {
-      f->init_stats_from_file = true;
-      vstorage_.UpdateAccumulatedStats(f);
-    }
+    vstorage.MaybeAddFile(level, f);
   }
 
   void UpdateVersionStorageInfo() {
-    vstorage_.ComputeCompactionScore(mutable_cf_options_, fifo_options_,
-                                    size_being_compacted_);
-    vstorage_.UpdateFilesBySize();
-    vstorage_.UpdateNumNonEmptyLevels();
-    vstorage_.GenerateFileIndexer();
-    vstorage_.GenerateLevelFilesBrief();
-    vstorage_.SetFinalized();
+    vstorage.ComputeCompactionScore(mutable_cf_options, fifo_options,
+                                    size_being_compacted);
+    vstorage.UpdateFilesBySize();
+    vstorage.UpdateNumNonEmptyLevels();
+    vstorage.GenerateFileIndexer();
+    vstorage.GenerateLevelFilesBrief();
+    vstorage.SetFinalized();
   }
 };
 
@@ -107,9 +99,9 @@ TEST(VersionBuilderTest, ApplyAndSaveTo) {
 
   EnvOptions env_options;
 
-  VersionBuilder version_builder(env_options, nullptr, &vstorage_);
+  VersionBuilder version_builder(env_options, nullptr, &vstorage);
 
-  VersionStorageInfo new_vstorage(&icmp_, ucmp_, options_.num_levels,
+  VersionStorageInfo new_vstorage(&icmp, ucmp, options.num_levels,
                                   kCompactionStyleLevel, nullptr);
   version_builder.Apply(&version_edit);
   version_builder.SaveTo(&new_vstorage);
@@ -126,28 +118,6 @@ TEST(VersionBuilderTest, ApplyAndSaveTo) {
   }
 }
 
-TEST(VersionBuilderTest, EstimatedActiveKeys) {
-  const uint32_t kTotalSamples = 20;
-  const uint32_t kNumLevels = 5;
-  const uint32_t kFilesPerLevel = 8;
-  const uint32_t kNumFiles = kNumLevels * kFilesPerLevel;
-  const uint32_t kEntriesPerFile = 1000;
-  const uint32_t kDeletionsPerFile = 100;
-  for (uint32_t i = 0; i < kNumFiles; ++i) {
-    Add(static_cast<int>(i / kFilesPerLevel), i + 1,
-        ToString((i + 100) * 1000).c_str(),
-        ToString((i + 100) * 1000 + 999).c_str(),
-        100U,  0, 100, 100,
-        kEntriesPerFile, kDeletionsPerFile,
-        (i < kTotalSamples));
-  }
-  // minus 2X for the number of deletion entries because:
-  // 1x for deletion entry does not count as a data entry.
-  // 1x for each deletion entry will actually remove one data entry.
-  ASSERT_EQ(vstorage_.GetEstimatedActiveKeys(),
-            (kEntriesPerFile - 2 * kDeletionsPerFile) * kNumFiles);
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }

db/version_edit.h

@@ -160,7 +160,7 @@ class VersionEdit {
   // Add the specified file at the specified number.
   // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
   // REQUIRES: "smallest" and "largest" are smallest and largest keys in file
-  void AddFile(int level, uint64_t file, uint32_t file_path_id,
+  void AddFile(int level, uint64_t file, uint64_t file_path_id,
                uint64_t file_size, const InternalKey& smallest,
                const InternalKey& largest, const SequenceNumber& smallest_seqno,
                const SequenceNumber& largest_seqno) {
@@ -180,7 +180,9 @@ class VersionEdit {
   }
 
   // Number of edits
-  size_t NumEntries() { return new_files_.size() + deleted_files_.size(); }
+  int NumEntries() {
+    return new_files_.size() + deleted_files_.size();
+  }
 
   bool IsColumnFamilyManipulation() {
     return is_column_family_add_ || is_column_family_drop_;

db/version_edit_test.cc

@@ -26,12 +26,11 @@ class VersionEditTest { };
 
 TEST(VersionEditTest, EncodeDecode) {
   static const uint64_t kBig = 1ull << 50;
-  static const uint32_t kBig32Bit = 1ull << 30;
 
   VersionEdit edit;
   for (int i = 0; i < 4; i++) {
     TestEncodeDecode(edit);
-    edit.AddFile(3, kBig + 300 + i, kBig32Bit + 400 + i, 0,
+    edit.AddFile(3, kBig + 300 + i, kBig + 400 + i, 0,
                  InternalKey("foo", kBig + 500 + i, kTypeValue),
                  InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
                  kBig + 500 + i, kBig + 600 + i);

db/version_set.cc

@@ -14,14 +14,13 @@
 #endif
 
 #include <inttypes.h>
-#include <stdio.h>
 #include <algorithm>
 #include <map>
 #include <set>
 #include <climits>
 #include <unordered_map>
 #include <vector>
-#include <string>
+#include <stdio.h>
 
 #include "db/filename.h"
 #include "db/log_reader.h"
@@ -31,7 +30,6 @@
 #include "db/table_cache.h"
 #include "db/compaction.h"
 #include "db/version_builder.h"
-#include "db/writebuffer.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
 #include "table/table_reader.h"
@@ -202,8 +200,8 @@ class FilePicker {
  private:
   unsigned int num_levels_;
   unsigned int curr_level_;
-  int32_t search_left_bound_;
-  int32_t search_right_bound_;
+  int search_left_bound_;
+  int search_right_bound_;
 #ifndef NDEBUG
   std::vector<FileMetaData*>* files_;
 #endif
@@ -259,13 +257,11 @@ class FilePicker {
           start_index = search_left_bound_;
         } else if (search_left_bound_ < search_right_bound_) {
           if (search_right_bound_ == FileIndexer::kLevelMaxIndex) {
-            search_right_bound_ =
-                static_cast<int32_t>(curr_file_level_->num_files) - 1;
+            search_right_bound_ = curr_file_level_->num_files - 1;
           }
-          start_index =
-              FindFileInRange(*internal_comparator_, *curr_file_level_, ikey_,
-                              static_cast<uint32_t>(search_left_bound_),
-                              static_cast<uint32_t>(search_right_bound_));
+          start_index = FindFileInRange(*internal_comparator_,
+              *curr_file_level_, ikey_,
+              search_left_bound_, search_right_bound_);
         } else {
           // search_left_bound > search_right_bound, key does not exist in
           // this level. Since no comparision is done in this level, it will
@@ -318,8 +314,7 @@ Version::~Version() {
 int FindFile(const InternalKeyComparator& icmp,
              const LevelFilesBrief& file_level,
              const Slice& key) {
-  return FindFileInRange(icmp, file_level, key, 0,
-                         static_cast<uint32_t>(file_level.num_files));
+  return FindFileInRange(icmp, file_level, key, 0, file_level.num_files);
 }
 
 void DoGenerateLevelFilesBrief(LevelFilesBrief* file_level,
@@ -416,7 +411,7 @@ class LevelFileNumIterator : public Iterator {
                        const LevelFilesBrief* flevel)
       : icmp_(icmp),
         flevel_(flevel),
-        index_(static_cast<uint32_t>(flevel->num_files)),
+        index_(flevel->num_files),
         current_value_(0, 0, 0) {  // Marks as invalid
   }
   virtual bool Valid() const {
@@ -427,9 +422,7 @@ class LevelFileNumIterator : public Iterator {
   }
   virtual void SeekToFirst() { index_ = 0; }
   virtual void SeekToLast() {
-    index_ = (flevel_->num_files == 0)
-                 ? 0
-                 : static_cast<uint32_t>(flevel_->num_files) - 1;
+    index_ = (flevel_->num_files == 0) ? 0 : flevel_->num_files - 1;
   }
   virtual void Next() {
     assert(Valid());
@@ -438,7 +431,7 @@ class LevelFileNumIterator : public Iterator {
   virtual void Prev() {
     assert(Valid());
     if (index_ == 0) {
-      index_ = static_cast<uint32_t>(flevel_->num_files);  // Marks as invalid
+      index_ = flevel_->num_files;  // Marks as invalid
     } else {
       index_--;
     }
@@ -606,49 +599,6 @@ size_t Version::GetMemoryUsageByTableReaders() {
   return total_usage;
 }
 
-void Version::GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta) {
-  assert(cf_meta);
-  assert(cfd_);
-
-  cf_meta->name = cfd_->GetName();
-  cf_meta->size = 0;
-  cf_meta->file_count = 0;
-  cf_meta->levels.clear();
-
-  auto* ioptions = cfd_->ioptions();
-  auto* vstorage = storage_info();
-
-  for (int level = 0; level < cfd_->NumberLevels(); level++) {
-    uint64_t level_size = 0;
-    cf_meta->file_count += vstorage->LevelFiles(level).size();
-    std::vector<SstFileMetaData> files;
-    for (const auto& file : vstorage->LevelFiles(level)) {
-      uint32_t path_id = file->fd.GetPathId();
-      std::string file_path;
-      if (path_id < ioptions->db_paths.size()) {
-        file_path = ioptions->db_paths[path_id].path;
-      } else {
-        assert(!ioptions->db_paths.empty());
-        file_path = ioptions->db_paths.back().path;
-      }
-      files.emplace_back(
-          MakeTableFileName("", file->fd.GetNumber()),
-          file_path,
-          file->fd.GetFileSize(),
-          file->smallest_seqno,
-          file->largest_seqno,
-          file->smallest.user_key().ToString(),
-          file->largest.user_key().ToString(),
-          file->being_compacted);
-      level_size += file->fd.GetFileSize();
-    }
-    cf_meta->levels.emplace_back(
-        level, level_size, std::move(files));
-    cf_meta->size += level_size;
-  }
-}
-
-
 uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const {
   // Estimation will be not accurate when:
   // (1) there is merge keys
@@ -659,16 +609,11 @@ uint64_t VersionStorageInfo::GetEstimatedActiveKeys() const {
     return 0;
   }
 
-  uint64_t file_count = 0;
-  for (int level = 0; level < num_levels_; ++level) {
-    file_count += files_[level].size();
-  }
-
-  if (num_samples_ < file_count) {
+  if (num_samples_ < files_->size()) {
     // casting to avoid overflowing
     return static_cast<uint64_t>(static_cast<double>(
         accumulated_num_non_deletions_ - accumulated_num_deletions_) *
-        static_cast<double>(file_count) / num_samples_);
+        files_->size() / num_samples_);
   } else {
     return accumulated_num_non_deletions_ - accumulated_num_deletions_;
   }
@@ -679,11 +624,6 @@ void Version::AddIterators(const ReadOptions& read_options,
                            MergeIteratorBuilder* merge_iter_builder) {
   assert(storage_info_.finalized_);
 
-  if (storage_info_.num_non_empty_levels() == 0) {
-    // No file in the Version.
-    return;
-  }
-
   // Merge all level zero files together since they may overlap
   for (size_t i = 0; i < storage_info_.LevelFilesBrief(0).num_files; i++) {
     const auto& file = storage_info_.LevelFilesBrief(0).files[i];
@@ -695,8 +635,8 @@ void Version::AddIterators(const ReadOptions& read_options,
   // For levels > 0, we can use a concatenating iterator that sequentially
   // walks through the non-overlapping files in the level, opening them
   // lazily.
-  for (int level = 1; level < storage_info_.num_non_empty_levels(); level++) {
-    if (storage_info_.LevelFilesBrief(level).num_files != 0) {
+  for (int level = 1; level < storage_info_.num_levels(); level++) {
+    if (storage_info_.level_files_brief_[level].num_files != 0) {
       merge_iter_builder->AddIterator(NewTwoLevelIterator(
           new LevelFileIteratorState(
               cfd_->table_cache(), read_options, soptions,
@@ -717,7 +657,7 @@ VersionStorageInfo::VersionStorageInfo(
       user_comparator_(user_comparator),
       // cfd is nullptr if Version is dummy
       num_levels_(levels),
-      num_non_empty_levels_(0),
+      num_non_empty_levels_(num_levels_),
       file_indexer_(user_comparator),
       compaction_style_(compaction_style),
       files_(new std::vector<FileMetaData*>[num_levels_]),
@@ -1052,7 +992,7 @@ bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) {
 
 } // anonymous namespace
 
-void VersionStorageInfo::AddFile(int level, FileMetaData* f) {
+void VersionStorageInfo::MaybeAddFile(int level, FileMetaData* f) {
   assert(level < num_levels());
   auto* level_files = &files_[level];
   // Must not overlap
@@ -1126,6 +1066,22 @@ bool Version::Unref() {
   return false;
 }
 
+bool VersionStorageInfo::NeedsCompaction() const {
+  // In universal compaction case, this check doesn't really
+  // check the compaction condition, but checks num of files threshold
+  // only. We are not going to miss any compaction opportunity
+  // but it's likely that more compactions are scheduled but
+  // ending up with nothing to do. We can improve it later.
+  // TODO(sdong): improve this function to be accurate for universal
+  //              compactions.
+  for (int i = 0; i <= MaxInputLevel(); i++) {
+    if (compaction_score_[i] >= 1) {
+      return true;
+    }
+  }
+  return false;
+}
+
 bool VersionStorageInfo::OverlapInLevel(int level,
                                         const Slice* smallest_user_key,
                                         const Slice* largest_user_key) {
@@ -1213,7 +1169,7 @@ void VersionStorageInfo::GetOverlappingInputs(
           i = 0;
         }
       } else if (file_index) {
-        *file_index = static_cast<int>(i) - 1;
+        *file_index = i-1;
       }
     }
   }
@@ -1229,7 +1185,7 @@ void VersionStorageInfo::GetOverlappingInputsBinarySearch(
   assert(level > 0);
   int min = 0;
   int mid = 0;
-  int max = static_cast<int>(files_[level].size()) - 1;
+  int max = files_[level].size() -1;
   bool foundOverlap = false;
   const Comparator* user_cmp = user_comparator_;
 
@@ -1491,11 +1447,9 @@ struct VersionSet::ManifestWriter {
 
 VersionSet::VersionSet(const std::string& dbname, const DBOptions* db_options,
                        const EnvOptions& storage_options, Cache* table_cache,
-                       WriteBuffer* write_buffer,
                        WriteController* write_controller)
     : column_family_set_(new ColumnFamilySet(
-          dbname, db_options, storage_options, table_cache,
-          write_buffer, write_controller)),
+          dbname, db_options, storage_options, table_cache, write_controller)),
       env_(db_options->env),
       dbname_(dbname),
       db_options_(db_options),
@@ -1615,7 +1569,7 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
   if (!descriptor_log_ ||
       manifest_file_size_ > db_options_->max_manifest_file_size) {
     pending_manifest_file_number_ = NewFileNumber();
-    batch_edits.back()->SetNextFile(next_file_number_.load());
+    batch_edits.back()->SetNextFile(next_file_number_);
     new_descriptor_log = true;
   } else {
     pending_manifest_file_number_ = manifest_file_number_;
@@ -1816,7 +1770,7 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
 
 void VersionSet::LogAndApplyCFHelper(VersionEdit* edit) {
   assert(edit->IsColumnFamilyManipulation());
-  edit->SetNextFile(next_file_number_.load());
+  edit->SetNextFile(next_file_number_);
   edit->SetLastSequence(last_sequence_);
   if (edit->is_column_family_drop_) {
     // if we drop column family, we have to make sure to save max column family,
@@ -1833,13 +1787,13 @@ void VersionSet::LogAndApplyHelper(ColumnFamilyData* cfd,
 
   if (edit->has_log_number_) {
     assert(edit->log_number_ >= cfd->GetLogNumber());
-    assert(edit->log_number_ < next_file_number_.load());
+    assert(edit->log_number_ < next_file_number_);
   }
 
   if (!edit->has_prev_log_number_) {
     edit->SetPrevLogNumber(prev_log_number_);
   }
-  edit->SetNextFile(next_file_number_.load());
+  edit->SetNextFile(next_file_number_);
   edit->SetLastSequence(last_sequence_);
 
   builder->Apply(edit);
@@ -2066,8 +2020,8 @@ Status VersionSet::Recover(
 
     column_family_set_->UpdateMaxColumnFamily(max_column_family);
 
-    MarkFileNumberUsedDuringRecovery(previous_log_number);
-    MarkFileNumberUsedDuringRecovery(log_number);
+    MarkFileNumberUsed(previous_log_number);
+    MarkFileNumberUsed(log_number);
   }
 
   // there were some column families in the MANIFEST that weren't specified
@@ -2107,7 +2061,7 @@ Status VersionSet::Recover(
     }
 
     manifest_file_size_ = current_manifest_file_size;
-    next_file_number_.store(next_file + 1);
+    next_file_number_ = next_file + 1;
     last_sequence_ = last_sequence;
     prev_log_number_ = previous_log_number;
 
@@ -2118,7 +2072,7 @@ Status VersionSet::Recover(
         "prev_log_number is %lu,"
         "max_column_family is %u\n",
         manifest_filename.c_str(), (unsigned long)manifest_file_number_,
-        (unsigned long)next_file_number_.load(), (unsigned long)last_sequence_,
+        (unsigned long)next_file_number_, (unsigned long)last_sequence_,
         (unsigned long)log_number, (unsigned long)prev_log_number_,
         column_family_set_->GetMaxColumnFamily());
 
@@ -2218,8 +2172,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
       options->max_open_files - 10, options->table_cache_numshardbits,
       options->table_cache_remove_scan_count_limit));
   WriteController wc;
-  WriteBuffer wb(options->db_write_buffer_size);
-  VersionSet versions(dbname, options, env_options, tc.get(), &wb, &wc);
+  VersionSet versions(dbname, options, env_options, tc.get(), &wc);
   Status status;
 
   std::vector<ColumnFamilyDescriptor> dummy;
@@ -2440,6 +2393,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
           v->storage_info()->num_levels() - 1);
       cfd->compaction_picker()->SizeBeingCompacted(size_being_compacted);
       v->PrepareApply(*cfd->GetLatestMutableCFOptions(), size_being_compacted);
+      delete builder;
 
       printf("--------------- Column family \"%s\"  (ID %u) --------------\n",
              cfd->GetName().c_str(), (unsigned int)cfd->GetID());
@@ -2454,19 +2408,14 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
       delete v;
     }
 
-    // Free builders
-    for (auto& builder : builders) {
-      delete builder.second;
-    }
-
-    next_file_number_.store(next_file + 1);
+    next_file_number_ = next_file + 1;
     last_sequence_ = last_sequence;
     prev_log_number_ = previous_log_number;
 
     printf(
         "next_file_number %lu last_sequence "
         "%lu  prev_log_number %lu max_column_family %u\n",
-        (unsigned long)next_file_number_.load(), (unsigned long)last_sequence,
+        (unsigned long)next_file_number_, (unsigned long)last_sequence,
         (unsigned long)previous_log_number,
         column_family_set_->GetMaxColumnFamily());
   }
@@ -2475,11 +2424,9 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
 }
 #endif  // ROCKSDB_LITE
 
-void VersionSet::MarkFileNumberUsedDuringRecovery(uint64_t number) {
-  // only called during recovery which is single threaded, so this works because
-  // there can't be concurrent calls
-  if (next_file_number_.load(std::memory_order_relaxed) <= number) {
-    next_file_number_.store(number + 1, std::memory_order_relaxed);
+void VersionSet::MarkFileNumberUsed(uint64_t number) {
+  if (next_file_number_ <= number) {
+    next_file_number_ = number + 1;
   }
 }
 
@@ -2627,7 +2574,7 @@ void VersionSet::AddLiveFiles(std::vector<FileDescriptor>* live_list) {
   }
 
   // just one time extension to the right size
-  live_list->reserve(live_list->size() + static_cast<size_t>(total_files));
+  live_list->reserve(live_list->size() + total_files);
 
   for (auto cfd : *column_family_set_) {
     Version* dummy_versions = cfd->dummy_versions();
@@ -2647,18 +2594,18 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
   auto cfd = c->column_family_data();
   ReadOptions read_options;
   read_options.verify_checksums =
-    c->mutable_cf_options()->verify_checksums_in_compaction;
+    cfd->options()->verify_checksums_in_compaction;
   read_options.fill_cache = false;
 
   // Level-0 files have to be merged together.  For other levels,
   // we will make a concatenating iterator per level.
   // TODO(opt): use concatenating iterator for level-0 if there is no overlap
-  const size_t space = (c->level() == 0 ? c->input_levels(0)->num_files +
-                                              c->num_input_levels() - 1
-                                        : c->num_input_levels());
-  Iterator** list = new Iterator* [space];
-  size_t num = 0;
-  for (size_t which = 0; which < c->num_input_levels(); which++) {
+  const int space = (c->level() == 0 ?
+      c->input_levels(0)->num_files + c->num_input_levels() - 1:
+      c->num_input_levels());
+  Iterator** list = new Iterator*[space];
+  int num = 0;
+  for (int which = 0; which < c->num_input_levels(); which++) {
     if (c->input_levels(which)->num_files != 0) {
       if (c->level(which) == 0) {
         const LevelFilesBrief* flevel = c->input_levels(which);
@@ -2680,9 +2627,8 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
     }
   }
   assert(num <= space);
-  Iterator* result =
-      NewMergingIterator(&c->column_family_data()->internal_comparator(), list,
-                         static_cast<int>(num));
+  Iterator* result = NewMergingIterator(
+      &c->column_family_data()->internal_comparator(), list, num);
   delete[] list;
   return result;
 }
@@ -2699,22 +2645,41 @@ bool VersionSet::VerifyCompactionFileConsistency(Compaction* c) {
         c->column_family_data()->GetName().c_str());
   }
 
-  for (size_t input = 0; input < c->num_input_levels(); ++input) {
-    int level = c->level(input);
-    for (size_t i = 0; i < c->num_input_files(input); ++i) {
-      uint64_t number = c->input(input, i)->fd.GetNumber();
-      bool found = false;
-      for (unsigned int j = 0; j < vstorage->files_[level].size(); j++) {
-        FileMetaData* f = vstorage->files_[level][j];
-        if (f->fd.GetNumber() == number) {
-          found = true;
-          break;
-        }
+  // verify files in level
+  int level = c->level();
+  for (int i = 0; i < c->num_input_files(0); i++) {
+    uint64_t number = c->input(0, i)->fd.GetNumber();
+
+    // look for this file in the current version
+    bool found = false;
+    for (unsigned int j = 0; j < vstorage->files_[level].size(); j++) {
+      FileMetaData* f = vstorage->files_[level][j];
+      if (f->fd.GetNumber() == number) {
+        found = true;
+        break;
       }
-      if (!found) {
-        return false;  // input files non existent in current version
+    }
+    if (!found) {
+      return false; // input files non existant in current version
+    }
+  }
+  // verify level+1 files
+  level++;
+  for (int i = 0; i < c->num_input_files(1); i++) {
+    uint64_t number = c->input(1, i)->fd.GetNumber();
+
+    // look for this file in the current version
+    bool found = false;
+    for (unsigned int j = 0; j < vstorage->files_[level].size(); j++) {
+      FileMetaData* f = vstorage->files_[level][j];
+      if (f->fd.GetNumber() == number) {
+        found = true;
+        break;
       }
     }
+    if (!found) {
+      return false; // input files non existant in current version
+    }
   }
 #endif
   return true;     // everything good

db/version_set.h

@@ -50,7 +50,6 @@ class LookupKey;
 class MemTable;
 class Version;
 class VersionSet;
-class WriteBuffer;
 class MergeContext;
 class ColumnFamilyData;
 class ColumnFamilySet;
@@ -95,7 +94,7 @@ class VersionStorageInfo {
 
   void Reserve(int level, size_t size) { files_[level].reserve(size); }
 
-  void AddFile(int level, FileMetaData* f);
+  void MaybeAddFile(int level, FileMetaData* f);
 
   void SetFinalized() { finalized_ = true; }
 
@@ -129,6 +128,9 @@ class VersionStorageInfo {
 
   int MaxInputLevel() const;
 
+  // Returns true iff some level needs a compaction.
+  bool NeedsCompaction() const;
+
   // Returns the maxmimum compaction score for levels 1 to max
   double max_compaction_score() const { return max_compaction_score_; }
 
@@ -192,7 +194,7 @@ class VersionStorageInfo {
   // REQUIRES: This version has been saved (see VersionSet::SaveTo)
   int NumLevelFiles(int level) const {
     assert(finalized_);
-    return static_cast<int>(files_[level].size());
+    return files_[level].size();
   }
 
   // Return the combined file size of all files at the specified level.
@@ -204,7 +206,6 @@ class VersionStorageInfo {
   }
 
   const rocksdb::LevelFilesBrief& LevelFilesBrief(int level) const {
-    assert(level < static_cast<int>(level_files_brief_.size()));
     return level_files_brief_[level];
   }
 
@@ -419,8 +420,6 @@ class Version {
 
   VersionSet* version_set() { return vset_; }
 
-  void GetColumnFamilyMetaData(ColumnFamilyMetaData* cf_meta);
-
  private:
   friend class VersionSet;
 
@@ -476,7 +475,7 @@ class VersionSet {
  public:
   VersionSet(const std::string& dbname, const DBOptions* db_options,
              const EnvOptions& env_options, Cache* table_cache,
-             WriteBuffer* write_buffer, WriteController* write_controller);
+             WriteController* write_controller);
   ~VersionSet();
 
   // Apply *edit to the current version to form a new descriptor that
@@ -531,10 +530,17 @@ class VersionSet {
     return pending_manifest_file_number_;
   }
 
-  uint64_t current_next_file_number() const { return next_file_number_.load(); }
-
   // Allocate and return a new file number
-  uint64_t NewFileNumber() { return next_file_number_.fetch_add(1); }
+  uint64_t NewFileNumber() { return next_file_number_++; }
+
+  // Arrange to reuse "file_number" unless a newer file number has
+  // already been allocated.
+  // REQUIRES: "file_number" was returned by a call to NewFileNumber().
+  void ReuseLogFileNumber(uint64_t file_number) {
+    if (next_file_number_ == file_number + 1) {
+      next_file_number_ = file_number;
+    }
+  }
 
   // Return the last sequence number.
   uint64_t LastSequence() const {
@@ -548,8 +554,7 @@ class VersionSet {
   }
 
   // Mark the specified file number as used.
-  // REQUIRED: this is only called during single-threaded recovery
-  void MarkFileNumberUsedDuringRecovery(uint64_t number);
+  void MarkFileNumberUsed(uint64_t number);
 
   // Return the log file number for the log file that is currently
   // being compacted, or zero if there is no such log file.
@@ -590,7 +595,7 @@ class VersionSet {
   Status GetMetadataForFile(uint64_t number, int* filelevel,
                             FileMetaData** metadata, ColumnFamilyData** cfd);
 
-  void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata);
+  void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
 
   void GetObsoleteFiles(std::vector<FileMetaData*>* files);
 
@@ -601,7 +606,6 @@ class VersionSet {
   struct ManifestWriter;
 
   friend class Version;
-  friend class DBImpl;
 
   struct LogReporter : public log::Reader::Reporter {
     Status* status;
@@ -626,7 +630,7 @@ class VersionSet {
   Env* const env_;
   const std::string dbname_;
   const DBOptions* const db_options_;
-  std::atomic<uint64_t> next_file_number_;
+  uint64_t next_file_number_;
   uint64_t manifest_file_number_;
   uint64_t pending_manifest_file_number_;
   std::atomic<uint64_t> last_sequence_;

db/wal_manager.cc

@@ -31,7 +31,6 @@
 #include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/sync_point.h"
-#include "util/string_util.h"
 
 namespace rocksdb {
 
@@ -356,9 +355,9 @@ Status WalManager::ReadFirstRecord(const WalFileType type,
                                    SequenceNumber* sequence) {
   if (type != kAliveLogFile && type != kArchivedLogFile) {
     Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
-        "[WalManger] Unknown file type %s", ToString(type).c_str());
+        "[WalManger] Unknown file type %s", std::to_string(type).c_str());
     return Status::NotSupported(
-        "File Type Not Known " + ToString(type));
+        "File Type Not Known " + std::to_string(type));
   }
   {
     MutexLock l(&read_first_record_cache_mutex_);

db/wal_manager.h

@@ -38,9 +38,9 @@ class WalManager {
         env_(db_options.env),
         purge_wal_files_last_run_(0) {}
 
-  Status GetSortedWalFiles(VectorLogPtr& files);
+  virtual Status GetSortedWalFiles(VectorLogPtr& files);
 
-  Status GetUpdatesSince(
+  virtual Status GetUpdatesSince(
       SequenceNumber seq_number, std::unique_ptr<TransactionLogIterator>* iter,
       const TransactionLogIterator::ReadOptions& read_options,
       VersionSet* version_set);

db/wal_manager_test.cc

@@ -13,7 +13,6 @@
 #include "db/log_writer.h"
 #include "db/column_family.h"
 #include "db/version_set.h"
-#include "db/writebuffer.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
 #include "table/mock_table.h"
@@ -29,7 +28,6 @@ class WalManagerTest {
       : env_(Env::Default()),
         dbname_(test::TmpDir() + "/wal_manager_test"),
         table_cache_(NewLRUCache(50000, 16, 8)),
-        write_buffer_(db_options_.db_write_buffer_size),
         current_log_number_(0) {
     DestroyDB(dbname_, Options());
   }
@@ -42,8 +40,7 @@ class WalManagerTest {
     db_options_.wal_dir = dbname_;
 
     versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
-                                   table_cache_.get(), &write_buffer_,
-                                   &write_controller_));
+                                   table_cache_.get(), &write_controller_));
 
     wal_manager_.reset(new WalManager(db_options_, env_options_));
   }
@@ -76,7 +73,7 @@ class WalManagerTest {
     for (int i = 1; i <= num_logs; ++i) {
       RollTheLog(true);
       for (int k = 0; k < entries_per_log; ++k) {
-        Put(ToString(k), std::string(1024, 'a'));
+        Put(std::to_string(k), std::string(1024, 'a'));
       }
     }
   }
@@ -96,7 +93,6 @@ class WalManagerTest {
   EnvOptions env_options_;
   std::shared_ptr<Cache> table_cache_;
   DBOptions db_options_;
-  WriteBuffer write_buffer_;
   std::unique_ptr<VersionSet> versions_;
   std::unique_ptr<WalManager> wal_manager_;
 

db/write_batch.cc

@@ -48,6 +48,20 @@ WriteBatch::~WriteBatch() { }
 
 WriteBatch::Handler::~Handler() { }
 
+void WriteBatch::Handler::Put(const Slice& key, const Slice& value) {
+  // you need to either implement Put or PutCF
+  throw std::runtime_error("Handler::Put not implemented!");
+}
+
+void WriteBatch::Handler::Merge(const Slice& key, const Slice& value) {
+  throw std::runtime_error("Handler::Merge not implemented!");
+}
+
+void WriteBatch::Handler::Delete(const Slice& key) {
+  // you need to either implement Delete or DeleteCF
+  throw std::runtime_error("Handler::Delete not implemented!");
+}
+
 void WriteBatch::Handler::LogData(const Slice& blob) {
   // If the user has not specified something to do with blobs, then we ignore
   // them.
@@ -360,7 +374,7 @@ class MemTableInserter : public WriteBatch::Handler {
         Status s = db_->Get(ropts, cf_handle, key, &prev_value);
 
         char* prev_buffer = const_cast<char*>(prev_value.c_str());
-        uint32_t prev_size = static_cast<uint32_t>(prev_value.size());
+        uint32_t prev_size = prev_value.size();
         auto status = moptions->inplace_callback(s.ok() ? prev_buffer : nullptr,
                                                  s.ok() ? &prev_size : nullptr,
                                                  value, &merged_value);

db/write_batch_internal.h

@@ -26,14 +26,15 @@ class ColumnFamilyMemTables {
   // been processed)
   virtual uint64_t GetLogNumber() const = 0;
   virtual MemTable* GetMemTable() const = 0;
+  virtual const Options* GetOptions() const = 0;
   virtual ColumnFamilyHandle* GetColumnFamilyHandle() = 0;
   virtual void CheckMemtableFull() = 0;
 };
 
 class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
  public:
-  explicit ColumnFamilyMemTablesDefault(MemTable* mem)
-      : ok_(false), mem_(mem) {}
+  ColumnFamilyMemTablesDefault(MemTable* mem, const Options* options)
+      : ok_(false), mem_(mem), options_(options) {}
 
   bool Seek(uint32_t column_family_id) override {
     ok_ = (column_family_id == 0);
@@ -47,6 +48,11 @@ class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
     return mem_;
   }
 
+  const Options* GetOptions() const override {
+    assert(ok_);
+    return options_;
+  }
+
   ColumnFamilyHandle* GetColumnFamilyHandle() override { return nullptr; }
 
   void CheckMemtableFull() override {}
@@ -54,6 +60,7 @@ class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
  private:
   bool ok_;
   MemTable* mem_;
+  const Options* const options_;
 };
 
 // WriteBatchInternal provides static methods for manipulating a

db/write_batch_test.cc

@@ -13,7 +13,6 @@
 #include "db/memtable.h"
 #include "db/column_family.h"
 #include "db/write_batch_internal.h"
-#include "db/writebuffer.h"
 #include "rocksdb/env.h"
 #include "rocksdb/memtablerep.h"
 #include "rocksdb/utilities/write_batch_with_index.h"
@@ -29,12 +28,11 @@ static std::string PrintContents(WriteBatch* b) {
   Options options;
   options.memtable_factory = factory;
   ImmutableCFOptions ioptions(options);
-  WriteBuffer wb(options.db_write_buffer_size);
   MemTable* mem = new MemTable(cmp, ioptions,
-                               MutableCFOptions(options, ioptions), &wb);
+                               MutableCFOptions(options, ioptions));
   mem->Ref();
   std::string state;
-  ColumnFamilyMemTablesDefault cf_mems_default(mem);
+  ColumnFamilyMemTablesDefault cf_mems_default(mem, &options);
   Status s = WriteBatchInternal::InsertInto(b, &cf_mems_default);
   int count = 0;
   Arena arena;
@@ -157,7 +155,7 @@ namespace {
       if (column_family_id == 0) {
         seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
       } else {
-        seen += "PutCF(" + ToString(column_family_id) + ", " +
+        seen += "PutCF(" + std::to_string(column_family_id) + ", " +
                 key.ToString() + ", " + value.ToString() + ")";
       }
       return Status::OK();
@@ -167,7 +165,7 @@ namespace {
       if (column_family_id == 0) {
         seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
       } else {
-        seen += "MergeCF(" + ToString(column_family_id) + ", " +
+        seen += "MergeCF(" + std::to_string(column_family_id) + ", " +
                 key.ToString() + ", " + value.ToString() + ")";
       }
       return Status::OK();
@@ -179,7 +177,7 @@ namespace {
       if (column_family_id == 0) {
         seen += "Delete(" + key.ToString() + ")";
       } else {
-        seen += "DeleteCF(" + ToString(column_family_id) + ", " +
+        seen += "DeleteCF(" + std::to_string(column_family_id) + ", " +
                 key.ToString() + ")";
       }
       return Status::OK();
@@ -187,39 +185,6 @@ namespace {
   };
 }
 
-TEST(WriteBatchTest, MergeNotImplemented) {
-  WriteBatch batch;
-  batch.Merge(Slice("foo"), Slice("bar"));
-  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Merge(foo, bar)@0",
-            PrintContents(&batch));
-
-  WriteBatch::Handler handler;
-  ASSERT_OK(batch.Iterate(&handler));
-}
-
-TEST(WriteBatchTest, PutNotImplemented) {
-  WriteBatch batch;
-  batch.Put(Slice("k1"), Slice("v1"));
-  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Put(k1, v1)@0",
-            PrintContents(&batch));
-
-  WriteBatch::Handler handler;
-  ASSERT_OK(batch.Iterate(&handler));
-}
-
-TEST(WriteBatchTest, DeleteNotImplemented) {
-  WriteBatch batch;
-  batch.Delete(Slice("k2"));
-  ASSERT_EQ(1, batch.Count());
-  ASSERT_EQ("Delete(k2)@0",
-            PrintContents(&batch));
-
-  WriteBatch::Handler handler;
-  ASSERT_OK(batch.Iterate(&handler));
-}
-
 TEST(WriteBatchTest, Blob) {
   WriteBatch batch;
   batch.Put(Slice("k1"), Slice("v1"));

db/writebuffer.h

@@ -1,44 +0,0 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// WriteBuffer is for managing memory allocation for one or more MemTables.
-
-#pragma once
-
-namespace rocksdb {
-
-class WriteBuffer {
- public:
-  explicit WriteBuffer(size_t _buffer_size)
-    : buffer_size_(_buffer_size), memory_used_(0) {}
-
-  ~WriteBuffer() {}
-
-  size_t memory_usage() const { return memory_used_; }
-  size_t buffer_size() const { return buffer_size_; }
-
-  // Should only be called from write thread
-  bool ShouldFlush() const {
-    return buffer_size() > 0 && memory_usage() >= buffer_size();
-  }
-
-  // Should only be called from write thread
-  void ReserveMem(size_t mem) { memory_used_ += mem; }
-  void FreeMem(size_t mem) { memory_used_ -= mem; }
-
- private:
-  const size_t buffer_size_;
-  size_t memory_used_;
-
-  // No copying allowed
-  WriteBuffer(const WriteBuffer&);
-  void operator=(const WriteBuffer&);
-};
-
-}  // namespace rocksdb

examples/Makefile

@@ -1,8 +1,8 @@
 include ../build_config.mk
 
-.PHONY: clean
+.PHONY: main clean
 
-all: simple_example column_families_example compact_files_example
+all: simple_example column_families_example
 
 simple_example: simple_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
@@ -10,8 +10,5 @@ simple_example: simple_example.cc
 column_families_example: column_families_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
-compact_files_example: compact_files_example.cc
-	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
-
-clean: simple_example column_families_example compact_files_example
-	rm -rf ./simple_example ./column_families_example ./compact_files_example
+clean: simple_example column_families_example
+	rm -rf ./simple_example ./column_families_example

examples/compact_files_example.cc

@@ -1,175 +0,0 @@
-// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-//
-// An example code demonstrating how to use CompactFiles, EventListener,
-// and GetColumnFamilyMetaData APIs to implement custom compaction algorithm.
-
-#include <mutex>
-#include <string>
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/options.h"
-
-using namespace rocksdb;
-std::string kDBPath = "/tmp/rocksdb_compact_files_example";
-class CompactionTask;
-
-// This is an example interface of external-compaction algorithm.
-// Compaction algorithm can be implemented outside the core-RocksDB
-// code by using the pluggable compaction APIs that RocksDb provides.
-class Compactor : public EventListener {
- public:
-  // Picks and returns a compaction task given the specified DB
-  // and column family.  It is the caller's responsibility to
-  // destroy the returned CompactionTask.  Returns "nullptr"
-  // if it cannot find a proper compaction task.
-  virtual CompactionTask* PickCompaction(
-      DB* db, const std::string& cf_name) = 0;
-
-  // Schedule and run the specified compaction task in background.
-  virtual void ScheduleCompaction(CompactionTask *task) = 0;
-};
-
-// Example structure that describes a compaction task.
-struct CompactionTask {
-  CompactionTask(
-      DB* db, Compactor* compactor,
-      const std::string& column_family_name,
-      const std::vector<std::string>& input_file_names,
-      const int output_level,
-      const CompactionOptions& compact_options,
-      bool retry_on_fail)
-          : db(db),
-            compactor(compactor),
-            column_family_name(column_family_name),
-            input_file_names(input_file_names),
-            output_level(output_level),
-            compact_options(compact_options),
-            retry_on_fail(false) {}
-  DB* db;
-  Compactor* compactor;
-  const std::string& column_family_name;
-  std::vector<std::string> input_file_names;
-  int output_level;
-  CompactionOptions compact_options;
-  bool retry_on_fail;
-};
-
-// A simple compaction algorithm that always compacts everything
-// to the highest level whenever possible.
-class FullCompactor : public Compactor {
- public:
-  explicit FullCompactor(const Options options) : options_(options) {
-    compact_options_.compression = options_.compression;
-    compact_options_.output_file_size_limit =
-        options_.target_file_size_base;
-  }
-
-  // When flush happens, it determins whether to trigger compaction.
-  // If triggered_writes_stop is true, it will also set the retry
-  // flag of compaction-task to true.
-  void OnFlushCompleted(
-      DB* db, const std::string& cf_name,
-      const std::string& file_path,
-      bool triggered_writes_slowdown,
-      bool triggered_writes_stop) override {
-    CompactionTask* task = PickCompaction(db, cf_name);
-    if (task != nullptr) {
-      if (triggered_writes_stop) {
-        task->retry_on_fail = true;
-      }
-      // Schedule compaction in a different thread.
-      ScheduleCompaction(task);
-    }
-  }
-
-  // Always pick a compaction which includes all files whenever possible.
-  CompactionTask* PickCompaction(
-      DB* db, const std::string& cf_name) override {
-    ColumnFamilyMetaData cf_meta;
-    db->GetColumnFamilyMetaData(&cf_meta);
-
-    std::vector<std::string> input_file_names;
-    for (auto level : cf_meta.levels) {
-      for (auto file : level.files) {
-        if (file.being_compacted) {
-          return nullptr;
-        }
-        input_file_names.push_back(file.name);
-      }
-    }
-    return new CompactionTask(
-        db, this, cf_name, input_file_names,
-        options_.num_levels - 1, compact_options_, false);
-  }
-
-  // Schedule the specified compaction task in background.
-  void ScheduleCompaction(CompactionTask* task) override {
-    options_.env->Schedule(&FullCompactor::CompactFiles, task);
-  }
-
-  static void CompactFiles(void* arg) {
-    CompactionTask* task = reinterpret_cast<CompactionTask*>(arg);
-    assert(task);
-    assert(task->db);
-    Status s = task->db->CompactFiles(
-        task->compact_options,
-        task->input_file_names,
-        task->output_level);
-    printf("CompactFiles() finished with status %s\n", s.ToString().c_str());
-    if (!s.ok() && !s.IsIOError() && task->retry_on_fail) {
-      // If a compaction task with its retry_on_fail=true failed,
-      // try to schedule another compaction in case the reason
-      // is not an IO error.
-      CompactionTask* new_task = task->compactor->PickCompaction(
-          task->db, task->column_family_name);
-      task->compactor->ScheduleCompaction(new_task);
-    }
-    // release the task
-    delete task;
-  }
-
- private:
-  Options options_;
-  CompactionOptions compact_options_;
-};
-
-int main() {
-  Options options;
-  options.create_if_missing = true;
-  // Disable RocksDB background compaction.
-  options.compaction_style = kCompactionStyleNone;
-  // Small slowdown and stop trigger for experimental purpose.
-  options.level0_slowdown_writes_trigger = 3;
-  options.level0_stop_writes_trigger = 5;
-  options.IncreaseParallelism(5);
-  options.listeners.emplace_back(new FullCompactor(options));
-
-  DB* db = nullptr;
-  DestroyDB(kDBPath, options);
-  Status s = DB::Open(options, kDBPath, &db);
-  assert(s.ok());
-  assert(db);
-
-  // if background compaction is not working, write will stall
-  // because of options.level0_stop_writes_trigger
-  for (int i = 1000; i < 99999; ++i) {
-    db->Put(WriteOptions(), ToString(i),
-                            std::string(500, 'a' + (i % 26)));
-  }
-
-  // verify the values are still there
-  std::string value;
-  for (int i = 1000; i < 99999; ++i) {
-    db->Get(ReadOptions(), ToString(i),
-                           &value);
-    assert(value == std::string(500, 'a' + (i % 26)));
-  }
-
-  // close the db.
-  delete db;
-
-  return 0;
-}

hdfs/env_hdfs.h

@@ -93,8 +93,6 @@ class HdfsEnv : public Env {
 
   virtual Status RenameFile(const std::string& src, const std::string& target);
 
-  virtual Status LinkFile(const std::string& src, const std::string& target);
-
   virtual Status LockFile(const std::string& fname, FileLock** lock);
 
   virtual Status UnlockFile(FileLock* lock);
@@ -238,7 +236,7 @@ class HdfsEnv : public Env {
   explicit HdfsEnv(const std::string& fsname) {
     fprintf(stderr, "You have not build rocksdb with HDFS support\n");
     fprintf(stderr, "Please see hdfs/README for details\n");
-    abort();
+    throw std::exception();
   }
 
   virtual ~HdfsEnv() {
@@ -293,10 +291,6 @@ class HdfsEnv : public Env {
 
   virtual Status RenameFile(const std::string& src, const std::string& target){return notsup;}
 
-  virtual Status LinkFile(const std::string& src, const std::string& target) {
-    return notsup;
-  }
-
   virtual Status LockFile(const std::string& fname, FileLock** lock){return notsup;}
 
   virtual Status UnlockFile(FileLock* lock){return notsup;}

include/rocksdb/c.h

@@ -77,8 +77,6 @@ typedef struct rocksdb_mergeoperator_t   rocksdb_mergeoperator_t;
 typedef struct rocksdb_options_t         rocksdb_options_t;
 typedef struct rocksdb_block_based_table_options_t
     rocksdb_block_based_table_options_t;
-typedef struct rocksdb_cuckoo_table_options_t
-    rocksdb_cuckoo_table_options_t;
 typedef struct rocksdb_randomfile_t      rocksdb_randomfile_t;
 typedef struct rocksdb_readoptions_t     rocksdb_readoptions_t;
 typedef struct rocksdb_seqfile_t         rocksdb_seqfile_t;
@@ -378,25 +376,6 @@ extern void rocksdb_block_based_options_set_whole_key_filtering(
 extern void rocksdb_options_set_block_based_table_factory(
     rocksdb_options_t *opt, rocksdb_block_based_table_options_t* table_options);
 
-/* Cuckoo table options */
-
-extern rocksdb_cuckoo_table_options_t*
-    rocksdb_cuckoo_options_create();
-extern void rocksdb_cuckoo_options_destroy(
-    rocksdb_cuckoo_table_options_t* options);
-extern void rocksdb_cuckoo_options_set_hash_ratio(
-    rocksdb_cuckoo_table_options_t* options, double v);
-extern void rocksdb_cuckoo_options_set_max_search_depth(
-    rocksdb_cuckoo_table_options_t* options, uint32_t v);
-extern void rocksdb_cuckoo_options_set_cuckoo_block_size(
-    rocksdb_cuckoo_table_options_t* options, uint32_t v);
-extern void rocksdb_cuckoo_options_set_identity_as_first_hash(
-    rocksdb_cuckoo_table_options_t* options, unsigned char v);
-extern void rocksdb_cuckoo_options_set_use_module_hash(
-    rocksdb_cuckoo_table_options_t* options, unsigned char v);
-extern void rocksdb_options_set_cuckoo_table_factory(
-    rocksdb_options_t *opt, rocksdb_cuckoo_table_options_t* table_options);
-
 /* Options */
 
 extern rocksdb_options_t* rocksdb_options_create();
@@ -440,7 +419,6 @@ extern void rocksdb_options_set_info_log(rocksdb_options_t*, rocksdb_logger_t*);
 extern void rocksdb_options_set_info_log_level(rocksdb_options_t*, int);
 extern void rocksdb_options_set_write_buffer_size(rocksdb_options_t*, size_t);
 extern void rocksdb_options_set_max_open_files(rocksdb_options_t*, int);
-extern void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n);
 extern void rocksdb_options_set_compression_options(
     rocksdb_options_t*, int, int, int);
 extern void rocksdb_options_set_prefix_extractor(

include/rocksdb/comparator.h

@@ -63,7 +63,7 @@ class Comparator {
 extern const Comparator* BytewiseComparator();
 
 // Return a builtin comparator that uses reverse lexicographic byte-wise
-// ordering.
+// ordering. 
 extern const Comparator* ReverseBytewiseComparator();
 
 }  // namespace rocksdb

include/rocksdb/db.h

@@ -15,34 +15,19 @@
 #include <vector>
 #include <string>
 #include <unordered_map>
-#include "rocksdb/metadata.h"
 #include "rocksdb/version.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
 #include "rocksdb/types.h"
 #include "rocksdb/transaction_log.h"
-#include "rocksdb/listener.h"
 
 namespace rocksdb {
 
-struct Options;
-struct DBOptions;
-struct ColumnFamilyOptions;
-struct ReadOptions;
-struct WriteOptions;
-struct FlushOptions;
-struct CompactionOptions;
-struct TableProperties;
-class WriteBatch;
-class Env;
-class EventListener;
-
 using std::unique_ptr;
 
 class ColumnFamilyHandle {
  public:
   virtual ~ColumnFamilyHandle() {}
-  virtual const std::string& GetName() const = 0;
 };
 extern const std::string kDefaultColumnFamilyName;
 
@@ -59,6 +44,27 @@ struct ColumnFamilyDescriptor {
 static const int kMajorVersion = __ROCKSDB_MAJOR__;
 static const int kMinorVersion = __ROCKSDB_MINOR__;
 
+struct Options;
+struct ReadOptions;
+struct WriteOptions;
+struct FlushOptions;
+struct TableProperties;
+class WriteBatch;
+class Env;
+
+// Metadata associated with each SST file.
+struct LiveFileMetaData {
+  std::string column_family_name;  // Name of the column family
+  std::string db_path;
+  std::string name;                // Name of the file
+  int level;               // Level at which this file resides.
+  size_t size;             // File size in bytes.
+  std::string smallestkey; // Smallest user defined key in the file.
+  std::string largestkey;  // Largest user defined key in the file.
+  SequenceNumber smallest_seqno; // smallest seqno in file
+  SequenceNumber largest_seqno;  // largest seqno in file
+};
+
 // Abstract handle to particular state of a DB.
 // A Snapshot is an immutable object and can therefore be safely
 // accessed from multiple threads without any external synchronization.
@@ -100,9 +106,6 @@ class DB {
   // that modify data, like put/delete, will return error.
   // If the db is opened in read only mode, then no compactions
   // will happen.
-  //
-  // Not supported in ROCKSDB_LITE, in which case the function will
-  // return Status::NotSupported.
   static Status OpenForReadOnly(const Options& options,
       const std::string& name, DB** dbptr,
       bool error_if_log_file_exist = false);
@@ -112,9 +115,6 @@ class DB {
   // database that should be opened. However, you always need to specify default
   // column family. The default column family name is 'default' and it's stored
   // in rocksdb::kDefaultColumnFamilyName
-  //
-  // Not supported in ROCKSDB_LITE, in which case the function will
-  // return Status::NotSupported.
   static Status OpenForReadOnly(
       const DBOptions& db_options, const std::string& name,
       const std::vector<ColumnFamilyDescriptor>& column_families,
@@ -370,26 +370,6 @@ class DB {
     return SetOptions(DefaultColumnFamily(), new_options);
   }
 
-  // CompactFiles() inputs a list of files specified by file numbers
-  // and compacts them to the specified level.  Note that the behavior
-  // is different from CompactRange in that CompactFiles() will
-  // perform the compaction job using the CURRENT thread.
-  //
-  // @see GetDataBaseMetaData
-  // @see GetColumnFamilyMetaData
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1) = 0;
-
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1) {
-    return CompactFiles(compact_options, DefaultColumnFamily(),
-                        input_file_names, output_level, output_path_id);
-  }
   // Number of levels used for this DB.
   virtual int NumberLevels(ColumnFamilyHandle* column_family) = 0;
   virtual int NumberLevels() { return NumberLevels(DefaultColumnFamily()); }
@@ -496,21 +476,6 @@ class DB {
   // and end key
   virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {}
 
-  // Obtains the meta data of the specified column family of the DB.
-  // Status::NotFound() will be returned if the current DB does not have
-  // any column family match the specified name.
-  //
-  // If cf_name is not specified, then the metadata of the default
-  // column family will be returned.
-  virtual void GetColumnFamilyMetaData(
-      ColumnFamilyHandle* column_family,
-      ColumnFamilyMetaData* metadata) {}
-
-  // Get the metadata of the default column family.
-  void GetColumnFamilyMetaData(
-      ColumnFamilyMetaData* metadata) {
-    GetColumnFamilyMetaData(DefaultColumnFamily(), metadata);
-  }
 #endif  // ROCKSDB_LITE
 
   // Sets the globally unique ID created at database creation time by invoking
@@ -547,7 +512,6 @@ Status DestroyDB(const std::string& name, const Options& options);
 Status RepairDB(const std::string& dbname, const Options& options);
 #endif
 
-
 }  // namespace rocksdb
 
 #endif  // STORAGE_ROCKSDB_INCLUDE_DB_H_

include/rocksdb/env.h

@@ -20,7 +20,6 @@
 #include <cstdarg>
 #include <string>
 #include <memory>
-#include <limits>
 #include <vector>
 #include <stdint.h>
 #include "rocksdb/status.h"
@@ -178,11 +177,6 @@ class Env {
   virtual Status RenameFile(const std::string& src,
                             const std::string& target) = 0;
 
-  // Hard Link file src to target.
-  virtual Status LinkFile(const std::string& src, const std::string& target) {
-    return Status::NotSupported("LinkFile is not supported for this Env");
-  }
-
   // Lock the specified file.  Used to prevent concurrent access to
   // the same db by multiple processes.  On failure, stores nullptr in
   // *lock and returns non-OK.
@@ -482,8 +476,8 @@ class WritableFile {
     if (new_last_preallocated_block > last_preallocated_block_) {
       size_t num_spanned_blocks =
         new_last_preallocated_block - last_preallocated_block_;
-      Allocate(static_cast<off_t>(block_size * last_preallocated_block_),
-               static_cast<off_t>(block_size * num_spanned_blocks));
+      Allocate(block_size * last_preallocated_block_,
+               block_size * num_spanned_blocks);
       last_preallocated_block_ = new_last_preallocated_block;
     }
   }
@@ -586,8 +580,7 @@ enum InfoLogLevel : unsigned char {
 // An interface for writing log messages.
 class Logger {
  public:
-  size_t kDoNotSupportGetLogFileSize = std::numeric_limits<size_t>::max();
-
+  enum { DO_NOT_SUPPORT_GET_LOG_FILE_SIZE = -1 };
   explicit Logger(const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL)
       : log_level_(log_level) {}
   virtual ~Logger();
@@ -620,7 +613,9 @@ class Logger {
       Logv(new_format, ap);
     }
   }
-  virtual size_t GetLogFileSize() const { return kDoNotSupportGetLogFileSize; }
+  virtual size_t GetLogFileSize() const {
+    return DO_NOT_SUPPORT_GET_LOG_FILE_SIZE;
+  }
   // Flush to the OS buffers
   virtual void Flush() {}
   virtual InfoLogLevel GetInfoLogLevel() const { return log_level_; }
@@ -752,11 +747,6 @@ class EnvWrapper : public Env {
   Status RenameFile(const std::string& s, const std::string& t) {
     return target_->RenameFile(s, t);
   }
-
-  Status LinkFile(const std::string& s, const std::string& t) {
-    return target_->LinkFile(s, t);
-  }
-
   Status LockFile(const std::string& f, FileLock** l) {
     return target_->LockFile(f, l);
   }

include/rocksdb/immutable_options.h

@@ -90,12 +90,6 @@ struct ImmutableCFOptions {
   Options::AccessHint access_hint_on_compaction_start;
 
   int num_levels;
-
-#ifndef ROCKSDB_LITE
-  // A vector of EventListeners which call-back functions will be called
-  // when specific RocksDB event happens.
-  std::vector<std::shared_ptr<EventListener>> listeners;
-#endif  // ROCKSDB_LITE
 };
 
 }  // namespace rocksdb

include/rocksdb/listener.h

@@ -1,66 +0,0 @@
-// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-
-#ifndef ROCKSDB_LITE
-
-#include <string>
-#include "rocksdb/status.h"
-
-namespace rocksdb {
-
-class DB;
-class Status;
-
-// EventListener class contains a set of call-back functions that will
-// be called when specific RocksDB event happens such as flush.  It can
-// be used as a building block for developing custom features such as
-// stats-collector or external compaction algorithm.
-//
-// Note that call-back functions should not run for an extended period of
-// time before the function returns, otherwise RocksDB may be blocked.
-// For example, it is not suggested to do DB::CompactFiles() (as it may
-// run for a long while) or issue many of DB::Put() (as Put may be blocked
-// in certain cases) in the same thread in the EventListener callback.
-// However, doing DB::CompactFiles() and DB::Put() in another thread is
-// considered safe.
-//
-// [Threading] All EventListener callback will be called using the
-// actual thread that involves in that specific event.   For example, it
-// is the RocksDB background flush thread that does the actual flush to
-// call EventListener::OnFlushCompleted().
-class EventListener {
- public:
-  // A call-back function to RocksDB which will be called whenever a
-  // registered RocksDB flushes a file.  The default implementation is
-  // no-op.
-  //
-  // Note that the this function must be implemented in a way such that
-  // it should not run for an extended period of time before the function
-  // returns.  Otherwise, RocksDB may be blocked.
-  //
-  // @param db a pointer to the rocksdb instance which just flushed
-  //     a memtable to disk.
-  // @param column_family_id the id of the flushed column family.
-  // @param file_path the path to the newly created file.
-  // @param triggered_writes_slowdown true when rocksdb is currently
-  //     slowing-down all writes to prevent creating too many Level 0
-  //     files as compaction seems not able to catch up the write request
-  //     speed.  This indicates that there're too many files in Level 0.
-  // @param triggered_writes_stop true when rocksdb is currently blocking
-  //     any writes to prevent creating more L0 files.  This indicates that
-  //     there're too many files in level 0.  Compactions should try to
-  //     compact L0 files down to lower levels as soon as possible.
-  virtual void OnFlushCompleted(
-      DB* db, const std::string& column_family_name,
-      const std::string& file_path,
-      bool triggered_writes_slowdown,
-      bool triggered_writes_stop) {}
-  virtual ~EventListener() {}
-};
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE

include/rocksdb/memtablerep.h

@@ -14,8 +14,8 @@
 //  (4) Items are never deleted.
 // The liberal use of assertions is encouraged to enforce (1).
 //
-// The factory will be passed an MemTableAllocator object when a new MemTableRep
-// is requested.
+// The factory will be passed an Arena object when a new MemTableRep is
+// requested. The API for this object is in rocksdb/arena.h.
 //
 // Users can implement their own memtable representations. We include three
 // types built in:
@@ -41,7 +41,6 @@
 namespace rocksdb {
 
 class Arena;
-class MemTableAllocator;
 class LookupKey;
 class Slice;
 class SliceTransform;
@@ -66,7 +65,7 @@ class MemTableRep {
     virtual ~KeyComparator() { }
   };
 
-  explicit MemTableRep(MemTableAllocator* allocator) : allocator_(allocator) {}
+  explicit MemTableRep(Arena* arena) : arena_(arena) {}
 
   // Allocate a buf of len size for storing key. The idea is that a specific
   // memtable representation knows its underlying data structure better. By
@@ -102,7 +101,7 @@ class MemTableRep {
                    bool (*callback_func)(void* arg, const char* entry));
 
   // Report an approximation of how much memory has been used other than memory
-  // that was allocated through the allocator.
+  // that was allocated through the arena.
   virtual size_t ApproximateMemoryUsage() = 0;
 
   virtual ~MemTableRep() { }
@@ -151,7 +150,7 @@ class MemTableRep {
 
   // Return an iterator that has a special Seek semantics. The result of
   // a Seek might only include keys with the same prefix as the target key.
-  // arena: If not null, the arena is used to allocate the Iterator.
+  // arena: If not null, the arena needs to be used to allocate the Iterator.
   //        When destroying the iterator, the caller will not call "delete"
   //        but Iterator::~Iterator() directly. The destructor needs to destroy
   //        all the states but those allocated in arena.
@@ -172,7 +171,7 @@ class MemTableRep {
   // user key.
   virtual Slice UserKey(const char* key) const;
 
-  MemTableAllocator* allocator_;
+  Arena* arena_;
 };
 
 // This is the base class for all factories that are used by RocksDB to create
@@ -181,8 +180,7 @@ class MemTableRepFactory {
  public:
   virtual ~MemTableRepFactory() {}
   virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
-                                         MemTableAllocator*,
-                                         const SliceTransform*,
+                                         Arena*, const SliceTransform*,
                                          Logger* logger) = 0;
   virtual const char* Name() const = 0;
 };
@@ -199,8 +197,7 @@ class SkipListFactory : public MemTableRepFactory {
   explicit SkipListFactory(size_t lookahead = 0) : lookahead_(lookahead) {}
 
   virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
-                                         MemTableAllocator*,
-                                         const SliceTransform*,
+                                         Arena*, const SliceTransform*,
                                          Logger* logger) override;
   virtual const char* Name() const override { return "SkipListFactory"; }
 
@@ -223,8 +220,7 @@ class VectorRepFactory : public MemTableRepFactory {
  public:
   explicit VectorRepFactory(size_t count = 0) : count_(count) { }
   virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
-                                         MemTableAllocator*,
-                                         const SliceTransform*,
+                                         Arena*, const SliceTransform*,
                                          Logger* logger) override;
   virtual const char* Name() const override {
     return "VectorRepFactory";

include/rocksdb/metadata.h

@@ -1,90 +0,0 @@
-// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-
-#include <limits>
-#include <string>
-#include <vector>
-
-#include "rocksdb/types.h"
-
-#pragma once
-
-namespace rocksdb {
-struct ColumnFamilyMetaData;
-struct LevelMetaData;
-struct SstFileMetaData;
-
-// The metadata that describes a column family.
-struct ColumnFamilyMetaData {
-  ColumnFamilyMetaData() : size(0), name("") {}
-  ColumnFamilyMetaData(const std::string& _name, uint64_t _size,
-                       const std::vector<LevelMetaData>&& _levels) :
-      size(_size), name(_name), levels(_levels) {}
-
-  // The size of this column family in bytes, which is equal to the sum of
-  // the file size of its "levels".
-  uint64_t size;
-  // The number of files in this column family.
-  size_t file_count;
-  // The name of the column family.
-  std::string name;
-  // The metadata of all levels in this column family.
-  std::vector<LevelMetaData> levels;
-};
-
-// The metadata that describes a level.
-struct LevelMetaData {
-  LevelMetaData(int _level, uint64_t _size,
-                const std::vector<SstFileMetaData>&& _files) :
-      level(_level), size(_size),
-      files(_files) {}
-
-  // The level which this meta data describes.
-  const int level;
-  // The size of this level in bytes, which is equal to the sum of
-  // the file size of its "files".
-  const uint64_t size;
-  // The metadata of all sst files in this level.
-  const std::vector<SstFileMetaData> files;
-};
-
-// The metadata that describes a SST file.
-struct SstFileMetaData {
-  SstFileMetaData() {}
-  SstFileMetaData(const std::string& _file_name,
-                  const std::string& _path, uint64_t _size,
-                  SequenceNumber _smallest_seqno,
-                  SequenceNumber _largest_seqno,
-                  const std::string& _smallestkey,
-                  const std::string& _largestkey,
-                  bool _being_compacted) :
-    size(_size), name(_file_name),
-    db_path(_path), smallest_seqno(_smallest_seqno), largest_seqno(_largest_seqno),
-    smallestkey(_smallestkey), largestkey(_largestkey),
-    being_compacted(_being_compacted) {}
-
-  // File size in bytes.
-  uint64_t size;
-  // The name of the file.
-  std::string name;
-  // The full path where the file locates.
-  std::string db_path;
-
-  SequenceNumber smallest_seqno;  // Smallest sequence number in file.
-  SequenceNumber largest_seqno;   // Largest sequence number in file.
-  std::string smallestkey;     // Smallest user defined key in the file.
-  std::string largestkey;      // Largest user defined key in the file.
-  bool being_compacted;  // true if the file is currently being compacted.
-};
-
-// The full set of metadata associated with each SST file.
-struct LiveFileMetaData : SstFileMetaData {
-  std::string column_family_name;  // Name of the column family
-  int level;               // Level at which this file resides.
-};
-
-
-
-}  // namespace rocksdb

include/rocksdb/options.h

@@ -10,16 +10,13 @@
 #define STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
 
 #include <stddef.h>
-#include <stdint.h>
 #include <string>
 #include <memory>
 #include <vector>
-#include <limits>
 #include <stdint.h>
 #include <unordered_map>
 
 #include "rocksdb/version.h"
-#include "rocksdb/listener.h"
 #include "rocksdb/universal_compaction.h"
 
 namespace rocksdb {
@@ -56,20 +53,12 @@ enum CompressionType : char {
 };
 
 enum CompactionStyle : char {
-  // level based compaction style
-  kCompactionStyleLevel = 0x0,
-  // Universal compaction style
-  // Not supported in ROCKSDB_LITE.
-  kCompactionStyleUniversal = 0x1,
-  // FIFO compaction style
-  // Not supported in ROCKSDB_LITE
-  kCompactionStyleFIFO = 0x2,
-  // Disable background compaction. Compaction jobs are submitted
-  // via CompactFiles().
-  // Not supported in ROCKSDB_LITE
-  kCompactionStyleNone = 0x3,
+  kCompactionStyleLevel = 0x0,      // level based compaction style
+  kCompactionStyleUniversal = 0x1,  // Universal compaction style
+  kCompactionStyleFIFO = 0x2,       // FIFO compaction style
 };
 
+
 struct CompactionOptionsFIFO {
   // once the total sum of table files reaches this, we will delete the oldest
   // table file
@@ -110,8 +99,6 @@ struct ColumnFamilyOptions {
 
   // Use this if you don't need to keep the data sorted, i.e. you'll never use
   // an iterator, only Put() and Get() API calls
-  //
-  // Not supported in ROCKSDB_LITE
   ColumnFamilyOptions* OptimizeForPointLookup(
       uint64_t block_cache_size_mb);
 
@@ -129,8 +116,6 @@ struct ColumnFamilyOptions {
   // biggest performance gains.
   // Note: we might use more memory than memtable_memory_budget during high
   // write rate period
-  //
-  // OptimizeUniversalStyleCompaction is not supported in ROCKSDB_LITE
   ColumnFamilyOptions* OptimizeLevelStyleCompaction(
       uint64_t memtable_memory_budget = 512 * 1024 * 1024);
   ColumnFamilyOptions* OptimizeUniversalStyleCompaction(
@@ -205,9 +190,6 @@ struct ColumnFamilyOptions {
   // Also, a larger write buffer will result in a longer recovery time
   // the next time the database is opened.
   //
-  // Note that write_buffer_size is enforced per column family.
-  // See db_write_buffer_size for sharing memory across column families.
-  //
   // Default: 4MB
   //
   // Dynamically changeable through SetOptions() API
@@ -322,7 +304,6 @@ struct ColumnFamilyOptions {
   //
   // Dynamically changeable through SetOptions() API
   uint64_t target_file_size_base;
-
   // By default target_file_size_multiplier is 1, which means
   // by default files in different levels will have similar size.
   //
@@ -342,7 +323,6 @@ struct ColumnFamilyOptions {
   //
   // Dynamically changeable through SetOptions() API
   uint64_t max_bytes_for_level_base;
-
   // Default: 10.
   //
   // Dynamically changeable through SetOptions() API
@@ -435,10 +415,7 @@ struct ColumnFamilyOptions {
 
   // If true, compaction will verify checksum on every read that happens
   // as part of compaction
-  //
   // Default: true
-  //
-  // Dynamically changeable through SetOptions() API
   bool verify_checksums_in_compaction;
 
   // The options needed to support Universal Style compactions
@@ -609,12 +586,6 @@ struct ColumnFamilyOptions {
   // Default: 2
   uint32_t min_partial_merge_operands;
 
-#ifndef ROCKSDB_LITE
-  // A vector of EventListeners which call-back functions will be called
-  // when specific RocksDB event happens.
-  std::vector<std::shared_ptr<EventListener>> listeners;
-#endif  // ROCKSDB_LITE
-
   // Create ColumnFamilyOptions with default values for all fields
   ColumnFamilyOptions();
   // Create ColumnFamilyOptions from Options
@@ -626,14 +597,12 @@ struct ColumnFamilyOptions {
 struct DBOptions {
   // Some functions that make it easier to optimize RocksDB
 
-#ifndef ROCKSDB_LITE
   // By default, RocksDB uses only one background thread for flush and
   // compaction. Calling this function will set it up such that total of
   // `total_threads` is used. Good value for `total_threads` is the number of
   // cores. You almost definitely want to call this function if your system is
   // bottlenecked by RocksDB.
   DBOptions* IncreaseParallelism(int total_threads = 16);
-#endif  // ROCKSDB_LITE
 
   // If true, the database will be created if it is missing.
   // Default: false
@@ -647,11 +616,14 @@ struct DBOptions {
   // Default: false
   bool error_if_exists;
 
-  // If true, RocksDB will aggressively check consistency of the data.
-  // Also, if any of the  writes to the database fails (Put, Delete, Merge,
-  // Write), the database will switch to read-only mode and fail all other
+  // If true, the implementation will do aggressive checking of the
+  // data it is processing and will stop early if it detects any
+  // errors.  This may have unforeseen ramifications: for example, a
+  // corruption of one DB entry may cause a large number of entries to
+  // become unreadable or for the entire DB to become unopenable.
+  // If any of the  writes to the database fails (Put, Delete, Merge, Write),
+  // the database will switch to read-only mode and fail all other
   // Write operations.
-  // In most cases you want this to be set to true.
   // Default: true
   bool paranoid_checks;
 
@@ -686,7 +658,7 @@ struct DBOptions {
   // column families whose memtables are backed by the oldest live WAL file
   // (i.e. the ones that are causing all the space amplification). If set to 0
   // (default), we will dynamically choose the WAL size limit to be
-  // [sum of all write_buffer_size * max_write_buffer_number] * 4
+  // [sum of all write_buffer_size * max_write_buffer_number] * 2
   // Default: 0
   uint64_t max_total_wal_size;
 
@@ -864,18 +836,6 @@ struct DBOptions {
   // Default: true
   bool advise_random_on_open;
 
-  // Amount of data to build up in memtables across all column
-  // families before writing to disk.
-  //
-  // This is distinct from write_buffer_size, which enforces a limit
-  // for a single memtable.
-  //
-  // This feature is disabled by default. Specify a non-zero value
-  // to enable it.
-  //
-  // Default: 0 (disabled)
-  size_t db_write_buffer_size;
-
   // Specify the file access pattern once a compaction is started.
   // It will be applied to all input files of a compaction.
   // Default: NORMAL
@@ -910,12 +870,6 @@ struct DBOptions {
   // When rate limiter is enabled, it automatically enables bytes_per_sync
   // to 1MB.
   uint64_t bytes_per_sync;
-
-  // If true, then the status of the threads involved in this DB will
-  // be tracked and available via GetThreadList() API.
-  //
-  // Default: false
-  bool enable_thread_tracking;
 };
 
 // Options to control the behavior of a database (passed to DB::Open)
@@ -1113,19 +1067,6 @@ extern Options GetOptions(size_t total_write_buffer_limit,
                           int write_amplification_threshold = 32,
                           uint64_t target_db_size = 68719476736 /* 64GB */);
 
-// CompactionOptions are used in CompactFiles() call.
-struct CompactionOptions {
-  // Compaction output compression type
-  // Default: snappy
-  CompressionType compression;
-  // Compaction will create files of size `output_file_size_limit`.
-  // Default: MAX, which means that compaction will create a single file
-  uint64_t output_file_size_limit;
-
-  CompactionOptions()
-      : compression(kSnappyCompression),
-        output_file_size_limit(std::numeric_limits<uint64_t>::max()) {}
-};
 }  // namespace rocksdb
 
 #endif  // STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_

include/rocksdb/slice.h

@@ -123,7 +123,7 @@ inline bool operator!=(const Slice& x, const Slice& y) {
 }
 
 inline int Slice::compare(const Slice& b) const {
-  const size_t min_len = (size_ < b.size_) ? size_ : b.size_;
+  const int min_len = (size_ < b.size_) ? size_ : b.size_;
   int r = memcmp(data_, b.data_, min_len);
   if (r == 0) {
     if (size_ < b.size_) r = -1;

include/rocksdb/sst_dump_tool.h

@@ -1,17 +0,0 @@
-// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-#ifndef ROCKSDB_LITE
-#pragma once
-
-namespace rocksdb {
-
-class SSTDumpTool {
- public:
-  void Run(int argc, char** argv);
-};
-
-}  // namespace rocksdb
-
-#endif  // ROCKSDB_LITE

include/rocksdb/status.h

@@ -61,9 +61,6 @@ class Status {
   static Status Incomplete(const Slice& msg, const Slice& msg2 = Slice()) {
     return Status(kIncomplete, msg, msg2);
   }
-  static Status ShutdownInProgress() {
-    return Status(kShutdownInProgress);
-  }
   static Status ShutdownInProgress(const Slice& msg,
                                    const Slice& msg2 = Slice()) {
     return Status(kShutdownInProgress, msg, msg2);
@@ -74,12 +71,6 @@ class Status {
   static Status TimedOut(const Slice& msg, const Slice& msg2 = Slice()) {
     return Status(kTimedOut, msg, msg2);
   }
-  static Status Aborted() {
-    return Status(kAborted);
-  }
-  static Status Aborted(const Slice& msg, const Slice& msg2 = Slice()) {
-    return Status(kAborted, msg, msg2);
-  }
 
   // Returns true iff the status indicates success.
   bool ok() const { return code() == kOk; }
@@ -110,8 +101,6 @@ class Status {
 
   bool IsTimedOut() const { return code() == kTimedOut; }
 
-  bool IsAborted() const { return code() == kAborted; }
-
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
   std::string ToString() const;
@@ -126,8 +115,7 @@ class Status {
     kMergeInProgress = 6,
     kIncomplete = 7,
     kShutdownInProgress = 8,
-    kTimedOut = 9,
-    kAborted = 10
+    kTimedOut = 9
   };
 
   Code code() const {

include/rocksdb/table.h

@@ -369,14 +369,13 @@ class TableFactory {
 };
 
 #ifndef ROCKSDB_LITE
-// Create a special table factory that can open either of the supported
-// table formats, based on setting inside the SST files. It should be used to
+// Create a special table factory that can open both of block based table format
+// and plain table, based on setting inside the SST files. It should be used to
 // convert a DB from one table format to another.
 // @table_factory_to_write: the table factory used when writing to new files.
 // @block_based_table_factory:  block based table factory to use. If NULL, use
 //                              a default one.
 // @plain_table_factory: plain table factory to use. If NULL, use a default one.
-// @cuckoo_table_factory: cuckoo table factory to use. If NULL, use a default one.
 extern TableFactory* NewAdaptiveTableFactory(
     std::shared_ptr<TableFactory> table_factory_to_write = nullptr,
     std::shared_ptr<TableFactory> block_based_table_factory = nullptr,

include/rocksdb/utilities/backupable_db.h

@@ -177,8 +177,6 @@ class BackupEngineReadOnly {
   // You can GetBackupInfo safely, even with other BackupEngine performing
   // backups on the same directory
   virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info) = 0;
-  virtual void GetCorruptedBackups(
-      std::vector<BackupID>* corrupt_backup_ids) = 0;
 
   // Restoring DB from backup is NOT safe when there is another BackupEngine
   // running that might call DeleteBackup() or PurgeOldBackups(). It is caller's
@@ -198,12 +196,7 @@ class BackupEngine {
   virtual ~BackupEngine() {}
 
   static BackupEngine* NewBackupEngine(Env* db_env,
-                                       const BackupableDBOptions& options)
-    __attribute__((deprecated("Please use Open() instead")));
-
-  static Status Open(Env* db_env,
-                     const BackupableDBOptions& options,
-                     BackupEngine** backup_engine_ptr);
+                                       const BackupableDBOptions& options);
 
   virtual Status CreateNewBackup(DB* db, bool flush_before_backup = false) = 0;
   virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0;
@@ -211,16 +204,12 @@ class BackupEngine {
   virtual void StopBackup() = 0;
 
   virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info) = 0;
-  virtual void GetCorruptedBackups(
-      std::vector<BackupID>* corrupt_backup_ids) = 0;
   virtual Status RestoreDBFromBackup(
       BackupID backup_id, const std::string& db_dir, const std::string& wal_dir,
       const RestoreOptions& restore_options = RestoreOptions()) = 0;
   virtual Status RestoreDBFromLatestBackup(
       const std::string& db_dir, const std::string& wal_dir,
       const RestoreOptions& restore_options = RestoreOptions()) = 0;
-
-  virtual Status GarbageCollect() = 0;
 };
 
 // Stack your DB with BackupableDB to be able to backup the DB
@@ -239,8 +228,6 @@ class BackupableDB : public StackableDB {
   Status CreateNewBackup(bool flush_before_backup = false);
   // Returns info about backups in backup_info
   void GetBackupInfo(std::vector<BackupInfo>* backup_info);
-  // Returns info about corrupt backups in corrupt_backups
-  void GetCorruptedBackups(std::vector<BackupID>* corrupt_backup_ids);
   // deletes old backups, keeping latest num_backups_to_keep alive
   Status PurgeOldBackups(uint32_t num_backups_to_keep);
   // deletes a specific backup
@@ -254,11 +241,6 @@ class BackupableDB : public StackableDB {
   // next time you create BackupableDB or RestoreBackupableDB.
   void StopBackup();
 
-  // Will delete all the files we don't need anymore
-  // It will do the full scan of the files/ directory and delete all the
-  // files that are not referenced.
-  Status GarbageCollect();
-
  private:
   BackupEngine* backup_engine_;
 };
@@ -271,8 +253,6 @@ class RestoreBackupableDB {
 
   // Returns info about backups in backup_info
   void GetBackupInfo(std::vector<BackupInfo>* backup_info);
-  // Returns info about corrupt backups in corrupt_backups
-  void GetCorruptedBackups(std::vector<BackupID>* corrupt_backup_ids);
 
   // restore from backup with backup_id
   // IMPORTANT -- if options_.share_table_files == true and you restore DB
@@ -299,11 +279,6 @@ class RestoreBackupableDB {
   // deletes a specific backup
   Status DeleteBackup(BackupID backup_id);
 
-  // Will delete all the files we don't need anymore
-  // It will do the full scan of the files/ directory and delete all the
-  // files that are not referenced.
-  Status GarbageCollect();
-
  private:
   BackupEngine* backup_engine_;
 };

include/rocksdb/utilities/checkpoint.h

@@ -1,34 +0,0 @@
-// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree. An additional grant
-// of patent rights can be found in the PATENTS file in the same directory.
-//
-// A checkpoint is an openable snapshot of a database at a point in time.
-
-#pragma once
-
-#include <string>
-#include "rocksdb/status.h"
-
-namespace rocksdb {
-
-class DB;
-
-class Checkpoint {
- public:
-  // Creates a Checkpoint object to be used for creating openable sbapshots
-  static Status Create(DB* db, Checkpoint** checkpoint_ptr);
-
-  // Builds an openable snapshot of RocksDB on the same disk, which
-  // accepts an output directory on the same disk, and under the directory
-  // (1) hard-linked SST files pointing to existing live SST files
-  // SST files will be copied if output directory is on a different filesystem
-  // (2) a copied manifest files and other files
-  // The directory should not already exist and will be created by this API.
-  // The directory will be an absolute path
-  virtual Status CreateCheckpoint(const std::string& checkpoint_dir);
-
-  virtual ~Checkpoint() {}
-};
-
-}  // namespace rocksdb

include/rocksdb/utilities/leveldb_options.h

@@ -1,144 +0,0 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-
-#include <stddef.h>
-
-namespace rocksdb {
-
-class Cache;
-class Comparator;
-class Env;
-class FilterPolicy;
-class Logger;
-struct Options;
-class Snapshot;
-
-enum CompressionType : char;
-
-// Options to control the behavior of a database (passed to
-// DB::Open). A LevelDBOptions object can be initialized as though
-// it were a LevelDB Options object, and then it can be converted into
-// a RocksDB Options object.
-struct LevelDBOptions {
-  // -------------------
-  // Parameters that affect behavior
-
-  // Comparator used to define the order of keys in the table.
-  // Default: a comparator that uses lexicographic byte-wise ordering
-  //
-  // REQUIRES: The client must ensure that the comparator supplied
-  // here has the same name and orders keys *exactly* the same as the
-  // comparator provided to previous open calls on the same DB.
-  const Comparator* comparator;
-
-  // If true, the database will be created if it is missing.
-  // Default: false
-  bool create_if_missing;
-
-  // If true, an error is raised if the database already exists.
-  // Default: false
-  bool error_if_exists;
-
-  // If true, the implementation will do aggressive checking of the
-  // data it is processing and will stop early if it detects any
-  // errors.  This may have unforeseen ramifications: for example, a
-  // corruption of one DB entry may cause a large number of entries to
-  // become unreadable or for the entire DB to become unopenable.
-  // Default: false
-  bool paranoid_checks;
-
-  // Use the specified object to interact with the environment,
-  // e.g. to read/write files, schedule background work, etc.
-  // Default: Env::Default()
-  Env* env;
-
-  // Any internal progress/error information generated by the db will
-  // be written to info_log if it is non-NULL, or to a file stored
-  // in the same directory as the DB contents if info_log is NULL.
-  // Default: NULL
-  Logger* info_log;
-
-  // -------------------
-  // Parameters that affect performance
-
-  // Amount of data to build up in memory (backed by an unsorted log
-  // on disk) before converting to a sorted on-disk file.
-  //
-  // Larger values increase performance, especially during bulk loads.
-  // Up to two write buffers may be held in memory at the same time,
-  // so you may wish to adjust this parameter to control memory usage.
-  // Also, a larger write buffer will result in a longer recovery time
-  // the next time the database is opened.
-  //
-  // Default: 4MB
-  size_t write_buffer_size;
-
-  // Number of open files that can be used by the DB.  You may need to
-  // increase this if your database has a large working set (budget
-  // one open file per 2MB of working set).
-  //
-  // Default: 1000
-  int max_open_files;
-
-  // Control over blocks (user data is stored in a set of blocks, and
-  // a block is the unit of reading from disk).
-
-  // If non-NULL, use the specified cache for blocks.
-  // If NULL, leveldb will automatically create and use an 8MB internal cache.
-  // Default: NULL
-  Cache* block_cache;
-
-  // Approximate size of user data packed per block.  Note that the
-  // block size specified here corresponds to uncompressed data.  The
-  // actual size of the unit read from disk may be smaller if
-  // compression is enabled.  This parameter can be changed dynamically.
-  //
-  // Default: 4K
-  size_t block_size;
-
-  // Number of keys between restart points for delta encoding of keys.
-  // This parameter can be changed dynamically.  Most clients should
-  // leave this parameter alone.
-  //
-  // Default: 16
-  int block_restart_interval;
-
-  // Compress blocks using the specified compression algorithm.  This
-  // parameter can be changed dynamically.
-  //
-  // Default: kSnappyCompression, which gives lightweight but fast
-  // compression.
-  //
-  // Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz:
-  //    ~200-500MB/s compression
-  //    ~400-800MB/s decompression
-  // Note that these speeds are significantly faster than most
-  // persistent storage speeds, and therefore it is typically never
-  // worth switching to kNoCompression.  Even if the input data is
-  // incompressible, the kSnappyCompression implementation will
-  // efficiently detect that and will switch to uncompressed mode.
-  CompressionType compression;
-
-  // If non-NULL, use the specified filter policy to reduce disk reads.
-  // Many applications will benefit from passing the result of
-  // NewBloomFilterPolicy() here.
-  //
-  // Default: NULL
-  const FilterPolicy* filter_policy;
-
-  // Create a LevelDBOptions object with default values for all fields.
-  LevelDBOptions();
-};
-
-// Converts a LevelDBOptions object into a RocksDB Options object.
-Options ConvertOptions(const LevelDBOptions& leveldb_options);
-
-}  // namespace rocksdb

include/rocksdb/utilities/spatial_db.h

@@ -222,9 +222,7 @@ class SpatialDB : public StackableDB {
 
   // Calling Compact() after inserting a bunch of elements should speed up
   // reading. This is especially useful if you use SpatialDBOptions::bulk_load
-  // Num threads determines how many threads we'll use for compactions. Setting
-  // this to bigger number will use more IO and CPU, but finish faster
-  virtual Status Compact(int num_threads = 1) = 0;
+  virtual Status Compact() = 0;
 
   // Query the specified spatial_index. Query will return all elements that
   // intersect bbox, but it may also return some extra elements.

include/rocksdb/utilities/stackable_db.h

@@ -133,17 +133,6 @@ class StackableDB : public DB {
                              target_level, target_path_id);
   }
 
-  using DB::CompactFiles;
-  virtual Status CompactFiles(
-      const CompactionOptions& compact_options,
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& input_file_names,
-      const int output_level, const int output_path_id = -1) override {
-    return db_->CompactFiles(
-        compact_options, column_family, input_file_names,
-        output_level, output_path_id);
-  }
-
   using DB::NumberLevels;
   virtual int NumberLevels(ColumnFamilyHandle* column_family) override {
     return db_->NumberLevels(column_family);
@@ -181,8 +170,6 @@ class StackableDB : public DB {
     return db_->Flush(fopts, column_family);
   }
 
-#ifndef ROCKSDB_LITE
-
   virtual Status DisableFileDeletions() override {
     return db_->DisableFileDeletions();
   }
@@ -196,14 +183,6 @@ class StackableDB : public DB {
     db_->GetLiveFilesMetaData(metadata);
   }
 
-  virtual void GetColumnFamilyMetaData(
-      ColumnFamilyHandle *column_family,
-      ColumnFamilyMetaData* cf_meta) override {
-    db_->GetColumnFamilyMetaData(column_family, cf_meta);
-  }
-
-#endif  // ROCKSDB_LITE
-
   virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs,
                               bool flush_memtable = true) override {
       return db_->GetLiveFiles(vec, mfs, flush_memtable);

include/rocksdb/version.h

@@ -5,8 +5,8 @@
 #pragma once
 
 #define ROCKSDB_MAJOR 3
-#define ROCKSDB_MINOR 9
-#define ROCKSDB_PATCH 1
+#define ROCKSDB_MINOR 7
+#define ROCKSDB_PATCH 0
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these