Merge pull request #764 from dmittendorf/fix-java-static-packaging

Fix crossbuild jar packaging
Conflicts:
	Makefile
	java/crossbuild/build-linux-centos.sh

.travis.yml

@@ -1,19 +1,43 @@
+sudo: false
 language: cpp
-compiler: gcc
-before_install:
-# As of this writing (10 May 2014) the Travis build environment is Ubuntu 12.04,
-# which needs the following ugly dependency incantations to build RocksDB:
- - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
- - sudo apt-get update -qq
- - sudo apt-get install -y -qq gcc-4.9 g++-4.9 zlib1g-dev libbz2-dev libsnappy-dev
- - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-4.9 50
- - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-4.9 50
- - wget https://gflags.googlecode.com/files/libgflags0_2.0-1_amd64.deb
- - sudo dpkg -i libgflags0_2.0-1_amd64.deb
- - wget https://gflags.googlecode.com/files/libgflags-dev_2.0-1_amd64.deb
- - sudo dpkg -i libgflags-dev_2.0-1_amd64.deb
+
+matrix:
+  include:
+    - os: linux
+      compiler: clang
+      env: COMPILER=clang++-3.6
+      addons:
+         apt:
+            sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.6']
+            packages: ['clang-3.6', 'clang-format-3.6', 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl']
+    - os: osx
+      compiler: clang
+
+install:
+  # Build gflags
+  # TODO(noetzli): Remove when gflags available through Travis
+  - pushd /tmp/ && curl -L https://github.com/gflags/gflags/archive/v2.1.2.tar.gz -o gflags.tar.gz && tar xfz gflags.tar.gz && cd gflags-2.1.2 && cmake . && make && popd
+  # Download clang-format-diff.py to check source code formatting
+  - pushd /tmp/ && curl -L http://llvm.org/svn/llvm-project/cfe/trunk/tools/clang-format/clang-format-diff.py -o clang-format-diff.py && chmod +x clang-format-diff.py && popd
+
+before_script:
+  # Add gflags to include/library paths
+  # TODO(noetzli): Remove when gflags available through Travis
+  - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/gflags-2.1.2/lib"
+  - export LIBRARY_PATH="$LIBRARY_PATH:/tmp/gflags-2.1.2/lib"
+  - export CPLUS_INCLUDE_PATH="$CPLUS_INCLUDE_PATH:/tmp/gflags-2.1.2/include"
+  - if [ -n "${COMPILER}" ]; then CXX=${COMPILER}; fi
+  - if [[ "${TRAVIS_OS_NAME}" == 'osx' ]]; then brew install gflags snappy; fi
+  - ulimit -n 2000 || true
+
 # Lousy hack to disable use and testing of fallocate, which doesn't behave quite
 # as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
-script: OPT=-DTRAVIS V=1 make unity && make clean && OPT=-DTRAVIS V=1 make db_test && ./db_test && OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest
+script:
+  - if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then OPT=-DTRAVIS CLANG_FORMAT_DIFF=/tmp/clang-format-diff.py make format; fi
+  - OPT=-DTRAVIS V=1 make -j4 check && OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest
+
 notifications:
-    email: false
+    email:
+      - leveldb@fb.com
+    webhooks:
+      - https://buildtimetrend.herokuapp.com/travis

CMakeLists.txt

@@ -74,6 +74,7 @@ set(SOURCES
         db/column_family.cc
         db/compacted_db_impl.cc
         db/compaction.cc
+        db/compaction_iterator.cc
         db/compaction_job.cc
         db/compaction_picker.cc
         db/convenience.cc
@@ -102,6 +103,7 @@ set(SOURCES
         db/merge_operator.cc
         db/repair.cc
         db/slice.cc
+        db/snapshot_impl.cc
         db/table_cache.cc
         db/table_properties_collector.cc
         db/transaction_log_impl.cc
@@ -190,6 +192,7 @@ set(SOURCES
         util/sst_dump_tool.cc
         util/statistics.cc
         util/status.cc
+        util/status_message.cc
         util/string_util.cc
         util/sync_point.cc
         util/testharness.cc
@@ -218,8 +221,14 @@ set(SOURCES
         utilities/redis/redis_lists.cc
         utilities/spatialdb/spatial_db.cc
         utilities/table_properties_collectors/compact_on_deletion_collector.cc
-        utilities/transactions/optimistic_transaction_impl.cc        
+        utilities/transactions/optimistic_transaction_impl.cc
         utilities/transactions/optimistic_transaction_db_impl.cc
+        utilities/transactions/transaction_base.cc
+        utilities/transactions/transaction_impl.cc
+        utilities/transactions/transaction_db_impl.cc
+        utilities/transactions/transaction_db_mutex_impl.cc
+        utilities/transactions/transaction_lock_mgr.cc
+        utilities/transactions/transaction_util.cc
         utilities/ttl/db_ttl_impl.cc
         utilities/write_batch_with_index/write_batch_with_index.cc
         utilities/write_batch_with_index/write_batch_with_index_internal.cc
@@ -332,6 +341,7 @@ set(TESTS
         utilities/spatialdb/spatial_db_test.cc
         utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
         utilities/transactions/optimistic_transaction_test.cc
+        utilities/transactions/transaction_test.cc
         utilities/ttl/ttl_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc
 )

HISTORY.md

@@ -1,5 +1,18 @@
 # Rocksdb Change Log
 
+## 4.0.0 (9/9/2015)
+### New Features
+* Added support for transactions.  See include/rocksdb/utilities/transaction.h for more info.
+* DB::GetProperty() now accept "rocksdb.aggregated-table-properties" and "rocksdb.aggregated-table-properties-at-levelN", in which case it returns aggregated table properties of the target column family, or the aggregated table properties of the specified level N if the "at-level" version is used.
+* Add compression option kZSTDNotFinalCompression for people to experiment ZSTD although its format is not finalized.
+* We removed the need for LATEST_BACKUP file in BackupEngine. We still keep writing it when we create new backups (because of backward compatibility), but we don't read it anymore.
+
+### Public API Changes
+* Removed class Env::RandomRWFile and Env::NewRandomRWFile().
+* Renamed DBOptions.num_subcompactions to DBOptions.max_subcompactions to make the name better match the actual functionality of the option.
+* Added Equal() method to the Comparator interface that can optionally be overwritten in cases where equality comparisons can be done more efficiently than three-way comparisons.
+* Previous 'experimental' OptimisticTransaction class has been replaced by Transaction class.
+
 ## 3.13.0 (8/6/2015)
 ### New Features
 * RollbackToSavePoint() in WriteBatch/WriteBatchWithIndex

INSTALL.md

@@ -82,4 +82,4 @@ your make commands, like this: `PORTABLE=1 make static_lib`
   * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
 
 * **Windows**:
-  * Read the follow the instructions at CMakeLists.txt
+  * Read and follow the instructions at CMakeLists.txt

Makefile

@@ -304,7 +304,8 @@ TESTS = \
 	write_callback_test \
 	heap_test \
 	compact_on_deletion_collector_test \
-	compaction_job_stats_test
+	compaction_job_stats_test \
+	transaction_test
 
 SUBSET :=  $(shell echo $(TESTS) |sed s/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/)
 
@@ -376,20 +377,23 @@ endif  # PLATFORM_SHARED_EXT
 .PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests package \
 	release tags valgrind_check whitebox_crash_test format static_lib shared_lib all \
 	dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
-	analyze
+	analyze tools
 
-all: $(LIBRARY) $(BENCHMARKS) $(TOOLS) $(TESTS)
+
+all: $(LIBRARY) $(BENCHMARKS) tools $(TESTS)
 
 static_lib: $(LIBRARY)
 
 shared_lib: $(SHARED)
 
-dbg: $(LIBRARY) $(BENCHMARKS) $(TOOLS) $(TESTS)
+tools: $(TOOLS)
+
+dbg: $(LIBRARY) $(BENCHMARKS) tools $(TESTS)
 
 # creates static library and programs
 release:
 	$(MAKE) clean
-	OPT="-DNDEBUG -O2" $(MAKE) static_lib $(TOOLS) db_bench
+	OPT="-DNDEBUG -O2" $(MAKE) static_lib tools db_bench
 
 coverage:
 	$(MAKE) clean
@@ -919,6 +923,9 @@ write_callback_test: db/write_callback_test.o $(LIBOBJECTS) $(TESTHARNESS)
 heap_test: util/heap_test.o $(GTEST)
 	$(AM_LINK)
 
+transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)
 
@@ -995,7 +1002,7 @@ libbz2.a:
 	-rm -rf bzip2-1.0.6
 	curl -O  http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
 	tar xvzf bzip2-1.0.6.tar.gz
-	cd bzip2-1.0.6 && make CFLAGS='-fPIC -Wall -Winline -O2 -g -D_FILE_OFFSET_BITS=64'
+	cd bzip2-1.0.6 && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64'
 	cp bzip2-1.0.6/libbz2.a .
 
 libsnappy.a:
@@ -1014,21 +1021,23 @@ liblz4.a:
 	   cd lz4-r127/lib && make CFLAGS='-fPIC' all
 	   cp lz4-r127/lib/liblz4.a .
 
-# A version of each $(LIBOBJECTS) compiled with -fPIC
-java_libobjects = $(patsubst %,jl/%,$(LIBOBJECTS))
-CLEAN_FILES += jl
+# A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
+java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
+CLEAN_FILES += jls
 
-$(java_libobjects): jl/%.o: %.cc
-	$(AM_V_CC)mkdir -p $(@D)
-	@$(CXX) $(CXXFLAGS) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
+JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4
+JAVA_STATIC_INCLUDES = -I./zlib-1.2.8 -I./bzip2-1.0.6 -I./snappy-1.1.1 -I./lz4-r127/lib
 
-rocksdbjavastatic: $(java_libobjects) libz.a libbz2.a libsnappy.a liblz4.a
+$(java_static_libobjects): jls/%.o: %.cc libz.a libbz2.a libsnappy.a liblz4.a
+	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
+
+rocksdbjavastatic: $(java_static_libobjects)
 	cd java;$(MAKE) javalib;
 	rm -f ./java/target/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
-	  $(java_libobjects) $(COVERAGEFLAGS) \
-	  libz.a libbz2.a libsnappy.a liblz4.a $(LDFLAGS)
+	  $(java_static_libobjects) $(COVERAGEFLAGS) \
+	  libz.a libbz2.a libsnappy.a liblz4.a $(JAVA_STATIC_LDFLAGS)
 	cd java/target;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
@@ -1050,6 +1059,13 @@ rocksdbjavastaticpublish: rocksdbjavastaticrelease
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar -Dclassifier=osx
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
 
+# A version of each $(LIBOBJECTS) compiled with -fPIC
+java_libobjects = $(patsubst %,jl/%,$(LIBOBJECTS))
+CLEAN_FILES += jl
+
+$(java_libobjects): jl/%.o: %.cc
+	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
+
 rocksdbjava: $(java_libobjects)
 	cd java;$(MAKE) javalib;
 	rm -f ./java/target/$(ROCKSDBJNILIB)

USERS.md

@@ -34,3 +34,6 @@ Tango is using RocksDB as a graph storage to store all users' connection data an
 ## Turn
 Turn is using RocksDB as a storage layer for their key/value store, serving at peak 2.4MM QPS out of different datacenters.
 Check out our RocksDB Protobuf merge operator at: https://github.com/vladb38/rocksdb_protobuf
+
+## Santanader UK/Cloudera Profession Services
+Check out their blog post: http://blog.cloudera.com/blog/2015/08/inside-santanders-near-real-time-data-ingest-architecture/

appveyor.yml

@@ -0,0 +1,10 @@
+version: 1.0.{build}
+before_build:
+- md %APPVEYOR_BUILD_FOLDER%\build
+- cd %APPVEYOR_BUILD_FOLDER%\build
+- cmake -G "Visual Studio 12 Win64" ..
+- cd ..
+build:
+  project: build\ALL_BUILD.vcxproj
+  verbosity: minimal
+test: off

arcanist_util/__phutil_library_map__.php

@@ -11,11 +11,14 @@ phutil_register_library_map(array(
   'class' =>
   array(
     'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
+    'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
     'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php',
     'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
     'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php',
+    'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
+    'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
+    'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
     'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
-    'PfffCppLinter' => 'cpp_linter/PfffCppLinter.php',
   ),
   'function' =>
   array(
@@ -23,10 +26,13 @@ phutil_register_library_map(array(
   'xmap' =>
   array(
     'ArcanistCpplintLinter' => 'ArcanistLinter',
+    'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
     'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
     'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
     'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+    'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
+    'FacebookHowtoevenLinter' => 'ArcanistLinter',
+    'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
     'FbcodeCppLinter' => 'ArcanistLinter',
-    'PfffCppLinter' => 'ArcanistLinter',
   ),
 ));

arcanist_util/config/FacebookArcanistConfiguration.php

@@ -1,5 +1,8 @@
 <?php
 // Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
 
 class FacebookArcanistConfiguration extends ArcanistConfiguration {
 

arcanist_util/cpp_linter/BaseDirectoryScopedFormatLinter.php

@@ -0,0 +1,74 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+abstract class BaseDirectoryScopedFormatLinter extends ArcanistLinter {
+
+  const LINT_FORMATTING = 1;
+
+  private $changedLines = array();
+  private $rawLintOutput = array();
+
+  abstract protected function getPathsToLint();
+
+  protected function shouldLintPath($path) {
+    foreach ($this->getPathsToLint() as $p) {
+      // check if $path starts with $p
+      if (strncmp($path, $p, strlen($p)) === 0) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // API to tell this linter which lines were changed
+  final public function setPathChangedLines($path, $changed) {
+    $this->changedLines[$path] = $changed;
+  }
+
+  final public function willLintPaths(array $paths) {
+    $futures = array();
+    foreach ($paths as $path) {
+      if (!$this->shouldLintPath($path)) {
+        continue;
+      }
+
+      $changed = $this->changedLines[$path];
+      if (!isset($changed)) {
+        // do not run linter if there are no changes
+        continue;
+      }
+
+      $futures[$path] = $this->getFormatFuture($path, $changed);
+    }
+
+    foreach (Futures($futures)->limit(8) as $p => $f) {
+      $this->rawLintOutput[$p] = $f->resolvex();
+    }
+  }
+
+  abstract protected function getFormatFuture($path, array $changed);
+  abstract protected function getLintMessage($diff);
+
+  final public function lintPath($path) {
+    if (!isset($this->rawLintOutput[$path])) {
+      return;
+    }
+
+    list($new_content) = $this->rawLintOutput[$path];
+    $old_content = $this->getData($path);
+
+    if ($new_content != $old_content) {
+      $diff = ArcanistDiffUtils::renderDifferences($old_content, $new_content);
+      $this->raiseLintAtOffset(
+        0,
+        self::LINT_FORMATTING,
+        $this->getLintMessage($diff),
+        $old_content,
+        $new_content);
+    }
+  }
+
+}

arcanist_util/cpp_linter/FacebookHowtoevenLinter.php

@@ -0,0 +1,223 @@
+<?php
+// Copyright 2015-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FacebookHowtoevenLinter extends ArcanistLinter {
+
+  const VERSION = 'fd9192f324c36d28136d14380f0b552a1385b59b';
+
+  private $parsedTargets = array();
+
+  public function getLinterName() {
+    return 'Howtoeven';
+  }
+
+  protected function getSeverity($code) {
+    $severities = array(
+      ArcanistLintSeverity::SEVERITY_DISABLED,
+      ArcanistLintSeverity::SEVERITY_ADVICE,
+      ArcanistLintSeverity::SEVERITY_WARNING,
+      ArcanistLintSeverity::SEVERITY_ERROR,
+    );
+    return idx($severities, $code, ArcanistLintSeverity::SEVERITY_WARNING);
+  }
+
+  public function willLintPaths(array $paths) {
+    // Cleanup previous runs.
+    $this->localExecx("rm -rf _build/_lint");
+
+    // Build compilation database.
+    $lintable_paths = $this->getLintablePaths($paths);
+    $interesting_paths = $this->getInterestingPaths($lintable_paths);
+
+    if (!$lintable_paths) {
+      return;
+    }
+
+    // Run lint.
+    try {
+      $this->localExecx(
+        "%C %C -p _build/dev/ %Ls",
+        $this->getBinaryPath(),
+        $this->getFilteredIssues(),
+        $lintable_paths);
+    } catch (CommandException $exception) {
+      PhutilConsole::getConsole()->writeErr($exception->getMessage());
+    }
+
+    // Load results.
+    $result = id(
+      new SQLite3(
+        $this->getProjectRoot().'/_build/_lint/lint.db',
+        SQLITE3_OPEN_READONLY))
+      ->query("SELECT * FROM raised_issues");
+
+    while ($issue = $result->fetchArray(SQLITE3_ASSOC)) {
+      // Skip issues not part of the linted file.
+      if (in_array($issue['file'], $interesting_paths)) {
+        $this->addLintMessage(id(new ArcanistLintMessage())
+          ->setPath($issue['file'])
+          ->setLine($issue['line'])
+          ->setChar($issue['column'])
+          ->setCode('Howtoeven')
+          ->setSeverity($this->getSeverity($issue['severity']))
+          ->setName('Hte-'.$issue['name'])
+          ->setDescription(
+            sprintf(
+              "%s\n\n%s",
+              ($issue['message']) ? $issue['message'] : $issue['description'],
+              $issue['explanation']))
+          ->setOriginalText(idx($issue, 'original', ''))
+          ->setReplacementText(idx($issue, 'replacement', '')));
+      }
+    }
+  }
+
+  public function lintPath($path) {
+  }
+
+  /**
+   * Get the paths that we know how to lint.
+   *
+   * The strategy is to first look whether there's an existing compilation
+   * database and use that if it's exhaustive. We generate our own only if
+   * necessary.
+   */
+  private function getLintablePaths($paths) {
+    // Replace headers with existing sources.
+    for ($i = 0; $i < count($paths); $i++) {
+      if (preg_match("/\.h$/", $paths[$i])) {
+        $header = preg_replace("/\.h$/", ".cpp", $paths[$i]);
+        if (file_exists($header)) {
+          $paths[$i] = $header;
+        }
+      }
+    }
+
+    // Check if database exists and is exhaustive.
+    $available_paths = $this->getAvailablePaths();
+    $lintable_paths = array_intersect($paths, $available_paths);
+    if ($paths === $lintable_paths) {
+      return $lintable_paths;
+    }
+
+    // Generate our own database.
+    $targets = $this->getTargetsFor($paths);
+    if (!$targets) {
+      PhutilConsole::getConsole()->writeErr(
+        "No build targets found for %s\n",
+        implode(', ', $paths));
+      return array();
+    }
+
+    $this->localExecx("./tools/build/bin/fbconfig.par -r %Ls", $targets);
+    $this->localExecx("./tools/build/bin/fbmake.par gen_cdb");
+
+    $available_paths = $this->getAvailablePaths();
+    $lintable_paths = array_intersect($paths, $available_paths);
+    if ($paths != $lintable_paths) {
+      PhutilConsole::getConsole()->writeErr(
+        "Can't lint %s\n",
+        implode(', ', array_diff($paths, $available_paths)));
+    }
+
+    // Return what we know how to lint.
+    return $lintable_paths;
+  }
+
+  /**
+   * Get the available paths in the current compilation database.
+   */
+  private function getAvailablePaths() {
+    $database_path = $this->getProjectRoot()
+      .'/_build/dev/compile_commands.json';
+    if (!file_exists($database_path)) {
+      return array();
+    }
+
+    $entries = json_decode(file_get_contents($database_path), true);
+    $paths = array();
+    foreach ($entries as $entry) {
+      $paths[] = $entry['file'];
+    }
+    return $paths;
+  }
+
+  /**
+   * Search for the targets directories for the given files.
+   */
+  private static function getTargetsFor($paths) {
+    $targets = array();
+    foreach ($paths as $path) {
+      while (($path = dirname($path)) !== '.') {
+        if (in_array('TARGETS', scandir($path))) {
+          $contents = file_get_contents($path.'/TARGETS');
+          if (strpos($contents, 'cpp_binary') !== false) {
+            $targets[] = $path;
+            break;
+          }
+        }
+      }
+    }
+    return array_unique($targets);
+  }
+
+  /**
+   * The paths that we actually want to report on.
+   */
+  private function getInterestingPaths($paths) {
+    $headers = array();
+    foreach ($paths as $path) {
+      $headers[] = preg_replace("/\.cpp$/", ".h", $path);
+    }
+    return array_merge($paths, $headers);
+  }
+
+  /**
+   * The path where the binary is located. Will return the current dewey binary
+   * unless the `HOWTOEVEN_BUILD` environment variable is set.
+   */
+  private function getBinaryPath() {
+    $path = sprintf(
+      "/mnt/dewey/fbcode/.commits/%s/builds/howtoeven/client",
+      self::VERSION);
+
+    $build = getenv('HOWTOEVEN_BUILD');
+    if ($build) {
+      $path = sprintf(
+        "./_build/%s/tools/howtoeven/client",
+        $build);
+      if (!file_exists($path)) {
+        PhutilConsole::getConsole()->writeErr(">> %s does not exist\n", $path);
+        exit(1);
+      }
+    }
+
+    return $path;
+  }
+
+  /**
+   * Execute the command in the root directory.
+   */
+  private function localExecx($command /* , ... */) {
+    $arguments = func_get_args();
+    return newv('ExecFuture', $arguments)
+      ->setCWD($this->getProjectRoot())
+      ->resolvex();
+  }
+
+  /**
+   * The root of the project.
+   */
+  private function getProjectRoot() {
+    return $this->getEngine()->getWorkingCopy()->getProjectRoot();
+  }
+
+  private function getFilteredIssues() {
+    $issues = getenv('HOWTOEVEN_ISSUES');
+    return ($issues) ? csprintf('-issues %s', $issues) : '';
+  }
+
+}

arcanist_util/cpp_linter/FbcodeClangFormatLinter.php

@@ -0,0 +1,58 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FbcodeClangFormatLinter extends BaseDirectoryScopedFormatLinter {
+
+  const LINT_FORMATTING = 1;
+  const CLANG_FORMAT_BINARY = '/mnt/vol/engshare/admin/scripts/clang-format';
+
+  protected function getPathsToLint() {
+    return array('');
+  }
+
+  public function getLinterName() {
+    return 'CLANG_FORMAT';
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::LINT_FORMATTING => ArcanistLintSeverity::SEVERITY_ADVICE,
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::LINT_FORMATTING => pht('Changes are not clang-formatted'),
+    );
+  }
+
+  protected function getFormatFuture($path, array $changed) {
+    $args = "";
+    foreach ($changed as $key => $value) {
+      $args .= " --lines=$key:$key";
+    }
+
+    $binary = self::CLANG_FORMAT_BINARY;
+    if (!file_exists($binary)) {
+      // trust the $PATH
+      $binary = "clang-format";
+    }
+
+    return new ExecFuture(
+      "%s %s $args",
+      $binary,
+      $this->getEngine()->getFilePathOnDisk($path));
+  }
+
+  protected function getLintMessage($diff) {
+    $link_to_clang_format =
+      "[[ http://fburl.com/clang-format | clang-format ]]";
+    return <<<LINT_MSG
+Changes in this file were not formatted using $link_to_clang_format.
+Please run build_tools/format-diff.sh or `make format`
+LINT_MSG;
+  }
+}

arcanist_util/cpp_linter/FbcodeCppLinter.php

@@ -1,44 +1,40 @@
 <?php
+// Copyright 2004-present Facebook.  All rights reserved.
 
 class FbcodeCppLinter extends ArcanistLinter {
-  const CPPLINT      = "/home/engshare/tools/cpplint";
+  const FLINT      = "/home/engshare/tools/flint";
   const LINT_ERROR   = 1;
   const LINT_WARNING = 2;
+  const LINT_ADVICE  = 3;
   const C_FLAG = "--c_mode=true";
+
   private $rawLintOutput = array();
 
   public function willLintPaths(array $paths) {
+    if (!file_exists(self::FLINT)) {
+      return;
+    }
     $futures = array();
-    $ret_value = 0;
-    $last_line = system("which cpplint", $ret_value);
-    $CPP_LINT = false;
-    if ($ret_value == 0) {
-      $CPP_LINT = $last_line;
-    } else if (file_exists(self::CPPLINT)) {
-      $CPP_LINT = self::CPPLINT;
-    }
-
-    if ($CPP_LINT) {
-      foreach ($paths as $p) {
-        $lpath = $this->getEngine()->getFilePathOnDisk($p);
-        $lpath_file = file($lpath);
-        if (preg_match('/\.(c)$/', $lpath) ||
-            preg_match('/-\*-.*Mode: C[; ].*-\*-/', $lpath_file[0]) ||
-            preg_match('/vim(:.*)*:\s*(set\s+)?filetype=c\s*:/', $lpath_file[0])
-            ) {
-          $futures[$p] = new ExecFuture("%s %s %s 2>&1",
-                             $CPP_LINT, self::C_FLAG,
-                             $this->getEngine()->getFilePathOnDisk($p));
-        } else {
-          $futures[$p] = new ExecFuture("%s %s 2>&1",
-            $CPP_LINT, $this->getEngine()->getFilePathOnDisk($p));
-        }
-      }
-
-      foreach (Futures($futures)->limit(8) as $p => $f) {
-        $this->rawLintOutput[$p] = $f->resolvex();
+    foreach ($paths as $p) {
+      $lpath = $this->getEngine()->getFilePathOnDisk($p);
+      $lpath_file = file($lpath);
+      if (preg_match('/\.(c)$/', $lpath) ||
+          preg_match('/-\*-.*Mode: C[; ].*-\*-/', $lpath_file[0]) ||
+          preg_match('/vim(:.*)*:\s*(set\s+)?filetype=c\s*:/', $lpath_file[0])
+          ) {
+        $futures[$p] = new ExecFuture("%s %s %s 2>&1",
+                           self::FLINT, self::C_FLAG,
+                           $this->getEngine()->getFilePathOnDisk($p));
+      } else {
+        $futures[$p] = new ExecFuture("%s %s 2>&1",
+          self::FLINT, $this->getEngine()->getFilePathOnDisk($p));
       }
     }
+
+    foreach (Futures($futures)->limit(8) as $p => $f) {
+      $this->rawLintOutput[$p] = $f->resolvex();
+    }
+
     return;
   }
 
@@ -47,44 +43,70 @@ class FbcodeCppLinter extends ArcanistLinter {
   }
 
   public function lintPath($path) {
+    $this->runCppLint($path);
+  }
+
+  private function runCppLint($path) {
     $msgs = $this->getCppLintOutput($path);
     foreach ($msgs as $m) {
       $this->raiseLintAtLine($m['line'], 0, $m['severity'], $m['msg']);
     }
   }
 
+  private function adviseOnEachPattern(
+    $path,
+    $regex,
+    $message,
+    $lint_type = self::LINT_ADVICE,
+    $match_idx = 0) {
+      $file_data = $this->getData($path);
+      $matches = array();
+      if (!preg_match_all($regex, $file_data, $matches, PREG_OFFSET_CAPTURE)) {
+        return;
+      }
+
+      foreach ($matches[$match_idx] as $match) {
+        list($match_str, $offset) = $match;
+        $this->raiseLintAtOffset($offset, $lint_type, $message, $match_str);
+      }
+  }
+
   public function getLintSeverityMap() {
     return array(
       self::LINT_WARNING => ArcanistLintSeverity::SEVERITY_WARNING,
+      self::LINT_ADVICE  => ArcanistLintSeverity::SEVERITY_ADVICE,
       self::LINT_ERROR   => ArcanistLintSeverity::SEVERITY_ERROR
     );
   }
 
   public function getLintNameMap() {
     return array(
-      self::LINT_WARNING => "CppLint Warning",
-      self::LINT_ERROR   => "CppLint Error"
+      self::LINT_ADVICE   => "CppLint Advice",
+      self::LINT_WARNING  => "CppLint Warning",
+      self::LINT_ERROR    => "CppLint Error"
     );
   }
 
   private function getCppLintOutput($path) {
-    if (!array_key_exists($path, $this->rawLintOutput)) {
-      return array();
-    }
     list($output) = $this->rawLintOutput[$path];
 
     $msgs = array();
     $current = null;
+    $matches = array();
     foreach (explode("\n", $output) as $line) {
-      if (preg_match('/[^:]*\((\d+)\):(.*)$/', $line, $matches)) {
+      if (preg_match('/.*?:(\d+):(.*)/', $line, $matches)) {
         if ($current) {
           $msgs[] = $current;
         }
         $line = $matches[1];
         $text = $matches[2];
-        $sev  = preg_match('/.*Warning.*/', $text)
-                  ? self::LINT_WARNING
-                  : self::LINT_ERROR;
+        if (preg_match('/.*Warning.*/', $text)) {
+          $sev = self::LINT_WARNING;
+        } else if (preg_match('/.*Advice.*/', $text)) {
+          $sev = self::LINT_ADVICE;
+        } else {
+          $sev = self::LINT_ERROR;
+        }
         $current = array('line'     => $line,
                          'msg'      => $text,
                          'severity' => $sev);
@@ -99,4 +121,3 @@ class FbcodeCppLinter extends ArcanistLinter {
     return $msgs;
   }
 }
-

arcanist_util/cpp_linter/PfffCppLinter.php

@@ -1,68 +0,0 @@
-<?php
-// Copyright 2004-present Facebook.  All rights reserved.
-
-class PfffCppLinter extends ArcanistLinter {
-  const PROGRAM      = "/home/engshare/tools/checkCpp";
-
-  public function getLinterName() {
-    return "checkCpp";
-  }
-  public function getLintNameMap() {
-    return array(
-    );
-  }
-
-  public function getLintSeverityMap() {
-    return array(
-    );
-  }
-
-  public function willLintPaths(array $paths) {
-    $program = false;
-    $ret_value = 0;
-    $last_line = system("which checkCpp", $ret_value);
-    if ($ret_value == 0) {
-      $program = $last_line;
-    } else if (file_exists(self::PROGRAM)) {
-      $program = self::PROGRAM;
-    }
-    if ($program) {
-      $futures = array();
-      foreach ($paths as $p) {
-        $futures[$p] = new ExecFuture("%s --lint %s 2>&1",
-          $program, $this->getEngine()->getFilePathOnDisk($p));
-      }
-      foreach (Futures($futures)->limit(8) as $p => $f) {
-
-        list($stdout, $stderr) = $f->resolvex();
-        $raw = json_decode($stdout, true);
-        if (!is_array($raw)) {
-          throw new Exception(
-            "checkCpp returned invalid JSON!".
-            "Stdout: {$stdout} Stderr: {$stderr}"
-          );
-        }
-        foreach($raw as $err) {
-          $this->addLintMessage(
-            ArcanistLintMessage::newFromDictionary(
-              array(
-                'path' => $err['file'],
-                'line' => $err['line'],
-                'char' => 0,
-                'name' => $err['name'],
-                'description' => $err['info'],
-                'code' => $this->getLinterName(),
-                'severity' => ArcanistLintSeverity::SEVERITY_WARNING,
-              )
-            )
-          );
-        }
-      }
-    }
-    return;
-  }
-
-  public function lintPath($path) {
-    return;
-  }
-}

arcanist_util/lint_engine/FacebookFbcodeLintEngine.php

@@ -1,5 +1,8 @@
 <?php
 // Copyright 2004-present Facebook.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
 
 class FacebookFbcodeLintEngine extends ArcanistLintEngine {
 
@@ -39,14 +42,13 @@ class FacebookFbcodeLintEngine extends ArcanistLintEngine {
     $python_linter = new ArcanistPEP8Linter();
     $linters[] = $python_linter;
 
-   // Currently we can't run cpplint in commit hook mode, because it
-    // depends on having access to the working directory.
     if (!$this->getCommitHookMode()) {
       $cpp_linters = array();
-      $google_linter = new ArcanistCpplintLinter();
-      $cpp_linters[] = $linters[] = $google_linter;
+      $cpp_linters[] = $linters[] = new ArcanistCpplintLinter();
       $cpp_linters[] = $linters[] = new FbcodeCppLinter();
-      $cpp_linters[] = $linters[] = new PfffCppLinter();
+
+      $clang_format_linter = new FbcodeClangFormatLinter();
+      $linters[] = $clang_format_linter;
     }
 
     $spelling_linter = new ArcanistSpellingLinter();
@@ -93,6 +95,11 @@ class FacebookFbcodeLintEngine extends ArcanistLintEngine {
           $linter->addPath($path);
           $linter->addData($path, $this->loadData($path));
         }
+
+        $clang_format_linter->addPath($path);
+        $clang_format_linter->addData($path, $this->loadData($path));
+        $clang_format_linter->setPathChangedLines(
+          $path, $this->getPathChangedLines($path));
       }
 
       // Match *.py and contbuild config files

arcanist_util/lint_engine/FacebookHowtoevenLintEngine.php

@@ -0,0 +1,27 @@
+<?php
+// Copyright 2015-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FacebookHowtoevenLintEngine extends ArcanistLintEngine {
+
+  public function buildLinters() {
+    $paths = array();
+
+    foreach ($this->getPaths() as $path) {
+      // Don't try to lint deleted files or changed directories.
+      if (!Filesystem::pathExists($path) || is_dir($path)) {
+        continue;
+      }
+
+      if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)) {
+        $paths[] = $path;
+      }
+    }
+
+    $howtoeven = new FacebookHowtoevenLinter();
+    $howtoeven->setPaths($paths);
+    return array($howtoeven);
+  }
+}

arcanist_util/unit_engine/FacebookFbcodeUnitTestEngine.php

@@ -1,5 +1,8 @@
 <?php
 // Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
 
 class FacebookFbcodeUnitTestEngine extends ArcanistBaseUnitTestEngine {
 

build_tools/build_detect_platform

@@ -8,6 +8,7 @@
 #   CXX                         C++ Compiler path
 #   PLATFORM_LDFLAGS            Linker flags
 #   JAVA_LDFLAGS                Linker flags for RocksDBJava
+#   JAVA_STATIC_LDFLAGS         Linker flags for RocksDBJava static build
 #   PLATFORM_SHARED_EXT         Extension for shared libraries
 #   PLATFORM_SHARED_LDFLAGS     Flags for building shared library
 #   PLATFORM_SHARED_CFLAGS      Flags for compiling objects for shared library
@@ -22,6 +23,7 @@
 #       -DLEVELDB_PLATFORM_NOATOMIC if it is not
 #       -DSNAPPY                    if the Snappy library is present
 #       -DLZ4                       if the LZ4 library is present
+#       -DZSTD                      if the ZSTD library is present
 #       -DNUMA                      if the NUMA library is present
 #
 # Using gflags in rocksdb:
@@ -180,6 +182,7 @@ esac
 
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
 JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
+JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"
 
 if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
     # Cross-compiling; do not try any compilation tests.
@@ -269,6 +272,17 @@ EOF
         JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
     fi
 
+    # Test whether zstd library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <zstd.h>
+      int main() {}
+EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
+    fi
+
     # Test whether numa is available
     $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
       #include <numa.h>
@@ -362,6 +376,7 @@ echo "CXX=$CXX" >> "$OUTPUT"
 echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
 echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
 echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
+echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
 echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
 echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
 echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"

build_tools/fbcode_config.sh

@@ -42,6 +42,11 @@ if test -z $PIC_BUILD; then
   LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/79d2943e2dd7208a3e0b06cf95e9f85f05fe9e1b/r124/gcc-4.9-glibc-2.20/4230243/include/"
   LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/79d2943e2dd7208a3e0b06cf95e9f85f05fe9e1b/r124/gcc-4.9-glibc-2.20/4230243/lib/liblz4.a"
   CFLAGS+=" -DLZ4"
+
+  ZSTD_REV=8df2d01673ae6afcc8c8d16fec862b2d67ecc1e9
+  ZSTD_INCLUDE=" -I /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.1.1/gcc-4.8.1-glibc-2.17/c3f970a/include"
+  ZSTD_LIBS=" /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.1.1/gcc-4.8.1-glibc-2.17/c3f970a/lib/libzstd.a"
+  CFLAGS+=" -DZSTD"
 fi
 
 # location of gflags headers and libraries
@@ -73,7 +78,7 @@ export USE_SSE=1
 BINUTILS="/mnt/gvfs/third-party2/binutils/0b6ad0c88ddd903333a48ae8bff134efac468e4a/2.25/centos6-native/da39a3e/bin"
 AR="$BINUTILS/ar"
 
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
 
 GCC_BASE="/mnt/gvfs/third-party2/gcc/1c67a0b88f64d4d9ced0382d141c76aaa7d62fba/4.9.x/centos6-native/1317bc4"
 STDLIBS="-L $GCC_BASE/lib64"
@@ -114,14 +119,14 @@ CFLAGS+=" $DEPS_INCLUDE"
 CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
 CXXFLAGS+=" $CFLAGS"
 
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
 EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.9-glibc-2.20/lib"
 
 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
 
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
 
 VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/6c45ef049cbf11c2df593addb712cd891049e737/3.10.0/gcc-4.9-glibc-2.20/4230243/bin/"
 

build_tools/fbcode_config4.8.1.sh

@@ -30,6 +30,10 @@ LZ4_REV=065ec7e38fe83329031f6668c43bef83eff5808b
 LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/include"
 LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a"
 
+ZSTD_REV=8df2d01673ae6afcc8c8d16fec862b2d67ecc1e9
+ZSTD_INCLUDE=" -I /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.1.1/gcc-4.8.1-glibc-2.17/c3f970a/include"
+ZSTD_LIBS=" /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.1.1/gcc-4.8.1-glibc-2.17/c3f970a/lib/libzstd.a"
+
 # location of gflags headers and libraries
 GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/1ad047a6e6f6673991918ecadc670868205a243a/1.6/gcc-4.8.1-glibc-2.17/c3f970a/include/"
 GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/1ad047a6e6f6673991918ecadc670868205a243a/1.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libgflags.a"
@@ -53,7 +57,7 @@ export USE_SSE=1
 BINUTILS="/mnt/gvfs/third-party2/binutils/2aff2e7b474cd3e6ab23495ad1224b7d214b9f8e/2.21.1/centos6-native/da39a3e/bin"
 AR="$BINUTILS/ar"
 
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
 
 GCC_BASE="/mnt/gvfs/third-party2/gcc/1ec615e23800f0815d474478ba476a0adc3fe788/4.8.1/centos6-native/cc6c9dc"
 STDLIBS="-L $GCC_BASE/lib64"
@@ -88,17 +92,17 @@ fi
 
 CFLAGS+=" $DEPS_INCLUDE"
 CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
-CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DNUMA"
+CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA"
 CXXFLAGS+=" $CFLAGS"
 
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
 EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"
 
 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
 
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
 
 VALGRIND_REV=b2a9f85e4b70cd03abc85a7f3027fbc4cef35bd0
 VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/$VALGRIND_REV/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a/bin/"

build_tools/format-diff.sh

@@ -79,6 +79,12 @@ echo -e "Detect lines that doesn't follow the format rules:\r"
 echo "$diffs" | 
   sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" |
   sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/"
+
+if [[ "$OPT" == *"-DTRAVIS"* ]]
+then
+  exit 1
+fi
+
 echo -e "Would you like to fix the format automatically (y/n): \c"
 
 # Make sure under any mode, we can read user input.

build_tools/rocksdb-lego-determinator

@@ -0,0 +1,259 @@
+#!/bin/bash
+# This script is executed by Sandcastle
+# to determine next steps to run
+
+#
+# Helper variables
+#
+REPORT_EMAIL="'report': [
+    {
+        'type':'email',
+        'triggers':['fail'],
+        'emails':['leveldb@fb.com']
+    }
+]"
+
+CLEANUP_ENV="
+{
+  'name':'Cleanup environment',
+  'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && make clean',
+  'user':'root'
+}"
+
+DEBUG="OPT=-g"
+SHM="TEST_TMPDIR=/dev/shm/rocksdb"
+GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
+ASAN="COMPILE_WITH_ASAN=1"
+CLANG="USE_CLANG=1"
+LITE="OPT=-DROCKSDB_LITE"
+TSAN="COMPILE_WITH_TSAN=1"
+
+#
+# RocksDB unit test in parallel
+# Currently we always have noise in our parallel runs. This job is to help
+# manage the noise
+#
+PARALLEL_UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Parallel Unit Test',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version',
+                'shell':'$SHM $DEBUG make check 2>&1 > /dev/null || cat t/log-*',
+                'user':'root'
+            },
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version under gcc-4.8.1',
+                'shell':'$SHM $GCC_481 $DEBUG make check 2>&1 > /dev/null || cat t/log-*',
+                'user':'root'
+            },
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+#
+# RocksDB unit test
+#
+UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version',
+                'shell':'$SHM $DEBUG make J=1 check',
+                'user':'root'
+            },
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+#
+# RocksDB unit test on gcc-4.8.1
+#
+UNIT_TEST_COMMANDS_481="[
+    {
+        'name':'Rocksdb Unit Test on GCC 4.8.1',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version',
+                'shell':'$SHM $GCC_481 $DEBUG make J=1 check',
+                'user':'root'
+            },
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+#
+# RocksDB unit test with CLANG
+#
+CLANG_UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug',
+                'shell':'$CLANG $SHM $DEBUG make check',
+                'user':'root'
+            },
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+#
+# Build RocksDB lite
+#
+LITE_BUILD_COMMANDS="[
+    {
+        'name':'Rocksdb Lite build',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build RocksDB debug version',
+                'shell':'$LITE $DEBUG make J=1 static_lib',
+                'user':'root'
+            },
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+#
+# RocksDB stress/crash test
+#
+STRESS_CRASH_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Stress/Crash Test',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and run RocksDB debug stress tests',
+                'shell':'$SHM $DEBUG make J=1 db_stress',
+                'user':'root'
+            },
+            {
+                'name':'Build and run RocksDB debug crash tests',
+                'shell':'$SHM $DEBUG make J=1 crash_test',
+                'user':'root'
+            }
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+
+#
+# RocksDB test under address sanitizer
+#
+ASAN_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under ASAN',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Test RocksDB debug under ASAN',
+                'shell':'set -o pipefail && $SHM $ASAN $DEBUG make J=1 asan_check |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
+                'user':'root'
+            }
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+#
+# RocksDB crash testing under address sanitizer
+#
+ASAN_CRASH_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb crash test under ASAN',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and run RocksDB debug asan_crash_test',
+                'shell':'$SHM $DEBUG make J=1 asan_crash_test',
+                'user':'root'
+            },
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+#
+# RocksDB unit test under valgrind
+#
+VALGRIND_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under valgrind',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Run RocksDB debug unit tests',
+                'shell':'$SHM $DEBUG ./build_tools/valgrind_test.sh',
+                'user':'root'
+            },
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+#
+# RocksDB test under TSAN
+#
+TSAN_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under TSAN',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Run RocksDB debug unit test',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN make J=1 check',
+                'user':'root'
+            },
+        ],
+        $REPORT_EMAIL
+    }
+]"
+
+case $1 in
+  punit)
+    echo $PARALLEL_UNIT_TEST_COMMANDS
+    ;;
+  unit)
+    echo $UNIT_TEST_COMMANDS
+    ;;
+  unit_481)
+    echo $UNIT_TEST_COMMANDS_481
+    ;;
+  clang_unit)
+    echo $CLANG_UNIT_TEST_COMMANDS
+    ;;
+  lite)
+    echo $LITE_BUILD_COMMANDS
+    ;;
+  stress_crash)
+    echo $STRESS_CRASH_TEST_COMMANDS
+    ;;
+  asan)
+    echo $ASAN_TEST_COMMANDS
+    ;;
+  asan_crash)
+    echo $ASAN_CRASH_TEST_COMMANDS
+    ;;
+  valgrind)
+    echo $VALGRIND_TEST_COMMANDS
+    ;;
+  tsan)
+    echo $TSAN_TEST_COMMANDS
+    ;;
+  *)
+    echo "Invalid determinator command"
+    ;;
+esac

db/builder.cc

@@ -9,9 +9,14 @@
 
 #include "db/builder.h"
 
+#include <algorithm>
+#include <deque>
 #include <vector>
+
+#include "db/compaction_iterator.h"
 #include "db/dbformat.h"
 #include "db/filename.h"
+#include "db/internal_stats.h"
 #include "db/merge_helper.h"
 #include "db/table_cache.h"
 #include "db/version_edit.h"
@@ -23,8 +28,8 @@
 #include "table/block_based_table_builder.h"
 #include "util/file_reader_writer.h"
 #include "util/iostats_context_imp.h"
-#include "util/thread_status_util.h"
 #include "util/stop_watch.h"
+#include "util/thread_status_util.h"
 
 namespace rocksdb {
 
@@ -50,26 +55,16 @@ Status BuildTable(
     FileMetaData* meta, const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    const SequenceNumber newest_snapshot,
-    const SequenceNumber earliest_seqno_in_memtable,
-    const CompressionType compression,
+    std::vector<SequenceNumber> snapshots, const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
-    const Env::IOPriority io_priority, TableProperties* table_properties) {
+    InternalStats* internal_stats, const Env::IOPriority io_priority,
+    TableProperties* table_properties) {
   // Reports the IOStats for flush for every following bytes.
   const size_t kReportFlushIOStatsEvery = 1048576;
   Status s;
   meta->fd.file_size = 0;
-  meta->smallest_seqno = meta->largest_seqno = 0;
   iter->SeekToFirst();
 
-  // If the sequence number of the smallest entry in the memtable is
-  // smaller than the most recent snapshot, then we do not trigger
-  // removal of duplicate/deleted keys as part of this builder.
-  bool purge = true;
-  if (earliest_seqno_in_memtable <= newest_snapshot) {
-    purge = false;
-  }
-
   std::string fname = TableFileName(ioptions.db_paths, meta->fd.GetNumber(),
                                     meta->fd.GetPathId());
   if (iter->Valid()) {
@@ -90,138 +85,31 @@ Status BuildTable(
           file_writer.get(), compression, compression_opts);
     }
 
-    {
-      // the first key is the smallest key
-      Slice key = iter->key();
-      meta->smallest.DecodeFrom(key);
-      meta->smallest_seqno = GetInternalKeySeqno(key);
-      meta->largest_seqno = meta->smallest_seqno;
-    }
-
     MergeHelper merge(internal_comparator.user_comparator(),
                       ioptions.merge_operator, ioptions.info_log,
                       ioptions.min_partial_merge_operands,
                       true /* internal key corruption is not ok */);
 
-    if (purge) {
-      // Ugly walkaround to avoid compiler error for release build
-      bool ok __attribute__((unused)) = true;
+    CompactionIterator c_iter(iter, internal_comparator.user_comparator(),
+                              &merge, kMaxSequenceNumber, &snapshots, env,
+                              true /* internal key corruption is not ok */);
+    c_iter.SeekToFirst();
+    for (; c_iter.Valid(); c_iter.Next()) {
+      const Slice& key = c_iter.key();
+      const Slice& value = c_iter.value();
+      builder->Add(key, value);
+      meta->UpdateBoundaries(key, c_iter.ikey().sequence);
 
-      // Will write to builder if current key != prev key
-      ParsedInternalKey prev_ikey;
-      std::string prev_key;
-      bool is_first_key = true;    // Also write if this is the very first key
-
-      while (iter->Valid()) {
-        bool iterator_at_next = false;
-
-        // Get current key
-        ParsedInternalKey this_ikey;
-        Slice key = iter->key();
-        Slice value = iter->value();
-
-        // In-memory key corruption is not ok;
-        // TODO: find a clean way to treat in memory key corruption
-        ok = ParseInternalKey(key, &this_ikey);
-        assert(ok);
-        assert(this_ikey.sequence >= earliest_seqno_in_memtable);
-
-        // If the key is the same as the previous key (and it is not the
-        // first key), then we skip it, since it is an older version.
-        // Otherwise we output the key and mark it as the "new" previous key.
-        if (!is_first_key && !internal_comparator.user_comparator()->Compare(
-                                  prev_ikey.user_key, this_ikey.user_key)) {
-          // seqno within the same key are in decreasing order
-          assert(this_ikey.sequence < prev_ikey.sequence);
-        } else {
-          is_first_key = false;
-
-          if (this_ikey.type == kTypeMerge) {
-            // TODO(tbd): Add a check here to prevent RocksDB from crash when
-            // reopening a DB w/o properly specifying the merge operator.  But
-            // currently we observed a memory leak on failing in RocksDB
-            // recovery, so we decide to let it crash instead of causing
-            // memory leak for now before we have identified the real cause
-            // of the memory leak.
-
-            // Handle merge-type keys using the MergeHelper
-            // TODO: pass statistics to MergeUntil
-            merge.MergeUntil(iter, 0 /* don't worry about snapshot */);
-            iterator_at_next = true;
-            if (merge.IsSuccess()) {
-              // Merge completed correctly.
-              // Add the resulting merge key/value and continue to next
-              builder->Add(merge.key(), merge.value());
-              prev_key.assign(merge.key().data(), merge.key().size());
-              ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
-              assert(ok);
-            } else {
-              // Merge did not find a Put/Delete.
-              // Can not compact these merges into a kValueType.
-              // Write them out one-by-one. (Proceed back() to front())
-              const std::deque<std::string>& keys = merge.keys();
-              const std::deque<std::string>& values = merge.values();
-              assert(keys.size() == values.size() && keys.size() >= 1);
-              std::deque<std::string>::const_reverse_iterator key_iter;
-              std::deque<std::string>::const_reverse_iterator value_iter;
-              for (key_iter=keys.rbegin(), value_iter = values.rbegin();
-                   key_iter != keys.rend() && value_iter != values.rend();
-                   ++key_iter, ++value_iter) {
-
-                builder->Add(Slice(*key_iter), Slice(*value_iter));
-              }
-
-              // Sanity check. Both iterators should end at the same time
-              assert(key_iter == keys.rend() && value_iter == values.rend());
-
-              prev_key.assign(keys.front());
-              ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
-              assert(ok);
-            }
-          } else {
-            // Handle Put/Delete-type keys by simply writing them
-            builder->Add(key, value);
-            prev_key.assign(key.data(), key.size());
-            ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
-            assert(ok);
-          }
-        }
-
-        if (io_priority == Env::IO_HIGH &&
-            IOSTATS(bytes_written) >= kReportFlushIOStatsEvery) {
-          ThreadStatusUtil::IncreaseThreadOperationProperty(
-              ThreadStatus::FLUSH_BYTES_WRITTEN,
-              IOSTATS(bytes_written));
-          IOSTATS_RESET(bytes_written);
-        }
-        if (!iterator_at_next) iter->Next();
-      }
-
-      // The last key is the largest key
-      meta->largest.DecodeFrom(Slice(prev_key));
-      SequenceNumber seqno = GetInternalKeySeqno(Slice(prev_key));
-      meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
-      meta->largest_seqno = std::max(meta->largest_seqno, seqno);
-
-    } else {
-      for (; iter->Valid(); iter->Next()) {
-        Slice key = iter->key();
-        meta->largest.DecodeFrom(key);
-        builder->Add(key, iter->value());
-        SequenceNumber seqno = GetInternalKeySeqno(key);
-        meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
-        meta->largest_seqno = std::max(meta->largest_seqno, seqno);
-        if (io_priority == Env::IO_HIGH &&
-            IOSTATS(bytes_written) >= kReportFlushIOStatsEvery) {
-          ThreadStatusUtil::IncreaseThreadOperationProperty(
-              ThreadStatus::FLUSH_BYTES_WRITTEN,
-              IOSTATS(bytes_written));
-          IOSTATS_RESET(bytes_written);
-        }
+      // TODO(noetzli): Update stats after flush, too.
+      if (io_priority == Env::IO_HIGH &&
+          IOSTATS(bytes_written) >= kReportFlushIOStatsEvery) {
+        ThreadStatusUtil::SetThreadOperationProperty(
+            ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
       }
     }
 
     // Finish and check for builder errors
+    s = c_iter.status();
     if (s.ok()) {
       s = builder->Finish();
     } else {
@@ -248,8 +136,11 @@ Status BuildTable(
 
     if (s.ok()) {
       // Verify that the table is usable
-      Iterator* it = table_cache->NewIterator(ReadOptions(), env_options,
-                                              internal_comparator, meta->fd);
+      Iterator* it = table_cache->NewIterator(
+          ReadOptions(), env_options, internal_comparator, meta->fd, nullptr,
+          (internal_stats == nullptr) ? nullptr
+                                      : internal_stats->GetFileReadHist(0),
+          false);
       s = it->status();
       if (s.ok() && paranoid_file_checks) {
         for (it->SeekToFirst(); it->Valid(); it->Next()) {}

db/builder.h

@@ -30,6 +30,7 @@ class TableCache;
 class VersionEdit;
 class TableBuilder;
 class WritableFileWriter;
+class InternalStats;
 
 TableBuilder* NewTableBuilder(
     const ImmutableCFOptions& options,
@@ -51,10 +52,9 @@ extern Status BuildTable(
     FileMetaData* meta, const InternalKeyComparator& internal_comparator,
     const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
         int_tbl_prop_collector_factories,
-    const SequenceNumber newest_snapshot,
-    const SequenceNumber earliest_seqno_in_memtable,
-    const CompressionType compression,
+    std::vector<SequenceNumber> snapshots, const CompressionType compression,
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
+    InternalStats* internal_stats,
     const Env::IOPriority io_priority = Env::IO_HIGH,
     TableProperties* table_properties = nullptr);
 

db/column_family.cc

@@ -21,13 +21,12 @@
 
 #include "db/compaction_picker.h"
 #include "db/db_impl.h"
-#include "db/job_context.h"
-#include "db/version_set.h"
-#include "db/writebuffer.h"
 #include "db/internal_stats.h"
+#include "db/job_context.h"
 #include "db/table_properties_collector.h"
 #include "db/version_set.h"
 #include "db/write_controller.h"
+#include "db/writebuffer.h"
 #include "util/autovector.h"
 #include "util/compression.h"
 #include "util/hash_skiplist_rep.h"
@@ -126,7 +125,12 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
   // if user sets arena_block_size, we trust user to use this value. Otherwise,
   // calculate a proper value from writer_buffer_size;
   if (result.arena_block_size <= 0) {
-    result.arena_block_size = result.write_buffer_size / 10;
+    result.arena_block_size = result.write_buffer_size / 8;
+
+    // Align up to 4k
+    const size_t align = 4 * 1024;
+    result.arena_block_size =
+        ((result.arena_block_size + align - 1) / align) * align;
   }
   result.min_write_buffer_number_to_merge =
       std::min(result.min_write_buffer_number_to_merge,
@@ -242,6 +246,9 @@ void SuperVersion::Cleanup() {
   imm->Unref(&to_delete);
   MemTable* m = mem->Unref();
   if (m != nullptr) {
+    auto* memory_usage = current->cfd()->imm()->current_memory_usage();
+    assert(*memory_usage >= m->ApproximateMemoryUsage());
+    *memory_usage -= m->ApproximateMemoryUsage();
     to_delete.push_back(m);
   }
   current->Unref();
@@ -476,6 +483,10 @@ uint64_t ColumnFamilyData::GetNumLiveVersions() const {
   return VersionSet::GetNumLiveVersions(dummy_versions_);
 }
 
+uint64_t ColumnFamilyData::GetTotalSstFilesSize() const {
+  return VersionSet::GetTotalSstFilesSize(dummy_versions_);
+}
+
 MemTable* ColumnFamilyData::ConstructNewMemtable(
     const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
   assert(current_ != nullptr);

db/column_family.h

@@ -226,6 +226,7 @@ class ColumnFamilyData {
   Version* dummy_versions() { return dummy_versions_; }
   void SetCurrent(Version* current);
   uint64_t GetNumLiveVersions() const;  // REQUIRE: DB mutex held
+  uint64_t GetTotalSstFilesSize() const;  // REQUIRE: DB mutex held
   void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
 
   // See Memtable constructor for explanation of earliest_seq param.

db/column_family_test.cc

@@ -10,6 +10,7 @@
 #include <algorithm>
 #include <vector>
 #include <string>
+#include <thread>
 
 #include "db/db_impl.h"
 #include "rocksdb/db.h"
@@ -19,6 +20,7 @@
 #include "util/testharness.h"
 #include "util/testutil.h"
 #include "util/coding.h"
+#include "util/sync_point.h"
 #include "utilities/merge_operators.h"
 
 namespace rocksdb {
@@ -625,6 +627,7 @@ TEST_F(ColumnFamilyTest, FlushTest) {
 // Makes sure that obsolete log files get deleted
 TEST_F(ColumnFamilyTest, LogDeletionTest) {
   db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max();
+  column_family_options_.arena_block_size = 4 * 1024;
   column_family_options_.write_buffer_size = 100000;  // 100KB
   Open();
   CreateColumnFamilies({"one", "two", "three", "four"});
@@ -702,18 +705,22 @@ TEST_F(ColumnFamilyTest, DifferentWriteBufferSizes) {
   // "two" -> 1MB memtable, start flushing with three immutable memtables
   // "three" -> 90KB memtable, start flushing with four immutable memtables
   default_cf.write_buffer_size = 100000;
+  default_cf.arena_block_size = 4 * 4096;
   default_cf.max_write_buffer_number = 10;
   default_cf.min_write_buffer_number_to_merge = 1;
   default_cf.max_write_buffer_number_to_maintain = 0;
   one.write_buffer_size = 200000;
+  one.arena_block_size = 4 * 4096;
   one.max_write_buffer_number = 10;
   one.min_write_buffer_number_to_merge = 2;
   one.max_write_buffer_number_to_maintain = 1;
   two.write_buffer_size = 1000000;
+  two.arena_block_size = 4 * 4096;
   two.max_write_buffer_number = 10;
   two.min_write_buffer_number_to_merge = 3;
   two.max_write_buffer_number_to_maintain = 2;
-  three.write_buffer_size = 90000;
+  three.write_buffer_size = 4096 * 22 + 2048;
+  three.arena_block_size = 4096;
   three.max_write_buffer_number = 10;
   three.min_write_buffer_number_to_merge = 4;
   three.max_write_buffer_number_to_maintain = -1;
@@ -737,15 +744,15 @@ TEST_F(ColumnFamilyTest, DifferentWriteBufferSizes) {
   env_->SleepForMicroseconds(micros_wait_for_flush);
   AssertNumberOfImmutableMemtables({0, 1, 2, 0});
   AssertCountLiveLogFiles(4);
-  PutRandomData(3, 90, 1000);
+  PutRandomData(3, 91, 990);
   env_->SleepForMicroseconds(micros_wait_for_flush);
   AssertNumberOfImmutableMemtables({0, 1, 2, 1});
   AssertCountLiveLogFiles(5);
-  PutRandomData(3, 90, 1000);
+  PutRandomData(3, 90, 990);
   env_->SleepForMicroseconds(micros_wait_for_flush);
   AssertNumberOfImmutableMemtables({0, 1, 2, 2});
   AssertCountLiveLogFiles(6);
-  PutRandomData(3, 90, 1000);
+  PutRandomData(3, 90, 990);
   env_->SleepForMicroseconds(micros_wait_for_flush);
   AssertNumberOfImmutableMemtables({0, 1, 2, 3});
   AssertCountLiveLogFiles(7);
@@ -757,11 +764,11 @@ TEST_F(ColumnFamilyTest, DifferentWriteBufferSizes) {
   WaitForFlush(2);
   AssertNumberOfImmutableMemtables({0, 1, 0, 3});
   AssertCountLiveLogFiles(9);
-  PutRandomData(3, 90, 1000);
+  PutRandomData(3, 90, 990);
   WaitForFlush(3);
   AssertNumberOfImmutableMemtables({0, 1, 0, 0});
   AssertCountLiveLogFiles(10);
-  PutRandomData(3, 90, 1000);
+  PutRandomData(3, 90, 990);
   env_->SleepForMicroseconds(micros_wait_for_flush);
   AssertNumberOfImmutableMemtables({0, 1, 0, 1});
   AssertCountLiveLogFiles(11);
@@ -769,9 +776,9 @@ TEST_F(ColumnFamilyTest, DifferentWriteBufferSizes) {
   WaitForFlush(1);
   AssertNumberOfImmutableMemtables({0, 0, 0, 1});
   AssertCountLiveLogFiles(5);
-  PutRandomData(3, 240, 1000);
+  PutRandomData(3, 90 * 3, 990);
   WaitForFlush(3);
-  PutRandomData(3, 300, 1000);
+  PutRandomData(3, 90 * 4, 990);
   WaitForFlush(3);
   AssertNumberOfImmutableMemtables({0, 0, 0, 0});
   AssertCountLiveLogFiles(12);
@@ -779,7 +786,7 @@ TEST_F(ColumnFamilyTest, DifferentWriteBufferSizes) {
   WaitForFlush(0);
   AssertNumberOfImmutableMemtables({0, 0, 0, 0});
   AssertCountLiveLogFiles(12);
-  PutRandomData(2, 3*100, 10000);
+  PutRandomData(2, 3 * 1000, 1000);
   WaitForFlush(2);
   AssertNumberOfImmutableMemtables({0, 0, 0, 0});
   AssertCountLiveLogFiles(12);
@@ -864,7 +871,7 @@ TEST_F(ColumnFamilyTest, DifferentCompactionStyles) {
   one.num_levels = 1;
   // trigger compaction if there are >= 4 files
   one.level0_file_num_compaction_trigger = 4;
-  one.write_buffer_size = 100000;
+  one.write_buffer_size = 120000;
 
   two.compaction_style = kCompactionStyleLevel;
   two.num_levels = 4;
@@ -875,23 +882,27 @@ TEST_F(ColumnFamilyTest, DifferentCompactionStyles) {
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 1; ++i) {
-    PutRandomData(1, 11, 10000);
+    PutRandomData(1, 10, 12000);
+    PutRandomData(1, 1, 10);
     WaitForFlush(1);
     AssertFilesPerLevel(ToString(i + 1), 1);
   }
 
   // SETUP column family "two" -- level style with 4 levels
   for (int i = 0; i < two.level0_file_num_compaction_trigger - 1; ++i) {
-    PutRandomData(2, 15, 10000);
+    PutRandomData(2, 10, 12000);
+    PutRandomData(2, 1, 10);
     WaitForFlush(2);
     AssertFilesPerLevel(ToString(i + 1), 2);
   }
 
   // TRIGGER compaction "one"
-  PutRandomData(1, 12, 10000);
+  PutRandomData(1, 10, 12000);
+  PutRandomData(1, 1, 10);
 
   // TRIGGER compaction "two"
-  PutRandomData(2, 10, 10000);
+  PutRandomData(2, 10, 12000);
+  PutRandomData(2, 1, 10);
 
   // WAIT for compactions
   WaitForCompaction();
@@ -1045,6 +1056,7 @@ TEST_F(ColumnFamilyTest, FlushStaleColumnFamilies) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   default_cf.write_buffer_size = 100000;  // small write buffer size
+  default_cf.arena_block_size = 4096;
   default_cf.disable_auto_compactions = true;
   one.disable_auto_compactions = true;
   two.disable_auto_compactions = true;
@@ -1092,6 +1104,9 @@ TEST_F(ColumnFamilyTest, SanitizeOptions) {
             original.level0_stop_writes_trigger = i;
             original.level0_slowdown_writes_trigger = j;
             original.level0_file_num_compaction_trigger = k;
+            original.write_buffer_size =
+                l * 4 * 1024 * 1024 + i * 1024 * 1024 + j * 1024 + k;
+
             ColumnFamilyOptions result =
                 SanitizeOptions(db_options, nullptr, original);
             ASSERT_TRUE(result.level0_stop_writes_trigger >=
@@ -1108,6 +1123,16 @@ TEST_F(ColumnFamilyTest, SanitizeOptions) {
                 ASSERT_EQ(result.num_levels, original.num_levels);
               }
             }
+
+            // Make sure Sanitize options sets arena_block_size to 1/8 of
+            // the write_buffer_size, rounded up to a multiple of 4k.
+            size_t expected_arena_block_size =
+                l * 4 * 1024 * 1024 / 8 + i * 1024 * 1024 / 8;
+            if (j + k != 0) {
+              // not a multiple of 4k, round up 4k
+              expected_arena_block_size += 4 * 1024;
+            }
+            ASSERT_EQ(expected_arena_block_size, result.arena_block_size);
           }
         }
       }
@@ -1164,6 +1189,7 @@ TEST_F(ColumnFamilyTest, ReadDroppedColumnFamily) {
         ASSERT_OK(iterator->status());
         ++count;
       }
+      ASSERT_OK(iterator->status());
       ASSERT_EQ(count, kKeysNum * ((i == 2) ? 1 : 2));
     }
 
@@ -1172,6 +1198,67 @@ TEST_F(ColumnFamilyTest, ReadDroppedColumnFamily) {
   }
 }
 
+TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) {
+  db_options_.create_missing_column_families = true;
+  Open({"default", "one"});
+  ColumnFamilyOptions options;
+  options.level0_file_num_compaction_trigger = 100;
+  options.level0_slowdown_writes_trigger = 200;
+  options.level0_stop_writes_trigger = 200;
+  options.max_write_buffer_number = 20;
+  options.write_buffer_size = 100000;  // small write buffer size
+  Reopen({options, options});
+
+  rocksdb::SyncPoint::GetInstance()->LoadDependency(
+      {{"VersionSet::LogAndApply::ColumnFamilyDrop:1"
+        "FlushJob::InstallResults"},
+       {"FlushJob::InstallResults",
+        "VersionSet::LogAndApply::ColumnFamilyDrop:2", }});
+
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+  test::SleepingBackgroundTask sleeping_task;
+
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task,
+                 Env::Priority::HIGH);
+
+  // 1MB should create ~10 files for each CF
+  int kKeysNum = 10000;
+  PutRandomData(1, kKeysNum, 100);
+
+  std::vector<std::thread> threads;
+  threads.emplace_back([&] { ASSERT_OK(db_->DropColumnFamily(handles_[1])); });
+
+  sleeping_task.WakeUp();
+  sleeping_task.WaitUntilDone();
+  sleeping_task.Reset();
+  // now we sleep again. this is just so we're certain that flush job finished
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task,
+                 Env::Priority::HIGH);
+  sleeping_task.WakeUp();
+  sleeping_task.WaitUntilDone();
+
+  {
+    // Since we didn't delete CF handle, RocksDB's contract guarantees that
+    // we're still able to read dropped CF
+    std::unique_ptr<Iterator> iterator(
+        db_->NewIterator(ReadOptions(), handles_[1]));
+    int count = 0;
+    for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+      ASSERT_OK(iterator->status());
+      ++count;
+    }
+    ASSERT_OK(iterator->status());
+    ASSERT_EQ(count, kKeysNum);
+  }
+  for (auto& t : threads) {
+    t.join();
+  }
+
+  Close();
+  Destroy();
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compaction.cc

@@ -101,8 +101,7 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
       score_(_score),
       bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
       is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
-      is_manual_compaction_(_manual_compaction),
-      level_ptrs_(std::vector<size_t>(number_levels_, 0)) {
+      is_manual_compaction_(_manual_compaction) {
   MarkFilesBeingCompacted(true);
 
 #ifndef NDEBUG
@@ -187,8 +186,11 @@ void Compaction::AddInputDeletions(VersionEdit* out_edit) {
   }
 }
 
-bool Compaction::KeyNotExistsBeyondOutputLevel(const Slice& user_key) {
+bool Compaction::KeyNotExistsBeyondOutputLevel(
+    const Slice& user_key, std::vector<size_t>* level_ptrs) const {
   assert(input_version_ != nullptr);
+  assert(level_ptrs != nullptr);
+  assert(level_ptrs->size() == static_cast<size_t>(number_levels_));
   assert(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO);
   if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
     return bottommost_level_;
@@ -198,8 +200,8 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(const Slice& user_key) {
   for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
     const std::vector<FileMetaData*>& files =
         input_version_->storage_info()->LevelFiles(lvl);
-    for (; level_ptrs_[lvl] < files.size(); ) {
-      FileMetaData* f = files[level_ptrs_[lvl]];
+    for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
+      auto* f = files[level_ptrs->at(lvl)];
       if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
         // We've advanced far enough
         if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
@@ -209,7 +211,6 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(const Slice& user_key) {
         }
         break;
       }
-      level_ptrs_[lvl]++;
     }
   }
   return true;
@@ -376,4 +377,21 @@ std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
       context);
 }
 
+bool Compaction::IsOutputLevelEmpty() const {
+  return inputs_.back().level != output_level_ || inputs_.back().empty();
+}
+
+bool Compaction::ShouldFormSubcompactions() const {
+  if (mutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
+    return false;
+  }
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+    return start_level_ == 0 && !IsOutputLevelEmpty();
+  } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
+    return number_levels_ > 1 && output_level_ > 0;
+  } else {
+    return false;
+  }
+}
+
 }  // namespace rocksdb

db/compaction.h

@@ -128,7 +128,8 @@ class Compaction {
 
   // Returns true if the available information we have guarantees that
   // the input "user_key" does not exist in any level beyond "output_level()".
-  bool KeyNotExistsBeyondOutputLevel(const Slice& user_key);
+  bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
+                                     std::vector<size_t>* level_ptrs) const;
 
   // Returns true iff we should stop building the current output
   // before processing "internal_key".
@@ -168,6 +169,9 @@ class Compaction {
   // are non-overlapping and can be trivially moved.
   bool is_trivial_move() { return is_trivial_move_; }
 
+  // How many total levels are there?
+  int number_levels() const { return number_levels_; }
+
   // Return the MutableCFOptions that should be used throughout the compaction
   // procedure
   const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
@@ -194,16 +198,11 @@ class Compaction {
   // Create a CompactionFilter from compaction_filter_factory
   std::unique_ptr<CompactionFilter> CreateCompactionFilter() const;
 
-  // Should this compaction be broken up into smaller ones run in parallel?
-  bool IsSubCompaction() const {
-    return start_level_ == 0 && output_level_ == 1
-        && mutable_cf_options_.num_subcompactions > 1;
-  }
+  // Is the input level corresponding to output_level_ empty?
+  bool IsOutputLevelEmpty() const;
 
-  // If is_sub_compaction == true, how many smaller compactions should execute
-  int NumSubCompactions() const {
-    return mutable_cf_options_.num_subcompactions;
-  }
+  // Should this compaction be broken up into smaller ones run in parallel?
+  bool ShouldFormSubcompactions() const;
 
  private:
   // mark (or clear) all files that are being compacted
@@ -258,16 +257,8 @@ class Compaction {
 
   // True if we can do trivial move in Universal multi level
   // compaction
-
   bool is_trivial_move_;
 
-  // "level_ptrs_" holds indices into "input_version_->levels_", where each
-  // index remembers which file of an associated level we are currently used
-  // to check KeyNotExistsBeyondOutputLevel() for deletion operation.
-  // As it is for checking KeyNotExistsBeyondOutputLevel(), it only
-  // records indices for all levels beyond "output_level_".
-  std::vector<size_t> level_ptrs_;
-
   // Does input compression match the output compression?
   bool InputCompressionMatchesOutput() const;
 };

db/compaction_iterator.cc

@@ -0,0 +1,269 @@
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include "db/compaction_iterator.h"
+
+namespace rocksdb {
+
+CompactionIterator::CompactionIterator(
+    Iterator* input, const Comparator* cmp, MergeHelper* merge_helper,
+    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
+    Env* env, bool expect_valid_internal_key, Statistics* stats,
+    Compaction* compaction, const CompactionFilter* compaction_filter,
+    LogBuffer* log_buffer)
+    : input_(input),
+      cmp_(cmp),
+      merge_helper_(merge_helper),
+      snapshots_(snapshots),
+      env_(env),
+      expect_valid_internal_key_(expect_valid_internal_key),
+      stats_(stats),
+      compaction_(compaction),
+      compaction_filter_(compaction_filter),
+      log_buffer_(log_buffer),
+      merge_out_iter_(merge_helper_) {
+  assert(compaction_filter_ == nullptr || compaction_ != nullptr);
+  bottommost_level_ =
+      compaction_ == nullptr ? false : compaction_->bottommost_level();
+  if (compaction_ != nullptr) {
+    level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
+  }
+
+  if (snapshots_->size() == 0) {
+    // optimize for fast path if there are no snapshots
+    visible_at_tip_ = last_sequence;
+    earliest_snapshot_ = visible_at_tip_;
+    latest_snapshot_ = 0;
+  } else {
+    visible_at_tip_ = 0;
+    earliest_snapshot_ = snapshots_->at(0);
+    latest_snapshot_ = snapshots_->back();
+  }
+}
+
+void CompactionIterator::ResetRecordCounts() {
+  iter_stats_.num_record_drop_user = 0;
+  iter_stats_.num_record_drop_hidden = 0;
+  iter_stats_.num_record_drop_obsolete = 0;
+}
+
+void CompactionIterator::SeekToFirst() {
+  NextFromInput();
+  PrepareOutput();
+}
+
+void CompactionIterator::Next() {
+  // If there is a merge output, return it before continuing to process the
+  // input.
+  if (merge_out_iter_.Valid()) {
+    merge_out_iter_.Next();
+
+    // Check if we returned all records of the merge output.
+    if (merge_out_iter_.Valid()) {
+      key_ = merge_out_iter_.key();
+      value_ = merge_out_iter_.value();
+      bool valid_key __attribute__((__unused__)) =
+          ParseInternalKey(key_, &ikey_);
+      // MergeUntil stops when it encounters a corrupt key and does not
+      // include them in the result, so we expect the keys here to be valid.
+      assert(valid_key);
+      valid_ = true;
+    } else {
+      // MergeHelper moves the iterator to the first record after the merged
+      // records, so even though we reached the end of the merge output, we do
+      // not want to advance the iterator.
+      NextFromInput();
+    }
+  } else {
+    // Only advance the input iterator if there is no merge output.
+    input_->Next();
+    NextFromInput();
+  }
+
+  PrepareOutput();
+}
+
+void CompactionIterator::NextFromInput() {
+  valid_ = false;
+
+  while (input_->Valid()) {
+    key_ = input_->key();
+    value_ = input_->value();
+    iter_stats_.num_input_records++;
+
+    if (!ParseInternalKey(key_, &ikey_)) {
+      // If `expect_valid_internal_key_` is false, return the corrupted key
+      // and let the caller decide what to do with it.
+      // TODO(noetzli): We should have a more elegant solution for this.
+      if (expect_valid_internal_key_) {
+        assert(!"corrupted internal key is not expected");
+        break;
+      }
+      current_user_key_.Clear();
+      has_current_user_key_ = false;
+      current_user_key_sequence_ = kMaxSequenceNumber;
+      current_user_key_snapshot_ = 0;
+      iter_stats_.num_input_corrupt_records++;
+      valid_ = true;
+      break;
+    }
+
+    // Update input statistics
+    if (ikey_.type == kTypeDeletion) {
+      iter_stats_.num_input_deletion_records++;
+    }
+    iter_stats_.total_input_raw_key_bytes += key_.size();
+    iter_stats_.total_input_raw_value_bytes += value_.size();
+
+    if (!has_current_user_key_ ||
+        cmp_->Compare(ikey_.user_key, current_user_key_.GetKey()) != 0) {
+      // First occurrence of this user key
+      current_user_key_.SetKey(ikey_.user_key);
+      has_current_user_key_ = true;
+      current_user_key_sequence_ = kMaxSequenceNumber;
+      current_user_key_snapshot_ = 0;
+      // apply the compaction filter to the first occurrence of the user key
+      if (compaction_filter_ != nullptr && ikey_.type == kTypeValue &&
+          (visible_at_tip_ || ikey_.sequence > latest_snapshot_)) {
+        // If the user has specified a compaction filter and the sequence
+        // number is greater than any external snapshot, then invoke the
+        // filter. If the return value of the compaction filter is true,
+        // replace the entry with a deletion marker.
+        bool value_changed = false;
+        bool to_delete = false;
+        compaction_filter_value_.clear();
+        {
+          StopWatchNano timer(env_, true);
+          to_delete = compaction_filter_->Filter(
+              compaction_->level(), ikey_.user_key, value_,
+              &compaction_filter_value_, &value_changed);
+          iter_stats_.total_filter_time +=
+              env_ != nullptr ? timer.ElapsedNanos() : 0;
+        }
+        if (to_delete) {
+          // make a copy of the original key and convert it to a delete
+          delete_key_.SetInternalKey(ExtractUserKey(key_), ikey_.sequence,
+                                     kTypeDeletion);
+          // anchor the key again
+          key_ = delete_key_.GetKey();
+          // needed because ikey_ is backed by key
+          ParseInternalKey(key_, &ikey_);
+          // no value associated with delete
+          value_.clear();
+          iter_stats_.num_record_drop_user++;
+        } else if (value_changed) {
+          value_ = compaction_filter_value_;
+        }
+      }
+    }
+
+    // If there are no snapshots, then this kv affect visibility at tip.
+    // Otherwise, search though all existing snapshots to find the earliest
+    // snapshot that is affected by this kv.
+    SequenceNumber last_sequence __attribute__((__unused__)) =
+        current_user_key_sequence_;
+    current_user_key_sequence_ = ikey_.sequence;
+    SequenceNumber last_snapshot = current_user_key_snapshot_;
+    SequenceNumber prev_snapshot = 0;  // 0 means no previous snapshot
+    current_user_key_snapshot_ =
+        visible_at_tip_ ? visible_at_tip_ : findEarliestVisibleSnapshot(
+                                                ikey_.sequence, &prev_snapshot);
+
+    if (last_snapshot == current_user_key_snapshot_) {
+      // If the earliest snapshot is which this key is visible in
+      // is the same as the visibility of a previous instance of the
+      // same key, then this kv is not visible in any snapshot.
+      // Hidden by an newer entry for same user key
+      // TODO: why not > ?
+      assert(last_sequence >= current_user_key_sequence_);
+      ++iter_stats_.num_record_drop_hidden;  // (A)
+    } else if (compaction_ != nullptr && ikey_.type == kTypeDeletion &&
+               ikey_.sequence <= earliest_snapshot_ &&
+               compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
+                                                          &level_ptrs_)) {
+      // TODO(noetzli): This is the only place where we use compaction_
+      // (besides the constructor). We should probably get rid of this
+      // dependency and find a way to do similar filtering during flushes.
+      //
+      // For this user key:
+      // (1) there is no data in higher levels
+      // (2) data in lower levels will have larger sequence numbers
+      // (3) data in layers that are being compacted here and have
+      //     smaller sequence numbers will be dropped in the next
+      //     few iterations of this loop (by rule (A) above).
+      // Therefore this deletion marker is obsolete and can be dropped.
+      ++iter_stats_.num_record_drop_obsolete;
+    } else if (ikey_.type == kTypeMerge) {
+      if (!merge_helper_->HasOperator()) {
+        LogToBuffer(log_buffer_, "Options::merge_operator is null.");
+        status_ = Status::InvalidArgument(
+            "merge_operator is not properly initialized.");
+        return;
+      }
+
+      // We know the merge type entry is not hidden, otherwise we would
+      // have hit (A)
+      // We encapsulate the merge related state machine in a different
+      // object to minimize change to the existing flow.
+      merge_helper_->MergeUntil(input_, prev_snapshot, bottommost_level_,
+                                stats_, env_);
+      merge_out_iter_.SeekToFirst();
+
+      // NOTE: key, value, and ikey_ refer to old entries.
+      //       These will be correctly set below.
+      key_ = merge_out_iter_.key();
+      value_ = merge_out_iter_.value();
+      bool valid_key __attribute__((__unused__)) =
+          ParseInternalKey(key_, &ikey_);
+      // MergeUntil stops when it encounters a corrupt key and does not
+      // include them in the result, so we expect the keys here to valid.
+      assert(valid_key);
+      valid_ = true;
+      break;
+    } else {
+      valid_ = true;
+      break;
+    }
+
+    input_->Next();
+  }
+}
+
+void CompactionIterator::PrepareOutput() {
+  // Zeroing out the sequence number leads to better compression.
+  // If this is the bottommost level (no files in lower levels)
+  // and the earliest snapshot is larger than this seqno
+  // then we can squash the seqno to zero.
+  if (bottommost_level_ && valid_ && ikey_.sequence < earliest_snapshot_ &&
+      ikey_.type != kTypeMerge) {
+    assert(ikey_.type != kTypeDeletion);
+    // make a copy because updating in place would cause problems
+    // with the priority queue that is managing the input key iterator
+    updated_key_.assign(key_.data(), key_.size());
+    UpdateInternalKey(&updated_key_, (uint64_t)0, ikey_.type);
+    key_ = Slice(updated_key_);
+  }
+}
+
+inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot(
+    SequenceNumber in, SequenceNumber* prev_snapshot) {
+  assert(snapshots_->size());
+  SequenceNumber prev __attribute__((unused)) = 0;
+  for (const auto cur : *snapshots_) {
+    assert(prev <= cur);
+    if (cur >= in) {
+      *prev_snapshot = prev;
+      return cur;
+    }
+    prev = cur;
+    assert(prev);
+  }
+  *prev_snapshot = prev;
+  return kMaxSequenceNumber;
+}
+
+}  // namespace rocksdb

db/compaction_iterator.h

@@ -0,0 +1,126 @@
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+#pragma once
+
+#include <algorithm>
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "db/compaction.h"
+#include "db/merge_helper.h"
+#include "rocksdb/compaction_filter.h"
+#include "util/log_buffer.h"
+
+namespace rocksdb {
+
+struct CompactionIteratorStats {
+  // Compaction statistics
+  int64_t num_record_drop_user = 0;
+  int64_t num_record_drop_hidden = 0;
+  int64_t num_record_drop_obsolete = 0;
+  uint64_t total_filter_time = 0;
+
+  // Input statistics
+  // TODO(noetzli): The stats are incomplete. They are lacking everything
+  // consumed by MergeHelper.
+  uint64_t num_input_records = 0;
+  uint64_t num_input_deletion_records = 0;
+  uint64_t num_input_corrupt_records = 0;
+  uint64_t total_input_raw_key_bytes = 0;
+  uint64_t total_input_raw_value_bytes = 0;
+};
+
+class CompactionIterator {
+ public:
+  CompactionIterator(Iterator* input, const Comparator* cmp,
+                     MergeHelper* merge_helper, SequenceNumber last_sequence,
+                     std::vector<SequenceNumber>* snapshots, Env* env,
+                     bool expect_valid_internal_key,
+                     Statistics* stats = nullptr,
+                     Compaction* compaction = nullptr,
+                     const CompactionFilter* compaction_filter = nullptr,
+                     LogBuffer* log_buffer = nullptr);
+
+  void ResetRecordCounts();
+
+  // Seek to the beginning of the compaction iterator output.
+  //
+  // REQUIRED: Call only once.
+  void SeekToFirst();
+
+  // Produces the next record in the compaction.
+  //
+  // REQUIRED: SeekToFirst() has been called.
+  void Next();
+
+  // Getters
+  const Slice& key() const { return key_; }
+  const Slice& value() const { return value_; }
+  const Status& status() const { return status_; }
+  const ParsedInternalKey& ikey() const { return ikey_; }
+  bool Valid() const { return valid_; }
+  Slice user_key() const { return current_user_key_.GetKey(); }
+  const CompactionIteratorStats& iter_stats() const { return iter_stats_; }
+
+ private:
+  // Processes the input stream to find the next output
+  void NextFromInput();
+
+  // Do last preparations before presenting the output to the callee. At this
+  // point this only zeroes out the sequence number if possible for better
+  // compression.
+  void PrepareOutput();
+
+  // Given a sequence number, return the sequence number of the
+  // earliest snapshot that this sequence number is visible in.
+  // The snapshots themselves are arranged in ascending order of
+  // sequence numbers.
+  // Employ a sequential search because the total number of
+  // snapshots are typically small.
+  inline SequenceNumber findEarliestVisibleSnapshot(
+      SequenceNumber in, SequenceNumber* prev_snapshot);
+
+  Iterator* input_;
+  const Comparator* cmp_;
+  MergeHelper* merge_helper_;
+  const std::vector<SequenceNumber>* snapshots_;
+  Env* env_;
+  bool expect_valid_internal_key_;
+  Statistics* stats_;
+  Compaction* compaction_;
+  const CompactionFilter* compaction_filter_;
+  LogBuffer* log_buffer_;
+  bool bottommost_level_;
+  bool valid_ = false;
+  SequenceNumber visible_at_tip_;
+  SequenceNumber earliest_snapshot_;
+  SequenceNumber latest_snapshot_;
+
+  // State
+  Slice key_;
+  Slice value_;
+  Status status_;
+  ParsedInternalKey ikey_;
+  bool has_current_user_key_ = false;
+  IterKey current_user_key_;
+  SequenceNumber current_user_key_sequence_;
+  SequenceNumber current_user_key_snapshot_;
+  MergeOutputIterator merge_out_iter_;
+  std::string updated_key_;
+  std::string compaction_filter_value_;
+  IterKey delete_key_;
+  // "level_ptrs" holds indices that remember which file of an associated
+  // level we were last checking during the last call to compaction->
+  // KeyNotExistsBeyondOutputLevel(). This allows future calls to the function
+  // to pick off where it left off since each subcompaction's key range is
+  // increasing so a later call to the function must be looking for a key that
+  // is in or beyond the last file checked during the previous call
+  std::vector<size_t> level_ptrs_;
+  CompactionIteratorStats iter_stats_;
+};
+}  // namespace rocksdb

db/compaction_job.h

@@ -10,35 +10,36 @@
 
 #include <atomic>
 #include <deque>
+#include <functional>
 #include <limits>
 #include <set>
+#include <string>
 #include <utility>
 #include <vector>
-#include <string>
-#include <functional>
 
-#include "db/dbformat.h"
-#include "db/log_writer.h"
 #include "db/column_family.h"
-#include "db/version_edit.h"
+#include "db/compaction_iterator.h"
+#include "db/dbformat.h"
+#include "db/flush_scheduler.h"
+#include "db/internal_stats.h"
+#include "db/job_context.h"
+#include "db/log_writer.h"
 #include "db/memtable_list.h"
+#include "db/version_edit.h"
+#include "db/write_controller.h"
+#include "db/write_thread.h"
 #include "port/port.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/compaction_job_stats.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/memtablerep.h"
-#include "rocksdb/compaction_filter.h"
-#include "rocksdb/compaction_job_stats.h"
 #include "rocksdb/transaction_log.h"
 #include "util/autovector.h"
 #include "util/event_logger.h"
+#include "util/scoped_arena_iterator.h"
 #include "util/stop_watch.h"
 #include "util/thread_local.h"
-#include "util/scoped_arena_iterator.h"
-#include "db/internal_stats.h"
-#include "db/write_controller.h"
-#include "db/flush_scheduler.h"
-#include "db/write_thread.h"
-#include "db/job_context.h"
 
 namespace rocksdb {
 
@@ -57,8 +58,8 @@ class CompactionJob {
                 Directory* db_directory, Directory* output_directory,
                 Statistics* stats,
                 std::vector<SequenceNumber> existing_snapshots,
-                std::shared_ptr<Cache> table_cache,
-                EventLogger* event_logger, bool paranoid_file_checks,
+                std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
+                bool paranoid_file_checks, bool measure_io_stats,
                 const std::string& dbname,
                 CompactionJobStats* compaction_job_stats);
 
@@ -75,48 +76,39 @@ class CompactionJob {
   Status Run();
 
   // REQUIRED: mutex held
-  // status is the return of Run()
-  void Install(Status* status, const MutableCFOptions& mutable_cf_options,
-               InstrumentedMutex* db_mutex);
+  Status Install(const MutableCFOptions& mutable_cf_options,
+                 InstrumentedMutex* db_mutex);
 
  private:
-  // REQUIRED: mutex not held
-  Status SubCompactionRun(Slice* start, Slice* end);
+  struct SubcompactionState;
+
+  void AggregateStatistics();
+  void GenSubcompactionBoundaries();
 
-  void GetSubCompactionBoundaries();
   // update the thread status for starting a compaction.
   void ReportStartedCompaction(Compaction* compaction);
   void AllocateCompactionOutputFileNumbers();
   // Call compaction filter. Then iterate through input and compact the
   // kv-pairs
-  Status ProcessKeyValueCompaction(int64_t* imm_micros, Iterator* input,
-                                    Slice* start = nullptr,
-                                    Slice* end = nullptr);
+  void ProcessKeyValueCompaction(SubcompactionState* sub_compact);
 
-  Status WriteKeyValue(const Slice& key, const Slice& value,
-                       const ParsedInternalKey& ikey,
-                       const Status& input_status);
-
-  Status FinishCompactionOutputFile(const Status& input_status);
-  Status InstallCompactionResults(InstrumentedMutex* db_mutex,
-                                  const MutableCFOptions& mutable_cf_options);
-  SequenceNumber findEarliestVisibleSnapshot(
-      SequenceNumber in, const std::vector<SequenceNumber>& snapshots,
-      SequenceNumber* prev_snapshot);
+  Status FinishCompactionOutputFile(const Status& input_status,
+                                    SubcompactionState* sub_compact);
+  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options,
+                                  InstrumentedMutex* db_mutex);
   void RecordCompactionIOStats();
-  Status OpenCompactionOutputFile();
-  void CleanupCompaction(const Status& status);
+  Status OpenCompactionOutputFile(SubcompactionState* sub_compact);
+  void CleanupCompaction();
   void UpdateCompactionJobStats(
     const InternalStats::CompactionStats& stats) const;
-  void RecordDroppedKeys(int64_t* key_drop_user,
-                         int64_t* key_drop_newer_entry,
-                         int64_t* key_drop_obsolete);
+  void RecordDroppedKeys(const CompactionIteratorStats& c_iter_stats,
+                         CompactionJobStats* compaction_job_stats = nullptr);
 
   void UpdateCompactionStats();
   void UpdateCompactionInputStatsHelper(
       int* num_files, uint64_t* bytes_read, int input_level);
 
-  void LogCompaction(ColumnFamilyData* cfd, Compaction* compaction);
+  void LogCompaction();
 
   int job_id_;
 
@@ -124,12 +116,6 @@ class CompactionJob {
   struct CompactionState;
   CompactionState* compact_;
   CompactionJobStats* compaction_job_stats_;
-
-  bool bottommost_level_;
-  SequenceNumber earliest_snapshot_;
-  SequenceNumber visible_at_tip_;
-  SequenceNumber latest_snapshot_;
-
   InternalStats::CompactionStats compaction_stats_;
 
   // DBImpl state
@@ -152,8 +138,13 @@ class CompactionJob {
 
   EventLogger* event_logger_;
 
+  bool bottommost_level_;
   bool paranoid_file_checks_;
-  std::vector<Slice> sub_compaction_boundaries_;
+  bool measure_io_stats_;
+  // Stores the Slices that designate the boundaries for each subcompaction
+  std::vector<Slice> boundaries_;
+  // Stores the approx size of keys covered in the range of each subcompaction
+  std::vector<uint64_t> sizes_;
 };
 
 }  // namespace rocksdb

db/compaction_job_stats_test.cc

@@ -92,7 +92,7 @@ class CompactionJobStatsTest : public testing::Test,
   Env* env_;
   DB* db_;
   std::vector<ColumnFamilyHandle*> handles_;
-  uint32_t num_subcompactions_;
+  uint32_t max_subcompactions_;
 
   Options last_options_;
 
@@ -103,8 +103,8 @@ class CompactionJobStatsTest : public testing::Test,
     alternative_wal_dir_ = dbname_ + "/wal";
     Options options;
     options.create_if_missing = true;
-    num_subcompactions_ = GetParam();
-    options.num_subcompactions = num_subcompactions_;
+    max_subcompactions_ = GetParam();
+    options.max_subcompactions = max_subcompactions_;
     auto delete_options = options;
     delete_options.wal_dir = alternative_wal_dir_;
     EXPECT_OK(DestroyDB(dbname_, delete_options));
@@ -417,14 +417,25 @@ class CompactionJobStatsTest : public testing::Test,
 // test CompactionJobStatsTest.
 class CompactionJobStatsChecker : public EventListener {
  public:
-  CompactionJobStatsChecker() : compression_enabled_(false) {}
+  CompactionJobStatsChecker()
+      : compression_enabled_(false), verify_next_comp_io_stats_(false) {}
 
   size_t NumberOfUnverifiedStats() { return expected_stats_.size(); }
 
+  void set_verify_next_comp_io_stats(bool v) { verify_next_comp_io_stats_ = v; }
+
   // Once a compaction completed, this function will verify the returned
   // CompactionJobInfo with the oldest CompactionJobInfo added earlier
   // in "expected_stats_" which has not yet being used for verification.
   virtual void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) {
+    if (verify_next_comp_io_stats_) {
+      ASSERT_GT(ci.stats.file_write_nanos, 0);
+      ASSERT_GT(ci.stats.file_range_sync_nanos, 0);
+      ASSERT_GT(ci.stats.file_fsync_nanos, 0);
+      ASSERT_GT(ci.stats.file_prepare_write_nanos, 0);
+      verify_next_comp_io_stats_ = false;
+    }
+
     std::lock_guard<std::mutex> lock(mutex_);
     if (expected_stats_.size()) {
       Verify(ci.stats, expected_stats_.front());
@@ -498,10 +509,13 @@ class CompactionJobStatsChecker : public EventListener {
     compression_enabled_ = flag;
   }
 
+  bool verify_next_comp_io_stats() const { return verify_next_comp_io_stats_; }
+
  private:
   std::mutex mutex_;
   std::queue<CompactionJobStats> expected_stats_;
   bool compression_enabled_;
+  bool verify_next_comp_io_stats_;
 };
 
 // An EventListener which helps verify the compaction statistics in
@@ -642,8 +656,10 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
   options.level0_file_num_compaction_trigger = kTestScale + 1;
   options.num_levels = 3;
   options.compression = kNoCompression;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
+  options.bytes_per_sync = 512 * 1024;
 
+  options.compaction_measure_io_stats = true;
   for (int test = 0; test < 2; ++test) {
     DestroyAndReopen(options);
     CreateAndReopenWithCF({"pikachu"}, options);
@@ -724,7 +740,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
     // by 1 because multiple threads are consuming the input and generating
     // output files without coordinating to see if the output could fit into
     // a smaller number of files like it does when it runs sequentially
-    int num_output_files = options.num_subcompactions > 1 ? 2 : 1;
+    int num_output_files = options.max_subcompactions > 1 ? 2 : 1;
     for (uint64_t start_key = key_base;
          num_L0_files > 1;
          start_key += key_base * sparseness) {
@@ -742,9 +758,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
               num_keys_per_L0_file));
       ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U);
       Compact(1, smallest_key, largest_key);
-      // TODO(aekmekji): account for whether parallel L0-L1 compaction is
-      // enabled or not. If so then num_L1_files will increase by 1
-      if (options.num_subcompactions == 1) {
+      if (options.max_subcompactions == 1) {
         --num_L1_files;
       }
       snprintf(buf, kBufSize, "%d,%d", --num_L0_files, num_L1_files);
@@ -766,7 +780,8 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
             num_keys_per_L0_file));
     ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 1U);
     Compact(1, smallest_key, largest_key);
-    num_L1_files = options.num_subcompactions > 1 ? 7 : 4;
+
+    num_L1_files = options.max_subcompactions > 1 ? 7 : 4;
     char L1_buf[4];
     snprintf(L1_buf, sizeof(L1_buf), "0,%d", num_L1_files);
     std::string L1_files(L1_buf);
@@ -777,6 +792,61 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
     }
     stats_checker->EnableCompression(true);
     compression_ratio = kCompressionRatio;
+
+    for (int i = 0; i < 5; i++) {
+      ASSERT_OK(Put(1, Slice(Key(key_base + i, 10)),
+                    Slice(RandomString(&rnd, 512 * 1024, 1))));
+    }
+
+    ASSERT_OK(Flush(1));
+    reinterpret_cast<DBImpl*>(db_)->TEST_WaitForCompact();
+
+    stats_checker->set_verify_next_comp_io_stats(true);
+    std::atomic<bool> first_prepare_write(true);
+    rocksdb::SyncPoint::GetInstance()->SetCallBack(
+        "WritableFileWriter::Append:BeforePrepareWrite", [&](void* arg) {
+          if (first_prepare_write.load()) {
+            options.env->SleepForMicroseconds(3);
+            first_prepare_write.store(false);
+          }
+        });
+
+    std::atomic<bool> first_flush(true);
+    rocksdb::SyncPoint::GetInstance()->SetCallBack(
+        "WritableFileWriter::Flush:BeforeAppend", [&](void* arg) {
+          if (first_flush.load()) {
+            options.env->SleepForMicroseconds(3);
+            first_flush.store(false);
+          }
+        });
+
+    std::atomic<bool> first_sync(true);
+    rocksdb::SyncPoint::GetInstance()->SetCallBack(
+        "WritableFileWriter::SyncInternal:0", [&](void* arg) {
+          if (first_sync.load()) {
+            options.env->SleepForMicroseconds(3);
+            first_sync.store(false);
+          }
+        });
+
+    std::atomic<bool> first_range_sync(true);
+    rocksdb::SyncPoint::GetInstance()->SetCallBack(
+        "WritableFileWriter::RangeSync:0", [&](void* arg) {
+          if (first_range_sync.load()) {
+            options.env->SleepForMicroseconds(3);
+            first_range_sync.store(false);
+          }
+        });
+    rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+    Compact(1, smallest_key, largest_key);
+
+    ASSERT_TRUE(!stats_checker->verify_next_comp_io_stats());
+    ASSERT_TRUE(!first_prepare_write.load());
+    ASSERT_TRUE(!first_flush.load());
+    ASSERT_TRUE(!first_sync.load());
+    ASSERT_TRUE(!first_range_sync.load());
+    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
   }
   ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U);
 }
@@ -808,7 +878,7 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
   options.num_levels = 3;
   options.compression = kNoCompression;
   options.max_bytes_for_level_multiplier = 2;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
 
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
@@ -900,7 +970,7 @@ TEST_P(CompactionJobStatsTest, UniversalCompactionTest) {
   options.compaction_style = kCompactionStyleUniversal;
   options.compaction_options_universal.size_ratio = 1;
   options.compaction_options_universal.max_size_amplification_percent = 1000;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
 
   DestroyAndReopen(options);
   CreateAndReopenWithCF({"pikachu"}, options);
@@ -944,8 +1014,8 @@ TEST_P(CompactionJobStatsTest, UniversalCompactionTest) {
         &rnd, start_key, start_key + key_base - 1,
         kKeySize, kValueSize, key_interval,
         compression_ratio, 1);
+    reinterpret_cast<DBImpl*>(db_)->TEST_WaitForCompact();
   }
-  reinterpret_cast<DBImpl*>(db_)->TEST_WaitForCompact();
   ASSERT_EQ(stats_checker->NumberOfUnverifiedStats(), 0U);
 }
 

db/compaction_job_test.cc

@@ -3,25 +3,29 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
+#include <algorithm>
 #include <map>
 #include <string>
+#include <tuple>
 
 #include "db/compaction_job.h"
 #include "db/column_family.h"
 #include "db/version_set.h"
 #include "db/writebuffer.h"
 #include "rocksdb/cache.h"
-#include "rocksdb/options.h"
 #include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "table/mock_table.h"
 #include "util/file_reader_writer.h"
 #include "util/string_util.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
-#include "table/mock_table.h"
+#include "utilities/merge_operators.h"
 
 namespace rocksdb {
 
 namespace {
+
 void VerifyInitializationOfCompactionJobStats(
       const CompactionJobStats& compaction_job_stats) {
 #if !defined(IOS_CROSS_COMPILE)
@@ -34,7 +38,7 @@ void VerifyInitializationOfCompactionJobStats(
   ASSERT_EQ(compaction_job_stats.num_output_records, 0U);
   ASSERT_EQ(compaction_job_stats.num_output_files, 0U);
 
-  ASSERT_EQ(compaction_job_stats.is_manual_compaction, false);
+  ASSERT_EQ(compaction_job_stats.is_manual_compaction, true);
 
   ASSERT_EQ(compaction_job_stats.total_input_bytes, 0U);
   ASSERT_EQ(compaction_job_stats.total_output_bytes, 0U);
@@ -73,12 +77,6 @@ class CompactionJobTest : public testing::Test {
     EXPECT_OK(env_->CreateDirIfMissing(dbname_));
     db_options_.db_paths.emplace_back(dbname_,
                                       std::numeric_limits<uint64_t>::max());
-    NewDB();
-    std::vector<ColumnFamilyDescriptor> column_families;
-    cf_options_.table_factory = mock_table_factory_;
-    column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
-
-    EXPECT_OK(versions_->Recover(column_families, false));
   }
 
   std::string GenerateFileName(uint64_t file_number) {
@@ -89,16 +87,64 @@ class CompactionJobTest : public testing::Test {
     return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
   }
 
-  // Corrupts key by changing the type
-  void CorruptKey(InternalKey* ikey) {
-    std::string keystr = ikey->Encode().ToString();
-    keystr[keystr.size() - 8] = kTypeLogData;
-    ikey->DecodeFrom(Slice(keystr.data(), keystr.size()));
+  std::string KeyStr(const std::string& user_key, const SequenceNumber seq_num,
+      const ValueType t) {
+    return InternalKey(user_key, seq_num, t).Encode().ToString();
+  }
+
+  void AddMockFile(const stl_wrappers::KVMap& contents, int level = 0) {
+    assert(contents.size() > 0);
+
+    bool first_key = true;
+    std::string smallest, largest;
+    InternalKey smallest_key, largest_key;
+    SequenceNumber smallest_seqno = kMaxSequenceNumber;
+    SequenceNumber largest_seqno = 0;
+    for (auto kv : contents) {
+      ParsedInternalKey key;
+      std::string skey;
+      std::string value;
+      std::tie(skey, value) = kv;
+      ParseInternalKey(skey, &key);
+
+      smallest_seqno = std::min(smallest_seqno, key.sequence);
+      largest_seqno = std::max(largest_seqno, key.sequence);
+
+      if (first_key ||
+          cfd_->user_comparator()->Compare(key.user_key, smallest) < 0) {
+        smallest.assign(key.user_key.data(), key.user_key.size());
+        smallest_key.DecodeFrom(skey);
+      }
+      if (first_key ||
+          cfd_->user_comparator()->Compare(key.user_key, largest) > 0) {
+        largest.assign(key.user_key.data(), key.user_key.size());
+        largest_key.DecodeFrom(skey);
+      }
+
+      first_key = false;
+    }
+
+    uint64_t file_number = versions_->NewFileNumber();
+    EXPECT_OK(mock_table_factory_->CreateMockTable(
+        env_, GenerateFileName(file_number), std::move(contents)));
+
+    VersionEdit edit;
+    edit.AddFile(level, file_number, 0, 10, smallest_key, largest_key,
+        smallest_seqno, largest_seqno, false);
+
+    mutex_.Lock();
+    versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
+                           mutable_cf_options_, &edit, &mutex_);
+    mutex_.Unlock();
+  }
+
+  void SetLastSequence(const SequenceNumber sequence_number) {
+    versions_->SetLastSequence(sequence_number + 1);
   }
 
   // returns expected result after compaction
-  mock::MockFileContents CreateTwoFiles(bool gen_corrupted_keys) {
-    mock::MockFileContents expected_results;
+  stl_wrappers::KVMap CreateTwoFiles(bool gen_corrupted_keys) {
+    auto expected_results = mock::MakeMockFile();
     const int kKeysPerFile = 10000;
     const int kCorruptKeysPerFile = 200;
     const int kMatchingKeys = kKeysPerFile / 2;
@@ -109,9 +155,7 @@ class CompactionJobTest : public testing::Test {
     };
 
     for (int i = 0; i < 2; ++i) {
-      mock::MockFileContents contents;
-      SequenceNumber smallest_seqno = 0, largest_seqno = 0;
-      InternalKey smallest, largest;
+      auto contents = mock::MakeMockFile();
       for (int k = 0; k < kKeysPerFile; ++k) {
         auto key = ToString(i * kMatchingKeys + k);
         auto value = ToString(i * kKeysPerFile + k);
@@ -120,41 +164,25 @@ class CompactionJobTest : public testing::Test {
         // file
         InternalKey bottommost_internal_key(key, 0, kTypeValue);
         if (corrupt_id(k)) {
-          CorruptKey(&internal_key);
-          CorruptKey(&bottommost_internal_key);
+          test::CorruptKeyType(&internal_key);
+          test::CorruptKeyType(&bottommost_internal_key);
         }
-        if (k == 0) {
-          smallest = internal_key;
-          smallest_seqno = sequence_number;
-        } else if (k == kKeysPerFile - 1) {
-          largest = internal_key;
-          largest_seqno = sequence_number;
-        }
-        contents.insert({internal_key.Encode().ToString(), value});
+        contents.insert({ internal_key.Encode().ToString(), value });
         if (i == 1 || k < kMatchingKeys || corrupt_id(k - kMatchingKeys)) {
           expected_results.insert(
-              {bottommost_internal_key.Encode().ToString(), value});
+              { bottommost_internal_key.Encode().ToString(), value });
         }
       }
 
-      uint64_t file_number = versions_->NewFileNumber();
-      EXPECT_OK(mock_table_factory_->CreateMockTable(
-          env_, GenerateFileName(file_number), std::move(contents)));
-
-      VersionEdit edit;
-      edit.AddFile(0, file_number, 0, 10, smallest, largest, smallest_seqno,
-                   largest_seqno, false);
-
-      mutex_.Lock();
-      versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
-                             mutable_cf_options_, &edit, &mutex_);
-      mutex_.Unlock();
+      AddMockFile(contents);
     }
-    versions_->SetLastSequence(sequence_number + 1);
+
+    SetLastSequence(sequence_number);
+
     return expected_results;
   }
 
-  void NewDB() {
+  void NewDB(std::shared_ptr<MergeOperator> merge_operator = nullptr) {
     VersionEdit new_db;
     new_db.SetLogNumber(0);
     new_db.SetNextFile(2);
@@ -166,7 +194,7 @@ class CompactionJobTest : public testing::Test {
         manifest, &file, env_->OptimizeForManifestWrite(env_options_));
     ASSERT_OK(s);
     unique_ptr<WritableFileWriter> file_writer(
-        new WritableFileWriter(std::move(file), EnvOptions()));
+        new WritableFileWriter(std::move(file), env_options_));
     {
       log::Writer log(std::move(file_writer));
       std::string record;
@@ -176,44 +204,63 @@ class CompactionJobTest : public testing::Test {
     ASSERT_OK(s);
     // Make "CURRENT" file that points to the new manifest file.
     s = SetCurrentFile(env_, dbname_, 1, nullptr);
+
+    std::vector<ColumnFamilyDescriptor> column_families;
+    cf_options_.table_factory = mock_table_factory_;
+    cf_options_.merge_operator = merge_operator;
+    column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
+
+    EXPECT_OK(versions_->Recover(column_families, false));
+    cfd_ = versions_->GetColumnFamilySet()->GetDefault();
   }
 
-  void RunCompaction(const std::vector<FileMetaData*>& files) {
+  void RunCompaction(const std::vector<std::vector<FileMetaData*>>& input_files,
+                     const stl_wrappers::KVMap& expected_results) {
     auto cfd = versions_->GetColumnFamilySet()->GetDefault();
 
-    CompactionInputFiles compaction_input_files;
-    compaction_input_files.level = 0;
-    for (auto file : files) {
-      compaction_input_files.files.push_back(file);
+    size_t num_input_files = 0;
+    std::vector<CompactionInputFiles> compaction_input_files;
+    for (size_t level = 0; level < input_files.size(); level++) {
+      auto level_files = input_files[level];
+      CompactionInputFiles compaction_level;
+      compaction_level.level = static_cast<int>(level);
+      compaction_level.files.insert(compaction_level.files.end(),
+          level_files.begin(), level_files.end());
+      compaction_input_files.push_back(compaction_level);
+      num_input_files += level_files.size();
     }
+
     Compaction compaction(cfd->current()->storage_info(),
                           *cfd->GetLatestMutableCFOptions(),
-                          {compaction_input_files}, 1, 1024 * 1024, 10, 0,
-                          kNoCompression, {});
+                          compaction_input_files, 1, 1024 * 1024, 10, 0,
+                          kNoCompression, {}, true);
     compaction.SetInputVersion(cfd->current());
 
     LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
     mutex_.Lock();
     EventLogger event_logger(db_options_.info_log.get());
-    CompactionJob compaction_job(
-        0, &compaction, db_options_, env_options_, versions_.get(),
-        &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, {},
-        table_cache_, &event_logger, false, dbname_, &compaction_job_stats_);
+    CompactionJob compaction_job(0, &compaction, db_options_, env_options_,
+                                 versions_.get(), &shutting_down_, &log_buffer,
+                                 nullptr, nullptr, nullptr, {}, table_cache_,
+                                 &event_logger, false, false, dbname_,
+                                 &compaction_job_stats_);
 
     VerifyInitializationOfCompactionJobStats(compaction_job_stats_);
 
     compaction_job.Prepare();
     mutex_.Unlock();
-    ASSERT_OK(compaction_job.Run());
-    mutex_.Lock();
     Status s;
-    compaction_job.Install(&s, *cfd->GetLatestMutableCFOptions(), &mutex_);
+    s = compaction_job.Run();
     ASSERT_OK(s);
+    mutex_.Lock();
+    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(),
+                                     &mutex_));
     mutex_.Unlock();
 
     ASSERT_GE(compaction_job_stats_.elapsed_micros, 0U);
-    ASSERT_EQ(compaction_job_stats_.num_input_files, files.size());
+    ASSERT_EQ(compaction_job_stats_.num_input_files, num_input_files);
     ASSERT_EQ(compaction_job_stats_.num_output_files, 1U);
+    mock_table_factory_->AssertLatestFile(expected_results);
   }
 
   Env* env_;
@@ -230,26 +277,146 @@ class CompactionJobTest : public testing::Test {
   std::atomic<bool> shutting_down_;
   std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
   CompactionJobStats compaction_job_stats_;
+  ColumnFamilyData* cfd_;
 };
 
 TEST_F(CompactionJobTest, Simple) {
+  NewDB();
+
   auto expected_results = CreateTwoFiles(false);
   auto cfd = versions_->GetColumnFamilySet()->GetDefault();
   auto files = cfd->current()->storage_info()->LevelFiles(0);
   ASSERT_EQ(2U, files.size());
-
-  RunCompaction(files);
-  mock_table_factory_->AssertLatestFile(expected_results);
+  RunCompaction({ files }, expected_results);
 }
 
 TEST_F(CompactionJobTest, SimpleCorrupted) {
+  NewDB();
+
   auto expected_results = CreateTwoFiles(true);
   auto cfd = versions_->GetColumnFamilySet()->GetDefault();
   auto files = cfd->current()->storage_info()->LevelFiles(0);
-
-  RunCompaction(files);
+  RunCompaction({ files }, expected_results);
   ASSERT_EQ(compaction_job_stats_.num_corrupt_keys, 400U);
-  mock_table_factory_->AssertLatestFile(expected_results);
+}
+
+TEST_F(CompactionJobTest, SimpleDeletion) {
+  NewDB();
+
+  auto file1 = mock::MakeMockFile({{KeyStr("c", 4U, kTypeDeletion), ""},
+                                   {KeyStr("c", 3U, kTypeValue), "val"}});
+  AddMockFile(file1);
+
+  auto file2 = mock::MakeMockFile({{KeyStr("b", 2U, kTypeValue), "val"},
+                                   {KeyStr("b", 1U, kTypeValue), "val"}});
+  AddMockFile(file2);
+
+  auto expected_results =
+      mock::MakeMockFile({{KeyStr("b", 0U, kTypeValue), "val"}});
+
+  SetLastSequence(4U);
+  auto files = cfd_->current()->storage_info()->LevelFiles(0);
+  RunCompaction({ files }, expected_results);
+}
+
+TEST_F(CompactionJobTest, SimpleOverwrite) {
+  NewDB();
+
+  auto file1 = mock::MakeMockFile({
+      {KeyStr("a", 3U, kTypeValue), "val2"},
+      {KeyStr("b", 4U, kTypeValue), "val3"},
+  });
+  AddMockFile(file1);
+
+  auto file2 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"},
+                                   {KeyStr("b", 2U, kTypeValue), "val"}});
+  AddMockFile(file2);
+
+  auto expected_results =
+      mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "val2"},
+                          {KeyStr("b", 0U, kTypeValue), "val3"}});
+
+  SetLastSequence(4U);
+  auto files = cfd_->current()->storage_info()->LevelFiles(0);
+  RunCompaction({ files }, expected_results);
+}
+
+TEST_F(CompactionJobTest, SimpleNonLastLevel) {
+  NewDB();
+
+  auto file1 = mock::MakeMockFile({
+      {KeyStr("a", 5U, kTypeValue), "val2"},
+      {KeyStr("b", 6U, kTypeValue), "val3"},
+  });
+  AddMockFile(file1);
+
+  auto file2 = mock::MakeMockFile({{KeyStr("a", 3U, kTypeValue), "val"},
+                                   {KeyStr("b", 4U, kTypeValue), "val"}});
+  AddMockFile(file2, 1);
+
+  auto file3 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"},
+                                   {KeyStr("b", 2U, kTypeValue), "val"}});
+  AddMockFile(file3, 2);
+
+  // Because level 1 is not the last level, the sequence numbers of a and b
+  // cannot be set to 0
+  auto expected_results =
+      mock::MakeMockFile({{KeyStr("a", 5U, kTypeValue), "val2"},
+                          {KeyStr("b", 6U, kTypeValue), "val3"}});
+
+  SetLastSequence(6U);
+  auto lvl0_files = cfd_->current()->storage_info()->LevelFiles(0);
+  auto lvl1_files = cfd_->current()->storage_info()->LevelFiles(1);
+  RunCompaction({ lvl0_files, lvl1_files }, expected_results);
+}
+
+TEST_F(CompactionJobTest, SimpleMerge) {
+  auto merge_op = MergeOperators::CreateStringAppendOperator();
+  NewDB(merge_op);
+
+  auto file1 = mock::MakeMockFile({
+      {KeyStr("a", 5U, kTypeMerge), "5"},
+      {KeyStr("a", 4U, kTypeMerge), "4"},
+      {KeyStr("a", 3U, kTypeValue), "3"},
+  });
+  AddMockFile(file1);
+
+  auto file2 = mock::MakeMockFile(
+      {{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeValue), "1"}});
+  AddMockFile(file2);
+
+  auto expected_results =
+      mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
+                          {KeyStr("b", 0U, kTypeValue), "1,2"}});
+
+  SetLastSequence(5U);
+  auto files = cfd_->current()->storage_info()->LevelFiles(0);
+  RunCompaction({ files }, expected_results);
+}
+
+TEST_F(CompactionJobTest, NonAssocMerge) {
+  auto merge_op = MergeOperators::CreateStringAppendTESTOperator();
+  NewDB(merge_op);
+
+  auto file1 = mock::MakeMockFile({
+      {KeyStr("a", 5U, kTypeMerge), "5"},
+      {KeyStr("a", 4U, kTypeMerge), "4"},
+      {KeyStr("a", 3U, kTypeMerge), "3"},
+  });
+  AddMockFile(file1);
+
+  auto file2 = mock::MakeMockFile(
+      {{KeyStr("b", 2U, kTypeMerge), "2"}, {KeyStr("b", 1U, kTypeMerge), "1"}});
+  AddMockFile(file2);
+
+  auto expected_results =
+      mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
+                          {KeyStr("b", 2U, kTypeMerge), "2"},
+                          {KeyStr("b", 1U, kTypeMerge), "1"}});
+
+  SetLastSequence(5U);
+  auto files = cfd_->current()->storage_info()->LevelFiles(0);
+  RunCompaction({ files }, expected_results);
 }
 
 }  // namespace rocksdb

db/compaction_picker.cc

@@ -261,10 +261,10 @@ Compaction* CompactionPicker::FormCompaction(
           mutable_cf_options.MaxGrandParentOverlapBytes(output_level + 1) :
           std::numeric_limits<uint64_t>::max();
   assert(input_files.size());
-  return new Compaction(vstorage, mutable_cf_options, input_files, output_level,
-                        compact_options.output_file_size_limit,
-                        max_grandparent_overlap_bytes, output_path_id,
-                        compact_options.compression, /* grandparents */ {});
+  return new Compaction(
+      vstorage, mutable_cf_options, input_files, output_level,
+      compact_options.output_file_size_limit, max_grandparent_overlap_bytes,
+      output_path_id, compact_options.compression, /* grandparents */ {}, true);
 }
 
 Status CompactionPicker::GetCompactionInputsFromFileNumbers(

db/compaction_picker.h

@@ -8,22 +8,20 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
-#include <vector>
-#include <memory>
-#include <set>
-#include <unordered_set>
 
-#include "db/version_set.h"
-#include "db/compaction.h"
-#include "rocksdb/status.h"
-#include "rocksdb/options.h"
-#include "rocksdb/env.h"
-#include "util/mutable_cf_options.h"
-
-#include <vector>
 #include <memory>
 #include <set>
 #include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "db/compaction.h"
+#include "db/version_set.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+#include "util/mutable_cf_options.h"
+
 
 namespace rocksdb {
 
@@ -90,7 +88,8 @@ class CompactionPicker {
   // Returns true if any one of the specified files are being compacted
   bool FilesInCompaction(const std::vector<FileMetaData*>& files);
 
-  // Takes a list of CompactionInputFiles and returns a Compaction object.
+  // Takes a list of CompactionInputFiles and returns a (manual) Compaction
+  // object.
   Compaction* FormCompaction(
       const CompactionOptions& compact_options,
       const std::vector<CompactionInputFiles>& input_files, int output_level,

db/compaction_picker_test.cc

@@ -552,6 +552,91 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys3) {
   ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
 }
 
+TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) {
+  int num_levels = ioptions_.num_levels;
+  ioptions_.level_compaction_dynamic_level_bytes = false;
+  mutable_cf_options_.level0_file_num_compaction_trigger = 3;
+  mutable_cf_options_.max_bytes_for_level_base = 1000;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  NewVersionStorage(num_levels, kCompactionStyleLevel);
+  Add(0, 1U, "150", "200", 200);
+  Add(0, 2U, "150", "200", 200);
+  Add(0, 3U, "150", "200", 200);
+  // Level 1 is over target by 200
+  Add(1, 4U, "400", "500", 600);
+  Add(1, 5U, "600", "700", 600);
+  // Level 2 is less than target 10000 even added size of level 1
+  Add(2, 6U, "150", "200", 2500);
+  Add(2, 7U, "201", "210", 2000);
+  Add(2, 8U, "300", "310", 2500);
+  Add(2, 9U, "400", "500", 2500);
+  // Level 3 exceeds target 100,000 of 1000
+  Add(3, 10U, "400", "500", 101000);
+  // Level 4 exceeds target 1,000,000 of 500 after adding size from level 3
+  Add(4, 11U, "400", "500", 999500);
+  Add(5, 11U, "400", "500", 8000000);
+
+  UpdateVersionStorageInfo();
+
+  ASSERT_EQ(2200u + 11000u + 5500u,
+            vstorage_->estimated_compaction_needed_bytes());
+}
+
+TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded2) {
+  int num_levels = ioptions_.num_levels;
+  ioptions_.level_compaction_dynamic_level_bytes = false;
+  mutable_cf_options_.level0_file_num_compaction_trigger = 3;
+  mutable_cf_options_.max_bytes_for_level_base = 1000;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  NewVersionStorage(num_levels, kCompactionStyleLevel);
+  Add(0, 1U, "150", "200", 200);
+  Add(0, 2U, "150", "200", 200);
+  Add(0, 4U, "150", "200", 200);
+  Add(0, 5U, "150", "200", 200);
+  Add(0, 6U, "150", "200", 200);
+  // Level 1 is over target by
+  Add(1, 7U, "400", "500", 200);
+  Add(1, 8U, "600", "700", 200);
+  // Level 2 is less than target 10000 even added size of level 1
+  Add(2, 9U, "150", "200", 9500);
+  Add(3, 10U, "400", "500", 101000);
+
+  UpdateVersionStorageInfo();
+
+  ASSERT_EQ(1400u + 4400u + 11000u,
+            vstorage_->estimated_compaction_needed_bytes());
+}
+
+TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
+  int num_levels = ioptions_.num_levels;
+  ioptions_.level_compaction_dynamic_level_bytes = true;
+  mutable_cf_options_.level0_file_num_compaction_trigger = 3;
+  mutable_cf_options_.max_bytes_for_level_base = 1000;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  NewVersionStorage(num_levels, kCompactionStyleLevel);
+
+  // Set Last level size 50000
+  // num_levels - 1 target 5000
+  // num_levels - 2 is base level with taret 500
+  Add(num_levels - 1, 10U, "400", "500", 50000);
+
+  Add(0, 1U, "150", "200", 200);
+  Add(0, 2U, "150", "200", 200);
+  Add(0, 4U, "150", "200", 200);
+  Add(0, 5U, "150", "200", 200);
+  Add(0, 6U, "150", "200", 200);
+  // num_levels - 3 is over target by 100 + 1000
+  Add(num_levels - 3, 7U, "400", "500", 300);
+  Add(num_levels - 3, 8U, "600", "700", 300);
+  // Level 2 is over target by 1100 + 100
+  Add(num_levels - 2, 9U, "150", "200", 5100);
+
+  UpdateVersionStorageInfo();
+
+  ASSERT_EQ(1600u + 12100u + 13200u,
+            vstorage_->estimated_compaction_needed_bytes());
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/comparator_db_test.cc

@@ -10,6 +10,7 @@
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "util/hash.h"
+#include "util/stl_wrappers.h"
 #include "util/string_util.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
@@ -22,18 +23,10 @@ namespace {
 
 static const Comparator* comparator;
 
-// A comparator for std::map, using comparator
-struct MapComparator {
-  bool operator()(const std::string& a, const std::string& b) const {
-    return comparator->Compare(a, b) < 0;
-  }
-};
-
-typedef std::map<std::string, std::string, MapComparator> KVMap;
-
 class KVIter : public Iterator {
  public:
-  explicit KVIter(const KVMap* map) : map_(map), iter_(map_->end()) {}
+  explicit KVIter(const stl_wrappers::KVMap* map)
+      : map_(map), iter_(map_->end()) {}
   virtual bool Valid() const override { return iter_ != map_->end(); }
   virtual void SeekToFirst() override { iter_ = map_->begin(); }
   virtual void SeekToLast() override {
@@ -60,8 +53,8 @@ class KVIter : public Iterator {
   virtual Status status() const override { return Status::OK(); }
 
  private:
-  const KVMap* const map_;
-  KVMap::const_iterator iter_;
+  const stl_wrappers::KVMap* const map_;
+  stl_wrappers::KVMap::const_iterator iter_;
 };
 
 void AssertItersEqual(Iterator* iter1, Iterator* iter2) {
@@ -77,7 +70,7 @@ void AssertItersEqual(Iterator* iter1, Iterator* iter2) {
 void DoRandomIteraratorTest(DB* db, std::vector<std::string> source_strings,
                             Random* rnd, int num_writes, int num_iter_ops,
                             int num_trigger_flush) {
-  KVMap map;
+  stl_wrappers::KVMap map((stl_wrappers::LessOfComparator(comparator)));
 
   for (int i = 0; i < num_writes; i++) {
     if (num_trigger_flush > 0 && i != 0 && i % num_trigger_flush == 0) {

db/db_bench.cc

@@ -54,7 +54,8 @@ int main() {
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/utilities/flashcache.h"
-#include "rocksdb/utilities/optimistic_transaction.h"
+#include "rocksdb/utilities/transaction.h"
+#include "rocksdb/utilities/transaction_db.h"
 #include "rocksdb/utilities/optimistic_transaction_db.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -111,8 +112,8 @@ DEFINE_string(benchmarks,
               "fillseekseq,"
               "randomtransaction",
 
-              "Comma-separated list of operations to run in the specified order"
-              "Actual benchmarks:\n"
+              "Comma-separated list of operations to run in the specified"
+              " order. Available benchmarks:\n"
               "\tfillseq       -- write N values in sequential key"
               " order in async mode\n"
               "\tfillrandom    -- write N values in random key order in async"
@@ -220,6 +221,15 @@ static bool ValidateKeySize(const char* flagname, int32_t value) {
   return true;
 }
 
+static bool ValidateUint32Range(const char* flagname, uint64_t value) {
+  if (value > std::numeric_limits<uint32_t>::max()) {
+    fprintf(stderr, "Invalid value for --%s: %lu, overflow\n", flagname,
+            (unsigned long)value);
+    return false;
+  }
+  return true;
+}
+
 DEFINE_int32(key_size, 16, "size of each key");
 
 DEFINE_int32(num_multi_db, 0,
@@ -285,6 +295,13 @@ DEFINE_int32(max_background_compactions,
              "The maximum number of concurrent background compactions"
              " that can occur in parallel.");
 
+DEFINE_uint64(subcompactions, 1,
+              "Maximum number of subcompactions to divide L0-L1 compactions "
+              "into.");
+static const bool FLAGS_subcompactions_dummy
+    __attribute__((unused)) = RegisterFlagValidator(&FLAGS_subcompactions,
+                                                    &ValidateUint32Range);
+
 DEFINE_int32(max_background_flushes,
              rocksdb::Options().max_background_flushes,
              "The maximum number of concurrent background flushes"
@@ -312,7 +329,7 @@ DEFINE_int32(universal_compression_size_percent, -1,
              "compaction. -1 means compress everything.");
 
 DEFINE_bool(universal_allow_trivial_move, false,
-            "Sllow trivial move in universal compaction.");
+            "Allow trivial move in universal compaction.");
 
 DEFINE_int64(cache_size, -1, "Number of bytes to use as a cache of uncompressed"
              "data. Negative means use default settings.");
@@ -340,6 +357,11 @@ DEFINE_int32(open_files, rocksdb::Options().max_open_files,
              "Maximum number of files to keep open at the same time"
              " (use default if == 0)");
 
+DEFINE_int32(new_table_reader_for_compaction_inputs, true,
+             "If true, uses a separate file handle for compaction inputs");
+
+DEFINE_int32(compaction_readahead_size, 0, "Compaction readahead size");
+
 DEFINE_int32(bloom_bits, -1, "Bloom filter bits per key. Negative means"
              " use default settings.");
 DEFINE_int32(memtable_bloom_bits, 0, "Bloom filter bits per key for memtable. "
@@ -349,6 +371,11 @@ DEFINE_bool(use_existing_db, false, "If true, do not destroy the existing"
             " database.  If you set this flag and also specify a benchmark that"
             " wants a fresh database, that benchmark will fail.");
 
+DEFINE_bool(show_table_properties, false,
+            "If true, then per-level table"
+            " properties will be printed on every stats-interval when"
+            " stats_interval is set and stats_per_interval is on.");
+
 DEFINE_string(db, "", "Use the db with the following name.");
 
 static bool ValidateCacheNumshardbits(const char* flagname, int32_t value) {
@@ -448,18 +475,33 @@ DEFINE_int32(deletepercent, 2, "Percentage of deletes out of reads/writes/"
 DEFINE_uint64(delete_obsolete_files_period_micros, 0,
               "Ignored. Left here for backward compatibility");
 
-DEFINE_bool(transaction_db, false,
+DEFINE_bool(optimistic_transaction_db, false,
             "Open a OptimisticTransactionDB instance. "
             "Required for randomtransaction benchmark.");
 
+DEFINE_bool(transaction_db, false,
+            "Open a TransactionDB instance. "
+            "Required for randomtransaction benchmark.");
+
 DEFINE_uint64(transaction_sets, 2,
               "Number of keys each transaction will "
               "modify (use in RandomTransaction only).  Max: 9999");
 
+DEFINE_bool(transaction_set_snapshot, false,
+            "Setting to true will have each transaction call SetSnapshot()"
+            " upon creation.");
+
 DEFINE_int32(transaction_sleep, 0,
              "Max microseconds to sleep in between "
              "reading and writing a value (used in RandomTransaction only). ");
 
+DEFINE_uint64(transaction_lock_timeout, 100,
+              "If using a transaction_db, specifies the lock wait timeout in"
+              " milliseconds before failing a transaction waiting on a lock");
+
+DEFINE_bool(compaction_measure_io_stats, false,
+            "Measure times spents on I/Os while in compactions. ");
+
 namespace {
 enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
   assert(ctype);
@@ -476,6 +518,8 @@ enum rocksdb::CompressionType StringToCompressionType(const char* ctype) {
     return rocksdb::kLZ4Compression;
   else if (!strcasecmp(ctype, "lz4hc"))
     return rocksdb::kLZ4HCCompression;
+  else if (!strcasecmp(ctype, "zstd"))
+    return rocksdb::kZSTDNotFinalCompression;
 
   fprintf(stdout, "Cannot parse compression type '%s'\n", ctype);
   return rocksdb::kSnappyCompression; //default value
@@ -919,7 +963,7 @@ static void AppendWithSpace(std::string* str, Slice msg) {
 struct DBWithColumnFamilies {
   std::vector<ColumnFamilyHandle*> cfh;
   DB* db;
-  OptimisticTransactionDB* txn_db;
+  OptimisticTransactionDB* opt_txn_db;
   std::atomic<size_t> num_created;  // Need to be updated after all the
                                     // new entries in cfh are set.
   size_t num_hot;  // Number of column families to be queried at each moment.
@@ -927,7 +971,7 @@ struct DBWithColumnFamilies {
                    // Column families will be created and used to be queried.
   port::Mutex create_cf_mutex;  // Only one thread can execute CreateNewCf()
 
-  DBWithColumnFamilies() : db(nullptr), txn_db(nullptr) {
+  DBWithColumnFamilies() : db(nullptr), opt_txn_db(nullptr) {
     cfh.clear();
     num_created = 0;
     num_hot = 0;
@@ -936,7 +980,7 @@ struct DBWithColumnFamilies {
   DBWithColumnFamilies(const DBWithColumnFamilies& other)
       : cfh(other.cfh),
         db(other.db),
-        txn_db(other.txn_db),
+        opt_txn_db(other.opt_txn_db),
         num_created(other.num_created.load()),
         num_hot(other.num_hot) {}
 
@@ -944,9 +988,9 @@ struct DBWithColumnFamilies {
     std::for_each(cfh.begin(), cfh.end(),
                   [](ColumnFamilyHandle* cfhi) { delete cfhi; });
     cfh.clear();
-    if (txn_db) {
-      delete txn_db;
-      txn_db = nullptr;
+    if (opt_txn_db) {
+      delete opt_txn_db;
+      opt_txn_db = nullptr;
     } else {
       delete db;
     }
@@ -1230,10 +1274,37 @@ class Stats {
                 if (db->GetProperty(db_with_cfh->cfh[i], "rocksdb.cfstats",
                                     &stats))
                   fprintf(stderr, "%s\n", stats.c_str());
+                if (FLAGS_show_table_properties) {
+                  for (int level = 0; level < FLAGS_num_levels; ++level) {
+                    if (db->GetProperty(
+                            db_with_cfh->cfh[i],
+                            "rocksdb.aggregated-table-properties-at-level" +
+                                ToString(level),
+                            &stats)) {
+                      if (stats.find("# entries=0") == std::string::npos) {
+                        fprintf(stderr, "Level[%d]: %s\n", level,
+                                stats.c_str());
+                      }
+                    }
+                  }
+                }
+              }
+            } else if (db) {
+              if (db->GetProperty("rocksdb.stats", &stats)) {
+                fprintf(stderr, "%s\n", stats.c_str());
+              }
+              if (FLAGS_show_table_properties) {
+                for (int level = 0; level < FLAGS_num_levels; ++level) {
+                  if (db->GetProperty(
+                          "rocksdb.aggregated-table-properties-at-level" +
+                              ToString(level),
+                          &stats)) {
+                    if (stats.find("# entries=0") == std::string::npos) {
+                      fprintf(stderr, "Level[%d]: %s\n", level, stats.c_str());
+                    }
+                  }
+                }
               }
-
-            } else if (db && db->GetProperty("rocksdb.stats", &stats)) {
-              fprintf(stderr, "%s\n", stats.c_str());
             }
           }
 
@@ -1403,6 +1474,39 @@ class Benchmark {
     return true;
   }
 
+  inline bool CompressSlice(const Slice& input, std::string* compressed) {
+    bool ok = true;
+    switch (FLAGS_compression_type_e) {
+      case rocksdb::kSnappyCompression:
+        ok = Snappy_Compress(Options().compression_opts, input.data(),
+                             input.size(), compressed);
+        break;
+      case rocksdb::kZlibCompression:
+        ok = Zlib_Compress(Options().compression_opts, 2, input.data(),
+                           input.size(), compressed);
+        break;
+      case rocksdb::kBZip2Compression:
+        ok = BZip2_Compress(Options().compression_opts, 2, input.data(),
+                            input.size(), compressed);
+        break;
+      case rocksdb::kLZ4Compression:
+        ok = LZ4_Compress(Options().compression_opts, 2, input.data(),
+                          input.size(), compressed);
+        break;
+      case rocksdb::kLZ4HCCompression:
+        ok = LZ4HC_Compress(Options().compression_opts, 2, input.data(),
+                            input.size(), compressed);
+        break;
+      case rocksdb::kZSTDNotFinalCompression:
+        ok = ZSTD_Compress(Options().compression_opts, input.data(),
+                           input.size(), compressed);
+        break;
+      default:
+        ok = false;
+    }
+    return ok;
+  }
+
   void PrintHeader() {
     PrintEnvironment();
     fprintf(stdout, "Keys:       %d bytes each\n", FLAGS_key_size);
@@ -1432,26 +1536,10 @@ class Benchmark {
       }
 #endif
     }
-    switch (FLAGS_compression_type_e) {
-      case rocksdb::kNoCompression:
-        fprintf(stdout, "Compression: none\n");
-        break;
-      case rocksdb::kSnappyCompression:
-        fprintf(stdout, "Compression: snappy\n");
-        break;
-      case rocksdb::kZlibCompression:
-        fprintf(stdout, "Compression: zlib\n");
-        break;
-      case rocksdb::kBZip2Compression:
-        fprintf(stdout, "Compression: bzip2\n");
-        break;
-      case rocksdb::kLZ4Compression:
-        fprintf(stdout, "Compression: lz4\n");
-        break;
-      case rocksdb::kLZ4HCCompression:
-        fprintf(stdout, "Compression: lz4hc\n");
-        break;
-    }
+
+    const char* compression =
+        CompressionTypeToString(FLAGS_compression_type_e).c_str();
+    fprintf(stdout, "Compression: %s\n", compression);
 
     switch (FLAGS_rep_factory) {
       case kPrefixHash:
@@ -1472,11 +1560,11 @@ class Benchmark {
     }
     fprintf(stdout, "Perf Level: %d\n", FLAGS_perf_level);
 
-    PrintWarnings();
+    PrintWarnings(compression);
     fprintf(stdout, "------------------------------------------------\n");
   }
 
-  void PrintWarnings() {
+  void PrintWarnings(const char* compression) {
 #if defined(__GNUC__) && !defined(__OPTIMIZE__)
     fprintf(stdout,
             "WARNING: Optimization is disabled: benchmarks unnecessarily slow\n"
@@ -1489,51 +1577,17 @@ class Benchmark {
     if (FLAGS_compression_type_e != rocksdb::kNoCompression) {
       // The test string should not be too small.
       const int len = FLAGS_block_size;
-      char* text = (char*) malloc(len+1);
-      bool result = true;
-      const char* name = nullptr;
+      std::string input_str(len, 'y');
       std::string compressed;
-
-      memset(text, (int) 'y', len);
-      text[len] = '\0';
-      switch (FLAGS_compression_type_e) {
-        case kSnappyCompression:
-          result = Snappy_Compress(Options().compression_opts, text,
-                                   strlen(text), &compressed);
-          name = "Snappy";
-          break;
-        case kZlibCompression:
-          result = Zlib_Compress(Options().compression_opts, 2, text,
-                                 strlen(text), &compressed);
-          name = "Zlib";
-          break;
-        case kBZip2Compression:
-          result = BZip2_Compress(Options().compression_opts, 2, text,
-                                  strlen(text), &compressed);
-          name = "BZip2";
-          break;
-        case kLZ4Compression:
-          result = LZ4_Compress(Options().compression_opts, 2, text,
-                                strlen(text), &compressed);
-          name = "LZ4";
-          break;
-        case kLZ4HCCompression:
-          result = LZ4HC_Compress(Options().compression_opts, 2, text,
-                                  strlen(text), &compressed);
-          name = "LZ4HC";
-          break;
-        case kNoCompression:
-          assert(false); // cannot happen
-          break;
-      }
+      bool result = CompressSlice(Slice(input_str), &compressed);
 
       if (!result) {
-        fprintf(stdout, "WARNING: %s compression is not enabled\n", name);
-      } else if (name && compressed.size() >= strlen(text)) {
-        fprintf(stdout, "WARNING: %s compression is not effective\n", name);
+        fprintf(stdout, "WARNING: %s compression is not enabled\n",
+                compression);
+      } else if (compressed.size() >= input_str.size()) {
+        fprintf(stdout, "WARNING: %s compression is not effective\n",
+                compression);
       }
-
-      free(text);
     }
   }
 
@@ -1731,18 +1785,9 @@ class Benchmark {
     }
     PrintHeader();
     Open(&open_options_);
-    const char* benchmarks = FLAGS_benchmarks.c_str();
-    while (benchmarks != nullptr) {
-      const char* sep = strchr(benchmarks, ',');
-      Slice name;
-      if (sep == nullptr) {
-        name = benchmarks;
-        benchmarks = nullptr;
-      } else {
-        name = Slice(benchmarks, sep - benchmarks);
-        benchmarks = sep + 1;
-      }
-
+    std::stringstream benchmark_stream(FLAGS_benchmarks);
+    std::string name;
+    while (std::getline(benchmark_stream, name, ',')) {
       // Sanitize parameters
       num_ = FLAGS_num;
       reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
@@ -1763,138 +1808,136 @@ class Benchmark {
       bool fresh_db = false;
       int num_threads = FLAGS_threads;
 
-      if (name == Slice("fillseq")) {
+      if (name == "fillseq") {
         fresh_db = true;
         method = &Benchmark::WriteSeq;
-      } else if (name == Slice("fillbatch")) {
+      } else if (name == "fillbatch") {
         fresh_db = true;
         entries_per_batch_ = 1000;
         method = &Benchmark::WriteSeq;
-      } else if (name == Slice("fillrandom")) {
+      } else if (name == "fillrandom") {
         fresh_db = true;
         method = &Benchmark::WriteRandom;
-      } else if (name == Slice("filluniquerandom")) {
+      } else if (name == "filluniquerandom") {
         fresh_db = true;
         if (num_threads > 1) {
-          fprintf(stderr, "filluniquerandom multithreaded not supported"
-                           ", use 1 thread");
+          fprintf(stderr,
+                  "filluniquerandom multithreaded not supported"
+                  ", use 1 thread");
           num_threads = 1;
         }
         method = &Benchmark::WriteUniqueRandom;
-      } else if (name == Slice("overwrite")) {
-        fresh_db = false;
+      } else if (name == "overwrite") {
         method = &Benchmark::WriteRandom;
-      } else if (name == Slice("fillsync")) {
+      } else if (name == "fillsync") {
         fresh_db = true;
         num_ /= 1000;
         write_options_.sync = true;
         method = &Benchmark::WriteRandom;
-      } else if (name == Slice("fill100K")) {
+      } else if (name == "fill100K") {
         fresh_db = true;
         num_ /= 1000;
         value_size_ = 100 * 1000;
         method = &Benchmark::WriteRandom;
-      } else if (name == Slice("readseq")) {
+      } else if (name == "readseq") {
         method = &Benchmark::ReadSequential;
-      } else if (name == Slice("readtocache")) {
+      } else if (name == "readtocache") {
         method = &Benchmark::ReadSequential;
         num_threads = 1;
         reads_ = num_;
-      } else if (name == Slice("readreverse")) {
+      } else if (name == "readreverse") {
         method = &Benchmark::ReadReverse;
-      } else if (name == Slice("readrandom")) {
+      } else if (name == "readrandom") {
         method = &Benchmark::ReadRandom;
-      } else if (name == Slice("readrandomfast")) {
+      } else if (name == "readrandomfast") {
         method = &Benchmark::ReadRandomFast;
-      } else if (name == Slice("multireadrandom")) {
+      } else if (name == "multireadrandom") {
         fprintf(stderr, "entries_per_batch = %" PRIi64 "\n",
                 entries_per_batch_);
         method = &Benchmark::MultiReadRandom;
-      } else if (name == Slice("readmissing")) {
+      } else if (name == "readmissing") {
         ++key_size_;
         method = &Benchmark::ReadRandom;
-      } else if (name == Slice("newiterator")) {
+      } else if (name == "newiterator") {
         method = &Benchmark::IteratorCreation;
-      } else if (name == Slice("newiteratorwhilewriting")) {
+      } else if (name == "newiteratorwhilewriting") {
         num_threads++;  // Add extra thread for writing
         method = &Benchmark::IteratorCreationWhileWriting;
-      } else if (name == Slice("seekrandom")) {
+      } else if (name == "seekrandom") {
         method = &Benchmark::SeekRandom;
-      } else if (name == Slice("seekrandomwhilewriting")) {
+      } else if (name == "seekrandomwhilewriting") {
         num_threads++;  // Add extra thread for writing
         method = &Benchmark::SeekRandomWhileWriting;
-      } else if (name == Slice("seekrandomwhilemerging")) {
+      } else if (name == "seekrandomwhilemerging") {
         num_threads++;  // Add extra thread for merging
         method = &Benchmark::SeekRandomWhileMerging;
-      } else if (name == Slice("readrandomsmall")) {
+      } else if (name == "readrandomsmall") {
         reads_ /= 1000;
         method = &Benchmark::ReadRandom;
-      } else if (name == Slice("deleteseq")) {
+      } else if (name == "deleteseq") {
         method = &Benchmark::DeleteSeq;
-      } else if (name == Slice("deleterandom")) {
+      } else if (name == "deleterandom") {
         method = &Benchmark::DeleteRandom;
-      } else if (name == Slice("readwhilewriting")) {
+      } else if (name == "readwhilewriting") {
         num_threads++;  // Add extra thread for writing
         method = &Benchmark::ReadWhileWriting;
-      } else if (name == Slice("readwhilemerging")) {
+      } else if (name == "readwhilemerging") {
         num_threads++;  // Add extra thread for writing
         method = &Benchmark::ReadWhileMerging;
-      } else if (name == Slice("readrandomwriterandom")) {
+      } else if (name == "readrandomwriterandom") {
         method = &Benchmark::ReadRandomWriteRandom;
-      } else if (name == Slice("readrandommergerandom")) {
+      } else if (name == "readrandommergerandom") {
         if (FLAGS_merge_operator.empty()) {
           fprintf(stdout, "%-12s : skipped (--merge_operator is unknown)\n",
-                  name.ToString().c_str());
+                  name.c_str());
           exit(1);
         }
         method = &Benchmark::ReadRandomMergeRandom;
-      } else if (name == Slice("updaterandom")) {
+      } else if (name == "updaterandom") {
         method = &Benchmark::UpdateRandom;
-      } else if (name == Slice("appendrandom")) {
+      } else if (name == "appendrandom") {
         method = &Benchmark::AppendRandom;
-      } else if (name == Slice("mergerandom")) {
+      } else if (name == "mergerandom") {
         if (FLAGS_merge_operator.empty()) {
           fprintf(stdout, "%-12s : skipped (--merge_operator is unknown)\n",
-                  name.ToString().c_str());
+                  name.c_str());
           exit(1);
         }
         method = &Benchmark::MergeRandom;
-      } else if (name == Slice("randomwithverify")) {
+      } else if (name == "randomwithverify") {
         method = &Benchmark::RandomWithVerify;
-      } else if (name == Slice("fillseekseq")) {
+      } else if (name == "fillseekseq") {
         method = &Benchmark::WriteSeqSeekSeq;
-      } else if (name == Slice("compact")) {
+      } else if (name == "compact") {
         method = &Benchmark::Compact;
-      } else if (name == Slice("crc32c")) {
+      } else if (name == "crc32c") {
         method = &Benchmark::Crc32c;
-      } else if (name == Slice("xxhash")) {
+      } else if (name == "xxhash") {
         method = &Benchmark::xxHash;
-      } else if (name == Slice("acquireload")) {
+      } else if (name == "acquireload") {
         method = &Benchmark::AcquireLoad;
-      } else if (name == Slice("compress")) {
+      } else if (name == "compress") {
         method = &Benchmark::Compress;
-      } else if (name == Slice("uncompress")) {
+      } else if (name == "uncompress") {
         method = &Benchmark::Uncompress;
-      } else if (name == Slice("randomtransaction")) {
+      } else if (name == "randomtransaction") {
         method = &Benchmark::RandomTransaction;
         post_process_method = &Benchmark::RandomTransactionVerify;
-      } else if (name == Slice("stats")) {
+      } else if (name == "stats") {
         PrintStats("rocksdb.stats");
-      } else if (name == Slice("levelstats")) {
+      } else if (name == "levelstats") {
         PrintStats("rocksdb.levelstats");
-      } else if (name == Slice("sstables")) {
+      } else if (name == "sstables") {
         PrintStats("rocksdb.sstables");
-      } else {
-        if (name != Slice()) {  // No error message for empty name
-          fprintf(stderr, "unknown benchmark '%s'\n", name.ToString().c_str());
-          exit(1);
-        }
+      } else if (!name.empty()) {  // No error message for empty name
+        fprintf(stderr, "unknown benchmark '%s'\n", name.c_str());
+        exit(1);
       }
 
       if (fresh_db) {
         if (FLAGS_use_existing_db) {
           fprintf(stdout, "%-12s : skipped (--use_existing_db is true)\n",
-                  name.ToString().c_str());
+                  name.c_str());
           method = nullptr;
         } else {
           if (db_.db != nullptr) {
@@ -2098,30 +2141,7 @@ class Benchmark {
 
     // Compress 1G
     while (ok && bytes < int64_t(1) << 30) {
-      switch (FLAGS_compression_type_e) {
-      case rocksdb::kSnappyCompression:
-        ok = Snappy_Compress(Options().compression_opts, input.data(),
-                             input.size(), &compressed);
-        break;
-      case rocksdb::kZlibCompression:
-        ok = Zlib_Compress(Options().compression_opts, 2, input.data(),
-                           input.size(), &compressed);
-        break;
-      case rocksdb::kBZip2Compression:
-        ok = BZip2_Compress(Options().compression_opts, 2, input.data(),
-                            input.size(), &compressed);
-        break;
-      case rocksdb::kLZ4Compression:
-        ok = LZ4_Compress(Options().compression_opts, 2, input.data(),
-                          input.size(), &compressed);
-        break;
-      case rocksdb::kLZ4HCCompression:
-        ok = LZ4HC_Compress(Options().compression_opts, 2, input.data(),
-                            input.size(), &compressed);
-        break;
-      default:
-        ok = false;
-      }
+      ok = CompressSlice(input, &compressed);
       produced += compressed.size();
       bytes += input.size();
       thread->stats.FinishedOps(nullptr, nullptr, 1);
@@ -2143,32 +2163,7 @@ class Benchmark {
     Slice input = gen.Generate(FLAGS_block_size);
     std::string compressed;
 
-    bool ok;
-    switch (FLAGS_compression_type_e) {
-    case rocksdb::kSnappyCompression:
-      ok = Snappy_Compress(Options().compression_opts, input.data(),
-                           input.size(), &compressed);
-      break;
-    case rocksdb::kZlibCompression:
-      ok = Zlib_Compress(Options().compression_opts, 2, input.data(),
-                         input.size(), &compressed);
-      break;
-    case rocksdb::kBZip2Compression:
-      ok = BZip2_Compress(Options().compression_opts, 2, input.data(),
-                          input.size(), &compressed);
-      break;
-    case rocksdb::kLZ4Compression:
-      ok = LZ4_Compress(Options().compression_opts, 2, input.data(),
-                        input.size(), &compressed);
-      break;
-    case rocksdb::kLZ4HCCompression:
-      ok = LZ4HC_Compress(Options().compression_opts, 2, input.data(),
-                          input.size(), &compressed);
-      break;
-    default:
-      ok = false;
-    }
-
+    bool ok = CompressSlice(input, &compressed);
     int64_t bytes = 0;
     int decompress_size;
     while (ok && bytes < 1024 * 1048576) {
@@ -2200,6 +2195,11 @@ class Benchmark {
                                       &decompress_size, 2);
         ok = uncompressed != nullptr;
         break;
+      case rocksdb::kZSTDNotFinalCompression:
+        uncompressed = ZSTD_Uncompress(compressed.data(), compressed.size(),
+                                       &decompress_size);
+        ok = uncompressed != nullptr;
+        break;
       default:
         ok = false;
       }
@@ -2230,6 +2230,7 @@ class Benchmark {
     options.max_write_buffer_number_to_maintain =
         FLAGS_max_write_buffer_number_to_maintain;
     options.max_background_compactions = FLAGS_max_background_compactions;
+    options.max_subcompactions = static_cast<uint32_t>(FLAGS_subcompactions);
     options.max_background_flushes = FLAGS_max_background_flushes;
     options.compaction_style = FLAGS_compaction_style_e;
     if (FLAGS_prefix_size != 0) {
@@ -2246,6 +2247,9 @@ class Benchmark {
     options.memtable_prefix_bloom_bits = FLAGS_memtable_bloom_bits;
     options.bloom_locality = FLAGS_bloom_locality;
     options.max_open_files = FLAGS_open_files;
+    options.new_table_reader_for_compaction_inputs =
+        FLAGS_new_table_reader_for_compaction_inputs;
+    options.compaction_readahead_size = FLAGS_compaction_readahead_size;
     options.statistics = dbstats;
     if (FLAGS_enable_io_prio) {
       FLAGS_env->LowerThreadPoolIOPriority(Env::LOW);
@@ -2262,7 +2266,7 @@ class Benchmark {
       flashcache_aware_env_ =
           std::move(NewFlashcacheAwareEnv(FLAGS_env, cachedev_fd_));
       if (flashcache_aware_env_.get() == nullptr) {
-        fprintf(stderr, "Failed to open flashcahce device at %s\n",
+        fprintf(stderr, "Failed to open flashcache device at %s\n",
                 FLAGS_flashcache_dev.c_str());
         std::abort();
       }
@@ -2454,6 +2458,7 @@ class Benchmark {
       exit(1);
     }
     options.max_successive_merges = FLAGS_max_successive_merges;
+    options.compaction_measure_io_stats = FLAGS_compaction_measure_io_stats;
 
     // set universal style compaction configurations, if applicable
     if (FLAGS_universal_size_ratio != 0) {
@@ -2525,11 +2530,19 @@ class Benchmark {
       if (FLAGS_readonly) {
         s = DB::OpenForReadOnly(options, db_name, column_families,
             &db->cfh, &db->db);
-      } else if (FLAGS_transaction_db) {
+      } else if (FLAGS_optimistic_transaction_db) {
         s = OptimisticTransactionDB::Open(options, db_name, column_families,
-                                          &db->cfh, &db->txn_db);
+                                          &db->cfh, &db->opt_txn_db);
         if (s.ok()) {
-          db->db = db->txn_db->GetBaseDB();
+          db->db = db->opt_txn_db->GetBaseDB();
+        }
+      } else if (FLAGS_transaction_db) {
+        TransactionDB* ptr;
+        TransactionDBOptions txn_db_options;
+        s = TransactionDB::Open(options, txn_db_options, db_name,
+                                column_families, &db->cfh, &ptr);
+        if (s.ok()) {
+          db->db = ptr;
         }
       } else {
         s = DB::Open(options, db_name, column_families, &db->cfh, &db->db);
@@ -2539,11 +2552,19 @@ class Benchmark {
       db->num_hot = num_hot;
     } else if (FLAGS_readonly) {
       s = DB::OpenForReadOnly(options, db_name, &db->db);
-    } else if (FLAGS_transaction_db) {
-      s = OptimisticTransactionDB::Open(options, db_name, &db->txn_db);
+    } else if (FLAGS_optimistic_transaction_db) {
+      s = OptimisticTransactionDB::Open(options, db_name, &db->opt_txn_db);
       if (s.ok()) {
-        db->db = db->txn_db->GetBaseDB();
+        db->db = db->opt_txn_db->GetBaseDB();
       }
+    } else if (FLAGS_transaction_db) {
+      TransactionDB* ptr;
+      TransactionDBOptions txn_db_options;
+      s = TransactionDB::Open(options, txn_db_options, db_name, &ptr);
+      if (s.ok()) {
+        db->db = ptr;
+      }
+
     } else {
       s = DB::Open(options, db_name, &db->db);
     }
@@ -3430,7 +3451,7 @@ class Benchmark {
       // Update the value (by appending data)
       Slice operand = gen.Generate(value_size_);
       if (value.size() > 0) {
-        // Use a delimeter to match the semantics for StringAppendOperator
+        // Use a delimiter to match the semantics for StringAppendOperator
         value.append(1,',');
       }
       value.append(operand.data(), operand.size());
@@ -3610,7 +3631,6 @@ class Benchmark {
     uint64_t transactions_aborted = 0;
     Status s;
     uint64_t num_prefix_ranges = FLAGS_transaction_sets;
-    bool use_txn = FLAGS_transaction_db;
 
     if (num_prefix_ranges == 0 || num_prefix_ranges > 9999) {
       fprintf(stderr, "invalid value for transaction_sets\n");
@@ -3625,19 +3645,32 @@ class Benchmark {
     }
 
     while (!duration.Done(1)) {
-      OptimisticTransaction* txn = nullptr;
+      Transaction* txn = nullptr;
       WriteBatch* batch = nullptr;
 
-      if (use_txn) {
-        txn = db_.txn_db->BeginTransaction(write_options_);
+      if (FLAGS_optimistic_transaction_db) {
+        txn = db_.opt_txn_db->BeginTransaction(write_options_);
+        assert(txn);
+      } else if (FLAGS_transaction_db) {
+        TransactionDB* txn_db = reinterpret_cast<TransactionDB*>(db_.db);
+
+        TransactionOptions txn_options;
+        txn_options.lock_timeout = FLAGS_transaction_lock_timeout;
+
+        txn = txn_db->BeginTransaction(write_options_, txn_options);
         assert(txn);
       } else {
         batch = new WriteBatch();
       }
 
+      if (txn && FLAGS_transaction_set_snapshot) {
+        txn->SetSnapshot();
+      }
+
       // pick a random number to use to increment a key in each set
       uint64_t incr = (thread->rand.Next() % 100) + 1;
 
+      bool failed = false;
       // For each set, pick a key at random and increment it
       for (uint8_t i = 0; i < num_prefix_ranges; i++) {
         uint64_t int_value;
@@ -3652,8 +3685,8 @@ class Benchmark {
         std::string full_key = std::string(prefix_buf) + base_key.ToString();
         Slice key(full_key);
 
-        if (use_txn) {
-          s = txn->Get(read_options, key, &value);
+        if (txn) {
+          s = txn->GetForUpdate(read_options, key, &value);
         } else {
           s = db->Get(read_options, key, &value);
         }
@@ -3668,9 +3701,13 @@ class Benchmark {
           }
         } else if (s.IsNotFound()) {
           int_value = 0;
-        } else {
-          fprintf(stderr, "Get returned an error: %s\n", s.ToString().c_str());
+        } else if (!(s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) {
+          fprintf(stderr, "Get returned an unexpected error: %s\n",
+                  s.ToString().c_str());
           abort();
+        } else {
+          failed = true;
+          break;
         }
 
         if (FLAGS_transaction_sleep > 0) {
@@ -3679,24 +3716,41 @@ class Benchmark {
         }
 
         std::string sum = ToString(int_value + incr);
-        if (use_txn) {
-          txn->Put(key, sum);
+        if (txn) {
+          s = txn->Put(key, sum);
+          if (!s.ok()) {
+            // Since we did a GetForUpdate, Put should not fail.
+            fprintf(stderr, "Put returned an unexpected error: %s\n",
+                    s.ToString().c_str());
+            abort();
+          }
         } else {
           batch->Put(key, sum);
         }
       }
 
-      if (use_txn) {
-        s = txn->Commit();
+      if (txn) {
+        if (failed) {
+          transactions_aborted++;
+          txn->Rollback();
+          s = Status::OK();
+        } else {
+          s = txn->Commit();
+        }
       } else {
         s = db->Write(write_options_, batch);
       }
 
       if (!s.ok()) {
+        failed = true;
+
         // Ideally, we'd want to run this stress test with enough concurrency
         // on a small enough set of keys that we get some failed transactions
         // due to conflicts.
-        if (use_txn && s.IsBusy()) {
+        if (FLAGS_optimistic_transaction_db &&
+            (s.IsBusy() || s.IsTimedOut() || s.IsTryAgain())) {
+          transactions_aborted++;
+        } else if (FLAGS_transaction_db && s.IsExpired()) {
           transactions_aborted++;
         } else {
           fprintf(stderr, "Unexpected write error: %s\n", s.ToString().c_str());
@@ -3711,11 +3765,15 @@ class Benchmark {
         delete batch;
       }
 
+      if (!failed) {
+        thread->stats.FinishedOps(nullptr, db, 1);
+      }
+
       transactions_done++;
     }
 
     char msg[100];
-    if (use_txn) {
+    if (FLAGS_optimistic_transaction_db || FLAGS_transaction_db) {
       snprintf(msg, sizeof(msg),
                "( transactions:%" PRIu64 " aborts:%" PRIu64 ")",
                transactions_done, transactions_aborted);
@@ -3733,7 +3791,7 @@ class Benchmark {
   // Since each iteration of RandomTransaction() incremented a key in each set
   // by the same value, the sum of the keys in each set should be the same.
   void RandomTransactionVerify() {
-    if (!FLAGS_transaction_db) {
+    if (!FLAGS_transaction_db && !FLAGS_optimistic_transaction_db) {
       // transactions not used, nothing to verify.
       return;
     }
@@ -3781,8 +3839,7 @@ class Benchmark {
       prev_total = total;
     }
 
-    fprintf(stdout, "RandomTransactionVerify Success! Total:%" PRIu64 "\n",
-            prev_total);
+    fprintf(stdout, "RandomTransactionVerify Success!\n");
   }
 
   void Compact(ThreadState* thread) {

db/db_compaction_filter_test.cc

@@ -374,7 +374,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
 
     // push all files to  lower levels
     ASSERT_OK(Flush(1));
-    if (option_config_ != kUniversalCompactionMultiLevel) {
+    if (option_config_ != kUniversalCompactionMultiLevel &&
+        option_config_ != kUniversalSubcompactions) {
       dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
       dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
     } else {
@@ -392,7 +393,8 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
     // push all files to  lower levels. This should
     // invoke the compaction filter for all 100000 keys.
     ASSERT_OK(Flush(1));
-    if (option_config_ != kUniversalCompactionMultiLevel) {
+    if (option_config_ != kUniversalCompactionMultiLevel &&
+        option_config_ != kUniversalSubcompactions) {
       dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
       dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[1]);
     } else {
@@ -535,6 +537,42 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
   }
 }
 
+// Compaction filters should only be applied to records that are newer than the
+// latest snapshot. This test inserts records and applies a delete filter.
+TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
+  Options options;
+  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
+  options.disable_auto_compactions = true;
+  options.create_if_missing = true;
+  options = CurrentOptions(options);
+  DestroyAndReopen(options);
+
+  // Put some data.
+  const Snapshot* snapshot = nullptr;
+  for (int table = 0; table < 4; ++table) {
+    for (int i = 0; i < 10; ++i) {
+      Put(ToString(table * 100 + i), "val");
+    }
+    Flush();
+
+    if (table == 0) {
+      snapshot = db_->GetSnapshot();
+    }
+  }
+  assert(snapshot != nullptr);
+
+  cfilter_count = 0;
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  // The filter should delete 10 records.
+  ASSERT_EQ(30U, cfilter_count);
+
+  // Release the snapshot and compact again -> now all records should be
+  // removed.
+  db_->ReleaseSnapshot(snapshot);
+  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+  ASSERT_EQ(0U, CountLiveFiles());
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_compaction_test.cc

@@ -13,6 +13,10 @@
 #include "util/sync_point.h"
 namespace rocksdb {
 
+// SYNC_POINT is not supported in released Windows mode.
+#if !(defined NDEBUG) || !defined(OS_WIN)
+
+
 class DBCompactionTest : public DBTestBase {
  public:
   DBCompactionTest() : DBTestBase("/db_compaction_test") {}
@@ -22,14 +26,14 @@ class DBCompactionTestWithParam : public DBTestBase,
                                 public testing::WithParamInterface<uint32_t> {
  public:
   DBCompactionTestWithParam() : DBTestBase("/db_compaction_test") {
-    num_subcompactions_ = GetParam();
+    max_subcompactions_ = GetParam();
   }
 
   // Required if inheriting from testing::WithParamInterface<>
   static void SetUpTestCase() {}
   static void TearDownTestCase() {}
 
-  uint32_t num_subcompactions_;
+  uint32_t max_subcompactions_;
 };
 
 namespace {
@@ -210,12 +214,12 @@ const SstFileMetaData* PickFileRandomly(
 }  // anonymous namespace
 
 // All the TEST_P tests run once with sub_compactions disabled (i.e.
-// options.num_subcompactions = 1) and once with it enabled
+// options.max_subcompactions = 1) and once with it enabled
 TEST_P(DBCompactionTestWithParam, CompactionDeletionTrigger) {
   for (int tid = 0; tid < 3; ++tid) {
     uint64_t db_size[2];
     Options options = CurrentOptions(DeletionTriggerOptions());
-    options.num_subcompactions = num_subcompactions_;
+    options.max_subcompactions = max_subcompactions_;
 
     if (tid == 1) {
       // the following only disable stats update in DB::Open()
@@ -230,7 +234,7 @@ TEST_P(DBCompactionTestWithParam, CompactionDeletionTrigger) {
     DestroyAndReopen(options);
     Random rnd(301);
 
-    const int kTestSize = kCDTKeysPerBuffer * 512;
+    const int kTestSize = kCDTKeysPerBuffer * 1024;
     std::vector<std::string> values;
     for (int k = 0; k < kTestSize; ++k) {
       values.push_back(RandomString(&rnd, kCDTValueSize));
@@ -296,12 +300,99 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
   ASSERT_GT(env_->random_file_open_counter_.load(), 5);
 }
 
+TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
+  Options options;
+  options = CurrentOptions(options);
+  options.env = env_;
+  options.new_table_reader_for_compaction_inputs = true;
+  options.max_open_files = 100;
+  options.level0_file_num_compaction_trigger = 3;
+  DestroyAndReopen(options);
+  Random rnd(301);
+
+  int num_table_cache_lookup = 0;
+  int num_new_table_reader = 0;
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "TableCache::FindTable:0", [&](void* arg) {
+        assert(arg != nullptr);
+        bool no_io = *(reinterpret_cast<bool*>(arg));
+        if (!no_io) {
+          // filter out cases for table properties queries.
+          num_table_cache_lookup++;
+        }
+      });
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "TableCache::GetTableReader:0",
+      [&](void* arg) { num_new_table_reader++; });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  for (int k = 0; k < options.level0_file_num_compaction_trigger; ++k) {
+    ASSERT_OK(Put(Key(k), Key(k)));
+    ASSERT_OK(Put(Key(10 - k), "bar"));
+    if (k < options.level0_file_num_compaction_trigger - 1) {
+      num_table_cache_lookup = 0;
+      Flush();
+      dbfull()->TEST_WaitForCompact();
+      // preloading iterator issues one table cache lookup and create
+      // a new table reader.
+      ASSERT_EQ(num_table_cache_lookup, 1);
+      ASSERT_EQ(num_new_table_reader, 1);
+
+      num_table_cache_lookup = 0;
+      num_new_table_reader = 0;
+      ASSERT_EQ(Key(k), Get(Key(k)));
+      // lookup iterator from table cache and no need to create a new one.
+      ASSERT_EQ(num_table_cache_lookup, 1);
+      ASSERT_EQ(num_new_table_reader, 0);
+    }
+  }
+
+  num_table_cache_lookup = 0;
+  num_new_table_reader = 0;
+  Flush();
+  dbfull()->TEST_WaitForCompact();
+  // Preloading iterator issues one table cache lookup and creates
+  // a new table reader. One file is created for flush and one for compaction.
+  // Compaction inputs make no table cache look-up.
+  ASSERT_EQ(num_table_cache_lookup, 2);
+  // Create new iterator for:
+  // (1) 1 for verifying flush results
+  // (2) 3 for compaction input files
+  // (3) 1 for verifying compaction results.
+  ASSERT_EQ(num_new_table_reader, 5);
+
+  num_table_cache_lookup = 0;
+  num_new_table_reader = 0;
+  ASSERT_EQ(Key(1), Get(Key(1)));
+  ASSERT_EQ(num_table_cache_lookup, 1);
+  ASSERT_EQ(num_new_table_reader, 0);
+
+  num_table_cache_lookup = 0;
+  num_new_table_reader = 0;
+  CompactRangeOptions cro;
+  cro.change_level = true;
+  cro.target_level = 2;
+  cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
+  db_->CompactRange(cro, nullptr, nullptr);
+  // Only verifying compaction outputs issues one table cache lookup.
+  ASSERT_EQ(num_table_cache_lookup, 1);
+  // One for compaction input, one for verifying compaction results.
+  ASSERT_EQ(num_new_table_reader, 2);
+
+  num_table_cache_lookup = 0;
+  num_new_table_reader = 0;
+  ASSERT_EQ(Key(1), Get(Key(1)));
+  ASSERT_EQ(num_table_cache_lookup, 1);
+  ASSERT_EQ(num_new_table_reader, 0);
+
+  rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
+}
 
 TEST_P(DBCompactionTestWithParam, CompactionDeletionTriggerReopen) {
   for (int tid = 0; tid < 2; ++tid) {
     uint64_t db_size[3];
     Options options = CurrentOptions(DeletionTriggerOptions());
-    options.num_subcompactions = num_subcompactions_;
+    options.max_subcompactions = max_subcompactions_;
 
     if (tid == 1) {
       // second pass with universal compaction
@@ -414,10 +505,11 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
 
 TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
   Options options;
-  options.write_buffer_size = 100 << 10;  // 100KB
+  options.write_buffer_size = 110 << 10;  // 110KB
+  options.arena_block_size = 4 << 10;
   options.num_levels = 3;
   options.level0_file_num_compaction_trigger = 3;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -426,9 +518,9 @@ TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
   for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
        num++) {
     std::vector<std::string> values;
-    // Write 120KB (12 values, each 10K)
-    for (int i = 0; i < 12; i++) {
-      values.push_back(RandomString(&rnd, 10000));
+    // Write 100KB (100 values, each 1K)
+    for (int i = 0; i < 100; i++) {
+      values.push_back(RandomString(&rnd, 990));
       ASSERT_OK(Put(1, Key(i), values[i]));
     }
     dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
@@ -437,8 +529,8 @@ TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
 
   // generate one more file in level-0, and should trigger level-0 compaction
   std::vector<std::string> values;
-  for (int i = 0; i < 12; i++) {
-    values.push_back(RandomString(&rnd, 10000));
+  for (int i = 0; i < 100; i++) {
+    values.push_back(RandomString(&rnd, 990));
     ASSERT_OK(Put(1, Key(i), values[i]));
   }
   dbfull()->TEST_WaitForCompact();
@@ -450,7 +542,7 @@ TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
 TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
   Options options;
   options.write_buffer_size = 100000000;        // Large write buffer
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -537,7 +629,7 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
 
   Options options;
   options.write_buffer_size = 100000000;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   options = CurrentOptions(options);
   DestroyAndReopen(options);
 
@@ -596,7 +688,7 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveNonOverlappingFiles) {
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.write_buffer_size = 10 * 1024 * 1024;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
 
   DestroyAndReopen(options);
   // non overlapping ranges
@@ -693,7 +785,7 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveTargetLevel) {
   options.disable_auto_compactions = true;
   options.write_buffer_size = 10 * 1024 * 1024;
   options.num_levels = 7;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
 
   DestroyAndReopen(options);
   int32_t value_size = 10 * 1024;  // 10 KB
@@ -749,7 +841,7 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
 
   Options options;
   options.write_buffer_size = 100000000;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   options = CurrentOptions(options);
   DestroyAndReopen(options);
 
@@ -801,11 +893,12 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionThirdPath) {
   options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
   options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
   options.compaction_style = kCompactionStyleLevel;
-  options.write_buffer_size = 100 << 10;  // 100KB
+  options.write_buffer_size = 110 << 10;  // 110KB
+  options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = 4;
   options.max_bytes_for_level_base = 400 * 1024;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   //  options = CurrentOptions(options);
 
   std::vector<std::string> filenames;
@@ -895,7 +988,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionThirdPath) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Reopen(options);
@@ -903,7 +996,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionThirdPath) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Destroy(options);
@@ -915,11 +1008,12 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) {
   options.db_paths.emplace_back(dbname_ + "_2", 4 * 1024 * 1024);
   options.db_paths.emplace_back(dbname_ + "_3", 1024 * 1024 * 1024);
   options.compaction_style = kCompactionStyleLevel;
-  options.write_buffer_size = 100 << 10;  // 100KB
+  options.write_buffer_size = 110 << 10;  // 110KB
+  options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = 4;
   options.max_bytes_for_level_base = 400 * 1024;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   //  options = CurrentOptions(options);
 
   std::vector<std::string> filenames;
@@ -1010,7 +1104,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Reopen(options);
@@ -1018,7 +1112,7 @@ TEST_P(DBCompactionTestWithParam, LevelCompactionPathUse) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Destroy(options);
@@ -1031,14 +1125,15 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
 
   // Stage 1: generate a db with level compaction
   Options options;
-  options.write_buffer_size = 100 << 10;  // 100KB
+  options.write_buffer_size = 110 << 10;  // 110KB
+  options.arena_block_size = 4 << 10;
   options.num_levels = 4;
   options.level0_file_num_compaction_trigger = 3;
   options.max_bytes_for_level_base = 500 << 10;  // 500KB
   options.max_bytes_for_level_multiplier = 1;
   options.target_file_size_base = 200 << 10;  // 200KB
   options.target_file_size_multiplier = 1;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -1089,7 +1184,8 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
   options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = 4;
-  options.write_buffer_size = 100 << 10;  // 100KB
+  options.write_buffer_size = 110 << 10;  // 110KB
+  options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 3;
   options = CurrentOptions(options);
   ReopenWithColumnFamilies({"default", "pikachu"}, options);
@@ -1176,7 +1272,7 @@ TEST_F(DBCompactionTest, L0_CompactionBug_Issue44_b) {
 
 TEST_P(DBCompactionTestWithParam, ManualCompaction) {
   Options options = CurrentOptions();
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   CreateAndReopenWithCF({"pikachu"}, options);
 
   // iter - 0 with 7 levels
@@ -1228,7 +1324,7 @@ TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
   options.db_paths.emplace_back(dbname_ + "_2", 2 * 10485760);
   options.db_paths.emplace_back(dbname_ + "_3", 100 * 10485760);
   options.db_paths.emplace_back(dbname_ + "_4", 120 * 10485760);
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   CreateAndReopenWithCF({"pikachu"}, options);
 
   // iter - 0 with 7 levels
@@ -1284,11 +1380,8 @@ TEST_P(DBCompactionTestWithParam, ManualLevelCompactionOutputPathId) {
     compact_options.target_path_id = 1;
     db_->CompactRange(compact_options, handles_[1], nullptr, nullptr);
 
-    int num_files = options.num_subcompactions > 1 ? 2 : 1;
-    std::string files_string = options.num_subcompactions > 1 ? "0,2" : "0,1";
-
-    ASSERT_EQ(files_string, FilesPerLevel(1));
-    ASSERT_EQ(num_files, GetSstFileCount(options.db_paths[1].path));
+    ASSERT_EQ("0,1", FilesPerLevel(1));
+    ASSERT_EQ(1, GetSstFileCount(options.db_paths[1].path));
     ASSERT_EQ(0, GetSstFileCount(options.db_paths[0].path));
     ASSERT_EQ(0, GetSstFileCount(dbname_));
 
@@ -1335,7 +1428,7 @@ TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
   options.level0_stop_writes_trigger = 2;
   options.max_bytes_for_level_multiplier = 2;
   options.compression = kNoCompression;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   options = CurrentOptions(options);
   CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -1395,7 +1488,7 @@ TEST_P(DBCompactionTestWithParam, PartialCompactionFailure) {
       options.target_file_size_base;
   options.max_bytes_for_level_multiplier = 2;
   options.compression = kNoCompression;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
 
   env_->SetBackgroundThreads(1, Env::HIGH);
   env_->SetBackgroundThreads(1, Env::LOW);
@@ -1476,7 +1569,7 @@ TEST_P(DBCompactionTestWithParam, DeleteMovedFileAfterCompaction) {
         2;  // trigger compaction when we have 2 files
     OnFileDeletionListener* listener = new OnFileDeletionListener();
     options.listeners.emplace_back(listener);
-    options.num_subcompactions = num_subcompactions_;
+    options.max_subcompactions = max_subcompactions_;
     DestroyAndReopen(options);
 
     Random rnd(301);
@@ -1546,11 +1639,12 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
   }
   Options options = CurrentOptions();
   options.compaction_style = kCompactionStyleLevel;
-  options.write_buffer_size = 100 << 10;  // 100KB
+  options.write_buffer_size = 110 << 10;  // 110KB
+  options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = 4;
   options.max_bytes_for_level_base = 400 * 1024;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   // First two levels have no compression, so that a trivial move between
   // them will be allowed. Level 2 has Zlib compression so that a trivial
   // move to level 3 will not be allowed
@@ -1636,7 +1730,7 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Reopen(options);
@@ -1644,20 +1738,22 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Destroy(options);
 }
 
-// SYNC_POINT is not supported in released Windows mode.
-#if !(defined NDEBUG) || !defined(OS_WIN)
 // This tests for a bug that could cause two level0 compactions running
 // concurrently
-TEST_P(DBCompactionTestWithParam, SuggestCompactRangeNoTwoLevel0Compactions) {
+// TODO(aekmekji): Make sure that the reason this fails when run with
+// max_subcompactions > 1 is not a correctness issue but just inherent to
+// running parallel L0-L1 compactions
+TEST_F(DBCompactionTest, SuggestCompactRangeNoTwoLevel0Compactions) {
   Options options = CurrentOptions();
   options.compaction_style = kCompactionStyleLevel;
   options.write_buffer_size = 110 << 10;
+  options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 4;
   options.num_levels = 4;
   options.compression = kNoCompression;
@@ -1665,7 +1761,6 @@ TEST_P(DBCompactionTestWithParam, SuggestCompactRangeNoTwoLevel0Compactions) {
   options.target_file_size_base = 98 << 10;
   options.max_write_buffer_number = 2;
   options.max_background_compactions = 2;
-  options.num_subcompactions = num_subcompactions_;
 
   DestroyAndReopen(options);
 
@@ -1708,7 +1803,6 @@ TEST_P(DBCompactionTestWithParam, SuggestCompactRangeNoTwoLevel0Compactions) {
   dbfull()->TEST_WaitForCompact();
 }
 
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
 
 TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
   int32_t trivial_move = 0;
@@ -1723,7 +1817,7 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
 
   Options options;
   options.write_buffer_size = 100000000;
-  options.num_subcompactions = num_subcompactions_;
+  options.max_subcompactions = max_subcompactions_;
   options = CurrentOptions(options);
   DestroyAndReopen(options);
 
@@ -1795,7 +1889,7 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
 
 INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
                         ::testing::Values(1, 4));
-
+#endif  // !(defined NDEBUG) || !defined(OS_WIN)
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_impl.cc

@@ -14,25 +14,27 @@
 #endif
 
 #include <inttypes.h>
+#include <stdint.h>
+
 #include <algorithm>
 #include <climits>
 #include <cstdio>
 #include <set>
 #include <stdexcept>
-#include <stdint.h>
 #include <string>
-#include <unordered_set>
 #include <unordered_map>
+#include <unordered_set>
 #include <utility>
 #include <vector>
 
 #include "db/builder.h"
-#include "db/flush_job.h"
 #include "db/compaction_job.h"
 #include "db/db_iter.h"
 #include "db/dbformat.h"
 #include "db/event_helpers.h"
 #include "db/filename.h"
+#include "db/flush_job.h"
+#include "db/forward_iterator.h"
 #include "db/job_context.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
@@ -43,24 +45,23 @@
 #include "db/merge_helper.h"
 #include "db/table_cache.h"
 #include "db/table_properties_collector.h"
-#include "db/forward_iterator.h"
 #include "db/transaction_log_impl.h"
 #include "db/version_set.h"
-#include "db/writebuffer.h"
 #include "db/write_batch_internal.h"
 #include "db/write_callback.h"
+#include "db/writebuffer.h"
+#include "port/likely.h"
 #include "port/port.h"
 #include "rocksdb/cache.h"
-#include "port/likely.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/db.h"
 #include "rocksdb/delete_scheduler.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
-#include "rocksdb/version.h"
 #include "rocksdb/statistics.h"
 #include "rocksdb/status.h"
 #include "rocksdb/table.h"
+#include "rocksdb/version.h"
 #include "table/block.h"
 #include "table/block_based_table_factory.h"
 #include "table/merger.h"
@@ -75,16 +76,16 @@
 #include "util/db_info_dumper.h"
 #include "util/file_reader_writer.h"
 #include "util/file_util.h"
-#include "util/hash_skiplist_rep.h"
 #include "util/hash_linklist_rep.h"
-#include "util/logging.h"
+#include "util/hash_skiplist_rep.h"
+#include "util/iostats_context_imp.h"
 #include "util/log_buffer.h"
+#include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/perf_context_imp.h"
-#include "util/iostats_context_imp.h"
 #include "util/stop_watch.h"
-#include "util/sync_point.h"
 #include "util/string_util.h"
+#include "util/sync_point.h"
 #include "util/thread_status_updater.h"
 #include "util/thread_status_util.h"
 #include "util/xfunc.h"
@@ -122,7 +123,11 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
 
   // result.max_open_files means an "infinite" open files.
   if (result.max_open_files != -1) {
-    ClipToRange(&result.max_open_files, 20, 1000000);
+    int max_max_open_files = port::GetMaxOpenFiles();
+    if (max_max_open_files == -1) {
+      max_max_open_files = 1000000;
+    }
+    ClipToRange(&result.max_open_files, 20, max_max_open_files);
   }
 
   if (result.info_log == nullptr) {
@@ -156,6 +161,10 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
     result.db_paths.emplace_back(dbname, std::numeric_limits<uint64_t>::max());
   }
 
+  if (result.compaction_readahead_size > 0) {
+    result.new_table_reader_for_compaction_inputs = true;
+  }
+
   return result;
 }
 
@@ -529,6 +538,9 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
 
   // don't delete files that might be currently written to from compaction
   // threads
+  // Since job_context->min_pending_output is set, until file scan finishes,
+  // mutex_ cannot be released. Otherwise, we might see no min_pending_output
+  // here but later find newer generated unfinalized files while scannint.
   if (!pending_outputs_.empty()) {
     job_context->min_pending_output = *pending_outputs_.begin();
   } else {
@@ -541,37 +553,6 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
   versions_->GetObsoleteFiles(&job_context->sst_delete_files,
                               job_context->min_pending_output);
 
-  if (!alive_log_files_.empty()) {
-    uint64_t min_log_number = versions_->MinLogNumber();
-    // find newly obsoleted log files
-    while (alive_log_files_.begin()->number < min_log_number) {
-      auto& earliest = *alive_log_files_.begin();
-      job_context->log_delete_files.push_back(earliest.number);
-      total_log_size_ -= earliest.size;
-      alive_log_files_.pop_front();
-      // Current log should always stay alive since it can't have
-      // number < MinLogNumber().
-      assert(alive_log_files_.size());
-    }
-    while (!logs_.empty() && logs_.front().number < min_log_number) {
-      auto& log = logs_.front();
-      if (log.getting_synced) {
-        log_sync_cv_.Wait();
-        // logs_ could have changed while we were waiting.
-        continue;
-      }
-      logs_to_free_.push_back(log.ReleaseWriter());
-      logs_.pop_front();
-    }
-    // Current log cannot be obsolete.
-    assert(!logs_.empty());
-  }
-
-  // We're just cleaning up for DB::Write().
-  assert(job_context->logs_to_free.empty());
-  job_context->logs_to_free = logs_to_free_;
-  logs_to_free_.clear();
-
   // store the current filenum, lognum, etc
   job_context->manifest_file_number = versions_->manifest_file_number();
   job_context->pending_manifest_file_number =
@@ -613,6 +594,37 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
       }
     }
   }
+
+  if (!alive_log_files_.empty()) {
+    uint64_t min_log_number = versions_->MinLogNumber();
+    // find newly obsoleted log files
+    while (alive_log_files_.begin()->number < min_log_number) {
+      auto& earliest = *alive_log_files_.begin();
+      job_context->log_delete_files.push_back(earliest.number);
+      total_log_size_ -= earliest.size;
+      alive_log_files_.pop_front();
+      // Current log should always stay alive since it can't have
+      // number < MinLogNumber().
+      assert(alive_log_files_.size());
+    }
+    while (!logs_.empty() && logs_.front().number < min_log_number) {
+      auto& log = logs_.front();
+      if (log.getting_synced) {
+        log_sync_cv_.Wait();
+        // logs_ could have changed while we were waiting.
+        continue;
+      }
+      logs_to_free_.push_back(log.ReleaseWriter());
+      logs_.pop_front();
+    }
+    // Current log cannot be obsolete.
+    assert(!logs_.empty());
+  }
+
+  // We're just cleaning up for DB::Write().
+  assert(job_context->logs_to_free.empty());
+  job_context->logs_to_free = logs_to_free_;
+  logs_to_free_.clear();
 }
 
 namespace {
@@ -750,9 +762,8 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
     }
 #endif  // !ROCKSDB_LITE
     Status file_deletion_status;
-    if (db_options_.delete_scheduler != nullptr && type == kTableFile &&
-        path_id == 0) {
-      file_deletion_status = db_options_.delete_scheduler->DeleteFile(fname);
+    if (type == kTableFile && path_id == 0) {
+      file_deletion_status = DeleteOrMoveToTrash(&db_options_, fname);
     } else {
       file_deletion_status = env_->DeleteFile(fname);
     }
@@ -1140,8 +1151,12 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
 
       MaybeIgnoreError(&status);
       if (!status.ok()) {
-        return status;
+        // We are treating this as a failure while reading since we read valid
+        // blocks that do not form coherent data
+        reporter.Corruption(record.size(), status);
+        continue;
       }
+
       const SequenceNumber last_seq = WriteBatchInternal::Sequence(&batch) +
                                       WriteBatchInternal::Count(&batch) - 1;
       if ((*max_sequence == kMaxSequenceNumber) || (last_seq > *max_sequence)) {
@@ -1175,12 +1190,11 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     }
 
     if (!status.ok()) {
-      // The hook function is designed to ignore all IO errors from reader
-      // during recovery for kSkipAnyCorruptedRecords. Status variable is
-      // unmodified by the reader.
-      assert(db_options_.wal_recovery_mode !=
-             WALRecoveryMode::kSkipAnyCorruptedRecords);
       if (db_options_.wal_recovery_mode ==
+             WALRecoveryMode::kSkipAnyCorruptedRecords) {
+        // We should ignore all errors unconditionally
+        status = Status::OK();
+      } else if (db_options_.wal_recovery_mode ==
                  WALRecoveryMode::kPointInTimeRecovery) {
         // We should ignore the error but not continue replaying
         status = Status::OK();
@@ -1189,8 +1203,11 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
         Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
             "Point in time recovered to log #%" PRIu64 " seq #%" PRIu64,
             log_number, *max_sequence);
-      } else if (db_options_.wal_recovery_mode !=
-                 WALRecoveryMode::kSkipAnyCorruptedRecords) {
+      } else {
+        assert(db_options_.wal_recovery_mode ==
+                  WALRecoveryMode::kTolerateCorruptedTailRecords
+               || db_options_.wal_recovery_mode ==
+                  WALRecoveryMode::kAbsoluteConsistency);
         return status;
       }
     }
@@ -1275,9 +1292,6 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
   TableProperties table_properties;
   {
     ScopedArenaIterator iter(mem->NewIterator(ro, &arena));
-    const SequenceNumber newest_snapshot = snapshots_.GetNewest();
-    const SequenceNumber earliest_seqno_in_memtable =
-        mem->GetFirstSequenceNumber();
     Log(InfoLogLevel::DEBUG_LEVEL, db_options_.info_log,
         "[%s] [WriteLevel0TableForRecovery]"
         " Level-0 table #%" PRIu64 ": started",
@@ -1291,10 +1305,10 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
       s = BuildTable(
           dbname_, env_, *cfd->ioptions(), env_options_, cfd->table_cache(),
           iter.get(), &meta, cfd->internal_comparator(),
-          cfd->int_tbl_prop_collector_factories(), newest_snapshot,
-          earliest_seqno_in_memtable, GetCompressionFlush(*cfd->ioptions()),
-          cfd->ioptions()->compression_opts, paranoid_file_checks, Env::IO_HIGH,
-          &info.table_properties);
+          cfd->int_tbl_prop_collector_factories(), snapshots_.GetAll(),
+          GetCompressionFlush(*cfd->ioptions()),
+          cfd->ioptions()->compression_opts, paranoid_file_checks,
+          cfd->internal_stats(), Env::IO_HIGH, &info.table_properties);
       LogFlush(db_options_.info_log);
       Log(InfoLogLevel::DEBUG_LEVEL, db_options_.info_log,
           "[%s] [WriteLevel0TableForRecovery]"
@@ -1342,14 +1356,14 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
 
 Status DBImpl::FlushMemTableToOutputFile(
     ColumnFamilyData* cfd, const MutableCFOptions& mutable_cf_options,
-    bool* madeProgress, JobContext* job_context, LogBuffer* log_buffer) {
+    bool* made_progress, JobContext* job_context, LogBuffer* log_buffer) {
   mutex_.AssertHeld();
   assert(cfd->imm()->NumNotFlushed() != 0);
   assert(cfd->imm()->IsFlushPending());
 
   FlushJob flush_job(dbname_, cfd, db_options_, mutable_cf_options,
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     snapshots_.GetNewest(), job_context, log_buffer,
+                     snapshots_.GetAll(), job_context, log_buffer,
                      directories_.GetDbDir(), directories_.GetDataDir(0U),
                      GetCompressionFlush(*cfd->ioptions()), stats_,
                      &event_logger_);
@@ -1367,8 +1381,8 @@ Status DBImpl::FlushMemTableToOutputFile(
   if (s.ok()) {
     InstallSuperVersionAndScheduleWorkWrapper(cfd, job_context,
                                               mutable_cf_options);
-    if (madeProgress) {
-      *madeProgress = 1;
+    if (made_progress) {
+      *made_progress = 1;
     }
     VersionStorageInfo::LevelSummaryStorage tmp;
     LogToBuffer(log_buffer, "[%s] Level summary: %s\n", cfd->GetName().c_str(),
@@ -1473,7 +1487,7 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
   } else {
     for (int level = 0; level <= max_level_with_files; level++) {
       int output_level;
-      // in case the compaction is unversal or if we're compacting the
+      // in case the compaction is universal or if we're compacting the
       // bottom-most level, the output level will be the same as input one.
       // level 0 can never be the bottommost level (i.e. if all files are in
       // level 0, we will compact to level 1)
@@ -1489,8 +1503,8 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
                        BottommostLevelCompaction::kIfHaveCompactionFilter &&
                    cfd->ioptions()->compaction_filter == nullptr &&
                    cfd->ioptions()->compaction_filter_factory == nullptr) {
-          // Skip bottommost level compaction since we dont have
-          // compaction filter
+          // Skip bottommost level compaction since we don't have a compaction
+          // filter
           continue;
         }
         output_level = level;
@@ -1669,7 +1683,8 @@ Status DBImpl::CompactFilesImpl(
       &shutting_down_, log_buffer, directories_.GetDbDir(),
       directories_.GetDataDir(c->output_path_id()), stats_, snapshots_.GetAll(),
       table_cache_, &event_logger_,
-      c->mutable_cf_options()->paranoid_file_checks, dbname_,
+      c->mutable_cf_options()->paranoid_file_checks,
+      c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
       nullptr);  // Here we pass a nullptr for CompactionJobStats because
                  // CompactFiles does not trigger OnCompactionCompleted(),
                  // which is the only place where CompactionJobStats is
@@ -1686,10 +1701,10 @@ Status DBImpl::CompactFilesImpl(
   compaction_job.Prepare();
 
   mutex_.Unlock();
-  Status status = compaction_job.Run();
+  compaction_job.Run();
   mutex_.Lock();
 
-  compaction_job.Install(&status, *c->mutable_cf_options(), &mutex_);
+  Status status = compaction_job.Install(*c->mutable_cf_options(), &mutex_);
   if (status.ok()) {
     InstallSuperVersionAndScheduleWorkWrapper(
         c->column_family_data(), job_context, *c->mutable_cf_options());
@@ -2019,12 +2034,13 @@ void DBImpl::MarkLogsSynced(
     assert(log.getting_synced);
     if (status.ok() && logs_.size() > 1) {
       logs_to_free_.push_back(log.ReleaseWriter());
-      logs_.erase(it++);
+      it = logs_.erase(it);
     } else {
       log.getting_synced = false;
       ++it;
     }
   }
+  assert(logs_.empty() || (logs_.size() == 1 && !logs_[0].getting_synced));
   log_sync_cv_.SignalAll();
 }
 
@@ -2130,14 +2146,13 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
       return Status::OK();
     }
 
-    WriteThread::Writer w(&mutex_);
-    write_thread_.EnterWriteThread(&w);
-    assert(!w.done);  // Nobody should do our job
+    WriteThread::Writer w;
+    write_thread_.EnterUnbatched(&w, &mutex_);
 
     // SwitchMemtable() will release and reacquire mutex
     // during execution
     s = SwitchMemtable(cfd, &context);
-    write_thread_.ExitWriteThread(&w, &w, s);
+    write_thread_.ExitUnbatched(&w);
 
     cfd->imm()->FlushRequested();
 
@@ -2280,7 +2295,7 @@ void DBImpl::BGWorkCompaction(void* db) {
   reinterpret_cast<DBImpl*>(db)->BackgroundCallCompaction();
 }
 
-Status DBImpl::BackgroundFlush(bool* madeProgress, JobContext* job_context,
+Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
                                LogBuffer* log_buffer) {
   mutex_.AssertHeld();
 
@@ -2321,7 +2336,7 @@ Status DBImpl::BackgroundFlush(bool* madeProgress, JobContext* job_context,
         cfd->GetName().c_str(),
         db_options_.max_background_flushes - bg_flush_scheduled_,
         db_options_.max_background_compactions - bg_compaction_scheduled_);
-    status = FlushMemTableToOutputFile(cfd, mutable_cf_options, madeProgress,
+    status = FlushMemTableToOutputFile(cfd, mutable_cf_options, made_progress,
                                        job_context, log_buffer);
     if (cfd->Unref()) {
       delete cfd;
@@ -2331,7 +2346,7 @@ Status DBImpl::BackgroundFlush(bool* madeProgress, JobContext* job_context,
 }
 
 void DBImpl::BackgroundCallFlush() {
-  bool madeProgress = false;
+  bool made_progress = false;
   JobContext job_context(next_job_id_.fetch_add(1), true);
   assert(bg_flush_scheduled_);
 
@@ -2342,7 +2357,7 @@ void DBImpl::BackgroundCallFlush() {
     auto pending_outputs_inserted_elem =
         CaptureCurrentFileNumberInPendingOutputs();
 
-    Status s = BackgroundFlush(&madeProgress, &job_context, &log_buffer);
+    Status s = BackgroundFlush(&made_progress, &job_context, &log_buffer);
     if (!s.ok() && !s.IsShutdownInProgress()) {
       // Wait a little bit before retrying background flush in
       // case this is an environmental problem and we do not want to
@@ -2396,7 +2411,7 @@ void DBImpl::BackgroundCallFlush() {
 }
 
 void DBImpl::BackgroundCallCompaction() {
-  bool madeProgress = false;
+  bool made_progress = false;
   JobContext job_context(next_job_id_.fetch_add(1), true);
 
   MaybeDumpStats();
@@ -2408,7 +2423,7 @@ void DBImpl::BackgroundCallCompaction() {
         CaptureCurrentFileNumberInPendingOutputs();
 
     assert(bg_compaction_scheduled_);
-    Status s = BackgroundCompaction(&madeProgress, &job_context, &log_buffer);
+    Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer);
     if (!s.ok() && !s.IsShutdownInProgress()) {
       // Wait a little bit before retrying background compaction in
       // case this is an environmental problem and we do not want to
@@ -2457,9 +2472,9 @@ void DBImpl::BackgroundCallCompaction() {
 
     // See if there's more work to be done
     MaybeScheduleFlushOrCompaction();
-    if (madeProgress || bg_compaction_scheduled_ == 0 || bg_manual_only_ > 0) {
+    if (made_progress || bg_compaction_scheduled_ == 0 || bg_manual_only_ > 0) {
       // signal if
-      // * madeProgress -- need to wakeup DelayWrite
+      // * made_progress -- need to wakeup DelayWrite
       // * bg_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
       // * bg_manual_only_ > 0 -- need to wakeup RunManualCompaction
       // If none of this is true, there is no need to signal since nobody is
@@ -2473,9 +2488,10 @@ void DBImpl::BackgroundCallCompaction() {
   }
 }
 
-Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
+Status DBImpl::BackgroundCompaction(bool* made_progress,
+                                    JobContext* job_context,
                                     LogBuffer* log_buffer) {
-  *madeProgress = false;
+  *made_progress = false;
   mutex_.AssertHeld();
 
   bool is_manual = (manual_compaction_ != nullptr) &&
@@ -2612,7 +2628,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
     LogToBuffer(log_buffer, "[%s] Deleted %d files\n",
                 c->column_family_data()->GetName().c_str(),
                 c->num_input_files(0));
-    *madeProgress = true;
+    *made_progress = true;
   } else if (!trivial_move_disallowed && c->IsTrivialMove()) {
     TEST_SYNC_POINT("DBImpl::BackgroundCompaction:TrivialMove");
     // Instrument for event update
@@ -2669,7 +2685,7 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
         c->column_family_data()->GetName().c_str(), moved_files,
         c->output_level(), moved_bytes, status.ToString().c_str(),
         c->column_family_data()->current()->storage_info()->LevelSummary(&tmp));
-    *madeProgress = true;
+    *made_progress = true;
 
     // Clear Instrument
     ThreadStatusUtil::ResetThreadStatus();
@@ -2683,28 +2699,29 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
         versions_.get(), &shutting_down_, log_buffer, directories_.GetDbDir(),
         directories_.GetDataDir(c->output_path_id()), stats_,
         snapshots_.GetAll(), table_cache_, &event_logger_,
-        c->mutable_cf_options()->paranoid_file_checks, dbname_,
+        c->mutable_cf_options()->paranoid_file_checks,
+        c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
         &compaction_job_stats);
     compaction_job.Prepare();
 
     mutex_.Unlock();
-    status = compaction_job.Run();
+    compaction_job.Run();
     TEST_SYNC_POINT("DBImpl::BackgroundCompaction:NonTrivial:AfterRun");
     mutex_.Lock();
 
-    compaction_job.Install(&status, *c->mutable_cf_options(), &mutex_);
+    status = compaction_job.Install(*c->mutable_cf_options(), &mutex_);
     if (status.ok()) {
       InstallSuperVersionAndScheduleWorkWrapper(
           c->column_family_data(), job_context, *c->mutable_cf_options());
     }
-    *madeProgress = true;
+    *made_progress = true;
   }
   if (c != nullptr) {
     NotifyOnCompactionCompleted(
         c->column_family_data(), c.get(), status,
         compaction_job_stats, job_context->job_id);
     c->ReleaseCompactionFiles(status);
-    *madeProgress = true;
+    *made_progress = true;
   }
   // this will unref its input_version and column_family_data
   c.reset();
@@ -3071,15 +3088,14 @@ Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
     // ColumnFamilyData object
     Options opt(db_options_, cf_options);
     {  // write thread
-      WriteThread::Writer w(&mutex_);
-      write_thread_.EnterWriteThread(&w);
-      assert(!w.done);  // Nobody should do our job
+      WriteThread::Writer w;
+      write_thread_.EnterUnbatched(&w, &mutex_);
       // LogAndApply will both write the creation in MANIFEST and create
       // ColumnFamilyData object
       s = versions_->LogAndApply(
           nullptr, MutableCFOptions(opt, ImmutableCFOptions(opt)), &edit,
           &mutex_, directories_.GetDbDir(), false, &cf_options);
-      write_thread_.ExitWriteThread(&w, &w, s);
+      write_thread_.ExitUnbatched(&w);
     }
     if (s.ok()) {
       single_column_family_mode_ = false;
@@ -3133,12 +3149,11 @@ Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) {
     }
     if (s.ok()) {
       // we drop column family from a single write thread
-      WriteThread::Writer w(&mutex_);
-      write_thread_.EnterWriteThread(&w);
-      assert(!w.done);  // Nobody should do our job
+      WriteThread::Writer w;
+      write_thread_.EnterUnbatched(&w, &mutex_);
       s = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
                                  &edit, &mutex_);
-      write_thread_.ExitWriteThread(&w, &w, s);
+      write_thread_.ExitUnbatched(&w);
     }
 
     if (!cf_support_snapshot) {
@@ -3273,8 +3288,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
     // |       |                       |
     // +-------+-----------------------+
     //
-    // ArenaWrappedDBIter inlines an arena area where all the iterartor in the
-    // the iterator tree is allocated in the order of being accessed when
+    // ArenaWrappedDBIter inlines an arena area where all the iterators in
+    // the iterator tree are allocated in the order of being accessed when
     // querying.
     // Laying out the iterators in the order of being accessed makes it more
     // likely that any iterator pointer is close to the iterator it points to so
@@ -3427,6 +3442,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   }
 
   Status status;
+  bool callback_failed = false;
+
   bool xfunc_attempted_write = false;
   XFUNC_TEST("transaction", "transaction_xftest_write_impl",
              xf_transaction_write1, xf_transaction_write, write_options,
@@ -3438,7 +3455,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   }
 
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  WriteThread::Writer w(&mutex_);
+  WriteThread::Writer w;
   w.batch = my_batch;
   w.sync = write_options.sync;
   w.disableWAL = write_options.disableWAL;
@@ -3452,6 +3469,14 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
   StopWatch write_sw(env_, db_options_.statistics.get(), DB_WRITE);
 
+  write_thread_.JoinBatchGroup(&w);
+  if (w.done) {
+    // write was done by someone else, no need to grab mutex
+    RecordTick(stats_, WRITE_DONE_BY_OTHER);
+    return w.status;
+  }
+  // else we are the leader of the write batch group
+
   WriteContext context;
   mutex_.Lock();
 
@@ -3459,15 +3484,6 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     default_cf_internal_stats_->AddDBStats(InternalStats::WRITE_WITH_WAL, 1);
   }
 
-  write_thread_.EnterWriteThread(&w);
-  if (w.done) {  // write was done by someone else
-    default_cf_internal_stats_->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER,
-                                           1);
-    mutex_.Unlock();
-    RecordTick(stats_, WRITE_DONE_BY_OTHER);
-    return w.status;
-  }
-
   RecordTick(stats_, WRITE_DONE_BY_SELF);
   default_cf_internal_stats_->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1);
 
@@ -3556,8 +3572,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
 
   if (status.ok()) {
-    last_batch_group_size_ =
-        write_thread_.BuildBatchGroup(&last_writer, &write_batch_group);
+    last_batch_group_size_ = write_thread_.EnterAsBatchGroupLeader(
+        &w, &last_writer, &write_batch_group);
 
     if (need_log_sync) {
       while (logs_.front().getting_synced) {
@@ -3580,6 +3596,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
       // If this write has a validation callback, check to see if this write
       // is able to be written.  Must be called on the write thread.
       status = callback->Callback(this);
+      callback_failed = true;
     }
   } else {
     mutex_.Unlock();
@@ -3686,7 +3703,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     mutex_.Lock();
   }
 
-  if (db_options_.paranoid_checks && !status.ok() &&
+  if (db_options_.paranoid_checks && !status.ok() && !callback_failed &&
       !status.IsBusy() && bg_error_.ok()) {
     bg_error_ = status; // stop compaction & fail any further writes
   }
@@ -3697,10 +3714,20 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     MarkLogsSynced(logfile_number_, need_log_dir_sync, status);
   }
 
-  write_thread_.ExitWriteThread(&w, last_writer, status);
+  uint64_t writes_for_other = write_batch_group.size() - 1;
+  if (writes_for_other > 0) {
+    default_cf_internal_stats_->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER,
+                                           writes_for_other);
+    if (!write_options.disableWAL) {
+      default_cf_internal_stats_->AddDBStats(InternalStats::WRITE_WITH_WAL,
+                                             writes_for_other);
+    }
+  }
 
   mutex_.Unlock();
 
+  write_thread_.ExitAsBatchGroupLeader(&w, last_writer, status);
+
   return status;
 }
 
@@ -3944,6 +3971,22 @@ SuperVersion* DBImpl::GetAndRefSuperVersion(uint32_t column_family_id) {
   return GetAndRefSuperVersion(cfd);
 }
 
+// REQUIRED:  mutex is NOT held
+SuperVersion* DBImpl::GetAndRefSuperVersionUnlocked(uint32_t column_family_id) {
+  ColumnFamilyData* cfd;
+  {
+    InstrumentedMutexLock l(&mutex_);
+    auto column_family_set = versions_->GetColumnFamilySet();
+    cfd = column_family_set->GetColumnFamily(column_family_id);
+  }
+
+  if (!cfd) {
+    return nullptr;
+  }
+
+  return GetAndRefSuperVersion(cfd);
+}
+
 void DBImpl::ReturnAndCleanupSuperVersion(ColumnFamilyData* cfd,
                                           SuperVersion* sv) {
   bool unref_sv = !cfd->ReturnThreadLocalSuperVersion(sv);
@@ -3974,6 +4017,22 @@ void DBImpl::ReturnAndCleanupSuperVersion(uint32_t column_family_id,
   ReturnAndCleanupSuperVersion(cfd, sv);
 }
 
+// REQUIRED: Mutex should NOT be held.
+void DBImpl::ReturnAndCleanupSuperVersionUnlocked(uint32_t column_family_id,
+                                                  SuperVersion* sv) {
+  ColumnFamilyData* cfd;
+  {
+    InstrumentedMutexLock l(&mutex_);
+    auto column_family_set = versions_->GetColumnFamilySet();
+    cfd = column_family_set->GetColumnFamily(column_family_id);
+  }
+
+  // If SuperVersion is held, and we successfully fetched a cfd using
+  // GetAndRefSuperVersion(), it must still exist.
+  assert(cfd != nullptr);
+  ReturnAndCleanupSuperVersion(cfd, sv);
+}
+
 // REQUIRED: this function should only be called on the write thread or if the
 // mutex is held.
 ColumnFamilyHandle* DBImpl::GetColumnFamilyHandle(uint32_t column_family_id) {
@@ -3986,6 +4045,20 @@ ColumnFamilyHandle* DBImpl::GetColumnFamilyHandle(uint32_t column_family_id) {
   return cf_memtables->GetColumnFamilyHandle();
 }
 
+// REQUIRED: mutex is NOT held.
+ColumnFamilyHandle* DBImpl::GetColumnFamilyHandleUnlocked(
+    uint32_t column_family_id) {
+  ColumnFamilyMemTables* cf_memtables = column_family_memtables_.get();
+
+  InstrumentedMutexLock l(&mutex_);
+
+  if (!cf_memtables->Seek(column_family_id)) {
+    return nullptr;
+  }
+
+  return cf_memtables->GetColumnFamilyHandle();
+}
+
 void DBImpl::GetApproximateSizes(ColumnFamilyHandle* column_family,
                                  const Range* range, int n, uint64_t* sizes,
                                  bool include_memtable) {
@@ -4461,10 +4534,13 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
       if (ParseFileName(filenames[i], &number, info_log_prefix.prefix, &type) &&
           type != kDBLockFile) {  // Lock file will be deleted at end
         Status del;
+        std::string path_to_delete = dbname + "/" + filenames[i];
         if (type == kMetaDatabase) {
-          del = DestroyDB(dbname + "/" + filenames[i], options);
+          del = DestroyDB(path_to_delete, options);
+        } else if (type == kTableFile) {
+          del = DeleteOrMoveToTrash(&options, path_to_delete);
         } else {
-          del = env->DeleteFile(dbname + "/" + filenames[i]);
+          del = env->DeleteFile(path_to_delete);
         }
         if (result.ok() && !del.ok()) {
           result = del;
@@ -4472,12 +4548,19 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
       }
     }
 
-    for (auto& db_path : options.db_paths) {
+    for (size_t path_id = 0; path_id < options.db_paths.size(); path_id++) {
+      const auto& db_path = options.db_paths[path_id];
       env->GetChildren(db_path.path, &filenames);
       for (size_t i = 0; i < filenames.size(); i++) {
         if (ParseFileName(filenames[i], &number, &type) &&
             type == kTableFile) {  // Lock file will be deleted at end
-          Status del = env->DeleteFile(db_path.path + "/" + filenames[i]);
+          Status del;
+          std::string table_path = db_path.path + "/" + filenames[i];
+          if (path_id == 0) {
+            del = DeleteOrMoveToTrash(&options, table_path);
+          } else {
+            del = env->DeleteFile(table_path);
+          }
           if (result.ok() && !del.ok()) {
             result = del;
           }

db/db_impl.h

@@ -11,21 +11,25 @@
 #include <atomic>
 #include <deque>
 #include <limits>
+#include <list>
+#include <list>
 #include <set>
-#include <list>
-#include <utility>
-#include <list>
-#include <vector>
 #include <string>
+#include <utility>
+#include <vector>
 
-#include "db/dbformat.h"
-#include "db/log_writer.h"
-#include "db/snapshot.h"
 #include "db/column_family.h"
 #include "db/compaction_job.h"
+#include "db/dbformat.h"
 #include "db/flush_job.h"
+#include "db/flush_scheduler.h"
+#include "db/internal_stats.h"
+#include "db/log_writer.h"
+#include "db/snapshot_impl.h"
 #include "db/version_edit.h"
 #include "db/wal_manager.h"
+#include "db/write_controller.h"
+#include "db/write_thread.h"
 #include "db/writebuffer.h"
 #include "memtable_list.h"
 #include "port/port.h"
@@ -36,15 +40,11 @@
 #include "util/autovector.h"
 #include "util/event_logger.h"
 #include "util/hash.h"
-#include "util/stop_watch.h"
-#include "util/thread_local.h"
-#include "util/scoped_arena_iterator.h"
 #include "util/hash.h"
 #include "util/instrumented_mutex.h"
-#include "db/internal_stats.h"
-#include "db/write_controller.h"
-#include "db/flush_scheduler.h"
-#include "db/write_thread.h"
+#include "util/scoped_arena_iterator.h"
+#include "util/stop_watch.h"
+#include "util/thread_local.h"
 
 namespace rocksdb {
 
@@ -326,6 +326,9 @@ class DBImpl : public DB {
   // mutex is held.
   SuperVersion* GetAndRefSuperVersion(uint32_t column_family_id);
 
+  // Same as above, should called without mutex held and not on write thread.
+  SuperVersion* GetAndRefSuperVersionUnlocked(uint32_t column_family_id);
+
   // Un-reference the super version and return it to thread local cache if
   // needed. If it is the last reference of the super version. Clean it up
   // after un-referencing it.
@@ -336,11 +339,18 @@ class DBImpl : public DB {
   // REQUIRED: this function should only be called on the write thread.
   void ReturnAndCleanupSuperVersion(uint32_t colun_family_id, SuperVersion* sv);
 
+  // Same as above, should called without mutex held and not on write thread.
+  void ReturnAndCleanupSuperVersionUnlocked(uint32_t colun_family_id,
+                                            SuperVersion* sv);
+
   // REQUIRED: this function should only be called on the write thread or if the
   // mutex is held.  Return value only valid until next call to this function or
   // mutex is released.
   ColumnFamilyHandle* GetColumnFamilyHandle(uint32_t column_family_id);
 
+  // Same as above, should called without mutex held and not on write thread.
+  ColumnFamilyHandle* GetColumnFamilyHandleUnlocked(uint32_t column_family_id);
+
  protected:
   Env* const env_;
   const std::string dbname_;

db/db_impl_debug.cc

@@ -131,15 +131,14 @@ void DBImpl::TEST_UnlockMutex() {
 }
 
 void* DBImpl::TEST_BeginWrite() {
-  auto w = new WriteThread::Writer(&mutex_);
-  write_thread_.EnterWriteThread(w);
-  assert(!w->done);  // Nobody should do our job
+  auto w = new WriteThread::Writer();
+  write_thread_.EnterUnbatched(w, &mutex_);
   return reinterpret_cast<void*>(w);
 }
 
 void DBImpl::TEST_EndWrite(void* w) {
   auto writer = reinterpret_cast<WriteThread::Writer*>(w);
-  write_thread_.ExitWriteThread(writer, writer, Status::OK());
+  write_thread_.ExitUnbatched(writer);
   delete writer;
 }
 

db/db_iter.cc

@@ -298,7 +298,7 @@ void DBIter::MergeValuesNewToOld() {
       continue;
     }
 
-    if (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0) {
+    if (!user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
       // hit the next user key, stop right here
       break;
     }
@@ -400,7 +400,7 @@ void DBIter::PrevInternal() {
         return;
       }
       FindParseableKey(&ikey, kReverse);
-      if (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) {
+      if (user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
         FindPrevUserKey();
       }
       return;
@@ -409,8 +409,7 @@ void DBIter::PrevInternal() {
       break;
     }
     FindParseableKey(&ikey, kReverse);
-    if (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) {
-
+    if (user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
       FindPrevUserKey();
     }
   }
@@ -434,7 +433,7 @@ bool DBIter::FindValueForCurrentKey() {
 
   size_t num_skipped = 0;
   while (iter_->Valid() && ikey.sequence <= sequence_ &&
-         (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0)) {
+         user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
     // We iterate too much: let's use Seek() to avoid too much key comparisons
     if (num_skipped >= max_skip_) {
       return FindValueForCurrentKeyUsingSeek();
@@ -461,7 +460,7 @@ bool DBIter::FindValueForCurrentKey() {
     }
 
     PERF_COUNTER_ADD(internal_key_skipped_count, 1);
-    assert(user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0);
+    assert(user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()));
     iter_->Prev();
     ++num_skipped;
     FindParseableKey(&ikey, kReverse);
@@ -531,7 +530,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   // in operands
   std::deque<std::string> operands;
   while (iter_->Valid() &&
-         (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) &&
+         user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) &&
          ikey.type == kTypeMerge) {
     operands.push_front(iter_->value().ToString());
     iter_->Next();
@@ -539,7 +538,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   }
 
   if (!iter_->Valid() ||
-      (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0) ||
+      !user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) ||
       ikey.type == kTypeDeletion) {
     {
       StopWatchNano timer(env_, statistics_ != nullptr);
@@ -550,7 +549,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     }
     // Make iter_ valid and point to saved_key_
     if (!iter_->Valid() ||
-        (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0)) {
+        !user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
       iter_->Seek(last_key);
       RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
     }
@@ -581,7 +580,7 @@ void DBIter::FindNextUserKey() {
   ParsedInternalKey ikey;
   FindParseableKey(&ikey, kForward);
   while (iter_->Valid() &&
-         user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) != 0) {
+         !user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
     iter_->Next();
     FindParseableKey(&ikey, kForward);
   }
@@ -595,19 +594,23 @@ void DBIter::FindPrevUserKey() {
   size_t num_skipped = 0;
   ParsedInternalKey ikey;
   FindParseableKey(&ikey, kReverse);
-  while (iter_->Valid() &&
-         user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) == 0) {
-    if (num_skipped >= max_skip_) {
-      num_skipped = 0;
-      IterKey last_key;
-      last_key.SetInternalKey(ParsedInternalKey(
-          saved_key_.GetKey(), kMaxSequenceNumber, kValueTypeForSeek));
-      iter_->Seek(last_key.GetKey());
-      RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
+  int cmp;
+  while (iter_->Valid() && ((cmp = user_comparator_->Compare(
+                                 ikey.user_key, saved_key_.GetKey())) == 0 ||
+                            (cmp > 0 && ikey.sequence > sequence_))) {
+    if (cmp == 0) {
+      if (num_skipped >= max_skip_) {
+        num_skipped = 0;
+        IterKey last_key;
+        last_key.SetInternalKey(ParsedInternalKey(
+            saved_key_.GetKey(), kMaxSequenceNumber, kValueTypeForSeek));
+        iter_->Seek(last_key.GetKey());
+        RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
+      } else {
+        ++num_skipped;
+      }
     }
-
     iter_->Prev();
-    ++num_skipped;
     FindParseableKey(&ikey, kReverse);
   }
 }
@@ -625,21 +628,6 @@ void DBIter::FindParseableKey(ParsedInternalKey* ikey, Direction direction) {
 
 void DBIter::Seek(const Slice& target) {
   StopWatch sw(env_, statistics_, DB_SEEK);
-
-  // total ordering is not guaranteed if prefix_extractor is set
-  // hence prefix based seeks will not give correct results
-  if (iterate_upper_bound_ != nullptr && prefix_extractor_ != nullptr) {
-    if (!prefix_extractor_->InDomain(*iterate_upper_bound_) ||
-        !prefix_extractor_->InDomain(target) ||
-        prefix_extractor_->Transform(*iterate_upper_bound_).compare(
-          prefix_extractor_->Transform(target)) != 0) {
-      status_ = Status::InvalidArgument("read_options.iterate_*_bound "
-                  " and seek target need to have the same prefix.");
-      valid_ = false;
-      return;
-    }
-  }
-
   saved_key_.Clear();
   // now savved_key is used to store internal key.
   saved_key_.SetInternalKey(target, sequence_);
@@ -652,7 +640,7 @@ void DBIter::Seek(const Slice& target) {
   if (iter_->Valid()) {
     direction_ = kForward;
     ClearSavedValue();
-    FindNextUserEntry(false /*not skipping */);
+    FindNextUserEntry(false /* not skipping */);
   } else {
     valid_ = false;
   }
@@ -660,7 +648,7 @@ void DBIter::Seek(const Slice& target) {
 
 void DBIter::SeekToFirst() {
   // Don't use iter_::Seek() if we set a prefix extractor
-  // because prefix seek wiil be used.
+  // because prefix seek will be used.
   if (prefix_extractor_ != nullptr) {
     max_skip_ = std::numeric_limits<uint64_t>::max();
   }
@@ -681,7 +669,7 @@ void DBIter::SeekToFirst() {
 
 void DBIter::SeekToLast() {
   // Don't use iter_::Seek() if we set a prefix extractor
-  // because prefix seek wiil be used.
+  // because prefix seek will be used.
   if (prefix_extractor_ != nullptr) {
     max_skip_ = std::numeric_limits<uint64_t>::max();
   }

db/db_iter_test.cc

@@ -8,14 +8,17 @@
 #include <algorithm>
 #include <utility>
 
+#include "db/db_iter.h"
 #include "db/dbformat.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/options.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/statistics.h"
-#include "db/db_iter.h"
+#include "table/iterator_wrapper.h"
+#include "table/merger.h"
 #include "util/string_util.h"
+#include "util/sync_point.h"
 #include "util/testharness.h"
 #include "utilities/merge_operators.h"
 
@@ -48,11 +51,23 @@ class TestIterator : public Iterator {
   }
 
   void Add(std::string argkey, ValueType type, std::string argvalue) {
+    Add(argkey, type, argvalue, sequence_number_++);
+  }
+
+  void Add(std::string argkey, ValueType type, std::string argvalue,
+           size_t seq_num, bool update_iter = false) {
     valid_ = true;
-    ParsedInternalKey internal_key(argkey, sequence_number_++, type);
+    ParsedInternalKey internal_key(argkey, seq_num, type);
     data_.push_back(
         std::pair<std::string, std::string>(std::string(), argvalue));
     AppendInternalKey(&data_.back().first, internal_key);
+    if (update_iter && valid_ && cmp.Compare(data_.back().first, key()) < 0) {
+      // insert a key smaller than current key
+      Finish();
+      // data_[iter_] is not anymore the current element of the iterator.
+      // Increment it to reposition it to the right position.
+      iter_++;
+    }
   }
 
   // should be called before operations with iterator
@@ -1772,6 +1787,453 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
   ASSERT_FALSE(db_iter->Valid());
 }
 
+class DBIterWithMergeIterTest : public testing::Test {
+ public:
+  DBIterWithMergeIterTest()
+      : env_(Env::Default()), icomp_(BytewiseComparator()) {
+    options_.merge_operator = nullptr;
+
+    internal_iter1_ = new TestIterator(BytewiseComparator());
+    internal_iter1_->Add("a", kTypeValue, "1", 3u);
+    internal_iter1_->Add("f", kTypeValue, "2", 5u);
+    internal_iter1_->Add("g", kTypeValue, "3", 7u);
+    internal_iter1_->Finish();
+
+    internal_iter2_ = new TestIterator(BytewiseComparator());
+    internal_iter2_->Add("a", kTypeValue, "4", 6u);
+    internal_iter2_->Add("b", kTypeValue, "5", 1u);
+    internal_iter2_->Add("c", kTypeValue, "6", 2u);
+    internal_iter2_->Add("d", kTypeValue, "7", 3u);
+    internal_iter2_->Finish();
+
+    std::vector<Iterator*> child_iters;
+    child_iters.push_back(internal_iter1_);
+    child_iters.push_back(internal_iter2_);
+    InternalKeyComparator icomp(BytewiseComparator());
+    Iterator* merge_iter = NewMergingIterator(&icomp_, &child_iters[0], 2u);
+
+    db_iter_.reset(NewDBIterator(env_, ImmutableCFOptions(options_),
+                                 BytewiseComparator(), merge_iter,
+                                 8 /* read data earlier than seqId 8 */,
+                                 3 /* max iterators before reseek */));
+  }
+
+  Env* env_;
+  Options options_;
+  TestIterator* internal_iter1_;
+  TestIterator* internal_iter2_;
+  InternalKeyComparator icomp_;
+  Iterator* merge_iter_;
+  std::unique_ptr<Iterator> db_iter_;
+};
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIterator1) {
+  db_iter_->SeekToFirst();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+  db_iter_->Next();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Next();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Next();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Next();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+  db_iter_->Next();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "g");
+  ASSERT_EQ(db_iter_->value().ToString(), "3");
+  db_iter_->Next();
+  ASSERT_FALSE(db_iter_->Valid());
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIterator2) {
+  // Test Prev() when one child iterator is at its end.
+  db_iter_->Seek("g");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "g");
+  ASSERT_EQ(db_iter_->value().ToString(), "3");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace1) {
+  // Test Prev() when one child iterator is at its end but more rows
+  // are added.
+  db_iter_->Seek("f");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+
+  // Test call back inserts a key in the end of the mem table after
+  // MergeIterator::Prev() realized the mem table iterator is at its end
+  // and before an SeekToLast() is called.
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "MergeIterator::Prev:BeforeSeekToLast",
+      [&](void* arg) { internal_iter2_->Add("z", kTypeValue, "7", 12u); });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace2) {
+  // Test Prev() when one child iterator is at its end but more rows
+  // are added.
+  db_iter_->Seek("f");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+
+  // Test call back inserts entries for update a key in the end of the
+  // mem table after MergeIterator::Prev() realized the mem tableiterator is at
+  // its end and before an SeekToLast() is called.
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "MergeIterator::Prev:BeforeSeekToLast", [&](void* arg) {
+        internal_iter2_->Add("z", kTypeValue, "7", 12u);
+        internal_iter2_->Add("z", kTypeValue, "7", 11u);
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace3) {
+  // Test Prev() when one child iterator is at its end but more rows
+  // are added and max_skipped is triggered.
+  db_iter_->Seek("f");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+
+  // Test call back inserts entries for update a key in the end of the
+  // mem table after MergeIterator::Prev() realized the mem table iterator is at
+  // its end and before an SeekToLast() is called.
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "MergeIterator::Prev:BeforeSeekToLast", [&](void* arg) {
+        internal_iter2_->Add("z", kTypeValue, "7", 16u, true);
+        internal_iter2_->Add("z", kTypeValue, "7", 15u, true);
+        internal_iter2_->Add("z", kTypeValue, "7", 14u, true);
+        internal_iter2_->Add("z", kTypeValue, "7", 13u, true);
+        internal_iter2_->Add("z", kTypeValue, "7", 12u, true);
+        internal_iter2_->Add("z", kTypeValue, "7", 11u, true);
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace4) {
+  // Test Prev() when one child iterator has more rows inserted
+  // between Seek() and Prev() when changing directions.
+  internal_iter2_->Add("z", kTypeValue, "9", 4u);
+
+  db_iter_->Seek("g");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "g");
+  ASSERT_EQ(db_iter_->value().ToString(), "3");
+
+  // Test call back inserts entries for update a key before "z" in
+  // mem table after MergeIterator::Prev() calls mem table iterator's
+  // Seek() and before calling Prev()
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
+        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        if (it->key().starts_with("z")) {
+          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 15u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 14u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 13u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 12u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 11u, true);
+        }
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace5) {
+  internal_iter2_->Add("z", kTypeValue, "9", 4u);
+
+  // Test Prev() when one child iterator has more rows inserted
+  // between Seek() and Prev() when changing directions.
+  db_iter_->Seek("g");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "g");
+  ASSERT_EQ(db_iter_->value().ToString(), "3");
+
+  // Test call back inserts entries for update a key before "z" in
+  // mem table after MergeIterator::Prev() calls mem table iterator's
+  // Seek() and before calling Prev()
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
+        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        if (it->key().starts_with("z")) {
+          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 15u, true);
+        }
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace6) {
+  internal_iter2_->Add("z", kTypeValue, "9", 4u);
+
+  // Test Prev() when one child iterator has more rows inserted
+  // between Seek() and Prev() when changing directions.
+  db_iter_->Seek("g");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "g");
+  ASSERT_EQ(db_iter_->value().ToString(), "3");
+
+  // Test call back inserts an entry for update a key before "z" in
+  // mem table after MergeIterator::Prev() calls mem table iterator's
+  // Seek() and before calling Prev()
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
+        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        if (it->key().starts_with("z")) {
+          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
+        }
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace7) {
+  internal_iter1_->Add("u", kTypeValue, "10", 4u);
+  internal_iter1_->Add("v", kTypeValue, "11", 4u);
+  internal_iter1_->Add("w", kTypeValue, "12", 4u);
+  internal_iter2_->Add("z", kTypeValue, "9", 4u);
+
+  // Test Prev() when one child iterator has more rows inserted
+  // between Seek() and Prev() when changing directions.
+  db_iter_->Seek("g");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "g");
+  ASSERT_EQ(db_iter_->value().ToString(), "3");
+
+  // Test call back inserts entries for update a key before "z" in
+  // mem table after MergeIterator::Prev() calls mem table iterator's
+  // Seek() and before calling Prev()
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
+        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        if (it->key().starts_with("z")) {
+          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 15u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 14u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 13u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 12u, true);
+          internal_iter2_->Add("x", kTypeValue, "7", 11u, true);
+        }
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "c");
+  ASSERT_EQ(db_iter_->value().ToString(), "6");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "b");
+  ASSERT_EQ(db_iter_->value().ToString(), "5");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "a");
+  ASSERT_EQ(db_iter_->value().ToString(), "4");
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+TEST_F(DBIterWithMergeIterTest, InnerMergeIteratorDataRace8) {
+  // internal_iter1_: a, f, g
+  // internal_iter2_: a, b, c, d, adding (z)
+  internal_iter2_->Add("z", kTypeValue, "9", 4u);
+
+  // Test Prev() when one child iterator has more rows inserted
+  // between Seek() and Prev() when changing directions.
+  db_iter_->Seek("g");
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "g");
+  ASSERT_EQ(db_iter_->value().ToString(), "3");
+
+  // Test call back inserts two keys before "z" in mem table after
+  // MergeIterator::Prev() calls mem table iterator's Seek() and
+  // before calling Prev()
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "MergeIterator::Prev:BeforePrev", [&](void* arg) {
+        IteratorWrapper* it = reinterpret_cast<IteratorWrapper*>(arg);
+        if (it->key().starts_with("z")) {
+          internal_iter2_->Add("x", kTypeValue, "7", 16u, true);
+          internal_iter2_->Add("y", kTypeValue, "7", 17u, true);
+        }
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "f");
+  ASSERT_EQ(db_iter_->value().ToString(), "2");
+  db_iter_->Prev();
+  ASSERT_TRUE(db_iter_->Valid());
+  ASSERT_EQ(db_iter_->key().ToString(), "d");
+  ASSERT_EQ(db_iter_->value().ToString(), "7");
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_tailing_iter_test.cc

@@ -12,6 +12,7 @@
 // which is a pity, it is a good test
 #if !(defined NDEBUG) || !defined(OS_WIN)
 
+#include "db/forward_iterator.h"
 #include "port/stack_trace.h"
 #include "util/db_test_util.h"
 
@@ -68,6 +69,7 @@ TEST_F(DBTestTailingIterator, TailingIteratorSeekToNext) {
   read_options.tailing = true;
 
   std::unique_ptr<Iterator> iter(db_->NewIterator(read_options, handles_[1]));
+  std::unique_ptr<Iterator> itern(db_->NewIterator(read_options, handles_[1]));
   std::string value(1024, 'a');
 
   const int num_records = 1000;
@@ -88,7 +90,140 @@ TEST_F(DBTestTailingIterator, TailingIteratorSeekToNext) {
     iter->Seek(target);
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(iter->key().compare(key), 0);
+    if (i == 1) {
+      itern->SeekToFirst();
+    } else {
+      itern->Next();
+    }
+    ASSERT_TRUE(itern->Valid());
+    ASSERT_EQ(itern->key().compare(key), 0);
   }
+  rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+  for (int i = 2 * num_records; i > 0; --i) {
+    char buf1[32];
+    char buf2[32];
+    snprintf(buf1, sizeof(buf1), "00a0%016d", i * 5);
+
+    Slice key(buf1, 20);
+    ASSERT_OK(Put(1, key, value));
+
+    if (i % 100 == 99) {
+      ASSERT_OK(Flush(1));
+    }
+
+    snprintf(buf2, sizeof(buf2), "00a0%016d", i * 5 - 2);
+    Slice target(buf2, 20);
+    iter->Seek(target);
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().compare(key), 0);
+  }
+}
+
+TEST_F(DBTestTailingIterator, TailingIteratorTrimSeekToNext) {
+  const uint64_t k150KB = 150 * 1024;
+  Options options;
+  options.write_buffer_size = k150KB;
+  options.max_write_buffer_number = 3;
+  options.min_write_buffer_number_to_merge = 2;
+  CreateAndReopenWithCF({"pikachu"}, options);
+  ReadOptions read_options;
+  read_options.tailing = true;
+  int num_iters, deleted_iters;
+
+  char bufe[32];
+  snprintf(bufe, sizeof(bufe), "00b0%016d", 0);
+  Slice keyu(bufe, 20);
+  read_options.iterate_upper_bound = &keyu;
+  std::unique_ptr<Iterator> iter(db_->NewIterator(read_options, handles_[1]));
+  std::unique_ptr<Iterator> itern(db_->NewIterator(read_options, handles_[1]));
+  std::unique_ptr<Iterator> iterh(db_->NewIterator(read_options, handles_[1]));
+  std::string value(1024, 'a');
+  bool file_iters_deleted = false;
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "ForwardIterator::SeekInternal:Return", [&](void* arg) {
+        ForwardIterator* fiter = reinterpret_cast<ForwardIterator*>(arg);
+        ASSERT_TRUE(!file_iters_deleted ||
+                    fiter->TEST_CheckDeletedIters(&deleted_iters, &num_iters));
+      });
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "ForwardIterator::Next:Return", [&](void* arg) {
+        ForwardIterator* fiter = reinterpret_cast<ForwardIterator*>(arg);
+        ASSERT_TRUE(!file_iters_deleted ||
+                    fiter->TEST_CheckDeletedIters(&deleted_iters, &num_iters));
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+  const int num_records = 1000;
+  for (int i = 1; i < num_records; ++i) {
+    char buf1[32];
+    char buf2[32];
+    char buf3[32];
+    char buf4[32];
+    snprintf(buf1, sizeof(buf1), "00a0%016d", i * 5);
+    snprintf(buf3, sizeof(buf1), "00b0%016d", i * 5);
+
+    Slice key(buf1, 20);
+    ASSERT_OK(Put(1, key, value));
+    Slice keyn(buf3, 20);
+    ASSERT_OK(Put(1, keyn, value));
+
+    if (i % 100 == 99) {
+      ASSERT_OK(Flush(1));
+      dbfull()->TEST_WaitForCompact();
+      if (i == 299) {
+        file_iters_deleted = true;
+      }
+      snprintf(buf4, sizeof(buf1), "00a0%016d", i * 5 / 2);
+      Slice target(buf4, 20);
+      iterh->Seek(target);
+      ASSERT_TRUE(iter->Valid());
+      for (int j = (i + 1) * 5 / 2; j < i * 5; j += 5) {
+        iterh->Next();
+        ASSERT_TRUE(iterh->Valid());
+      }
+      if (i == 299) {
+        file_iters_deleted = false;
+      }
+    }
+
+    file_iters_deleted = true;
+    snprintf(buf2, sizeof(buf2), "00a0%016d", i * 5 - 2);
+    Slice target(buf2, 20);
+    iter->Seek(target);
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(iter->key().compare(key), 0);
+    ASSERT_LE(num_iters, 1);
+    if (i == 1) {
+      itern->SeekToFirst();
+    } else {
+      itern->Next();
+    }
+    ASSERT_TRUE(itern->Valid());
+    ASSERT_EQ(itern->key().compare(key), 0);
+    ASSERT_LE(num_iters, 1);
+    file_iters_deleted = false;
+  }
+  iter = 0;
+  itern = 0;
+  iterh = 0;
+  BlockBasedTableOptions table_options;
+  table_options.no_block_cache = true;
+  table_options.block_cache_compressed = nullptr;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  ReopenWithColumnFamilies({"default", "pikachu"}, options);
+  read_options.read_tier = kBlockCacheTier;
+  std::unique_ptr<Iterator> iteri(db_->NewIterator(read_options, handles_[1]));
+  char buf5[32];
+  snprintf(buf5, sizeof(buf5), "00a0%016d", (num_records / 2) * 5 - 2);
+  Slice target1(buf5, 20);
+  iteri->Seek(target1);
+  ASSERT_TRUE(iteri->status().IsIncomplete());
+  iteri = 0;
+
+  read_options.read_tier = kReadAllTier;
+  options.table_factory.reset(NewBlockBasedTableFactory());
+  ReopenWithColumnFamilies({"default", "pikachu"}, options);
+  iter.reset(db_->NewIterator(read_options, handles_[1]));
   for (int i = 2 * num_records; i > 0; --i) {
     char buf1[32];
     char buf2[32];
@@ -240,6 +375,49 @@ TEST_F(DBTestTailingIterator, TailingIteratorSeekToSame) {
   ASSERT_EQ(found, iter->key().ToString());
 }
 
+// Sets iterate_upper_bound and verifies that ForwardIterator doesn't call
+// Seek() on immutable iterators when target key is >= prev_key and all
+// iterators, including the memtable iterator, are over the upper bound.
+TEST_F(DBTestTailingIterator, TailingIteratorUpperBound) {
+  CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
+
+  const Slice upper_bound("20", 3);
+  ReadOptions read_options;
+  read_options.tailing = true;
+  read_options.iterate_upper_bound = &upper_bound;
+
+  ASSERT_OK(Put(1, "11", "11"));
+  ASSERT_OK(Put(1, "12", "12"));
+  ASSERT_OK(Put(1, "22", "22"));
+  ASSERT_OK(Flush(1));  // flush all those keys to an immutable SST file
+
+  // Add another key to the memtable.
+  ASSERT_OK(Put(1, "21", "21"));
+
+  std::unique_ptr<Iterator> it(db_->NewIterator(read_options, handles_[1]));
+  it->Seek("12");
+  ASSERT_TRUE(it->Valid());
+  ASSERT_EQ("12", it->key().ToString());
+
+  it->Next();
+  // Not valid since "21" is over the upper bound.
+  ASSERT_FALSE(it->Valid());
+
+  // This keeps track of the number of times NeedToSeekImmutable() was true.
+  int immutable_seeks = 0;
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "ForwardIterator::SeekInternal:Immutable",
+      [&](void* arg) { ++immutable_seeks; });
+
+  // Seek to 13. This should not require any immutable seeks.
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+  it->Seek("13");
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+
+  ASSERT_FALSE(it->Valid());
+  ASSERT_EQ(0, immutable_seeks);
+}
+
 TEST_F(DBTestTailingIterator, ManagedTailingIteratorSingle) {
   ReadOptions read_options;
   read_options.tailing = true;

db/db_universal_compaction_test.cc

@@ -11,7 +11,6 @@
 #include "util/db_test_util.h"
 #if !(defined NDEBUG) || !defined(OS_WIN)
 #include "util/sync_point.h"
-#endif
 
 namespace rocksdb {
 
@@ -113,9 +112,6 @@ class DelayFilterFactory : public CompactionFilterFactory {
 };
 }  // namespace
 
-// SyncPoint is not supported in released Windows mode.
-#if !(defined NDEBUG) || !defined(OS_WIN)
-
 // TODO(kailiu) The tests on UniversalCompaction has some issues:
 //  1. A lot of magic numbers ("11" or "12").
 //  2. Made assumption on the memtable flush conditions, which may change from
@@ -124,7 +120,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
   Options options;
   options.compaction_style = kCompactionStyleUniversal;
   options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;     // 100KB
+  options.write_buffer_size = 105 << 10;  // 105KB
+  options.arena_block_size = 4 << 10;
   options.target_file_size_base = 32 << 10;  // 32KB
   // trigger compaction if there are >= 4 files
   options.level0_file_num_compaction_trigger = 4;
@@ -155,22 +152,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
   //   compaction.
   for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
        num++) {
-    // Write 110KB (11 values, each 10K)
-    for (int i = 0; i < 12; i++) {
-      ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
-      key_idx++;
-    }
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-    ASSERT_EQ(NumSortedRuns(1), num + 1);
+    // Write 100KB
+    GenerateNewFile(1, &rnd, &key_idx);
   }
 
   // Generate one more file at level-0, which should trigger level-0
   // compaction.
-  for (int i = 0; i < 11; i++) {
-    ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
-    key_idx++;
-  }
-  dbfull()->TEST_WaitForCompact();
+  GenerateNewFile(1, &rnd, &key_idx);
   // Suppose each file flushed from mem table has size 1. Now we compact
   // (level0_file_num_compaction_trigger+1)=4 files and should have a big
   // file of size 4.
@@ -187,22 +175,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
   ASSERT_OK(Flush(1));
   for (int num = 0; num < options.level0_file_num_compaction_trigger - 3;
        num++) {
-    // Write 110KB (11 values, each 10K)
-    for (int i = 0; i < 11; i++) {
-      ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
-      key_idx++;
-    }
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
+    GenerateNewFile(1, &rnd, &key_idx);
     ASSERT_EQ(NumSortedRuns(1), num + 3);
   }
 
   // Generate one more file at level-0, which should trigger level-0
   // compaction.
-  for (int i = 0; i < 11; i++) {
-    ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
-    key_idx++;
-  }
-  dbfull()->TEST_WaitForCompact();
+  GenerateNewFile(1, &rnd, &key_idx);
   // Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1.
   // After compaction, we should have 2 files, with size 4, 2.4.
   ASSERT_EQ(NumSortedRuns(1), 2);
@@ -212,22 +191,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
   //   generating new files at level 0.
   for (int num = 0; num < options.level0_file_num_compaction_trigger - 3;
        num++) {
-    // Write 110KB (11 values, each 10K)
-    for (int i = 0; i < 11; i++) {
-      ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
-      key_idx++;
-    }
-    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
+    GenerateNewFile(1, &rnd, &key_idx);
     ASSERT_EQ(NumSortedRuns(1), num + 3);
   }
 
   // Generate one more file at level-0, which should trigger level-0
   // compaction.
-  for (int i = 0; i < 12; i++) {
-    ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
-    key_idx++;
-  }
-  dbfull()->TEST_WaitForCompact();
+  GenerateNewFile(1, &rnd, &key_idx);
   // Before compaction, we have 4 files at level 0, with size 4, 2.4, 1, 1.
   // After compaction, we should have 3 files, with size 4, 2.4, 2.
   ASSERT_EQ(NumSortedRuns(1), 3);
@@ -235,10 +205,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
   // Stage 4:
   //   Now we have 3 files at level 0, with size 4, 2.4, 2. Let's generate a
   //   new file of size 1.
-  for (int i = 0; i < 11; i++) {
-    ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
-    key_idx++;
-  }
+  GenerateNewFile(1, &rnd, &key_idx);
   dbfull()->TEST_WaitForCompact();
   // Level-0 compaction is triggered, but no file will be picked up.
   ASSERT_EQ(NumSortedRuns(1), 4);
@@ -247,17 +214,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
   //   Now we have 4 files at level 0, with size 4, 2.4, 2, 1. Let's generate
   //   a new file of size 1.
   filter->expect_full_compaction_.store(true);
-  for (int i = 0; i < 11; i++) {
-    ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
-    key_idx++;
-  }
+  GenerateNewFile(1, &rnd, &key_idx);
   dbfull()->TEST_WaitForCompact();
   // All files at level 0 will be compacted into a single one.
   ASSERT_EQ(NumSortedRuns(1), 1);
 
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
 
 TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
   Options options;
@@ -340,9 +303,10 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
   }
 
   // expect fail since universal compaction only allow L0 output
-  ASSERT_TRUE(!dbfull()->CompactFiles(
-      CompactionOptions(), handles_[1],
-      compaction_input_file_names, 1).ok());
+  ASSERT_FALSE(dbfull()
+                   ->CompactFiles(CompactionOptions(), handles_[1],
+                                  compaction_input_file_names, 1)
+                   .ok());
 
   // expect ok and verify the compacted files no longer exist.
   ASSERT_OK(dbfull()->CompactFiles(
@@ -573,7 +537,8 @@ INSTANTIATE_TEST_CASE_P(DBTestUniversalCompactionParallel,
 TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
   Options options;
   options.compaction_style = kCompactionStyleUniversal;
-  options.write_buffer_size = 100 << 10;     // 100KB
+  options.write_buffer_size = 105 << 10;    // 105KB
+  options.arena_block_size = 4 << 10;       // 4KB
   options.target_file_size_base = 32 << 10;  // 32KB
   options.level0_file_num_compaction_trigger = 4;
   options.num_levels = num_levels_;
@@ -586,9 +551,9 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
   int key_idx = 0;
 
   for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
-    // Write 110KB (11 values, each 10K)
-    for (int i = 0; i < 11; i++) {
-      ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 10000)));
+    // Write 100KB (100 values, each 1K)
+    for (int i = 0; i < 100; i++) {
+      ASSERT_OK(Put(1, Key(key_idx), RandomString(&rnd, 990)));
       key_idx++;
     }
     dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
@@ -605,7 +570,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
 TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
   Options options = CurrentOptions();
   options.compaction_style = kCompactionStyleUniversal;
-  options.write_buffer_size = 100 << 10;     // 100KB
+  options.write_buffer_size = 105 << 10;    // 105KB
+  options.arena_block_size = 4 << 10;       // 4KB
   options.target_file_size_base = 32 << 10;  // 32KB
   // trigger compaction if there are >= 4 files
   options.level0_file_num_compaction_trigger = 4;
@@ -623,9 +589,9 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
   //   compaction.
   for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
        num++) {
-    // Write 110KB (11 values, each 10K)
-    for (int i = 0; i < 11; i++) {
-      ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
+    // Write 100KB (100 values, each 1K)
+    for (int i = 0; i < 100; i++) {
+      ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 990)));
       key_idx++;
     }
     dbfull()->TEST_WaitForFlushMemTable();
@@ -634,8 +600,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
 
   // Generate one more file at level-0, which should trigger level-0
   // compaction.
-  for (int i = 0; i < 11; i++) {
-    ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
+  for (int i = 0; i < 100; i++) {
+    ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 990)));
     key_idx++;
   }
   dbfull()->TEST_WaitForCompact();
@@ -655,8 +621,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
   for (int num = 0; num < options.level0_file_num_compaction_trigger - 3;
        num++) {
     // Write 110KB (11 values, each 10K)
-    for (int i = 0; i < 11; i++) {
-      ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
+    for (int i = 0; i < 100; i++) {
+      ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 990)));
       key_idx++;
     }
     dbfull()->TEST_WaitForFlushMemTable();
@@ -665,8 +631,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
 
   // Generate one more file at level-0, which should trigger level-0
   // compaction.
-  for (int i = 0; i < 11; i++) {
-    ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
+  for (int i = 0; i < 100; i++) {
+    ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 990)));
     key_idx++;
   }
   dbfull()->TEST_WaitForCompact();
@@ -676,8 +642,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionStopStyleSimilarSize) {
   // Stage 3:
   //   Now we have 3 files at level 0, with size 4, 0.4, 2. Generate one
   //   more file at level-0, which should trigger level-0 compaction.
-  for (int i = 0; i < 11; i++) {
-    ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
+  for (int i = 0; i < 100; i++) {
+    ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 990)));
     key_idx++;
   }
   dbfull()->TEST_WaitForCompact();
@@ -886,7 +852,8 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
   options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
   options.db_paths.emplace_back(dbname_ + "_4", 1024 * 1024 * 1024);
   options.compaction_style = kCompactionStyleUniversal;
-  options.write_buffer_size = 100 << 10;  // 100KB
+  options.write_buffer_size = 110 << 10;  // 105KB
+  options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = 1;
   options = CurrentOptions(options);
@@ -964,7 +931,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Reopen(options);
@@ -972,7 +939,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Destroy(options);
@@ -1080,7 +1047,9 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
   options.db_paths.emplace_back(dbname_, 500 * 1024);
   options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024);
   options.compaction_style = kCompactionStyleUniversal;
-  options.write_buffer_size = 100 << 10;  // 100KB
+  options.write_buffer_size = 110 << 10;  // 105KB
+  options.arena_block_size = 4 * 1024;
+  options.arena_block_size = 4 << 10;
   options.level0_file_num_compaction_trigger = 2;
   options.num_levels = 1;
   options = CurrentOptions(options);
@@ -1155,7 +1124,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Reopen(options);
@@ -1163,7 +1132,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
   for (int i = 0; i < key_idx; i++) {
     auto v = Get(Key(i));
     ASSERT_NE(v, "NOT_FOUND");
-    ASSERT_TRUE(v.size() == 1 || v.size() == 10000);
+    ASSERT_TRUE(v.size() == 1 || v.size() == 990);
   }
 
   Destroy(options);
@@ -1241,6 +1210,8 @@ INSTANTIATE_TEST_CASE_P(DBTestUniversalManualCompactionOutputPathId,
 
 }  // namespace rocksdb
 
+#endif  // !(defined NDEBUG) || !defined(OS_WIN)
+
 int main(int argc, char** argv) {
 #if !(defined NDEBUG) || !defined(OS_WIN)
   rocksdb::port::InstallStackTraceHandler();

db/dbformat.h

@@ -166,6 +166,7 @@ class InternalKey {
   }
 
   Slice user_key() const { return ExtractUserKey(rep_); }
+  size_t size() { return rep_.size(); }
 
   void SetFrom(const ParsedInternalKey& p) {
     rep_.clear();

db/fault_injection_test.cc

@@ -11,6 +11,8 @@
 // the last "sync". It then checks for data loss errors by purposely dropping
 // file data (or entire files) not protected by a "sync".
 
+#if !(defined NDEBUG) || !defined(OS_WIN)
+
 #include <map>
 #include <set>
 #include "db/db_impl.h"
@@ -931,7 +933,13 @@ INSTANTIATE_TEST_CASE_P(FaultTest, FaultInjectionTest, ::testing::Bool());
 
 }  // namespace rocksdb
 
+#endif // #if !(defined NDEBUG) || !defined(OS_WIN)
+
 int main(int argc, char** argv) {
+#if !(defined NDEBUG) || !defined(OS_WIN)
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
+#else
+  return 0;
+#endif
 }

db/flush_job.cc

@@ -14,6 +14,7 @@
 #endif
 
 #include <inttypes.h>
+
 #include <algorithm>
 #include <vector>
 
@@ -28,8 +29,8 @@
 #include "db/memtable_list.h"
 #include "db/merge_context.h"
 #include "db/version_set.h"
-#include "port/port.h"
 #include "port/likely.h"
+#include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/statistics.h"
@@ -43,11 +44,11 @@
 #include "util/coding.h"
 #include "util/event_logger.h"
 #include "util/file_util.h"
-#include "util/logging.h"
+#include "util/iostats_context_imp.h"
 #include "util/log_buffer.h"
+#include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/perf_context_imp.h"
-#include "util/iostats_context_imp.h"
 #include "util/stop_watch.h"
 #include "util/sync_point.h"
 #include "util/thread_status_util.h"
@@ -60,9 +61,9 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                    const EnvOptions& env_options, VersionSet* versions,
                    InstrumentedMutex* db_mutex,
                    std::atomic<bool>* shutting_down,
-                   SequenceNumber newest_snapshot, JobContext* job_context,
-                   LogBuffer* log_buffer, Directory* db_directory,
-                   Directory* output_file_directory,
+                   std::vector<SequenceNumber> existing_snapshots,
+                   JobContext* job_context, LogBuffer* log_buffer,
+                   Directory* db_directory, Directory* output_file_directory,
                    CompressionType output_compression, Statistics* stats,
                    EventLogger* event_logger)
     : dbname_(dbname),
@@ -73,7 +74,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
       versions_(versions),
       db_mutex_(db_mutex),
       shutting_down_(shutting_down),
-      newest_snapshot_(newest_snapshot),
+      existing_snapshots_(std::move(existing_snapshots)),
       job_context_(job_context),
       log_buffer_(log_buffer),
       db_directory_(db_directory),
@@ -87,7 +88,6 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
 }
 
 FlushJob::~FlushJob() {
-  TEST_SYNC_POINT("FlushJob::~FlushJob()");
   ThreadStatusUtil::ResetThreadStatus();
 }
 
@@ -111,9 +111,8 @@ void FlushJob::ReportFlushInputSize(const autovector<MemTable*>& mems) {
 }
 
 void FlushJob::RecordFlushIOStats() {
-  ThreadStatusUtil::IncreaseThreadOperationProperty(
+  ThreadStatusUtil::SetThreadOperationProperty(
       ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
-  IOSTATS_RESET(bytes_written);
 }
 
 Status FlushJob::Run(FileMetaData* file_meta) {
@@ -154,6 +153,7 @@ Status FlushJob::Run(FileMetaData* file_meta) {
   if (!s.ok()) {
     cfd_->imm()->RollbackMemtableFlush(mems, meta.fd.GetNumber());
   } else {
+    TEST_SYNC_POINT("FlushJob::InstallResults");
     // Replace immutable memtable with the generated Table
     s = cfd_->imm()->InstallMemtableFlushResults(
         cfd_, mutable_cf_options_, mems, versions_, db_mutex_,
@@ -189,8 +189,6 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
   // path 0 for level 0 file.
   meta->fd = FileDescriptor(versions_->NewFileNumber(), 0, 0);
 
-  const SequenceNumber earliest_seqno_in_memtable =
-      mems[0]->GetFirstSequenceNumber();
   Version* base = cfd_->current();
   base->Ref();  // it is likely that we do not need this reference
   Status s;
@@ -221,6 +219,7 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
                          << total_num_entries << "num_deletes"
                          << total_num_deletes << "memory_usage"
                          << total_memory_usage;
+
     TableFileCreationInfo info;
     {
       ScopedArenaIterator iter(
@@ -232,14 +231,13 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
 
       TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:output_compression",
                                &output_compression_);
-      s = BuildTable(dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
-                     cfd_->table_cache(), iter.get(), meta,
-                     cfd_->internal_comparator(),
-                     cfd_->int_tbl_prop_collector_factories(), newest_snapshot_,
-                     earliest_seqno_in_memtable, output_compression_,
-                     cfd_->ioptions()->compression_opts,
-                     mutable_cf_options_.paranoid_file_checks, Env::IO_HIGH,
-                     &info.table_properties);
+      s = BuildTable(
+          dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
+          cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(),
+          cfd_->int_tbl_prop_collector_factories(), existing_snapshots_,
+          output_compression_, cfd_->ioptions()->compression_opts,
+          mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
+          Env::IO_HIGH, &info.table_properties);
       LogFlush(db_options_.info_log);
     }
     Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
@@ -262,6 +260,7 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
       EventHelpers::LogAndNotifyTableFileCreation(
           event_logger_, db_options_.listeners,
           meta->fd, info);
+      TEST_SYNC_POINT("FlushJob::LogAndNotifyTableFileCreation()");
     }
 
     if (!db_options_.disableDataSync && output_file_directory_ != nullptr) {

db/flush_job.h

@@ -17,11 +17,11 @@
 #include <string>
 
 #include "db/dbformat.h"
-#include "db/log_writer.h"
-#include "db/snapshot.h"
 #include "db/column_family.h"
-#include "db/version_edit.h"
+#include "db/log_writer.h"
 #include "db/memtable_list.h"
+#include "db/snapshot_impl.h"
+#include "db/version_edit.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
@@ -57,10 +57,11 @@ class FlushJob {
            const MutableCFOptions& mutable_cf_options,
            const EnvOptions& env_options, VersionSet* versions,
            InstrumentedMutex* db_mutex, std::atomic<bool>* shutting_down,
-           SequenceNumber newest_snapshot, JobContext* job_context,
-           LogBuffer* log_buffer, Directory* db_directory,
-           Directory* output_file_directory, CompressionType output_compression,
-           Statistics* stats, EventLogger* event_logger);
+           std::vector<SequenceNumber> existing_snapshots,
+           JobContext* job_context, LogBuffer* log_buffer,
+           Directory* db_directory, Directory* output_file_directory,
+           CompressionType output_compression, Statistics* stats,
+           EventLogger* event_logger);
 
   ~FlushJob();
 
@@ -80,7 +81,7 @@ class FlushJob {
   VersionSet* versions_;
   InstrumentedMutex* db_mutex_;
   std::atomic<bool>* shutting_down_;
-  SequenceNumber newest_snapshot_;
+  std::vector<SequenceNumber> existing_snapshots_;
   JobContext* job_context_;
   LogBuffer* log_buffer_;
   Directory* db_directory_;

db/flush_job_test.cc

@@ -3,6 +3,7 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
+#include <algorithm>
 #include <map>
 #include <string>
 
@@ -91,7 +92,7 @@ TEST_F(FlushJobTest, Empty) {
   FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     SequenceNumber(), &job_context, nullptr, nullptr, nullptr,
+                     {}, &job_context, nullptr, nullptr, nullptr,
                      kNoCompression, nullptr, &event_logger);
   ASSERT_OK(flush_job.Run());
   job_context.Clean();
@@ -103,10 +104,18 @@ TEST_F(FlushJobTest, NonEmpty) {
   auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
                                            kMaxSequenceNumber);
   new_mem->Ref();
-  std::map<std::string, std::string> inserted_keys;
+  auto inserted_keys = mock::MakeMockFile();
+  // Test data:
+  //   seqno [    1,    2 ... 8998, 8999, 9000, 9001, 9002 ... 9999 ]
+  //   key   [ 1001, 1002 ... 9998, 9999,    0,    1,    2 ...  999 ]
+  // Expected:
+  //   smallest_key   = "0"
+  //   largest_key    = "9999"
+  //   smallest_seqno = 1
+  //   smallest_seqno = 9999
   for (int i = 1; i < 10000; ++i) {
-    std::string key(ToString(i));
-    std::string value("value" + ToString(i));
+    std::string key(ToString((i + 1000) % 10000));
+    std::string value("value" + key);
     new_mem->Add(SequenceNumber(i), kTypeValue, key, value);
     InternalKey internal_key(key, SequenceNumber(i), kTypeValue);
     inserted_keys.insert({internal_key.Encode().ToString(), value});
@@ -122,7 +131,71 @@ TEST_F(FlushJobTest, NonEmpty) {
   FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
                      db_options_, *cfd->GetLatestMutableCFOptions(),
                      env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     SequenceNumber(), &job_context, nullptr, nullptr, nullptr,
+                     {}, &job_context, nullptr, nullptr, nullptr,
+                     kNoCompression, nullptr, &event_logger);
+  FileMetaData fd;
+  mutex_.Lock();
+  ASSERT_OK(flush_job.Run(&fd));
+  mutex_.Unlock();
+  ASSERT_EQ(ToString(0), fd.smallest.user_key().ToString());
+  ASSERT_EQ(ToString(9999), fd.largest.user_key().ToString());
+  ASSERT_EQ(1, fd.smallest_seqno);
+  ASSERT_EQ(9999, fd.largest_seqno);
+  mock_table_factory_->AssertSingleFile(inserted_keys);
+  job_context.Clean();
+}
+
+TEST_F(FlushJobTest, Snapshots) {
+  JobContext job_context(0);
+  auto cfd = versions_->GetColumnFamilySet()->GetDefault();
+  auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions(),
+                                           kMaxSequenceNumber);
+
+  std::vector<SequenceNumber> snapshots;
+  std::set<SequenceNumber> snapshots_set;
+  int keys = 10000;
+  int max_inserts_per_keys = 8;
+
+  Random rnd(301);
+  for (int i = 0; i < keys / 2; ++i) {
+    snapshots.push_back(rnd.Uniform(keys * (max_inserts_per_keys / 2)) + 1);
+    snapshots_set.insert(snapshots.back());
+  }
+  std::sort(snapshots.begin(), snapshots.end());
+
+  new_mem->Ref();
+  SequenceNumber current_seqno = 0;
+  auto inserted_keys = mock::MakeMockFile();
+  for (int i = 1; i < keys; ++i) {
+    std::string key(ToString(i));
+    int insertions = rnd.Uniform(max_inserts_per_keys);
+    for (int j = 0; j < insertions; ++j) {
+      std::string value(test::RandomHumanReadableString(&rnd, 10));
+      auto seqno = ++current_seqno;
+      new_mem->Add(SequenceNumber(seqno), kTypeValue, key, value);
+      // a key is visible only if:
+      // 1. it's the last one written (j == insertions - 1)
+      // 2. there's a snapshot pointing at it
+      bool visible = (j == insertions - 1) ||
+                     (snapshots_set.find(seqno) != snapshots_set.end());
+      if (visible) {
+        InternalKey internal_key(key, seqno, kTypeValue);
+        inserted_keys.insert({internal_key.Encode().ToString(), value});
+      }
+    }
+  }
+
+  autovector<MemTable*> to_delete;
+  cfd->imm()->Add(new_mem, &to_delete);
+  for (auto& m : to_delete) {
+    delete m;
+  }
+
+  EventLogger event_logger(db_options_.info_log.get());
+  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
+                     db_options_, *cfd->GetLatestMutableCFOptions(),
+                     env_options_, versions_.get(), &mutex_, &shutting_down_,
+                     snapshots, &job_context, nullptr, nullptr, nullptr,
                      kNoCompression, nullptr, &event_logger);
   mutex_.Lock();
   ASSERT_OK(flush_job.Run());

db/forward_iterator.cc

@@ -19,6 +19,7 @@
 #include "rocksdb/slice_transform.h"
 #include "table/merger.h"
 #include "db/dbformat.h"
+#include "util/sync_point.h"
 
 namespace rocksdb {
 
@@ -47,7 +48,8 @@ class LevelIterator : public Iterator {
     assert(file_index_ < files_.size());
     file_iter_.reset(cfd_->table_cache()->NewIterator(
         read_options_, *(cfd_->soptions()), cfd_->internal_comparator(),
-        files_[file_index_]->fd, nullptr /* table_reader_ptr */, false));
+        files_[file_index_]->fd, nullptr /* table_reader_ptr */, nullptr,
+        false));
   }
   void SeekToLast() override {
     status_ = Status::NotSupported("LevelIterator::SeekToLast()");
@@ -115,7 +117,8 @@ class LevelIterator : public Iterator {
 };
 
 ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
-    ColumnFamilyData* cfd, SuperVersion* current_sv)
+                                 ColumnFamilyData* cfd,
+                                 SuperVersion* current_sv)
     : db_(db),
       read_options_(read_options),
       cfd_(cfd),
@@ -125,9 +128,11 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
       sv_(current_sv),
       mutable_iter_(nullptr),
       current_(nullptr),
+      valid_(false),
       status_(Status::OK()),
       immutable_status_(Status::OK()),
-      valid_(false),
+      has_iter_trimmed_for_upper_bound_(false),
+      current_over_upper_bound_(false),
       is_prev_set_(false),
       is_prev_inclusive_(false) {
   if (sv_) {
@@ -175,7 +180,8 @@ void ForwardIterator::Cleanup(bool release_sv) {
 }
 
 bool ForwardIterator::Valid() const {
-  return valid_;
+  // See UpdateCurrent().
+  return valid_ ? !current_over_upper_bound_ : false;
 }
 
 void ForwardIterator::SeekToFirst() {
@@ -188,7 +194,17 @@ void ForwardIterator::SeekToFirst() {
   SeekInternal(Slice(), true);
 }
 
+bool ForwardIterator::IsOverUpperBound(const Slice& internal_key) const {
+  return !(read_options_.iterate_upper_bound == nullptr ||
+           cfd_->internal_comparator().user_comparator()->Compare(
+               ExtractUserKey(internal_key),
+               *read_options_.iterate_upper_bound) < 0);
+}
+
 void ForwardIterator::Seek(const Slice& internal_key) {
+  if (IsOverUpperBound(internal_key)) {
+    valid_ = false;
+  }
   if (sv_ == nullptr ||
       sv_ ->version_number != cfd_->GetSuperVersionNumber()) {
     RebuildIterators(true);
@@ -211,11 +227,19 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
   // an option to turn it off.
   if (seek_to_first || NeedToSeekImmutable(internal_key)) {
     immutable_status_ = Status::OK();
+    if (has_iter_trimmed_for_upper_bound_) {
+      // Some iterators are trimmed. Need to rebuild.
+      RebuildIterators(true);
+      // Already seeked mutable iter, so seek again
+      seek_to_first ? mutable_iter_->SeekToFirst()
+                    : mutable_iter_->Seek(internal_key);
+    }
     {
       auto tmp = MinIterHeap(MinIterComparator(&cfd_->internal_comparator()));
       immutable_min_heap_.swap(tmp);
     }
-    for (auto* m : imm_iters_) {
+    for (size_t i = 0; i < imm_iters_.size(); i++) {
+      auto* m = imm_iters_[i];
       seek_to_first ? m->SeekToFirst() : m->Seek(internal_key);
       if (!m->status().ok()) {
         immutable_status_ = m->status();
@@ -231,6 +255,9 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
     const VersionStorageInfo* vstorage = sv_->current->storage_info();
     const std::vector<FileMetaData*>& l0 = vstorage->LevelFiles(0);
     for (uint32_t i = 0; i < l0.size(); ++i) {
+      if (!l0_iters_[i]) {
+        continue;
+      }
       if (seek_to_first) {
         l0_iters_[i]->SeekToFirst();
       } else {
@@ -238,6 +265,11 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         // won't go over this file.
         if (user_comparator_->Compare(user_key,
               l0[i]->largest.user_key()) > 0) {
+          if (read_options_.iterate_upper_bound != nullptr) {
+            has_iter_trimmed_for_upper_bound_ = true;
+            delete l0_iters_[i];
+            l0_iters_[i] = nullptr;
+          }
           continue;
         }
         l0_iters_[i]->Seek(internal_key);
@@ -246,7 +278,13 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
       if (!l0_iters_[i]->status().ok()) {
         immutable_status_ = l0_iters_[i]->status();
       } else if (l0_iters_[i]->Valid()) {
-        immutable_min_heap_.push(l0_iters_[i]);
+        if (!IsOverUpperBound(l0_iters_[i]->key())) {
+          immutable_min_heap_.push(l0_iters_[i]);
+        } else {
+          has_iter_trimmed_for_upper_bound_ = true;
+          delete l0_iters_[i];
+          l0_iters_[i] = nullptr;
+        }
       }
     }
 
@@ -260,7 +298,9 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         search_right_bound = FileIndexer::kLevelMaxIndex;
         continue;
       }
-      assert(level_iters_[level - 1] != nullptr);
+      if (level_iters_[level - 1] == nullptr) {
+        continue;
+      }
       uint32_t f_idx = 0;
       const auto& indexer = vstorage->file_indexer();
       if (!seek_to_first) {
@@ -294,14 +334,10 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         if (f_idx < level_files.size()) {
           int cmp_smallest = user_comparator_->Compare(
               user_key, level_files[f_idx]->smallest.user_key());
-          int cmp_largest = -1;
-          if (cmp_smallest >= 0) {
-            cmp_smallest = user_comparator_->Compare(
-                user_key, level_files[f_idx]->smallest.user_key());
-          }
-          indexer.GetNextLevelIndex(level, f_idx,
-              cmp_smallest, cmp_largest,
-              &search_left_bound, &search_right_bound);
+          assert(user_comparator_->Compare(
+                     user_key, level_files[f_idx]->largest.user_key()) <= 0);
+          indexer.GetNextLevelIndex(level, f_idx, cmp_smallest, -1,
+                                    &search_left_bound, &search_right_bound);
         } else {
           indexer.GetNextLevelIndex(
               level, level_files.size() - 1,
@@ -318,7 +354,14 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
         if (!level_iters_[level - 1]->status().ok()) {
           immutable_status_ = level_iters_[level - 1]->status();
         } else if (level_iters_[level - 1]->Valid()) {
-          immutable_min_heap_.push(level_iters_[level - 1]);
+          if (!IsOverUpperBound(level_iters_[level - 1]->key())) {
+            immutable_min_heap_.push(level_iters_[level - 1]);
+          } else {
+            // Nothing in this level is interesting. Remove.
+            has_iter_trimmed_for_upper_bound_ = true;
+            delete level_iters_[level - 1];
+            level_iters_[level - 1] = nullptr;
+          }
         }
       }
     }
@@ -330,16 +373,20 @@ void ForwardIterator::SeekInternal(const Slice& internal_key,
       is_prev_set_ = true;
       is_prev_inclusive_ = true;
     }
+
+    TEST_SYNC_POINT_CALLBACK("ForwardIterator::SeekInternal:Immutable", this);
   } else if (current_ && current_ != mutable_iter_) {
     // current_ is one of immutable iterators, push it back to the heap
     immutable_min_heap_.push(current_);
   }
 
   UpdateCurrent();
+  TEST_SYNC_POINT_CALLBACK("ForwardIterator::SeekInternal:Return", this);
 }
 
 void ForwardIterator::Next() {
   assert(valid_);
+  bool update_prev_key = false;
 
   if (sv_ == nullptr ||
       sv_->version_number != cfd_->GetSuperVersionNumber()) {
@@ -354,14 +401,16 @@ void ForwardIterator::Next() {
   } else if (current_ != mutable_iter_) {
     // It is going to advance immutable iterator
 
-    bool update_prev_key = true;
     if (is_prev_set_ && prefix_extractor_) {
       // advance prev_key_ to current_ only if they share the same prefix
       update_prev_key =
         prefix_extractor_->Transform(prev_key_.GetKey()).compare(
           prefix_extractor_->Transform(current_->key())) == 0;
+    } else {
+      update_prev_key = true;
     }
 
+
     if (update_prev_key) {
       prev_key_.SetKey(current_->key());
       is_prev_set_ = true;
@@ -373,12 +422,21 @@ void ForwardIterator::Next() {
   if (current_ != mutable_iter_) {
     if (!current_->status().ok()) {
       immutable_status_ = current_->status();
-    } else if (current_->Valid()) {
+    } else if ((current_->Valid()) && (!IsOverUpperBound(current_->key()))) {
       immutable_min_heap_.push(current_);
+    } else {
+      if ((current_->Valid()) && (IsOverUpperBound(current_->key()))) {
+        // remove the current iterator
+        DeleteCurrentIter();
+        current_ = nullptr;
+      }
+      if (update_prev_key) {
+        mutable_iter_->Seek(prev_key_.GetKey());
+      }
     }
   }
-
   UpdateCurrent();
+  TEST_SYNC_POINT_CALLBACK("ForwardIterator::Next:Return", this);
 }
 
 Slice ForwardIterator::key() const {
@@ -410,11 +468,19 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
   }
   mutable_iter_ = sv_->mem->NewIterator(read_options_, &arena_);
   sv_->imm->AddIterators(read_options_, &imm_iters_, &arena_);
+  has_iter_trimmed_for_upper_bound_ = false;
 
   const auto* vstorage = sv_->current->storage_info();
   const auto& l0_files = vstorage->LevelFiles(0);
   l0_iters_.reserve(l0_files.size());
   for (const auto* l0 : l0_files) {
+    if ((read_options_.iterate_upper_bound != nullptr) &&
+        cfd_->internal_comparator().user_comparator()->Compare(
+            l0->smallest.user_key(), *read_options_.iterate_upper_bound) > 0) {
+      has_iter_trimmed_for_upper_bound_ = true;
+      l0_iters_.push_back(nullptr);
+      continue;
+    }
     l0_iters_.push_back(cfd_->table_cache()->NewIterator(
         read_options_, *cfd_->soptions(), cfd_->internal_comparator(), l0->fd));
   }
@@ -422,8 +488,15 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
   for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
     const auto& level_files = vstorage->LevelFiles(level);
 
-    if (level_files.empty()) {
+    if ((level_files.empty()) ||
+        ((read_options_.iterate_upper_bound != nullptr) &&
+         (user_comparator_->Compare(*read_options_.iterate_upper_bound,
+                                    level_files[0]->smallest.user_key()) <
+          0))) {
       level_iters_.push_back(nullptr);
+      if (!level_files.empty()) {
+        has_iter_trimmed_for_upper_bound_ = true;
+      }
     } else {
       level_iters_.push_back(
           new LevelIterator(cfd_, read_options_, level_files));
@@ -438,7 +511,7 @@ void ForwardIterator::ResetIncompleteIterators() {
   const auto& l0_files = sv_->current->storage_info()->LevelFiles(0);
   for (uint32_t i = 0; i < l0_iters_.size(); ++i) {
     assert(i < l0_files.size());
-    if (!l0_iters_[i]->status().IsIncomplete()) {
+    if (!l0_iters_[i] || !l0_iters_[i]->status().IsIncomplete()) {
       continue;
     }
     delete l0_iters_[i];
@@ -482,6 +555,13 @@ void ForwardIterator::UpdateCurrent() {
   if (!status_.ok()) {
     status_ = Status::OK();
   }
+
+  // Upper bound doesn't apply to the memtable iterator. We want Valid() to
+  // return false when all iterators are over iterate_upper_bound, but can't
+  // just set valid_ to false, as that would effectively disable the tailing
+  // optimization (Seek() would be called on all immutable iterators regardless
+  // of whether the target key is greater than prev_key_).
+  current_over_upper_bound_ = valid_ && IsOverUpperBound(current_->key());
 }
 
 bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
@@ -517,6 +597,71 @@ bool ForwardIterator::NeedToSeekImmutable(const Slice& target) {
   return false;
 }
 
+void ForwardIterator::DeleteCurrentIter() {
+  const VersionStorageInfo* vstorage = sv_->current->storage_info();
+  const std::vector<FileMetaData*>& l0 = vstorage->LevelFiles(0);
+  for (uint32_t i = 0; i < l0.size(); ++i) {
+    if (!l0_iters_[i]) {
+      continue;
+    }
+    if (l0_iters_[i] == current_) {
+      has_iter_trimmed_for_upper_bound_ = true;
+      delete l0_iters_[i];
+      l0_iters_[i] = nullptr;
+      return;
+    }
+  }
+
+  for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
+    if (level_iters_[level - 1] == nullptr) {
+      continue;
+    }
+    if (level_iters_[level - 1] == current_) {
+      has_iter_trimmed_for_upper_bound_ = true;
+      delete level_iters_[level - 1];
+      level_iters_[level - 1] = nullptr;
+    }
+  }
+}
+
+bool ForwardIterator::TEST_CheckDeletedIters(int* pdeleted_iters,
+                                             int* pnum_iters) {
+  bool retval = false;
+  int deleted_iters = 0;
+  int num_iters = 0;
+
+  const VersionStorageInfo* vstorage = sv_->current->storage_info();
+  const std::vector<FileMetaData*>& l0 = vstorage->LevelFiles(0);
+  for (uint32_t i = 0; i < l0.size(); ++i) {
+    if (!l0_iters_[i]) {
+      retval = true;
+      deleted_iters++;
+    } else {
+      num_iters++;
+    }
+  }
+
+  for (int32_t level = 1; level < vstorage->num_levels(); ++level) {
+    if ((level_iters_[level - 1] == nullptr) &&
+        (!vstorage->LevelFiles(level).empty())) {
+      retval = true;
+      deleted_iters++;
+    } else if (!vstorage->LevelFiles(level).empty()) {
+      num_iters++;
+    }
+  }
+  if ((!retval) && num_iters <= 1) {
+    retval = true;
+  }
+  if (pdeleted_iters) {
+    *pdeleted_iters = deleted_iters;
+  }
+  if (pnum_iters) {
+    *pnum_iters = num_iters;
+  }
+  return retval;
+}
+
 uint32_t ForwardIterator::FindFileInRange(
     const std::vector<FileMetaData*>& files, const Slice& internal_key,
     uint32_t left, uint32_t right) {

db/forward_iterator.h

@@ -70,6 +70,7 @@ class ForwardIterator : public Iterator {
   virtual Slice key() const override;
   virtual Slice value() const override;
   virtual Status status() const override;
+  bool TEST_CheckDeletedIters(int* deleted_iters, int* num_iters);
 
  private:
   void Cleanup(bool release_sv);
@@ -78,10 +79,13 @@ class ForwardIterator : public Iterator {
   void SeekInternal(const Slice& internal_key, bool seek_to_first);
   void UpdateCurrent();
   bool NeedToSeekImmutable(const Slice& internal_key);
+  void DeleteCurrentIter();
   uint32_t FindFileInRange(
     const std::vector<FileMetaData*>& files, const Slice& internal_key,
     uint32_t left, uint32_t right);
 
+  bool IsOverUpperBound(const Slice& internal_key) const;
+
   DBImpl* const db_;
   const ReadOptions read_options_;
   ColumnFamilyData* const cfd_;
@@ -95,14 +99,30 @@ class ForwardIterator : public Iterator {
   std::vector<Iterator*> l0_iters_;
   std::vector<LevelIterator*> level_iters_;
   Iterator* current_;
-  // internal iterator status
-  Status status_;
-  Status immutable_status_;
   bool valid_;
 
+  // Internal iterator status; set only by one of the unsupported methods.
+  Status status_;
+  // Status of immutable iterators, maintained here to avoid iterating over
+  // all of them in status().
+  Status immutable_status_;
+  // Indicates that at least one of the immutable iterators pointed to a key
+  // larger than iterate_upper_bound and was therefore destroyed. Seek() may
+  // need to rebuild such iterators.
+  bool has_iter_trimmed_for_upper_bound_;
+  // Is current key larger than iterate_upper_bound? If so, makes Valid()
+  // return false.
+  bool current_over_upper_bound_;
+
+  // Left endpoint of the range of keys that immutable iterators currently
+  // cover. When Seek() is called with a key that's within that range, immutable
+  // iterators don't need to be moved; see NeedToSeekImmutable(). This key is
+  // included in the range after a Seek(), but excluded when advancing the
+  // iterator using Next().
   IterKey prev_key_;
   bool is_prev_set_;
   bool is_prev_inclusive_;
+
   Arena arena_;
 };
 

db/internal_stats.cc

@@ -13,6 +13,7 @@
 #endif
 
 #include <inttypes.h>
+#include <string>
 #include <algorithm>
 #include <vector>
 #include "db/column_family.h"
@@ -26,6 +27,7 @@ namespace rocksdb {
 namespace {
 const double kMB = 1048576.0;
 const double kGB = kMB * 1024;
+const double kMicrosInSec = 1000000.0;
 
 void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
   snprintf(
@@ -49,7 +51,7 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
       stats.bytes_read_non_output_levels + stats.bytes_read_output_level;
   int64_t bytes_new =
       stats.bytes_written - stats.bytes_read_output_level;
-  double elapsed = (stats.micros + 1) / 1000000.0;
+  double elapsed = (stats.micros + 1) / kMicrosInSec;
   std::string num_input_records = NumberToHumanString(stats.num_input_records);
   std::string num_dropped_records =
       NumberToHumanString(stats.num_dropped_records);
@@ -72,16 +74,15 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
            " "      /* Stall(cnt) */
            "%7s "   /* KeyIn */
            "%6s\n", /* KeyDrop */
-           name.c_str(), num_files, being_compacted, total_file_size / kMB,
-           score, bytes_read / kGB, stats.bytes_read_non_output_levels / kGB,
+           name.c_str(),
+           num_files, being_compacted, total_file_size / kMB, score,
+           bytes_read / kGB, stats.bytes_read_non_output_levels / kGB,
            stats.bytes_read_output_level / kGB, stats.bytes_written / kGB,
-           bytes_new / kGB, stats.bytes_moved / kGB,
-           w_amp, bytes_read / kMB / elapsed,
-           stats.bytes_written / kMB / elapsed, stats.micros / 1000000.0,
-           stats.count,
-           stats.count == 0 ? 0 : stats.micros / 1000000.0 / stats.count,
-           stalls,
-           num_input_records.c_str(), num_dropped_records.c_str());
+           bytes_new / kGB, stats.bytes_moved / kGB, w_amp,
+           bytes_read / kMB / elapsed, stats.bytes_written / kMB / elapsed,
+           stats.micros / kMicrosInSec, stats.count,
+           stats.count == 0 ? 0 : stats.micros / kMicrosInSec / stats.count,
+           stalls, num_input_records.c_str(), num_dropped_records.c_str());
 }
 }
 
@@ -101,7 +102,9 @@ static const std::string compaction_pending = "compaction-pending";
 static const std::string background_errors = "background-errors";
 static const std::string cur_size_active_mem_table =
                           "cur-size-active-mem-table";
-static const std::string cur_size_all_mem_tables = "cur-size-all-mem-tables";
+static const std::string cur_size_unflushed_mem_tables =
+    "cur-size-all-mem-tables";
+static const std::string cur_size_all_mem_tables = "size-all-mem-tables";
 static const std::string num_entries_active_mem_table =
                           "num-entries-active-mem-table";
 static const std::string num_entries_imm_mem_tables =
@@ -120,6 +123,13 @@ static const std::string oldest_snapshot_time = "oldest-snapshot-time";
 static const std::string num_live_versions = "num-live-versions";
 static const std::string estimate_live_data_size = "estimate-live-data-size";
 static const std::string base_level = "base-level";
+static const std::string total_sst_files_size = "total-sst-files-size";
+static const std::string estimate_pending_comp_bytes =
+    "estimate-pending-compaction-bytes";
+static const std::string aggregated_table_properties =
+    "aggregated-table-properties";
+static const std::string aggregated_table_properties_at_level =
+    aggregated_table_properties + "-at-level";
 
 const std::string DB::Properties::kNumFilesAtLevelPrefix =
                       rocksdb_prefix + num_files_at_level_prefix;
@@ -138,7 +148,9 @@ const std::string DB::Properties::kBackgroundErrors =
 const std::string DB::Properties::kCurSizeActiveMemTable =
                       rocksdb_prefix + cur_size_active_mem_table;
 const std::string DB::Properties::kCurSizeAllMemTables =
-                      rocksdb_prefix + cur_size_all_mem_tables;
+    rocksdb_prefix + cur_size_unflushed_mem_tables;
+const std::string DB::Properties::kSizeAllMemTables =
+    rocksdb_prefix + cur_size_all_mem_tables;
 const std::string DB::Properties::kNumEntriesActiveMemTable =
                       rocksdb_prefix + num_entries_active_mem_table;
 const std::string DB::Properties::kNumEntriesImmMemTables =
@@ -161,6 +173,14 @@ const std::string DB::Properties::kNumLiveVersions =
                       rocksdb_prefix + num_live_versions;
 const std::string DB::Properties::kEstimateLiveDataSize =
                       rocksdb_prefix + estimate_live_data_size;
+const std::string DB::Properties::kTotalSstFilesSize =
+                      rocksdb_prefix + total_sst_files_size;
+const std::string DB::Properties::kEstimatePendingCompactionBytes =
+    rocksdb_prefix + estimate_pending_comp_bytes;
+const std::string DB::Properties::kAggregatedTableProperties =
+    rocksdb_prefix + aggregated_table_properties;
+const std::string DB::Properties::kAggregatedTablePropertiesAtLevel =
+    rocksdb_prefix + aggregated_table_properties_at_level;
 
 DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
                                bool* need_out_of_mutex) {
@@ -187,6 +207,10 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
     return kDBStats;
   } else if (in == sstables) {
     return kSsTables;
+  } else if (in == aggregated_table_properties) {
+    return kAggregatedTableProperties;
+  } else if (in.starts_with(aggregated_table_properties_at_level)) {
+    return kAggregatedTablePropertiesAtLevel;
   }
 
   *is_int_property = true;
@@ -202,8 +226,10 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
     return kBackgroundErrors;
   } else if (in == cur_size_active_mem_table) {
     return kCurSizeActiveMemTable;
-  } else if (in == cur_size_all_mem_tables) {
+  } else if (in == cur_size_unflushed_mem_tables) {
     return kCurSizeAllMemTables;
+  } else if (in == cur_size_all_mem_tables) {
+    return kSizeAllMemTables;
   } else if (in == num_entries_active_mem_table) {
     return kNumEntriesInMutableMemtable;
   } else if (in == num_entries_imm_mem_tables) {
@@ -230,6 +256,10 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
     return kEstimateLiveDataSize;
   } else if (in == base_level) {
     return kBaseLevel;
+  } else if (in == total_sst_files_size) {
+    return kTotalSstFilesSize;
+  } else if (in == estimate_pending_comp_bytes) {
+    return kEstimatePendingCompactionBytes;
   }
   return kUnknown;
 }
@@ -311,6 +341,32 @@ bool InternalStats::GetStringProperty(DBPropertyType property_type,
     case kSsTables:
       *value = current->DebugString();
       return true;
+    case kAggregatedTableProperties: {
+      std::shared_ptr<const TableProperties> tp;
+      auto s = cfd_->current()->GetAggregatedTableProperties(&tp);
+      if (!s.ok()) {
+        return false;
+      }
+      *value = tp->ToString();
+      return true;
+    }
+    case kAggregatedTablePropertiesAtLevel: {
+      in.remove_prefix(
+          DB::Properties::kAggregatedTablePropertiesAtLevel.length());
+      uint64_t level;
+      bool ok = ConsumeDecimalNumber(&in, &level) && in.empty();
+      if (!ok || static_cast<int>(level) >= number_levels_) {
+        return false;
+      }
+      std::shared_ptr<const TableProperties> tp;
+      auto s = cfd_->current()->GetAggregatedTableProperties(
+          &tp, static_cast<int>(level));
+      if (!s.ok()) {
+        return false;
+      }
+      *value = tp->ToString();
+      return true;
+    }
     default:
       return false;
   }
@@ -333,7 +389,7 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
       *value = (cfd_->imm()->IsFlushPending() ? 1 : 0);
       return true;
     case kCompactionPending:
-      // 1 if the system already determines at least one compacdtion is needed.
+      // 1 if the system already determines at least one compaction is needed.
       // 0 otherwise,
       *value = (cfd_->compaction_picker()->NeedsCompaction(vstorage) ? 1 : 0);
       return true;
@@ -347,6 +403,10 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
       return true;
     case kCurSizeAllMemTables:
       // Current size of the active memtable + immutable memtables
+      *value = cfd_->mem()->ApproximateMemoryUsage() +
+               cfd_->imm()->ApproximateUnflushedMemTablesMemoryUsage();
+      return true;
+    case kSizeAllMemTables:
       *value = cfd_->mem()->ApproximateMemoryUsage() +
                cfd_->imm()->ApproximateMemoryUsage();
       return true;
@@ -385,14 +445,18 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
     case kNumLiveVersions:
       *value = cfd_->GetNumLiveVersions();
       return true;
-#ifndef ROCKSDB_LITE
     case kIsFileDeletionEnabled:
       *value = db->IsFileDeletionsEnabled();
       return true;
-#endif
     case kBaseLevel:
       *value = vstorage->base_level();
       return true;
+    case kTotalSstFilesSize:
+      *value = cfd_->GetTotalSstFilesSize();
+      return true;
+    case kEstimatePendingCompactionBytes:
+      *value = vstorage->estimated_compaction_needed_bytes();
+      return true;
     default:
       return false;
   }
@@ -401,7 +465,7 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
 void InternalStats::DumpDBStats(std::string* value) {
   char buf[1000];
   // DB-level stats, only available from default column family
-  double seconds_up = (env_->NowMicros() - started_at_ + 1) / 1000000.0;
+  double seconds_up = (env_->NowMicros() - started_at_ + 1) / kMicrosInSec;
   double interval_seconds_up = seconds_up - db_stats_snapshot_.seconds_up;
   snprintf(buf, sizeof(buf),
            "\n** DB Stats **\nUptime(secs): %.1f total, %.1f interval\n",
@@ -460,11 +524,9 @@ void InternalStats::DumpDBStats(std::string* value) {
   snprintf(buf, sizeof(buf),
            "Cumulative compaction: %.2f GB write, %.2f MB/s write, "
            "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
-           compact_bytes_write / kGB,
-           compact_bytes_write / kMB / seconds_up,
-           compact_bytes_read / kGB,
-           compact_bytes_read / kMB / seconds_up,
-           compact_micros / 1000000.0);
+           compact_bytes_write / kGB, compact_bytes_write / kMB / seconds_up,
+           compact_bytes_read / kGB, compact_bytes_read / kMB / seconds_up,
+           compact_micros / kMicrosInSec);
   value->append(buf);
   // Stall
   AppendHumanMicros(write_stall_micros, human_micros, kHumanMicrosLen, true);
@@ -518,16 +580,15 @@ void InternalStats::DumpDBStats(std::string* value) {
   uint64_t interval_compact_micros =
       compact_micros - db_stats_snapshot_.compact_micros;
 
-  snprintf(buf, sizeof(buf),
-           "Interval compaction: %.2f GB write, %.2f MB/s write, "
-           "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
-           interval_compact_bytes_write / kGB,
-           interval_compact_bytes_write / kMB /
-               std::max(interval_seconds_up, 0.001),
-           interval_compact_bytes_read / kGB,
-           interval_compact_bytes_read / kMB /
-               std::max(interval_seconds_up, 0.001),
-           interval_compact_micros / 1000000.0);
+  snprintf(
+      buf, sizeof(buf),
+      "Interval compaction: %.2f GB write, %.2f MB/s write, "
+      "%.2f GB read, %.2f MB/s read, %.1f seconds\n",
+      interval_compact_bytes_write / kGB,
+      interval_compact_bytes_write / kMB / std::max(interval_seconds_up, 0.001),
+      interval_compact_bytes_read / kGB,
+      interval_compact_bytes_read / kMB / std::max(interval_seconds_up, 0.001),
+      interval_compact_micros / kMicrosInSec);
   value->append(buf);
 
   // Stall
@@ -542,6 +603,16 @@ void InternalStats::DumpDBStats(std::string* value) {
                10000.0 / std::max(interval_seconds_up, 0.001));
   value->append(buf);
 
+  for (int level = 0; level < number_levels_; level++) {
+    if (!file_read_latency_[level].Empty()) {
+      char buf2[5000];
+      snprintf(buf2, sizeof(buf2),
+               "** Level %d read latency histogram (micros):\n%s\n", level,
+               file_read_latency_[level].ToString().c_str());
+      value->append(buf2);
+    }
+  }
+
   db_stats_snapshot_.seconds_up = seconds_up;
   db_stats_snapshot_.ingest_bytes = user_bytes_written;
   db_stats_snapshot_.write_other = write_other;

db/internal_stats.h

@@ -41,8 +41,10 @@ enum DBPropertyType : uint32_t {
   kCompactionPending,      // Return 1 if a compaction is pending. Otherwise 0.
   kBackgroundErrors,       // Return accumulated background errors encountered.
   kCurSizeActiveMemTable,  // Return current size of the active memtable
-  kCurSizeAllMemTables,    // Return current size of all (active + immutable)
-                           // memtables
+  kCurSizeAllMemTables,    // Return current size of unflushed
+                           // (active + immutable) memtables
+  kSizeAllMemTables,       // Return current size of all (active + immutable
+                           // + pinned) memtables
   kNumEntriesInMutableMemtable,    // Return number of deletes in the mutable
                                    // memtable.
   kNumEntriesInImmutableMemtable,  // Return sum of number of entries in all
@@ -58,8 +60,15 @@ enum DBPropertyType : uint32_t {
   kNumSnapshots,                  // Number of snapshots in the system
   kOldestSnapshotTime,            // Unix timestamp of the first snapshot
   kNumLiveVersions,
-  kEstimateLiveDataSize,          // Estimated amount of live data in bytes
-  kBaseLevel,  // The level that L0 data is compacted to
+  kEstimateLiveDataSize,            // Estimated amount of live data in bytes
+  kTotalSstFilesSize,               // Total size of all sst files.
+  kBaseLevel,                       // The level that L0 data is compacted to
+  kEstimatePendingCompactionBytes,  // Estimated bytes to compaction
+  kAggregatedTableProperties,  // Return a string that contains the aggregated
+                               // table properties.
+  kAggregatedTablePropertiesAtLevel,  // Return a string that contains the
+                                      // aggregated
+  // table properties at the specified level.
 };
 
 extern DBPropertyType GetPropertyType(const Slice& property,
@@ -98,6 +107,7 @@ class InternalStats {
         comp_stats_(num_levels),
         stall_leveln_slowdown_count_hard_(num_levels),
         stall_leveln_slowdown_count_soft_(num_levels),
+        file_read_latency_(num_levels),
         bg_error_count_(0),
         number_levels_(num_levels),
         env_(env),
@@ -238,6 +248,10 @@ class InternalStats {
     db_stats_[type] += value;
   }
 
+  HistogramImpl* GetFileReadHist(int level) {
+    return &file_read_latency_[level];
+  }
+
   uint64_t GetBackgroundErrorCount() const { return bg_error_count_; }
 
   uint64_t BumpAndGetBackgroundErrorCount() { return ++bg_error_count_; }
@@ -265,6 +279,7 @@ class InternalStats {
   // These count the number of microseconds for which MakeRoomForWrite stalls.
   std::vector<uint64_t> stall_leveln_slowdown_count_hard_;
   std::vector<uint64_t> stall_leveln_slowdown_count_soft_;
+  std::vector<HistogramImpl> file_read_latency_;
 
   // Used to compute per-interval statistics
   struct CFStatsSnapshot {
@@ -389,6 +404,8 @@ class InternalStats {
 
   void AddDBStats(InternalDBStatsType type, uint64_t value) {}
 
+  HistogramImpl* GetFileReadHist(int level) { return nullptr; }
+
   uint64_t GetBackgroundErrorCount() const { return 0; }
 
   uint64_t BumpAndGetBackgroundErrorCount() { return 0; }

db/listener_test.cc

@@ -2,8 +2,9 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-#include "db/dbformat.h"
+
 #include "db/db_impl.h"
+#include "db/dbformat.h"
 #include "db/filename.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
@@ -12,25 +13,25 @@
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/filter_policy.h"
+#include "rocksdb/options.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/table.h"
-#include "rocksdb/options.h"
 #include "rocksdb/table_properties.h"
 #include "table/block_based_table_factory.h"
 #include "table/plain_table_factory.h"
 #include "util/hash.h"
 #include "util/hash_linklist_rep.h"
-#include "utilities/merge_operators.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/rate_limiter.h"
 #include "util/statistics.h"
 #include "util/string_util.h"
-#include "util/testharness.h"
 #include "util/sync_point.h"
+#include "util/testharness.h"
 #include "util/testutil.h"
+#include "utilities/merge_operators.h"
 
 #ifndef ROCKSDB_LITE
 
@@ -134,7 +135,7 @@ class EventListenerTest : public testing::Test {
     return db_->Put(wo, handles_[cf], k, v);
   }
 
-  Status Flush(int cf = 0) {
+  Status Flush(size_t cf = 0) {
     FlushOptions opt = FlushOptions();
     opt.wait = true;
     if (cf == 0) {
@@ -144,6 +145,8 @@ class EventListenerTest : public testing::Test {
     }
   }
 
+  const size_t k110KB = 110 << 10;
+
   DB* db_;
   std::string dbname_;
   std::vector<ColumnFamilyHandle*> handles_;
@@ -198,7 +201,7 @@ TEST_F(EventListenerTest, OnSingleDBCompactionTest) {
   ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
   ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
   for (size_t i = 1; i < 8; ++i) {
-    ASSERT_OK(Flush(static_cast<int>(i)));
+    ASSERT_OK(Flush(i));
     const Slice kStart = "a";
     const Slice kEnd = "z";
     ASSERT_OK(dbfull()->CompactRange(CompactRangeOptions(), handles_[i],
@@ -287,7 +290,7 @@ class TestFlushListener : public EventListener {
 
 TEST_F(EventListenerTest, OnSingleDBFlushTest) {
   Options options;
-  options.write_buffer_size = 100000;
+  options.write_buffer_size = k110KB;
 #if ROCKSDB_USING_THREAD_STATUS
   options.enable_thread_tracking = true;
 #endif  // ROCKSDB_USING_THREAD_STATUS
@@ -306,7 +309,7 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
   ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
   ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
   for (size_t i = 1; i < 8; ++i) {
-    ASSERT_OK(Flush(static_cast<int>(i)));
+    ASSERT_OK(Flush(i));
     dbfull()->TEST_WaitForFlushMemTable();
     ASSERT_EQ(listener->flushed_dbs_.size(), i);
     ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
@@ -321,7 +324,7 @@ TEST_F(EventListenerTest, OnSingleDBFlushTest) {
 
 TEST_F(EventListenerTest, MultiCF) {
   Options options;
-  options.write_buffer_size = 100000;
+  options.write_buffer_size = k110KB;
 #if ROCKSDB_USING_THREAD_STATUS
   options.enable_thread_tracking = true;
 #endif  // ROCKSDB_USING_THREAD_STATUS
@@ -340,7 +343,7 @@ TEST_F(EventListenerTest, MultiCF) {
   ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
   ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
   for (size_t i = 1; i < 8; ++i) {
-    ASSERT_OK(Flush(static_cast<int>(i)));
+    ASSERT_OK(Flush(i));
     ASSERT_EQ(listener->flushed_dbs_.size(), i);
     ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
   }

db/log_writer.h

@@ -6,10 +6,12 @@
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-
 #pragma once
-#include <memory>
+
 #include <stdint.h>
+
+#include <memory>
+
 #include "db/log_format.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"

db/memtable.cc

@@ -112,7 +112,7 @@ bool MemTable::ShouldFlushNow() const {
   // In a lot of times, we cannot allocate arena blocks that exactly matches the
   // buffer size. Thus we have to decide if we should over-allocate or
   // under-allocate.
-  // This constant avariable can be interpreted as: if we still have more than
+  // This constant variable can be interpreted as: if we still have more than
   // "kAllowOverAllocationRatio * kArenaBlockSize" space left, we'd try to over
   // allocate one more block.
   const double kAllowOverAllocationRatio = 0.6;
@@ -404,8 +404,8 @@ static bool SaveValue(void* arg, const char* entry) {
   // all entries with overly large sequence numbers.
   uint32_t key_length;
   const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
-  if (s->mem->GetInternalKeyComparator().user_comparator()->Compare(
-          Slice(key_ptr, key_length - 8), s->key->user_key()) == 0) {
+  if (s->mem->GetInternalKeyComparator().user_comparator()->Equal(
+          Slice(key_ptr, key_length - 8), s->key->user_key())) {
     // Correct user key
     const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
     ValueType type;
@@ -534,7 +534,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
 
   // No change to value, since we have not yet found a Put/Delete
   if (!found_final_value && merge_in_progress) {
-    *s = Status::MergeInProgress("");
+    *s = Status::MergeInProgress();
   }
   PERF_COUNTER_ADD(get_from_memtable_count, 1);
   return found_final_value;
@@ -563,8 +563,8 @@ void MemTable::Update(SequenceNumber seq,
     const char* entry = iter->key();
     uint32_t key_length = 0;
     const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
-    if (comparator_.comparator.user_comparator()->Compare(
-        Slice(key_ptr, key_length - 8), lkey.user_key()) == 0) {
+    if (comparator_.comparator.user_comparator()->Equal(
+            Slice(key_ptr, key_length - 8), lkey.user_key())) {
       // Correct user key
       const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
       ValueType type;
@@ -624,8 +624,8 @@ bool MemTable::UpdateCallback(SequenceNumber seq,
     const char* entry = iter->key();
     uint32_t key_length = 0;
     const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
-    if (comparator_.comparator.user_comparator()->Compare(
-        Slice(key_ptr, key_length - 8), lkey.user_key()) == 0) {
+    if (comparator_.comparator.user_comparator()->Equal(
+            Slice(key_ptr, key_length - 8), lkey.user_key())) {
       // Correct user key
       const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
       ValueType type;
@@ -695,8 +695,8 @@ size_t MemTable::CountSuccessiveMergeEntries(const LookupKey& key) {
     const char* entry = iter->key();
     uint32_t key_length = 0;
     const char* iter_key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
-    if (comparator_.comparator.user_comparator()->Compare(
-            Slice(iter_key_ptr, key_length - 8), key.user_key()) != 0) {
+    if (!comparator_.comparator.user_comparator()->Equal(
+            Slice(iter_key_ptr, key_length - 8), key.user_key())) {
       break;
     }
 

db/memtable_list.cc

@@ -27,9 +27,26 @@ class InternalKeyComparator;
 class Mutex;
 class VersionSet;
 
-MemTableListVersion::MemTableListVersion(MemTableListVersion* old)
+void MemTableListVersion::AddMemTable(MemTable* m) {
+  memlist_.push_front(m);
+  *parent_memtable_list_memory_usage_ += m->ApproximateMemoryUsage();
+}
+
+void MemTableListVersion::UnrefMemTable(autovector<MemTable*>* to_delete,
+                                        MemTable* m) {
+  if (m->Unref()) {
+    to_delete->push_back(m);
+    assert(*parent_memtable_list_memory_usage_ >= m->ApproximateMemoryUsage());
+    *parent_memtable_list_memory_usage_ -= m->ApproximateMemoryUsage();
+  } else {
+  }
+}
+
+MemTableListVersion::MemTableListVersion(
+    size_t* parent_memtable_list_memory_usage, MemTableListVersion* old)
     : max_write_buffer_number_to_maintain_(
-          old->max_write_buffer_number_to_maintain_) {
+          old->max_write_buffer_number_to_maintain_),
+      parent_memtable_list_memory_usage_(parent_memtable_list_memory_usage) {
   if (old != nullptr) {
     memlist_ = old->memlist_;
     for (auto& m : memlist_) {
@@ -44,12 +61,14 @@ MemTableListVersion::MemTableListVersion(MemTableListVersion* old)
 }
 
 MemTableListVersion::MemTableListVersion(
+    size_t* parent_memtable_list_memory_usage,
     int max_write_buffer_number_to_maintain)
-    : max_write_buffer_number_to_maintain_(
-          max_write_buffer_number_to_maintain) {}
+    : max_write_buffer_number_to_maintain_(max_write_buffer_number_to_maintain),
+      parent_memtable_list_memory_usage_(parent_memtable_list_memory_usage) {}
 
 void MemTableListVersion::Ref() { ++refs_; }
 
+// called by superversion::clean()
 void MemTableListVersion::Unref(autovector<MemTable*>* to_delete) {
   assert(refs_ >= 1);
   --refs_;
@@ -58,16 +77,10 @@ void MemTableListVersion::Unref(autovector<MemTable*>* to_delete) {
     // that refs_ will not be zero
     assert(to_delete != nullptr);
     for (const auto& m : memlist_) {
-      MemTable* x = m->Unref();
-      if (x != nullptr) {
-        to_delete->push_back(x);
-      }
+      UnrefMemTable(to_delete, m);
     }
     for (const auto& m : memlist_history_) {
-      MemTable* x = m->Unref();
-      if (x != nullptr) {
-        to_delete->push_back(x);
-      }
+      UnrefMemTable(to_delete, m);
     }
     delete this;
   }
@@ -180,7 +193,7 @@ SequenceNumber MemTableListVersion::GetEarliestSequenceNumber(
 // caller is responsible for referencing m
 void MemTableListVersion::Add(MemTable* m, autovector<MemTable*>* to_delete) {
   assert(refs_ == 1);  // only when refs_ == 1 is MemTableListVersion mutable
-  memlist_.push_front(m);
+  AddMemTable(m);
 
   TrimHistory(to_delete);
 }
@@ -195,9 +208,7 @@ void MemTableListVersion::Remove(MemTable* m,
     memlist_history_.push_front(m);
     TrimHistory(to_delete);
   } else {
-    if (m->Unref()) {
-      to_delete->push_back(m);
-    }
+    UnrefMemTable(to_delete, m);
   }
 }
 
@@ -209,9 +220,7 @@ void MemTableListVersion::TrimHistory(autovector<MemTable*>* to_delete) {
     MemTable* x = memlist_history_.back();
     memlist_history_.pop_back();
 
-    if (x->Unref()) {
-      to_delete->push_back(x);
-    }
+    UnrefMemTable(to_delete, x);
   }
 }
 
@@ -315,7 +324,7 @@ Status MemTableList::InstallMemtableFlushResults(
 
     // All the later memtables that have the same filenum
     // are part of the same batch. They can be committed now.
-    uint64_t mem_id = 1;  // how many memtables has been flushed.
+    uint64_t mem_id = 1;  // how many memtables have been flushed.
     do {
       if (s.ok()) { // commit new state
         LogToBuffer(log_buffer, "[%s] Level-0 commit table #%" PRIu64
@@ -324,7 +333,7 @@ Status MemTableList::InstallMemtableFlushResults(
         assert(m->file_number_ > 0);
         current_->Remove(m, to_delete);
       } else {
-        //commit failed. setup state so that we can flush again.
+        // commit failed. setup state so that we can flush again.
         LogToBuffer(log_buffer, "Level-0 commit table #%" PRIu64
                                 ": memtable #%" PRIu64 " failed",
                     m->file_number_, mem_id);
@@ -361,7 +370,7 @@ void MemTableList::Add(MemTable* m, autovector<MemTable*>* to_delete) {
 }
 
 // Returns an estimate of the number of bytes of data in use.
-size_t MemTableList::ApproximateMemoryUsage() {
+size_t MemTableList::ApproximateUnflushedMemTablesMemoryUsage() {
   size_t total_size = 0;
   for (auto& memtable : current_->memlist_) {
     total_size += memtable->ApproximateMemoryUsage();
@@ -369,13 +378,15 @@ size_t MemTableList::ApproximateMemoryUsage() {
   return total_size;
 }
 
+size_t MemTableList::ApproximateMemoryUsage() { return current_memory_usage_; }
+
 void MemTableList::InstallNewVersion() {
   if (current_->refs_ == 1) {
     // we're the only one using the version, just keep using it
   } else {
     // somebody else holds the current version, we need to create new one
     MemTableListVersion* version = current_;
-    current_ = new MemTableListVersion(current_);
+    current_ = new MemTableListVersion(&current_memory_usage_, current_);
     current_->Ref();
     version->Unref();
   }

db/memtable_list.h

@@ -10,14 +10,11 @@
 #include <vector>
 #include <set>
 #include <deque>
-#include "rocksdb/db.h"
-#include "rocksdb/options.h"
-#include "rocksdb/iterator.h"
 
 #include "db/dbformat.h"
 #include "db/filename.h"
-#include "db/skiplist.h"
 #include "db/memtable.h"
+#include "db/skiplist.h"
 #include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
@@ -41,8 +38,10 @@ class MergeIteratorBuilder;
 // (such as holding the db mutex or being on the write thread).
 class MemTableListVersion {
  public:
-  explicit MemTableListVersion(MemTableListVersion* old = nullptr);
-  explicit MemTableListVersion(int max_write_buffer_number_to_maintain);
+  explicit MemTableListVersion(size_t* parent_memtable_list_memory_usage,
+                               MemTableListVersion* old = nullptr);
+  explicit MemTableListVersion(size_t* parent_memtable_list_memory_usage,
+                               int max_write_buffer_number_to_maintain);
 
   void Ref();
   void Unref(autovector<MemTable*>* to_delete = nullptr);
@@ -104,6 +103,10 @@ class MemTableListVersion {
                    std::string* value, Status* s, MergeContext* merge_context,
                    SequenceNumber* seq);
 
+  void AddMemTable(MemTable* m);
+
+  void UnrefMemTable(autovector<MemTable*>* to_delete, MemTable* m);
+
   friend class MemTableList;
 
   // Immutable MemTables that have not yet been flushed.
@@ -118,6 +121,8 @@ class MemTableListVersion {
   const int max_write_buffer_number_to_maintain_;
 
   int refs_ = 0;
+
+  size_t* parent_memtable_list_memory_usage_;
 };
 
 // This class stores references to all the immutable memtables.
@@ -138,11 +143,13 @@ class MemTableList {
                         int max_write_buffer_number_to_maintain)
       : imm_flush_needed(false),
         min_write_buffer_number_to_merge_(min_write_buffer_number_to_merge),
-        current_(new MemTableListVersion(max_write_buffer_number_to_maintain)),
+        current_(new MemTableListVersion(&current_memory_usage_,
+                                         max_write_buffer_number_to_maintain)),
         num_flush_not_started_(0),
         commit_in_progress_(false),
         flush_requested_(false) {
     current_->Ref();
+    current_memory_usage_ = 0;
   }
 
   // Should not delete MemTableList without making sure MemTableList::current()
@@ -190,6 +197,10 @@ class MemTableList {
   // Returns an estimate of the number of bytes of data in use.
   size_t ApproximateMemoryUsage();
 
+  // Returns an estimate of the number of bytes of data used by
+  // the unflushed mem-tables.
+  size_t ApproximateUnflushedMemTablesMemoryUsage();
+
   // Request a flush of all existing memtables to storage.  This will
   // cause future calls to IsFlushPending() to return true if this list is
   // non-empty (regardless of the min_write_buffer_number_to_merge
@@ -201,6 +212,8 @@ class MemTableList {
   // MemTableList(const MemTableList&);
   // void operator=(const MemTableList&);
 
+  size_t* current_memory_usage() { return &current_memory_usage_; }
+
  private:
   // DB mutex held
   void InstallNewVersion();
@@ -218,6 +231,8 @@ class MemTableList {
   // Requested a flush of all memtables to storage
   bool flush_requested_;
 
+  // The current memory usage.
+  size_t current_memory_usage_;
 };
 
 }  // namespace rocksdb

db/memtablerep_bench.cc

@@ -312,9 +312,10 @@ class ReadBenchmarkThread : public BenchmarkThread {
     assert(callback_args != nullptr);
     uint32_t key_length;
     const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
-    if ((callback_args->comparator)->user_comparator()->Compare(
-            Slice(key_ptr, key_length - 8), callback_args->key->user_key()) ==
-        0) {
+    if ((callback_args->comparator)
+            ->user_comparator()
+            ->Equal(Slice(key_ptr, key_length - 8),
+                    callback_args->key->user_key())) {
       callback_args->found = true;
     }
     return false;

db/merge_helper.cc

@@ -2,18 +2,18 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//
+
+#include "db/merge_helper.h"
 
 #include <stdio.h>
 #include <string>
 
-#include "merge_helper.h"
 #include "db/dbformat.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/db.h"
 #include "rocksdb/merge_operator.h"
-#include "util/statistics.h"
 #include "util/perf_context_imp.h"
+#include "util/statistics.h"
 #include "util/stop_watch.h"
 
 namespace rocksdb {
@@ -58,9 +58,9 @@ Status MergeHelper::TimedFullMerge(const Slice& key, const Slice* value,
 //       keys_ stores the list of keys encountered while merging.
 //       operands_ stores the list of merge operands encountered while merging.
 //       keys_[i] corresponds to operands_[i] for each i.
-void MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
-                             const bool at_bottom, Statistics* stats,
-                             Env* env_) {
+Status MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
+                               const bool at_bottom, Statistics* stats,
+                               Env* env_) {
   // Get a copy of the internal key, before it's invalidated by iter->Next()
   // Also maintain the list of merge operands seen.
   assert(HasOperator());
@@ -70,8 +70,6 @@ void MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
   operands_.push_front(iter->value().ToString());
   assert(user_merge_operator_);
 
-  success_ = false;   // Will become true if we hit Put/Delete or bottom
-
   // We need to parse the internal key again as the parsed key is
   // backed by the internal key!
   // Assume no internal key corruption as it has been successfully parsed
@@ -80,6 +78,7 @@ void MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
   ParsedInternalKey orig_ikey;
   ParseInternalKey(keys_.back(), &orig_ikey);
 
+  Status s;
   bool hit_the_next_user_key = false;
   for (iter->Next(); iter->Valid(); iter->Next()) {
     ParsedInternalKey ikey;
@@ -92,15 +91,11 @@ void MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
         assert(!"corrupted internal key is not expected");
       }
       break;
-    }
-
-    if (user_comparator_->Compare(ikey.user_key, orig_ikey.user_key) != 0) {
+    } else if (!user_comparator_->Equal(ikey.user_key, orig_ikey.user_key)) {
       // hit a different user key, stop right here
       hit_the_next_user_key = true;
       break;
-    }
-
-    if (stop_before && ikey.sequence <= stop_before) {
+    } else if (stop_before && ikey.sequence <= stop_before) {
       // hit an entry that's visible by the previous snapshot, can't touch that
       break;
     }
@@ -114,28 +109,33 @@ void MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
       //   => store result in operands_.back() (and update keys_.back())
       //   => change the entry type to kTypeValue for keys_.back()
       // We are done! Success!
+      //
+      // TODO(noetzli) If the merge operator returns false, we are currently
+      // (almost) silently dropping the put/delete. That's probably not what we
+      // want.
       const Slice val = iter->value();
       const Slice* val_ptr = (kTypeValue == ikey.type) ? &val : nullptr;
       std::string merge_result;
-      Status s =
-          TimedFullMerge(ikey.user_key, val_ptr, operands_,
+      s = TimedFullMerge(ikey.user_key, val_ptr, operands_,
                          user_merge_operator_, stats, env_, logger_,
                          &merge_result);
 
       // We store the result in keys_.back() and operands_.back()
       // if nothing went wrong (i.e.: no operand corruption on disk)
       if (s.ok()) {
-        std::string& original_key =
-            keys_.back();  // The original key encountered
+        // The original key encountered
+        std::string original_key = std::move(keys_.back());
         orig_ikey.type = kTypeValue;
         UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type);
-        operands_.back() = std::move(merge_result);
-        success_ = true;
+        keys_.clear();
+        operands_.clear();
+        keys_.emplace_front(std::move(original_key));
+        operands_.emplace_front(std::move(merge_result));
       }
 
       // move iter to the next entry
       iter->Next();
-      return;
+      return s;
     } else {
       // hit a merge
       //   => merge the operand into the front of the operands_ list
@@ -173,28 +173,28 @@ void MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
     assert(operands_.size() >= 1);
     assert(operands_.size() == keys_.size());
     std::string merge_result;
-    {
-      StopWatchNano timer(env_, stats != nullptr);
-      PERF_TIMER_GUARD(merge_operator_time_nanos);
-      success_ = user_merge_operator_->FullMerge(
-          orig_ikey.user_key, nullptr, operands_, &merge_result, logger_);
-      RecordTick(stats, MERGE_OPERATION_TOTAL_TIME,
-                 env_ != nullptr ? timer.ElapsedNanos() : 0);
-    }
-    if (success_) {
-      std::string& original_key = keys_.back();  // The original key encountered
+    s = TimedFullMerge(orig_ikey.user_key, nullptr, operands_,
+                       user_merge_operator_, stats, env_, logger_,
+                       &merge_result);
+    if (s.ok()) {
+      // The original key encountered
+      std::string original_key = std::move(keys_.back());
       orig_ikey.type = kTypeValue;
       UpdateInternalKey(&original_key, orig_ikey.sequence, orig_ikey.type);
-      operands_.back() = std::move(merge_result);
-    } else {
-      RecordTick(stats, NUMBER_MERGE_FAILURES);
-      // Do nothing if not success_. Leave keys() and operands() as they are.
+      keys_.clear();
+      operands_.clear();
+      keys_.emplace_front(std::move(original_key));
+      operands_.emplace_front(std::move(merge_result));
     }
   } else {
     // We haven't seen the beginning of the key nor a Put/Delete.
     // Attempt to use the user's associative merge function to
     // merge the stacked merge operands into a single operand.
-
+    //
+    // TODO(noetzli) The docblock of MergeUntil suggests that a successful
+    // partial merge returns Status::OK(). Should we change the status code
+    // after a successful partial merge?
+    s = Status::MergeInProgress();
     if (operands_.size() >= 2 &&
         operands_.size() >= min_partial_merge_operands_) {
       bool merge_success = false;
@@ -218,6 +218,28 @@ void MergeHelper::MergeUntil(Iterator* iter, const SequenceNumber stop_before,
       }
     }
   }
+
+  return s;
+}
+
+MergeOutputIterator::MergeOutputIterator(const MergeHelper* merge_helper)
+    : merge_helper_(merge_helper) {
+  it_keys_ = merge_helper_->keys().rend();
+  it_values_ = merge_helper_->values().rend();
+}
+
+void MergeOutputIterator::SeekToFirst() {
+  const auto& keys = merge_helper_->keys();
+  const auto& values = merge_helper_->values();
+  assert(keys.size() > 0);
+  assert(keys.size() == values.size());
+  it_keys_ = keys.rbegin();
+  it_values_ = values.rbegin();
+}
+
+void MergeOutputIterator::Next() {
+  ++it_keys_;
+  ++it_values_;
 }
 
 } // namespace rocksdb

db/merge_helper.h

@@ -33,12 +33,17 @@ class MergeHelper {
         min_partial_merge_operands_(min_partial_merge_operands),
         assert_valid_internal_key_(assert_valid_internal_key),
         keys_(),
-        operands_(),
-        success_(false) {}
+        operands_() {
+    assert(user_comparator_ != nullptr);
+  }
 
   // Wrapper around MergeOperator::FullMerge() that records perf statistics.
   // Result of merge will be written to result if status returned is OK.
   // If operands is empty, the value will simply be copied to result.
+  // Returns one of the following statuses:
+  // - OK: Entries were successfully merged.
+  // - Corruption: Merge operator reported unsuccessful merge.
+  // - NotSupported: Merge operator is missing.
   static Status TimedFullMerge(const Slice& key, const Slice* value,
                                const std::deque<std::string>& operands,
                                const MergeOperator* merge_operator,
@@ -57,18 +62,23 @@ class MergeHelper {
   //                   0 means no restriction
   // at_bottom:   (IN) true if the iterator covers the bottem level, which means
   //                   we could reach the start of the history of this user key.
-  void MergeUntil(Iterator* iter, const SequenceNumber stop_before = 0,
-                  const bool at_bottom = false, Statistics* stats = nullptr,
-                  Env* env_ = nullptr);
+  // Returns one of the following statuses:
+  // - OK: Entries were successfully merged.
+  // - MergeInProgress: Put/Delete not encountered and unable to merge operands.
+  // - Corruption: Merge operator reported unsuccessful merge.
+  //
+  // REQUIRED: The first key in the input is not corrupted.
+  Status MergeUntil(Iterator* iter, const SequenceNumber stop_before = 0,
+                    const bool at_bottom = false, Statistics* stats = nullptr,
+                    Env* env_ = nullptr);
 
   // Query the merge result
   // These are valid until the next MergeUntil call
   // If the merging was successful:
-  //   - IsSuccess() will be true
-  //   - key() will have the latest sequence number of the merges.
-  //           The type will be Put or Merge. See IMPORTANT 1 note, below.
-  //   - value() will be the result of merging all the operands together
-  //   - The user should ignore keys() and values().
+  //   - keys() contains a single element with the latest sequence number of
+  //     the merges. The type will be Put or Merge. See IMPORTANT 1 note, below.
+  //   - values() contains a single element with the result of merging all the
+  //     operands together
   //
   //   IMPORTANT 1: the key type could change after the MergeUntil call.
   //        Put/Delete + Merge + ... + Merge => Put
@@ -76,7 +86,6 @@ class MergeHelper {
   //
   // If the merge operator is not associative, and if a Put/Delete is not found
   // then the merging will be unsuccessful. In this case:
-  //   - IsSuccess() will be false
   //   - keys() contains the list of internal keys seen in order of iteration.
   //   - values() contains the list of values (merges) seen in the same order.
   //              values() is parallel to keys() so that the first entry in
@@ -84,20 +93,12 @@ class MergeHelper {
   //              and so on. These lists will be the same length.
   //              All of these pairs will be merges over the same user key.
   //              See IMPORTANT 2 note below.
-  //   - The user should ignore key() and value().
   //
   //   IMPORTANT 2: The entries were traversed in order from BACK to FRONT.
   //                So keys().back() was the first key seen by iterator.
   // TODO: Re-style this comment to be like the first one
-  bool IsSuccess() const { return success_; }
-  Slice key() const { assert(success_); return Slice(keys_.back()); }
-  Slice value() const { assert(success_); return Slice(operands_.back()); }
-  const std::deque<std::string>& keys() const {
-    assert(!success_); return keys_;
-  }
-  const std::deque<std::string>& values() const {
-    assert(!success_); return operands_;
-  }
+  const std::deque<std::string>& keys() const { return keys_; }
+  const std::deque<std::string>& values() const { return operands_; }
   bool HasOperator() const { return user_merge_operator_ != nullptr; }
 
  private:
@@ -111,7 +112,27 @@ class MergeHelper {
   // valid up to the next MergeUntil call
   std::deque<std::string> keys_;    // Keeps track of the sequence of keys seen
   std::deque<std::string> operands_;  // Parallel with keys_; stores the values
-  bool success_;
+};
+
+// MergeOutputIterator can be used to iterate over the result of a merge.
+class MergeOutputIterator {
+ public:
+  // The MergeOutputIterator is bound to a MergeHelper instance.
+  explicit MergeOutputIterator(const MergeHelper* merge_helper);
+
+  // Seeks to the first record in the output.
+  void SeekToFirst();
+  // Advances to the next record in the output.
+  void Next();
+
+  Slice key() { return Slice(*it_keys_); }
+  Slice value() { return Slice(*it_values_); }
+  bool Valid() { return it_keys_ != merge_helper_->keys().rend(); }
+
+ private:
+  const MergeHelper* merge_helper_;
+  std::deque<std::string>::const_reverse_iterator it_keys_;
+  std::deque<std::string>::const_reverse_iterator it_values_;
 };
 
 } // namespace rocksdb

db/merge_helper_test.cc

@@ -21,34 +21,38 @@ class MergeHelperTest : public testing::Test {
   MergeHelperTest() = default;
   ~MergeHelperTest() = default;
 
-  void RunUInt64MergeHelper(SequenceNumber stop_before, bool at_bottom) {
+  Status RunUInt64MergeHelper(SequenceNumber stop_before, bool at_bottom) {
     InitIterator();
     merge_op_ = MergeOperators::CreateUInt64AddOperator();
     merge_helper_.reset(new MergeHelper(BytewiseComparator(), merge_op_.get(),
-          nullptr, 2U, true));
-    merge_helper_->MergeUntil(iter_.get(), stop_before, at_bottom, nullptr,
-                              Env::Default());
+                                        nullptr, 2U, false));
+    return merge_helper_->MergeUntil(iter_.get(), stop_before, at_bottom,
+                                     nullptr, Env::Default());
   }
 
-  void RunStringAppendMergeHelper(SequenceNumber stop_before, bool at_bottom) {
+  Status RunStringAppendMergeHelper(SequenceNumber stop_before,
+                                    bool at_bottom) {
     InitIterator();
     merge_op_ = MergeOperators::CreateStringAppendTESTOperator();
     merge_helper_.reset(new MergeHelper(BytewiseComparator(), merge_op_.get(),
-          nullptr, 2U, true));
-    merge_helper_->MergeUntil(iter_.get(), stop_before, at_bottom, nullptr,
-                              Env::Default());
+                                        nullptr, 2U, false));
+    return merge_helper_->MergeUntil(iter_.get(), stop_before, at_bottom,
+                                     nullptr, Env::Default());
   }
 
   std::string Key(const std::string& user_key, const SequenceNumber& seq,
       const ValueType& t) {
-    std::string ikey;
-    AppendInternalKey(&ikey, ParsedInternalKey(Slice(user_key), seq, t));
-    return ikey;
+    return InternalKey(user_key, seq, t).Encode().ToString();
   }
 
   void AddKeyVal(const std::string& user_key, const SequenceNumber& seq,
-      const ValueType& t, const std::string& val) {
-    ks_.push_back(Key(user_key, seq, t));
+                 const ValueType& t, const std::string& val,
+                 bool corrupt = false) {
+    InternalKey ikey = InternalKey(user_key, seq, t);
+    if (corrupt) {
+      test::CorruptKeyType(&ikey);
+    }
+    ks_.push_back(ikey.Encode().ToString());
     vs_.push_back(val);
   }
 
@@ -63,15 +67,6 @@ class MergeHelperTest : public testing::Test {
     return result;
   }
 
-  void CheckState(bool success, int iter_pos) {
-    ASSERT_EQ(success, merge_helper_->IsSuccess());
-    if (iter_pos == -1) {
-      ASSERT_FALSE(iter_->Valid());
-    } else {
-      ASSERT_EQ(ks_[iter_pos], iter_->key());
-    }
-  }
-
   std::unique_ptr<test::VectorIterator> iter_;
   std::shared_ptr<MergeOperator> merge_op_;
   std::unique_ptr<MergeHelper> merge_helper_;
@@ -86,10 +81,12 @@ TEST_F(MergeHelperTest, MergeAtBottomSuccess) {
   AddKeyVal("a", 10, kTypeMerge, EncodeInt(3U));
   AddKeyVal("b", 10, kTypeMerge, EncodeInt(4U));  // <- Iterator after merge
 
-  RunUInt64MergeHelper(0, true);
-  CheckState(true, 2);
-  ASSERT_EQ(Key("a", 20, kTypeValue), merge_helper_->key());
-  ASSERT_EQ(EncodeInt(4U), merge_helper_->value());
+  ASSERT_TRUE(RunUInt64MergeHelper(0, true).ok());
+  ASSERT_EQ(ks_[2], iter_->key());
+  ASSERT_EQ(Key("a", 20, kTypeValue), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(4U), merge_helper_->values()[0]);
+  ASSERT_EQ(1U, merge_helper_->keys().size());
+  ASSERT_EQ(1U, merge_helper_->values().size());
 }
 
 // Merging with a value results in a successful merge.
@@ -99,10 +96,12 @@ TEST_F(MergeHelperTest, MergeValue) {
   AddKeyVal("a", 20, kTypeValue, EncodeInt(4U));  // <- Iterator after merge
   AddKeyVal("a", 10, kTypeMerge, EncodeInt(1U));
 
-  RunUInt64MergeHelper(0, false);
-  CheckState(true, 3);
-  ASSERT_EQ(Key("a", 40, kTypeValue), merge_helper_->key());
-  ASSERT_EQ(EncodeInt(8U), merge_helper_->value());
+  ASSERT_TRUE(RunUInt64MergeHelper(0, false).ok());
+  ASSERT_EQ(ks_[3], iter_->key());
+  ASSERT_EQ(Key("a", 40, kTypeValue), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(8U), merge_helper_->values()[0]);
+  ASSERT_EQ(1U, merge_helper_->keys().size());
+  ASSERT_EQ(1U, merge_helper_->values().size());
 }
 
 // Merging stops before a snapshot.
@@ -113,10 +112,12 @@ TEST_F(MergeHelperTest, SnapshotBeforeValue) {
   AddKeyVal("a", 20, kTypeValue, EncodeInt(4U));
   AddKeyVal("a", 10, kTypeMerge, EncodeInt(1U));
 
-  RunUInt64MergeHelper(31, true);
-  CheckState(false, 2);
+  ASSERT_TRUE(RunUInt64MergeHelper(31, true).IsMergeInProgress());
+  ASSERT_EQ(ks_[2], iter_->key());
   ASSERT_EQ(Key("a", 50, kTypeMerge), merge_helper_->keys()[0]);
   ASSERT_EQ(EncodeInt(4U), merge_helper_->values()[0]);
+  ASSERT_EQ(1U, merge_helper_->keys().size());
+  ASSERT_EQ(1U, merge_helper_->values().size());
 }
 
 // MergeHelper preserves the operand stack for merge operators that
@@ -126,12 +127,26 @@ TEST_F(MergeHelperTest, NoPartialMerge) {
   AddKeyVal("a", 40, kTypeMerge, "v");  // <- Iterator after merge
   AddKeyVal("a", 30, kTypeMerge, "v");
 
-  RunStringAppendMergeHelper(31, true);
-  CheckState(false, 2);
+  ASSERT_TRUE(RunStringAppendMergeHelper(31, true).IsMergeInProgress());
+  ASSERT_EQ(ks_[2], iter_->key());
   ASSERT_EQ(Key("a", 40, kTypeMerge), merge_helper_->keys()[0]);
   ASSERT_EQ("v", merge_helper_->values()[0]);
   ASSERT_EQ(Key("a", 50, kTypeMerge), merge_helper_->keys()[1]);
   ASSERT_EQ("v2", merge_helper_->values()[1]);
+  ASSERT_EQ(2U, merge_helper_->keys().size());
+  ASSERT_EQ(2U, merge_helper_->values().size());
+}
+
+// A single operand can not be merged.
+TEST_F(MergeHelperTest, SingleOperand) {
+  AddKeyVal("a", 50, kTypeMerge, EncodeInt(1U));
+
+  ASSERT_TRUE(RunUInt64MergeHelper(31, true).IsMergeInProgress());
+  ASSERT_FALSE(iter_->Valid());
+  ASSERT_EQ(Key("a", 50, kTypeMerge), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(1U), merge_helper_->values()[0]);
+  ASSERT_EQ(1U, merge_helper_->keys().size());
+  ASSERT_EQ(1U, merge_helper_->values().size());
 }
 
 // Merging with a deletion turns the deletion into a value
@@ -139,10 +154,27 @@ TEST_F(MergeHelperTest, MergeDeletion) {
   AddKeyVal("a", 30, kTypeMerge, EncodeInt(3U));
   AddKeyVal("a", 20, kTypeDeletion, "");
 
-  RunUInt64MergeHelper(15, false);
-  CheckState(true, -1);
-  ASSERT_EQ(Key("a", 30, kTypeValue), merge_helper_->key());
-  ASSERT_EQ(EncodeInt(3U), merge_helper_->value());
+  ASSERT_TRUE(RunUInt64MergeHelper(15, false).ok());
+  ASSERT_FALSE(iter_->Valid());
+  ASSERT_EQ(Key("a", 30, kTypeValue), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(3U), merge_helper_->values()[0]);
+  ASSERT_EQ(1U, merge_helper_->keys().size());
+  ASSERT_EQ(1U, merge_helper_->values().size());
+}
+
+// The merge helper stops upon encountering a corrupt key
+TEST_F(MergeHelperTest, CorruptKey) {
+  AddKeyVal("a", 30, kTypeMerge, EncodeInt(3U));
+  AddKeyVal("a", 25, kTypeMerge, EncodeInt(1U));
+  // Corrupt key
+  AddKeyVal("a", 20, kTypeDeletion, "", true);  // <- Iterator after merge
+
+  ASSERT_TRUE(RunUInt64MergeHelper(15, false).IsMergeInProgress());
+  ASSERT_EQ(ks_[2], iter_->key());
+  ASSERT_EQ(Key("a", 30, kTypeMerge), merge_helper_->keys()[0]);
+  ASSERT_EQ(EncodeInt(4U), merge_helper_->values()[0]);
+  ASSERT_EQ(1U, merge_helper_->keys().size());
+  ASSERT_EQ(1U, merge_helper_->values().size());
 }
 
 }  // namespace rocksdb

db/plain_table_db_test.cc

@@ -1012,7 +1012,7 @@ static std::string RandomString(Random* rnd, int len) {
 
 TEST_F(PlainTableDBTest, CompactionTrigger) {
   Options options = CurrentOptions();
-  options.write_buffer_size = 100 << 10; //100KB
+  options.write_buffer_size = 120 << 10;  // 100KB
   options.num_levels = 3;
   options.level0_file_num_compaction_trigger = 3;
   Reopen(&options);
@@ -1022,11 +1022,12 @@ TEST_F(PlainTableDBTest, CompactionTrigger) {
   for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
       num++) {
     std::vector<std::string> values;
-    // Write 120KB (12 values, each 10K)
-    for (int i = 0; i < 12; i++) {
-      values.push_back(RandomString(&rnd, 10000));
+    // Write 120KB (10 values, each 12K)
+    for (int i = 0; i < 10; i++) {
+      values.push_back(RandomString(&rnd, 12000));
       ASSERT_OK(Put(Key(i), values[i]));
     }
+    ASSERT_OK(Put(Key(999), ""));
     dbfull()->TEST_WaitForFlushMemTable();
     ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
   }
@@ -1037,6 +1038,7 @@ TEST_F(PlainTableDBTest, CompactionTrigger) {
     values.push_back(RandomString(&rnd, 10000));
     ASSERT_OK(Put(Key(i), values[i]));
   }
+  ASSERT_OK(Put(Key(999), ""));
   dbfull()->TEST_WaitForCompact();
 
   ASSERT_EQ(NumTableFilesAtLevel(0), 0);

db/repair.cc

@@ -292,8 +292,8 @@ class Repairer {
       ScopedArenaIterator iter(mem->NewIterator(ro, &arena));
       status = BuildTable(dbname_, env_, ioptions_, env_options_, table_cache_,
                           iter.get(), &meta, icmp_,
-                          &int_tbl_prop_collector_factories_, 0, 0,
-                          kNoCompression, CompressionOptions(), false);
+                          &int_tbl_prop_collector_factories_, {},
+                          kNoCompression, CompressionOptions(), false, nullptr);
     }
     delete mem->Unref();
     delete cf_mems_default;

db/skiplist.h

@@ -120,7 +120,10 @@ class SkipList {
   // values are ok.
   std::atomic<int> max_height_;  // Height of the entire list
 
-  // Used for optimizing sequential insert patterns
+  // Used for optimizing sequential insert patterns.  Tricky.  prev_[i] for
+  // i up to max_height_ is the predecessor of prev_[0] and prev_height_
+  // is the height of prev_[0].  prev_[0] can only be equal to head before
+  // insertion, in which case max_height_ and prev_height_ are 1.
   Node** prev_;
   int32_t prev_height_;
 
@@ -138,16 +141,15 @@ class SkipList {
   // Return true if key is greater than the data stored in "n"
   bool KeyIsAfterNode(const Key& key, Node* n) const;
 
-  // Return the earliest node that comes at or after key.
+  // Returns the earliest node with a key >= key.
   // Return nullptr if there is no such node.
-  //
-  // If prev is non-nullptr, fills prev[level] with pointer to previous
-  // node at "level" for every level in [0..max_height_-1].
-  Node* FindGreaterOrEqual(const Key& key, Node** prev) const;
+  Node* FindGreaterOrEqual(const Key& key) const;
 
   // Return the latest node with a key < key.
   // Return head_ if there is no such node.
-  Node* FindLessThan(const Key& key) const;
+  // Fills prev[level] with pointer to previous node at "level" for every
+  // level in [0..max_height_-1], if prev is non-null.
+  Node* FindLessThan(const Key& key, Node** prev = nullptr) const;
 
   // Return the last node in the list.
   // Return head_ if list is empty.
@@ -244,7 +246,7 @@ inline void SkipList<Key, Comparator>::Iterator::Prev() {
 
 template<typename Key, class Comparator>
 inline void SkipList<Key, Comparator>::Iterator::Seek(const Key& target) {
-  node_ = list_->FindGreaterOrEqual(target, nullptr);
+  node_ = list_->FindGreaterOrEqual(target);
 }
 
 template<typename Key, class Comparator>
@@ -280,59 +282,61 @@ bool SkipList<Key, Comparator>::KeyIsAfterNode(const Key& key, Node* n) const {
 
 template<typename Key, class Comparator>
 typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::
-  FindGreaterOrEqual(const Key& key, Node** prev) const {
-  // Use prev as an optimization hint and fallback to slow path
-  if (prev && !KeyIsAfterNode(key, prev[0]->Next(0))) {
-    Node* x = prev[0];
-    Node* next = x->Next(0);
-    if ((x == head_) || KeyIsAfterNode(key, x)) {
-      // Adjust all relevant insertion points to the previous entry
-      for (int i = 1; i < prev_height_; i++) {
-        prev[i] = x;
-      }
-      return next;
-    }
-  }
-  // Normal lookup
+  FindGreaterOrEqual(const Key& key) const {
+  // Note: It looks like we could reduce duplication by implementing
+  // this function as FindLessThan(key)->Next(0), but we wouldn't be able
+  // to exit early on equality and the result wouldn't even be correct.
+  // A concurrent insert might occur after FindLessThan(key) but before
+  // we get a chance to call Next(0).
   Node* x = head_;
   int level = GetMaxHeight() - 1;
+  Node* last_bigger = nullptr;
   while (true) {
     Node* next = x->Next(level);
-    // Make sure the lists are sorted.
-    // If x points to head_ or next points nullptr, it is trivially satisfied.
-    assert((x == head_) || (next == nullptr) || KeyIsAfterNode(next->key, x));
-    if (KeyIsAfterNode(key, next)) {
+    // Make sure the lists are sorted
+    assert(x == head_ || next == nullptr || KeyIsAfterNode(next->key, x));
+    // Make sure we haven't overshot during our search
+    assert(x == head_ || KeyIsAfterNode(key, x));
+    int cmp = (next == nullptr || next == last_bigger)
+        ? 1 : compare_(next->key, key);
+    if (cmp == 0 || (cmp > 0 && level == 0)) {
+      return next;
+    } else if (cmp < 0) {
       // Keep searching in this list
       x = next;
     } else {
-      if (prev != nullptr) prev[level] = x;
-      if (level == 0) {
-        return next;
-      } else {
-        // Switch to next list
-        level--;
-      }
+      // Switch to next list, reuse compare_() result
+      last_bigger = next;
+      level--;
     }
   }
 }
 
 template<typename Key, class Comparator>
 typename SkipList<Key, Comparator>::Node*
-SkipList<Key, Comparator>::FindLessThan(const Key& key) const {
+SkipList<Key, Comparator>::FindLessThan(const Key& key, Node** prev) const {
   Node* x = head_;
   int level = GetMaxHeight() - 1;
+  // KeyIsAfter(key, last_not_after) is definitely false
+  Node* last_not_after = nullptr;
   while (true) {
-    assert(x == head_ || compare_(x->key, key) < 0);
     Node* next = x->Next(level);
-    if (next == nullptr || compare_(next->key, key) >= 0) {
+    assert(x == head_ || next == nullptr || KeyIsAfterNode(next->key, x));
+    assert(x == head_ || KeyIsAfterNode(key, x));
+    if (next != last_not_after && KeyIsAfterNode(key, next)) {
+      // Keep searching in this list
+      x = next;
+    } else {
+      if (prev != nullptr) {
+        prev[level] = x;
+      }
       if (level == 0) {
         return x;
       } else {
-        // Switch to next list
+        // Switch to next list, reuse KeyIUsAfterNode() result
+        last_not_after = next;
         level--;
       }
-    } else {
-      x = next;
     }
   }
 }
@@ -408,12 +412,27 @@ SkipList<Key, Comparator>::SkipList(const Comparator cmp, Allocator* allocator,
 
 template<typename Key, class Comparator>
 void SkipList<Key, Comparator>::Insert(const Key& key) {
-  // TODO(opt): We can use a barrier-free variant of FindGreaterOrEqual()
-  // here since Insert() is externally synchronized.
-  Node* x = FindGreaterOrEqual(key, prev_);
+  // fast path for sequential insertion
+  if (!KeyIsAfterNode(key, prev_[0]->NoBarrier_Next(0)) &&
+      (prev_[0] == head_ || KeyIsAfterNode(key, prev_[0]))) {
+    assert(prev_[0] != head_ || (prev_height_ == 1 && GetMaxHeight() == 1));
+
+    // Outside of this method prev_[1..max_height_] is the predecessor
+    // of prev_[0], and prev_height_ refers to prev_[0].  Inside Insert
+    // prev_[0..max_height - 1] is the predecessor of key.  Switch from
+    // the external state to the internal
+    for (int i = 1; i < prev_height_; i++) {
+      prev_[i] = prev_[0];
+    }
+  } else {
+    // TODO(opt): we could use a NoBarrier predecessor search as an
+    // optimization for architectures where memory_order_acquire needs
+    // a synchronization instruction.  Doesn't matter on x86
+    FindLessThan(key, prev_);
+  }
 
   // Our data structure does not allow duplicate insertion
-  assert(x == nullptr || !Equal(key, x->key));
+  assert(prev_[0]->Next(0) == nullptr || !Equal(key, prev_[0]->Next(0)->key));
 
   int height = RandomHeight();
   if (height > GetMaxHeight()) {
@@ -432,7 +451,7 @@ void SkipList<Key, Comparator>::Insert(const Key& key) {
     max_height_.store(height, std::memory_order_relaxed);
   }
 
-  x = NewNode(key, height);
+  Node* x = NewNode(key, height);
   for (int i = 0; i < height; i++) {
     // NoBarrier_SetNext() suffices since we will add a barrier when
     // we publish a pointer to "x" in prev[i].
@@ -445,7 +464,7 @@ void SkipList<Key, Comparator>::Insert(const Key& key) {
 
 template<typename Key, class Comparator>
 bool SkipList<Key, Comparator>::Contains(const Key& key) const {
-  Node* x = FindGreaterOrEqual(key, nullptr);
+  Node* x = FindGreaterOrEqual(key);
   if (x != nullptr && Equal(key, x->key)) {
     return true;
   } else {

db/snapshot_impl.cc

@@ -0,0 +1,23 @@
+//  Copyright (c) 2015, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include "rocksdb/snapshot.h"
+
+#include "rocksdb/db.h"
+
+namespace rocksdb {
+
+ManagedSnapshot::ManagedSnapshot(DB* db) : db_(db),
+                                           snapshot_(db->GetSnapshot()) {}
+
+ManagedSnapshot::~ManagedSnapshot() {
+  if (snapshot_) {
+    db_->ReleaseSnapshot(snapshot_);
+  }
+}
+
+const Snapshot* ManagedSnapshot::snapshot() { return snapshot_;}
+
+}  // namespace rocksdb

db/table_cache.cc

@@ -21,6 +21,7 @@
 #include "util/file_reader_writer.h"
 #include "util/perf_context_imp.h"
 #include "util/stop_watch.h"
+#include "util/sync_point.h"
 
 namespace rocksdb {
 
@@ -38,6 +39,11 @@ static void UnrefEntry(void* arg1, void* arg2) {
   cache->Release(h);
 }
 
+static void DeleteTableReader(void* arg1, void* arg2) {
+  TableReader* table_reader = reinterpret_cast<TableReader*>(arg1);
+  delete table_reader;
+}
+
 static Slice GetSliceForFileNumber(const uint64_t* file_number) {
   return Slice(reinterpret_cast<const char*>(file_number),
                sizeof(*file_number));
@@ -76,40 +82,58 @@ void TableCache::ReleaseHandle(Cache::Handle* handle) {
   cache_->Release(handle);
 }
 
+Status TableCache::GetTableReader(
+    const EnvOptions& env_options,
+    const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
+    bool sequential_mode, bool record_read_stats, HistogramImpl* file_read_hist,
+    unique_ptr<TableReader>* table_reader) {
+  std::string fname =
+      TableFileName(ioptions_.db_paths, fd.GetNumber(), fd.GetPathId());
+  unique_ptr<RandomAccessFile> file;
+  Status s = ioptions_.env->NewRandomAccessFile(fname, &file, env_options);
+  if (sequential_mode && ioptions_.compaction_readahead_size > 0) {
+    file = NewReadaheadRandomAccessFile(std::move(file),
+                                        ioptions_.compaction_readahead_size);
+  }
+  RecordTick(ioptions_.statistics, NO_FILE_OPENS);
+  if (s.ok()) {
+    if (!sequential_mode && ioptions_.advise_random_on_open) {
+      file->Hint(RandomAccessFile::RANDOM);
+    }
+    StopWatch sw(ioptions_.env, ioptions_.statistics, TABLE_OPEN_IO_MICROS);
+    std::unique_ptr<RandomAccessFileReader> file_reader(
+        new RandomAccessFileReader(std::move(file), ioptions_.env,
+                                   ioptions_.statistics, record_read_stats,
+                                   file_read_hist));
+    s = ioptions_.table_factory->NewTableReader(
+        ioptions_, env_options, internal_comparator, std::move(file_reader),
+        fd.GetFileSize(), table_reader);
+    TEST_SYNC_POINT("TableCache::GetTableReader:0");
+  }
+  return s;
+}
+
 Status TableCache::FindTable(const EnvOptions& env_options,
                              const InternalKeyComparator& internal_comparator,
                              const FileDescriptor& fd, Cache::Handle** handle,
-                             const bool no_io, bool record_read_stats) {
+                             const bool no_io, bool record_read_stats,
+                             HistogramImpl* file_read_hist) {
   PERF_TIMER_GUARD(find_table_nanos);
   Status s;
   uint64_t number = fd.GetNumber();
   Slice key = GetSliceForFileNumber(&number);
   *handle = cache_->Lookup(key);
+  TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0",
+                           const_cast<bool*>(&no_io));
+
   if (*handle == nullptr) {
-    if (no_io) { // Dont do IO and return a not-found status
+    if (no_io) {  // Don't do IO and return a not-found status
       return Status::Incomplete("Table not found in table_cache, no_io is set");
     }
-    std::string fname =
-        TableFileName(ioptions_.db_paths, fd.GetNumber(), fd.GetPathId());
-    unique_ptr<RandomAccessFile> file;
     unique_ptr<TableReader> table_reader;
-    s = ioptions_.env->NewRandomAccessFile(fname, &file, env_options);
-    RecordTick(ioptions_.statistics, NO_FILE_OPENS);
-    if (s.ok()) {
-      if (ioptions_.advise_random_on_open) {
-        file->Hint(RandomAccessFile::RANDOM);
-      }
-      StopWatch sw(ioptions_.env, ioptions_.statistics, TABLE_OPEN_IO_MICROS);
-      std::unique_ptr<RandomAccessFileReader> file_reader(
-          new RandomAccessFileReader(
-              std::move(file), ioptions_.env,
-              record_read_stats ? ioptions_.statistics : nullptr,
-              SST_READ_MICROS));
-      s = ioptions_.table_factory->NewTableReader(
-          ioptions_, env_options, internal_comparator, std::move(file_reader),
-          fd.GetFileSize(), &table_reader);
-    }
-
+    s = GetTableReader(env_options, internal_comparator, fd,
+                       false /* sequential mode */, record_read_stats,
+                       file_read_hist, &table_reader);
     if (!s.ok()) {
       assert(table_reader == nullptr);
       RecordTick(ioptions_.statistics, NO_FILE_ERRORS);
@@ -128,35 +152,56 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
                                   const InternalKeyComparator& icomparator,
                                   const FileDescriptor& fd,
                                   TableReader** table_reader_ptr,
+                                  HistogramImpl* file_read_hist,
                                   bool for_compaction, Arena* arena) {
   PERF_TIMER_GUARD(new_table_iterator_nanos);
 
   if (table_reader_ptr != nullptr) {
     *table_reader_ptr = nullptr;
   }
-  TableReader* table_reader = fd.table_reader;
+
+  TableReader* table_reader = nullptr;
   Cache::Handle* handle = nullptr;
-  Status s;
-  if (table_reader == nullptr) {
-    s = FindTable(env_options, icomparator, fd, &handle,
-                  options.read_tier == kBlockCacheTier, !for_compaction);
+  bool create_new_table_reader =
+      (for_compaction && ioptions_.new_table_reader_for_compaction_inputs);
+  if (create_new_table_reader) {
+    unique_ptr<TableReader> table_reader_unique_ptr;
+    Status s = GetTableReader(
+        env_options, icomparator, fd, /* sequential mode */ true,
+        /* record stats */ false, nullptr, &table_reader_unique_ptr);
     if (!s.ok()) {
       return NewErrorIterator(s, arena);
     }
-    table_reader = GetTableReaderFromHandle(handle);
+    table_reader = table_reader_unique_ptr.release();
+  } else {
+    table_reader = fd.table_reader;
+    if (table_reader == nullptr) {
+      Status s =
+          FindTable(env_options, icomparator, fd, &handle,
+                    options.read_tier == kBlockCacheTier /* no_io */,
+                    !for_compaction /* record read_stats */, file_read_hist);
+      if (!s.ok()) {
+        return NewErrorIterator(s, arena);
+      }
+      table_reader = GetTableReaderFromHandle(handle);
+    }
   }
 
   Iterator* result = table_reader->NewIterator(options, arena);
-  if (handle != nullptr) {
+
+  if (create_new_table_reader) {
+    assert(handle == nullptr);
+    result->RegisterCleanup(&DeleteTableReader, table_reader, nullptr);
+  } else if (handle != nullptr) {
     result->RegisterCleanup(&UnrefEntry, cache_, handle);
   }
-  if (table_reader_ptr != nullptr) {
-    *table_reader_ptr = table_reader;
-  }
 
   if (for_compaction) {
     table_reader->SetupForCompaction();
   }
+  if (table_reader_ptr != nullptr) {
+    *table_reader_ptr = table_reader;
+  }
 
   return result;
 }
@@ -164,7 +209,7 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
 Status TableCache::Get(const ReadOptions& options,
                        const InternalKeyComparator& internal_comparator,
                        const FileDescriptor& fd, const Slice& k,
-                       GetContext* get_context) {
+                       GetContext* get_context, HistogramImpl* file_read_hist) {
   TableReader* t = fd.table_reader;
   Status s;
   Cache::Handle* handle = nullptr;
@@ -210,7 +255,8 @@ Status TableCache::Get(const ReadOptions& options,
 
   if (!t) {
     s = FindTable(env_options_, internal_comparator, fd, &handle,
-                  options.read_tier == kBlockCacheTier);
+                  options.read_tier == kBlockCacheTier /* no_io */,
+                  true /* record_read_stats */, file_read_hist);
     if (s.ok()) {
       t = GetTableReaderFromHandle(handle);
     }

db/table_cache.h

@@ -28,6 +28,7 @@ class Env;
 class Arena;
 struct FileDescriptor;
 class GetContext;
+class HistogramImpl;
 
 class TableCache {
  public:
@@ -46,6 +47,7 @@ class TableCache {
                         const InternalKeyComparator& internal_comparator,
                         const FileDescriptor& file_fd,
                         TableReader** table_reader_ptr = nullptr,
+                        HistogramImpl* file_read_hist = nullptr,
                         bool for_compaction = false, Arena* arena = nullptr);
 
   // If a seek to internal key "k" in specified file finds an entry,
@@ -54,7 +56,7 @@ class TableCache {
   Status Get(const ReadOptions& options,
              const InternalKeyComparator& internal_comparator,
              const FileDescriptor& file_fd, const Slice& k,
-             GetContext* get_context);
+             GetContext* get_context, HistogramImpl* file_read_hist = nullptr);
 
   // Evict any entry for the specified file number
   static void Evict(Cache* cache, uint64_t file_number);
@@ -63,7 +65,8 @@ class TableCache {
   Status FindTable(const EnvOptions& toptions,
                    const InternalKeyComparator& internal_comparator,
                    const FileDescriptor& file_fd, Cache::Handle**,
-                   const bool no_io = false, bool record_read_stats = true);
+                   const bool no_io = false, bool record_read_stats = true,
+                   HistogramImpl* file_read_hist = nullptr);
 
   // Get TableReader from a cache handle.
   TableReader* GetTableReaderFromHandle(Cache::Handle* handle);
@@ -81,7 +84,7 @@ class TableCache {
                             bool no_io = false);
 
   // Return total memory usage of the table reader of the file.
-  // 0 of table reader of the file is not loaded.
+  // 0 if table reader of the file is not loaded.
   size_t GetMemoryUsageByTableReader(
       const EnvOptions& toptions,
       const InternalKeyComparator& internal_comparator,
@@ -91,6 +94,13 @@ class TableCache {
   void ReleaseHandle(Cache::Handle* handle);
 
  private:
+  // Build a table reader
+  Status GetTableReader(const EnvOptions& env_options,
+                        const InternalKeyComparator& internal_comparator,
+                        const FileDescriptor& fd, bool sequential_mode,
+                        bool record_read_stats, HistogramImpl* file_read_hist,
+                        unique_ptr<TableReader>* table_reader);
+
   const ImmutableCFOptions& ioptions_;
   const EnvOptions& env_options_;
   Cache* const cache_;

db/table_properties_collector_test.cc

@@ -11,8 +11,8 @@
 #include "db/db_impl.h"
 #include "db/dbformat.h"
 #include "db/table_properties_collector.h"
-#include "rocksdb/table.h"
 #include "rocksdb/immutable_options.h"
+#include "rocksdb/table.h"
 #include "table/block_based_table_factory.h"
 #include "table/meta_blocks.h"
 #include "table/plain_table_factory.h"

db/version_builder.cc

@@ -15,12 +15,16 @@
 
 #include <inttypes.h>
 #include <algorithm>
+#include <atomic>
 #include <set>
+#include <thread>
 #include <unordered_map>
 #include <unordered_set>
+#include <utility>
 #include <vector>
 
 #include "db/dbformat.h"
+#include "db/internal_stats.h"
 #include "db/table_cache.h"
 #include "db/version_set.h"
 #include "table/table_reader.h"
@@ -278,21 +282,52 @@ class VersionBuilder::Rep {
     CheckConsistency(vstorage);
   }
 
-  void LoadTableHandlers() {
+  void LoadTableHandlers(InternalStats* internal_stats, int max_threads) {
     assert(table_cache_ != nullptr);
+    // <file metadata, level>
+    std::vector<std::pair<FileMetaData*, int>> files_meta;
     for (int level = 0; level < base_vstorage_->num_levels(); level++) {
       for (auto& file_meta_pair : levels_[level].added_files) {
         auto* file_meta = file_meta_pair.second;
         assert(!file_meta->table_reader_handle);
-        table_cache_->FindTable(
-            env_options_, *(base_vstorage_->InternalComparator()),
-            file_meta->fd, &file_meta->table_reader_handle, false);
+        files_meta.emplace_back(file_meta, level);
+      }
+    }
+
+    std::atomic<size_t> next_file_meta_idx(0);
+    std::function<void()> load_handlers_func = [&]() {
+      while (true) {
+        size_t file_idx = next_file_meta_idx.fetch_add(1);
+        if (file_idx >= files_meta.size()) {
+          break;
+        }
+
+        auto* file_meta = files_meta[file_idx].first;
+        int level = files_meta[file_idx].second;
+        table_cache_->FindTable(env_options_,
+                                *(base_vstorage_->InternalComparator()),
+                                file_meta->fd, &file_meta->table_reader_handle,
+                                false /*no_io */, true /* record_read_stats */,
+                                internal_stats->GetFileReadHist(level));
         if (file_meta->table_reader_handle != nullptr) {
           // Load table_reader
           file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle(
               file_meta->table_reader_handle);
         }
       }
+    };
+
+    if (max_threads <= 1) {
+      load_handlers_func();
+    } else {
+      std::vector<std::thread> threads;
+      for (int i = 0; i < max_threads; i++) {
+        threads.emplace_back(load_handlers_func);
+      }
+
+      for (auto& t : threads) {
+        t.join();
+      }
     }
   }
 
@@ -321,7 +356,10 @@ void VersionBuilder::Apply(VersionEdit* edit) { rep_->Apply(edit); }
 void VersionBuilder::SaveTo(VersionStorageInfo* vstorage) {
   rep_->SaveTo(vstorage);
 }
-void VersionBuilder::LoadTableHandlers() { rep_->LoadTableHandlers(); }
+void VersionBuilder::LoadTableHandlers(InternalStats* internal_stats,
+                                       int max_threads) {
+  rep_->LoadTableHandlers(internal_stats, max_threads);
+}
 void VersionBuilder::MaybeAddFile(VersionStorageInfo* vstorage, int level,
                                   FileMetaData* f) {
   rep_->MaybeAddFile(vstorage, level, f);

db/version_builder.h

@@ -16,6 +16,7 @@ class TableCache;
 class VersionStorageInfo;
 class VersionEdit;
 struct FileMetaData;
+class InternalStats;
 
 // A helper class so we can efficiently apply a whole sequence
 // of edits to a particular state without creating intermediate
@@ -30,7 +31,7 @@ class VersionBuilder {
                                   int level);
   void Apply(VersionEdit* edit);
   void SaveTo(VersionStorageInfo* vstorage);
-  void LoadTableHandlers();
+  void LoadTableHandlers(InternalStats* internal_stats, int max_threads = 1);
   void MaybeAddFile(VersionStorageInfo* vstorage, int level, FileMetaData* f);
 
  private:

db/version_edit.h

@@ -93,6 +93,8 @@ struct FileMetaData {
   FileMetaData()
       : refs(0),
         being_compacted(false),
+        smallest_seqno(kMaxSequenceNumber),
+        largest_seqno(0),
         table_reader_handle(nullptr),
         compensated_file_size(0),
         num_entries(0),
@@ -101,6 +103,17 @@ struct FileMetaData {
         raw_value_size(0),
         init_stats_from_file(false),
         marked_for_compaction(false) {}
+
+  // REQUIRED: Keys must be given to the function in sorted order (it expects
+  // the last key to be the largest).
+  void UpdateBoundaries(const Slice& key, SequenceNumber seqno) {
+    if (smallest.size() == 0) {
+      smallest.DecodeFrom(key);
+    }
+    largest.DecodeFrom(key);
+    smallest_seqno = std::min(smallest_seqno, seqno);
+    largest_seqno = std::max(largest_seqno, seqno);
+  }
 };
 
 // A compressed copy of file meta data that just contain
@@ -179,7 +192,12 @@ class VersionEdit {
     f.smallest_seqno = smallest_seqno;
     f.largest_seqno = largest_seqno;
     f.marked_for_compaction = marked_for_compaction;
-    new_files_.push_back(std::make_pair(level, f));
+    new_files_.emplace_back(level, f);
+  }
+
+  void AddFile(int level, const FileMetaData& f) {
+    assert(f.smallest_seqno <= f.largest_seqno);
+    new_files_.emplace_back(level, f);
   }
 
   // Delete the specified "file" from the specified "level".

db/version_set.cc

@@ -24,6 +24,7 @@
 #include <string>
 
 #include "db/filename.h"
+#include "db/internal_stats.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
 #include "db/memtable.h"
@@ -474,13 +475,18 @@ class LevelFileNumIterator : public Iterator {
 class LevelFileIteratorState : public TwoLevelIteratorState {
  public:
   LevelFileIteratorState(TableCache* table_cache,
-    const ReadOptions& read_options, const EnvOptions& env_options,
-    const InternalKeyComparator& icomparator, bool for_compaction,
-    bool prefix_enabled)
-    : TwoLevelIteratorState(prefix_enabled),
-      table_cache_(table_cache), read_options_(read_options),
-      env_options_(env_options), icomparator_(icomparator),
-      for_compaction_(for_compaction) {}
+                         const ReadOptions& read_options,
+                         const EnvOptions& env_options,
+                         const InternalKeyComparator& icomparator,
+                         HistogramImpl* file_read_hist, bool for_compaction,
+                         bool prefix_enabled)
+      : TwoLevelIteratorState(prefix_enabled),
+        table_cache_(table_cache),
+        read_options_(read_options),
+        env_options_(env_options),
+        icomparator_(icomparator),
+        file_read_hist_(file_read_hist),
+        for_compaction_(for_compaction) {}
 
   Iterator* NewSecondaryIterator(const Slice& meta_handle) override {
     if (meta_handle.size() != sizeof(FileDescriptor)) {
@@ -491,7 +497,8 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
           reinterpret_cast<const FileDescriptor*>(meta_handle.data());
       return table_cache_->NewIterator(
           read_options_, env_options_, icomparator_, *fd,
-          nullptr /* don't need reference to table*/, for_compaction_);
+          nullptr /* don't need reference to table*/, file_read_hist_,
+          for_compaction_);
     }
   }
 
@@ -504,6 +511,7 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
   const ReadOptions read_options_;
   const EnvOptions& env_options_;
   const InternalKeyComparator& icomparator_;
+  HistogramImpl* file_read_hist_;
   bool for_compaction_;
 };
 
@@ -584,26 +592,58 @@ Status Version::GetTableProperties(std::shared_ptr<const TableProperties>* tp,
 }
 
 Status Version::GetPropertiesOfAllTables(TablePropertiesCollection* props) {
+  Status s;
   for (int level = 0; level < storage_info_.num_levels_; level++) {
-    for (const auto& file_meta : storage_info_.files_[level]) {
-      auto fname =
-          TableFileName(vset_->db_options_->db_paths, file_meta->fd.GetNumber(),
-                        file_meta->fd.GetPathId());
-      // 1. If the table is already present in table cache, load table
-      // properties from there.
-      std::shared_ptr<const TableProperties> table_properties;
-      Status s = GetTableProperties(&table_properties, file_meta, &fname);
-      if (s.ok()) {
-        props->insert({fname, table_properties});
-      } else {
-        return s;
-      }
+    s = GetPropertiesOfAllTables(props, level);
+    if (!s.ok()) {
+      return s;
     }
   }
 
   return Status::OK();
 }
 
+Status Version::GetPropertiesOfAllTables(TablePropertiesCollection* props,
+                                         int level) {
+  for (const auto& file_meta : storage_info_.files_[level]) {
+    auto fname =
+        TableFileName(vset_->db_options_->db_paths, file_meta->fd.GetNumber(),
+                      file_meta->fd.GetPathId());
+    // 1. If the table is already present in table cache, load table
+    // properties from there.
+    std::shared_ptr<const TableProperties> table_properties;
+    Status s = GetTableProperties(&table_properties, file_meta, &fname);
+    if (s.ok()) {
+      props->insert({fname, table_properties});
+    } else {
+      return s;
+    }
+  }
+
+  return Status::OK();
+}
+
+Status Version::GetAggregatedTableProperties(
+    std::shared_ptr<const TableProperties>* tp, int level) {
+  TablePropertiesCollection props;
+  Status s;
+  if (level < 0) {
+    s = GetPropertiesOfAllTables(&props);
+  } else {
+    s = GetPropertiesOfAllTables(&props, level);
+  }
+  if (!s.ok()) {
+    return s;
+  }
+
+  auto* new_tp = new TableProperties();
+  for (const auto& item : props) {
+    new_tp->Add(*item.second);
+  }
+  tp->reset(new_tp);
+  return Status::OK();
+}
+
 size_t Version::GetMemoryUsageByTableReaders() {
   size_t total_usage = 0;
   for (auto& file_level : storage_info_.level_files_brief_) {
@@ -705,7 +745,7 @@ void Version::AddIterators(const ReadOptions& read_options,
     const auto& file = storage_info_.LevelFilesBrief(0).files[i];
     merge_iter_builder->AddIterator(cfd_->table_cache()->NewIterator(
         read_options, soptions, cfd_->internal_comparator(), file.fd, nullptr,
-        false, arena));
+        cfd_->internal_stats()->GetFileReadHist(0), false, arena));
   }
 
   // For levels > 0, we can use a concatenating iterator that sequentially
@@ -714,10 +754,12 @@ void Version::AddIterators(const ReadOptions& read_options,
   for (int level = 1; level < storage_info_.num_non_empty_levels(); level++) {
     if (storage_info_.LevelFilesBrief(level).num_files != 0) {
       auto* mem = arena->AllocateAligned(sizeof(LevelFileIteratorState));
-      auto* state = new (mem) LevelFileIteratorState(
-          cfd_->table_cache(), read_options, soptions,
-          cfd_->internal_comparator(), false /* for_compaction */,
-          cfd_->ioptions()->prefix_extractor != nullptr);
+      auto* state = new (mem)
+          LevelFileIteratorState(cfd_->table_cache(), read_options, soptions,
+                                 cfd_->internal_comparator(),
+                                 cfd_->internal_stats()->GetFileReadHist(level),
+                                 false /* for_compaction */,
+                                 cfd_->ioptions()->prefix_extractor != nullptr);
       mem = arena->AllocateAligned(sizeof(LevelFileNumIterator));
       auto* first_level_iter = new (mem) LevelFileNumIterator(
           cfd_->internal_comparator(), &storage_info_.LevelFilesBrief(level));
@@ -752,6 +794,7 @@ VersionStorageInfo::VersionStorageInfo(
       accumulated_num_non_deletions_(0),
       accumulated_num_deletions_(0),
       num_samples_(0),
+      estimated_compaction_needed_bytes_(0),
       finalized_(false) {
   if (ref_vstorage != nullptr) {
     accumulated_file_size_ = ref_vstorage->accumulated_file_size_;
@@ -810,8 +853,9 @@ void Version::Get(const ReadOptions& read_options,
       user_comparator(), internal_comparator());
   FdWithKeyRange* f = fp.GetNextFile();
   while (f != nullptr) {
-    *status = table_cache_->Get(read_options, *internal_comparator(), f->fd,
-                                ikey, &get_context);
+    *status = table_cache_->Get(
+        read_options, *internal_comparator(), f->fd, ikey, &get_context,
+        cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()));
     // TODO: examine the behavior for corrupted key
     if (!status->ok()) {
       return;
@@ -1003,6 +1047,62 @@ int VersionStorageInfo::MaxInputLevel() const {
   return 0;
 }
 
+void VersionStorageInfo::EstimateCompactionBytesNeeded(
+    const MutableCFOptions& mutable_cf_options) {
+  // Only implemented for level-based compaction
+  if (compaction_style_ != kCompactionStyleLevel) {
+    return;
+  }
+
+  // Start from Level 0, if level 0 qualifies compaction to level 1,
+  // we estimate the size of compaction.
+  // Then we move on to the next level and see whether it qualifies compaction
+  // to the next level. The size of the level is estimated as the actual size
+  // on the level plus the input bytes from the previous level if there is any.
+  // If it exceeds, take the exceeded bytes as compaction input and add the size
+  // of the compaction size to tatal size.
+  // We keep doing it to Level 2, 3, etc, until the last level and return the
+  // accumulated bytes.
+
+  size_t bytes_compact_to_next_level = 0;
+  // Level 0
+  bool level0_compact_triggered = false;
+  if (static_cast<int>(files_[0].size()) >
+      mutable_cf_options.level0_file_num_compaction_trigger) {
+    level0_compact_triggered = true;
+    for (auto* f : files_[0]) {
+      bytes_compact_to_next_level += f->fd.GetFileSize();
+    }
+    estimated_compaction_needed_bytes_ = bytes_compact_to_next_level;
+  } else {
+    estimated_compaction_needed_bytes_ = 0;
+  }
+
+  // Level 1 and up.
+  for (int level = base_level(); level <= MaxInputLevel(); level++) {
+    size_t level_size = 0;
+    for (auto* f : files_[level]) {
+      level_size += f->fd.GetFileSize();
+    }
+    if (level == base_level() && level0_compact_triggered) {
+      // Add base level size to compaction if level0 compaction triggered.
+      estimated_compaction_needed_bytes_ += level_size;
+    }
+    // Add size added by previous compaction
+    level_size += bytes_compact_to_next_level;
+    bytes_compact_to_next_level = 0;
+    size_t level_target = MaxBytesForLevel(level);
+    if (level_size > level_target) {
+      bytes_compact_to_next_level = level_size - level_target;
+      // Simplify to assume the actual compaction fan-out ratio is always
+      // mutable_cf_options.max_bytes_for_level_multiplier.
+      estimated_compaction_needed_bytes_ +=
+          bytes_compact_to_next_level *
+          (1 + mutable_cf_options.max_bytes_for_level_multiplier);
+    }
+  }
+}
+
 void VersionStorageInfo::ComputeCompactionScore(
     const MutableCFOptions& mutable_cf_options,
     const CompactionOptionsFIFO& compaction_options_fifo) {
@@ -1087,6 +1187,7 @@ void VersionStorageInfo::ComputeCompactionScore(
     }
   }
   ComputeFilesMarkedForCompaction();
+  EstimateCompactionBytesNeeded(mutable_cf_options);
 }
 
 void VersionStorageInfo::ComputeFilesMarkedForCompaction() {
@@ -1130,7 +1231,6 @@ bool CompareCompensatedSizeDescending(const Fsize& first, const Fsize& second) {
 } // anonymous namespace
 
 void VersionStorageInfo::AddFile(int level, FileMetaData* f) {
-  assert(level < num_levels());
   auto* level_files = &files_[level];
   // Must not overlap
   assert(level <= 0 || level_files->empty() ||
@@ -1481,7 +1581,7 @@ bool VersionStorageInfo::HasOverlappingUserKey(
         files[last_file].largest_key);
     const Slice first_key_after = ExtractUserKey(
         files[last_file+1].smallest_key);
-    if (user_cmp->Compare(last_key_in_input, first_key_after) == 0) {
+    if (user_cmp->Equal(last_key_in_input, first_key_after)) {
       // The last user key in input overlaps with the next file's first key
       return true;
     }
@@ -1496,7 +1596,7 @@ bool VersionStorageInfo::HasOverlappingUserKey(
         files[first_file].smallest_key);
     const Slice& last_key_before = ExtractUserKey(
         files[first_file-1].largest_key);
-    if (user_cmp->Compare(first_key_in_input, last_key_before) == 0) {
+    if (user_cmp->Equal(first_key_in_input, last_key_before)) {
       // The first user key in input overlaps with the previous file's last key
       return true;
     }
@@ -1879,7 +1979,8 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
     if (!manifest_writers_.empty()) {
       manifest_writers_.front()->cv.Signal();
     }
-    return Status::OK();
+    // we steal this code to also inform about cf-drop
+    return Status::ShutdownInProgress();
   }
 
   std::vector<VersionEdit*> batch_edits;
@@ -1945,7 +2046,8 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
         db_options_->max_open_files == -1) {
       // unlimited table cache. Pre-load table handle now.
       // Need to do it out of the mutex.
-      builder_guard->version_builder()->LoadTableHandlers();
+      builder_guard->version_builder()->LoadTableHandlers(
+          column_family_data->internal_stats());
     }
 
     // This is fine because everything inside of this block is serialized --
@@ -2040,6 +2142,11 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
       new_manifest_file_size = descriptor_log_->file()->GetFileSize();
     }
 
+    if (edit->is_column_family_drop_) {
+      TEST_SYNC_POINT("VersionSet::LogAndApply::ColumnFamilyDrop:1");
+      TEST_SYNC_POINT("VersionSet::LogAndApply::ColumnFamilyDrop:2");
+    }
+
     LogFlush(db_options_->info_log);
     mu->Lock();
   }
@@ -2394,9 +2501,10 @@ Status VersionSet::Recover(
       auto* builder = builders_iter->second->version_builder();
 
       if (db_options_->max_open_files == -1) {
-      // unlimited table cache. Pre-load table handle now.
-      // Need to do it out of the mutex.
-        builder->LoadTableHandlers();
+        // unlimited table cache. Pre-load table handle now.
+        // Need to do it out of the mutex.
+        builder->LoadTableHandlers(cfd->internal_stats(),
+                                   db_options_->max_file_opening_threads);
       }
 
       Version* v = new Version(cfd, this, current_version_number_++);
@@ -2900,15 +3008,27 @@ bool VersionSet::ManifestContains(uint64_t manifest_file_num,
   return result;
 }
 
+// TODO(aekmekji): in CompactionJob::GenSubcompactionBoundaries(), this
+// function is called repeatedly with consecutive pairs of slices. For example
+// if the slice list is [a, b, c, d] this function is called with arguments
+// (a,b) then (b,c) then (c,d). Knowing this, an optimization is possible where
+// we avoid doing binary search for the keys b and c twice and instead somehow
+// maintain state of where they first appear in the files.
 uint64_t VersionSet::ApproximateSize(Version* v, const Slice& start,
-                                     const Slice& end) {
+                                     const Slice& end, int start_level,
+                                     int end_level) {
   // pre-condition
   assert(v->cfd_->internal_comparator().Compare(start, end) <= 0);
 
   uint64_t size = 0;
   const auto* vstorage = v->storage_info();
+  end_level = end_level == -1
+                  ? vstorage->num_non_empty_levels()
+                  : std::min(end_level, vstorage->num_non_empty_levels());
 
-  for (int level = 0; level < vstorage->num_non_empty_levels(); level++) {
+  assert(start_level <= end_level);
+
+  for (int level = start_level; level < end_level; level++) {
     const LevelFilesBrief& files_brief = vstorage->LevelFilesBrief(level);
     if (!files_brief.num_files) {
       // empty level, skip exploration
@@ -3040,7 +3160,7 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
   read_options.verify_checksums =
     c->mutable_cf_options()->verify_checksums_in_compaction;
   read_options.fill_cache = false;
-  if (c->IsSubCompaction()) {
+  if (c->ShouldFormSubcompactions()) {
     read_options.total_order_seek = true;
   }
 
@@ -3060,14 +3180,17 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
           list[num++] = cfd->table_cache()->NewIterator(
               read_options, env_options_compactions_,
               cfd->internal_comparator(), flevel->files[i].fd, nullptr,
+              nullptr, /* no per level latency histogram*/
               true /* for compaction */);
         }
       } else {
         // Create concatenating iterator for the files from this level
-        list[num++] = NewTwoLevelIterator(new LevelFileIteratorState(
-              cfd->table_cache(), read_options, env_options_,
-              cfd->internal_comparator(), true /* for_compaction */,
-              false /* prefix enabled */),
+        list[num++] = NewTwoLevelIterator(
+            new LevelFileIteratorState(
+                cfd->table_cache(), read_options, env_options_,
+                cfd->internal_comparator(),
+                nullptr /* no per level latency histogram */,
+                true /* for_compaction */, false /* prefix enabled */),
             new LevelFileNumIterator(cfd->internal_comparator(),
                                      c->input_levels(which)));
       }
@@ -3226,4 +3349,22 @@ uint64_t VersionSet::GetNumLiveVersions(Version* dummy_versions) {
   return count;
 }
 
+uint64_t VersionSet::GetTotalSstFilesSize(Version* dummy_versions) {
+  std::unordered_set<uint64_t> unique_files;
+  uint64_t total_files_size = 0;
+  for (Version* v = dummy_versions->next_; v != dummy_versions; v = v->next_) {
+    VersionStorageInfo* storage_info = v->storage_info();
+    for (int level = 0; level < storage_info->num_levels_; level++) {
+      for (const auto& file_meta : storage_info->LevelFiles(level)) {
+        if (unique_files.find(file_meta->fd.packed_number_and_path_id) ==
+            unique_files.end()) {
+          unique_files.insert(file_meta->fd.packed_number_and_path_id);
+          total_files_size += file_meta->fd.GetFileSize();
+        }
+      }
+    }
+  }
+  return total_files_size;
+}
+
 }  // namespace rocksdb

db/version_set.h

@@ -121,6 +121,10 @@ class VersionStorageInfo {
       const MutableCFOptions& mutable_cf_options,
       const CompactionOptionsFIFO& compaction_options_fifo);
 
+  // Estimate est_comp_needed_bytes_
+  void EstimateCompactionBytesNeeded(
+      const MutableCFOptions& mutable_cf_options);
+
   // This computes files_marked_for_compaction_ and is called by
   // ComputeCompactionScore()
   void ComputeFilesMarkedForCompaction();
@@ -315,6 +319,10 @@ class VersionStorageInfo {
   // Returns an estimate of the amount of live data in bytes.
   uint64_t EstimateLiveDataSize() const;
 
+  uint64_t estimated_compaction_needed_bytes() const {
+    return estimated_compaction_needed_bytes_;
+  }
+
  private:
   const InternalKeyComparator* internal_comparator_;
   const Comparator* user_comparator_;
@@ -389,6 +397,9 @@ class VersionStorageInfo {
   uint64_t accumulated_num_deletions_;
   // the number of samples
   uint64_t num_samples_;
+  // Estimated bytes needed to be compacted until all levels' size is down to
+  // target sizes.
+  uint64_t estimated_compaction_needed_bytes_;
 
   bool finalized_;
 
@@ -451,6 +462,14 @@ class Version {
   // tables' propertis, represented as shared_ptr.
   Status GetPropertiesOfAllTables(TablePropertiesCollection* props);
 
+  Status GetPropertiesOfAllTables(TablePropertiesCollection* props, int level);
+
+  // REQUIRES: lock is held
+  // On success, "tp" will contains the aggregated table property amoug
+  // the table properties of all sst files in this version.
+  Status GetAggregatedTableProperties(
+      std::shared_ptr<const TableProperties>* tp, int level = -1);
+
   uint64_t GetEstimatedActiveKeys() {
     return storage_info_.GetEstimatedActiveKeys();
   }
@@ -627,7 +646,10 @@ class VersionSet {
   void AddLiveFiles(std::vector<FileDescriptor>* live_list);
 
   // Return the approximate size of data to be scanned for range [start, end)
-  uint64_t ApproximateSize(Version* v, const Slice& start, const Slice& end);
+  // in levels [start_level, end_level). If end_level == 0 it will search
+  // through all non-empty levels
+  uint64_t ApproximateSize(Version* v, const Slice& start, const Slice& end,
+                           int start_level = 0, int end_level = -1);
 
   // Return the size of the current manifest file
   uint64_t manifest_file_size() const { return manifest_file_size_; }
@@ -651,6 +673,8 @@ class VersionSet {
 
   static uint64_t GetNumLiveVersions(Version* dummy_versions);
 
+  static uint64_t GetTotalSstFilesSize(Version* dummy_versions);
+
  private:
   struct ManifestWriter;
 

db/write_batch.cc

@@ -27,16 +27,16 @@
 #include <stack>
 #include <stdexcept>
 
-#include "rocksdb/merge_operator.h"
-#include "db/dbformat.h"
-#include "db/db_impl.h"
 #include "db/column_family.h"
+#include "db/db_impl.h"
+#include "db/dbformat.h"
 #include "db/memtable.h"
-#include "db/snapshot.h"
+#include "db/snapshot_impl.h"
 #include "db/write_batch_internal.h"
+#include "rocksdb/merge_operator.h"
 #include "util/coding.h"
-#include "util/statistics.h"
 #include "util/perf_context_imp.h"
+#include "util/statistics.h"
 
 namespace rocksdb {
 

db/write_thread.cc

@@ -7,82 +7,102 @@
 
 namespace rocksdb {
 
-void WriteThread::EnterWriteThread(WriteThread::Writer* w) {
-  // the following code block pushes the current writer "w" into the writer
-  // queue "writers_" and wait until one of the following conditions met:
-  // 1. the job of "w" has been done by some other writers.
-  // 2. "w" becomes the first writer in "writers_"
-  // 3. "w" timed-out.
-  writers_.push_back(w);
-
-  while (!w->done && w != writers_.front()) {
-    w->cv.Wait();
-  }
+void WriteThread::Await(Writer* w) {
+  std::unique_lock<std::mutex> guard(w->JoinMutex());
+  w->JoinCV().wait(guard, [w] { return w->joined; });
 }
 
-void WriteThread::ExitWriteThread(WriteThread::Writer* w,
-                                  WriteThread::Writer* last_writer,
-                                  Status status) {
-  // Pop out the current writer and all writers being pushed before the
-  // current writer from the writer queue.
-  while (!writers_.empty()) {
-    Writer* ready = writers_.front();
-    writers_.pop_front();
-    if (ready != w) {
-      ready->status = status;
-      ready->done = true;
-      ready->cv.Signal();
+void WriteThread::MarkJoined(Writer* w) {
+  std::lock_guard<std::mutex> guard(w->JoinMutex());
+  assert(!w->joined);
+  w->joined = true;
+  w->JoinCV().notify_one();
+}
+
+void WriteThread::LinkOne(Writer* w, bool* wait_needed) {
+  assert(!w->joined && !w->done);
+
+  Writer* writers = newest_writer_.load(std::memory_order_relaxed);
+  while (true) {
+    w->link_older = writers;
+    if (writers != nullptr) {
+      w->CreateMutex();
+    }
+    if (newest_writer_.compare_exchange_strong(writers, w)) {
+      // Success.
+      *wait_needed = (writers != nullptr);
+      return;
     }
-    if (ready == last_writer) break;
-  }
-
-  // Notify new head of write queue
-  if (!writers_.empty()) {
-    writers_.front()->cv.Signal();
   }
 }
 
-// This function will be called only when the first writer succeeds.
-// All writers in the to-be-built batch group will be processed.
-//
-// REQUIRES: Writer list must be non-empty
-// REQUIRES: First writer must have a non-nullptr batch
-size_t WriteThread::BuildBatchGroup(
-    WriteThread::Writer** last_writer,
-    autovector<WriteBatch*>* write_batch_group) {
-  assert(!writers_.empty());
-  Writer* first = writers_.front();
-  assert(first->batch != nullptr);
+void WriteThread::CreateMissingNewerLinks(Writer* head) {
+  while (true) {
+    Writer* next = head->link_older;
+    if (next == nullptr || next->link_newer != nullptr) {
+      assert(next == nullptr || next->link_newer == head);
+      break;
+    }
+    next->link_newer = head;
+    head = next;
+  }
+}
 
-  size_t size = WriteBatchInternal::ByteSize(first->batch);
-  write_batch_group->push_back(first->batch);
+void WriteThread::JoinBatchGroup(Writer* w) {
+  assert(w->batch != nullptr);
+  bool wait_needed;
+  LinkOne(w, &wait_needed);
+  if (wait_needed) {
+    Await(w);
+  }
+}
+
+size_t WriteThread::EnterAsBatchGroupLeader(
+    Writer* leader, WriteThread::Writer** last_writer,
+    autovector<WriteBatch*>* write_batch_group) {
+  assert(leader->link_older == nullptr);
+  assert(leader->batch != nullptr);
+
+  size_t size = WriteBatchInternal::ByteSize(leader->batch);
+  write_batch_group->push_back(leader->batch);
 
   // Allow the group to grow up to a maximum size, but if the
   // original write is small, limit the growth so we do not slow
   // down the small write too much.
   size_t max_size = 1 << 20;
-  if (size <= (128<<10)) {
-    max_size = size + (128<<10);
+  if (size <= (128 << 10)) {
+    max_size = size + (128 << 10);
   }
 
-  *last_writer = first;
+  *last_writer = leader;
 
-  if (first->has_callback) {
+  if (leader->has_callback) {
     // TODO(agiardullo:) Batching not currently supported as this write may
     // fail if the callback function decides to abort this write.
     return size;
   }
 
-  std::deque<Writer*>::iterator iter = writers_.begin();
-  ++iter;  // Advance past "first"
-  for (; iter != writers_.end(); ++iter) {
-    Writer* w = *iter;
-    if (w->sync && !first->sync) {
+  Writer* newest_writer = newest_writer_.load(std::memory_order_acquire);
+
+  // This is safe regardless of any db mutex status of the caller. Previous
+  // calls to ExitAsGroupLeader either didn't call CreateMissingNewerLinks
+  // (they emptied the list and then we added ourself as leader) or had to
+  // explicitly wake up us (the list was non-empty when we added ourself,
+  // so we have already received our MarkJoined).
+  CreateMissingNewerLinks(newest_writer);
+
+  // Tricky. Iteration start (leader) is exclusive and finish
+  // (newest_writer) is inclusive. Iteration goes from old to new.
+  Writer* w = leader;
+  while (w != newest_writer) {
+    w = w->link_newer;
+
+    if (w->sync && !leader->sync) {
       // Do not include a sync write into a batch handled by a non-sync write.
       break;
     }
 
-    if (!w->disableWAL && first->disableWAL) {
+    if (!w->disableWAL && leader->disableWAL) {
       // Do not include a write that needs WAL into a batch that has
       // WAL disabled.
       break;
@@ -113,4 +133,68 @@ size_t WriteThread::BuildBatchGroup(
   return size;
 }
 
+void WriteThread::ExitAsBatchGroupLeader(Writer* leader, Writer* last_writer,
+                                         Status status) {
+  assert(leader->link_older == nullptr);
+
+  Writer* head = newest_writer_.load(std::memory_order_acquire);
+  if (head != last_writer ||
+      !newest_writer_.compare_exchange_strong(head, nullptr)) {
+    // Either w wasn't the head during the load(), or it was the head
+    // during the load() but somebody else pushed onto the list before
+    // we did the compare_exchange_strong (causing it to fail).  In the
+    // latter case compare_exchange_strong has the effect of re-reading
+    // its first param (head).  No need to retry a failing CAS, because
+    // only a departing leader (which we are at the moment) can remove
+    // nodes from the list.
+    assert(head != last_writer);
+
+    // After walking link_older starting from head (if not already done)
+    // we will be able to traverse w->link_newer below. This function
+    // can only be called from an active leader, only a leader can
+    // clear newest_writer_, we didn't, and only a clear newest_writer_
+    // could cause the next leader to start their work without a call
+    // to MarkJoined, so we can definitely conclude that no other leader
+    // work is going on here (with or without db mutex).
+    CreateMissingNewerLinks(head);
+    assert(last_writer->link_newer->link_older == last_writer);
+    last_writer->link_newer->link_older = nullptr;
+
+    // Next leader didn't self-identify, because newest_writer_ wasn't
+    // nullptr when they enqueued (we were definitely enqueued before them
+    // and are still in the list).  That means leader handoff occurs when
+    // we call MarkJoined
+    MarkJoined(last_writer->link_newer);
+  }
+  // else nobody else was waiting, although there might already be a new
+  // leader now
+
+  while (last_writer != leader) {
+    last_writer->status = status;
+    last_writer->done = true;
+    // We must read link_older before calling MarkJoined, because as
+    // soon as it is marked the other thread's AwaitJoined may return
+    // and deallocate the Writer.
+    auto next = last_writer->link_older;
+    MarkJoined(last_writer);
+    last_writer = next;
+  }
+}
+
+void WriteThread::EnterUnbatched(Writer* w, InstrumentedMutex* mu) {
+  assert(w->batch == nullptr);
+  bool wait_needed;
+  LinkOne(w, &wait_needed);
+  if (wait_needed) {
+    mu->Unlock();
+    Await(w);
+    mu->Lock();
+  }
+}
+
+void WriteThread::ExitUnbatched(Writer* w) {
+  Status dummy_status;
+  ExitAsBatchGroupLeader(w, w, dummy_status);
+}
+
 }  // namespace rocksdb

db/write_thread.h

@@ -5,76 +5,145 @@
 
 #pragma once
 
+#include <assert.h>
 #include <stdint.h>
-#include <deque>
-#include <limits>
+#include <atomic>
+#include <condition_variable>
+#include <mutex>
+#include <type_traits>
 #include "rocksdb/status.h"
 #include "db/write_batch_internal.h"
 #include "util/autovector.h"
-#include "port/port.h"
 #include "util/instrumented_mutex.h"
 
 namespace rocksdb {
 
 class WriteThread {
  public:
-  // Information kept for every waiting writer
+  // Information kept for every waiting writer.
   struct Writer {
-    Status status;
     WriteBatch* batch;
     bool sync;
     bool disableWAL;
     bool in_batch_group;
     bool done;
     bool has_callback;
-    InstrumentedCondVar cv;
+    Status status;
+    bool made_waitable;  // records lazy construction of mutex and cv
+    bool joined;         // read/write only under JoinMutex() (or pre-link)
+    std::aligned_storage<sizeof(std::mutex)>::type join_mutex_bytes;
+    std::aligned_storage<sizeof(std::condition_variable)>::type join_cv_bytes;
+    Writer* link_older;  // read/write only before linking, or as leader
+    Writer* link_newer;  // lazy, read/write only before linking, or as leader
 
-    explicit Writer(InstrumentedMutex* mu)
+    Writer()
         : batch(nullptr),
           sync(false),
           disableWAL(false),
           in_batch_group(false),
           done(false),
           has_callback(false),
-          cv(mu) {}
+          made_waitable(false),
+          joined(false),
+          link_older(nullptr),
+          link_newer(nullptr) {}
+
+    ~Writer() {
+      if (made_waitable) {
+        JoinMutex().~mutex();
+        JoinCV().~condition_variable();
+      }
+    }
+
+    void CreateMutex() {
+      assert(!joined);
+      if (!made_waitable) {
+        made_waitable = true;
+        new (&join_mutex_bytes) std::mutex;
+        new (&join_cv_bytes) std::condition_variable;
+      }
+    }
+
+    // No other mutexes may be acquired while holding JoinMutex(), it is
+    // always last in the order
+    std::mutex& JoinMutex() {
+      assert(made_waitable);
+      return *static_cast<std::mutex*>(static_cast<void*>(&join_mutex_bytes));
+    }
+
+    std::condition_variable& JoinCV() {
+      assert(made_waitable);
+      return *static_cast<std::condition_variable*>(
+          static_cast<void*>(&join_cv_bytes));
+    }
   };
 
-  WriteThread() = default;
-  ~WriteThread() = default;
+  WriteThread() : newest_writer_(nullptr) {}
 
-  // Before applying write operation (such as DBImpl::Write, DBImpl::Flush)
-  // thread should grab the mutex_ and be the first on writers queue.
-  // EnterWriteThread is used for it.
-  // Be aware! Writer's job can be done by other thread (see DBImpl::Write
-  // for examples), so check it via w.done before applying changes.
+  // IMPORTANT: None of the methods in this class rely on the db mutex
+  // for correctness. All of the methods except JoinBatchGroup and
+  // EnterUnbatched may be called either with or without the db mutex held.
+  // Correctness is maintained by ensuring that only a single thread is
+  // a leader at a time.
+
+  // Registers w as ready to become part of a batch group, and blocks
+  // until some other thread has completed the write (in which case
+  // w->done will be set to true) or this write has become the leader
+  // of a batch group (w->done will remain unset).  The db mutex SHOULD
+  // NOT be held when calling this function, because it will block.
+  // If !w->done then JoinBatchGroup should be followed by a call to
+  // EnterAsBatchGroupLeader and ExitAsBatchGroupLeader.
   //
-  // Writer* w:                writer to be placed in the queue
-  // See also: ExitWriteThread
-  // REQUIRES: db mutex held
-  void EnterWriteThread(Writer* w);
+  // Writer* w:        Writer to be executed as part of a batch group
+  void JoinBatchGroup(Writer* w);
 
-  // After doing write job, we need to remove already used writers from
-  // writers_ queue and notify head of the queue about it.
-  // ExitWriteThread is used for this.
+  // Constructs a write batch group led by leader, which should be a
+  // Writer passed to JoinBatchGroup on the current thread.
   //
-  // Writer* w:           Writer, that was added by EnterWriteThread function
-  // Writer* last_writer: Since we can join a few Writers (as DBImpl::Write
-  //                      does)
-  //                      we should pass last_writer as a parameter to
-  //                      ExitWriteThread
-  //                      (if you don't touch other writers, just pass w)
-  // Status status:       Status of write operation
-  // See also: EnterWriteThread
-  // REQUIRES: db mutex held
-  void ExitWriteThread(Writer* w, Writer* last_writer, Status status);
+  // Writer* leader:         Writer passed to JoinBatchGroup, but !done
+  // Writer** last_writer:   Out-param for use by ExitAsBatchGroupLeader
+  // autovector<WriteBatch*>* write_batch_group: Out-param of group members
+  // returns:                Total batch group size
+  size_t EnterAsBatchGroupLeader(Writer* leader, Writer** last_writer,
+                                 autovector<WriteBatch*>* write_batch_group);
 
-  // return total batch group size
-  size_t BuildBatchGroup(Writer** last_writer,
-                         autovector<WriteBatch*>* write_batch_group);
+  // Unlinks the Writer-s in a batch group, wakes up the non-leaders, and
+  // wakes up the next leader (if any).
+  //
+  // Writer* leader:         From EnterAsBatchGroupLeader
+  // Writer* last_writer:    Value of out-param of EnterAsBatchGroupLeader
+  // Status status:          Status of write operation
+  void ExitAsBatchGroupLeader(Writer* leader, Writer* last_writer,
+                              Status status);
+
+  // Waits for all preceding writers (unlocking mu while waiting), then
+  // registers w as the currently proceeding writer.
+  //
+  // Writer* w:              A Writer not eligible for batching
+  // InstrumentedMutex* mu:  The db mutex, to unlock while waiting
+  // REQUIRES: db mutex held
+  void EnterUnbatched(Writer* w, InstrumentedMutex* mu);
+
+  // Completes a Writer begun with EnterUnbatched, unblocking subsequent
+  // writers.
+  void ExitUnbatched(Writer* w);
 
  private:
-  // Queue of writers.
-  std::deque<Writer*> writers_;
+  // Points to the newest pending Writer.  Only leader can remove
+  // elements, adding can be done lock-free by anybody
+  std::atomic<Writer*> newest_writer_;
+
+  void Await(Writer* w);
+  void MarkJoined(Writer* w);
+
+  // Links w into the newest_writer_ list. Sets *wait_needed to false
+  // if w was linked directly into the leader position, true otherwise.
+  // Safe to call from multiple threads without external locking.
+  void LinkOne(Writer* w, bool* wait_needed);
+
+  // Computes any missing link_newer links.  Should not be called
+  // concurrently with itself.
+  void CreateMissingNewerLinks(Writer* head);
 };
 
 }  // namespace rocksdb

examples/.gitignore

@@ -3,3 +3,4 @@ simple_example
 c_simple_example
 compact_files_example
 transaction_example
+optimistic_transaction_example

examples/Makefile

@@ -1,26 +1,32 @@
 include ../make_config.mk
 
-.PHONY: clean
+.PHONY: clean librocksdb
 
-all: simple_example column_families_example compact_files_example c_simple_example transaction_example
+all: simple_example column_families_example compact_files_example c_simple_example optimistic_transaction_example transaction_example
 
-simple_example: simple_example.cc
+simple_example: librocksdb simple_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
-column_families_example: column_families_example.cc
+column_families_example: librocksdb column_families_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
-compact_files_example: compact_files_example.cc
+compact_files_example: librocksdb compact_files_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
 .c.o:
 	$(CC) $(CFLAGS) -c $< -o $@ -I../include
 
-c_simple_example: c_simple_example.o
+c_simple_example: librocksdb c_simple_example.o
 	$(CXX) $@.o -o$@ ../librocksdb.a $(PLATFORM_LDFLAGS) $(EXEC_LDFLAGS)
 
-transaction_example: transaction_example.cc
+optimistic_transaction_example: librocksdb optimistic_transaction_example.cc
+	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
+
+transaction_example: librocksdb transaction_example.cc
 	$(CXX) $(CXXFLAGS) $@.cc -o$@ ../librocksdb.a -I../include -O2 -std=c++11 $(PLATFORM_LDFLAGS) $(PLATFORM_CXXFLAGS) $(EXEC_LDFLAGS)
 
 clean:
-	rm -rf ./simple_example ./column_families_example ./compact_files_example ./c_simple_example c_simple_example.o ./transaction_example
+	rm -rf ./simple_example ./column_families_example ./compact_files_example ./c_simple_example c_simple_example.o ./optimistic_transaction_example ./transaction_example
+
+librocksdb:
+	cd .. && $(MAKE) librocksdb.a

examples/compact_files_example.cc

@@ -67,9 +67,9 @@ class FullCompactor : public Compactor {
         options_.target_file_size_base;
   }
 
-  // When flush happens, it determins whether to trigger compaction.
-  // If triggered_writes_stop is true, it will also set the retry
-  // flag of compaction-task to true.
+  // When flush happens, it determines whether to trigger compaction. If
+  // triggered_writes_stop is true, it will also set the retry flag of
+  // compaction-task to true.
   void OnFlushCompleted(
       DB* db, const FlushJobInfo& info) override {
     CompactionTask* task = PickCompaction(db, info.cf_name);
@@ -108,7 +108,8 @@ class FullCompactor : public Compactor {
   }
 
   static void CompactFiles(void* arg) {
-    CompactionTask* task = reinterpret_cast<CompactionTask*>(arg);
+    std::unique_ptr<CompactionTask> task(
+        reinterpret_cast<CompactionTask*>(arg));
     assert(task);
     assert(task->db);
     Status s = task->db->CompactFiles(
@@ -124,8 +125,6 @@ class FullCompactor : public Compactor {
           task->db, task->column_family_name);
       task->compactor->ScheduleCompaction(new_task);
     }
-    // release the task
-    delete task;
   }
 
  private:

examples/optimistic_transaction_example.cc

@@ -0,0 +1,142 @@
+// Copyright (c) 2015, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+#ifndef ROCKSDB_LITE
+
+#include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/utilities/transaction.h"
+#include "rocksdb/utilities/optimistic_transaction_db.h"
+
+using namespace rocksdb;
+
+std::string kDBPath = "/tmp/rocksdb_transaction_example";
+
+int main() {
+  // open DB
+  Options options;
+  options.create_if_missing = true;
+  DB* db;
+  OptimisticTransactionDB* txn_db;
+
+  Status s = OptimisticTransactionDB::Open(options, kDBPath, &txn_db);
+  assert(s.ok());
+  db = txn_db->GetBaseDB();
+
+  WriteOptions write_options;
+  ReadOptions read_options;
+  OptimisticTransactionOptions txn_options;
+  std::string value;
+
+  ////////////////////////////////////////////////////////
+  //
+  // Simple OptimisticTransaction Example ("Read Committed")
+  //
+  ////////////////////////////////////////////////////////
+
+  // Start a transaction
+  Transaction* txn = txn_db->BeginTransaction(write_options);
+  assert(txn);
+
+  // Read a key in this transaction
+  s = txn->Get(read_options, "abc", &value);
+  assert(s.IsNotFound());
+
+  // Write a key in this transaction
+  txn->Put("abc", "def");
+
+  // Read a key OUTSIDE this transaction. Does not affect txn.
+  s = db->Get(read_options, "abc", &value);
+
+  // Write a key OUTSIDE of this transaction.
+  // Does not affect txn since this is an unrelated key.  If we wrote key 'abc'
+  // here, the transaction would fail to commit.
+  s = db->Put(write_options, "xyz", "zzz");
+
+  // Commit transaction
+  s = txn->Commit();
+  assert(s.ok());
+  delete txn;
+
+  ////////////////////////////////////////////////////////
+  //
+  // "Repeatable Read" (Snapshot Isolation) Example
+  //   -- Using a single Snapshot
+  //
+  ////////////////////////////////////////////////////////
+
+  // Set a snapshot at start of transaction by setting set_snapshot=true
+  txn_options.set_snapshot = true;
+  txn = txn_db->BeginTransaction(write_options, txn_options);
+
+  const Snapshot* snapshot = txn->GetSnapshot();
+
+  // Write a key OUTSIDE of transaction
+  db->Put(write_options, "abc", "xyz");
+
+  // Read a key using the snapshot
+  read_options.snapshot = snapshot;
+  s = txn->GetForUpdate(read_options, "abc", &value);
+  assert(value == "def");
+
+  // Attempt to commit transaction
+  s = txn->Commit();
+
+  // Transaction could not commit since the write outside of the txn conflicted
+  // with the read!
+  assert(s.IsBusy());
+
+  delete txn;
+  // Clear snapshot from read options since it is no longer valid
+  read_options.snapshot = nullptr;
+  snapshot = nullptr;
+
+  ////////////////////////////////////////////////////////
+  //
+  // "Read Committed" (Monotonic Atomic Views) Example
+  //   --Using multiple Snapshots
+  //
+  ////////////////////////////////////////////////////////
+
+  // In this example, we set the snapshot multiple times.  This is probably
+  // only necessary if you have very strict isolation requirements to
+  // implement.
+
+  // Set a snapshot at start of transaction
+  txn_options.set_snapshot = true;
+  txn = txn_db->BeginTransaction(write_options, txn_options);
+
+  // Do some reads and writes to key "x"
+  read_options.snapshot = db->GetSnapshot();
+  s = txn->Get(read_options, "x", &value);
+  txn->Put("x", "x");
+
+  // Do a write outside of the transaction to key "y"
+  s = db->Put(write_options, "y", "y");
+
+  // Set a new snapshot in the transaction
+  txn->SetSnapshot();
+  read_options.snapshot = db->GetSnapshot();
+
+  // Do some reads and writes to key "y"
+  s = txn->GetForUpdate(read_options, "y", &value);
+  txn->Put("y", "y");
+
+  // Commit.  Since the snapshot was advanced, the write done outside of the
+  // transaction does not prevent this transaction from Committing.
+  s = txn->Commit();
+  assert(s.ok());
+  delete txn;
+  // Clear snapshot from read options since it is no longer valid
+  read_options.snapshot = nullptr;
+
+  // Cleanup
+  delete txn_db;
+  DestroyDB(kDBPath, options);
+  return 0;
+}
+
+#endif  // ROCKSDB_LITE

examples/transaction_example.cc

@@ -8,8 +8,8 @@
 #include "rocksdb/db.h"
 #include "rocksdb/options.h"
 #include "rocksdb/slice.h"
-#include "rocksdb/utilities/optimistic_transaction.h"
-#include "rocksdb/utilities/optimistic_transaction_db.h"
+#include "rocksdb/utilities/transaction.h"
+#include "rocksdb/utilities/transaction_db.h"
 
 using namespace rocksdb;
 
@@ -18,17 +18,16 @@ std::string kDBPath = "/tmp/rocksdb_transaction_example";
 int main() {
   // open DB
   Options options;
+  TransactionDBOptions txn_db_options;
   options.create_if_missing = true;
-  DB* db;
-  OptimisticTransactionDB* txn_db;
+  TransactionDB* txn_db;
 
-  Status s = OptimisticTransactionDB::Open(options, kDBPath, &txn_db);
+  Status s = TransactionDB::Open(options, txn_db_options, kDBPath, &txn_db);
   assert(s.ok());
-  db = txn_db->GetBaseDB();
 
   WriteOptions write_options;
   ReadOptions read_options;
-  OptimisticTransactionOptions txn_options;
+  TransactionOptions txn_options;
   std::string value;
 
   ////////////////////////////////////////////////////////
@@ -38,7 +37,7 @@ int main() {
   ////////////////////////////////////////////////////////
 
   // Start a transaction
-  OptimisticTransaction* txn = txn_db->BeginTransaction(write_options);
+  Transaction* txn = txn_db->BeginTransaction(write_options);
   assert(txn);
 
   // Read a key in this transaction
@@ -46,15 +45,16 @@ int main() {
   assert(s.IsNotFound());
 
   // Write a key in this transaction
-  txn->Put("abc", "def");
+  s = txn->Put("abc", "def");
+  assert(s.ok());
 
   // Read a key OUTSIDE this transaction. Does not affect txn.
-  s = db->Get(read_options, "abc", &value);
+  s = txn_db->Get(read_options, "abc", &value);
 
   // Write a key OUTSIDE of this transaction.
   // Does not affect txn since this is an unrelated key.  If we wrote key 'abc'
   // here, the transaction would fail to commit.
-  s = db->Put(write_options, "xyz", "zzz");
+  s = txn_db->Put(write_options, "xyz", "zzz");
 
   // Commit transaction
   s = txn->Commit();
@@ -75,20 +75,17 @@ int main() {
   const Snapshot* snapshot = txn->GetSnapshot();
 
   // Write a key OUTSIDE of transaction
-  db->Put(write_options, "abc", "xyz");
+  s = txn_db->Put(write_options, "abc", "xyz");
+  assert(s.ok());
 
-  // Read a key using the snapshot
+  // Attempt to read a key using the snapshot.  This will fail since
+  // the previous write outside this txn conflicts with this read.
   read_options.snapshot = snapshot;
   s = txn->GetForUpdate(read_options, "abc", &value);
-  assert(value == "def");
-
-  // Attempt to commit transaction
-  s = txn->Commit();
-
-  // Transaction could not commit since the write outside of the txn conflicted
-  // with the read!
   assert(s.IsBusy());
 
+  txn->Rollback();
+
   delete txn;
   // Clear snapshot from read options since it is no longer valid
   read_options.snapshot = nullptr;
@@ -110,23 +107,28 @@ int main() {
   txn = txn_db->BeginTransaction(write_options, txn_options);
 
   // Do some reads and writes to key "x"
-  read_options.snapshot = db->GetSnapshot();
+  read_options.snapshot = txn_db->GetSnapshot();
   s = txn->Get(read_options, "x", &value);
   txn->Put("x", "x");
 
   // Do a write outside of the transaction to key "y"
-  s = db->Put(write_options, "y", "y");
+  s = txn_db->Put(write_options, "y", "y");
 
   // Set a new snapshot in the transaction
   txn->SetSnapshot();
-  read_options.snapshot = db->GetSnapshot();
+  txn->SetSavePoint();
+  read_options.snapshot = txn_db->GetSnapshot();
 
   // Do some reads and writes to key "y"
+  // Since the snapshot was advanced, the write done outside of the
+  // transaction does not conflict.
   s = txn->GetForUpdate(read_options, "y", &value);
   txn->Put("y", "y");
 
-  // Commit.  Since the snapshot was advanced, the write done outside of the
-  // transaction does not prevent this transaction from Committing.
+  // Decide we want to revert the last write from this transaction.
+  txn->RollbackToSavePoint();
+
+  // Commit.
   s = txn->Commit();
   assert(s.ok());
   delete txn;

hdfs/env_hdfs.h

@@ -66,10 +66,6 @@ class HdfsEnv : public Env {
                                  std::unique_ptr<WritableFile>* result,
                                  const EnvOptions& options);
 
-  virtual Status NewRandomRWFile(const std::string& fname,
-                                 std::unique_ptr<RandomRWFile>* result,
-                                 const EnvOptions& options);
-
   virtual Status NewDirectory(const std::string& name,
                               std::unique_ptr<Directory>* result);
 
@@ -268,12 +264,6 @@ class HdfsEnv : public Env {
     return notsup;
   }
 
-  virtual Status NewRandomRWFile(const std::string& fname,
-                                 unique_ptr<RandomRWFile>* result,
-                                 const EnvOptions& options) override {
-    return notsup;
-  }
-
   virtual Status NewDirectory(const std::string& name,
                               unique_ptr<Directory>* result) override {
     return notsup;

include/rocksdb/compaction_job_stats.h

@@ -12,6 +12,8 @@ namespace rocksdb {
 struct CompactionJobStats {
   CompactionJobStats() { Reset(); }
   void Reset();
+  // Aggregate the CompactionJobStats from another instance with this one
+  void Add(const CompactionJobStats& stats);
 
   // the elapsed time in micro of this compaction.
   uint64_t elapsed_micros;
@@ -58,6 +60,21 @@ struct CompactionJobStats {
   // the key) encountered and written out.
   uint64_t num_corrupt_keys;
 
+  // Following counters are only populated if
+  // options.compaction_measure_io_stats = true;
+
+  // Time spent on file's Append() call.
+  uint64_t file_write_nanos;
+
+  // Time spent on sync file range.
+  uint64_t file_range_sync_nanos;
+
+  // Time spent on file fsync.
+  uint64_t file_fsync_nanos;
+
+  // Time spent on preparing file write (falocate, etc)
+  uint64_t file_prepare_write_nanos;
+
   // 0-terminated strings storing the first 8 bytes of the smallest and
   // largest key in the output.
   static const size_t kMaxPrefixLength = 8;

include/rocksdb/comparator.h

@@ -29,6 +29,15 @@ class Comparator {
   //   > 0 iff "a" > "b"
   virtual int Compare(const Slice& a, const Slice& b) const = 0;
 
+  // Compares two slices for equality. The following invariant should always
+  // hold (and is the default implementation):
+  //   Equal(a, b) iff Compare(a, b) == 0
+  // Overwrite only if equality comparisons can be done more efficiently than
+  // three-way comparisons.
+  virtual bool Equal(const Slice& a, const Slice& b) const {
+    return Compare(a, b) == 0;
+  }
+
   // The name of the comparator.  Used to check for comparator
   // mismatches (i.e., a DB created with one comparator is
   // accessed using a different comparator.

include/rocksdb/convenience.h

@@ -48,6 +48,12 @@ Status GetDBOptionsFromString(
     const std::string& opts_str,
     DBOptions* new_options);
 
+Status GetStringFromDBOptions(const DBOptions& db_options,
+                              std::string* opts_str);
+
+Status GetStringFromColumnFamilyOptions(const ColumnFamilyOptions& db_options,
+                                        std::string* opts_str);
+
 Status GetBlockBasedTableOptionsFromString(
     const BlockBasedTableOptions& table_options,
     const std::string& opts_str,

include/rocksdb/db.h

@@ -22,6 +22,7 @@
 #include "rocksdb/types.h"
 #include "rocksdb/transaction_log.h"
 #include "rocksdb/listener.h"
+#include "rocksdb/snapshot.h"
 #include "rocksdb/thread_status.h"
 
 #ifdef _WIN32
@@ -68,18 +69,6 @@ struct ColumnFamilyDescriptor {
 static const int kMajorVersion = __ROCKSDB_MAJOR__;
 static const int kMinorVersion = __ROCKSDB_MINOR__;
 
-// Abstract handle to particular state of a DB.
-// A Snapshot is an immutable object and can therefore be safely
-// accessed from multiple threads without any external synchronization.
-class Snapshot {
- public:
-  // returns Snapshot's sequence number
-  virtual SequenceNumber GetSequenceNumber() const = 0;
-
- protected:
-  virtual ~Snapshot();
-};
-
 // A range of keys
 struct Range {
   Slice start;          // Included in the range
@@ -323,22 +312,34 @@ class DB {
   //  "rocksdb.compaction-pending" - 1 if at least one compaction is pending
   //  "rocksdb.background-errors" - accumulated number of background errors
   //  "rocksdb.cur-size-active-mem-table"
-  //  "rocksdb.cur-size-all-mem-tables"
-  //  "rocksdb.num-entries-active-mem-table"
-  //  "rocksdb.num-entries-imm-mem-tables"
-  //  "rocksdb.num-deletes-active-mem-table"
-  //  "rocksdb.num-deletes-imm-mem-tables"
-  //  "rocksdb.estimate-num-keys" - estimated keys in the column family
-  //  "rocksdb.estimate-table-readers-mem" - estimated memory used for reding
-  //      SST tables, that is not counted as a part of block cache.
-  //  "rocksdb.is-file-deletions-enabled"
-  //  "rocksdb.num-snapshots"
-  //  "rocksdb.oldest-snapshot-time"
-  //  "rocksdb.num-live-versions" - `version` is an internal data structure.
-  //      See version_set.h for details. More live versions often mean more SST
-  //      files are held from being deleted, by iterators or unfinished
-  //      compactions.
-  //  "rocksdb.estimate-live-data-size"
+//  "rocksdb.size-all-mem-tables"
+//  "rocksdb.num-entries-active-mem-table"
+//  "rocksdb.num-entries-imm-mem-tables"
+//  "rocksdb.num-deletes-active-mem-table"
+//  "rocksdb.num-deletes-imm-mem-tables"
+//  "rocksdb.estimate-num-keys" - estimated keys in the column family
+//  "rocksdb.estimate-table-readers-mem" - estimated memory used for reding
+//      SST tables, that is not counted as a part of block cache.
+//  "rocksdb.is-file-deletions-enabled"
+//  "rocksdb.num-snapshots"
+//  "rocksdb.oldest-snapshot-time"
+//  "rocksdb.num-live-versions" - `version` is an internal data structure.
+//      See version_set.h for details. More live versions often mean more SST
+//      files are held from being deleted, by iterators or unfinished
+//      compactions.
+//  "rocksdb.estimate-live-data-size"
+//  "rocksdb.total-sst-files-size" - total size of all used sst files, this may
+//      slow down online queries if there are too many files.
+//  "rocksdb.base-level"
+//  "rocksdb.estimate-pending-compaction-bytes" - estimated total number of
+//      bytes compaction needs to rewrite the data to get all levels down
+//      to under target size. Not valid for other compactions than level-based.
+//  "rocksdb.aggregated-table-properties" - returns a string representation of
+//      the aggregated table properties of the target column family.
+//  "rocksdb.aggregated-table-properties-at-level<N>", same as the previous
+//      one but only returns the aggregated table properties of the specified
+//      level "N" at the target column family.
+//  replaced by the target level.
 #ifndef ROCKSDB_LITE
   struct Properties {
     static const std::string kNumFilesAtLevelPrefix;
@@ -352,6 +353,7 @@ class DB {
     static const std::string kBackgroundErrors;
     static const std::string kCurSizeActiveMemTable;
     static const std::string kCurSizeAllMemTables;
+    static const std::string kSizeAllMemTables;
     static const std::string kNumEntriesActiveMemTable;
     static const std::string kNumEntriesImmMemTables;
     static const std::string kNumDeletesActiveMemTable;
@@ -363,6 +365,10 @@ class DB {
     static const std::string kOldestSnapshotTime;
     static const std::string kNumLiveVersions;
     static const std::string kEstimateLiveDataSize;
+    static const std::string kTotalSstFilesSize;
+    static const std::string kEstimatePendingCompactionBytes;
+    static const std::string kAggregatedTableProperties;
+    static const std::string kAggregatedTablePropertiesAtLevel;
   };
 #endif /* ROCKSDB_LITE */
 
@@ -381,6 +387,7 @@ class DB {
   //  "rocksdb.background-errors"
   //  "rocksdb.cur-size-active-mem-table"
   //  "rocksdb.cur-size-all-mem-tables"
+  //  "rocksdb.size-all-mem-tables"
   //  "rocksdb.num-entries-active-mem-table"
   //  "rocksdb.num-entries-imm-mem-tables"
   //  "rocksdb.num-deletes-active-mem-table"
@@ -392,6 +399,9 @@ class DB {
   //  "rocksdb.oldest-snapshot-time"
   //  "rocksdb.num-live-versions"
   //  "rocksdb.estimate-live-data-size"
+  //  "rocksdb.total-sst-files-size"
+  //  "rocksdb.base-level"
+  //  "rocksdb.estimate-pending-compaction-bytes"
   virtual bool GetIntProperty(ColumnFamilyHandle* column_family,
                               const Slice& property, uint64_t* value) = 0;
   virtual bool GetIntProperty(const Slice& property, uint64_t* value) {
@@ -482,10 +492,10 @@ class DB {
     return SetOptions(DefaultColumnFamily(), new_options);
   }
 
-  // CompactFiles() inputs a list of files specified by file numbers
-  // and compacts them to the specified level.  Note that the behavior
-  // is different from CompactRange in that CompactFiles() will
-  // perform the compaction job using the CURRENT thread.
+  // CompactFiles() inputs a list of files specified by file numbers and
+  // compacts them to the specified level. Note that the behavior is different
+  // from CompactRange() in that CompactFiles() performs the compaction job
+  // using the CURRENT thread.
   //
   // @see GetDataBaseMetaData
   // @see GetColumnFamilyMetaData

include/rocksdb/delete_scheduler.h

@@ -35,6 +35,10 @@ class DeleteScheduler {
   // Return a map containing errors that happened in the background thread
   // file_path => error status
   virtual std::map<std::string, Status> GetBackgroundErrors() = 0;
+
+  // Wait for all files being deleteing in the background to finish or for
+  // destructor to be called.
+  virtual void WaitForEmptyTrash() = 0;
 };
 
 // Create a new DeleteScheduler that can be shared among multiple RocksDB

include/rocksdb/env.h

@@ -40,7 +40,6 @@ class RandomAccessFile;
 class SequentialFile;
 class Slice;
 class WritableFile;
-class RandomRWFile;
 class Directory;
 struct DBOptions;
 class RateLimiter;
@@ -137,15 +136,6 @@ class Env {
                                  unique_ptr<WritableFile>* result,
                                  const EnvOptions& options) = 0;
 
-  // Create an object that both reads and writes to a file on
-  // specified offsets (random access). If file already exists,
-  // does not overwrite it. On success, stores a pointer to the
-  // new file in *result and returns OK. On failure stores nullptr
-  // in *result and returns non-OK.
-  virtual Status NewRandomRWFile(const std::string& fname,
-                                 unique_ptr<RandomRWFile>* result,
-                                 const EnvOptions& options) = 0;
-
   // Create an object that represents a directory. Will fail if directory
   // doesn't exist. If the directory exists, it will open the directory
   // and create a new Directory object.
@@ -267,6 +257,8 @@ class Env {
 
   // Returns the number of micro-seconds since some fixed point in time. Only
   // useful for computing deltas of time.
+  // However, it is often used as system time such as in GenericRateLimiter
+  // and other places so a port needs to return system time in order to work.
   virtual uint64_t NowMicros() = 0;
 
   // Returns the number of nano-seconds since some fixed point in time. Only
@@ -467,6 +459,10 @@ class WritableFile {
     return false;
   }
 
+  // Indicates the upper layers if the current WritableFile implementation
+  // uses direct IO.
+  virtual bool UseDirectIO() const { return false; }
+
   /*
    * Change the priority in rate limiter if rate limiting is enabled.
    * If rate limiting is not enabled, this call has no effect.
@@ -568,55 +564,6 @@ class WritableFile {
   Env::IOPriority io_priority_;
 };
 
-// A file abstraction for random reading and writing.
-class RandomRWFile {
- public:
-  RandomRWFile() {}
-  virtual ~RandomRWFile() {}
-
-  // Write data from Slice data to file starting from offset
-  // Returns IOError on failure, but does not guarantee
-  // atomicity of a write.  Returns OK status on success.
-  //
-  // Safe for concurrent use.
-  virtual Status Write(uint64_t offset, const Slice& data) = 0;
-  // Read up to "n" bytes from the file starting at "offset".
-  // "scratch[0..n-1]" may be written by this routine.  Sets "*result"
-  // to the data that was read (including if fewer than "n" bytes were
-  // successfully read).  May set "*result" to point at data in
-  // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
-  // "*result" is used.  If an error was encountered, returns a non-OK
-  // status.
-  //
-  // Safe for concurrent use by multiple threads.
-  virtual Status Read(uint64_t offset, size_t n, Slice* result,
-                      char* scratch) const = 0;
-  virtual Status Close() = 0; // closes the file
-  virtual Status Sync() = 0; // sync data
-
-  /*
-   * Sync data and/or metadata as well.
-   * By default, sync only data.
-   * Override this method for environments where we need to sync
-   * metadata as well.
-   */
-  virtual Status Fsync() {
-    return Sync();
-  }
-
-  /*
-   * Pre-allocate space for a file.
-   */
-  virtual Status Allocate(off_t offset, off_t len) {
-    return Status::OK();
-  }
-
- private:
-  // No copying allowed
-  RandomRWFile(const RandomRWFile&);
-  void operator=(const RandomRWFile&);
-};
-
 // Directory object represents collection of files and implements
 // filesystem operations that can be executed on directories.
 class Directory {
@@ -766,10 +713,6 @@ class EnvWrapper : public Env {
                          const EnvOptions& options) override {
     return target_->NewWritableFile(f, r, options);
   }
-  Status NewRandomRWFile(const std::string& f, unique_ptr<RandomRWFile>* r,
-                         const EnvOptions& options) override {
-    return target_->NewRandomRWFile(f, r, options);
-  }
   virtual Status NewDirectory(const std::string& name,
                               unique_ptr<Directory>* result) override {
     return target_->NewDirectory(name, result);

include/rocksdb/immutable_options.h

@@ -89,6 +89,10 @@ struct ImmutableCFOptions {
 
   Options::AccessHint access_hint_on_compaction_start;
 
+  bool new_table_reader_for_compaction_inputs;
+
+  size_t compaction_readahead_size;
+
   int num_levels;
 
   bool optimize_filters_for_hits;

include/rocksdb/iostats_context.h

@@ -38,7 +38,10 @@ struct IOStatsContext {
   uint64_t read_nanos;
   // time spent in sync_file_range().
   uint64_t range_sync_nanos;
-
+  // time spent in fsync
+  uint64_t fsync_nanos;
+  // time spent in preparing write (fallocate etc).
+  uint64_t prepare_write_nanos;
   // time spent in Logger::Logv().
   uint64_t logger_nanos;
 };

include/rocksdb/merge_operator.h

@@ -6,9 +6,10 @@
 #ifndef STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_
 #define STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_
 
+#include <deque>
 #include <memory>
 #include <string>
-#include <deque>
+
 #include "rocksdb/slice.h"
 
 namespace rocksdb {

include/rocksdb/options.h

@@ -15,7 +15,6 @@
 #include <memory>
 #include <vector>
 #include <limits>
-#include <stdint.h>
 #include <unordered_map>
 
 #include "rocksdb/version.h"
@@ -56,8 +55,14 @@ class InternalKeyComparator;
 enum CompressionType : char {
   // NOTE: do not change the values of existing entries, as these are
   // part of the persistent format on disk.
-  kNoCompression = 0x0, kSnappyCompression = 0x1, kZlibCompression = 0x2,
-  kBZip2Compression = 0x3, kLZ4Compression = 0x4, kLZ4HCCompression = 0x5
+  kNoCompression = 0x0,
+  kSnappyCompression = 0x1,
+  kZlibCompression = 0x2,
+  kBZip2Compression = 0x3,
+  kLZ4Compression = 0x4,
+  kLZ4HCCompression = 0x5,
+  // zstd format is not finalized yet so it's subject to changes.
+  kZSTDNotFinalCompression = 0x40,
 };
 
 enum CompactionStyle : char {
@@ -271,10 +276,9 @@ struct ColumnFamilyOptions {
   // If this value is set to -1, 'max_write_buffer_number' will be used.
   //
   // Default:
-  // If using an OptimisticTransactionDB, the default value will be set to the
-  // value
-  // of 'max_write_buffer_number' if it is not explicitly set by the user.
-  // Otherwise, the default is 0.
+  // If using a TransactionDB/OptimisticTransactionDB, the default value will
+  // be set to the value of 'max_write_buffer_number' if it is not explicitly
+  // set by the user.  Otherwise, the default is 0.
   int max_write_buffer_number_to_maintain;
 
   // Compress blocks using the specified compression algorithm.  This
@@ -510,8 +514,8 @@ struct ColumnFamilyOptions {
   unsigned int rate_limit_delay_max_milliseconds;
 
   // size of one block in arena memory allocation.
-  // If <= 0, a proper value is automatically calculated (usually 1/10 of
-  // writer_buffer_size).
+  // If <= 0, a proper value is automatically calculated (usually 1/8 of
+  // writer_buffer_size, rounded up to a multiple of 4KB).
   //
   // There are two additonal restriction of the The specified size:
   // (1) size should be in the range of [4096, 2 << 30] and
@@ -736,6 +740,10 @@ struct ColumnFamilyOptions {
   // Default: false
   bool paranoid_file_checks;
 
+  // Measure IO stats in compactions, if true.
+  // Default: false
+  bool compaction_measure_io_stats;
+
   // Create ColumnFamilyOptions with default values for all fields
   ColumnFamilyOptions();
   // Create ColumnFamilyOptions from Options
@@ -807,9 +815,14 @@ struct DBOptions {
   // files opened are always kept open. You can estimate number of files based
   // on target_file_size_base and target_file_size_multiplier for level-based
   // compaction. For universal-style compaction, you can usually set it to -1.
-  // Default: 5000
+  // Default: 5000 or ulimit value of max open files (whichever is smaller)
   int max_open_files;
 
+  // If max_open_files is -1, DB will open all files on DB::Open(). You can
+  // use this option to increase the number of threads used to open the files.
+  // Default: 1
+  int max_file_opening_threads;
+
   // Once write-ahead logs exceed this size, we will start forcing the flush of
   // column families whose memtables are backed by the oldest live WAL file
   // (i.e. the ones that are causing all the space amplification). If set to 0
@@ -894,13 +907,10 @@ struct DBOptions {
   int max_background_compactions;
 
   // This integer represents the maximum number of threads that will
-  // concurrently perform a level-based compaction from L0 to L1. A value
-  // of 1 means there is no parallelism, and a greater number enables a
-  // multi-threaded version of the L0-L1 compaction that divides the compaction
-  // into multiple, smaller ones that are run simultaneously. This is still
-  // under development and is only available for level-based compaction.
-  // Default: 1
-  uint32_t num_subcompactions;
+  // concurrently perform a compaction job by breaking it into multiple,
+  // smaller ones that are run simultaneously.
+  // Default: 1 (i.e. no subcompactions)
+  uint32_t max_subcompactions;
 
   // Maximum number of concurrent background memtable flush jobs, submitted to
   // the HIGH priority thread pool.
@@ -1018,6 +1028,28 @@ struct DBOptions {
   };
   AccessHint access_hint_on_compaction_start;
 
+  // If true, always create a new file descriptor and new table reader
+  // for compaction inputs. Turn this parameter on may introduce extra
+  // memory usage in the table reader, if it allocates extra memory
+  // for indexes. This will allow file descriptor prefetch options
+  // to be set for compaction input files and not to impact file
+  // descriptors for the same file used by user queries.
+  // Suggest to enable BlockBasedTableOptions.cache_index_and_filter_blocks
+  // for this mode if using block-based table.
+  //
+  // Default: false
+  bool new_table_reader_for_compaction_inputs;
+
+  // If non-zero, we perform bigger reads when doing compaction. If you're
+  // running RocksDB on spinning disks, you should set this to at least 2MB.
+  // That way RocksDB's compaction is doing sequential instead of random reads.
+  //
+  // When non-zero, we also force new_table_reader_for_compaction_inputs to
+  // true.
+  //
+  // Default: 0
+  size_t compaction_readahead_size;
+
   // Use adaptive mutex, which spins in the user space before resorting
   // to kernel. This could reduce context switch when the mutex is not
   // heavily contended. However, if the mutex is hot, we could end up