table/block.h: change memset

Summary: In gcc-7 the following is an error identified by -Werror=class-memaccess In file included from ./table/get_context.h:14:0, from db/version_set.cc:43: ./table/block.h: In constructor ‘rocksdb::BlockReadAmpBitmap::BlockReadAmpBitmap(size_t, size_t, rocksdb::Statistics*)’: ./table/block.h:73:53: error: ‘void* memset(void*, int, size_t)’ clearing an object of type ‘struct std::atomic<unsigned int>’ with no trivial copy-assignment; use value-initialization instead [-Werror=class-memaccess] memset(bitmap_, 0, bitmap_size * kBytesPersEntry); ^ In file included from ./db/version_set.h:23:0, from db/version_set.cc:12: /toolchain/include/c++/8.0.0/atomic:684:12: note: ‘struct std::atomic<unsigned int>’ declared here struct atomic<unsigned int> : __atomic_base<unsigned int> ^~~~~~~~~~~~~~~~~~~~ As a solution the default initializer can be applied in list context. Signed-off-by: Daniel Black <daniel.black@au.ibm.com> Closes https://github.com/facebook/rocksdb/pull/2561 Differential Revision: D5398714 Pulled By: siying fbshipit-source-id: d883fb88ec7535eee60d551038fe91f14488be36
Fix GitHub issue #3716 : gcc-8 warnings
2026-11-07 11:32:34 -05:00 · 2019-10-30 14:00:55 -07:00 · 2019-10-30 13:54:49 -07:00 · 2019-10-30 13:54:49 -07:00 · 2019-10-30 13:54:49 -07:00 · 2017-11-28 12:25:25 -08:00
235 changed files with 8354 additions and 4494 deletions
@@ -0,0 +1,17 @@
+{
+  "project_id" : "rocksdb",
+  "conduit_uri" : "https://phabricator.fb.com/api/",
+  "copyright_holder" : "Facebook",
+  "load" : [
+    "arcanist_util"
+  ],
+  "lint.engine" : "FacebookFbcodeLintEngine",
+  "lint.engine.single.linter" : "FbcodeCppLinter",
+  "unit.engine" : "FacebookFbcodeUnitTestEngine",
+  "arcanist_configuration" : "FacebookArcanistConfiguration",
+  "base" : "git:HEAD^, hg:.^",
+  "git.default-relative-commit" : "HEAD^",
+  "git:arc.feature.start.default" : "origin/master",
+  "arc.feature.start.default" : "master",
+  "history.immutable" : false
+}
@@ -57,7 +57,7 @@ before_script:

 script:
  - ${CXX} --version
-  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then ccache -C && OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
+  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 make -j4 all; OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
  - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
@@ -174,6 +174,7 @@ else()
  endif()
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    add_definitions(-DNDEBUG)
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
    include(CheckCXXCompilerFlag)
    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
@@ -482,9 +483,6 @@ set(SOURCES
        utilities/blob_db/blob_log_writer.cc
        utilities/blob_db/blob_log_format.cc
        utilities/blob_db/ttl_extractor.cc
-        utilities/cassandra/cassandra_compaction_filter.cc
-        utilities/cassandra/format.cc
-        utilities/cassandra/merge_operator.cc
        utilities/checkpoint/checkpoint_impl.cc
        utilities/col_buf_decoder.cc
        utilities/col_buf_encoder.cc
@@ -503,6 +501,8 @@ set(SOURCES
        utilities/memory/memory_util.cc
        utilities/merge_operators/max.cc
        utilities/merge_operators/put.cc
+        utilities/merge_operators/cassandra/format.cc
+        utilities/merge_operators/cassandra/merge_operator.cc
        utilities/merge_operators/string_append/stringappend.cc
        utilities/merge_operators/string_append/stringappend2.cc
        utilities/merge_operators/uint64add.cc
@@ -518,14 +518,13 @@ set(SOURCES
        utilities/spatialdb/spatial_db.cc
        utilities/table_properties_collectors/compact_on_deletion_collector.cc
        utilities/transactions/optimistic_transaction_db_impl.cc
-        utilities/transactions/optimistic_transaction.cc
+        utilities/transactions/optimistic_transaction_impl.cc
        utilities/transactions/transaction_base.cc
-        utilities/transactions/pessimistic_transaction_db.cc
+        utilities/transactions/transaction_db_impl.cc
        utilities/transactions/transaction_db_mutex_impl.cc
-        utilities/transactions/pessimistic_transaction.cc
+        utilities/transactions/transaction_impl.cc
        utilities/transactions/transaction_lock_mgr.cc
        utilities/transactions/transaction_util.cc
-        utilities/transactions/write_prepared_txn.cc
        utilities/ttl/db_ttl_impl.cc
        utilities/write_batch_with_index/write_batch_with_index.cc
        utilities/write_batch_with_index/write_batch_with_index_internal.cc
@@ -707,10 +706,6 @@ set(TESTS
        util/thread_local_test.cc
        utilities/backupable/backupable_db_test.cc
        utilities/blob_db/blob_db_test.cc
-        utilities/cassandra/cassandra_functional_test.cc
-        utilities/cassandra/cassandra_format_test.cc
-        utilities/cassandra/cassandra_row_merge_test.cc
-        utilities/cassandra/cassandra_serialize_test.cc
        utilities/checkpoint/checkpoint_test.cc
        utilities/column_aware_encoding_test.cc
        utilities/date_tiered/date_tiered_test.cc
@@ -719,6 +714,10 @@ set(TESTS
        utilities/geodb/geodb_test.cc
        utilities/lua/rocks_lua_test.cc
        utilities/memory/memory_test.cc
+        utilities/merge_operators/cassandra/cassandra_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_format_test.cc
+        utilities/merge_operators/cassandra/cassandra_row_merge_test.cc
+        utilities/merge_operators/cassandra/cassandra_serialize_test.cc
        utilities/merge_operators/string_append/stringappend_test.cc
        utilities/object_registry_test.cc
        utilities/option_change_migration/option_change_migration_test.cc
@@ -727,7 +726,6 @@ set(TESTS
        utilities/persistent_cache/persistent_cache_test.cc
        utilities/redis/redis_lists_test.cc
        utilities/spatialdb/spatial_db_test.cc
-        utilities/simulator_cache/sim_cache_test.cc
        utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
        utilities/transactions/optimistic_transaction_test.cc
        utilities/transactions/transaction_test.cc
@@ -760,7 +758,7 @@ set(TESTUTIL_SOURCE
    monitoring/thread_status_updater_debug.cc
    table/mock_table.cc
    util/fault_injection_test_env.cc
-    utilities/cassandra/test_utils.cc
+    utilities/merge_operators/cassandra/test_utils.cc
 )
 # test utilities are only build in debug
 enable_testing()
@@ -1,12 +1,23 @@
 # Rocksdb Change Log
-## Unreleased
-### New Features
-* Add Iterator::Refresh(), which allows users to update the iterator state so that they can avoid some initialization costs of recreating iterators.
-* Replace dynamic_cast<> (except unit test) so people can choose to build with RTTI off. With make, release mode is by default built with -fno-rtti and debug mode is built without it. Users can override it by setting USE_RTTI=0 or 1.
-* Universal compactions including the bottom level can be executed in a dedicated thread pool. This alleviates head-of-line blocking in the compaction queue, which cause write stalling, particularly in multi-instance use cases. Users can enable this feature via `Env::SetBackgroundThreads(N, Env::Priority::BOTTOM)`, where `N > 0`.
-
+## 5.7.5 (11/28/2017)
+### Bug Fixes
+* Fix IOError on WAL write doesn't propagate to write group follower
+
+## 5.7.4 (08/31/2017)
+No significant changes.
+
+## 5.7.3 (08/29/2017)
+### Bug Fixes
+* Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled.
+* Fix potentially wrong file smallest key when range deletions separated by snapshot are written together.
+
+## 5.7.2 (08/15/2017)
+### Bug Fixes
+* Fix incorrect dropping of deletions issue with FIFO compaction.
+* Fix LITE build compiler error with missing abort().
+
+## 5.7.1 (08/13/2017)
 ### Bug Fixes
-* Fix wrong latencies in `rocksdb.db.get.micros`, `rocksdb.db.write.micros`, and `rocksdb.sst.read.micros`.
 * Fix incorrect dropping of deletions during intra-L0 compaction.

 ## 5.7.0 (07/13/2017)
@@ -113,8 +113,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 * **Windows**:
  * For building with MS Visual Studio 13 you will need Update 4 installed.
  * Read and follow the instructions at CMakeLists.txt
-  * Or install via [vcpkg](https://github.com/microsoft/vcpkg) 
-       * run `vcpkg install rocksdb`

 * **AIX 6.1**
    * Install AIX Toolbox rpms with gcc
@@ -101,19 +101,7 @@ endif
 ifeq ($(DEBUG_LEVEL),0)
 OPT += -DNDEBUG
 DISABLE_WARNING_AS_ERROR=1
-
-ifneq ($(USE_RTTI), 1)
-	CXXFLAGS += -fno-rtti
 else
-	CXXFLAGS += -DROCKSDB_USE_RTTI
-endif
-else
-ifneq ($(USE_RTTI), 0)
-	CXXFLAGS += -DROCKSDB_USE_RTTI
-else
-	CXXFLAGS += -fno-rtti
-endif
-
 $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production)
 endif

@@ -232,10 +220,6 @@ ifndef DISABLE_JEMALLOC
 		PLATFORM_CXXFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 		PLATFORM_CCFLAGS  += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
 	endif
-	ifdef WITH_JEMALLOC_FLAG
-		PLATFORM_LDFLAGS += -ljemalloc
-		JAVA_LDFLAGS += -ljemalloc
-	endif
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
 	PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE)
@@ -259,19 +243,6 @@ default: all
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
  -Wno-unused-parameter

-CCVERSION = $(shell $(CXX) -dumpversion)
-CCNAME = $(shell $(CXX) --version | awk 'NR==1' | cut -f1 -d " ")
-
-ifeq ($(CCNAME), clang)
-ifeq ($(CCVERSION), 4*)
-	CXXFLAGS += -faligned-new
-endif
-else
-ifeq ($(CCVERSION), 7)
-	CXXFLAGS += -faligned-new
-endif
-endif
-
 ifndef DISABLE_WARNING_AS_ERROR
 	WARNING_FLAGS += -Werror
 endif
@@ -434,7 +405,7 @@ TESTS = \
 	write_buffer_manager_test \
 	stringappend_test \
 	cassandra_format_test \
-	cassandra_functional_test \
+	cassandra_merge_test \
 	cassandra_row_merge_test \
 	cassandra_serialize_test \
 	ttl_test \
@@ -604,8 +575,6 @@ endif  # PLATFORM_SHARED_EXT

 all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)

-all_but_some_tests: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(SUBSET)
-
 static_lib: $(LIBRARY)

 shared_lib: $(SHARED)
@@ -809,8 +778,8 @@ ldb_tests: ldb
 crash_test: whitebox_crash_test blackbox_crash_test

 blackbox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
-	python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
+	python -u tools/db_crashtest.py --simple blackbox
+	python -u tools/db_crashtest.py blackbox

 ifeq ($(CRASH_TEST_KILL_ODD),)
  CRASH_TEST_KILL_ODD=888887
@@ -818,9 +787,9 @@ endif

 whitebox_crash_test: db_stress
 	python -u tools/db_crashtest.py --simple whitebox --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+      $(CRASH_TEST_KILL_ODD)
 	python -u tools/db_crashtest.py whitebox  --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
+      $(CRASH_TEST_KILL_ODD)

 asan_check:
 	$(MAKE) clean
@@ -1031,16 +1000,16 @@ option_change_migration_test: utilities/option_change_migration/option_change_mi
 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_format_test: utilities/cassandra/cassandra_format_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_format_test: utilities/merge_operators/cassandra/cassandra_format_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_functional_test: utilities/cassandra/cassandra_functional_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_merge_test: utilities/merge_operators/cassandra/cassandra_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_row_merge_test: utilities/cassandra/cassandra_row_merge_test.o utilities/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_row_merge_test: utilities/merge_operators/cassandra/cassandra_row_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-cassandra_serialize_test: utilities/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
+cassandra_serialize_test: utilities/merge_operators/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -6,6 +6,8 @@ REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
  "-fno-builtin-memcmp",
  "-DROCKSDB_PLATFORM_POSIX",
  "-DROCKSDB_LIB_IO_POSIX",
@@ -16,7 +18,6 @@ rocksdb_compiler_flags = [
  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
  "-DHAVE_SSE42",
  "-DOS_LINUX",
-  "-DROCKSDB_UBSAN_RUN",
  # Flags to enable libs we include
  "-DSNAPPY",
  "-DZLIB",
@@ -211,9 +212,6 @@ cpp_library(
      "utilities/blob_db/blob_log_writer.cc",
      "utilities/blob_db/blob_log_format.cc",
      "utilities/blob_db/ttl_extractor.cc",
-      "utilities/cassandra/cassandra_compaction_filter.cc",
-      "utilities/cassandra/format.cc",
-      "utilities/cassandra/merge_operator.cc",
      "utilities/checkpoint/checkpoint_impl.cc",
      "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
      "utilities/convenience/info_log_finder.cc",
@@ -228,6 +226,8 @@ cpp_library(
      "utilities/leveldb_options/leveldb_options.cc",
      "utilities/lua/rocks_lua_compaction_filter.cc",
      "utilities/memory/memory_util.cc",
+      "utilities/merge_operators/cassandra/format.cc",
+      "utilities/merge_operators/cassandra/merge_operator.cc",
      "utilities/merge_operators/max.cc",
      "utilities/merge_operators/put.cc",
      "utilities/merge_operators/string_append/stringappend.cc",
@@ -245,14 +245,13 @@ cpp_library(
      "utilities/spatialdb/spatial_db.cc",
      "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
      "utilities/transactions/optimistic_transaction_db_impl.cc",
-      "utilities/transactions/optimistic_transaction.cc",
+      "utilities/transactions/optimistic_transaction_impl.cc",
      "utilities/transactions/transaction_base.cc",
-      "utilities/transactions/pessimistic_transaction_db.cc",
+      "utilities/transactions/transaction_db_impl.cc",
      "utilities/transactions/transaction_db_mutex_impl.cc",
-      "utilities/transactions/pessimistic_transaction.cc",
+      "utilities/transactions/transaction_impl.cc",
      "utilities/transactions/transaction_lock_mgr.cc",
      "utilities/transactions/transaction_util.cc",
-      "utilities/transactions/write_prepared_txn.cc",
      "utilities/ttl/db_ttl_impl.cc",
      "utilities/write_batch_with_index/write_batch_with_index.cc",
      "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
@@ -276,7 +275,7 @@ cpp_library(
      "util/testharness.cc",
      "util/testutil.cc",
      "db/db_test_util.cc",
-      "utilities/cassandra/test_utils.cc",
+      "utilities/merge_operators/cassandra/test_utils.cc",
      "utilities/col_buf_encoder.cc",
      "utilities/col_buf_decoder.cc",
      "utilities/column_aware_encoding_util.cc",
@@ -326,16 +325,16 @@ ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
 ['c_test', 'db/c_test.c', 'serial'],
 ['cache_test', 'cache/cache_test.cc', 'serial'],
 ['cassandra_format_test',
-  'utilities/cassandra/cassandra_format_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_format_test.cc',
  'serial'],
- ['cassandra_functional_test',
-  'utilities/cassandra/cassandra_functional_test.cc',
+ ['cassandra_merge_test',
+  'utilities/merge_operators/cassandra/cassandra_merge_test.cc',
  'serial'],
 ['cassandra_row_merge_test',
-  'utilities/cassandra/cassandra_row_merge_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_row_merge_test.cc',
  'serial'],
 ['cassandra_serialize_test',
-  'utilities/cassandra/cassandra_serialize_test.cc',
+  'utilities/merge_operators/cassandra/cassandra_serialize_test.cc',
  'serial'],
 ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
 ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
@@ -80,6 +80,3 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.

 ## 360 Pika
 [360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
-
-## LzLabs
-LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
@@ -1,2 +0,0 @@
-arcanist_util are only used internaly, If you want to change it please check
-<internal_rocksdb_repo>/arcanist_util
@@ -0,0 +1,3 @@
+<?php
+
+phutil_register_library('arcanist_util', __FILE__);
@@ -0,0 +1,71 @@
+<?php
+
+/**
+ * This file is automatically generated. Use 'arc liberate' to rebuild it.
+ * @generated
+ * @phutil-library-version 2
+ */
+
+if (class_exists('ArcanistWorkflow')) {
+  phutil_register_library_map(array(
+    '__library_version__' => 2,
+    'class' =>
+    array(
+      'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
+      'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
+      'FacebookArcanistConfiguration' => 'config/FacebookArcanistConfiguration.php',
+      'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
+      'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookFbcodeUnitTestEngine.php',
+      'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
+      'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
+      'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
+      'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
+    ),
+    'function' =>
+    array(
+    ),
+    'xmap' =>
+    array(
+      'ArcanistCpplintLinter' => 'ArcanistLinter',
+      'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
+      'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
+      'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
+      'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+      'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
+      'FacebookHowtoevenLinter' => 'ArcanistLinter',
+      'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
+      'FbcodeCppLinter' => 'ArcanistLinter',
+    ),
+  ));
+} else {
+  phutil_register_library_map(array(
+    '__library_version__' => 2,
+    'class' =>
+    array(
+      'ArcanistCpplintLinter' => 'cpp_linter/ArcanistCpplintLinter.php',
+      'BaseDirectoryScopedFormatLinter' => 'cpp_linter/BaseDirectoryScopedFormatLinter.php',
+      'FacebookArcanistConfiguration' => 'config/FacebookOldArcanistConfiguration.php',
+      'FacebookFbcodeLintEngine' => 'lint_engine/FacebookFbcodeLintEngine.php',
+      'FacebookFbcodeUnitTestEngine' => 'unit_engine/FacebookOldFbcodeUnitTestEngine.php',
+      'FacebookHowtoevenLintEngine' => 'lint_engine/FacebookHowtoevenLintEngine.php',
+      'FacebookHowtoevenLinter' => 'cpp_linter/FacebookHowtoevenLinter.php',
+      'FbcodeClangFormatLinter' => 'cpp_linter/FbcodeClangFormatLinter.php',
+      'FbcodeCppLinter' => 'cpp_linter/FbcodeCppLinter.php',
+    ),
+    'function' =>
+    array(
+    ),
+    'xmap' =>
+    array(
+      'ArcanistCpplintLinter' => 'ArcanistLinter',
+      'BaseDirectoryScopedFormatLinter' => 'ArcanistLinter',
+      'FacebookArcanistConfiguration' => 'ArcanistConfiguration',
+      'FacebookFbcodeLintEngine' => 'ArcanistLintEngine',
+      'FacebookFbcodeUnitTestEngine' => 'ArcanistBaseUnitTestEngine',
+      'FacebookHowtoevenLintEngine' => 'ArcanistLintEngine',
+      'FacebookHowtoevenLinter' => 'ArcanistLinter',
+      'FbcodeClangFormatLinter' => 'BaseDirectoryScopedFormatLinter',
+      'FbcodeCppLinter' => 'ArcanistLinter',
+    ),
+  ));
+}
@@ -0,0 +1,43 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+require('RocksDBCommonHelper.php');
+
+define("DIFF_COMMAND", "diff");
+
+class FacebookArcanistConfiguration extends ArcanistConfiguration {
+  public function getCustomArgumentsForCommand($command) {
+    if ($command == "land") {
+      return array(
+          'async' => array('help' => 'Just to make tools happy'));
+    }
+    return array();
+  }
+
+  public function didRunWorkflow($command,
+                                 ArcanistWorkflow $workflow,
+                                 $error_code) {
+    // Default options don't terminate on failure, but that's what we want. In
+    // the current case we use assertions intentionally as "terminate on failure
+    // invariants".
+    assert_options(ASSERT_BAIL, true);
+
+    assert($workflow);
+    assert(strlen($command) > 0);
+
+    if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
+      $diffID = $workflow->getDiffId();
+
+      // When submitting a diff this code path gets executed multiple times in
+      // a row. We only care about the case when ID for the diff is provided
+      // because that's what we need to apply the diff and trigger the tests.
+      if (strlen($diffID) > 0) {
+        assert(is_numeric($diffID));
+        startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
+      }
+    }
+  }
+}
@@ -0,0 +1,43 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+require('RocksDBCommonHelper.php');
+
+define("DIFF_COMMAND", "diff");
+
+class FacebookArcanistConfiguration extends ArcanistConfiguration {
+  public function getCustomArgumentsForCommand($command) {
+    if ($command == "land") {
+      return array(
+          'async' => array('help' => 'Just to make tools happy'));
+    }
+    return array();
+  }
+
+  public function didRunWorkflow($command,
+                                 ArcanistBaseWorkflow $workflow,
+                                 $error_code) {
+    // Default options don't terminate on failure, but that's what we want. In
+    // the current case we use assertions intentionally as "terminate on failure
+    // invariants".
+    assert_options(ASSERT_BAIL, true);
+
+    assert($workflow);
+    assert(strlen($command) > 0);
+
+    if ($command == DIFF_COMMAND && !$workflow->isRawDiffSource()) {
+      $diffID = $workflow->getDiffId();
+
+      // When submitting a diff this code path gets executed multiple times in
+      // a row. We only care about the case when ID for the diff is provided
+      // because that's what we need to apply the diff and trigger the tests.
+      if (strlen($diffID) > 0) {
+        assert(is_numeric($diffID));
+        startTestsInSandcastle(true /* $applyDiff */, $workflow, $diffID);
+      }
+    }
+  }
+}
@@ -1,8 +1,8 @@
 <?php
 // Copyright 2004-present Facebook. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.

 // Name of the environment variables which need to be set by the entity which
 // triggers continuous runs so that code at the end of the file gets executed
@@ -94,17 +94,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
    "user" => "root"
  );

-  // This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory"
-  // Source:
-  // https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ
-  $fix_kernel_issue = array(
-    "name" => "Fix kernel issue with tsan",
-    "shell" => "echo 2 >/proc/sys/kernel/randomize_va_space",
-    "user" => "root"
-  );
-
  $steps[] = $fix_git_ignore;
-  $steps[] = $fix_kernel_issue;

  // This will be the command used to execute particular type of tests.
  $cmd = "";
@@ -251,7 +241,7 @@ function getSandcastleConfig() {
  // List of tests we want to run in Sandcastle.
  $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
                 "asan", "lite_test", "valgrind", "release", "release_481",
-                 "clang_release", "clang_analyze", "code_cov",
+                 "clang_release", "punit", "clang_analyze", "code_cov",
                 "java_build", "no_compression", "unity", "ubsan");

  $send_email_template = array(
@@ -0,0 +1,88 @@
+<?php
+
+/**
+ * Uses google's cpplint.py to check code. RocksDB team forked this file from
+ * phabricator's /src/lint/linter/ArcanistCpplintLinter.php, and customized it
+ * for its own use.
+ *
+ * You can get it here:
+ * http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
+ * @group linter
+ */
+final class ArcanistCpplintLinter extends ArcanistLinter {
+
+  public function willLintPaths(array $paths) {
+    return;
+  }
+
+  public function getLinterName() {
+    return 'cpplint.py';
+  }
+
+  public function getLintPath() {
+    $bin = 'cpplint.py';
+    // Search under current dir
+    list($err) = exec_manual('which %s/%s', $this->linterDir(), $bin);
+    if (!$err) {
+      return $this->linterDir().'/'.$bin;
+    }
+
+    // Look for globally installed cpplint.py
+    list($err) = exec_manual('which %s', $bin);
+    if ($err) {
+      throw new ArcanistUsageException(
+        "cpplint.py does not appear to be installed on this system. Install ".
+        "it (e.g., with 'wget \"http://google-styleguide.googlecode.com/".
+        "svn/trunk/cpplint/cpplint.py\"') ".
+        "in your .arcconfig to point to the directory where it resides. ".
+        "Also don't forget to chmod a+x cpplint.py!");
+    }
+
+    return $bin;
+  }
+
+  public function lintPath($path) {
+    $bin = $this->getLintPath();
+    $path = $this->rocksdbDir().'/'.$path;
+
+    $f = new ExecFuture("%C $path", $bin);
+
+    list($err, $stdout, $stderr) = $f->resolve();
+
+    if ($err === 2) {
+      throw new Exception("cpplint failed to run correctly:\n".$stderr);
+    }
+
+    $lines = explode("\n", $stderr);
+    $messages = array();
+    foreach ($lines as $line) {
+      $line = trim($line);
+      $matches = null;
+      $regex = '/^[^:]+:(\d+):\s*(.*)\s*\[(.*)\] \[(\d+)\]$/';
+      if (!preg_match($regex, $line, $matches)) {
+        continue;
+      }
+      foreach ($matches as $key => $match) {
+        $matches[$key] = trim($match);
+      }
+      $message = new ArcanistLintMessage();
+      $message->setPath($path);
+      $message->setLine($matches[1]);
+      $message->setCode($matches[3]);
+      $message->setName($matches[3]);
+      $message->setDescription($matches[2]);
+      $message->setSeverity(ArcanistLintSeverity::SEVERITY_WARNING);
+      $this->addLintMessage($message);
+    }
+  }
+
+  // The path of this linter
+  private function linterDir() {
+    return dirname(__FILE__);
+  }
+
+  // TODO(kaili) a quick and dirty way to figure out rocksdb's root dir.
+  private function rocksdbDir() {
+    return $this->linterDir()."/../..";
+  }
+}
@@ -0,0 +1,74 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+abstract class BaseDirectoryScopedFormatLinter extends ArcanistLinter {
+
+  const LINT_FORMATTING = 1;
+
+  private $changedLines = array();
+  private $rawLintOutput = array();
+
+  abstract protected function getPathsToLint();
+
+  protected function shouldLintPath($path) {
+    foreach ($this->getPathsToLint() as $p) {
+      // check if $path starts with $p
+      if (strncmp($path, $p, strlen($p)) === 0) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  // API to tell this linter which lines were changed
+  final public function setPathChangedLines($path, $changed) {
+    $this->changedLines[$path] = $changed;
+  }
+
+  final public function willLintPaths(array $paths) {
+    $futures = array();
+    foreach ($paths as $path) {
+      if (!$this->shouldLintPath($path)) {
+        continue;
+      }
+
+      $changed = $this->changedLines[$path];
+      if (!isset($changed)) {
+        // do not run linter if there are no changes
+        continue;
+      }
+
+      $futures[$path] = $this->getFormatFuture($path, $changed);
+    }
+
+    foreach (id(new FutureIterator($futures))->limit(8) as $p => $f) {
+      $this->rawLintOutput[$p] = $f->resolvex();
+    }
+  }
+
+  abstract protected function getFormatFuture($path, array $changed);
+  abstract protected function getLintMessage($diff);
+
+  final public function lintPath($path) {
+    if (!isset($this->rawLintOutput[$path])) {
+      return;
+    }
+
+    list($new_content) = $this->rawLintOutput[$path];
+    $old_content = $this->getData($path);
+
+    if ($new_content != $old_content) {
+      $diff = ArcanistDiffUtils::renderDifferences($old_content, $new_content);
+      $this->raiseLintAtOffset(
+        0,
+        self::LINT_FORMATTING,
+        $this->getLintMessage($diff),
+        $old_content,
+        $new_content);
+    }
+  }
+
+}
@@ -0,0 +1,223 @@
+<?php
+// Copyright 2015-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FacebookHowtoevenLinter extends ArcanistLinter {
+
+  const VERSION = 'fd9192f324c36d28136d14380f0b552a1385b59b';
+
+  private $parsedTargets = array();
+
+  public function getLinterName() {
+    return 'Howtoeven';
+  }
+
+  protected function getSeverity($code) {
+    $severities = array(
+      ArcanistLintSeverity::SEVERITY_DISABLED,
+      ArcanistLintSeverity::SEVERITY_ADVICE,
+      ArcanistLintSeverity::SEVERITY_WARNING,
+      ArcanistLintSeverity::SEVERITY_ERROR,
+    );
+    return idx($severities, $code, ArcanistLintSeverity::SEVERITY_WARNING);
+  }
+
+  public function willLintPaths(array $paths) {
+    // Cleanup previous runs.
+    $this->localExecx("rm -rf _build/_lint");
+
+    // Build compilation database.
+    $lintable_paths = $this->getLintablePaths($paths);
+    $interesting_paths = $this->getInterestingPaths($lintable_paths);
+
+    if (!$lintable_paths) {
+      return;
+    }
+
+    // Run lint.
+    try {
+      $this->localExecx(
+        "%C %C -p _build/dev/ %Ls",
+        $this->getBinaryPath(),
+        $this->getFilteredIssues(),
+        $lintable_paths);
+    } catch (CommandException $exception) {
+      PhutilConsole::getConsole()->writeErr($exception->getMessage());
+    }
+
+    // Load results.
+    $result = id(
+      new SQLite3(
+        $this->getProjectRoot().'/_build/_lint/lint.db',
+        SQLITE3_OPEN_READONLY))
+      ->query("SELECT * FROM raised_issues");
+
+    while ($issue = $result->fetchArray(SQLITE3_ASSOC)) {
+      // Skip issues not part of the linted file.
+      if (in_array($issue['file'], $interesting_paths)) {
+        $this->addLintMessage(id(new ArcanistLintMessage())
+          ->setPath($issue['file'])
+          ->setLine($issue['line'])
+          ->setChar($issue['column'])
+          ->setCode('Howtoeven')
+          ->setSeverity($this->getSeverity($issue['severity']))
+          ->setName('Hte-'.$issue['name'])
+          ->setDescription(
+            sprintf(
+              "%s\n\n%s",
+              ($issue['message']) ? $issue['message'] : $issue['description'],
+              $issue['explanation']))
+          ->setOriginalText(idx($issue, 'original', ''))
+          ->setReplacementText(idx($issue, 'replacement', '')));
+      }
+    }
+  }
+
+  public function lintPath($path) {
+  }
+
+  /**
+   * Get the paths that we know how to lint.
+   *
+   * The strategy is to first look whether there's an existing compilation
+   * database and use that if it's exhaustive. We generate our own only if
+   * necessary.
+   */
+  private function getLintablePaths($paths) {
+    // Replace headers with existing sources.
+    for ($i = 0; $i < count($paths); $i++) {
+      if (preg_match("/\.h$/", $paths[$i])) {
+        $header = preg_replace("/\.h$/", ".cpp", $paths[$i]);
+        if (file_exists($header)) {
+          $paths[$i] = $header;
+        }
+      }
+    }
+
+    // Check if database exists and is exhaustive.
+    $available_paths = $this->getAvailablePaths();
+    $lintable_paths = array_intersect($paths, $available_paths);
+    if ($paths === $lintable_paths) {
+      return $lintable_paths;
+    }
+
+    // Generate our own database.
+    $targets = $this->getTargetsFor($paths);
+    if (!$targets) {
+      PhutilConsole::getConsole()->writeErr(
+        "No build targets found for %s\n",
+        implode(', ', $paths));
+      return array();
+    }
+
+    $this->localExecx("./tools/build/bin/fbconfig.par -r %Ls", $targets);
+    $this->localExecx("./tools/build/bin/fbmake.par gen_cdb");
+
+    $available_paths = $this->getAvailablePaths();
+    $lintable_paths = array_intersect($paths, $available_paths);
+    if ($paths != $lintable_paths) {
+      PhutilConsole::getConsole()->writeErr(
+        "Can't lint %s\n",
+        implode(', ', array_diff($paths, $available_paths)));
+    }
+
+    // Return what we know how to lint.
+    return $lintable_paths;
+  }
+
+  /**
+   * Get the available paths in the current compilation database.
+   */
+  private function getAvailablePaths() {
+    $database_path = $this->getProjectRoot()
+      .'/_build/dev/compile_commands.json';
+    if (!file_exists($database_path)) {
+      return array();
+    }
+
+    $entries = json_decode(file_get_contents($database_path), true);
+    $paths = array();
+    foreach ($entries as $entry) {
+      $paths[] = $entry['file'];
+    }
+    return $paths;
+  }
+
+  /**
+   * Search for the targets directories for the given files.
+   */
+  private static function getTargetsFor($paths) {
+    $targets = array();
+    foreach ($paths as $path) {
+      while (($path = dirname($path)) !== '.') {
+        if (in_array('TARGETS', scandir($path))) {
+          $contents = file_get_contents($path.'/TARGETS');
+          if (strpos($contents, 'cpp_binary') !== false) {
+            $targets[] = $path;
+            break;
+          }
+        }
+      }
+    }
+    return array_unique($targets);
+  }
+
+  /**
+   * The paths that we actually want to report on.
+   */
+  private function getInterestingPaths($paths) {
+    $headers = array();
+    foreach ($paths as $path) {
+      $headers[] = preg_replace("/\.cpp$/", ".h", $path);
+    }
+    return array_merge($paths, $headers);
+  }
+
+  /**
+   * The path where the binary is located. Will return the current dewey binary
+   * unless the `HOWTOEVEN_BUILD` environment variable is set.
+   */
+  private function getBinaryPath() {
+    $path = sprintf(
+      "/mnt/dewey/fbcode/.commits/%s/builds/howtoeven/client",
+      self::VERSION);
+
+    $build = getenv('HOWTOEVEN_BUILD');
+    if ($build) {
+      $path = sprintf(
+        "./_build/%s/tools/howtoeven/client",
+        $build);
+      if (!file_exists($path)) {
+        PhutilConsole::getConsole()->writeErr(">> %s does not exist\n", $path);
+        exit(1);
+      }
+    }
+
+    return $path;
+  }
+
+  /**
+   * Execute the command in the root directory.
+   */
+  private function localExecx($command /* , ... */) {
+    $arguments = func_get_args();
+    return newv('ExecFuture', $arguments)
+      ->setCWD($this->getProjectRoot())
+      ->resolvex();
+  }
+
+  /**
+   * The root of the project.
+   */
+  private function getProjectRoot() {
+    return $this->getEngine()->getWorkingCopy()->getProjectRoot();
+  }
+
+  private function getFilteredIssues() {
+    $issues = getenv('HOWTOEVEN_ISSUES');
+    return ($issues) ? csprintf('-issues %s', $issues) : '';
+  }
+
+}
@@ -0,0 +1,58 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FbcodeClangFormatLinter extends BaseDirectoryScopedFormatLinter {
+
+  const LINT_FORMATTING = 1;
+  const CLANG_FORMAT_BINARY = '/mnt/vol/engshare/admin/scripts/clang-format';
+
+  protected function getPathsToLint() {
+    return array('');
+  }
+
+  public function getLinterName() {
+    return 'CLANG_FORMAT';
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::LINT_FORMATTING => ArcanistLintSeverity::SEVERITY_ADVICE,
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::LINT_FORMATTING => pht('Changes are not clang-formatted'),
+    );
+  }
+
+  protected function getFormatFuture($path, array $changed) {
+    $args = "";
+    foreach ($changed as $key => $value) {
+      $args .= " --lines=$key:$key";
+    }
+
+    $binary = self::CLANG_FORMAT_BINARY;
+    if (!file_exists($binary)) {
+      // trust the $PATH
+      $binary = "clang-format";
+    }
+
+    return new ExecFuture(
+      "%s %s $args",
+      $binary,
+      $this->getEngine()->getFilePathOnDisk($path));
+  }
+
+  protected function getLintMessage($diff) {
+    $link_to_clang_format =
+      "[[ http://fburl.com/clang-format | clang-format ]]";
+    return <<<LINT_MSG
+Changes in this file were not formatted using $link_to_clang_format.
+Please run build_tools/format-diff.sh or `make format`
+LINT_MSG;
+  }
+}
@@ -0,0 +1,126 @@
+<?php
+// Copyright 2004-present Facebook.  All rights reserved.
+
+class FbcodeCppLinter extends ArcanistLinter {
+  const FLINT      = "/home/engshare/tools/flint";
+  const LINT_ERROR   = 1;
+  const LINT_WARNING = 2;
+  const LINT_ADVICE  = 3;
+  const C_FLAG = "--c_mode=true";
+
+  private $rawLintOutput = array();
+
+  public function willLintPaths(array $paths) {
+    if (!file_exists(self::FLINT)) {
+      return;
+    }
+    $futures = array();
+    foreach ($paths as $p) {
+      $lpath = $this->getEngine()->getFilePathOnDisk($p);
+      $lpath_file = file($lpath);
+      if (preg_match('/\.(c)$/', $lpath) ||
+          preg_match('/-\*-.*Mode: C[; ].*-\*-/', $lpath_file[0]) ||
+          preg_match('/vim(:.*)*:\s*(set\s+)?filetype=c\s*:/', $lpath_file[0])
+          ) {
+        $futures[$p] = new ExecFuture("%s %s %s 2>&1",
+                           self::FLINT, self::C_FLAG,
+                           $this->getEngine()->getFilePathOnDisk($p));
+      } else {
+        $futures[$p] = new ExecFuture("%s %s 2>&1",
+          self::FLINT, $this->getEngine()->getFilePathOnDisk($p));
+      }
+    }
+
+    foreach (Futures($futures)->limit(8) as $p => $f) {
+      $this->rawLintOutput[$p] = $f->resolvex();
+    }
+
+    return;
+  }
+
+  public function getLinterName() {
+    return "FBCPP";
+  }
+
+  public function lintPath($path) {
+    $this->runCppLint($path);
+  }
+
+  private function runCppLint($path) {
+    $msgs = $this->getCppLintOutput($path);
+    foreach ($msgs as $m) {
+      $this->raiseLintAtLine($m['line'], 0, $m['severity'], $m['msg']);
+    }
+  }
+
+  private function adviseOnEachPattern(
+    $path,
+    $regex,
+    $message,
+    $lint_type = self::LINT_ADVICE,
+    $match_idx = 0) {
+      $file_data = $this->getData($path);
+      $matches = array();
+      if (!preg_match_all($regex, $file_data, $matches, PREG_OFFSET_CAPTURE)) {
+        return;
+      }
+
+      foreach ($matches[$match_idx] as $match) {
+        list($match_str, $offset) = $match;
+        $this->raiseLintAtOffset($offset, $lint_type, $message, $match_str);
+      }
+  }
+
+  public function getLintSeverityMap() {
+    return array(
+      self::LINT_WARNING => ArcanistLintSeverity::SEVERITY_WARNING,
+      self::LINT_ADVICE  => ArcanistLintSeverity::SEVERITY_ADVICE,
+      self::LINT_ERROR   => ArcanistLintSeverity::SEVERITY_ERROR
+    );
+  }
+
+  public function getLintNameMap() {
+    return array(
+      self::LINT_ADVICE   => "CppLint Advice",
+      self::LINT_WARNING  => "CppLint Warning",
+      self::LINT_ERROR    => "CppLint Error"
+    );
+  }
+
+  private function getCppLintOutput($path) {
+     if (!array_key_exists($path, $this->rawLintOutput)) {
+       return array();
+     }
+    list($output) = $this->rawLintOutput[$path];
+
+    $msgs = array();
+    $current = null;
+    $matches = array();
+    foreach (explode("\n", $output) as $line) {
+      if (preg_match('/.*?:(\d+):(.*)/', $line, $matches)) {
+        if ($current) {
+          $msgs[] = $current;
+        }
+        $line = $matches[1];
+        $text = $matches[2];
+        if (preg_match('/.*Warning.*/', $text)) {
+          $sev = self::LINT_WARNING;
+        } else if (preg_match('/.*Advice.*/', $text)) {
+          $sev = self::LINT_ADVICE;
+        } else {
+          $sev = self::LINT_ERROR;
+        }
+        $current = array('line'     => $line,
+                         'msg'      => $text,
+                         'severity' => $sev);
+      } else if ($current) {
+        $current['msg'] .= ' ' . $line;
+      }
+    }
+    if ($current) {
+      $msgs[] = $current;
+    }
+
+    return $msgs;
+  }
+}
@@ -0,0 +1,138 @@
+<?php
+// Copyright 2004-present Facebook.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeLintEngine extends ArcanistLintEngine {
+
+  public function buildLinters() {
+    $linters = array();
+    $paths = $this->getPaths();
+
+    // Remove all deleted files, which are not checked by the
+    // following linters.
+    foreach ($paths as $key => $path) {
+      if (!Filesystem::pathExists($this->getFilePathOnDisk($path))) {
+        unset($paths[$key]);
+      }
+    }
+
+    $generated_linter = new ArcanistGeneratedLinter();
+    $linters[] = $generated_linter;
+
+    $nolint_linter = new ArcanistNoLintLinter();
+    $linters[] = $nolint_linter;
+
+    $text_linter = new ArcanistTextLinter();
+    $text_linter->setCustomSeverityMap(array(
+      ArcanistTextLinter::LINT_LINE_WRAP
+        => ArcanistLintSeverity::SEVERITY_ADVICE,
+    ));
+    $linters[] = $text_linter;
+
+    $java_text_linter = new ArcanistTextLinter();
+    $java_text_linter->setMaxLineLength(100);
+    $java_text_linter->setCustomSeverityMap(array(
+      ArcanistTextLinter::LINT_LINE_WRAP
+        => ArcanistLintSeverity::SEVERITY_ADVICE,
+    ));
+    $linters[] = $java_text_linter;
+
+    $python_linter = new ArcanistPEP8Linter();
+    $linters[] = $python_linter;
+
+    $cpp_linters = array();
+    $cpp_linters[] = $linters[] = new ArcanistCpplintLinter();
+    $cpp_linters[] = $linters[] = new FbcodeCppLinter();
+
+    $clang_format_linter = new FbcodeClangFormatLinter();
+    $linters[] = $clang_format_linter;
+
+    $spelling_linter = new ArcanistSpellingLinter();
+    $linters[] = $spelling_linter;
+
+    foreach ($paths as $path) {
+      $is_text = false;
+
+      $text_extensions = (
+        '/\.('.
+        'cpp|cxx|c|cc|h|hpp|hxx|tcc|'.
+        'py|rb|hs|pl|pm|tw|'.
+        'php|phpt|css|js|'.
+        'java|'.
+        'thrift|'.
+        'lua|'.
+        'siv|'.
+        'txt'.
+        ')$/'
+      );
+      if (preg_match($text_extensions, $path)) {
+        $is_text = true;
+      }
+      if ($is_text) {
+        $nolint_linter->addPath($path);
+
+        $generated_linter->addPath($path);
+        $generated_linter->addData($path, $this->loadData($path));
+
+        if (preg_match('/\.java$/', $path)) {
+          $java_text_linter->addPath($path);
+          $java_text_linter->addData($path, $this->loadData($path));
+        } else {
+          $text_linter->addPath($path);
+          $text_linter->addData($path, $this->loadData($path));
+        }
+
+        $spelling_linter->addPath($path);
+        $spelling_linter->addData($path, $this->loadData($path));
+      }
+      if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)
+          && !preg_match('/third-party/', $path)) {
+        foreach ($cpp_linters as &$linter) {
+          $linter->addPath($path);
+          $linter->addData($path, $this->loadData($path));
+        }
+
+        $clang_format_linter->addPath($path);
+        $clang_format_linter->addData($path, $this->loadData($path));
+        $clang_format_linter->setPathChangedLines(
+          $path, $this->getPathChangedLines($path));
+      }
+
+      // Match *.py and contbuild config files
+      if (preg_match('/(\.(py|tw|smcprops)|^contbuild\/configs\/[^\/]*)$/',
+                    $path)) {
+        $space_count = 4;
+        $real_path = $this->getFilePathOnDisk($path);
+        $dir = dirname($real_path);
+        do {
+          if (file_exists($dir.'/.python2space')) {
+            $space_count = 2;
+            break;
+          }
+          $dir = dirname($dir);
+        } while ($dir != '/' && $dir != '.');
+
+        $cur_path_linter = $python_linter;
+        $cur_path_linter->addPath($path);
+        $cur_path_linter->addData($path, $this->loadData($path));
+
+        if (preg_match('/\.tw$/', $path)) {
+          $cur_path_linter->setCustomSeverityMap(array(
+            'E251' => ArcanistLintSeverity::SEVERITY_DISABLED,
+          ));
+        }
+      }
+    }
+
+    $name_linter = new ArcanistFilenameLinter();
+    $linters[] = $name_linter;
+    foreach ($paths as $path) {
+      $name_linter->addPath($path);
+    }
+
+    return $linters;
+  }
+
+}
@@ -0,0 +1,27 @@
+<?php
+// Copyright 2015-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+final class FacebookHowtoevenLintEngine extends ArcanistLintEngine {
+
+  public function buildLinters() {
+    $paths = array();
+
+    foreach ($this->getPaths() as $path) {
+      // Don't try to lint deleted files or changed directories.
+      if (!Filesystem::pathExists($path) || is_dir($path)) {
+        continue;
+      }
+
+      if (preg_match('/\.(cpp|c|cc|cxx|h|hh|hpp|hxx|tcc)$/', $path)) {
+        $paths[] = $path;
+      }
+    }
+
+    $howtoeven = new FacebookHowtoevenLinter();
+    $howtoeven->setPaths($paths);
+    return array($howtoeven);
+  }
+}
@@ -0,0 +1,17 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeUnitTestEngine extends ArcanistUnitTestEngine {
+
+  public function run() {
+      // For a call to `arc call-conduit differential.updateunitresults` to
+      // succeed we need at least one entry here.
+      $result = new ArcanistUnitTestResult();
+      $result->setName("dummy_placeholder_entry");
+      $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
+      return array($result);
+  }
+}
@@ -0,0 +1,17 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+class FacebookFbcodeUnitTestEngine extends ArcanistBaseUnitTestEngine {
+
+  public function run() {
+      // For a call to `arc call-conduit differential.updateunitresults` to
+      // succeed we need at least one entry here.
+      $result = new ArcanistUnitTestResult();
+      $result->setName("dummy_placeholder_entry");
+      $result->setResult(ArcanistUnitTestResult::RESULT_PASS);
+      return array($result);
+  }
+}
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"
@@ -10,6 +10,8 @@ REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
  "-fno-builtin-memcmp",
  "-DROCKSDB_PLATFORM_POSIX",
  "-DROCKSDB_LIB_IO_POSIX",
@@ -20,7 +22,6 @@ rocksdb_compiler_flags = [
  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
  "-DHAVE_SSE42",
  "-DOS_LINUX",
-  "-DROCKSDB_UBSAN_RUN",
  # Flags to enable libs we include
  "-DSNAPPY",
  "-DZLIB",
@@ -51,11 +51,13 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
    FBCODE_BUILD="true"
    # If we're compiling with TSAN we need pic build
    PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      source "$PWD/build_tools/fbcode_config.sh"
-    else
+    if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
      # we need this to build with MySQL. Don't use for other purposes.
      source "$PWD/build_tools/fbcode_config4.8.1.sh"
+    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
+      source "$PWD/build_tools/fbcode_config.sh"
+    else
+      source "$PWD/build_tools/fbcode_config_platform007.sh"
    fi
 fi

@@ -317,11 +319,9 @@ EOF
    # Test whether jemalloc is available
    if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
      2>/dev/null; then
-        # This will enable some preprocessor identifiers in the Makefile
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"
        JEMALLOC=1
-        # JEMALLOC can be enabled either using the flag (like here) or by
-        # providing direct link to the jemalloc library
-        WITH_JEMALLOC_FLAG=1
    else
        # jemalloc is not available. Let's try tcmalloc
        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
@@ -519,14 +519,7 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
 echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
 echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
-# This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
  echo "JEMALLOC=1" >> "$OUTPUT"
 fi
-# Indicates that jemalloc should be enabled using -ljemalloc flag
-# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
-# and JEMALLOC_INCLUDE
-if test -n "$WITH_JEMALLOC_FLAG"; then
-  echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
-fi
 echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 #
 # Copyright (c) 2016, Facebook. All rights reserved.
 #
@@ -67,7 +67,7 @@ function update_repo_status {
 #

 # Path to the determinator from the root of the RocksDB repo.
-CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php
+CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonHelper.php

 # Value of the previous commit.
 PREV_COMMIT=
@@ -0,0 +1,18 @@
+GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
+NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
@@ -1,2 +1,2 @@
-#!/usr/bin/env bash
+#!/bin/bash
 docker run -v $PWD:/rocks -w /rocks buildpack-deps make
@@ -0,0 +1,157 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile rocksdb using
+# fbcode settings.  It uses the latest g++ and clang compilers and also
+# uses jemalloc
+# Environment variables that change the behavior of this script:
+# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
+
+
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies_platform007.sh"
+
+CFLAGS=""
+
+# libgcc
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
+
+# glibc
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"
+
+# snappy
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
+if test -z $PIC_BUILD; then
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
+else
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
+fi
+CFLAGS+=" -DSNAPPY"
+
+if test -z $PIC_BUILD; then
+  # location of zlib headers and libraries
+  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
+  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
+  CFLAGS+=" -DZLIB"
+
+  # location of bzip headers and libraries
+  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
+  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
+  CFLAGS+=" -DBZIP2"
+
+  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
+  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
+  CFLAGS+=" -DLZ4"
+fi
+
+ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+if test -z $PIC_BUILD; then
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+else
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+fi
+CFLAGS+=" -DZSTD"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
+if test -z $PIC_BUILD; then
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
+else
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
+fi
+CFLAGS+=" -DGFLAGS=gflags"
+
+# location of jemalloc
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
+JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
+
+if test -z $PIC_BUILD; then
+  # location of numa
+  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
+  CFLAGS+=" -DNUMA"
+
+  # location of libunwind
+  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
+fi
+
+# location of TBB
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+if test -z $PIC_BUILD; then
+  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+else
+  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
+fi
+CFLAGS+=" -DTBB"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
+export PORTABLE=1
+
+BINUTILS="$BINUTILS_BASE/bin"
+AR="$BINUTILS/ar"
+
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
+
+STDLIBS="-L $GCC_BASE/lib64"
+
+CLANG_BIN="$CLANG_BASE/bin"
+CLANG_LIB="$CLANG_BASE/lib"
+CLANG_SRC="$CLANG_BASE/../../src"
+
+CLANG_ANALYZER="$CLANG_BIN/clang++"
+CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
+
+if [ -z "$USE_CLANG" ]; then
+  # gcc
+  CC="$GCC_BASE/bin/gcc"
+  CXX="$GCC_BASE/bin/g++"
+
+  CFLAGS+=" -B$BINUTILS/gold"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  JEMALLOC=1
+else
+  # clang
+  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
+  CC="$CLANG_BIN/clang"
+  CXX="$CLANG_BIN/clang++"
+
+  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
+
+  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $CLANG_INCLUDE"
+  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
+  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
+  CFLAGS+=" -Wno-expansion-to-defined "
+  CXXFLAGS="-nostdinc++"
+fi
+
+CFLAGS+=" $DEPS_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CXXFLAGS+=" $CFLAGS"
+
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
+EXEC_LDFLAGS+=" $LIBUNWIND"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
+
+VALGRIND_VER="$VALGRIND_BASE/bin/"
+
+# lua not supported because it's on track for deprecation, I think
+LUA_PATH=
+LUA_LIB=
+
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # If clang_format_diff.py command is not specfied, we assume we are able to
 # access directly without any path.
 if [ -z $CLANG_FORMAT_DIFF ]
@@ -0,0 +1,53 @@
+#!/bin/bash
+#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+#  This source code is licensed under the BSD-style license found in the
+#  LICENSE file in the root directory of this source tree. An additional grant
+#  of patent rights can be found in the PATENTS file in the same directory.
+
+set -e
+if [ -z "$GIT" ]
+then
+  GIT="git"
+fi
+
+# Print out the colored progress info so that it can be brainlessly
+# distinguished by users.
+function title() {
+  echo -e "\033[1;32m$*\033[0m"
+}
+
+usage="Create new RocksDB version and prepare it for the release process\n"
+usage+="USAGE: ./make_new_version.sh <version> [<remote>]\n"
+usage+="  version: specify a version without '.fb' suffix (e.g. 5.4).\n"
+usage+="  remote: name of the remote to push the branch to (default: origin)."
+
+# -- Pre-check
+if [[ $# < 1 ]]; then
+  echo -e $usage
+  exit 1
+fi
+
+ROCKSDB_VERSION=$1
+
+REMOTE="origin"
+if [[ $# > 1 ]]; then
+  REMOTE=$2
+fi
+
+GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+echo $GIT_BRANCH
+
+if [ $GIT_BRANCH != "master" ]; then
+  echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch."
+  exit 1
+fi
+
+title "Adding new tag for this release ..."
+BRANCH="$ROCKSDB_VERSION.fb"
+$GIT checkout -b $BRANCH
+
+# Setting up the proxy for remote repo access
+title "Pushing new branch to remote repo ..."
+git push $REMOTE --set-upstream $BRANCH
+
+title "Branch $BRANCH is pushed to github;"
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash

 set -e

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 # This script is executed by Sandcastle
 # to determine next steps to run

@@ -85,8 +85,9 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-LITE="OPT=\"-DROCKSDB_LITE -g\""
-TSAN="COMPILE_WITH_TSAN=1"
+# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
+# using platform007 gives us gcc-8 or higher which has that bug fixed.
+TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
@@ -564,7 +565,7 @@ TSAN_CRASH_TEST_COMMANDS="[
            {
                'name':'Compile and run',
                'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22  make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
                'user':'root',
                $PARSER
            },
@@ -583,7 +584,39 @@ run_format_compatible()
  rm -rf /dev/shm/rocksdb
  mkdir /dev/shm/rocksdb

-  tools/check_format_compatible.sh
+  echo '
+  if [ -e "build_tools/build_detect_platform" ]
+  then
+    sed "s/tcmalloc/nothingnothingnothing/g" build_tools/build_detect_platform > $TEST_TMPDIR/temp_build_file
+    rm -rf build_tools/build_detect_platform
+    cp $TEST_TMPDIR/temp_build_file build_tools/build_detect_platform
+    chmod +x build_tools/build_detect_platform
+  fi
+
+  if [ -e "build_detect_platform" ]
+  then
+    sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
+    rm -rf build_detect_platform
+    cp $TEST_TMPDIR/temp_build_file build_detect_platform
+    chmod +x build_detect_platform
+  fi
+
+  make ldb -j32
+
+  if [ -e "build_detect_platform" ]
+  then
+    git checkout -- build_detect_platform
+  fi
+
+  if [ -e "build_tools/build_detect_platform" ]
+  then
+    git checkout -- build_tools/build_detect_platform
+  fi
+  ' > temp_build_ldb.sh
+
+  sed "s/make ldb -j32/source temp_build_ldb.sh/g" tools/check_format_compatible.sh > tools/temp_check_format_compatible.sh
+  chmod +x tools/temp_check_format_compatible.sh
+  tools/temp_check_format_compatible.sh
 }

 FORMAT_COMPATIBLE_COMMANDS="[
@@ -52,6 +52,45 @@ function get_lib_base()
  log_variable $__res_var
 }

+###########################################################
+#                platform007 dependencies                 #
+###########################################################
+
+OUTPUT="$BASEDIR/dependencies_platform007.sh"
+
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
+echo "Writing dependencies to $OUTPUT"
+
+# Compilers locations
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
+
+log_variable GCC_BASE
+log_variable CLANG_BASE
+
+# Libraries locations
+get_lib_base libgcc     7.x     platform007
+get_lib_base glibc      2.26    platform007
+get_lib_base snappy     LATEST  platform007
+get_lib_base zlib       LATEST  platform007
+get_lib_base bzip2      LATEST  platform007
+get_lib_base lz4        LATEST  platform007
+get_lib_base zstd       LATEST  platform007
+get_lib_base gflags     LATEST  platform007
+get_lib_base jemalloc   LATEST  platform007
+get_lib_base numa       LATEST  platform007
+get_lib_base libunwind  LATEST  platform007
+get_lib_base tbb        LATEST  platform007
+
+get_lib_base kernel-headers fb platform007
+get_lib_base binutils   LATEST centos7-native
+get_lib_base valgrind   LATEST platform007
+get_lib_base lua        5.3.4  platform007
+
+git diff $OUTPUT
+
 ###########################################################
 #                   5.x dependencies                      #
 ###########################################################
@@ -27,11 +27,6 @@ std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
 #include <atomic>
 #include <deque>

-// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
-// Disable it so users can chooose to disable RTTI.
-#ifndef ROCKSDB_USE_RTTI
-#define TBB_USE_EXCEPTIONS 0
-#endif
 #include "tbb/concurrent_hash_map.h"

 #include "cache/sharded_cache.h"
@@ -22,7 +22,7 @@

 namespace rocksdb {

-LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
+LRUHandleTable::LRUHandleTable() : length_(0), elems_(0), list_(nullptr) {
  Resize();
 }

@@ -100,7 +100,7 @@ void LRUHandleTable::Resize() {
 }

 LRUCacheShard::LRUCacheShard()
-    : high_pri_pool_usage_(0), usage_(0), lru_usage_(0) {
+    : usage_(0), lru_usage_(0), high_pri_pool_usage_(0) {
  // Make empty circular linked list
  lru_.next = &lru_;
  lru_.prev = &lru_;
@@ -157,16 +157,6 @@ void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
  *lru_low_pri = lru_low_pri_;
 }

-size_t LRUCacheShard::TEST_GetLRUSize() {
-  LRUHandle* lru_handle = lru_.next;
-  size_t lru_size = 0;
-  while (lru_handle != &lru_) {
-    lru_size++;
-    lru_handle = lru_handle->next;
-  }
-  return lru_size;
-}
-
 void LRUCacheShard::LRU_Remove(LRUHandle* e) {
  assert(e->next != nullptr);
  assert(e->prev != nullptr);
@@ -233,14 +223,6 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
  }
 }

-void* LRUCacheShard::operator new(size_t size) {
-  return rocksdb::port::cacheline_aligned_alloc(size);
-}
-
-void LRUCacheShard::operator delete(void *memblock) {
-  rocksdb::port::cacheline_aligned_free(memblock);
-}
-
 void LRUCacheShard::SetCapacity(size_t capacity) {
  autovector<LRUHandle*> last_reference_list;
  {
@@ -456,18 +438,11 @@ std::string LRUCacheShard::GetPrintableOptions() const {
 LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                   bool strict_capacity_limit, double high_pri_pool_ratio)
    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
-  num_shards_ = 1 << num_shard_bits;
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4316) // We've validated the alignment with the new operators
-#endif
-  shards_ = new LRUCacheShard[num_shards_];
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
+  int num_shards = 1 << num_shard_bits;
+  shards_ = new LRUCacheShard[num_shards];
  SetCapacity(capacity);
  SetStrictCapacityLimit(strict_capacity_limit);
-  for (int i = 0; i < num_shards_; i++) {
+  for (int i = 0; i < num_shards; i++) {
    shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
  }
 }
@@ -496,14 +471,6 @@ uint32_t LRUCache::GetHash(Handle* handle) const {

 void LRUCache::DisownData() { shards_ = nullptr; }

-size_t LRUCache::TEST_GetLRUSize() {
-  size_t lru_size_of_all_shards = 0;
-  for (int i = 0; i < num_shards_; i++) {
-    lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
-  }
-  return lru_size_of_all_shards;
-}
-
 std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
                                   bool strict_capacity_limit,
                                   double high_pri_pool_ratio) {
@@ -148,13 +148,13 @@ class LRUHandleTable {

  // The table consists of an array of buckets where each bucket is
  // a linked list of cache entries that hash into the bucket.
-  LRUHandle** list_;
  uint32_t length_;
  uint32_t elems_;
+  LRUHandle** list_;
 };

 // A single shard of sharded cache.
-class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
+class LRUCacheShard : public CacheShard {
 public:
  LRUCacheShard();
  virtual ~LRUCacheShard();
@@ -198,15 +198,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {

  void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);

-  //  Retrieves number of elements in LRU, for unit test purpose only
-  //  not threadsafe
-  size_t TEST_GetLRUSize();
-
-  // Overloading to aligned it to cache line size
-  void* operator new(size_t);
-
-  void operator delete(void *);
-
 private:
  void LRU_Remove(LRUHandle* e);
  void LRU_Insert(LRUHandle* e);
@@ -228,6 +219,12 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Initialized before use.
  size_t capacity_;

+  // Memory size for entries residing in the cache
+  size_t usage_;
+
+  // Memory size for entries residing only in the LRU list
+  size_t lru_usage_;
+
  // Memory size for entries in high-pri pool.
  size_t high_pri_pool_usage_;

@@ -241,6 +238,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Remember the value to avoid recomputing each time.
  double high_pri_pool_capacity_;

+  // mutex_ protects the following state.
+  // We don't count mutex_ as the cache's internal state so semantically we
+  // don't mind mutex_ invoking the non-const actions.
+  mutable port::Mutex mutex_;
+
  // Dummy head of LRU list.
  // lru.prev is newest entry, lru.next is oldest entry.
  // LRU contains items which can be evicted, ie reference only by cache
@@ -249,29 +251,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
  // Pointer to head of low-pri pool in LRU list.
  LRUHandle* lru_low_pri_;

-  // ------------^^^^^^^^^^^^^-----------
-  // Not frequently modified data members
-  // ------------------------------------
-  //
-  // We separate data members that are updated frequently from the ones that
-  // are not frequently updated so that they don't share the same cache line
-  // which will lead into false cache sharing
-  //
-  // ------------------------------------
-  // Frequently modified data members
-  // ------------vvvvvvvvvvvvv-----------
  LRUHandleTable table_;
-
-  // Memory size for entries residing in the cache
-  size_t usage_;
-
-  // Memory size for entries residing only in the LRU list
-  size_t lru_usage_;
-
-  // mutex_ protects the following state.
-  // We don't count mutex_ as the cache's internal state so semantically we
-  // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
 };

 class LRUCache : public ShardedCache {
@@ -287,12 +267,8 @@ class LRUCache : public ShardedCache {
  virtual uint32_t GetHash(Handle* handle) const override;
  virtual void DisownData() override;

-  //  Retrieves number of elements in LRU, for unit test purpose only
-  size_t TEST_GetLRUSize();
-
 private:
  LRUCacheShard* shards_;
-  int num_shards_ = 0;
 };

 }  // namespace rocksdb
@@ -17,16 +17,7 @@ class LRUCacheTest : public testing::Test {
  ~LRUCacheTest() {}

  void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
-    cache_.reset(
-#if defined(_MSC_VER)
-#pragma warning(push)
-#pragma warning(disable: 4316) // We've validated the alignment with the new operators
-#endif
-      new LRUCacheShard()
-#if defined(_MSC_VER)
-#pragma warning(pop)
-#endif
-    );
+    cache_.reset(new LRUCacheShard());
    cache_->SetCapacity(capacity);
    cache_->SetStrictCapacityLimit(false);
    cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash

 # Exit on error.
 set -e
@@ -52,7 +52,6 @@ using rocksdb::CompressionType;
 using rocksdb::WALRecoveryMode;
 using rocksdb::DB;
 using rocksdb::DBOptions;
-using rocksdb::DbPath;
 using rocksdb::Env;
 using rocksdb::EnvOptions;
 using rocksdb::InfoLogLevel;
@@ -383,10 +382,6 @@ struct rocksdb_mergeoperator_t : public MergeOperator {
  }
 };

-struct rocksdb_dbpath_t {
-  DbPath rep;
-};
-
 struct rocksdb_env_t {
  Env* rep;
  bool is_default;
@@ -1377,23 +1372,24 @@ void rocksdb_writebatch_put_log_data(
  b->rep.PutLogData(Slice(blob, len));
 }

+class H : public WriteBatch::Handler {
+ public:
+  void* state_;
+  void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+  void (*deleted_)(void*, const char* k, size_t klen);
+  virtual void Put(const Slice& key, const Slice& value) override {
+    (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+  }
+  virtual void Delete(const Slice& key) override {
+    (*deleted_)(state_, key.data(), key.size());
+  }
+};
+
 void rocksdb_writebatch_iterate(
    rocksdb_writebatch_t* b,
    void* state,
    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
    void (*deleted)(void*, const char* k, size_t klen)) {
-  class H : public WriteBatch::Handler {
-   public:
-    void* state_;
-    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-    void (*deleted_)(void*, const char* k, size_t klen);
-    virtual void Put(const Slice& key, const Slice& value) override {
-      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-    }
-    virtual void Delete(const Slice& key) override {
-      (*deleted_)(state_, key.data(), key.size());
-    }
-  };
  H handler;
  handler.state_ = state;
  handler.put_ = put;
@@ -1638,18 +1634,6 @@ void rocksdb_writebatch_wi_iterate(
    void* state,
    void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
    void (*deleted)(void*, const char* k, size_t klen)) {
-  class H : public WriteBatch::Handler {
-   public:
-    void* state_;
-    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-    void (*deleted_)(void*, const char* k, size_t klen);
-    virtual void Put(const Slice& key, const Slice& value) override {
-      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-    }
-    virtual void Delete(const Slice& key) override {
-      (*deleted_)(state_, key.data(), key.size());
-    }
-  };
  H handler;
  handler.state_ = state;
  handler.put_ = put;
@@ -2014,16 +1998,6 @@ void rocksdb_options_set_paranoid_checks(
  opt->rep.paranoid_checks = v;
 }

-void rocksdb_options_set_db_paths(rocksdb_options_t* opt, 
-                                  const rocksdb_dbpath_t** dbpath_values, 
-                                  size_t num_paths) {
-  std::vector<DbPath> db_paths(num_paths);
-  for (size_t i = 0; i < num_paths; ++i) {
-    db_paths[i] = dbpath_values[i]->rep;
-  }
-  opt->rep.db_paths = db_paths;
-}
-
 void rocksdb_options_set_env(rocksdb_options_t* opt, rocksdb_env_t* env) {
  opt->rep.env = (env ? env->rep : nullptr);
 }
@@ -2102,10 +2076,6 @@ void rocksdb_options_enable_statistics(rocksdb_options_t* opt) {
  opt->rep.statistics = rocksdb::CreateDBStatistics();
 }

-void rocksdb_options_set_skip_stats_update_on_db_open(rocksdb_options_t* opt, unsigned char val) {
-  opt->rep.skip_stats_update_on_db_open = val;
-}
-
 void rocksdb_options_set_num_levels(rocksdb_options_t* opt, int n) {
  opt->rep.num_levels = n;
 }
@@ -2808,17 +2778,6 @@ size_t rocksdb_cache_get_pinned_usage(rocksdb_cache_t* cache) {
  return cache->rep->GetPinnedUsage();
 }

-rocksdb_dbpath_t* rocksdb_dbpath_create(const char* path, uint64_t target_size) {
-  rocksdb_dbpath_t* result = new rocksdb_dbpath_t;
-  result->rep.path = std::string(path);
-  result->rep.target_size = target_size;
-  return result;
-}
-
-void rocksdb_dbpath_destroy(rocksdb_dbpath_t* dbpath) {
-  delete dbpath;
-}
-
 rocksdb_env_t* rocksdb_create_default_env() {
  rocksdb_env_t* result = new rocksdb_env_t;
  result->rep = Env::Default();
@@ -2996,20 +2955,21 @@ void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) {
  delete st;
 }

+struct Wrapper : public rocksdb_slicetransform_t {
+  const SliceTransform* rep_;
+  ~Wrapper() { delete rep_; }
+  const char* Name() const override { return rep_->Name(); }
+  Slice Transform(const Slice& src) const override {
+    return rep_->Transform(src);
+  }
+  bool InDomain(const Slice& src) const override {
+    return rep_->InDomain(src);
+  }
+  bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+  static void DoNothing(void*) { }
+};
+
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t prefixLen) {
-  struct Wrapper : public rocksdb_slicetransform_t {
-    const SliceTransform* rep_;
-    ~Wrapper() { delete rep_; }
-    const char* Name() const override { return rep_->Name(); }
-    Slice Transform(const Slice& src) const override {
-      return rep_->Transform(src);
-    }
-    bool InDomain(const Slice& src) const override {
-      return rep_->InDomain(src);
-    }
-    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-    static void DoNothing(void*) { }
-  };
  Wrapper* wrapper = new Wrapper;
  wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
  wrapper->state_ = nullptr;
@@ -3018,19 +2978,6 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
 }

 rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() {
-  struct Wrapper : public rocksdb_slicetransform_t {
-    const SliceTransform* rep_;
-    ~Wrapper() { delete rep_; }
-    const char* Name() const override { return rep_->Name(); }
-    Slice Transform(const Slice& src) const override {
-      return rep_->Transform(src);
-    }
-    bool InDomain(const Slice& src) const override {
-      return rep_->InDomain(src);
-    }
-    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-    static void DoNothing(void*) { }
-  };
  Wrapper* wrapper = new Wrapper;
  wrapper->rep_ = rocksdb::NewNoopTransform();
  wrapper->state_ = nullptr;
@@ -3253,17 +3200,6 @@ void rocksdb_transaction_options_set_max_write_batch_size(
  opt->rep.max_write_batch_size = size;
 }

-rocksdb_column_family_handle_t* rocksdb_transactiondb_create_column_family(
-    rocksdb_transactiondb_t* txn_db,
-    const rocksdb_options_t* column_family_options,
-    const char* column_family_name, char** errptr) {
-  rocksdb_column_family_handle_t* handle = new rocksdb_column_family_handle_t;
-  SaveError(errptr, txn_db->rep->CreateColumnFamily(
-                        ColumnFamilyOptions(column_family_options->rep),
-                        std::string(column_family_name), &(handle->rep)));
-  return handle;
-}
-
 rocksdb_transactiondb_t* rocksdb_transactiondb_open(
    const rocksdb_options_t* options,
    const rocksdb_transactiondb_options_t* txn_db_options, const char* name,
@@ -3340,27 +3276,6 @@ char* rocksdb_transaction_get(rocksdb_transaction_t* txn,
  return result;
 }

-char* rocksdb_transaction_get_cf(rocksdb_transaction_t* txn,
-                                 const rocksdb_readoptions_t* options,
-                                 rocksdb_column_family_handle_t* column_family,
-                                 const char* key, size_t klen, size_t* vlen,
-                                 char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s =
-      txn->rep->Get(options->rep, column_family->rep, Slice(key, klen), &tmp);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
 // Read a key outside a transaction
 char* rocksdb_transactiondb_get(
    rocksdb_transactiondb_t* txn_db,
@@ -3383,26 +3298,6 @@ char* rocksdb_transactiondb_get(
  return result;
 }

-char* rocksdb_transactiondb_get_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, size_t* vallen, char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s = txn_db->rep->Get(options->rep, column_family->rep,
-                              Slice(key, keylen), &tmp);
-  if (s.ok()) {
-    *vallen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vallen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
 // Put a key inside a transaction
 void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
                             size_t klen, const char* val, size_t vlen,
@@ -3410,14 +3305,6 @@ void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
  SaveError(errptr, txn->rep->Put(Slice(key, klen), Slice(val, vlen)));
 }

-void rocksdb_transaction_put_cf(rocksdb_transaction_t* txn,
-                                rocksdb_column_family_handle_t* column_family,
-                                const char* key, size_t klen, const char* val,
-                                size_t vlen, char** errptr) {
-  SaveError(errptr, txn->rep->Put(column_family->rep, Slice(key, klen),
-                                  Slice(val, vlen)));
-}
-
 //Put a key outside a transaction
 void rocksdb_transactiondb_put(rocksdb_transactiondb_t* txn_db,
                               const rocksdb_writeoptions_t* options,
@@ -3427,37 +3314,12 @@ void rocksdb_transactiondb_put(rocksdb_transactiondb_t* txn_db,
            txn_db->rep->Put(options->rep, Slice(key, klen), Slice(val, vlen)));
 }

-void rocksdb_transactiondb_put_cf(rocksdb_transactiondb_t* txn_db,
-                                  const rocksdb_writeoptions_t* options,
-                                  rocksdb_column_family_handle_t* column_family,
-                                  const char* key, size_t keylen,
-                                  const char* val, size_t vallen,
-                                  char** errptr) {
-  SaveError(errptr, txn_db->rep->Put(options->rep, column_family->rep,
-                                     Slice(key, keylen), Slice(val, vallen)));
-}
-
-//Write batch into transaction db
-void rocksdb_transactiondb_write(
-        rocksdb_transactiondb_t* db,
-        const rocksdb_writeoptions_t* options,
-        rocksdb_writebatch_t* batch,
-        char** errptr) {
-  SaveError(errptr, db->rep->Write(options->rep, &batch->rep));
-}
-
 // Delete a key inside a transaction
 void rocksdb_transaction_delete(rocksdb_transaction_t* txn, const char* key,
                                size_t klen, char** errptr) {
  SaveError(errptr, txn->rep->Delete(Slice(key, klen)));
 }

-void rocksdb_transaction_delete_cf(
-    rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family,
-    const char* key, size_t klen, char** errptr) {
-  SaveError(errptr, txn->rep->Delete(column_family->rep, Slice(key, klen)));
-}
-
 // Delete a key outside a transaction
 void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
                                  const rocksdb_writeoptions_t* options,
@@ -3465,14 +3327,6 @@ void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
  SaveError(errptr, txn_db->rep->Delete(options->rep, Slice(key, klen)));
 }

-void rocksdb_transactiondb_delete_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Delete(options->rep, column_family->rep,
-                                        Slice(key, keylen)));
-}
-
 // Create an iterator inside a transaction
 rocksdb_iterator_t* rocksdb_transaction_create_iterator(
    rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options) {
@@ -41,7 +41,6 @@ static char dbname[200];
 static char sstfilename[200];
 static char dbbackupname[200];
 static char dbcheckpointname[200];
-static char dbpathname[200];

 static void StartPhase(const char* name) {
  fprintf(stderr, "=== Test %s\n", name);
@@ -348,25 +347,10 @@ static void CheckTxnDBGet(
        Free(&val);
 }

-static void CheckTxnDBGetCF(rocksdb_transactiondb_t* txn_db,
-                            const rocksdb_readoptions_t* options,
-                            rocksdb_column_family_handle_t* column_family,
-                            const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  char* val;
-  val = rocksdb_transactiondb_get_cf(txn_db, options, column_family, key,
-                                     strlen(key), &val_len, &err);
-  CheckNoError(err);
-  CheckEqual(expected, val, val_len);
-  Free(&val);
-}
-
 int main(int argc, char** argv) {
  rocksdb_t* db;
  rocksdb_comparator_t* cmp;
  rocksdb_cache_t* cache;
-  rocksdb_dbpath_t *dbpath;
  rocksdb_env_t* env;
  rocksdb_options_t* options;
  rocksdb_compactoptions_t* coptions;
@@ -401,14 +385,8 @@ int main(int argc, char** argv) {
           GetTempDir(),
           ((int)geteuid()));

-  snprintf(dbpathname, sizeof(dbpathname),
-           "%s/rocksdb_c_test-%d-dbpath",
-           GetTempDir(),
-           ((int) geteuid()));
-
  StartPhase("create_objects");
  cmp = rocksdb_comparator_create(NULL, CmpDestroy, CmpCompare, CmpName);
-  dbpath = rocksdb_dbpath_create(dbpathname, 1024 * 1024);
  env = rocksdb_create_default_env();
  cache = rocksdb_cache_create_lru(100000);

@@ -1379,18 +1357,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
    CheckTxnDBGet(txn_db, roptions, "foo", NULL);

-    // write batch into TransactionDB
-    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
-    rocksdb_writebatch_put(wb, "foo", 3, "a", 1);
-    rocksdb_writebatch_clear(wb);
-    rocksdb_writebatch_put(wb, "bar", 3, "b", 1);
-    rocksdb_writebatch_put(wb, "box", 3, "c", 1);
-    rocksdb_writebatch_delete(wb, "bar", 3);
-    rocksdb_transactiondb_write(txn_db, woptions, wb, &err);
-    rocksdb_writebatch_destroy(wb);
-    CheckTxnDBGet(txn_db, roptions, "box", "c");
-    CheckNoError(err);
-
    // begin a transaction
    txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL);
    // put
@@ -1447,23 +1413,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
    CheckTxnDBGet(txn_db, roptions, "bar", NULL);

-    // Column families.
-    rocksdb_column_family_handle_t* cfh;
-    cfh = rocksdb_transactiondb_create_column_family(txn_db, options,
-                                                     "txn_db_cf", &err);
-    CheckNoError(err);
-
-    rocksdb_transactiondb_put_cf(txn_db, woptions, cfh, "cf_foo", 6, "cf_hello",
-                                 8, &err);
-    CheckNoError(err);
-    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", "cf_hello");
-
-    rocksdb_transactiondb_delete_cf(txn_db, woptions, cfh, "cf_foo", 6, &err);
-    CheckNoError(err);
-    CheckTxnDBGetCF(txn_db, roptions, cfh, "cf_foo", NULL);
-
-    rocksdb_column_family_handle_destroy(cfh);
-
    // close and destroy
    rocksdb_transaction_destroy(txn);
    rocksdb_transactiondb_close(txn_db);
@@ -1491,18 +1440,6 @@ int main(int argc, char** argv) {
    CheckNoError(err);
  }

-  // Simple sanity check that options setting db_paths work.
-  StartPhase("open_db_paths");
-  {
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-
-    const rocksdb_dbpath_t* paths[1] = {dbpath};
-    rocksdb_options_set_db_paths(options, paths, 1);
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-  }
-  
  StartPhase("cleanup");
  rocksdb_close(db);
  rocksdb_options_destroy(options);
@@ -1512,7 +1449,6 @@ int main(int argc, char** argv) {
  rocksdb_compactoptions_destroy(coptions);
  rocksdb_cache_destroy(cache);
  rocksdb_comparator_destroy(cmp);
-  rocksdb_dbpath_destroy(dbpath);
  rocksdb_env_destroy(env);

  fprintf(stderr, "PASS\n");
@@ -521,7 +521,6 @@ TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) {
  }
 }

-#ifndef ROCKSDB_LITE
 TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {
  Open();

@@ -543,7 +542,6 @@ TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {

  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
-#endif  // !ROCKSDB_LITE

 class FlushEmptyCFTestWithParam : public ColumnFamilyTest,
                                  public testing::WithParamInterface<bool> {
@@ -169,7 +169,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
      bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
      is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
      is_manual_compaction_(_manual_compaction),
-      is_trivial_move_(false),
      compaction_reason_(_compaction_reason) {
  MarkFilesBeingCompacted(true);
  if (is_manual_compaction_) {
@@ -284,32 +283,32 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
  assert(input_version_ != nullptr);
  assert(level_ptrs != nullptr);
  assert(level_ptrs->size() == static_cast<size_t>(number_levels_));
-  assert(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO);
-  if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
-    return bottommost_level_;
-  }
-  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel &&
-      output_level_ == 0) {
-    return false;
-  }
-  // Maybe use binary search to find right entry instead of linear search?
-  const Comparator* user_cmp = cfd_->user_comparator();
-  for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
-    const std::vector<FileMetaData*>& files = input_vstorage_->LevelFiles(lvl);
-    for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
-      auto* f = files[level_ptrs->at(lvl)];
-      if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
-        // We've advanced far enough
-        if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
-          // Key falls in this file's range, so definitely
-          // exists beyond output level
-          return false;
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
+    if (output_level_ == 0) {
+      return false;
+    }
+    // Maybe use binary search to find right entry instead of linear search?
+    const Comparator* user_cmp = cfd_->user_comparator();
+    for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
+      const std::vector<FileMetaData*>& files =
+          input_vstorage_->LevelFiles(lvl);
+      for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
+        auto* f = files[level_ptrs->at(lvl)];
+        if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
+          // We've advanced far enough
+          if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
+            // Key falls in this file's range, so definitely
+            // exists beyond output level
+            return false;
+          }
+          break;
        }
-        break;
      }
    }
+    return true;
+  } else {
+    return bottommost_level_;
  }
-  return true;
 }

 // Mark (or clear) each file that is being compacted
@@ -594,9 +594,6 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {

  double read_write_amp = 0.0;
  double write_amp = 0.0;
-  double bytes_read_per_sec = 0;
-  double bytes_written_per_sec = 0;
-
  if (stats.bytes_read_non_output_levels > 0) {
    read_write_amp = (stats.bytes_written + stats.bytes_read_output_level +
                      stats.bytes_read_non_output_levels) /
@@ -604,22 +601,17 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
    write_amp = stats.bytes_written /
                static_cast<double>(stats.bytes_read_non_output_levels);
  }
-  if (stats.micros > 0) {
-    bytes_read_per_sec =
-        (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
-        static_cast<double>(stats.micros);
-    bytes_written_per_sec =
-        stats.bytes_written / static_cast<double>(stats.micros);
-  }
-
  ROCKS_LOG_BUFFER(
      log_buffer_,
      "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
      "files in(%d, %d) out(%d) "
      "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
      "write-amplify(%.1f) %s, records in: %d, records dropped: %d\n",
-      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
-      bytes_written_per_sec, compact_->compaction->output_level(),
+      cfd->GetName().c_str(), vstorage->LevelSummary(&tmp),
+      (stats.bytes_read_non_output_levels + stats.bytes_read_output_level) /
+          static_cast<double>(stats.micros),
+      stats.bytes_written / static_cast<double>(stats.micros),
+      compact_->compaction->output_level(),
      stats.num_input_files_in_non_output_levels,
      stats.num_input_files_in_output_level, stats.num_output_files,
      stats.bytes_read_non_output_levels / 1048576.0,
@@ -234,6 +234,11 @@ bool CompactionPicker::ExpandInputsToCleanCut(const std::string& cf_name,
  // If, after the expansion, there are files that are already under
  // compaction, then we must drop/cancel this compaction.
  if (AreFilesInCompaction(inputs->files)) {
+    ROCKS_LOG_WARN(
+        ioptions_.info_log,
+        "[%s] ExpandWhileOverlapping() failure because some of the necessary"
+        " compaction input files are currently being compacted.",
+        cf_name.c_str());
    return false;
  }
  return true;
@@ -1101,7 +1106,11 @@ void LevelCompactionBuilder::SetupInitialFiles() {
      }
      output_level_ =
          (start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
-      if (PickFileToCompact()) {
+      if (PickFileToCompact() &&
+          compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
+                                                     &start_level_inputs_) &&
+          !compaction_picker_->FilesRangeOverlapWithCompaction(
+              {start_level_inputs_}, output_level_)) {
        // found the compaction!
        if (start_level_ == 0) {
          // L0 score = `num L0 files` / `level0_file_num_compaction_trigger`
@@ -1323,10 +1332,12 @@ bool LevelCompactionBuilder::PickFileToCompact() {
  const std::vector<FileMetaData*>& level_files =
      vstorage_->LevelFiles(start_level_);

-  unsigned int cmp_idx;
-  for (cmp_idx = vstorage_->NextCompactionIndex(start_level_);
-       cmp_idx < file_size.size(); cmp_idx++) {
-    int index = file_size[cmp_idx];
+  // record the first file that is not yet compacted
+  int nextIndex = -1;
+
+  for (unsigned int i = vstorage_->NextCompactionIndex(start_level_);
+       i < file_size.size(); i++) {
+    int index = file_size[i];
    auto* f = level_files[index];

    // do not pick a file to compact if it is being compacted
@@ -1335,42 +1346,27 @@ bool LevelCompactionBuilder::PickFileToCompact() {
      continue;
    }

-    start_level_inputs_.files.push_back(f);
-    start_level_inputs_.level = start_level_;
-    if (!compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                    &start_level_inputs_) ||
-        compaction_picker_->FilesRangeOverlapWithCompaction(
-            {start_level_inputs_}, output_level_)) {
-      // A locked (pending compaction) input-level file was pulled in due to
-      // user-key overlap.
-      start_level_inputs_.clear();
-      continue;
+    // remember the startIndex for the next call to PickCompaction
+    if (nextIndex == -1) {
+      nextIndex = i;
    }

-    // Now that input level is fully expanded, we check whether any output files
-    // are locked due to pending compaction.
-    //
-    // Note we rely on ExpandInputsToCleanCut() to tell us whether any output-
-    // level files are locked, not just the extra ones pulled in for user-key
-    // overlap.
-    InternalKey smallest, largest;
-    compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
-    CompactionInputFiles output_level_inputs;
-    output_level_inputs.level = output_level_;
-    vstorage_->GetOverlappingInputs(output_level_, &smallest, &largest,
-                                    &output_level_inputs.files);
-    if (!output_level_inputs.empty() &&
-        !compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
-                                                    &output_level_inputs)) {
-      start_level_inputs_.clear();
+    // Do not pick this file if its parents at level+1 are being compacted.
+    // Maybe we can avoid redoing this work in SetupOtherInputs
+    parent_index_ = -1;
+    if (compaction_picker_->IsRangeInCompaction(vstorage_, &f->smallest,
+                                                &f->largest, output_level_,
+                                                &parent_index_)) {
      continue;
    }
+    start_level_inputs_.files.push_back(f);
+    start_level_inputs_.level = start_level_;
    base_index_ = index;
    break;
  }

  // store where to start the iteration in the next call to PickCompaction
-  vstorage_->SetNextCompactionIndex(start_level_, cmp_idx);
+  vstorage_->SetNextCompactionIndex(start_level_, nextIndex);

  return start_level_inputs_.size() > 0;
 }
@@ -512,7 +512,7 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
        kFileSize, 0, i * 100, i * 100 + 99);
    current_size += kFileSize;
    UpdateVersionStorageInfo();
-    ASSERT_EQ(fifo_compaction_picker.NeedsCompaction(vstorage_.get()),
+    ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
              vstorage_->CompactionScore(0) >= 1);
  }
 }
@@ -852,80 +852,6 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
  ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
 }

-TEST_F(CompactionPickerTest, OverlappingUserKeys10) {
-  // Locked file encountered when pulling in extra input-level files with same
-  // user keys. Verify we pick the next-best file from the same input level.
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  // file_number 2U is largest and thus first choice. But it overlaps with
-  // file_number 1U which is being compacted. So instead we pick the next-
-  // biggest file, 3U, which is eligible for compaction.
-  Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
-      "150" /* largest */, 1U /* file_size */);
-  file_map_[1U].first->being_compacted = true;
-  Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
-      "200" /* largest */, 1000000000U /* file_size */, 0 /* smallest_seq */,
-      0 /* largest_seq */);
-  Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 900000000U /* file_size */);
-  Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
-      "150" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "151" /* smallest */,
-      "200" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 1U /* file_size */);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys11) {
-  // Locked file encountered when pulling in extra output-level files with same
-  // user keys. Expected to skip that compaction and pick the next-best choice.
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  // score(L1) = 3.7
-  // score(L2) = 1.85
-  // There is no eligible file in L1 to compact since both candidates pull in
-  // file_number 5U, which overlaps with a file pending compaction (6U). The
-  // first eligible compaction is from L2->L3.
-  Add(1 /* level */, 2U /* file_number */, "151" /* smallest */,
-      "200" /* largest */, 1000000000U /* file_size */);
-  Add(1 /* level */, 3U /* file_number */, "201" /* smallest */,
-      "250" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 4U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 5000000000U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
-      "201" /* largest */, 1U /* file_size */);
-  Add(2 /* level */, 6U /* file_number */, "201" /* smallest */,
-      "249" /* largest */, 1U /* file_size */, 0 /* smallest_seq */,
-      0 /* largest_seq */);
-  file_map_[6U].first->being_compacted = true;
-  Add(3 /* level */, 7U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 1U /* file_size */);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(1U, compaction->num_input_files(1));
-  ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
-}
-
 TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
  NewVersionStorage(6, kCompactionStyleLevel);
  mutable_cf_options_.level0_file_num_compaction_trigger = 2;
@@ -1390,49 +1316,6 @@ TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
  ASSERT_FALSE(compaction->IsTrivialMove());
 }

-TEST_F(CompactionPickerTest, CacheNextCompactionIndex) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-
-  Add(1 /* level */, 1U /* file_number */, "100" /* smallest */,
-      "149" /* largest */, 1000000000U /* file_size */);
-  file_map_[1U].first->being_compacted = true;
-  Add(1 /* level */, 2U /* file_number */, "150" /* smallest */,
-      "199" /* largest */, 900000000U /* file_size */);
-  Add(1 /* level */, 3U /* file_number */, "200" /* smallest */,
-      "249" /* largest */, 800000000U /* file_size */);
-  Add(1 /* level */, 4U /* file_number */, "250" /* smallest */,
-      "299" /* largest */, 700000000U /* file_size */);
-  Add(2 /* level */, 5U /* file_number */, "150" /* smallest */,
-      "199" /* largest */, 1U /* file_size */);
-  file_map_[5U].first->being_compacted = true;
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(0U, compaction->num_input_files(1));
-  ASSERT_EQ(3U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2, vstorage_->NextCompactionIndex(1 /* level */));
-
-  compaction.reset(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(1U, compaction->num_input_levels());
-  ASSERT_EQ(1U, compaction->num_input_files(0));
-  ASSERT_EQ(0U, compaction->num_input_files(1));
-  ASSERT_EQ(4U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(3, vstorage_->NextCompactionIndex(1 /* level */));
-
-  compaction.reset(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() == nullptr);
-  ASSERT_EQ(4, vstorage_->NextCompactionIndex(1 /* level */));
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -9,50 +9,19 @@
 #include "rocksdb/convenience.h"

 #include "db/db_impl.h"
-#include "util/cast_util.h"

 namespace rocksdb {

 void CancelAllBackgroundWork(DB* db, bool wait) {
-  (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
-      ->CancelAllBackgroundWork(wait);
+  (dynamic_cast<DBImpl*>(db->GetRootDB()))->CancelAllBackgroundWork(wait);
 }

 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
                          const Slice* begin, const Slice* end) {
-  return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
+  return (dynamic_cast<DBImpl*>(db->GetRootDB()))
      ->DeleteFilesInRange(column_family, begin, end);
 }

-Status VerifySstFileChecksum(const Options& options,
-                             const EnvOptions& env_options,
-                             const std::string& file_path) {
-  unique_ptr<RandomAccessFile> file;
-  uint64_t file_size;
-  InternalKeyComparator internal_comparator(options.comparator);
-  ImmutableCFOptions ioptions(options);
-
-  Status s = ioptions.env->NewRandomAccessFile(file_path, &file, env_options);
-  if (s.ok()) {
-    s = ioptions.env->GetFileSize(file_path, &file_size);
-  } else {
-    return s;
-  }
-  unique_ptr<TableReader> table_reader;
-  std::unique_ptr<RandomAccessFileReader> file_reader(
-      new RandomAccessFileReader(std::move(file), file_path));
-  s = ioptions.table_factory->NewTableReader(
-      TableReaderOptions(ioptions, env_options, internal_comparator,
-                         false /* skip_filters */, -1 /* level */),
-      std::move(file_reader), file_size, &table_reader,
-      false /* prefetch_index_and_filter_in_cache */);
-  if (!s.ok()) {
-    return s;
-  }
-  s = table_reader->VerifyChecksum();
-  return s;
-}
-
 }  // namespace rocksdb

 #endif  // ROCKSDB_LITE
@@ -20,7 +20,6 @@
 #include "db/log_format.h"
 #include "db/version_set.h"
 #include "rocksdb/cache.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/env.h"
 #include "rocksdb/table.h"
 #include "rocksdb/write_batch.h"
@@ -153,7 +152,7 @@ class CorruptionTest : public testing::Test {
    struct stat sbuf;
    if (stat(fname.c_str(), &sbuf) != 0) {
      const char* msg = strerror(errno);
-      FAIL() << fname << ": " << msg;
+      ASSERT_TRUE(false) << fname << ": " << msg;
    }

    if (offset < 0) {
@@ -180,9 +179,6 @@ class CorruptionTest : public testing::Test {
    }
    s = WriteStringToFile(Env::Default(), contents, fname);
    ASSERT_TRUE(s.ok()) << s.ToString();
-    Options options;
-    EnvOptions env_options;
-    ASSERT_NOK(VerifySstFileChecksum(options, env_options, fname));
  }

  void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
@@ -217,7 +213,7 @@ class CorruptionTest : public testing::Test {
        return;
      }
    }
-    FAIL() << "no file found at level";
+    ASSERT_TRUE(false) << "no file found at level";
  }


@@ -316,7 +312,6 @@ TEST_F(CorruptionTest, TableFile) {

  Corrupt(kTableFile, 100, 1);
  Check(99, 99);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, TableFileIndexData) {
@@ -332,11 +327,9 @@ TEST_F(CorruptionTest, TableFileIndexData) {
  // corrupt an index block of an entire file
  Corrupt(kTableFile, -2000, 500);
  Reopen();
-  dbi = reinterpret_cast<DBImpl*>(db_);
  // one full file should be readable, since only one was corrupted
  // the other file should be fully non-readable, since index was corrupted
  Check(5000, 5000);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, MissingDescriptor) {
@@ -396,12 +389,10 @@ TEST_F(CorruptionTest, CompactionInputError) {

  Corrupt(kTableFile, 100, 1);
  Check(9, 9);
-  ASSERT_NOK(dbi->VerifyChecksum());

  // Force compactions by writing lots of values
  Build(10000);
  Check(10000, 10000);
-  ASSERT_NOK(dbi->VerifyChecksum());
 }

 TEST_F(CorruptionTest, CompactionInputErrorParanoid) {
@@ -433,7 +424,6 @@ TEST_F(CorruptionTest, CompactionInputErrorParanoid) {

  CorruptTableFileAtLevel(0, 100, 1);
  Check(9, 9);
-  ASSERT_NOK(dbi->VerifyChecksum());

  // Write must eventually fail because of corrupted table
  Status s;
@@ -455,7 +445,6 @@ TEST_F(CorruptionTest, UnrelatedKeys) {
  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
  dbi->TEST_FlushMemTable();
  Corrupt(kTableFile, 100, 1);
-  ASSERT_NOK(dbi->VerifyChecksum());

  std::string tmp1, tmp2;
  ASSERT_OK(db_->Put(WriteOptions(), Key(1000, &tmp1), Value(1000, &tmp2)));
@@ -175,8 +175,6 @@ TEST_F(DBBasicTest, LevelLimitReopen) {
  int i = 0;
  while (NumTableFilesAtLevel(2, 1) == 0) {
    ASSERT_OK(Put(1, Key(i++), value));
-    dbfull()->TEST_WaitForFlushMemTable();
-    dbfull()->TEST_WaitForCompact();
  }

  options.num_levels = 1;
@@ -497,7 +497,7 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
        options.compression = kNoCompression;
        break;
      default:
-        FAIL();
+        ASSERT_TRUE(false);
    }
    CreateAndReopenWithCF({"pikachu"}, options);
    // default column family doesn't have block cache
@@ -560,7 +560,7 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
        ASSERT_EQ(TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT), 0);
        break;
      default:
-        FAIL();
+        ASSERT_TRUE(false);
    }

    options.create_if_missing = true;
@@ -1,7 +1,9 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//  This source code is also licensed under the GPLv2 license found in the
+//  COPYING file in the root directory of this source tree.
 //
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
@@ -67,7 +67,6 @@
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
@@ -81,7 +80,6 @@
 #include "table/merging_iterator.h"
 #include "table/table_builder.h"
 #include "table/two_level_iterator.h"
-#include "tools/sst_dump_tool_imp.h"
 #include "util/auto_roll_logger.h"
 #include "util/autovector.h"
 #include "util/build_version.h"
@@ -170,7 +168,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
      last_batch_group_size_(0),
      unscheduled_flushes_(0),
      unscheduled_compactions_(0),
-      bg_bottom_compaction_scheduled_(0),
      bg_compaction_scheduled_(0),
      num_running_compactions_(0),
      bg_flush_scheduled_(0),
@@ -245,8 +242,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
    return;
  }
  // Wait for background work to finish
-  while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-         bg_flush_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_) {
    bg_cv_.Wait();
  }
 }
@@ -256,18 +252,15 @@ DBImpl::~DBImpl() {
  // marker. After this we do a variant of the waiting and unschedule work
  // (to consider: moving all the waiting into CancelAllBackgroundWork(true))
  CancelAllBackgroundWork(false);
-  int bottom_compactions_unscheduled =
-      env_->UnSchedule(this, Env::Priority::BOTTOM);
  int compactions_unscheduled = env_->UnSchedule(this, Env::Priority::LOW);
  int flushes_unscheduled = env_->UnSchedule(this, Env::Priority::HIGH);
  mutex_.Lock();
-  bg_bottom_compaction_scheduled_ -= bottom_compactions_unscheduled;
  bg_compaction_scheduled_ -= compactions_unscheduled;
  bg_flush_scheduled_ -= flushes_unscheduled;

  // Wait for background work to finish
-  while (bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-         bg_flush_scheduled_ || bg_purge_scheduled_) {
+  while (bg_compaction_scheduled_ || bg_flush_scheduled_ ||
+         bg_purge_scheduled_) {
    TEST_SYNC_POINT("DBImpl::~DBImpl:WaitJob");
    bg_cv_.Wait();
  }
@@ -1409,7 +1402,8 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
    return NewDBIterator(
        env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
        kMaxSequenceNumber,
-        sv->mutable_cf_options.max_sequential_skip_in_iterations);
+        sv->mutable_cf_options.max_sequential_skip_in_iterations,
+        sv->version_number);
 #endif
  } else {
    SequenceNumber latest_snapshot = versions_->LastSequence();
@@ -1464,10 +1458,9 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
    // likely that any iterator pointer is close to the iterator it points to so
    // that they are likely to be in the same cache line and/or page.
    ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(), snapshot,
+        env_, read_options, *cfd->ioptions(), cfd->user_comparator(), snapshot,
        sv->mutable_cf_options.max_sequential_skip_in_iterations,
-        sv->version_number,
-        ((read_options.snapshot != nullptr) ? nullptr : this), cfd);
+        sv->version_number);

    InternalIterator* internal_iter =
        NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
@@ -1518,7 +1511,8 @@ Status DBImpl::NewIterators(
      iterators->push_back(NewDBIterator(
          env_, read_options, *cfd->ioptions(), cfd->user_comparator(), iter,
          kMaxSequenceNumber,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations));
+          sv->mutable_cf_options.max_sequential_skip_in_iterations,
+          sv->version_number));
    }
 #endif
  } else {
@@ -1536,10 +1530,9 @@ Status DBImpl::NewIterators(
              : latest_snapshot;

      ArenaWrappedDBIter* db_iter = NewArenaWrappedDbIterator(
-          env_, read_options, *cfd->ioptions(), snapshot,
-          sv->mutable_cf_options.max_sequential_skip_in_iterations,
-          sv->version_number,
-          ((read_options.snapshot != nullptr) ? nullptr : this), cfd);
+          env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
+          snapshot, sv->mutable_cf_options.max_sequential_skip_in_iterations,
+          sv->version_number);
      InternalIterator* internal_iter =
          NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
                              db_iter->GetRangeDelAggregator());
@@ -2300,7 +2293,7 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
    // Delete log files in the WAL dir
    for (const auto& file : walDirFiles) {
      if (ParseFileName(file, &number, &type) && type == kLogFile) {
-        Status del = env->DeleteFile(LogFileName(soptions.wal_dir, number));
+        Status del = env->DeleteFile(soptions.wal_dir + "/" + file);
        if (result.ok() && !del.ok()) {
          result = del;
        }
@@ -2742,54 +2735,6 @@ Status DBImpl::IngestExternalFile(
  return status;
 }

-Status DBImpl::VerifyChecksum() {
-  Status s;
-  Options options;
-  EnvOptions env_options;
-  std::vector<ColumnFamilyData*> cfd_list;
-  {
-    InstrumentedMutexLock l(&mutex_);
-    for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (!cfd->IsDropped() && cfd->initialized()) {
-        cfd->Ref();
-        cfd_list.push_back(cfd);
-      }
-    }
-  }
-  std::vector<SuperVersion*> sv_list;
-  for (auto cfd : cfd_list) {
-    sv_list.push_back(cfd->GetReferencedSuperVersion(&mutex_));
-  }
-  for (auto& sv : sv_list) {
-    VersionStorageInfo* vstorage = sv->current->storage_info();
-    for (int i = 0; i < vstorage->num_non_empty_levels() && s.ok(); i++) {
-      for (size_t j = 0; j < vstorage->LevelFilesBrief(i).num_files && s.ok();
-           j++) {
-        const auto& fd = vstorage->LevelFilesBrief(i).files[j].fd;
-        std::string fname = TableFileName(immutable_db_options_.db_paths,
-                                          fd.GetNumber(), fd.GetPathId());
-        s = rocksdb::VerifySstFileChecksum(options, env_options, fname); 
-      }
-    }
-    if (!s.ok()) {
-      break;
-    }
-  }
-  {
-    InstrumentedMutexLock l(&mutex_);
-    for (auto sv : sv_list) {
-      if (sv && sv->Unref()) {
-        sv->Cleanup();
-        delete sv;
-      }
-    }
-    for (auto cfd : cfd_list) {
-        cfd->Unref();
-    }
-  }
-  return s;
-}
-
 void DBImpl::NotifyOnExternalFileIngested(
    ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job) {
 #ifndef ROCKSDB_LITE
@@ -235,11 +235,11 @@ class DBImpl : public DB {
      ColumnFamilyHandle* column_family,
      ColumnFamilyMetaData* metadata) override;

+  // experimental API
  Status SuggestCompactRange(ColumnFamilyHandle* column_family,
-                             const Slice* begin, const Slice* end) override;
+                             const Slice* begin, const Slice* end);

-  Status PromoteL0(ColumnFamilyHandle* column_family,
-                   int target_level) override;
+  Status PromoteL0(ColumnFamilyHandle* column_family, int target_level);

  // Similar to Write() but will call the callback once on the single write
  // thread to determine whether it is safe to perform the write.
@@ -293,8 +293,6 @@ class DBImpl : public DB {
      const std::vector<std::string>& external_files,
      const IngestExternalFileOptions& ingestion_options) override;

-  virtual Status VerifyChecksum() override;
-
 #endif  // ROCKSDB_LITE

  // Similar to GetSnapshot(), but also lets the db know that this snapshot
@@ -496,12 +494,6 @@ class DBImpl : public DB {

  const WriteController& write_controller() { return write_controller_; }

-  InternalIterator* NewInternalIterator(const ReadOptions&,
-                                        ColumnFamilyData* cfd,
-                                        SuperVersion* super_version,
-                                        Arena* arena,
-                                        RangeDelAggregator* range_del_agg);
-
  // hollow transactions shell used for recovery.
  // these will then be passed to TransactionDB so that
  // locks can be reacquired before writing can resume.
@@ -560,7 +552,6 @@ class DBImpl : public DB {
  void AddToLogsToFreeQueue(log::Writer* log_writer) {
    logs_to_free_queue_.push_back(log_writer);
  }
-  InstrumentedMutex* mutex() { return &mutex_; }

  Status NewDB();

@@ -575,6 +566,12 @@ class DBImpl : public DB {
  std::unordered_map<std::string, RecoveredTransaction*>
      recovered_transactions_;

+  InternalIterator* NewInternalIterator(const ReadOptions&,
+                                        ColumnFamilyData* cfd,
+                                        SuperVersion* super_version,
+                                        Arena* arena,
+                                        RangeDelAggregator* range_del_agg);
+
  // Except in DB::Open(), WriteOptionsFile can only be called when:
  // Persist options to options file.
  // If need_mutex_lock = false, the method will lock DB mutex.
@@ -633,9 +630,7 @@ class DBImpl : public DB {
 private:
  friend class DB;
  friend class InternalStats;
-  friend class PessimisticTransaction;
-  friend class WriteCommittedTxn;
-  friend class WritePreparedTxn;
+  friend class TransactionImpl;
 #ifndef ROCKSDB_LITE
  friend class ForwardIterator;
 #endif
@@ -660,7 +655,6 @@ class DBImpl : public DB {
    }
  };

-  struct PrepickedCompaction;
  struct PurgeFileInfo;

  // Recover the descriptor from persistent storage.  May do a significant
@@ -802,19 +796,14 @@ class DBImpl : public DB {
  void SchedulePendingPurge(std::string fname, FileType type, uint64_t number,
                            uint32_t path_id, int job_id);
  static void BGWorkCompaction(void* arg);
-  // Runs a pre-chosen universal compaction involving bottom level in a
-  // separate, bottom-pri thread pool.
-  static void BGWorkBottomCompaction(void* arg);
  static void BGWorkFlush(void* db);
  static void BGWorkPurge(void* arg);
  static void UnscheduleCallback(void* arg);
-  void BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
-                                Env::Priority bg_thread_pri);
+  void BackgroundCallCompaction(void* arg);
  void BackgroundCallFlush();
  void BackgroundCallPurge();
  Status BackgroundCompaction(bool* madeProgress, JobContext* job_context,
-                              LogBuffer* log_buffer,
-                              PrepickedCompaction* prepicked_compaction);
+                              LogBuffer* log_buffer, void* m = 0);
  Status BackgroundFlush(bool* madeProgress, JobContext* job_context,
                         LogBuffer* log_buffer);

@@ -1067,10 +1056,6 @@ class DBImpl : public DB {
  int unscheduled_flushes_;
  int unscheduled_compactions_;

-  // count how many background compactions are running or have been scheduled in
-  // the BOTTOM pool
-  int bg_bottom_compaction_scheduled_;
-
  // count how many background compactions are running or have been scheduled
  int bg_compaction_scheduled_;

@@ -1087,7 +1072,7 @@ class DBImpl : public DB {
  int bg_purge_scheduled_;

  // Information for a manual compaction
-  struct ManualCompactionState {
+  struct ManualCompaction {
    ColumnFamilyData* cfd;
    int input_level;
    int output_level;
@@ -1103,21 +1088,13 @@ class DBImpl : public DB {
    InternalKey* manual_end;      // how far we are compacting
    InternalKey tmp_storage;      // Used to keep track of compaction progress
    InternalKey tmp_storage1;     // Used to keep track of compaction progress
-  };
-  struct PrepickedCompaction {
-    // background compaction takes ownership of `compaction`.
    Compaction* compaction;
-    // caller retains ownership of `manual_compaction_state` as it is reused
-    // across background compactions.
-    ManualCompactionState* manual_compaction_state;  // nullptr if non-manual
  };
-  std::deque<ManualCompactionState*> manual_compaction_dequeue_;
+  std::deque<ManualCompaction*> manual_compaction_dequeue_;

  struct CompactionArg {
-    // caller retains ownership of `db`.
    DBImpl* db;
-    // background compaction takes ownership of `prepicked_compaction`.
-    PrepickedCompaction* prepicked_compaction;
+    ManualCompaction* m;
  };

  // Have we encountered a background error in paranoid mode?
@@ -1251,11 +1228,11 @@ class DBImpl : public DB {

  bool HasPendingManualCompaction();
  bool HasExclusiveManualCompaction();
-  void AddManualCompaction(ManualCompactionState* m);
-  void RemoveManualCompaction(ManualCompactionState* m);
-  bool ShouldntRunManualCompaction(ManualCompactionState* m);
+  void AddManualCompaction(ManualCompaction* m);
+  void RemoveManualCompaction(ManualCompaction* m);
+  bool ShouldntRunManualCompaction(ManualCompaction* m);
  bool HaveManualCompaction(ColumnFamilyData* cfd);
-  bool MCOverlap(ManualCompactionState* m, ManualCompactionState* m1);
+  bool MCOverlap(ManualCompaction* m, ManualCompaction* m1);

  size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;

@@ -612,8 +612,7 @@ Status DBImpl::CompactFilesImpl(
 Status DBImpl::PauseBackgroundWork() {
  InstrumentedMutexLock guard_lock(&mutex_);
  bg_compaction_paused_++;
-  while (bg_bottom_compaction_scheduled_ > 0 || bg_compaction_scheduled_ > 0 ||
-         bg_flush_scheduled_ > 0) {
+  while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_ > 0) {
    bg_cv_.Wait();
  }
  bg_work_paused_++;
@@ -809,7 +808,7 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,

  bool scheduled = false;
  bool manual_conflict = false;
-  ManualCompactionState manual;
+  ManualCompaction manual;
  manual.cfd = cfd;
  manual.input_level = input_level;
  manual.output_level = output_level;
@@ -859,8 +858,7 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
  AddManualCompaction(&manual);
  TEST_SYNC_POINT_CALLBACK("DBImpl::RunManualCompaction:NotScheduled", &mutex_);
  if (exclusive) {
-    while (bg_bottom_compaction_scheduled_ > 0 ||
-           bg_compaction_scheduled_ > 0) {
+    while (bg_compaction_scheduled_ > 0) {
      TEST_SYNC_POINT("DBImpl::RunManualCompaction:WaitScheduled");
      ROCKS_LOG_INFO(
          immutable_db_options_.info_log,
@@ -880,14 +878,14 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
  while (!manual.done) {
    assert(HasPendingManualCompaction());
    manual_conflict = false;
-    Compaction* compaction;
    if (ShouldntRunManualCompaction(&manual) || (manual.in_progress == true) ||
        scheduled ||
-        ((manual.manual_end = &manual.tmp_storage1) &&
-         ((compaction = manual.cfd->CompactRange(
-               *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
-               manual.output_level, manual.output_path_id, manual.begin,
-               manual.end, &manual.manual_end, &manual_conflict)) == nullptr) &&
+        ((manual.manual_end = &manual.tmp_storage1)&&(
+             (manual.compaction = manual.cfd->CompactRange(
+                  *manual.cfd->GetLatestMutableCFOptions(), manual.input_level,
+                  manual.output_level, manual.output_path_id, manual.begin,
+                  manual.end, &manual.manual_end, &manual_conflict)) ==
+             nullptr) &&
         manual_conflict)) {
      // exclusive manual compactions should not see a conflict during
      // CompactRange
@@ -900,16 +898,14 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
        manual.incomplete = false;
      }
    } else if (!scheduled) {
-      if (compaction == nullptr) {
+      if (manual.compaction == nullptr) {
        manual.done = true;
        bg_cv_.SignalAll();
        continue;
      }
      ca = new CompactionArg;
      ca->db = this;
-      ca->prepicked_compaction = new PrepickedCompaction;
-      ca->prepicked_compaction->manual_compaction_state = &manual;
-      ca->prepicked_compaction->compaction = compaction;
+      ca->m = &manual;
      manual.incomplete = false;
      bg_compaction_scheduled_++;
      env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
@@ -1051,7 +1047,7 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
         unscheduled_compactions_ > 0) {
    CompactionArg* ca = new CompactionArg;
    ca->db = this;
-    ca->prepicked_compaction = nullptr;
+    ca->m = nullptr;
    bg_compaction_scheduled_++;
    unscheduled_compactions_--;
    env_->Schedule(&DBImpl::BGWorkCompaction, ca, Env::Priority::LOW, this,
@@ -1156,23 +1152,7 @@ void DBImpl::BGWorkCompaction(void* arg) {
  delete reinterpret_cast<CompactionArg*>(arg);
  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::LOW);
  TEST_SYNC_POINT("DBImpl::BGWorkCompaction");
-  auto prepicked_compaction =
-      static_cast<PrepickedCompaction*>(ca.prepicked_compaction);
-  reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(
-      prepicked_compaction, Env::Priority::LOW);
-  delete prepicked_compaction;
-}
-
-void DBImpl::BGWorkBottomCompaction(void* arg) {
-  CompactionArg ca = *(static_cast<CompactionArg*>(arg));
-  delete static_cast<CompactionArg*>(arg);
-  IOSTATS_SET_THREAD_POOL_ID(Env::Priority::BOTTOM);
-  TEST_SYNC_POINT("DBImpl::BGWorkBottomCompaction");
-  auto* prepicked_compaction = ca.prepicked_compaction;
-  assert(prepicked_compaction && prepicked_compaction->compaction &&
-         !prepicked_compaction->manual_compaction_state);
-  ca.db->BackgroundCallCompaction(prepicked_compaction, Env::Priority::BOTTOM);
-  delete prepicked_compaction;
+  reinterpret_cast<DBImpl*>(ca.db)->BackgroundCallCompaction(ca.m);
 }

 void DBImpl::BGWorkPurge(void* db) {
@@ -1185,11 +1165,8 @@ void DBImpl::BGWorkPurge(void* db) {
 void DBImpl::UnscheduleCallback(void* arg) {
  CompactionArg ca = *(reinterpret_cast<CompactionArg*>(arg));
  delete reinterpret_cast<CompactionArg*>(arg);
-  if (ca.prepicked_compaction != nullptr) {
-    if (ca.prepicked_compaction->compaction != nullptr) {
-      delete ca.prepicked_compaction->compaction;
-    }
-    delete ca.prepicked_compaction;
+  if ((ca.m != nullptr) && (ca.m->compaction != nullptr)) {
+    delete ca.m->compaction;
  }
  TEST_SYNC_POINT("DBImpl::UnscheduleCallback");
 }
@@ -1316,9 +1293,9 @@ void DBImpl::BackgroundCallFlush() {
  }
 }

-void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
-                                      Env::Priority bg_thread_pri) {
+void DBImpl::BackgroundCallCompaction(void* arg) {
  bool made_progress = false;
+  ManualCompaction* m = reinterpret_cast<ManualCompaction*>(arg);
  JobContext job_context(next_job_id_.fetch_add(1), true);
  TEST_SYNC_POINT("BackgroundCallCompaction:0");
  MaybeDumpStats();
@@ -1336,11 +1313,9 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,
    auto pending_outputs_inserted_elem =
        CaptureCurrentFileNumberInPendingOutputs();

-    assert((bg_thread_pri == Env::Priority::BOTTOM &&
-            bg_bottom_compaction_scheduled_) ||
-           (bg_thread_pri == Env::Priority::LOW && bg_compaction_scheduled_));
-    Status s = BackgroundCompaction(&made_progress, &job_context, &log_buffer,
-                                    prepicked_compaction);
+    assert(bg_compaction_scheduled_);
+    Status s =
+        BackgroundCompaction(&made_progress, &job_context, &log_buffer, m);
    TEST_SYNC_POINT("BackgroundCallCompaction:1");
    if (!s.ok() && !s.IsShutdownInProgress()) {
      // Wait a little bit before retrying background compaction in
@@ -1386,24 +1361,17 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,

    assert(num_running_compactions_ > 0);
    num_running_compactions_--;
-    if (bg_thread_pri == Env::Priority::LOW) {
-      bg_compaction_scheduled_--;
-    } else {
-      assert(bg_thread_pri == Env::Priority::BOTTOM);
-      bg_bottom_compaction_scheduled_--;
-    }
+    bg_compaction_scheduled_--;

    versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();

    // See if there's more work to be done
    MaybeScheduleFlushOrCompaction();
-    if (made_progress ||
-        (bg_compaction_scheduled_ == 0 &&
-         bg_bottom_compaction_scheduled_ == 0) ||
+    if (made_progress || bg_compaction_scheduled_ == 0 ||
        HasPendingManualCompaction()) {
      // signal if
      // * made_progress -- need to wakeup DelayWrite
-      // * bg_{bottom,}_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
+      // * bg_compaction_scheduled_ == 0 -- need to wakeup ~DBImpl
      // * HasPendingManualCompaction -- need to wakeup RunManualCompaction
      // If none of this is true, there is no need to signal since nobody is
      // waiting for it
@@ -1418,23 +1386,14 @@ void DBImpl::BackgroundCallCompaction(PrepickedCompaction* prepicked_compaction,

 Status DBImpl::BackgroundCompaction(bool* made_progress,
                                    JobContext* job_context,
-                                    LogBuffer* log_buffer,
-                                    PrepickedCompaction* prepicked_compaction) {
-  ManualCompactionState* manual_compaction =
-      prepicked_compaction == nullptr
-          ? nullptr
-          : prepicked_compaction->manual_compaction_state;
+                                    LogBuffer* log_buffer, void* arg) {
+  ManualCompaction* manual_compaction =
+      reinterpret_cast<ManualCompaction*>(arg);
  *made_progress = false;
  mutex_.AssertHeld();
  TEST_SYNC_POINT("DBImpl::BackgroundCompaction:Start");

  bool is_manual = (manual_compaction != nullptr);
-  unique_ptr<Compaction> c;
-  if (prepicked_compaction != nullptr &&
-      prepicked_compaction->compaction != nullptr) {
-    c.reset(prepicked_compaction->compaction);
-  }
-  bool is_prepicked = is_manual || c;

  // (manual_compaction->in_progress == false);
  bool trivial_move_disallowed =
@@ -1451,6 +1410,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
      manual_compaction->status = status;
      manual_compaction->done = true;
      manual_compaction->in_progress = false;
+      delete manual_compaction->compaction;
      manual_compaction = nullptr;
    }
    return status;
@@ -1461,11 +1421,13 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
    manual_compaction->in_progress = true;
  }

+  unique_ptr<Compaction> c;
  // InternalKey manual_end_storage;
  // InternalKey* manual_end = &manual_end_storage;
  if (is_manual) {
-    ManualCompactionState* m = manual_compaction;
+    ManualCompaction* m = manual_compaction;
    assert(m->in_progress);
+    c.reset(std::move(m->compaction));
    if (!c) {
      m->done = true;
      m->manual_end = nullptr;
@@ -1487,7 +1449,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
               ? "(end)"
               : m->manual_end->DebugString().c_str()));
    }
-  } else if (!is_prepicked && !compaction_queue_.empty()) {
+  } else if (!compaction_queue_.empty()) {
    if (HaveManualCompaction(compaction_queue_.front())) {
      // Can't compact right now, but try again later
      TEST_SYNC_POINT("DBImpl::BackgroundCompaction()::Conflict");
@@ -1639,28 +1601,6 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,

    // Clear Instrument
    ThreadStatusUtil::ResetThreadStatus();
-  } else if (c->column_family_data()->ioptions()->compaction_style ==
-                 kCompactionStyleUniversal &&
-             !is_prepicked && c->output_level() > 0 &&
-             c->output_level() ==
-                 c->column_family_data()
-                     ->current()
-                     ->storage_info()
-                     ->MaxOutputLevel(
-                         immutable_db_options_.allow_ingest_behind) &&
-             env_->GetBackgroundThreads(Env::Priority::BOTTOM) > 0) {
-    // Forward universal compactions involving last level to the bottom pool
-    // if it exists, such that long-running compactions can't block short-
-    // lived ones, like L0->L0s.
-    TEST_SYNC_POINT("DBImpl::BackgroundCompaction:ForwardToBottomPriPool");
-    CompactionArg* ca = new CompactionArg;
-    ca->db = this;
-    ca->prepicked_compaction = new PrepickedCompaction;
-    ca->prepicked_compaction->compaction = c.release();
-    ca->prepicked_compaction->manual_compaction_state = nullptr;
-    ++bg_bottom_compaction_scheduled_;
-    env_->Schedule(&DBImpl::BGWorkBottomCompaction, ca, Env::Priority::BOTTOM,
-                   this, &DBImpl::UnscheduleCallback);
  } else {
    int output_level  __attribute__((unused)) = c->output_level();
    TEST_SYNC_POINT_CALLBACK("DBImpl::BackgroundCompaction:NonTrivial",
@@ -1724,7 +1664,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
  }

  if (is_manual) {
-    ManualCompactionState* m = manual_compaction;
+    ManualCompaction* m = manual_compaction;
    if (!status.ok()) {
      m->status = status;
      m->done = true;
@@ -1767,13 +1707,13 @@ bool DBImpl::HasPendingManualCompaction() {
  return (!manual_compaction_dequeue_.empty());
 }

-void DBImpl::AddManualCompaction(DBImpl::ManualCompactionState* m) {
+void DBImpl::AddManualCompaction(DBImpl::ManualCompaction* m) {
  manual_compaction_dequeue_.push_back(m);
 }

-void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
+void DBImpl::RemoveManualCompaction(DBImpl::ManualCompaction* m) {
  // Remove from queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if (m == (*it)) {
@@ -1786,17 +1726,16 @@ void DBImpl::RemoveManualCompaction(DBImpl::ManualCompactionState* m) {
  return;
 }

-bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {
+bool DBImpl::ShouldntRunManualCompaction(ManualCompaction* m) {
  if (num_running_ingest_file_ > 0) {
    // We need to wait for other IngestExternalFile() calls to finish
    // before running a manual compaction.
    return true;
  }
  if (m->exclusive) {
-    return (bg_bottom_compaction_scheduled_ > 0 ||
-            bg_compaction_scheduled_ > 0);
+    return (bg_compaction_scheduled_ > 0);
  }
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  bool seen = false;
  while (it != manual_compaction_dequeue_.end()) {
@@ -1817,7 +1756,7 @@ bool DBImpl::ShouldntRunManualCompaction(ManualCompactionState* m) {

 bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {
  // Remove from priority queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if ((*it)->exclusive) {
@@ -1835,7 +1774,7 @@ bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {

 bool DBImpl::HasExclusiveManualCompaction() {
  // Remove from priority queue
-  std::deque<ManualCompactionState*>::iterator it =
+  std::deque<ManualCompaction*>::iterator it =
      manual_compaction_dequeue_.begin();
  while (it != manual_compaction_dequeue_.end()) {
    if ((*it)->exclusive) {
@@ -1846,7 +1785,7 @@ bool DBImpl::HasExclusiveManualCompaction() {
  return false;
 }

-bool DBImpl::MCOverlap(ManualCompactionState* m, ManualCompactionState* m1) {
+bool DBImpl::MCOverlap(ManualCompaction* m, ManualCompaction* m1) {
  if ((m->exclusive) || (m1->exclusive)) {
    return true;
  }
@@ -112,9 +112,7 @@ Status DBImpl::TEST_WaitForCompact() {
  // OR flush to finish.

  InstrumentedMutexLock l(&mutex_);
-  while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
-          bg_flush_scheduled_) &&
-         bg_error_.ok()) {
+  while ((bg_compaction_scheduled_ || bg_flush_scheduled_) && bg_error_.ok()) {
    bg_cv_.Wait();
  }
  return bg_error_;
@@ -368,9 +368,6 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
    candidate_files.emplace_back(
        MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
        file->fd.GetPathId());
-    if (file->table_reader_handle) {
-      table_cache_->Release(file->table_reader_handle);
-    }
    delete file;
  }

@@ -58,7 +58,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
  SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
  SequenceNumber latest_snapshot = versions_->LastSequence();
  auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(),
+      env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
      (read_options.snapshot != nullptr
           ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                 ->number_
@@ -87,7 +87,7 @@ Status DBImplReadOnly::NewIterators(
    auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
    auto* sv = cfd->GetSuperVersion()->Ref();
    auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(),
+        env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
        (read_options.snapshot != nullptr
             ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                   ->number_
@@ -313,7 +313,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
      versions_->SetLastSequence(last_sequence);
    }
    MemTableInsertStatusCheck(w.status);
-    write_thread_.ExitAsBatchGroupLeader(write_group, w.status);
+    write_thread_.ExitAsBatchGroupLeader(write_group, status);
  }

  if (status.ok()) {
@@ -523,7 +523,7 @@ Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
  if (!w.CallbackFailed()) {
    WriteCallbackStatusCheck(status);
  }
-  nonmem_write_thread_.ExitAsBatchGroupLeader(write_group, w.status);
+  nonmem_write_thread_.ExitAsBatchGroupLeader(write_group, status);
  if (status.ok()) {
    status = w.FinalStatus();
  }
@@ -204,6 +204,7 @@ TEST_F(DBIOFailureTest, ManifestWriteError) {
    ASSERT_EQ("bar2", Get("foo2"));
  }
 }
+#endif  // ROCKSDB_LITE

 TEST_F(DBIOFailureTest, PutFailsParanoid) {
  // Test the following:
@@ -558,7 +559,6 @@ TEST_F(DBIOFailureTest, CompactionSstSyncError) {
  ASSERT_EQ("bar3", Get(1, "foo"));
 }
 #endif  // !(defined NDEBUG) || !defined(OS_WIN)
-#endif  // ROCKSDB_LITE
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -101,12 +101,12 @@ class DBIter: public Iterator {
    uint64_t bytes_read_;
  };

-  DBIter(Env* _env, const ReadOptions& read_options,
+  DBIter(Env* env, const ReadOptions& read_options,
         const ImmutableCFOptions& cf_options, const Comparator* cmp,
         InternalIterator* iter, SequenceNumber s, bool arena_mode,
-         uint64_t max_sequential_skip_in_iterations)
+         uint64_t max_sequential_skip_in_iterations, uint64_t version_number)
      : arena_mode_(arena_mode),
-        env_(_env),
+        env_(env),
        logger_(cf_options.info_log),
        user_comparator_(cmp),
        merge_operator_(cf_options.merge_operator),
@@ -116,6 +116,7 @@ class DBIter: public Iterator {
        valid_(false),
        current_entry_is_merged_(false),
        statistics_(cf_options.statistics),
+        version_number_(version_number),
        iterate_upper_bound_(read_options.iterate_upper_bound),
        prefix_same_as_start_(read_options.prefix_same_as_start),
        pin_thru_lifetime_(read_options.pin_data),
@@ -187,7 +188,10 @@ class DBIter: public Iterator {
    }
    if (prop_name == "rocksdb.iterator.super-version-number") {
      // First try to pass the value returned from inner iterator.
-      return iter_->GetProperty(prop_name, prop);
+      if (!iter_->GetProperty(prop_name, prop).ok()) {
+        *prop = ToString(version_number_);
+      }
+      return Status::OK();
    } else if (prop_name == "rocksdb.iterator.is-key-pinned") {
      if (valid_) {
        *prop = (pin_thru_lifetime_ && saved_key_.IsKeyPinned()) ? "1" : "0";
@@ -205,9 +209,6 @@ class DBIter: public Iterator {
  virtual void SeekForPrev(const Slice& target) override;
  virtual void SeekToFirst() override;
  virtual void SeekToLast() override;
-  Env* env() { return env_; }
-  void set_sequence(uint64_t s) { sequence_ = s; }
-  void set_valid(bool v) { valid_ = v; }

 private:
  void ReverseToForward();
@@ -259,7 +260,7 @@ class DBIter: public Iterator {
  const Comparator* const user_comparator_;
  const MergeOperator* const merge_operator_;
  InternalIterator* iter_;
-  SequenceNumber sequence_;
+  SequenceNumber const sequence_;

  Status status_;
  IterKey saved_key_;
@@ -273,6 +274,7 @@ class DBIter: public Iterator {
  uint64_t max_skip_;
  uint64_t max_skippable_internal_keys_;
  uint64_t num_internal_keys_skipped_;
+  uint64_t version_number_;
  const Slice* iterate_upper_bound_;
  IterKey prefix_start_buf_;
  Slice prefix_start_key_;
@@ -1155,15 +1157,18 @@ Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                        const Comparator* user_key_comparator,
                        InternalIterator* internal_iter,
                        const SequenceNumber& sequence,
-                        uint64_t max_sequential_skip_in_iterations) {
-  DBIter* db_iter = new DBIter(env, read_options, cf_options,
-                               user_key_comparator, internal_iter, sequence,
-                               false, max_sequential_skip_in_iterations);
+                        uint64_t max_sequential_skip_in_iterations,
+                        uint64_t version_number) {
+  DBIter* db_iter = new DBIter(
+      env, read_options, cf_options, user_key_comparator, internal_iter,
+      sequence, false, max_sequential_skip_in_iterations, version_number);
  return db_iter;
 }

 ArenaWrappedDBIter::~ArenaWrappedDBIter() { db_iter_->~DBIter(); }

+void ArenaWrappedDBIter::SetDBIter(DBIter* iter) { db_iter_ = iter; }
+
 RangeDelAggregator* ArenaWrappedDBIter::GetRangeDelAggregator() {
  return db_iter_->GetRangeDelAggregator();
 }
@@ -1188,67 +1193,26 @@ inline Slice ArenaWrappedDBIter::value() const { return db_iter_->value(); }
 inline Status ArenaWrappedDBIter::status() const { return db_iter_->status(); }
 inline Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
                                              std::string* prop) {
-  if (prop_name == "rocksdb.iterator.super-version-number") {
-    // First try to pass the value returned from inner iterator.
-    if (!db_iter_->GetProperty(prop_name, prop).ok()) {
-      *prop = ToString(sv_number_);
-    }
-    return Status::OK();
-  }
  return db_iter_->GetProperty(prop_name, prop);
 }
-
-void ArenaWrappedDBIter::Init(Env* env, const ReadOptions& read_options,
-                              const ImmutableCFOptions& cf_options,
-                              const SequenceNumber& sequence,
-                              uint64_t max_sequential_skip_in_iteration,
-                              uint64_t version_number) {
-  auto mem = arena_.AllocateAligned(sizeof(DBIter));
-  db_iter_ = new (mem)
-      DBIter(env, read_options, cf_options, cf_options.user_comparator, nullptr,
-             sequence, true, max_sequential_skip_in_iteration);
-  sv_number_ = version_number;
-}
-
-Status ArenaWrappedDBIter::Refresh() {
-  if (cfd_ == nullptr || db_impl_ == nullptr) {
-    return Status::NotSupported("Creating renew iterator is not allowed.");
-  }
-  assert(db_iter_ != nullptr);
-  SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
-  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
-  if (sv_number_ != cur_sv_number) {
-    Env* env = db_iter_->env();
-    db_iter_->~DBIter();
-    arena_.~Arena();
-    new (&arena_) Arena();
-
-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_->mutex());
-    Init(env, read_options_, *(cfd_->ioptions()), latest_seq,
-         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number);
-
-    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, db_iter_->GetRangeDelAggregator());
-    SetIterUnderDBIter(internal_iter);
-  } else {
-    db_iter_->set_sequence(latest_seq);
-    db_iter_->set_valid(false);
-  }
-  return Status::OK();
+void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
+                                         void* arg2) {
+  db_iter_->RegisterCleanup(function, arg1, arg2);
 }

 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    DBImpl* db_impl, ColumnFamilyData* cfd) {
+    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number) {
  ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
-  iter->Init(env, read_options, cf_options, sequence,
-             max_sequential_skip_in_iterations, version_number);
-  if (db_impl != nullptr && cfd != nullptr) {
-    iter->StoreRefreshInfo(read_options, db_impl, cfd);
-  }
+  Arena* arena = iter->GetArena();
+  auto mem = arena->AllocateAligned(sizeof(DBIter));
+  DBIter* db_iter = new (mem)
+      DBIter(env, read_options, cf_options, user_key_comparator, nullptr,
+             sequence, true, max_sequential_skip_in_iterations, version_number);
+
+  iter->SetDBIter(db_iter);

  return iter;
 }
@@ -10,7 +10,6 @@
 #pragma once
 #include <stdint.h>
 #include <string>
-#include "db/db_impl.h"
 #include "db/dbformat.h"
 #include "db/range_del_aggregator.h"
 #include "options/cf_options.h"
@@ -33,7 +32,8 @@ extern Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
                               const Comparator* user_key_comparator,
                               InternalIterator* internal_iter,
                               const SequenceNumber& sequence,
-                               uint64_t max_sequential_skip_in_iterations);
+                               uint64_t max_sequential_skip_in_iterations,
+                               uint64_t version_number);

 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed be allocated. This class is used as an entry point of
@@ -49,6 +49,10 @@ class ArenaWrappedDBIter : public Iterator {
  virtual Arena* GetArena() { return &arena_; }
  virtual RangeDelAggregator* GetRangeDelAggregator();

+  // Set the DB Iterator to be wrapped
+
+  virtual void SetDBIter(DBIter* iter);
+
  // Set the internal iterator wrapped inside the DB Iterator. Usually it is
  // a merging iterator.
  virtual void SetIterUnderDBIter(InternalIterator* iter);
@@ -62,39 +66,20 @@ class ArenaWrappedDBIter : public Iterator {
  virtual Slice key() const override;
  virtual Slice value() const override;
  virtual Status status() const override;
-  virtual Status Refresh() override;

+  void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
  virtual Status GetProperty(std::string prop_name, std::string* prop) override;

-  void Init(Env* env, const ReadOptions& read_options,
-            const ImmutableCFOptions& cf_options,
-            const SequenceNumber& sequence,
-            uint64_t max_sequential_skip_in_iterations,
-            uint64_t version_number);
-
-  void StoreRefreshInfo(const ReadOptions& read_options, DBImpl* db_impl,
-                        ColumnFamilyData* cfd) {
-    read_options_ = read_options;
-    db_impl_ = db_impl;
-    cfd_ = cfd;
-  }
-
 private:
  DBIter* db_iter_;
  Arena arena_;
-  uint64_t sv_number_;
-  ColumnFamilyData* cfd_ = nullptr;
-  DBImpl* db_impl_ = nullptr;
-  ReadOptions read_options_;
 };

 // Generate the arena wrapped iterator class.
-// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
-// be supported.
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr);
+    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number);

 }  // namespace rocksdb
@@ -193,7 +193,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -225,7 +225,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -251,7 +251,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -283,7 +283,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -318,7 +318,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
@@ -347,7 +347,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));

    SetPerfLevel(kEnableCount);
    ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -384,7 +384,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -409,7 +409,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
@@ -431,7 +431,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -466,7 +466,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));

    SetPerfLevel(kEnableCount);
    ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -495,7 +495,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -537,7 +537,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -568,7 +568,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    ReadOptions ro;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -597,7 +597,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -608,7 +608,7 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToFirst();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -630,7 +630,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 2, options.max_sequential_skip_in_iterations));
+      internal_iter, 2, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -673,7 +673,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      options.statistics = rocksdb::CreateDBStatistics();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -708,7 +708,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -736,7 +736,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 202,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -768,7 +768,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(!db_iter->Valid());

@@ -784,7 +784,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
    internal_iter->Finish();
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      200, options.max_sequential_skip_in_iterations));
+                      200, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -818,7 +818,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -852,7 +852,7 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -900,7 +900,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 0;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -946,7 +946,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -990,7 +990,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1028,7 +1028,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1063,7 +1063,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -1093,7 +1093,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1130,7 +1130,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
    ro.max_skippable_internal_keys = 2;
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -1167,7 +1167,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
      ro.max_skippable_internal_keys = i;
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 2 * i + 1,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));

      db_iter->SeekToFirst();
      ASSERT_TRUE(db_iter->Valid());
@@ -1220,7 +1220,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipInternalKeys) {
      ro.max_skippable_internal_keys = i;
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
          env_, ro, cf_options, BytewiseComparator(), internal_iter, 2 * i + 1,
-          options.max_sequential_skip_in_iterations));
+          options.max_sequential_skip_in_iterations, 0));

      db_iter->SeekToFirst();
      ASSERT_TRUE(db_iter->Valid());
@@ -1258,7 +1258,7 @@ TEST_F(DBIteratorTest, DBIterator1) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 1, options.max_sequential_skip_in_iterations));
+      internal_iter, 1, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1285,7 +1285,7 @@ TEST_F(DBIteratorTest, DBIterator2) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 0, options.max_sequential_skip_in_iterations));
+      internal_iter, 0, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1309,7 +1309,7 @@ TEST_F(DBIteratorTest, DBIterator3) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 2, options.max_sequential_skip_in_iterations));
+      internal_iter, 2, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1333,7 +1333,7 @@ TEST_F(DBIteratorTest, DBIterator4) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 4, options.max_sequential_skip_in_iterations));
+      internal_iter, 4, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1365,7 +1365,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1387,7 +1387,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      1, options.max_sequential_skip_in_iterations));
+                      1, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1409,7 +1409,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1431,7 +1431,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      3, options.max_sequential_skip_in_iterations));
+                      3, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1453,7 +1453,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1475,7 +1475,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1497,7 +1497,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1517,7 +1517,7 @@ TEST_F(DBIteratorTest, DBIterator5) {
    internal_iter->Finish();
    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      10, options.max_sequential_skip_in_iterations));
+                      10, options.max_sequential_skip_in_iterations, 0));
    db_iter->Seek("b");
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -1546,7 +1546,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1568,7 +1568,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      1, options.max_sequential_skip_in_iterations));
+                      1, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1590,7 +1590,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1612,7 +1612,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      3, options.max_sequential_skip_in_iterations));
+                      3, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -1630,7 +1630,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1652,7 +1652,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1674,7 +1674,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1715,7 +1715,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      0, options.max_sequential_skip_in_iterations));
+                      0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1749,7 +1749,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      2, options.max_sequential_skip_in_iterations));
+                      2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1789,7 +1789,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      4, options.max_sequential_skip_in_iterations));
+                      4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1829,7 +1829,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      5, options.max_sequential_skip_in_iterations));
+                      5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1874,7 +1874,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      6, options.max_sequential_skip_in_iterations));
+                      6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1920,7 +1920,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      7, options.max_sequential_skip_in_iterations));
+                      7, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1960,7 +1960,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      9, options.max_sequential_skip_in_iterations));
+                      9, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2006,7 +2006,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      13, options.max_sequential_skip_in_iterations));
+                      13, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2053,7 +2053,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(
        NewDBIterator(env_, ro, cf_options, BytewiseComparator(), internal_iter,
-                      14, options.max_sequential_skip_in_iterations));
+                      14, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -2083,7 +2083,7 @@ TEST_F(DBIteratorTest, DBIterator8) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 10, options.max_sequential_skip_in_iterations));
+      internal_iter, 10, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -2113,7 +2113,7 @@ TEST_F(DBIteratorTest, DBIterator9) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-        internal_iter, 10, options.max_sequential_skip_in_iterations));
+        internal_iter, 10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -2179,7 +2179,7 @@ TEST_F(DBIteratorTest, DBIterator10) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 10, options.max_sequential_skip_in_iterations));
+      internal_iter, 10, options.max_sequential_skip_in_iterations, 0));

  db_iter->Seek("c");
  ASSERT_TRUE(db_iter->Valid());
@@ -2218,7 +2218,7 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0 /* force seek */));
+                    internal_iter, 10, 0 /* force seek */, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -2246,7 +2246,7 @@ TEST_F(DBIteratorTest, DBIterator11) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-      internal_iter, 1, options.max_sequential_skip_in_iterations));
+      internal_iter, 1, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -2272,7 +2272,7 @@ TEST_F(DBIteratorTest, DBIterator12) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0));
+                    internal_iter, 10, 0, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -2309,7 +2309,7 @@ TEST_F(DBIteratorTest, DBIterator13) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 2, 3));
+                    internal_iter, 2, 3, 0));
  db_iter->Seek("b");
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), key);
@@ -2337,7 +2337,7 @@ TEST_F(DBIteratorTest, DBIterator14) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ro, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 4, 1));
+                    internal_iter, 4, 1, 0));
  db_iter->Seek("b");
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -2376,7 +2376,7 @@ class DBIterWithMergeIterTest : public testing::Test {
    db_iter_.reset(NewDBIterator(env_, ro_, ImmutableCFOptions(options_),
                                 BytewiseComparator(), merge_iter,
                                 8 /* read data earlier than seqId 8 */,
-                                 3 /* max iterators before reseek */));
+                                 3 /* max iterators before reseek */, 0));
  }

  Env* env_;
@@ -1909,65 +1909,6 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
                 NUMBER_OF_RESEEKS_IN_ITERATION));
 }

-TEST_F(DBIteratorTest, Refresh) {
-  ASSERT_OK(Put("x", "y"));
-
-  std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  ASSERT_OK(Put("c", "d"));
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter->Refresh();
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  dbfull()->Flush(FlushOptions());
-
-  ASSERT_OK(Put("m", "n"));
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter->Refresh();
-
-  iter->Seek(Slice("a"));
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("c")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("m")), 0);
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("x")), 0);
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-
-  iter.reset();
-}
-
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -868,6 +868,66 @@ TEST_F(DBRangeDelTest, SubcompactionHasEmptyDedicatedRangeDelFile) {
  db_->ReleaseSnapshot(snapshot);
 }

+TEST_F(DBRangeDelTest, MemtableBloomFilter) {
+  // regression test for #2743. the range delete tombstones in memtable should
+  // be added even when Get() skips searching due to its prefix bloom filter
+  const int kMemtableSize = 1 << 20;              // 1MB
+  const int kMemtablePrefixFilterSize = 1 << 13;  // 8KB
+  const int kNumKeys = 1000;
+  const int kPrefixLen = 8;
+  Options options = CurrentOptions();
+  options.memtable_prefix_bloom_size_ratio =
+      static_cast<double>(kMemtablePrefixFilterSize) / kMemtableSize;
+  options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(kPrefixLen));
+  options.write_buffer_size = kMemtableSize;
+  Reopen(options);
+
+  for (int i = 0; i < kNumKeys; ++i) {
+    ASSERT_OK(Put(Key(i), "val"));
+  }
+  Flush();
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
+                             Key(kNumKeys)));
+  for (int i = 0; i < kNumKeys; ++i) {
+    std::string value;
+    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound());
+  }
+}
+
+TEST_F(DBRangeDelTest, UnorderedTombstones) {
+  // Regression test for #2752. Range delete tombstones between
+  // different snapshot stripes are not stored in order, so the first
+  // tombstone of each snapshot stripe should be checked as a smallest
+  // candidate.
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+
+  auto cf = db_->DefaultColumnFamily();
+
+  ASSERT_OK(db_->Put(WriteOptions(), cf, "a", "a"));
+  ASSERT_OK(db_->Flush(FlushOptions(), cf));
+  ASSERT_EQ(1, NumTableFilesAtLevel(0));
+  ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
+  ASSERT_EQ(1, NumTableFilesAtLevel(1));
+
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "b", "c"));
+  // Hold a snapshot to separate these two delete ranges.
+  auto snapshot = db_->GetSnapshot();
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "a", "b"));
+  ASSERT_OK(db_->Flush(FlushOptions(), cf));
+  db_->ReleaseSnapshot(snapshot);
+
+  std::vector<std::vector<FileMetaData>> files;
+  dbfull()->TEST_GetFilesMetaData(cf, &files);
+  ASSERT_EQ(1, files[0].size());
+  ASSERT_EQ("a", files[0][0].smallest.user_key());
+  ASSERT_EQ("c", files[0][0].largest.user_key());
+
+  std::string v;
+  auto s = db_->Get(ReadOptions(), "a", &v);
+  ASSERT_TRUE(s.IsNotFound());
+}
+
 #endif  // ROCKSDB_LITE

 }  // namespace rocksdb
@@ -23,7 +23,6 @@
 #include <alloca.h>
 #endif

-#include "cache/lru_cache.h"
 #include "db/db_impl.h"
 #include "db/db_test_util.h"
 #include "db/dbformat.h"
@@ -2235,10 +2234,6 @@ class ModelDB : public DB {
    return Status::NotSupported("Not implemented.");
  }

-  virtual Status VerifyChecksum() override {
-    return Status::NotSupported("Not implemented.");
-  }
-
  using DB::GetPropertiesOfAllTables;
  virtual Status GetPropertiesOfAllTables(
      ColumnFamilyHandle* column_family,
@@ -2811,6 +2806,46 @@ TEST_F(DBTest, FIFOCompactionTestWithCompaction) {
            options.compaction_options_fifo.max_table_files_size);
 }

+TEST_F(DBTest, FIFOCompactionStyleWithCompactionAndDelete) {
+  Options options;
+  options.compaction_style = kCompactionStyleFIFO;
+  options.write_buffer_size = 20 << 10;  // 20K
+  options.arena_block_size = 4096;
+  options.compaction_options_fifo.max_table_files_size = 1500 << 10;  // 1MB
+  options.compaction_options_fifo.allow_compaction = true;
+  options.level0_file_num_compaction_trigger = 3;
+  options.compression = kNoCompression;
+  options.create_if_missing = true;
+  options = CurrentOptions(options);
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  for (int i = 0; i < 3; i++) {
+    // Each file contains a different key which will be dropped later.
+    ASSERT_OK(Put("a" + ToString(i), RandomString(&rnd, 500)));
+    ASSERT_OK(Put("key" + ToString(i), ""));
+    ASSERT_OK(Put("z" + ToString(i), RandomString(&rnd, 500)));
+    Flush();
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  }
+  ASSERT_EQ(NumTableFilesAtLevel(0), 1);
+  for (int i = 0; i < 3; i++) {
+    ASSERT_EQ("", Get("key" + ToString(i)));
+  }
+  for (int i = 0; i < 3; i++) {
+    // Each file contains a different key which will be dropped later.
+    ASSERT_OK(Put("a" + ToString(i), RandomString(&rnd, 500)));
+    ASSERT_OK(Delete("key" + ToString(i)));
+    ASSERT_OK(Put("z" + ToString(i), RandomString(&rnd, 500)));
+    Flush();
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+  }
+  ASSERT_EQ(NumTableFilesAtLevel(0), 2);
+  for (int i = 0; i < 3; i++) {
+    ASSERT_EQ("NOT_FOUND", Get("key" + ToString(i)));
+  }
+}
+
 // Check that FIFO-with-TTL is not supported with max_open_files != -1.
 TEST_F(DBTest, FIFOCompactionWithTTLAndMaxOpenFilesTest) {
  Options options;
@@ -2851,23 +2886,20 @@ TEST_F(DBTest, FIFOCompactionWithTTLAndVariousTableFormatsTest) {
  ASSERT_TRUE(TryReopen(options).IsNotSupported());
 }

-TEST_F(DBTest, FIFOCompactionWithTTLTest) {
+TEST_F(DBTest, DISABLED_FIFOCompactionWithTTLTest) {
  Options options;
  options.compaction_style = kCompactionStyleFIFO;
  options.write_buffer_size = 10 << 10;  // 10KB
  options.arena_block_size = 4096;
  options.compression = kNoCompression;
  options.create_if_missing = true;
-  env_->time_elapse_only_sleep_ = false;
-  options.env = env_;

  // Test to make sure that all files with expired ttl are deleted on next
  // manual compaction.
  {
-    env_->addon_time_.store(0);
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60 ;  // 1 hour
+    options.compaction_options_fifo.ttl = 600;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2878,21 +2910,19 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
-    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
-
-    // Sleep for 2 hours -- which is much greater than TTL.
-    // Note: Couldn't use SleepForMicroseconds because it takes an int instead
-    // of uint64_t. Hence used addon_time_ directly.
-    // env_->SleepForMicroseconds(2 * 60 * 60 * 1000 * 1000);
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-
-    // Since no flushes and compactions have run, the db should still be in
-    // the same state even after considerable time has passed.
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

+    // sleep for 5 seconds
+    env_->SleepForMicroseconds(5 * 1000 * 1000);
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
+    ASSERT_EQ(NumTableFilesAtLevel(0), 10);
+
+    // change ttl to 1 sec. So all files should be deleted on next compaction.
+    options.compaction_options_fifo.ttl = 1;
+    Reopen(options);
+
    dbfull()->CompactRange(CompactRangeOptions(), nullptr, nullptr);
    ASSERT_EQ(NumTableFilesAtLevel(0), 0);
  }
@@ -2902,7 +2932,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  {
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2913,13 +2943,11 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

@@ -2941,10 +2969,10 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  // Test that shows the fall back to size-based FIFO compaction if TTL-based
  // deletion doesn't move the total size to be less than max_table_files_size.
  {
-    options.write_buffer_size = 10 << 10;                              // 10KB
+    options.write_buffer_size = 110 << 10;                             // 10KB
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = false;
-    options.compaction_options_fifo.ttl =  1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options = CurrentOptions(options);
    DestroyAndReopen(options);

@@ -2955,13 +2983,11 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 3);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 3);

@@ -2970,8 +2996,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    // Size limit is still guaranteed.
    ASSERT_LE(SizeAtLevel(0),
              options.compaction_options_fifo.max_table_files_size);
@@ -2981,7 +3007,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
  {
    options.compaction_options_fifo.max_table_files_size = 150 << 10;  // 150KB
    options.compaction_options_fifo.allow_compaction = true;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 5;  // seconds
    options.level0_file_num_compaction_trigger = 6;
    options = CurrentOptions(options);
    DestroyAndReopen(options);
@@ -2993,16 +3019,15 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
    // With Intra-L0 compaction, out of 10 files, 6 files will be compacted to 1
    // (due to level0_file_num_compaction_trigger = 6).
    // So total files = 1 + remaining 4 = 5.
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);

-    // Sleep for 2 hours -- which is much greater than TTL.
-    env_->addon_time_.fetch_add(2 * 60 * 60);
-    // Just to make sure that we are in the same state even after sleeping.
+    // Sleep for a little over ttl time.
+    env_->SleepForMicroseconds(6 * 1000 * 1000);
    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);

@@ -3012,8 +3037,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    ASSERT_EQ(NumTableFilesAtLevel(0), 5);
    ASSERT_LE(SizeAtLevel(0),
              options.compaction_options_fifo.max_table_files_size);
@@ -3025,7 +3050,7 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
    options.write_buffer_size = 20 << 10;                               // 20K
    options.compaction_options_fifo.max_table_files_size = 1500 << 10;  // 1.5MB
    options.compaction_options_fifo.allow_compaction = true;
-    options.compaction_options_fifo.ttl = 1 * 60 * 60;  // 1 hour
+    options.compaction_options_fifo.ttl = 60 * 60;  // 1 hour
    options.level0_file_num_compaction_trigger = 6;
    options = CurrentOptions(options);
    DestroyAndReopen(options);
@@ -3037,8 +3062,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());
    // It should be compacted to 10 files.
    ASSERT_EQ(NumTableFilesAtLevel(0), 10);

@@ -3048,8 +3073,8 @@ TEST_F(DBTest, FIFOCompactionWithTTLTest) {
        ASSERT_OK(Put(ToString(i * 20 + j + 2000), RandomString(&rnd, 980)));
      }
      Flush();
-      ASSERT_OK(dbfull()->TEST_WaitForCompact());
    }
+    ASSERT_OK(dbfull()->TEST_WaitForCompact());

    // It should be compacted to no more than 20 files.
    ASSERT_GT(NumTableFilesAtLevel(0), 10);
@@ -5335,36 +5360,6 @@ TEST_F(DBTest, RowCache) {
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_HIT), 1);
  ASSERT_EQ(TestGetTickerCount(options, ROW_CACHE_MISS), 1);
 }
-
-TEST_F(DBTest, PinnableSliceAndRowCache) {
-  Options options = CurrentOptions();
-  options.statistics = rocksdb::CreateDBStatistics();
-  options.row_cache = NewLRUCache(8192);
-  DestroyAndReopen(options);
-
-  ASSERT_OK(Put("foo", "bar"));
-  ASSERT_OK(Flush());
-
-  ASSERT_EQ(Get("foo"), "bar");
-  ASSERT_EQ(
-      reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-      1);
-
-  {
-    PinnableSlice pin_slice;
-    ASSERT_EQ(Get("foo", &pin_slice), Status::OK());
-    ASSERT_EQ(pin_slice.ToString(), "bar");
-    // Entry is already in cache, lookup will remove the element from lru
-    ASSERT_EQ(
-        reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-        0);
-  }
-  // After PinnableSlice destruction element is added back in LRU
-  ASSERT_EQ(
-      reinterpret_cast<LRUCache*>(options.row_cache.get())->TEST_GetLRUSize(),
-      1);
-}
-
 #endif  // ROCKSDB_LITE

 TEST_F(DBTest, DeletingOldWalAfterDrop) {
@@ -2236,7 +2236,6 @@ TEST_F(DBTest2, LowPriWrite) {
  ASSERT_EQ(1, rate_limit_count.load());
 }

-#ifndef ROCKSDB_LITE
 TEST_F(DBTest2, RateLimitedCompactionReads) {
  // compaction input has 512KB data
  const int kNumKeysPerFile = 128;
@@ -2282,15 +2281,12 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
    // chose 1MB as the upper bound on the total bytes read.
    size_t rate_limited_bytes =
        options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW);
-    // Include the explict prefetch of the footer in direct I/O case.
-    size_t direct_io_extra = use_direct_io ? 512 * 1024 : 0;
-    ASSERT_GE(rate_limited_bytes,
-              static_cast<size_t>(kNumKeysPerFile * kBytesPerKey * kNumL0Files +
-                                  direct_io_extra));
+    ASSERT_GE(
+        rate_limited_bytes,
+        static_cast<size_t>(kNumKeysPerFile * kBytesPerKey * kNumL0Files));
    ASSERT_LT(
        rate_limited_bytes,
-        static_cast<size_t>(2 * kNumKeysPerFile * kBytesPerKey * kNumL0Files +
-                            direct_io_extra));
+        static_cast<size_t>(2 * kNumKeysPerFile * kBytesPerKey * kNumL0Files));

    Iterator* iter = db_->NewIterator(ReadOptions());
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
@@ -2303,7 +2299,7 @@ TEST_F(DBTest2, RateLimitedCompactionReads) {
                  options.rate_limiter->GetTotalBytesThrough(Env::IO_LOW)));
  }
 }
-#endif  // ROCKSDB_LITE
+
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -42,24 +42,13 @@ SpecialEnv::SpecialEnv(Env* base)
  non_writable_count_ = 0;
  table_write_callback_ = nullptr;
 }
-#ifndef ROCKSDB_LITE
+
 ROT13BlockCipher rot13Cipher_(16);
-#endif  // ROCKSDB_LITE

 DBTestBase::DBTestBase(const std::string path)
    : mem_env_(!getenv("MEM_ENV") ? nullptr : new MockEnv(Env::Default())),
-#ifndef ROCKSDB_LITE
-      encrypted_env_(
-          !getenv("ENCRYPTED_ENV")
-              ? nullptr
-              : NewEncryptedEnv(mem_env_ ? mem_env_ : Env::Default(),
-                                new CTREncryptionProvider(rot13Cipher_))),
-#else
-      encrypted_env_(nullptr),
-#endif  // ROCKSDB_LITE
-      env_(new SpecialEnv(encrypted_env_
-                              ? encrypted_env_
-                              : (mem_env_ ? mem_env_ : Env::Default()))),
+      encrypted_env_(!getenv("ENCRYPTED_ENV") ? nullptr : NewEncryptedEnv(mem_env_ ? mem_env_ : Env::Default(), new CTREncryptionProvider(rot13Cipher_))),
+      env_(new SpecialEnv(encrypted_env_ ? encrypted_env_ : (mem_env_ ? mem_env_ : Env::Default()))),
      option_config_(kDefault) {
  env_->SetBackgroundThreads(1, Env::LOW);
  env_->SetBackgroundThreads(1, Env::HIGH);
@@ -697,13 +686,6 @@ std::string DBTestBase::Get(int cf, const std::string& k,
  return result;
 }

-Status DBTestBase::Get(const std::string& k, PinnableSlice* v) {
-  ReadOptions options;
-  options.verify_checksums = true;
-  Status s = dbfull()->Get(options, dbfull()->DefaultColumnFamily(), k, v);
-  return s;
-}
-
 uint64_t DBTestBase::GetNumSnapshots() {
  uint64_t int_num;
  EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num));
@@ -803,8 +803,6 @@ class DBTestBase : public testing::Test {
  std::string Get(int cf, const std::string& k,
                  const Snapshot* snapshot = nullptr);

-  Status Get(const std::string& k, PinnableSlice* v);
-
  uint64_t GetNumSnapshots();

  uint64_t GetTimeOldestSnapshots();
@@ -1370,103 +1370,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
  Destroy(options);
 }

-TEST_P(DBTestUniversalCompaction, FullCompactionInBottomPriThreadPool) {
-  const int kNumFilesTrigger = 3;
-  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  for (bool allow_ingest_behind : {false, true}) {
-    Options options = CurrentOptions();
-    options.allow_ingest_behind = allow_ingest_behind;
-    options.compaction_style = kCompactionStyleUniversal;
-    options.num_levels = num_levels_;
-    options.write_buffer_size = 100 << 10;     // 100KB
-    options.target_file_size_base = 32 << 10;  // 32KB
-    options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-    // Trigger compaction if size amplification exceeds 110%
-    options.compaction_options_universal.max_size_amplification_percent = 110;
-    DestroyAndReopen(options);
-
-    int num_bottom_pri_compactions = 0;
-    SyncPoint::GetInstance()->SetCallBack(
-        "DBImpl::BGWorkBottomCompaction",
-        [&](void* arg) { ++num_bottom_pri_compactions; });
-    SyncPoint::GetInstance()->EnableProcessing();
-
-    Random rnd(301);
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      ASSERT_EQ(NumSortedRuns(), num);
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx);
-    }
-    dbfull()->TEST_WaitForCompact();
-
-    if (allow_ingest_behind || num_levels_ > 1) {
-      // allow_ingest_behind increases number of levels while sanitizing.
-      ASSERT_EQ(1, num_bottom_pri_compactions);
-    } else {
-      // for single-level universal, everything's bottom level so nothing should
-      // be executed in bottom-pri thread pool.
-      ASSERT_EQ(0, num_bottom_pri_compactions);
-    }
-    // Verify that size amplification did occur
-    ASSERT_EQ(NumSortedRuns(), 1);
-    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  }
-}
-
-TEST_P(DBTestUniversalCompaction, ConcurrentBottomPriLowPriCompactions) {
-  if (num_levels_ == 1) {
-    // for single-level universal, everything's bottom level so nothing should
-    // be executed in bottom-pri thread pool.
-    return;
-  }
-  const int kNumFilesTrigger = 3;
-  Env::Default()->SetBackgroundThreads(1, Env::Priority::BOTTOM);
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleUniversal;
-  options.num_levels = num_levels_;
-  options.write_buffer_size = 100 << 10;     // 100KB
-  options.target_file_size_base = 32 << 10;  // 32KB
-  options.level0_file_num_compaction_trigger = kNumFilesTrigger;
-  // Trigger compaction if size amplification exceeds 110%
-  options.compaction_options_universal.max_size_amplification_percent = 110;
-  DestroyAndReopen(options);
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {// wait for the full compaction to be picked before adding files intended
-       // for the second one.
-       {"DBImpl::BackgroundCompaction:ForwardToBottomPriPool",
-        "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0"},
-       // the full (bottom-pri) compaction waits until a partial (low-pri)
-       // compaction has started to verify they can run in parallel.
-       {"DBImpl::BackgroundCompaction:NonTrivial",
-        "DBImpl::BGWorkBottomCompaction"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  for (int i = 0; i < 2; ++i) {
-    for (int num = 0; num < kNumFilesTrigger; num++) {
-      int key_idx = 0;
-      GenerateNewFile(&rnd, &key_idx, true /* no_wait */);
-      // use no_wait above because that one waits for flush and compaction. We
-      // don't want to wait for compaction because the full compaction is
-      // intentionally blocked while more files are flushed.
-      dbfull()->TEST_WaitForFlushMemTable();
-    }
-    if (i == 0) {
-      TEST_SYNC_POINT(
-          "DBTestUniversalCompaction:ConcurrentBottomPriLowPriCompactions:0");
-    }
-  }
-  dbfull()->TEST_WaitForCompact();
-
-  // First compaction should output to bottom level. Second should output to L0
-  // since older L0 files pending compaction prevent it from being placed lower.
-  ASSERT_EQ(NumSortedRuns(), 2);
-  ASSERT_GT(NumTableFilesAtLevel(0), 0);
-  ASSERT_GT(NumTableFilesAtLevel(num_levels_ - 1), 0);
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
 INSTANTIATE_TEST_CASE_P(UniversalCompactionNumLevels, DBTestUniversalCompaction,
                        ::testing::Combine(::testing::Values(1, 3, 5),
                                           ::testing::Bool()));
@@ -3,12 +3,17 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).

+#include <atomic>
 #include <memory>
 #include <thread>
 #include <vector>
 #include "db/db_test_util.h"
 #include "db/write_batch_internal.h"
+#include "db/write_thread.h"
+#include "port/port.h"
 #include "port/stack_trace.h"
+#include "util/fault_injection_test_env.h"
+#include "util/string_util.h"
 #include "util/sync_point.h"

 namespace rocksdb {
@@ -18,7 +23,9 @@ class DBWriteTest : public DBTestBase, public testing::WithParamInterface<int> {
 public:
  DBWriteTest() : DBTestBase("/db_write_test") {}

-  void Open() { DBTestBase::Reopen(GetOptions(GetParam())); }
+  Options GetOptions() { return DBTestBase::GetOptions(GetParam()); }
+
+  void Open() { DBTestBase::Reopen(GetOptions()); }
 };

 // Sequence number should be return through input write batch.
@@ -67,6 +74,47 @@ TEST_P(DBWriteTest, ReturnSeuqneceNumberMultiThreaded) {
  }
 }

+TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
+  constexpr int kNumThreads = 5;
+  std::unique_ptr<FaultInjectionTestEnv> mock_env(
+      new FaultInjectionTestEnv(Env::Default()));
+  Options options = GetOptions();
+  options.env = mock_env.get();
+  Reopen(options);
+  std::atomic<int> ready_count{0};
+  std::atomic<int> leader_count{0};
+  std::vector<port::Thread> threads;
+  mock_env->SetFilesystemActive(false);
+  // Wait until all threads linked to write threads, to make sure
+  // all threads join the same batch group.
+  SyncPoint::GetInstance()->SetCallBack(
+      "WriteThread::JoinBatchGroup:Wait", [&](void* arg) {
+        ready_count++;
+        auto* w = reinterpret_cast<WriteThread::Writer*>(arg);
+        if (w->state == WriteThread::STATE_GROUP_LEADER) {
+          leader_count++;
+          while (ready_count < kNumThreads) {
+            // busy waiting
+          }
+        }
+      });
+  SyncPoint::GetInstance()->EnableProcessing();
+  for (int i = 0; i < kNumThreads; i++) {
+    threads.push_back(port::Thread(
+        [&](int index) {
+          // All threads should fail.
+          ASSERT_FALSE(Put("key" + ToString(index), "value").ok());
+        },
+        i));
+  }
+  for (int i = 0; i < kNumThreads; i++) {
+    threads[i].join();
+  }
+  ASSERT_EQ(1, leader_count);
+  // Close before mock_env destruct.
+  Close();
+}
+
 INSTANTIATE_TEST_CASE_P(DBWriteTestInstance, DBWriteTest,
                        testing::Values(DBTestBase::kDefault,
                                        DBTestBase::kConcurrentWALWrites,
@@ -84,12 +84,6 @@ struct ParsedInternalKey {
  ParsedInternalKey(const Slice& u, const SequenceNumber& seq, ValueType t)
      : user_key(u), sequence(seq), type(t) { }
  std::string DebugString(bool hex = false) const;
-
-  void clear() {
-    user_key.clear();
-    sequence = 0;
-    type = kTypeDeletion;
-  }
 };

 // Return the length of the encoding of "key".
@@ -157,9 +151,6 @@ class InternalKeyComparator : public Comparator {

  int Compare(const InternalKey& a, const InternalKey& b) const;
  int Compare(const ParsedInternalKey& a, const ParsedInternalKey& b) const;
-  virtual const Comparator* GetRootComparator() const override {
-    return user_comparator_->GetRootComparator();
-  }
 };

 // Modules in this directory should keep internal keys wrapped inside
@@ -14,18 +14,20 @@ namespace experimental {

 Status SuggestCompactRange(DB* db, ColumnFamilyHandle* column_family,
                           const Slice* begin, const Slice* end) {
-  if (db == nullptr) {
-    return Status::InvalidArgument("DB is empty");
+  auto dbimpl = dynamic_cast<DBImpl*>(db);
+  if (dbimpl == nullptr) {
+    return Status::InvalidArgument("Didn't recognize DB object");
  }

-  return db->SuggestCompactRange(column_family, begin, end);
+  return dbimpl->SuggestCompactRange(column_family, begin, end);
 }

 Status PromoteL0(DB* db, ColumnFamilyHandle* column_family, int target_level) {
-  if (db == nullptr) {
+  auto dbimpl = dynamic_cast<DBImpl*>(db);
+  if (dbimpl == nullptr) {
    return Status::InvalidArgument("Didn't recognize DB object");
  }
-  return db->PromoteL0(column_family, target_level);
+  return dbimpl->PromoteL0(column_family, target_level);
 }

 #else  // ROCKSDB_LITE
@@ -643,6 +643,14 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
  }
  PERF_TIMER_GUARD(get_from_memtable_time);

+  std::unique_ptr<InternalIterator> range_del_iter(
+      NewRangeTombstoneIterator(read_opts));
+  Status status = range_del_agg->AddTombstones(std::move(range_del_iter));
+  if (!status.ok()) {
+    *s = status;
+    return false;
+  }
+
  Slice user_key = key.user_key();
  bool found_final_value = false;
  bool merge_in_progress = s->IsMergeInProgress();
@@ -658,13 +666,6 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
    if (prefix_bloom_) {
      PERF_COUNTER_ADD(bloom_memtable_hit_count, 1);
    }
-    std::unique_ptr<InternalIterator> range_del_iter(
-        NewRangeTombstoneIterator(read_opts));
-    Status status = range_del_agg->AddTombstones(std::move(range_del_iter));
-    if (!status.ok()) {
-      *s = status;
-      return false;
-    }
    Saver saver;
    saver.status = s;
    saver.found_final_value = &found_final_value;
@@ -201,11 +201,12 @@ Status MergeHelper::MergeUntil(InternalIterator* iter,
          ikey.sequence <= latest_snapshot_
              ? CompactionFilter::Decision::kKeep
              : FilterMerge(orig_ikey.user_key, value_slice);
-      if (filter != CompactionFilter::Decision::kRemoveAndSkipUntil &&
-          range_del_agg != nullptr &&
+      if (range_del_agg != nullptr &&
+
          range_del_agg->ShouldDelete(
              iter->key(),
-              RangeDelAggregator::RangePositioningMode::kForwardTraversal)) {
+              RangeDelAggregator::RangePositioningMode::kForwardTraversal) &&
+          filter != CompactionFilter::Decision::kRemoveAndSkipUntil) {
        filter = CompactionFilter::Decision::kRemove;
      }
      if (filter == CompactionFilter::Decision::kKeep ||
@@ -412,8 +412,8 @@ void RangeDelAggregator::AddToBuilder(

  // Note the order in which tombstones are stored is insignificant since we
  // insert them into a std::map on the read path.
-  bool first_added = false;
  while (stripe_map_iter != rep_->stripe_map_.end()) {
+    bool first_added = false;
    for (auto tombstone_map_iter = stripe_map_iter->second.raw_map.begin();
         tombstone_map_iter != stripe_map_iter->second.raw_map.end();
         ++tombstone_map_iter) {
@@ -452,7 +452,7 @@ void RangeDelAggregator::AddToBuilder(
      builder->Add(ikey_and_end_key.first.Encode(), ikey_and_end_key.second);
      if (!first_added) {
        first_added = true;
-        InternalKey smallest_candidate = std::move(ikey_and_end_key.first);;
+        InternalKey smallest_candidate = std::move(ikey_and_end_key.first);
        if (lower_bound != nullptr &&
            icmp_.user_comparator()->Compare(smallest_candidate.user_key(),
                                             *lower_bound) <= 0) {
@@ -176,7 +176,6 @@ class Repairer {
      status = db_impl->NewDB();
      delete db_impl;
    }
-
    if (status.ok()) {
      // Recover using the fresh manifest created by NewDB()
      status =
@@ -247,21 +246,9 @@ class Repairer {
  Status FindFiles() {
    std::vector<std::string> filenames;
    bool found_file = false;
-    std::vector<std::string> to_search_paths;
-
    for (size_t path_id = 0; path_id < db_options_.db_paths.size(); path_id++) {
-        to_search_paths.push_back(db_options_.db_paths[path_id].path);
-    }
-
-    // search wal_dir if user uses a customize wal_dir
-    if (!db_options_.wal_dir.empty() && 
-        db_options_.wal_dir != dbname_) {
-        to_search_paths.push_back(db_options_.wal_dir);
-    }
-
-    for (size_t path_id = 0; path_id < to_search_paths.size(); path_id++) {
      Status status =
-          env_->GetChildren(to_search_paths[path_id], &filenames);
+          env_->GetChildren(db_options_.db_paths[path_id].path, &filenames);
      if (!status.ok()) {
        return status;
      }
@@ -274,12 +261,14 @@ class Repairer {
      for (size_t i = 0; i < filenames.size(); i++) {
        if (ParseFileName(filenames[i], &number, &type)) {
          if (type == kDescriptorFile) {
+            assert(path_id == 0);
            manifests_.push_back(filenames[i]);
          } else {
            if (number + 1 > next_file_number_) {
              next_file_number_ = number + 1;
            }
            if (type == kLogFile) {
+              assert(path_id == 0);
              logs_.push_back(number);
            } else if (type == kTableFile) {
              table_fds_.emplace_back(number, static_cast<uint32_t>(path_id),
@@ -299,8 +288,7 @@ class Repairer {

  void ConvertLogFilesToTables() {
    for (size_t i = 0; i < logs_.size(); i++) {
-      // we should use LogFileName(wal_dir, logs_[i]) here. user might uses wal_dir option.
-      std::string logname = LogFileName(db_options_.wal_dir, logs_[i]);
+      std::string logname = LogFileName(dbname_, logs_[i]);
      Status status = ConvertLogToTable(logs_[i]);
      if (!status.ok()) {
        ROCKS_LOG_WARN(db_options_.info_log,
@@ -324,7 +312,7 @@ class Repairer {
    };

    // Open the log file
-    std::string logname = LogFileName(db_options_.wal_dir, log);
+    std::string logname = LogFileName(dbname_, log);
    unique_ptr<SequentialFile> lfile;
    Status status = env_->NewSequentialFile(
        logname, &lfile, env_->OptimizeForLogRead(env_options_));
@@ -174,40 +174,6 @@ TEST_F(RepairTest, UnflushedSst) {
  ASSERT_EQ(Get("key"), "val");
 }

-TEST_F(RepairTest, SeparateWalDir) {
-  do {
-    Options options = CurrentOptions();
-    DestroyAndReopen(options);
-    Put("key", "val");
-    Put("foo", "bar");
-    VectorLogPtr wal_files;
-    ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
-    ASSERT_EQ(wal_files.size(), 1);
-    uint64_t total_ssts_size;
-    GetAllSSTFiles(&total_ssts_size);
-    ASSERT_EQ(total_ssts_size, 0);
-    std::string manifest_path =
-      DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
-
-    Close();
-    ASSERT_OK(env_->FileExists(manifest_path));
-    ASSERT_OK(env_->DeleteFile(manifest_path));
-    ASSERT_OK(RepairDB(dbname_, options));
-
-    // make sure that all WALs are converted to SSTables.
-    options.wal_dir = "";
-
-    Reopen(options);
-    ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
-    ASSERT_EQ(wal_files.size(), 0);
-    GetAllSSTFiles(&total_ssts_size);
-    ASSERT_GT(total_ssts_size, 0);
-    ASSERT_EQ(Get("key"), "val");
-    ASSERT_EQ(Get("foo"), "bar");
-
- } while(ChangeWalOptions());
-}
-
 TEST_F(RepairTest, RepairMultipleColumnFamilies) {
  // Verify repair logic associates SST files with their original column
  // families.
@@ -106,10 +106,10 @@ Status TableCache::GetTableReader(
    }
    StopWatch sw(ioptions_.env, ioptions_.statistics, TABLE_OPEN_IO_MICROS);
    std::unique_ptr<RandomAccessFileReader> file_reader(
-        new RandomAccessFileReader(
-            std::move(file), fname, ioptions_.env,
-            record_read_stats ? ioptions_.statistics : nullptr, SST_READ_MICROS,
-            file_read_hist, ioptions_.rate_limiter, for_compaction));
+        new RandomAccessFileReader(std::move(file), fname, ioptions_.env,
+                                   ioptions_.statistics, record_read_stats,
+                                   file_read_hist, ioptions_.rate_limiter,
+                                   for_compaction));
    s = ioptions_.table_factory->NewTableReader(
        TableReaderOptions(ioptions_, env_options, internal_comparator,
                           skip_filters, level),
@@ -225,7 +225,8 @@ InternalIterator* TableCache::NewIterator(
  }
  InternalIterator* result = nullptr;
  if (s.ok()) {
-    result = table_reader->NewIterator(options, arena, skip_filters);
+    result =
+      table_reader->NewIterator(options, arena, &icomparator, skip_filters);
    if (create_new_table_reader) {
      assert(handle == nullptr);
      result->RegisterCleanup(&DeleteTableReader, table_reader, nullptr);
@@ -310,7 +311,6 @@ Status TableCache::Get(const ReadOptions& options,
 #ifndef ROCKSDB_LITE
  IterKey row_cache_key;
  std::string row_cache_entry_buffer;
-
  // Check row cache if enabled. Since row cache does not currently store
  // sequence numbers, we cannot use it if we need to fetch the sequence.
  if (ioptions_.row_cache && !get_context->NeedToReadSequence()) {
@@ -334,26 +334,10 @@ Status TableCache::Get(const ReadOptions& options,

    if (auto row_handle =
            ioptions_.row_cache->Lookup(row_cache_key.GetUserKey())) {
-      // Cleanable routine to release the cache entry
-      Cleanable value_pinner;
-      auto release_cache_entry_func = [](void* cache_to_clean,
-                                         void* cache_handle) {
-        ((Cache*)cache_to_clean)->Release((Cache::Handle*)cache_handle);
-      };
      auto found_row_cache_entry = static_cast<const std::string*>(
          ioptions_.row_cache->Value(row_handle));
-      // If it comes here value is located on the cache.
-      // found_row_cache_entry points to the value on cache,
-      // and value_pinner has cleanup procedure for the cached entry.
-      // After replayGetContextLog() returns, get_context.pinnable_slice_
-      // will point to cache entry buffer (or a copy based on that) and
-      // cleanup routine under value_pinner will be delegated to
-      // get_context.pinnable_slice_. Cache entry is released when
-      // get_context.pinnable_slice_ is reset.
-      value_pinner.RegisterCleanup(release_cache_entry_func,
-                                   ioptions_.row_cache.get(), row_handle);
-      replayGetContextLog(*found_row_cache_entry, user_key, get_context,
-                          &value_pinner);
+      replayGetContextLog(*found_row_cache_entry, user_key, get_context);
+      ioptions_.row_cache->Release(row_handle);
      RecordTick(ioptions_.statistics, ROW_CACHE_HIT);
      done = true;
    } else {
@@ -328,6 +328,10 @@ Version::~Version() {
      assert(f->refs > 0);
      f->refs--;
      if (f->refs <= 0) {
+        if (f->table_reader_handle) {
+          cfd_->table_cache()->EraseHandle(f->fd, f->table_reader_handle);
+          f->table_reader_handle = nullptr;
+        }
        vset_->obsolete_files_.push_back(f);
      }
    }
@@ -1227,14 +1231,6 @@ int VersionStorageInfo::MaxInputLevel() const {
  return 0;
 }

-int VersionStorageInfo::MaxOutputLevel(bool allow_ingest_behind) const {
-  if (allow_ingest_behind) {
-    assert(num_levels() > 1);
-    return num_levels() - 2;
-  }
-  return num_levels() - 1;
-}
-
 void VersionStorageInfo::EstimateCompactionBytesNeeded(
    const MutableCFOptions& mutable_cf_options) {
  // Only implemented for level-based compaction
@@ -2335,14 +2331,10 @@ void CloseTables(void* ptr, size_t) {
 VersionSet::~VersionSet() {
  // we need to delete column_family_set_ because its destructor depends on
  // VersionSet
-  Cache* table_cache = column_family_set_->get_table_cache();
-  table_cache->ApplyToAllCacheEntries(&CloseTables, false /* thread_safe */);
+  column_family_set_->get_table_cache()->ApplyToAllCacheEntries(&CloseTables,
+                                                                false);
  column_family_set_.reset();
  for (auto file : obsolete_files_) {
-    if (file->table_reader_handle) {
-      table_cache->Release(file->table_reader_handle);
-      TableCache::Evict(table_cache, file->fd.GetNumber());
-    }
    delete file;
  }
  obsolete_files_.clear();
@@ -147,7 +147,6 @@ class VersionStorageInfo {
  }

  int MaxInputLevel() const;
-  int MaxOutputLevel(bool allow_ingest_behind) const;

  // Return level number that has idx'th highest score
  int CompactionScoreLevel(int idx) const { return compaction_level_[idx]; }
@@ -26,7 +26,6 @@
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/write_batch.h"
-#include "util/cast_util.h"
 #include "util/coding.h"
 #include "util/file_reader_writer.h"
 #include "util/filename.h"
@@ -274,8 +273,8 @@ namespace {
 struct CompareLogByPointer {
  bool operator()(const std::unique_ptr<LogFile>& a,
                  const std::unique_ptr<LogFile>& b) {
-    LogFileImpl* a_impl = static_cast_with_check<LogFileImpl, LogFile>(a.get());
-    LogFileImpl* b_impl = static_cast_with_check<LogFileImpl, LogFile>(b.get());
+    LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get());
+    LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get());
    return *a_impl < *b_impl;
  }
 };
@@ -60,7 +60,7 @@ static std::string PrintContents(WriteBatch* b) {
    }
    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
      ParsedInternalKey ikey;
-      ikey.clear();
+      memset((void*)&ikey, 0, sizeof(ikey));
      EXPECT_TRUE(ParseInternalKey(iter->key(), &ikey));
      switch (ikey.type) {
        case kTypeValue:
@@ -451,20 +451,20 @@ TEST_F(WriteBatchTest, DISABLED_ManyUpdates) {
    }
    virtual Status DeleteCF(uint32_t column_family_id,
                            const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status SingleDeleteCF(uint32_t column_family_id,
                                  const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                           const Slice& value) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
-    virtual void LogData(const Slice& blob) override { ADD_FAILURE(); }
+    virtual void LogData(const Slice& blob) override { EXPECT_TRUE(false); }
    virtual bool Continue() override { return num_seen < kNumUpdates; }
  } handler;

@@ -502,20 +502,20 @@ TEST_F(WriteBatchTest, DISABLED_LargeKeyValue) {
    }
    virtual Status DeleteCF(uint32_t column_family_id,
                            const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status SingleDeleteCF(uint32_t column_family_id,
                                  const Slice& key) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
                           const Slice& value) override {
-      ADD_FAILURE();
+      EXPECT_TRUE(false);
      return Status::OK();
    }
-    virtual void LogData(const Slice& blob) override { ADD_FAILURE(); }
+    virtual void LogData(const Slice& blob) override { EXPECT_TRUE(false); }
    virtual bool Continue() override { return num_seen < 2; }
  } handler;

@@ -16,6 +16,7 @@
 #include "rocksdb/db.h"
 #include "rocksdb/write_batch.h"
 #include "port/port.h"
+#include "util/logging.h"
 #include "util/random.h"
 #include "util/sync_point.h"
 #include "util/testharness.h"
@@ -106,10 +107,6 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
    std::vector<std::pair<string, string>> kvs_;
  };

-  // In each scenario we'll launch multiple threads to write.
-  // The size of each array equals to number of threads, and
-  // each boolean in it denote whether callback of corresponding
-  // thread should succeed or fail.
  std::vector<std::vector<WriteOP>> write_scenarios = {
      {true},
      {false},
@@ -137,6 +134,11 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
              options.allow_concurrent_memtable_write = allow_parallel;
              options.enable_pipelined_write = enable_pipelined_write;
              options.concurrent_prepare = two_queues;
+              if (options.enable_pipelined_write &&
+                  options.concurrent_prepare) {
+                // This combination is not supported
+                continue;
+              }

              ReadOptions read_options;
              DB* db;
@@ -148,37 +150,23 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
              db_impl = dynamic_cast<DBImpl*>(db);
              ASSERT_TRUE(db_impl);

-              // Writers that have called JoinBatchGroup.
-              std::atomic<uint64_t> threads_joining(0);
-              // Writers that have linked to the queue
-              std::atomic<uint64_t> threads_linked(0);
-              // Writers that pass WriteThread::JoinBatchGroup:Wait sync-point.
-              std::atomic<uint64_t> threads_verified(0);
-
+              std::atomic<uint64_t> threads_waiting(0);
              std::atomic<uint64_t> seq(db_impl->GetLatestSequenceNumber());
              ASSERT_EQ(db_impl->GetLatestSequenceNumber(), 0);

-              rocksdb::SyncPoint::GetInstance()->SetCallBack(
-                  "WriteThread::JoinBatchGroup:Start", [&](void*) {
-                    uint64_t cur_threads_joining = threads_joining.fetch_add(1);
-                    // Wait for the last joined writer to link to the queue.
-                    // In this way the writers link to the queue one by one.
-                    // This allows us to confidently detect the first writer
-                    // who increases threads_linked as the leader.
-                    while (threads_linked.load() < cur_threads_joining) {
-                    }
-                  });
-
-              // Verification once writers call JoinBatchGroup.
              rocksdb::SyncPoint::GetInstance()->SetCallBack(
                  "WriteThread::JoinBatchGroup:Wait", [&](void* arg) {
-                    uint64_t cur_threads_linked = threads_linked.fetch_add(1);
+                    uint64_t cur_threads_waiting = 0;
                    bool is_leader = false;
                    bool is_last = false;

                    // who am i
-                    is_leader = (cur_threads_linked == 0);
-                    is_last = (cur_threads_linked == write_group.size() - 1);
+                    do {
+                      cur_threads_waiting = threads_waiting.load();
+                      is_leader = (cur_threads_waiting == 0);
+                      is_last = (cur_threads_waiting == write_group.size() - 1);
+                    } while (!threads_waiting.compare_exchange_strong(
+                        cur_threads_waiting, cur_threads_waiting + 1));

                    // check my state
                    auto* writer = reinterpret_cast<WriteThread::Writer*>(arg);
@@ -202,10 +190,8 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
                                  !write_group.back().callback_.should_fail_);
                    }

-                    threads_verified.fetch_add(1);
-                    // Wait here until all verification in this sync-point
-                    // callback finish for all writers.
-                    while (threads_verified.load() < write_group.size()) {
+                    // wait for friends
+                    while (threads_waiting.load() < write_group.size()) {
                    }
                  });

@@ -230,20 +216,17 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {

              std::atomic<uint32_t> thread_num(0);
              std::atomic<char> dummy_key(0);
-
-              // Each write thread create a random write batch and write to DB
-              // with a write callback.
              std::function<void()> write_with_callback_func = [&]() {
                uint32_t i = thread_num.fetch_add(1);
                Random rnd(i);

                // leaders gotta lead
-                while (i > 0 && threads_verified.load() < 1) {
+                while (i > 0 && threads_waiting.load() < 1) {
                }

                // loser has to lose
                while (i == write_group.size() - 1 &&
-                       threads_verified.load() < write_group.size() - 1) {
+                       threads_waiting.load() < write_group.size() - 1) {
                }

                auto& write_op = write_group.at(i);
@@ -253,7 +236,11 @@ TEST_F(WriteCallbackTest, WriteWithCallbackTest) {
                // insert some keys
                for (uint32_t j = 0; j < rnd.Next() % 50; j++) {
                  // grab unique key
-                  char my_key = dummy_key.fetch_add(1);
+                  char my_key = 0;
+                  do {
+                    my_key = dummy_key.load();
+                  } while (
+                      !dummy_key.compare_exchange_strong(my_key, my_key + 1));

                  string skey(5, my_key);
                  string sval(10, my_key);
@@ -57,10 +57,6 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,
                                AdaptationContext* ctx) {
  uint8_t state;

-  // 1. Busy loop using "pause" for 1 micro sec
-  // 2. Else SOMETIMES busy loop using "yield" for 100 micro sec (default)
-  // 3. Else blocking wait
-
  // On a modern Xeon each loop takes about 7 nanoseconds (most of which
  // is the effect of the pause instruction), so 200 iterations is a bit
  // more than a microsecond.  This is long enough that waits longer than
@@ -118,21 +114,13 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,

  const size_t kMaxSlowYieldsWhileSpinning = 3;

-  // Whether the yield approach has any credit in this context. The credit is
-  // added by yield being succesfull before timing out, and decreased otherwise.
-  auto& yield_credit = ctx->value;
-  // Update the yield_credit based on sample runs or right after a hard failure
  bool update_ctx = false;
-  // Should we reinforce the yield credit
  bool would_spin_again = false;
-  // The samling base for updating the yeild credit. The sampling rate would be
-  // 1/sampling_base.
-  const int sampling_base = 256;

  if (max_yield_usec_ > 0) {
-    update_ctx = Random::GetTLSInstance()->OneIn(sampling_base);
+    update_ctx = Random::GetTLSInstance()->OneIn(256);

-    if (update_ctx || yield_credit.load(std::memory_order_relaxed) >= 0) {
+    if (update_ctx || ctx->value.load(std::memory_order_relaxed) >= 0) {
      // we're updating the adaptation statistics, or spinning has >
      // 50% chance of being shorter than max_yield_usec_ and causing no
      // involuntary context switches
@@ -161,7 +149,7 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,
          // accurate enough to measure the yield duration
          ++slow_yield_count;
          if (slow_yield_count >= kMaxSlowYieldsWhileSpinning) {
-            // Not just one ivcsw, but several.  Immediately update yield_credit
+            // Not just one ivcsw, but several.  Immediately update ctx
            // and fall back to blocking
            update_ctx = true;
            break;
@@ -177,19 +165,11 @@ uint8_t WriteThread::AwaitState(Writer* w, uint8_t goal_mask,
  }

  if (update_ctx) {
-    // Since our update is sample based, it is ok if a thread overwrites the
-    // updates by other threads. Thus the update does not have to be atomic.
-    auto v = yield_credit.load(std::memory_order_relaxed);
+    auto v = ctx->value.load(std::memory_order_relaxed);
    // fixed point exponential decay with decay constant 1/1024, with +1
    // and -1 scaled to avoid overflow for int32_t
-    //
-    // On each update the positive credit is decayed by a facor of 1/1024 (i.e.,
-    // 0.1%). If the sampled yield was successful, the credit is also increased
-    // by X. Setting X=2^17 ensures that the credit never exceeds
-    // 2^17*2^10=2^27, which is lower than 2^31 the upperbound of int32_t. Same
-    // logic applies to negative credits.
-    v = v - (v / 1024) + (would_spin_again ? 1 : -1) * 131072;
-    yield_credit.store(v, std::memory_order_relaxed);
+    v = v + (v / 1024) + (would_spin_again ? 1 : -1) * 16384;
+    ctx->value.store(v, std::memory_order_relaxed);
  }

  assert((state & goal_mask) != 0);
@@ -287,11 +267,10 @@ void WriteThread::CompleteFollower(Writer* w, WriteGroup& write_group) {
  SetState(w, STATE_COMPLETED);
 }

-static WriteThread::AdaptationContext jbg_ctx("JoinBatchGroup");
 void WriteThread::JoinBatchGroup(Writer* w) {
-  TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:Start", w);
-  assert(w->batch != nullptr);
+  static AdaptationContext ctx("JoinBatchGroup");

+  assert(w->batch != nullptr);
  bool linked_as_leader = LinkOne(w, &newest_writer_);
  if (linked_as_leader) {
    SetState(w, STATE_GROUP_LEADER);
@@ -315,7 +294,7 @@ void WriteThread::JoinBatchGroup(Writer* w) {
     */
    AwaitState(w, STATE_GROUP_LEADER | STATE_MEMTABLE_WRITER_LEADER |
                      STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED,
-               &jbg_ctx);
+               &ctx);
    TEST_SYNC_POINT_CALLBACK("WriteThread::JoinBatchGroup:DoneWaiting", w);
  }
 }
@@ -494,9 +473,9 @@ void WriteThread::LaunchParallelMemTableWriters(WriteGroup* write_group) {
  }
 }

-static WriteThread::AdaptationContext cpmtw_ctx("CompleteParallelMemTableWriter");
 // This method is called by both the leader and parallel followers
 bool WriteThread::CompleteParallelMemTableWriter(Writer* w) {
+  static AdaptationContext ctx("CompleteParallelMemTableWriter");

  auto* write_group = w->write_group;
  if (!w->status.ok()) {
@@ -506,7 +485,7 @@ bool WriteThread::CompleteParallelMemTableWriter(Writer* w) {

  if (write_group->running-- > 1) {
    // we're not the last one
-    AwaitState(w, STATE_COMPLETED, &cpmtw_ctx);
+    AwaitState(w, STATE_COMPLETED, &ctx);
    return false;
  }
  // else we're the last parallel worker and should perform exit duties.
@@ -525,13 +504,18 @@ void WriteThread::ExitAsBatchGroupFollower(Writer* w) {
  SetState(write_group->leader, STATE_COMPLETED);
 }

-static WriteThread::AdaptationContext eabgl_ctx("ExitAsBatchGroupLeader");
 void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
                                         Status status) {
+  static AdaptationContext ctx("ExitAsBatchGroupLeader");
  Writer* leader = write_group.leader;
  Writer* last_writer = write_group.last_writer;
  assert(leader->link_older == nullptr);

+  // Propagate memtable write error to the whole group.
+  if (status.ok() && !write_group.status.ok()) {
+    status = write_group.status;
+  }
+
  if (enable_pipelined_write_) {
    // Notify writers don't write to memtable to exit.
    for (Writer* w = last_writer; w != leader;) {
@@ -565,7 +549,7 @@ void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
    }
    AwaitState(leader, STATE_MEMTABLE_WRITER_LEADER |
                           STATE_PARALLEL_MEMTABLE_WRITER | STATE_COMPLETED,
-               &eabgl_ctx);
+               &ctx);
  } else {
    Writer* head = newest_writer_.load(std::memory_order_acquire);
    if (head != last_writer ||
@@ -612,15 +596,15 @@ void WriteThread::ExitAsBatchGroupLeader(WriteGroup& write_group,
  }
 }

-static WriteThread::AdaptationContext eu_ctx("EnterUnbatched");
 void WriteThread::EnterUnbatched(Writer* w, InstrumentedMutex* mu) {
+  static AdaptationContext ctx("EnterUnbatched");
  assert(w != nullptr && w->batch == nullptr);
  mu->Unlock();
  bool linked_as_leader = LinkOne(w, &newest_writer_);
  if (!linked_as_leader) {
    TEST_SYNC_POINT("WriteThread::EnterUnbatched:Wait");
    // Last leader will not pick us as a follower since our batch is nullptr
-    AwaitState(w, STATE_GROUP_LEADER, &eu_ctx);
+    AwaitState(w, STATE_GROUP_LEADER, &ctx);
  }
  if (enable_pipelined_write_) {
    WaitForMemTableWriters();
@@ -640,15 +624,15 @@ void WriteThread::ExitUnbatched(Writer* w) {
  }
 }

-static WriteThread::AdaptationContext wfmw_ctx("WaitForMemTableWriters");
 void WriteThread::WaitForMemTableWriters() {
+  static AdaptationContext ctx("WaitForMemTableWriters");
  assert(enable_pipelined_write_);
  if (newest_memtable_writer_.load() == nullptr) {
    return;
  }
  Writer w;
  if (!LinkOne(&w, &newest_memtable_writer_)) {
-    AwaitState(&w, STATE_MEMTABLE_WRITER_LEADER, &wfmw_ctx);
+    AwaitState(&w, STATE_MEMTABLE_WRITER_LEADER, &ctx);
  }
  newest_memtable_writer_.store(nullptr);
 }
@@ -55,7 +55,7 @@ class WriteThread {

    // The state used to inform a waiting writer that it has become a
    // parallel memtable writer. It can be the group leader who launch the
-    // parallel writer group, or one of the followers. The writer should then
+    // praallel writer group, or one of the followers. The writer should then
    // apply its batch to the memtable concurrently and call
    // CompleteParallelMemTableWriter.
    STATE_PARALLEL_MEMTABLE_WRITER = 8,
@@ -13,7 +13,7 @@ RocksDB is an embeddable persistent key-value store for fast storage. RocksDB ca

 RocksDB builds on [LevelDB](https://code.google.com/p/leveldb/) to be scalable to run on servers with many CPU cores, to efficiently use fast storage, to support IO-bound, in-memory and write-once workloads, and to be flexible to allow for innovation.

-For the latest details, watch [Mark Callaghan’s and Igor Canadi’s talk at CMU on 10/2015](https://scs.hosted.panopto.com/Panopto/Pages/Viewer.aspx?id=f4e0eb37-ae18-468f-9248-cb73edad3e56). [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages-old/intro.pdf?raw=true) from the Data @ Scale 2013 conference provides some perspective about how RocksDB has evolved.
+For the latest details, watch [Mark Callaghan’s and Igor Canadi’s talk at CMU on 10/2015](https://scs.hosted.panopto.com/Panopto/Pages/Viewer.aspx?id=f4e0eb37-ae18-468f-9248-cb73edad3e56). [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages/intro.pdf?raw=true) from the Data @ Scale 2013 conference provides some perspective about how RocksDB has evolved.

 ## How does performance compare?

@@ -11,7 +11,7 @@ The RocksDB library provides a persistent key value store. Keys and values are a

 The library is maintained by the Facebook Database Engineering Team, and is based on [LevelDB](https://github.com/google/leveldb), by Sanjay Ghemawat and Jeff Dean at Google.

-This overview gives some simple examples of how RocksDB is used. For the story of why RocksDB was created in the first place, see [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages-old/intro.pdf?raw=true) from the Data @ Scale 2013 conference.
+This overview gives some simple examples of how RocksDB is used. For the story of why RocksDB was created in the first place, see [Dhruba Borthakur’s introductory talk](https://github.com/facebook/rocksdb/blob/gh-pages/intro.pdf?raw=true) from the Data @ Scale 2013 conference.

 ## Opening A Database

@@ -1,22 +0,0 @@
---
-title: RocksDB 5.6.1 Released!
-layout: post
-author: yiwu
-category: blog
---
-
-### Public API Change
-* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads. See https://github.com/facebook/rocksdb/wiki/Thread-Pool for more details.
-* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
-* options.delayed_write_rate by default take the value of options.rate_limiter rate.
-* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`.
-
-### New Features
-* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads. See http://rocksdb.org/blog/2017/05/14/core-local-stats.html for more details.
-* Users can pass a cache object to write buffer manager, so that they can cap memory usage for memtable and block cache using one single limit.
-* Flush will be triggered when 7/8 of the limit introduced by write_buffer_manager or db_write_buffer_size is triggered, so that the hard threshold is hard to hit. See https://github.com/facebook/rocksdb/wiki/Write-Buffer-Manager for more details.
-* Introduce WriteOptions.low_pri. If it is true, low priority writes will be throttled if the compaction is behind. See https://github.com/facebook/rocksdb/wiki/Low-Priority-Write for more details.
-* `DB::IngestExternalFile()` now supports ingesting files into a database containing range deletions.
-
-### Bug Fixes
-* Shouldn't ignore return value of fsync() in flush.
@@ -1,7 +1,9 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//  This source code is also licensed under the GPLv2 license found in the
+//  COPYING file in the root directory of this source tree.

 #ifndef ROCKSDB_LITE

@@ -761,23 +761,23 @@ class PosixEnv : public Env {

  // Allow increasing the number of worker threads.
  virtual void SetBackgroundThreads(int num, Priority pri) override {
-    assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH);
+    assert(pri >= Priority::LOW && pri <= Priority::HIGH);
    thread_pools_[pri].SetBackgroundThreads(num);
  }

  virtual int GetBackgroundThreads(Priority pri) override {
-    assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH);
+    assert(pri >= Priority::LOW && pri <= Priority::HIGH);
    return thread_pools_[pri].GetBackgroundThreads();
  }

  // Allow increasing the number of worker threads.
  virtual void IncBackgroundThreadsIfNeeded(int num, Priority pri) override {
-    assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH);
+    assert(pri >= Priority::LOW && pri <= Priority::HIGH);
    thread_pools_[pri].IncBackgroundThreadsIfNeeded(num);
  }

  virtual void LowerThreadPoolIOPriority(Priority pool = LOW) override {
-    assert(pool >= Priority::BOTTOM && pool <= Priority::HIGH);
+    assert(pool >= Priority::LOW && pool <= Priority::HIGH);
 #ifdef OS_LINUX
    thread_pools_[pool].LowerIOPriority();
 #endif
@@ -883,7 +883,7 @@ PosixEnv::PosixEnv()

 void PosixEnv::Schedule(void (*function)(void* arg1), void* arg, Priority pri,
                        void* tag, void (*unschedFunction)(void* arg)) {
-  assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH);
+  assert(pri >= Priority::LOW && pri <= Priority::HIGH);
  thread_pools_[pri].Schedule(function, arg, tag, unschedFunction);
 }

@@ -892,7 +892,7 @@ int PosixEnv::UnSchedule(void* arg, Priority pri) {
 }

 unsigned int PosixEnv::GetThreadPoolQueueLen(Priority pri) const {
-  assert(pri >= Priority::BOTTOM && pri <= Priority::HIGH);
+  assert(pri >= Priority::LOW && pri <= Priority::HIGH);
  return thread_pools_[pri].GetQueueLen();
 }

@@ -125,14 +125,12 @@ static void SetBool(void* ptr) {
  reinterpret_cast<std::atomic<bool>*>(ptr)->store(true);
 }

-TEST_F(EnvPosixTest, RunImmediately) {
-  for (int pri = Env::BOTTOM; pri < Env::TOTAL; ++pri) {
-    std::atomic<bool> called(false);
-    env_->SetBackgroundThreads(1, static_cast<Env::Priority>(pri));
-    env_->Schedule(&SetBool, &called, static_cast<Env::Priority>(pri));
-    Env::Default()->SleepForMicroseconds(kDelayMicros);
-    ASSERT_TRUE(called.load());
-  }
+TEST_P(EnvPosixTestWithParam, RunImmediately) {
+  std::atomic<bool> called(false);
+  env_->Schedule(&SetBool, &called);
+  Env::Default()->SleepForMicroseconds(kDelayMicros);
+  ASSERT_TRUE(called.load());
+  WaitThreadPoolsEmpty();
 }

 TEST_P(EnvPosixTestWithParam, UnSchedule) {
@@ -11,7 +11,6 @@
 #include <algorithm>
 #include <chrono>
 #include "port/sys_time.h"
-#include "util/cast_util.h"
 #include "util/murmurhash.h"
 #include "util/random.h"
 #include "util/rate_limiter.h"
@@ -712,8 +711,7 @@ Status MockEnv::LockFile(const std::string& fname, FileLock** flock) {
 }

 Status MockEnv::UnlockFile(FileLock* flock) {
-  std::string fn =
-      static_cast_with_check<MockEnvFileLock, FileLock>(flock)->FileName();
+  std::string fn = dynamic_cast<MockEnvFileLock*>(flock)->FileName();
  {
    MutexLock lock(&mutex_);
    if (file_map_.find(fn) != file_map_.end()) {
@@ -8,10 +8,6 @@ ifndef DISABLE_JEMALLOC
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
 endif

-ifneq ($(USE_RTTI), 1)
-	CXXFLAGS += -fno-rtti
-endif
-
 .PHONY: clean librocksdb

 all: simple_example column_families_example compact_files_example c_simple_example optimistic_transaction_example transaction_example compaction_filter_example options_file_example
@@ -59,11 +59,7 @@ int main() {

  MyFilter filter;

-  int ret = system("rm -rf /tmp/rocksmergetest");
-  if (ret != 0) {
-    fprintf(stderr, "Error deleting /tmp/rocksmergetest, code: %d\n", ret);
-    return ret;
-  }
+  system("rm -rf /tmp/rocksmergetest");
  rocksdb::Options options;
  options.create_if_missing = true;
  options.merge_operator.reset(new MyMerge);
@@ -82,7 +82,6 @@ typedef struct rocksdb_compactionfiltercontext_t
 typedef struct rocksdb_compactionfilterfactory_t
    rocksdb_compactionfilterfactory_t;
 typedef struct rocksdb_comparator_t      rocksdb_comparator_t;
-typedef struct rocksdb_dbpath_t          rocksdb_dbpath_t;
 typedef struct rocksdb_env_t             rocksdb_env_t;
 typedef struct rocksdb_fifo_compaction_options_t rocksdb_fifo_compaction_options_t;
 typedef struct rocksdb_filelock_t        rocksdb_filelock_t;
@@ -714,9 +713,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_error_if_exists(
    rocksdb_options_t*, unsigned char);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_paranoid_checks(
    rocksdb_options_t*, unsigned char);
-extern ROCKSDB_LIBRARY_API void rocksdb_options_set_db_paths(rocksdb_options_t*,
-                                                             const rocksdb_dbpath_t** path_values, 
-                                                             size_t num_paths);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_env(rocksdb_options_t*,
                                                        rocksdb_env_t*);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_info_log(rocksdb_options_t*,
@@ -763,8 +759,6 @@ rocksdb_options_set_max_bytes_for_level_multiplier_additional(
    rocksdb_options_t*, int* level_values, size_t num_levels);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_enable_statistics(
    rocksdb_options_t*);
-extern ROCKSDB_LIBRARY_API void rocksdb_options_set_skip_stats_update_on_db_open(
-    rocksdb_options_t* opt, unsigned char val);

 /* returns a pointer to a malloc()-ed, null terminated string */
 extern ROCKSDB_LIBRARY_API char* rocksdb_options_statistics_get_string(
@@ -1095,11 +1089,6 @@ rocksdb_cache_get_usage(rocksdb_cache_t* cache);
 extern ROCKSDB_LIBRARY_API size_t
 rocksdb_cache_get_pinned_usage(rocksdb_cache_t* cache);

-/* DBPath */
-
-extern ROCKSDB_LIBRARY_API rocksdb_dbpath_t* rocksdb_dbpath_create(const char* path, uint64_t target_size);
-extern ROCKSDB_LIBRARY_API void rocksdb_dbpath_destroy(rocksdb_dbpath_t*);
-
 /* Env */

 extern ROCKSDB_LIBRARY_API rocksdb_env_t* rocksdb_create_default_env();
@@ -1258,12 +1247,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_delete_file_in_range_cf(

 /* Transactions */

-extern ROCKSDB_LIBRARY_API rocksdb_column_family_handle_t*
-rocksdb_transactiondb_create_column_family(
-    rocksdb_transactiondb_t* txn_db,
-    const rocksdb_options_t* column_family_options,
-    const char* column_family_name, char** errptr);
-
 extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* rocksdb_transactiondb_open(
    const rocksdb_options_t* options,
    const rocksdb_transactiondb_options_t* txn_db_options, const char* name,
@@ -1294,57 +1277,25 @@ extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get(
    rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options,
    const char* key, size_t klen, size_t* vlen, char** errptr);

-extern ROCKSDB_LIBRARY_API char* rocksdb_transaction_get_cf(
-    rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key, size_t klen,
-    size_t* vlen, char** errptr);
-
 extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get(
    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options,
    const char* key, size_t klen, size_t* vlen, char** errptr);

-extern ROCKSDB_LIBRARY_API char* rocksdb_transactiondb_get_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_readoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, size_t* vallen, char** errptr);
-
 extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put(
    rocksdb_transaction_t* txn, const char* key, size_t klen, const char* val,
    size_t vlen, char** errptr);

-extern ROCKSDB_LIBRARY_API void rocksdb_transaction_put_cf(
-    rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family,
-    const char* key, size_t klen, const char* val, size_t vlen, char** errptr);
-
 extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put(
    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
    const char* key, size_t klen, const char* val, size_t vlen, char** errptr);

-extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_put_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, const char* val, size_t vallen, char** errptr);
-
-extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_write(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
-    rocksdb_writebatch_t *batch, char** errptr);
-
 extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete(
    rocksdb_transaction_t* txn, const char* key, size_t klen, char** errptr);

-extern ROCKSDB_LIBRARY_API void rocksdb_transaction_delete_cf(
-    rocksdb_transaction_t* txn, rocksdb_column_family_handle_t* column_family,
-    const char* key, size_t klen, char** errptr);
-
 extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete(
    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
    const char* key, size_t klen, char** errptr);

-extern ROCKSDB_LIBRARY_API void rocksdb_transactiondb_delete_cf(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_writeoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, char** errptr);
-
 extern ROCKSDB_LIBRARY_API rocksdb_iterator_t*
 rocksdb_transaction_create_iterator(rocksdb_transaction_t* txn,
                                    const rocksdb_readoptions_t* options);
@@ -33,7 +33,7 @@ class Cleanable {
  typedef void (*CleanupFunction)(void* arg1, void* arg2);
  void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
  void DelegateCleanupsTo(Cleanable* other);
-  // DoCleanup and also resets the pointers for reuse
+  // DoCkeanup and also resets the pointers for reuse
  inline void Reset() {
    DoCleanup();
    cleanup_.function = nullptr;
@@ -64,10 +64,6 @@ class Comparator {
  // Simple comparator implementations may return with *key unchanged,
  // i.e., an implementation of this method that does nothing is correct.
  virtual void FindShortSuccessor(std::string* key) const = 0;
-
-  // if it is a wrapped comparator, may return the root one.
-  // return itself it is not wrapped.
-  virtual const Comparator* GetRootComparator() const { return this; }
 };

 // Return a builtin comparator that uses lexicographic byte-wise
@@ -329,11 +329,6 @@ void CancelAllBackgroundWork(DB* db, bool wait = false);
 // Snapshots before the delete might not see the data in the given range.
 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
                          const Slice* begin, const Slice* end);
-
-// Verify the checksum of file
-Status VerifySstFileChecksum(const Options& options,
-                             const EnvOptions& env_options,
-                             const std::string& file_path);
 #endif  // ROCKSDB_LITE

 }  // namespace rocksdb
Author	SHA1	Message	Date
Daniel Black	ede73d650c	table/block.h: change memset Summary: In gcc-7 the following is an error identified by -Werror=class-memaccess In file included from ./table/get_context.h:14:0, from db/version_set.cc:43: ./table/block.h: In constructor ‘rocksdb::BlockReadAmpBitmap::BlockReadAmpBitmap(size_t, size_t, rocksdb::Statistics)’: ./table/block.h:73:53: error: ‘void memset(void, int, size_t)’ clearing an object of type ‘struct std::atomic<unsigned int>’ with no trivial copy-assignment; use value-initialization instead [-Werror=class-memaccess] memset(bitmap_, 0, bitmap_size kBytesPersEntry); ^ In file included from ./db/version_set.h:23:0, from db/version_set.cc:12: /toolchain/include/c++/8.0.0/atomic:684:12: note: ‘struct std::atomic<unsigned int>’ declared here struct atomic<unsigned int> : __atomic_base<unsigned int> ^~~~~~~~~~~~~~~~~~~~ As a solution the default initializer can be applied in list context. Signed-off-by: Daniel Black <daniel.black@au.ibm.com> Closes https://github.com/facebook/rocksdb/pull/2561 Differential Revision: D5398714 Pulled By: siying fbshipit-source-id: d883fb88ec7535eee60d551038fe91f14488be36	2019-10-30 14:00:55 -07:00
przemyslaw.skibinski@percona.com	531d1186c4	Fix GitHub issue #3716 : gcc-8 warnings Summary: Fix the following gcc-8 warnings: - conflicting C language linkage declaration [-Werror] - writing to an object with no trivial copy-assignment [-Werror=class-memaccess] - array subscript -1 is below array bounds [-Werror=array-bounds] Solves https://github.com/facebook/rocksdb/issues/3716 Closes https://github.com/facebook/rocksdb/pull/3736 Differential Revision: D7684161 Pulled By: yiwu-arbug fbshipit-source-id: 47c0423d26b74add251f1d3595211eee1e41e54a	2019-10-30 13:54:49 -07:00
Vijay Nadimpalli	c8da4be8f5	Making platform 007 (gcc 7) default in build_detect_platform.sh (#5947 ) Summary: Making platform 007 (gcc 7) default in build_detect_platform.sh. Pull Request resolved: https://github.com/facebook/rocksdb/pull/5947 Differential Revision: D18038837 Pulled By: vjnadimpalli fbshipit-source-id: 9ac2ddaa93bf328a416faec028970e039886378e	2019-10-30 13:54:49 -07:00
Andrew Kryczka	90a87bbf61	Add latest toolchain (gcc-8, etc.) build support for fbcode users (#4923 ) Summary: - When building with internal dependencies, specify this toolchain by setting `ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1` - It is not enabled by default. However, it is enabled for TSAN builds in CI since there is a known problem with TSAN in gcc-5: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090 - I did not add support for Lua since (1) we agreed to deprecate it, and (2) we only have an internal build for v5.3 with this toolchain while that has breaking changes compared to our current version (v5.2). Pull Request resolved: https://github.com/facebook/rocksdb/pull/4923 Differential Revision: D13827226 Pulled By: ajkr fbshipit-source-id: 9aa3388ed3679777cfb15ef8cbcb83c07f62f947	2019-10-30 13:54:49 -07:00
Yi Wu	359e48333d	Bump version to 5.7.5	2017-11-28 12:25:25 -08:00
Yi Wu	30da984d07	Fix IOError on WAL write doesn't propagate to write group follower Summary: This is a simpler version of #3097 by removing all unrelated changes. Fixing the bug where concurrent writes may get Status::OK while it actually gets IOError on WAL write. This happens when multiple writes form a write batch group, and the leader get an IOError while writing to WAL. The leader failed to pass the error to followers in the group, and the followers end up returning Status::OK() while actually writing nothing. The bug only affect writes in a batch group. Future writes after the batch group will correctly return immediately with the IOError. Closes https://github.com/facebook/rocksdb/pull/3201 Differential Revision: D6421644 Pulled By: yiwu-arbug fbshipit-source-id: 1c2a455c5b73f6842423785eb8a9dbfbb191dc0e	2017-11-28 12:24:28 -08:00
Maysam Yabandeh	e3171c89c4	Exclude incompatible options in test Summary: options.enable_pipelined_write and options.concurrent_prepare are incompatible and should not be set together. Closes https://github.com/facebook/rocksdb/pull/2875 Differential Revision: D5818358 Pulled By: maysamyabandeh fbshipit-source-id: dad862508f00817ab302f8b61729accf38315fb8	2017-10-26 16:58:13 -07:00
gladiator	dcdf073654	fix missing manual_wal_flush for DBOptions ctor Summary: currently `ImmutableDBOptions::Dump` use default value for `concurrent_prepare` and `manual_wal_flush`, because DBOptions ctor does not init those member variables. so in LOG file, it will be ``` Options.concurrent_prepare: 0 Options.manual_wal_flush: 0 ``` Closes https://github.com/facebook/rocksdb/pull/2864 Differential Revision: D5816240 Pulled By: ajkr fbshipit-source-id: 82335e8bcae3dceedc6a99224e7998de5fad1e50	2017-09-14 15:37:51 -07:00
Yi Wu	096593bc60	Bump version to 5.7.4	2017-08-31 14:27:13 -07:00
Yi Wu	a03ae04493	make blob file close synchronous Summary: Fixing flaky blob_db_test. To close a blob file, blob db used to add a CloseSeqWrite job to the background thread to close it. Changing file close to be synchronous in order to simplify logic, and fix flaky blob_db_test. Closes https://github.com/facebook/rocksdb/pull/2787 Differential Revision: D5699387 Pulled By: yiwu-arbug fbshipit-source-id: dd07a945cd435cd3808fce7ee4ea57817409474a	2017-08-31 14:22:10 -07:00
yiwu-arbug	766fb69ec1	Blob db create a snapshot before every read Summary: If GC kicks in between * A Get() reads index entry from base db. * The Get() read from a blob file The GC can delete the corresponding blob file, making the key not found. Fortunately we have existing logic to avoid deleting a blob file if it is referenced by a snapshot. So the fix is to explicitly create a snapshot before reading index entry from base db. Closes https://github.com/facebook/rocksdb/pull/2754 Differential Revision: D5655956 Pulled By: yiwu-arbug fbshipit-source-id: e4ccbc51331362542e7343175bbcbdea5830f544	2017-08-31 14:22:05 -07:00
yiwu-arbug	e7de994745	GC the oldest file when out of space Summary: When out of space, blob db should GC the oldest file. The current implementation GC the newest one instead. Fixing it. Closes https://github.com/facebook/rocksdb/pull/2757 Differential Revision: D5657611 Pulled By: yiwu-arbug fbshipit-source-id: 56c30a4c52e6ab04551dda8c5c46006d4070b28d	2017-08-31 14:21:31 -07:00
yiwu-arbug	a4c3406242	Fix blob db crash during calculating write amp Summary: On initial call to BlobDBImpl::WaStats() `all_periods_write_` would be empty, so it will crash when we call pop_front() at line 1627. Apparently it is mean to pop only when `all_periods_write_.size() > kWriteAmplificationStatsPeriods`. The whole write amp calculation doesn't seems to be correct and it is not being exposed. Will work on it later. Test Plan Change kWriteAmplificationStatsPeriodMillisecs to 1000 (1 second) and run db_bench --use_blob_db for 5 minutes. Closes https://github.com/facebook/rocksdb/pull/2751 Differential Revision: D5648269 Pulled By: yiwu-arbug fbshipit-source-id: b843d9a09bb5f9e1b713d101ec7b87e54b5115a4	2017-08-31 14:21:01 -07:00
yiwu-arbug	672efad99a	Fix blob DB transaction usage while GC Summary: While GC, blob DB use optimistic transaction to delete or replace the index entry in LSM, to guarantee correctness if there's a normal write writing to the same key. However, the previous implementation doesn't call SetSnapshot() nor use GetForUpdate() of transaction API, instead it do its own sequence number checking before beginning the transaction. A normal write can sneak in after the sequence number check and overwrite the key, and the GC will delete or relocate the old version of the key by mistake. Update the code to property use GetForUpdate() to check the existing index entry. After the patch the sequence number store with each blob record is useless, So I'm considering remove the sequence number from blob record, in another patch. Closes https://github.com/facebook/rocksdb/pull/2703 Differential Revision: D5589178 Pulled By: yiwu-arbug fbshipit-source-id: 8dc960cd5f4e61b36024ba7c32d05584ce149c24	2017-08-31 14:20:55 -07:00
Yi Wu	b024e77f49	Avoid blob db call Sync() while writing Summary: The FsyncFiles background job call Fsync() periodically for blob files. However it can access WritableFileWriter concurrently with a Put() or Write(). And WritableFileWriter does not support concurrent access. It will lead to WritableFileWriter buffer being flush with same content twice, and blob file end up corrupted. Fixing by simply let FsyncFiles hold write_mutex_. Closes https://github.com/facebook/rocksdb/pull/2685 Differential Revision: D5561908 Pulled By: yiwu-arbug fbshipit-source-id: f0bb5bcab0e05694e053b8c49eab43640721e872	2017-08-31 14:17:33 -07:00
Yi Wu	6cc403ab71	Update all blob db TTL and timestamps to uint64_t Summary: The current blob db implementation use mix of int32_t, uint32_t and uint64_t for TTL and expiration. Update all timestamps to uint64_t for consistency. Closes https://github.com/facebook/rocksdb/pull/2683 Differential Revision: D5557103 Pulled By: yiwu-arbug fbshipit-source-id: e4eab2691629a755e614e8cf1eed9c3a681d0c42	2017-08-31 14:17:33 -07:00
Yi Wu	1339b299bc	Allow concurrent writes to blob db Summary: I'm going with brute-force solution, just letting Put() and Write() holding a mutex before writing. May improve concurrent writing with finer granularity locking later. Closes https://github.com/facebook/rocksdb/pull/2682 Differential Revision: D5552690 Pulled By: yiwu-arbug fbshipit-source-id: 039abd675b5d274a7af6428198d1733cafecef4c	2017-08-31 14:17:33 -07:00
Yi Wu	1b33ee8e7b	Blob DB garbage collection should keep keys with newer version Summary: Fix the bug where if blob db garbage collection revmoe keys with newer version. It shouldn't delete the key from base db when sequence number in base db is not equal to the one in blob log. Closes https://github.com/facebook/rocksdb/pull/2678 Differential Revision: D5549752 Pulled By: yiwu-arbug fbshipit-source-id: abb8649260963b5c389748023970fd746279d227	2017-08-31 14:16:43 -07:00
Yi Wu	b14207c0c5	Dump Blob DB options to info log Summary: * Dump blob db options to info log * Remove BlobDBOptionsImpl to disallow dynamic cast BlobDBOptions into BlobDBOptionsImpl. Move options there to be constants or into BlobDBOptions. The dynamic cast is broken after #2645 * Change some of the default options * Remove blob_db_options.min_blob_size, which is unimplemented. Will implement it soon. Closes https://github.com/facebook/rocksdb/pull/2671 Differential Revision: D5529912 Pulled By: yiwu-arbug fbshipit-source-id: dcd58ca981db5bcc7f123b65a0d6f6ae0dc703c7	2017-08-31 14:03:03 -07:00
Andrew Kryczka	acef93ed65	Bump version to 5.7.3	2017-08-29 23:08:32 -07:00
Andrew Kryczka	cf38a47a64	update HISTORY.md for DeleteRange bug fix Summary: fixed in #2799 Closes https://github.com/facebook/rocksdb/pull/2805 Differential Revision: D5734324 Pulled By: ajkr fbshipit-source-id: a285d4e84bf1018dc2257fd6c3e7c075a7243263	2017-08-29 23:05:48 -07:00
Huachao Huang	b722a18f67	Fix wrong smallest key of delete range tombstones Summary: Since tombstones are not stored in order, we may get a wrong smallest key if we only consider the first added tombstone. Check https://github.com/facebook/rocksdb/issues/2752 for more details. Closes https://github.com/facebook/rocksdb/pull/2799 Differential Revision: D5728217 Pulled By: ajkr fbshipit-source-id: 4a53edb0ca80d2a9fcf10749e52d47d57d6417d3	2017-08-29 23:04:42 -07:00
Andrew Kryczka	2b81d372eb	fix deleterange with memtable prefix bloom Summary: the range delete tombstones in memtable should be added to the aggregator even when the memtable's prefix bloom filter tells us the lookup key's not there. This bug could cause data to temporarily reappear until the memtable containing range deletions is flushed. Reported in #2743. Closes https://github.com/facebook/rocksdb/pull/2745 Differential Revision: D5639007 Pulled By: ajkr fbshipit-source-id: 04fc6facb6f978340a3f639536f4ca7c0d73dfc9	2017-08-29 23:02:34 -07:00
Yi Wu	cfaeb5846b	Bump version to 5.7.2	2017-08-15 14:19:55 -07:00
Siying Dong	62e9418a70	Fix false removal of tombstone issue in FIFO and kCompactionStyleNone Summary: Similar to the bug fixed by https://github.com/facebook/rocksdb/pull/2726, FIFO with compaction and kCompactionStyleNone during user customized CompactFiles() with output level to be 0 can suffer from the same problem. Fix it by leveraging the bottommost_level_ flag. Closes https://github.com/facebook/rocksdb/pull/2735 Differential Revision: D5626906 Pulled By: siying fbshipit-source-id: 2b148d0461c61dbd986d74655e384419ae442158	2017-08-15 14:12:19 -07:00
lxcode	03df818cff	Fix missing stdlib include required for abort() Summary: If ROCKSDB_LITE is defined, a call to abort() is introduced. This call requires stdlib.h. Build log of unpatched 5.7.1: http://beefy9.nyi.freebsd.org/data/110amd64-default/447974/logs/rocksdb-lite-5.7.1.log Closes https://github.com/facebook/rocksdb/pull/2744 Reviewed By: yiwu-arbug Differential Revision: D5632372 Pulled By: lxcode fbshipit-source-id: b2a8e692bf14ccf1f875f3a00463e87bba310a2b	2017-08-15 14:08:44 -07:00
Andrew Kryczka	96a092b1a3	empty commit	2017-08-14 10:27:56 -07:00
Andrew Kryczka	d59fdc3946	increment patch version for 5.7.1	2017-08-11 19:09:12 -07:00
Andrew Kryczka	e96d06c816	fix deletion dropping in intra-L0 Summary: `KeyNotExistsBeyondOutputLevel` didn't consider L0 files' key-ranges. So if a key only was covered by older L0 files' key-ranges, we would incorrectly drop deletions of that key. This PR just skips the deletion-dropping optimization when output level is L0. Closes https://github.com/facebook/rocksdb/pull/2726 Differential Revision: D5617286 Pulled By: ajkr fbshipit-source-id: 4bff1396b06d49a828ba4542f249191052915bce	2017-08-11 19:07:16 -07:00
Yi Wu	b86f89f25b	Move blob_db/ttl_extractor.h into blob_db/blob_db.h Summary: Move blob_db/ttl_extractor.h into blob_db/blob_db.h Also exclude TTLExtractor from LITE build. Closes https://github.com/facebook/rocksdb/pull/2665 Differential Revision: D5520009 Pulled By: yiwu-arbug fbshipit-source-id: 4813dcc272c7cc4bf2cdac285256d9a17d78c7b7	2017-07-28 14:39:11 -07:00
Yi Wu	48cf039064	Blob DB TTL extractor Summary: Introducing blob_db::TTLExtractor to replace extract_ttl_fn. The TTL extractor can be use to extract TTL from keys insert with Put or WriteBatch. Change over existing extract_ttl_fn are: * If value is changed, it will be return via std::string* (rather than Slice). With Slice the new value has to be part of the existing value. With std::string* the limitation is removed. * It can optionally return TTL or expiration. Other changes in this PR: * replace `std::chrono::system_clock` with `Env::NowMicros` so that I can mock time in tests. * add several TTL tests. * other minor naming change. Closes https://github.com/facebook/rocksdb/pull/2659 Differential Revision: D5512627 Pulled By: yiwu-arbug fbshipit-source-id: 0dfcb00d74d060b8534c6130c808e4d5d0a54440	2017-07-28 09:41:52 -07:00
Yi Wu	c39939d468	Reduce blob db noisy logging Summary: Remove some of the per-key logging by blob db to reduce noise. Closes https://github.com/facebook/rocksdb/pull/2587 Differential Revision: D5429115 Pulled By: yiwu-arbug fbshipit-source-id: b89328282fb8b3c64923ce48738c16017ce7feaf	2017-07-28 09:40:33 -07:00
Siying Dong	65afd8a1cc	Update java/rocksjni.pom Summary: Closes https://github.com/facebook/rocksdb/pull/2610 Differential Revision: D5445705 Pulled By: siying fbshipit-source-id: d5f97ffdf5bc18b853c3a106755aca96a650e428	2017-07-26 11:31:58 -07:00
Siying Dong	6df239cfee	Remove some left-over BSD headers Summary: Closes https://github.com/facebook/rocksdb/pull/2608 Differential Revision: D5444797 Pulled By: siying fbshipit-source-id: 690581d03f37822e059a16085088e8e2d8a45016	2017-07-26 11:31:42 -07:00
Siying Dong	54c4b3ab4f	Add back the LevelDB license file Summary: Closes https://github.com/facebook/rocksdb/pull/2591 Differential Revision: D5432696 Pulled By: siying fbshipit-source-id: a613230ab916de0b279a65ef429ede65460a8db2	2017-07-26 11:31:29 -07:00
Siying Dong	27768b0ec3	Remove the licensing description in CONTRIBUTING.md Summary: Closes https://github.com/facebook/rocksdb/pull/2590 Differential Revision: D5432539 Pulled By: siying fbshipit-source-id: 49902453bba3c95c1fb8354441b2198649e52bf4	2017-07-26 11:31:18 -07:00
Siying Dong	8673a8c567	Change RocksDB License Summary: Closes https://github.com/facebook/rocksdb/pull/2589 Differential Revision: D5431502 Pulled By: siying fbshipit-source-id: 8ebf8c87883daa9daa54b2303d11ce01ab1f6f75	2017-07-26 11:31:01 -07:00
Maysam Yabandeh	6124420d5d	Remove the orphan assert on !need_log_sync Summary: We initially had disabled support for write_options.sync when concurrent_prepare_ is set. We later added this support but the statement that asserts this combination is not used was left there. This patch cleans it up. Closes https://github.com/facebook/rocksdb/pull/2642 Differential Revision: D5496101 Pulled By: maysamyabandeh fbshipit-source-id: becbc503446f2a51bee24cc861958c090c724ec2	2017-07-26 07:31:36 -07:00
Maysam Yabandeh	258fd9ac46	Release note for partitioned index/filters Summary: Closes https://github.com/facebook/rocksdb/pull/2637 Differential Revision: D5489751 Pulled By: maysamyabandeh fbshipit-source-id: 0298f8960d4f86ce67959616615beee4d802c2e4	2017-07-25 11:20:15 -07:00