remove warning as error.

This reverts commit 61720850f6.

.gitignore

@@ -1,3 +1,4 @@
+TARGETS
 make_config.mk
 
 *.a
@@ -41,7 +42,6 @@ package/
 .phutil_module_cache
 unity.a
 tags
-etags
 rocksdb_dump
 rocksdb_undump
 db_test2
@@ -60,7 +60,7 @@ rocksdb.h
 unity.cc
 java/crossbuild/.vagrant
 .vagrant/
-java/**/*.asc
+java/**.asc
 java/javadoc
 
 scan_build_report/
@@ -71,4 +71,3 @@ db_logs/
 tp2/
 fbcode/
 fbcode
-buckifier/*.pyc

.travis.yml

@@ -1,70 +1,41 @@
 sudo: false
-dist: trusty
 language: cpp
 os:
   - linux
   - osx
 compiler:
   - clang
-  - gcc
 jdk:
   - oraclejdk7
-cache:
-  - ccache
-  - apt
 
 addons:
    apt:
       sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.6']
-      packages: ['clang-3.6' , 'g++-6', 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
+      packages: ['clang-3.6' , 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'gflags']
 env:
-  - TEST_GROUP=platform_dependent1
-  - TEST_GROUP=platform_dependent2
-  - TEST_GROUP=platform_dependent3
-  - TEST_GROUP=1
-  - TEST_GROUP=2
+  # Run all tests before db_block_cache_test (db_test, db_test2)
+  - JOB_NAME=unittests ROCKSDBTESTS_END=db_block_cache_test
+  # Run all tests starting from db_block_cache_test (db_block_cache_test, db_iter_test, ...)
+  - JOB_NAME=unittests ROCKSDBTESTS_START=db_block_cache_test
   # Run java tests
   - JOB_NAME=java_test
   # Build ROCKSDB_LITE
   - JOB_NAME=lite_build
-  # Build examples
-  - JOB_NAME=examples
-  - JOB_NAME=cmake
-  - JOB_NAME=cmake-mingw
 
-matrix:
-  exclude:
-  - os: osx
-    compiler: gcc
-  - os: osx
-    env: TEST_GROUP=1
-  - os: osx
-    env: TEST_GROUP=2
-  - os : osx
-    env: JOB_NAME=cmake-mingw
-  - os: linux
-    env: JOB_NAME=cmake-mingw
-    compiler: clang
+install:
+  # Build gflags
+  # TODO(noetzli): Remove when gflags available through Travis
+  - pushd /tmp/ && curl -L https://github.com/gflags/gflags/archive/v2.1.2.tar.gz -o gflags.tar.gz && tar xfz gflags.tar.gz && cd gflags-2.1.2 && cmake . && make && popd
 
 before_script:
-  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'clang++' ]]; then CXX=clang++-3.6; fi
-  # test one linux g++ build with g++-6
-  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'g++' && "${JOB_NAME}" == 'unittests' ]]; then CXX=g++-6; fi
+  - if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then CXX=clang++-3.6; fi
   # Limit the maximum number of open file descriptors to 8192
   - ulimit -n 8192 || true
 
 script:
-  - ${CXX} --version
-  - if [[ "${TEST_GROUP}" == 'platform_dependent1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=external_sst_file_basic_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == 'platform_dependent2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=external_sst_file_basic_test ROCKSDBTESTS_END=checkpoint_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == 'platform_dependent3' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=checkpoint_test ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == '1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
-  - if [[ "${TEST_GROUP}" == '2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
-  - if [[ "${JOB_NAME}" == 'java_test' ]]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
+  - if [[ "${JOB_NAME}" == 'unittests' ]]; then OPT=-DTRAVIS V=1 make -j4 check_some; fi
+  - if [[ "${JOB_NAME}" == 'java_test' ]]; then OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest; fi
   - if [[ "${JOB_NAME}" == 'lite_build' ]]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
-  - if [[ "${JOB_NAME}" == 'examples' ]]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
-  - if [[ "${JOB_NAME}" == 'cmake' ]]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
-  - if [[ "${JOB_NAME}" == 'cmake-mingw' ]]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi
 notifications:
     email:
       - leveldb@fb.com

CMakeLists.txt

@@ -1,5 +1,4 @@
-# Prerequisites for Windows:
-#     This cmake build is for Windows 64-bit only.
+# This cmake build is for Windows 64-bit only.
 #
 # Prerequisites:
 #     You must have at least Visual Studio 2015 Update 3. Start the Developer Command Prompt window that is a part of Visual Studio installation.
@@ -25,12 +24,6 @@
 # 5. And release mode (/m[:<N>] is also supported)
 #        msbuild rocksdb.sln /p:Configuration=Release
 #
-# Linux:
-#
-# 1. Install a recent toolchain such as devtoolset-3 if you're on a older distro. C++11 required.
-# 2. mkdir build; cd build
-# 3. cmake ..
-# 4. make -j
 
 cmake_minimum_required(VERSION 2.6)
 project(rocksdb)
@@ -41,20 +34,19 @@ endif()
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
 
-if(MSVC)
+if(WIN32)
   include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
 else()
   option(WITH_JEMALLOC "build with JeMalloc" OFF)
-  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
+  if(WITH_JEMALLOC)
+    find_package(JeMalloc REQUIRED)
+    add_definitions(-DROCKSDB_JEMALLOC)
+    include_directories(${JEMALLOC_INCLUDE_DIR})
+  endif()
+  if(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
     # FreeBSD has jemaloc as default malloc
-    # but it does not have all the jemalloc files in include/...
+    add_definitions(-DROCKSDB_JEMALLOC)
     set(WITH_JEMALLOC ON)
-  else()
-    if(WITH_JEMALLOC)
-      find_package(JeMalloc REQUIRED)
-      add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
-      include_directories(${JEMALLOC_INCLUDE_DIR})
-    endif()
   endif()
   option(WITH_SNAPPY "build with SNAPPY" OFF)
   if(WITH_SNAPPY)
@@ -66,8 +58,8 @@ else()
 endif()
 
 if(WIN32)
-  execute_process(COMMAND powershell -noprofile -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
-  execute_process(COMMAND powershell -noprofile -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
+  execute_process(COMMAND powershell -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
+  execute_process(COMMAND powershell -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
   string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE "${DATE}")
   string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME "${TIME}")
   set(GIT_DATE_TIME "${DATE} ${TIME}")
@@ -79,7 +71,7 @@ endif()
 
 find_package(Git)
 
-if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
+if (GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
   if(WIN32)
     execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
   else()
@@ -91,70 +83,20 @@ endif()
 
 string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
 
-if(NOT WIN32)
-  execute_process(COMMAND
-      "./build_tools/version.sh" "full"
-      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-      OUTPUT_VARIABLE ROCKSDB_VERSION
-  )
-  string(STRIP "${ROCKSDB_VERSION}" ROCKSDB_VERSION)
-  execute_process(COMMAND
-      "./build_tools/version.sh" "major"
-      WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
-      OUTPUT_VARIABLE ROCKSDB_VERSION_MAJOR
-  )
-  string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
-endif()
-
-if(WIN32)
-  option(WITH_AVX2 "build with AVX2" ON)
-  if(WITH_AVX2)
-    if(MSVC)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
-    else()
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
-    endif()
-  endif()
-
-  option(WITH_MD_LIBRARY "build with MD" ON)
-  if(MSVC)
-    if(WITH_MD_LIBRARY)
-      set(RUNTIME_LIBRARY "MD")
-    else()
-      set(RUNTIME_LIBRARY "MT")
-    endif()
-  endif()
-else()
-  option(WITH_SSE42 "build with SSE4.2" ON)
-  if(WITH_SSE42)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
-  endif()
-endif()
-
 set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
 configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
 add_library(build_version OBJECT ${BUILD_VERSION_CC})
 target_include_directories(build_version PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/util)
-if(MSVC)
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
+if(WIN32)
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo  /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
+
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W3 /wd4127 /wd4800 /wd4996 /wd4351")
 else()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers")
-  if(MINGW)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format")
-  endif()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-  if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
-    add_definitions(-DNDEBUG)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
-    include(CheckCXXCompilerFlag)
-    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
-    if(HAVE_OMIT_LEAF_FRAME_POINTER)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
-    endif()
-  endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -momit-leaf-frame-pointer")
 endif()
 
 option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
@@ -207,14 +149,14 @@ else()
    set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
 endif()
 
-if(MSVC)
+if(WIN32)
   if((${OPTIMIZE_DEBUG} EQUAL 1))
     message(STATUS "Debug optimization is enabled")
-    set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d")
+    set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd")
   else()
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d")
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
   endif()
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
+  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")
 
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
@@ -255,9 +197,6 @@ elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
   add_definitions(-DOS_ANDROID)
 elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
   add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
-  if(MINGW)
-    add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
-  endif()
 endif()
 
 if(NOT WIN32)
@@ -290,16 +229,24 @@ endif()
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
 include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.7.0/fused-src)
+
 find_package(Threads REQUIRED)
+if(WIN32)
+  set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
+else()
+  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
+endif()
+
+set(ROCKSDB_LIBS rocksdblib${ARTIFACT_SUFFIX})
+set(LIBS ${ROCKSDB_LIBS} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
 add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)
+add_subdirectory(tools)
 
 # Main library source code
 
 set(SOURCES
-        cache/clock_cache.cc
-        cache/lru_cache.cc
-        cache/sharded_cache.cc
+        db/auto_roll_logger.cc
         db/builder.cc
         db/c.cc
         db/column_family.cc
@@ -308,23 +255,19 @@ set(SOURCES
         db/compaction_iterator.cc
         db/compaction_job.cc
         db/compaction_picker.cc
-        db/compaction_picker_universal.cc
         db/convenience.cc
+        db/dbformat.cc
         db/db_filesnapshot.cc
         db/db_impl.cc
-        db/db_impl_write.cc
-        db/db_impl_compaction_flush.cc
-        db/db_impl_files.cc
-        db/db_impl_open.cc
         db/db_impl_debug.cc
         db/db_impl_experimental.cc
         db/db_impl_readonly.cc
         db/db_info_dumper.cc
         db/db_iter.cc
-        db/dbformat.cc
         db/event_helpers.cc
-        db/experimental.cc
         db/external_sst_file_ingestion_job.cc
+        db/experimental.cc
+        db/filename.cc
         db/file_indexer.cc
         db/flush_job.cc
         db/flush_scheduler.cc
@@ -334,6 +277,7 @@ set(SOURCES
         db/log_writer.cc
         db/managed_iterator.cc
         db/memtable.cc
+        db/memtable_allocator.cc
         db/memtable_list.cc
         db/merge_helper.cc
         db/merge_operator.cc
@@ -351,33 +295,12 @@ set(SOURCES
         db/write_batch_base.cc
         db/write_controller.cc
         db/write_thread.cc
-        env/env.cc
-        env/env_chroot.cc
-        env/env_hdfs.cc
-        env/memenv.cc
+        db/xfunc_test_points.cc
         memtable/hash_cuckoo_rep.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc
-        memtable/memtable_allocator.cc
         memtable/skiplistrep.cc
         memtable/vectorrep.cc
-        monitoring/histogram.cc
-        monitoring/histogram_windowing.cc
-        monitoring/instrumented_mutex.cc
-        monitoring/iostats_context.cc
-        monitoring/perf_context.cc
-        monitoring/perf_level.cc
-        monitoring/statistics.cc
-        monitoring/thread_status_impl.cc
-        monitoring/thread_status_updater.cc
-        monitoring/thread_status_util.cc
-        monitoring/thread_status_util_debug.cc
-        options/cf_options.cc
-        options/db_options.cc
-        options/options.cc
-        options/options_helper.cc
-        options/options_parser.cc
-        options/options_sanity_check.cc
         port/stack_trace.cc
         table/adaptive_table_factory.cc
         table/block.cc
@@ -395,47 +318,64 @@ set(SOURCES
         table/format.cc
         table/full_filter_block.cc
         table/get_context.cc
-        table/index_builder.cc
         table/iterator.cc
-        table/merging_iterator.cc
+        table/merger.cc
+        table/sst_file_writer.cc
         table/meta_blocks.cc
-        table/partitioned_filter_block.cc
-        table/persistent_cache_helper.cc
         table/plain_table_builder.cc
         table/plain_table_factory.cc
         table/plain_table_index.cc
         table/plain_table_key_coding.cc
         table/plain_table_reader.cc
-        table/sst_file_writer.cc
+        table/persistent_cache_helper.cc
         table/table_properties.cc
         table/two_level_iterator.cc
+        tools/sst_dump_tool.cc
         tools/db_bench_tool.cc
         tools/dump/db_dump_tool.cc
-        tools/ldb_cmd.cc
-        tools/ldb_tool.cc
-        tools/sst_dump_tool.cc
         util/arena.cc
-        util/auto_roll_logger.cc
         util/bloom.cc
+        util/cf_options.cc
+        util/clock_cache.cc
         util/coding.cc
         util/compaction_job_stats_impl.cc
         util/comparator.cc
         util/concurrent_arena.cc
         util/crc32c.cc
+        util/db_options.cc
         util/delete_scheduler.cc
         util/dynamic_bloom.cc
+        util/env.cc
+        util/env_chroot.cc
+        util/env_hdfs.cc
         util/event_logger.cc
-        util/file_reader_writer.cc
         util/file_util.cc
-        util/filename.cc
+        util/file_reader_writer.cc
+        util/sst_file_manager_impl.cc
         util/filter_policy.cc
         util/hash.cc
+        util/histogram.cc
+        util/histogram_windowing.cc
+        util/instrumented_mutex.cc
+        util/iostats_context.cc
+        util/lru_cache.cc
+        tools/ldb_cmd.cc
+        tools/ldb_tool.cc
+        util/logging.cc
         util/log_buffer.cc
+        util/memenv.cc
         util/murmurhash.cc
+        util/options.cc
+        util/options_helper.cc
+        util/options_parser.cc
+        util/options_sanity_check.cc
+        util/perf_context.cc
+        util/perf_level.cc
         util/random.cc
         util/rate_limiter.cc
+        util/sharded_cache.cc
         util/slice.cc
-        util/sst_file_manager_impl.cc
+        util/statistics.cc
         util/status.cc
         util/status_message.cc
         util/string_util.cc
@@ -443,36 +383,31 @@ set(SOURCES
         util/testutil.cc
         util/thread_local.cc
         util/threadpool_imp.cc
+        util/thread_status_impl.cc
+        util/thread_status_updater.cc
+        util/thread_status_util.cc
+        util/thread_status_util_debug.cc
         util/transaction_test_util.cc
+        util/xfunc.cc
         util/xxhash.cc
         utilities/backupable/backupable_db.cc
         utilities/blob_db/blob_db.cc
-        utilities/blob_db/blob_db_impl.cc
-        utilities/blob_db/blob_db_options_impl.cc
-        utilities/blob_db/blob_file.cc
-        utilities/blob_db/blob_log_reader.cc
-        utilities/blob_db/blob_log_writer.cc
-        utilities/blob_db/blob_log_format.cc
-        utilities/checkpoint/checkpoint_impl.cc
-        utilities/col_buf_decoder.cc
-        utilities/col_buf_encoder.cc
-        utilities/column_aware_encoding_util.cc
+        utilities/checkpoint/checkpoint.cc
         utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
         utilities/date_tiered/date_tiered_db_impl.cc
-        utilities/debug.cc
         utilities/document/document_db.cc
         utilities/document/json_document.cc
         utilities/document/json_document_builder.cc
         utilities/env_mirror.cc
-        utilities/env_timed.cc
+        utilities/env_registry.cc
         utilities/geodb/geodb_impl.cc
         utilities/leveldb_options/leveldb_options.cc
         utilities/lua/rocks_lua_compaction_filter.cc
         utilities/memory/memory_util.cc
-        utilities/merge_operators/max.cc
-        utilities/merge_operators/put.cc
         utilities/merge_operators/string_append/stringappend.cc
         utilities/merge_operators/string_append/stringappend2.cc
+        utilities/merge_operators/put.cc
+        utilities/merge_operators/max.cc
         utilities/merge_operators/uint64add.cc
         utilities/option_change_migration/option_change_migration.cc
         utilities/options/options_util.cc
@@ -485,17 +420,20 @@ set(SOURCES
         utilities/simulator_cache/sim_cache.cc
         utilities/spatialdb/spatial_db.cc
         utilities/table_properties_collectors/compact_on_deletion_collector.cc
-        utilities/transactions/optimistic_transaction_db_impl.cc
         utilities/transactions/optimistic_transaction_impl.cc
+        utilities/transactions/optimistic_transaction_db_impl.cc
         utilities/transactions/transaction_base.cc
+        utilities/transactions/transaction_impl.cc
         utilities/transactions/transaction_db_impl.cc
         utilities/transactions/transaction_db_mutex_impl.cc
-        utilities/transactions/transaction_impl.cc
         utilities/transactions/transaction_lock_mgr.cc
         utilities/transactions/transaction_util.cc
         utilities/ttl/db_ttl_impl.cc
         utilities/write_batch_with_index/write_batch_with_index.cc
         utilities/write_batch_with_index/write_batch_with_index_internal.cc
+        utilities/col_buf_encoder.cc
+        utilities/col_buf_decoder.cc
+        utilities/column_aware_encoding_util.cc
         $<TARGET_OBJECTS:build_version>)
 
 if(WIN32)
@@ -505,34 +443,12 @@ if(WIN32)
     port/win/env_default.cc
     port/win/port_win.cc
     port/win/win_logger.cc
-    port/win/win_thread.cc
     port/win/xpress_win.cc)
 else()
   list(APPEND SOURCES
     port/port_posix.cc
-    env/env_posix.cc
-    env/io_posix.cc)
-endif()
-
-set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
-set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
-set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
-if(WIN32)
-  set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
-  set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-else()
-  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT} rt)
-  set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-
-  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
-  target_link_libraries(${ROCKSDB_SHARED_LIB}
-    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-  set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
-                        LINKER_LANGUAGE CXX
-                        VERSION ${ROCKSDB_VERSION}
-                        SOVERSION ${ROCKSDB_VERSION_MAJOR}
-                        CXX_STANDARD 11
-                        OUTPUT_NAME "rocksdb")
+    util/env_posix.cc
+    util/io_posix.cc)
 endif()
 
 option(WITH_LIBRADOS "Build with librados" OFF)
@@ -542,22 +458,22 @@ if(WITH_LIBRADOS)
   list(APPEND THIRDPARTY_LIBS rados)
 endif()
 
-add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
-target_link_libraries(${ROCKSDB_STATIC_LIB}
+add_library(rocksdblib${ARTIFACT_SUFFIX} STATIC ${SOURCES})
+target_link_libraries(rocksdblib${ARTIFACT_SUFFIX}
   ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
 if(WIN32)
-  add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
-  target_link_libraries(${ROCKSDB_IMPORT_LIB}
-    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-  set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
-    COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS")
-  if(MSVC)
-    set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES
-      COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb")
-    set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
-      COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_IMPORT_LIB}.pdb")
-  endif()
+  set_target_properties(rocksdblib${ARTIFACT_SUFFIX} PROPERTIES
+    COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/rocksdblib${ARTIFACT_SUFFIX}.pdb")
+endif()
+
+add_library(rocksdb${ARTIFACT_SUFFIX} SHARED ${SOURCES})
+target_link_libraries(rocksdb${ARTIFACT_SUFFIX}
+  ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+
+if(WIN32)
+  set_target_properties(rocksdb${ARTIFACT_SUFFIX} PROPERTIES
+    COMPILE_FLAGS "-DROCKSDB_DLL -DROCKSDB_LIBRARY_EXPORTS /Fd${CMAKE_CFG_INTDIR}/rocksdb${ARTIFACT_SUFFIX}.pdb")
 endif()
 
 option(WITH_JNI "build with JNI" OFF)
@@ -569,61 +485,58 @@ else()
 endif()
 
 set(TESTS
-        cache/cache_test.cc
-        cache/lru_cache_test.cc
+        db/auto_roll_logger_test.cc
         db/column_family_test.cc
         db/compact_files_test.cc
         db/compaction_iterator_test.cc
-        db/compaction_job_stats_test.cc
         db/compaction_job_test.cc
+        db/compaction_job_stats_test.cc
         db/compaction_picker_test.cc
         db/comparator_db_test.cc
         db/corruption_test.cc
         db/cuckoo_table_db_test.cc
-        db/db_basic_test.cc
-        db/db_block_cache_test.cc
-        db/db_bloom_filter_test.cc
         db/db_compaction_filter_test.cc
         db/db_compaction_test.cc
         db/db_dynamic_level_test.cc
-        db/db_flush_test.cc
         db/db_inplace_update_test.cc
-        db/db_io_failure_test.cc
         db/db_iter_test.cc
-        db/db_iterator_test.cc
         db/db_log_iter_test.cc
         db/db_memtable_test.cc
         db/db_merge_operator_test.cc
         db/db_options_test.cc
         db/db_properties_test.cc
-        db/db_range_del_test.cc
-        db/db_sst_test.cc
-        db/db_statistics_test.cc
         db/db_table_properties_test.cc
         db/db_tailing_iter_test.cc
         db/db_test.cc
         db/db_test2.cc
+        db/db_block_cache_test.cc
+        db/db_bloom_filter_test.cc
+        db/db_flush_test.cc
+        db/db_iterator_test.cc
+        db/db_range_del_test.cc
+        db/db_sst_test.cc
+        db/external_sst_file_test.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc
         db/dbformat_test.cc
         db/deletefile_test.cc
-        db/external_sst_file_basic_test.cc
-        db/external_sst_file_test.cc
         db/fault_injection_test.cc
         db/file_indexer_test.cc
         db/filename_test.cc
         db/flush_job_test.cc
+        db/inlineskiplist_test.cc
         db/listener_test.cc
         db/log_test.cc
         db/manual_compaction_test.cc
         db/memtable_list_test.cc
-        db/merge_helper_test.cc
         db/merge_test.cc
+        db/merge_helper_test.cc
         db/options_file_test.cc
         db/perf_context_test.cc
         db/plain_table_db_test.cc
         db/prefix_test.cc
         db/repair_test.cc
+        db/skiplist_test.cc
         db/table_properties_collector_test.cc
         db/version_builder_test.cc
         db/version_edit_test.cc
@@ -632,16 +545,7 @@ set(TESTS
         db/write_batch_test.cc
         db/write_callback_test.cc
         db/write_controller_test.cc
-        env/env_basic_test.cc
-        env/env_test.cc
-        env/mock_env_test.cc
-        memtable/inlineskiplist_test.cc
-        memtable/skiplist_test.cc
-        monitoring/histogram_test.cc
-        monitoring/iostats_context_test.cc
-        monitoring/statistics_test.cc
-        options/options_settable_test.cc
-        options/options_test.cc
+        db/db_io_failure_test.cc
         table/block_based_filter_block_test.cc
         table/block_test.cc
         table/cuckoo_table_builder_test.cc
@@ -653,33 +557,40 @@ set(TESTS
         tools/reduce_levels_test.cc
         tools/sst_dump_test.cc
         util/arena_test.cc
-        util/auto_roll_logger_test.cc
         util/autovector_test.cc
         util/bloom_test.cc
+        util/cache_test.cc
         util/coding_test.cc
         util/crc32c_test.cc
         util/delete_scheduler_test.cc
         util/dynamic_bloom_test.cc
+        util/env_basic_test.cc
+        util/env_test.cc
         util/event_logger_test.cc
-        util/file_reader_writer_test.cc
         util/filelock_test.cc
+        util/file_reader_writer_test.cc
         util/heap_test.cc
+        util/histogram_test.cc
+        util/iostats_context_test.cc
+        util/lru_cache_test.cc
+        util/mock_env_test.cc
+        util/options_settable_test.cc
+        util/options_test.cc
         util/rate_limiter_test.cc
         util/slice_transform_test.cc
-        util/timer_queue_test.cc
+        util/statistics_test.cc
         util/thread_list_test.cc
         util/thread_local_test.cc
         utilities/backupable/backupable_db_test.cc
+        utilities/blob_db/blob_db_test.cc
         utilities/checkpoint/checkpoint_test.cc
-        utilities/column_aware_encoding_test.cc
         utilities/date_tiered/date_tiered_test.cc
         utilities/document/document_db_test.cc
         utilities/document/json_document_test.cc
+        utilities/env_registry_test.cc
         utilities/geodb/geodb_test.cc
-        utilities/lua/rocks_lua_test.cc
         utilities/memory/memory_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc
-        utilities/object_registry_test.cc
         utilities/option_change_migration/option_change_migration_test.cc
         utilities/options/options_util_test.cc
         utilities/persistent_cache/hash_table_test.cc
@@ -691,16 +602,18 @@ set(TESTS
         utilities/transactions/transaction_test.cc
         utilities/ttl/ttl_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc
+        utilities/column_aware_encoding_test.cc
+        utilities/lua/rocks_lua_test.cc
 )
 if(WITH_LIBRADOS)
   list(APPEND TESTS utilities/env_librados_test.cc)
 endif()
 
 set(BENCHMARKS
-  cache/cache_bench.cc
-  memtable/memtablerep_bench.cc
   tools/db_bench.cc
   table/table_reader_bench.cc
+  util/cache_bench.cc
+  db/memtablerep_bench.cc
   utilities/column_aware_encoding_exp.cc
   utilities/persistent_cache/hash_table_bench.cc)
 add_library(testharness OBJECT util/testharness.cc)
@@ -708,24 +621,24 @@ foreach(sourcefile ${BENCHMARKS})
   get_filename_component(exename ${sourcefile} NAME_WE)
   add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
     $<TARGET_OBJECTS:testharness>)
-  target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
+  target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${LIBS} gtest)
 endforeach(sourcefile ${BENCHMARKS})
 
 # For test util library that is build only in DEBUG mode
 # and linked to tests. Add test only code that is not #ifdefed for Release here.
 set(TESTUTIL_SOURCE
     db/db_test_util.cc
-    env/mock_env.cc
-    monitoring/thread_status_updater_debug.cc
     table/mock_table.cc
+    util/mock_env.cc
     util/fault_injection_test_env.cc
+    util/thread_status_updater_debug.cc
 )
 # test utilities are only build in debug
 enable_testing()
 add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
 set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
 add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
-if(MSVC)
+if(WIN32)
   set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
 endif()
 set_target_properties(${TESTUTILLIB}
@@ -746,7 +659,7 @@ foreach(sourcefile ${TEST_EXES})
       EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
       EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
       )
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} ${LIBS} gtest)
     if(NOT "${exename}" MATCHES "db_sanity_test")
       add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
       add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
@@ -766,18 +679,7 @@ foreach(sourcefile ${C_TEST_EXES})
       EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
       EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
       )
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} rocksdb${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX})
     add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
     add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
 endforeach(sourcefile ${C_TEST_EXES})
-add_subdirectory(tools)
-
-# Installation and packaging for Linux
-if(NOT WIN32)
-install(TARGETS ${ROCKSDB_STATIC_LIB} COMPONENT devel ARCHIVE DESTINATION lib64)
-install(TARGETS ${ROCKSDB_SHARED_LIB} COMPONENT runtime DESTINATION lib64)
-install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/rocksdb/"
-        COMPONENT devel
-        DESTINATION include/rocksdb)
-set(CMAKE_INSTALL_PREFIX /usr)
-endif()

COPYING

@@ -1,339 +0,0 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 2, June 1991
-
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-                            NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) year name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.

DEFAULT_OPTIONS_HISTORY.md

@@ -1,7 +1,4 @@
 # RocksDB default options change log
-## Unreleased
-* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
-
 ## 5.0 (11/17/2016)
 * Options::allow_concurrent_memtable_write and Options::enable_write_thread_adaptive_yield are now true by default
 * Options.level0_stop_writes_trigger default value changes from 24 to 32.

HISTORY.md

@@ -1,79 +1,9 @@
 # Rocksdb Change Log
-## 5.5.2 (07/13/2017)
-No change from 5.5.1.
-
-## 5.5.1 (06/27/2017)
-### Bug Fixes
-* Shouldn't ignore return value of fsync() in flush.
-
-## 5.5.0 (05/17/2017)
-### New Features
-* FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true.
-* DB::ResetStats() to reset internal stats.
-* Statistics::Reset() to reset user stats.
-* ldb add option --try_load_options, which will open DB with its own option file.
-* Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly.
-* Support dynamically change `max_open_files` option via SetDBOptions()
-* Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
-* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys.
-* Support file ingestion with universal compaction style
-* Support file ingestion behind with option `allow_ingest_behind`
-* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled.
-
-### Bug Fixes
-* Fix the bug that Direct I/O uses direct reads for non-SST file
-
-## 5.4.0 (04/11/2017)
-### Public API Change
-* random_access_max_buffer_size no longer has any effect
-* Removed Env::EnableReadAhead(), Env::ShouldForwardRawRequest()
-* Support dynamically change `stats_dump_period_sec` option via SetDBOptions().
-* Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped when using iterators.
-* DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases.
-    * PinnableSlice releases the pinned resources that contain the value when it is destructed or when ::Reset() is called on it.
-    * The old API that accepts std::string, although discouraged, is still supported.
-* Replace Options::use_direct_writes with Options::use_direct_io_for_flush_and_compaction. Read Direct IO wiki for details.
-* Added CompactionEventListener and EventListener::OnFlushBegin interfaces.
-
-### New Features
-* Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user.
-* Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
-* (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.
-
-## 5.3.0 (03/08/2017)
-### Public API Change
-* Remove disableDataSync option.
-* Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
-* Remove option min_partial_merge_operands. Partial merge operands will always be merged in flush or compaction if there are more than one.
-* Remove option verify_checksums_in_compaction. Compaction will always verify checksum.
-
-### Bug Fixes
-* Fix the bug that iterator may skip keys
-
-## 5.2.0 (02/08/2017)
-### Public API Change
-* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
-* Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
-* Options::use_direct_writes and Options::use_direct_reads are now ready to use.
-* (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition.
-
-### New Features
-* Added new overloaded function GetApproximateSizes that allows to specify if memtable stats should be computed only without computing SST files' stats approximations.
-* Added new function GetApproximateMemTableStats that approximates both number of records and size of memtables.
-* Add Direct I/O mode for SST file I/O
-
-### Bug Fixes
-* RangeSync() should work if ROCKSDB_FALLOCATE_PRESENT is not set
-* Fix wrong results in a data race case in Get()
-* Some fixes related to 2PC.
-* Fix bugs of data corruption in direct I/O
-
-## 5.1.0 (01/13/2017)
-* Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
-* Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
-* BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
-
-### Bug Fixes
+## Unreleased
+### Bug fixes
+* Fix the data corruption bug in the case that concurrent memtable write is enabled and 2PC is used.
+* OptimizeForPointLookup() doesn't work with the default DB setting of allow_concurrent_memtable_write=true. Fix it.
+* Fix a 2PC related bug where WAL files size grow too large.
 * Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.
 * When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying.
 
@@ -265,7 +195,7 @@ No change from 5.5.1.
 * Added a new way to report QPS from db_bench (check out --report_file and --report_interval_seconds)
 * Added a cache for individual rows. See DBOptions::row_cache for more info.
 * Several new features on EventListener (see include/rocksdb/listener.h):
- - OnCompationCompleted() now returns per-compaction job statistics, defined in include/rocksdb/compaction_job_stats.h.
+ - OnCompationCompleted() now returns per-compaciton job statistics, defined in include/rocksdb/compaction_job_stats.h.
  - Added OnTableFileCreated() and OnTableFileDeleted().
 * Add compaction_options_universal.enable_trivial_move to true, to allow trivial move while performing universal compaction. Trivial move will happen only when all the input files are non overlapping.
 

INSTALL.md

@@ -1,6 +1,6 @@
 ## Compilation
 
-**Important**: If you plan to run RocksDB in production, don't compile using default
+**Important**: If you plan to run RocksDB in production, don't compile using default 
 `make` or `make all`. That will compile RocksDB in debug mode, which is much slower
 than release mode.
 
@@ -22,18 +22,15 @@ use binaries compiled by `make all` in production.
 
 * By default the binary we produce is optimized for the platform you're compiling on
 (-march=native or the equivalent). If you want to build a portable binary, add 'PORTABLE=1' before
-your make commands, like this: `PORTABLE=1 make static_lib`. If you want to build a binary that
-makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SSE=1 make static_lib`.
+your make commands, like this: `PORTABLE=1 make static_lib`
 
 ## Dependencies
 
 * You can link RocksDB with following compression libraries:
   - [zlib](http://www.zlib.net/) - a library for data compression.
   - [bzip2](http://www.bzip.org/) - a library for data compression.
-  - [snappy](http://google.github.io/snappy/) - a library for fast
+  - [snappy](https://code.google.com/p/snappy/) - a library for fast
       data compression.
-  - [zstandard](http://www.zstd.net) - Fast real-time compression
-      algorithm.
 
 * All our tools depend on:
   - [gflags](https://gflags.github.io/gflags/) - a library that handles
@@ -51,45 +48,32 @@ makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SS
       `sudo apt-get install libsnappy-dev`.
     * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
     * Install bzip2: `sudo apt-get install libbz2-dev`.
-    * Install zstandard: `sudo apt-get install libzstd-dev`.
-
-* **Linux - CentOS / RHEL**
+* **Linux - CentOS**
     * Upgrade your gcc to version at least 4.8 to get C++11 support:
       `yum install gcc48-c++`
     * Install gflags:
 
-              git clone https://github.com/gflags/gflags.git
-              cd gflags
-              git checkout v2.0
+              wget https://gflags.googlecode.com/files/gflags-2.0-no-svn-files.tar.gz
+              tar -xzvf gflags-2.0-no-svn-files.tar.gz
+              cd gflags-2.0
               ./configure && make && sudo make install
 
-      **Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the
-      lib path to `LIBRARY_PATH`. If installed with default settings, the include path will be `/usr/local/include`
-      and the lib path will be `/usr/local/lib`.
-
     * Install snappy:
 
-              sudo yum install snappy snappy-devel
+              wget https://snappy.googlecode.com/files/snappy-1.1.1.tar.gz
+              tar -xzvf snappy-1.1.1.tar.gz
+              cd snappy-1.1.1
+              ./configure && make && sudo make install
 
     * Install zlib:
 
-              sudo yum install zlib zlib-devel
+              sudo yum install zlib
+              sudo yum install zlib-devel
 
     * Install bzip2:
 
-              sudo yum install bzip2 bzip2-devel
-
-    * Install ASAN (optional for debugging):
-
-              sudo yum install libasan
-
-    * Install zstandard:
-
-             wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
-             mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
-             tar zxvf zstd-1.1.3.tar.gz
-             cd zstd-1.1.3
-             make && sudo make install
+              sudo yum install bzip2
+              sudo yum install bzip2-devel
 
 * **OS X**:
     * Install latest C++ compiler that supports C++ 11:
@@ -105,30 +89,3 @@ makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SS
 * **Windows**:
   * For building with MS Visual Studio 13 you will need Update 4 installed.
   * Read and follow the instructions at CMakeLists.txt
-
-* **AIX 6.1**
-    * Install AIX Toolbox rpms with gcc
-    * Use these environment variables:
-  
-             export PORTABLE=1
-             export CC=gcc
-             export AR="ar -X64"
-             export EXTRA_ARFLAGS=-X64
-             export EXTRA_CFLAGS=-maix64
-             export EXTRA_CXXFLAGS=-maix64
-             export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
-             export LIBPATH=/opt/freeware/lib
-             export JAVA_HOME=/usr/java8_64
-             export PATH=/opt/freeware/bin:$PATH
-  
-* **Solaris Sparc**
-    * Install GCC 4.8.2 and higher.
-    * Use these environment variables:
-
-             export CC=gcc
-             export EXTRA_CFLAGS=-m64
-             export EXTRA_CXXFLAGS=-m64
-             export EXTRA_LDFLAGS=-m64
-             export PORTABLE=1
-             export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
-

LANGUAGE-BINDINGS.md

@@ -10,4 +10,3 @@ This is the list of all known third-party language bindings for RocksDB. If some
 * PHP - https://github.com/Photonios/rocksdb-php
 * C# - https://github.com/warrenfalk/rocksdb-sharp
 * Rust - https://github.com/spacejam/rust-rocksdb
-* D programming language - https://github.com/b1naryth1ef/rocksdb

Makefile

@@ -6,16 +6,12 @@
 
 #-----------------------------------------------
 
-BASH_EXISTS := $(shell which bash)
-SHELL := $(shell which bash)
-
 CLEAN_FILES = # deliberately empty, so we can append below.
 CFLAGS += ${EXTRA_CFLAGS}
 CXXFLAGS += ${EXTRA_CXXFLAGS}
 LDFLAGS += $(EXTRA_LDFLAGS)
 MACHINE ?= $(shell uname -m)
-ARFLAGS = ${EXTRA_ARFLAGS} rs
-STRIPFLAGS = -S -x
+ARFLAGS = rs
 
 # Transform parallel LOG output into something more readable.
 perl_command = perl -n \
@@ -141,7 +137,7 @@ endif
 
 AM_LINK = $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 # detect what platform we're building on
-dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; export PORTABLE="$(PORTABLE)"; "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
+dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; "$(CURDIR)/build_tools/build_detect_platform" "$(CURDIR)/make_config.mk"))
 # this file is generated by the previous line to set build flags and sources
 include make_config.mk
 CLEAN_FILES += make_config.mk
@@ -155,9 +151,7 @@ missing_make_config_paths := $(shell				\
 $(foreach path, $(missing_make_config_paths), \
 	$(warning Warning: $(path) dont exist))
 
-ifeq ($(PLATFORM), OS_AIX)
-# no debug info
-else ifneq ($(PLATFORM), IOS)
+ifneq ($(PLATFORM), IOS)
 CFLAGS += -g
 CXXFLAGS += -g
 else
@@ -165,11 +159,6 @@ else
 OPT += -DNDEBUG
 endif
 
-ifeq ($(PLATFORM), OS_AIX)
-ARFLAGS = -X64 rs
-STRIPFLAGS = -X64 -x
-endif
-
 ifeq ($(PLATFORM), OS_SOLARIS)
 	PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99
 endif
@@ -192,7 +181,7 @@ endif
 # TSAN doesn't work well with jemalloc. If we're compiling with TSAN, we should use regular malloc.
 ifdef COMPILE_WITH_TSAN
 	DISABLE_JEMALLOC=1
-	EXEC_LDFLAGS += -fsanitize=thread
+	EXEC_LDFLAGS += -fsanitize=thread -pie
 	PLATFORM_CCFLAGS += -fsanitize=thread -fPIC -DROCKSDB_TSAN_RUN
 	PLATFORM_CXXFLAGS += -fsanitize=thread -fPIC -DROCKSDB_TSAN_RUN
         # Turn off -pg when enabling TSAN testing, because that induces
@@ -202,11 +191,6 @@ ifdef COMPILE_WITH_TSAN
 	LUA_PATH =
 endif
 
-# AIX doesn't work with -pg
-ifeq ($(PLATFORM), OS_AIX)
-	PROFILING_FLAGS =
-endif
-
 # USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
 ifdef COMPILE_WITH_UBSAN
 	DISABLE_JEMALLOC=1
@@ -217,25 +201,18 @@ endif
 
 ifndef DISABLE_JEMALLOC
 	ifdef JEMALLOC
-		PLATFORM_CXXFLAGS += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
-		PLATFORM_CCFLAGS  += -DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE
+		PLATFORM_CXXFLAGS += "-DROCKSDB_JEMALLOC"
+		PLATFORM_CCFLAGS +=  "-DROCKSDB_JEMALLOC"
 	endif
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
 	PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE)
 endif
 
-export GTEST_THROW_ON_FAILURE=1
-export GTEST_HAS_EXCEPTIONS=1
+export GTEST_THROW_ON_FAILURE=1 GTEST_HAS_EXCEPTIONS=1
 GTEST_DIR = ./third-party/gtest-1.7.0/fused-src
-# AIX: pre-defined system headers are surrounded by an extern "C" block
-ifeq ($(PLATFORM), OS_AIX)
-	PLATFORM_CCFLAGS += -I$(GTEST_DIR)
-	PLATFORM_CXXFLAGS += -I$(GTEST_DIR)
-else
-	PLATFORM_CCFLAGS += -isystem $(GTEST_DIR)
-	PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
-endif
+PLATFORM_CCFLAGS += -isystem $(GTEST_DIR)
+PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
 
 # This (the first rule) must depend on "all".
 default: all
@@ -243,10 +220,6 @@ default: all
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
   -Wno-unused-parameter
 
-ifndef DISABLE_WARNING_AS_ERROR
-	WARNING_FLAGS += -Werror
-endif
-
 
 ifdef LUA_PATH
 
@@ -280,9 +253,6 @@ CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverl
 
 LDFLAGS += $(PLATFORM_LDFLAGS)
 
-# If NO_UPDATE_BUILD_VERSION is set we don't update util/build_version.cc, but
-# the file needs to already exist or else the build will fail
-ifndef NO_UPDATE_BUILD_VERSION
 date := $(shell date +%F)
 ifdef FORCE_GIT_SHA
 	git_sha := $(FORCE_GIT_SHA)
@@ -296,6 +266,7 @@ gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(da
 # as a regular source file as part of the compilation process.
 # One can run "strings executable_filename | grep _build_" to find
 # the version of the source that we used to build the executable file.
+CLEAN_FILES += util/build_version.cc:
 FORCE:
 util/build_version.cc: FORCE
 	$(AM_V_GEN)rm -f $@-t
@@ -303,7 +274,6 @@ util/build_version.cc: FORCE
 	$(AM_V_at)if test -f $@; then					\
 	  cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@;		\
 	else mv -f $@-t $@; fi
-endif
 
 LIBOBJECTS = $(LIB_SOURCES:.cc=.o)
 LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o)
@@ -322,26 +292,9 @@ BENCHTOOLOBJECTS = $(BENCH_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
 EXPOBJECTS = $(EXP_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
 
 TESTS = \
-	db_basic_test \
-	db_test2 \
-	external_sst_file_basic_test \
-	auto_roll_logger_test \
-	bloom_test \
-	dynamic_bloom_test \
-	c_test \
-	checkpoint_test \
-	crc32c_test \
-	coding_test \
-	inlineskiplist_test \
-	env_basic_test \
-	env_test \
-	thread_local_test \
-	rate_limiter_test \
-	perf_context_test \
-	iostats_context_test \
-	db_wal_test \
-	db_block_cache_test \
 	db_test \
+	db_test2 \
+	db_block_cache_test \
 	db_bloom_filter_test \
 	db_iter_test \
 	db_log_iter_test \
@@ -356,31 +309,40 @@ TESTS = \
 	db_options_test \
 	db_range_del_test \
 	db_sst_test \
+	external_sst_file_test \
 	db_tailing_iter_test \
 	db_universal_compaction_test \
+	db_wal_test \
 	db_io_failure_test \
 	db_properties_test \
 	db_table_properties_test \
-	db_statistics_test \
 	autovector_test \
-	cleanable_test \
 	column_family_test \
 	table_properties_collector_test \
 	arena_test \
+	auto_roll_logger_test \
 	block_test \
+	bloom_test \
+	dynamic_bloom_test \
+	c_test \
 	cache_test \
+	checkpoint_test \
+	coding_test \
 	corruption_test \
+	crc32c_test \
 	slice_transform_test \
 	dbformat_test \
+	env_basic_test \
+	env_test \
 	fault_injection_test \
 	filelock_test \
 	filename_test \
 	file_reader_writer_test \
 	block_based_filter_block_test \
 	full_filter_block_test \
-	partitioned_filter_block_test \
 	hash_table_test \
 	histogram_test \
+	inlineskiplist_test \
 	log_test \
 	manual_compaction_test \
 	mock_env_test \
@@ -395,13 +357,13 @@ TESTS = \
 	reduce_levels_test \
 	plain_table_db_test \
 	comparator_db_test \
-	external_sst_file_test \
 	prefix_test \
 	skiplist_test \
 	stringappend_test \
 	ttl_test \
 	date_tiered_test \
 	backupable_db_test \
+	blob_db_test \
 	document_db_test \
 	json_document_test \
 	sim_cache_test \
@@ -416,13 +378,14 @@ TESTS = \
 	write_controller_test\
 	deletefile_test \
 	table_test \
+	thread_local_test \
 	geodb_test \
+	rate_limiter_test \
 	delete_scheduler_test \
 	options_test \
 	options_settable_test \
 	options_util_test \
 	event_logger_test \
-	timer_queue_test \
 	cuckoo_table_builder_test \
 	cuckoo_table_reader_test \
 	cuckoo_table_db_test \
@@ -435,6 +398,7 @@ TESTS = \
 	sst_dump_test \
 	column_aware_encoding_test \
 	compact_files_test \
+	perf_context_test \
 	optimistic_transaction_test \
 	write_callback_test \
 	heap_test \
@@ -443,14 +407,11 @@ TESTS = \
 	option_change_migration_test \
 	transaction_test \
 	ldb_cmd_test \
+	iostats_context_test \
 	persistent_cache_test \
 	statistics_test \
 	lua_test \
-	range_del_aggregator_test \
 	lru_cache_test \
-	object_registry_test \
-	repair_test \
-	env_timed_test \
 
 PARALLEL_TEST = \
 	backupable_db_test \
@@ -546,14 +507,9 @@ $(SHARED3): $(SHARED4)
 	ln -fs $(SHARED4) $(SHARED3)
 endif
 
-shared_libobjects = $(patsubst %,shared-objects/%,$(LIBOBJECTS))
-CLEAN_FILES += shared-objects
-
-$(shared_libobjects): shared-objects/%.o: %.cc
-	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
-
-$(SHARED4): $(shared_libobjects)
-	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(shared_libobjects) $(LDFLAGS) -o $@
+$(SHARED4):
+	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LIB_SOURCES) $(TOOL_LIB_SOURCES) \
+		$(LDFLAGS) -o $@
 
 endif  # PLATFORM_SHARED_EXT
 
@@ -594,8 +550,7 @@ ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
 # and create a randomly-named rocksdb.XXXX directory therein.
 # We'll use that directory in the "make check" rules.
 ifeq ($(TMPD),)
-TMPDIR := $(shell echo $${TMPDIR:-/tmp})
-TMPD := $(shell f=/dev/shm; test -k $$f || f=$(TMPDIR);     \
+TMPD := $(shell f=/dev/shm; test -k $$f || f=/tmp;			\
   perl -le 'use File::Temp "tempdir";'					\
     -e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)')
 endif
@@ -704,8 +659,6 @@ check_0:
 	  | grep -E '$(tests-regexp)'					\
 	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
 
-valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized
-
 .PHONY: valgrind_check_0
 valgrind_check_0:
 	$(AM_V_GEN)export TEST_TMPDIR=$(TMPD);				\
@@ -719,7 +672,6 @@ valgrind_check_0:
 	}								\
 	  | $(prioritize_long_running_tests)				\
 	  | grep -E '$(tests-regexp)'					\
-	  | grep -E -v '$(valgrind-blacklist-regexp)'					\
 	  | build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu \
 	  '(if [[ "{}" == "./"* ]] ; then $(DRIVER) {}; else {}; fi) ' \
 	  '>& t/valgrind_log-{/}'
@@ -750,12 +702,10 @@ check: all
 	      echo "===== Running $$t"; ./$$t || exit 1; done;          \
 	fi
 	rm -rf $(TMPD)
-ifneq ($(PLATFORM), OS_AIX)
 ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
 	python tools/ldb_test.py
 	sh tools/rocksdb_dump_test.sh
 endif
-endif
 
 # TODO add ldb_tests
 check_some: $(SUBSET)
@@ -907,7 +857,6 @@ clean:
 tags:
 	ctags * -R
 	cscope -b `find . -name '*.cc'` `find . -name '*.h'` `find . -name '*.c'`
-	ctags -e -R -o etags *
 
 format:
 	build_tools/format-diff.sh
@@ -926,20 +875,20 @@ $(TOOLS_LIBRARY): $(BENCH_LIB_SOURCES:.cc=.o) $(TOOL_LIB_SOURCES:.cc=.o) $(LIB_S
 	$(AM_V_AR)rm -f $@
 	$(AM_V_at)$(AR) $(ARFLAGS) $@ $^
 
-librocksdb_env_basic_test.a: env/env_basic_test.o $(LIBOBJECTS) $(TESTHARNESS)
+librocksdb_env_basic_test.a: util/env_basic_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_V_AR)rm -f $@
 	$(AM_V_at)$(AR) $(ARFLAGS) $@ $^
 
 db_bench: tools/db_bench.o $(BENCHTOOLOBJECTS)
 	$(AM_LINK)
 
-cache_bench: cache/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
+cache_bench: util/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(AM_LINK)
 
 persistent_cache_bench: utilities/persistent_cache/persistent_cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(AM_LINK)
 
-memtablerep_bench: memtable/memtablerep_bench.o $(LIBOBJECTS) $(TESTUTIL)
+memtablerep_bench: db/memtablerep_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(AM_LINK)
 
 db_stress: tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL)
@@ -975,7 +924,7 @@ dynamic_bloom_test: util/dynamic_bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
 c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-cache_test: cache/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
+cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -993,7 +942,7 @@ redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
 hash_table_test: utilities/persistent_cache/hash_table_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-histogram_test: monitoring/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
+histogram_test: util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 thread_local_test: util/thread_local_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1008,9 +957,6 @@ crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
 slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_basic_test: db/db_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1059,12 +1005,6 @@ db_range_del_test: db/db_range_del_test.o db/db_test_util.o $(LIBOBJECTS) $(TEST
 db_sst_test: db/db_sst_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_statistics_test: db/db_statistics_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-external_sst_file_basic_test: db/external_sst_file_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 external_sst_file_test: db/external_sst_file_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1110,6 +1050,9 @@ prefix_test: db/prefix_test.o $(LIBOBJECTS) $(TESTHARNESS)
 backupable_db_test: utilities/backupable/backupable_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+blob_db_test: utilities/blob_db/blob_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 checkpoint_test: utilities/checkpoint/checkpoint_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1128,15 +1071,12 @@ spatial_db_test: utilities/spatialdb/spatial_db_test.o $(LIBOBJECTS) $(TESTHARNE
 env_mirror_test: utilities/env_mirror_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-env_timed_test: utilities/env_timed_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 ifdef ROCKSDB_USE_LIBRADOS
 env_librados_test: utilities/env_librados_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 endif
 
-object_registry_test: utilities/object_registry_test.o $(LIBOBJECTS) $(TESTHARNESS)
+env_registry_test: utilities/env_registry_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1169,10 +1109,10 @@ wal_manager_test: db/wal_manager_test.o $(LIBOBJECTS) $(TESTHARNESS)
 dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-env_basic_test: env/env_basic_test.o $(LIBOBJECTS) $(TESTHARNESS)
+env_basic_test: util/env_basic_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-env_test: env/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
+env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 fault_injection_test: db/fault_injection_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1196,25 +1136,19 @@ block_based_filter_block_test: table/block_based_filter_block_test.o $(LIBOBJECT
 full_filter_block_test: table/full_filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-partitioned_filter_block_test: table/partitioned_filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-cleanable_test: table/cleanable_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 block_test: table/block_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-inlineskiplist_test: memtable/inlineskiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
+inlineskiplist_test: db/inlineskiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-skiplist_test: memtable/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
+skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1289,10 +1223,10 @@ thread_list_test: util/thread_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
 compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-options_test: options/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
+options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-options_settable_test: options/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
+options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1304,9 +1238,6 @@ db_bench_tool_test: tools/db_bench_tool_test.o $(BENCHTOOLOBJECTS) $(TESTHARNESS
 event_logger_test: util/event_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-timer_queue_test: util/timer_queue_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 sst_dump_test: tools/sst_dump_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1316,7 +1247,7 @@ column_aware_encoding_test: utilities/column_aware_encoding_test.o $(TESTHARNESS
 optimistic_transaction_test: utilities/transactions/optimistic_transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-mock_env_test : env/mock_env_test.o $(LIBOBJECTS) $(TESTHARNESS)
+mock_env_test : util/mock_env_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 manual_compaction_test: db/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1325,7 +1256,7 @@ manual_compaction_test: db/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-auto_roll_logger_test: util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
+auto_roll_logger_test: db/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 memtable_list_test: db/memtable_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -1355,24 +1286,21 @@ ldb_cmd_test: tools/ldb_cmd_test.o $(LIBOBJECTS) $(TESTHARNESS)
 ldb: tools/ldb.o $(LIBOBJECTS)
 	$(AM_LINK)
 
-iostats_context_test: monitoring/iostats_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
+iostats_context_test: util/iostats_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
 
 persistent_cache_test: utilities/persistent_cache/persistent_cache_test.o  db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-statistics_test: monitoring/statistics_test.o $(LIBOBJECTS) $(TESTHARNESS)
+statistics_test: util/statistics_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-lru_cache_test: cache/lru_cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
+lru_cache_test: util/lru_cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 lua_test: utilities/lua/rocks_lua_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-range_del_aggregator_test: db/range_del_aggregator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -1420,38 +1348,15 @@ ifeq ($(PLATFORM), OS_SOLARIS)
 else
 	ARCH := $(shell getconf LONG_BIT)
 endif
-
-ifeq (,$(findstring ppc,$(MACHINE)))
-        ROCKSDBJNILIB = librocksdbjni-linux$(ARCH).so
-else
-        ROCKSDBJNILIB = librocksdbjni-linux-$(MACHINE).so
-endif
+ROCKSDBJNILIB = librocksdbjni-linux$(ARCH).so
 ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux$(ARCH).jar
 ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
 ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar
-SHA256_CMD = sha256sum
-
-ZLIB_VER ?= 1.2.11
-ZLIB_SHA256 ?= c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
-ZLIB_DOWNLOAD_BASE ?= http://zlib.net
-BZIP2_VER ?= 1.0.6
-BZIP2_SHA256 ?= a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
-BZIP2_DOWNLOAD_BASE ?= http://www.bzip.org
-SNAPPY_VER ?= 1.1.4
-SNAPPY_SHA256 ?= 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
-SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/releases/download
-LZ4_VER ?= 1.7.5
-LZ4_SHA256 ?= 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
-LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
-ZSTD_VER ?= 1.2.0
-ZSTD_SHA256 ?= 4a7e4593a3638276ca7f2a09dc4f38e674d8317bbea51626393ca73fc047cbfb
-ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
 
 ifeq ($(PLATFORM), OS_MACOSX)
 	ROCKSDBJNILIB = librocksdbjni-osx.jnilib
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
-	SHA256_CMD = openssl sha256 -r
 ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","")
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin
 else
@@ -1467,90 +1372,46 @@ ifeq ($(PLATFORM), OS_SOLARIS)
 	ROCKSDBJNILIB = librocksdbjni-solaris$(ARCH).so
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris
-	SHA256_CMD = digest -a sha256
-endif
-ifeq ($(PLATFORM), OS_AIX)
-	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/aix
-	ROCKSDBJNILIB = librocksdbjni-aix.so
-	EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf -
-	SNAPPY_MAKE_TARGET = libsnappy.la
 endif
 
 libz.a:
-	-rm -rf zlib-$(ZLIB_VER)
-	curl -O -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
-	ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \
-		echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf zlib-$(ZLIB_VER).tar.gz
-	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make
-	cp zlib-$(ZLIB_VER)/libz.a .
+	-rm -rf zlib-1.2.8
+	curl -O -L http://zlib.net/zlib-1.2.8.tar.gz
+	tar xvzf zlib-1.2.8.tar.gz
+	cd zlib-1.2.8 && CFLAGS='-fPIC' ./configure --static && make
+	cp zlib-1.2.8/libz.a .
 
 libbz2.a:
-	-rm -rf bzip2-$(BZIP2_VER)
-	curl -O -L ${BZIP2_DOWNLOAD_BASE}/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
-	BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \
-		echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf bzip2-$(BZIP2_VER).tar.gz
-	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
-	cp bzip2-$(BZIP2_VER)/libbz2.a .
+	-rm -rf bzip2-1.0.6
+	curl -O -L http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
+	tar xvzf bzip2-1.0.6.tar.gz
+	cd bzip2-1.0.6 && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64'
+	cp bzip2-1.0.6/libbz2.a .
 
 libsnappy.a:
-	-rm -rf snappy-$(SNAPPY_VER)
-	curl -O -L ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
-	SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
-		echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf snappy-$(SNAPPY_VER).tar.gz
-	cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
-	cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET}
-	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
+	-rm -rf snappy-1.1.3
+	curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz
+	tar xvzf snappy-1.1.3.tar.gz
+	cd snappy-1.1.3 && ./configure --with-pic --enable-static
+	cd snappy-1.1.3 && make
+	cp snappy-1.1.3/.libs/libsnappy.a .
 
 liblz4.a:
-	-rm -rf lz4-$(LZ4_VER)
-	curl -O -L ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
-	mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz
-	LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
-		echo lz4-$(LZ4_VER).tar.gz checksum mismatch, expected=\"$(LZ4_SHA256)\" actual=\"$$LZ4_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf lz4-$(LZ4_VER).tar.gz
-	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
-	cp lz4-$(LZ4_VER)/lib/liblz4.a .
-
-libzstd.a:
-	-rm -rf zstd-$(ZSTD_VER)
-	curl -O -L ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
-	mv v$(ZSTD_VER).tar.gz zstd-$(ZSTD_VER).tar.gz
-	ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \
-	if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \
-		echo zstd-$(ZSTD_VER).tar.gz checksum mismatch, expected=\"$(ZSTD_SHA256)\" actual=\"$$ZSTD_SHA256_ACTUAL\"; \
-		exit 1; \
-	fi
-	tar xvzf zstd-$(ZSTD_VER).tar.gz
-	cd zstd-$(ZSTD_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
-	cp zstd-$(ZSTD_VER)/lib/libzstd.a .
+	   -rm -rf lz4-r127
+	   curl -O -L https://codeload.github.com/Cyan4973/lz4/tar.gz/r127
+	   mv r127 lz4-r127.tar.gz
+	   tar xvzf lz4-r127.tar.gz
+	   cd lz4-r127/lib && make CFLAGS='-fPIC' all
+	   cp lz4-r127/lib/liblz4.a .
 
 # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
 java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
 CLEAN_FILES += jls
 
-ifneq ($(ROCKSDB_JAVA_NO_COMPRESSION), 1)
-JAVA_COMPRESSIONS = libz.a libbz2.a libsnappy.a liblz4.a libzstd.a
-endif
+JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4
+JAVA_STATIC_INCLUDES = -I./zlib-1.2.8 -I./bzip2-1.0.6 -I./snappy-1.1.3 -I./lz4-r127/lib
 
-JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD
-JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib
-
-$(java_static_libobjects): jls/%.o: %.cc $(JAVA_COMPRESSIONS)
+$(java_static_libobjects): jls/%.o: %.cc libz.a libbz2.a libsnappy.a liblz4.a
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
 
 rocksdbjavastatic: $(java_static_libobjects)
@@ -1559,8 +1420,8 @@ rocksdbjavastatic: $(java_static_libobjects)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
 	  $(java_static_libobjects) $(COVERAGEFLAGS) \
-	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
-	cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
+	  libz.a libbz2.a libsnappy.a liblz4.a $(JAVA_STATIC_LDFLAGS)
+	cd java/target;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
@@ -1573,25 +1434,8 @@ rocksdbjavastaticrelease: rocksdbjavastatic
 	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
 
-rocksdbjavastaticreleasedocker: rocksdbjavastatic
-	DOCKER_LINUX_X64_CONTAINER=`docker ps -aqf name=rocksdb_linux_x64-be`; \
-	if [ -z "$$DOCKER_LINUX_X64_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x64-be evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
-	fi
-	docker start -a rocksdb_linux_x64-be
-	DOCKER_LINUX_X86_CONTAINER=`docker ps -aqf name=rocksdb_linux_x86-be`; \
-	if [ -z "$$DOCKER_LINUX_X86_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
-	fi
-	docker start -a rocksdb_linux_x86-be
-	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
-	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
-	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
-
 rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral
 
-rocksdbjavastaticpublishdocker: rocksdbjavastaticreleasedocker rocksdbjavastaticpublishcentral
-
 rocksdbjavastaticpublishcentral:
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-javadoc.jar -Dclassifier=javadoc
 	mvn gpg:sign-and-deploy-file -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ -DrepositoryId=sonatype-nexus-staging -DpomFile=java/rocksjni.pom -Dfile=java/target/rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar -Dclassifier=sources
@@ -1636,6 +1480,13 @@ commit_prereq: build_tools/rocksdb-lego-determinator \
 	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit release release_481 clang_release tsan asan ubsan lite unit_non_shm
 	$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
 
+xfunc:
+	for xftest in $(XFUNC_TESTS); do \
+		echo "===== Running xftest $$xftest"; \
+		make check ROCKSDB_XFUNC_TEST="$$xftest" tests-regexp="DBTest" ;\
+	done
+
+
 # ---------------------------------------------------------------------------
 #  	Platform-specific compilation
 # ---------------------------------------------------------------------------

TARGETS

@@ -1,482 +0,0 @@
-REPO_PATH = "internal_repo_rocksdb/repo/"
-BUCK_BINS = "buck-out/gen/" + REPO_PATH
-TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
-rocksdb_compiler_flags = [
-  "-msse",
-  "-msse4.2",
-  "-fno-builtin-memcmp",
-  "-DROCKSDB_PLATFORM_POSIX",
-  "-DROCKSDB_LIB_IO_POSIX",
-  "-DROCKSDB_FALLOCATE_PRESENT",
-  "-DROCKSDB_MALLOC_USABLE_SIZE",
-  "-DROCKSDB_RANGESYNC_PRESENT",
-  "-DROCKSDB_SCHED_GETCPU_PRESENT",
-  "-DOS_LINUX",
-  # Flags to enable libs we include
-  "-DSNAPPY",
-  "-DZLIB",
-  "-DBZIP2",
-  "-DLZ4",
-  "-DZSTD",
-  "-DGFLAGS=gflags",
-  "-DNUMA",
-  "-DTBB",
-  # Needed to compile in fbcode
-  "-Wno-expansion-to-defined",
-]
-
-rocksdb_external_deps = [
-  ('bzip2', None, 'bz2'),
-  ('snappy', None, "snappy"),
-  ('zlib', None, 'z'),
-  ('gflags', None, 'gflags'),
-  ('lz4', None, 'lz4'),
-  ('zstd', None),
-  ('tbb', None),
-  ("numa", "2.0.8", "numa"),
-  ("googletest", None, "gtest"),
-]
-
-rocksdb_preprocessor_flags = [
-  # Directories with files for #include
-  "-I" + REPO_PATH + "include/",
-  "-I" + REPO_PATH,
-]
-
-cpp_library(
-    name = "rocksdb_lib",
-    headers = AutoHeaders.RECURSIVE_GLOB,
-    srcs = [
-      "cache/clock_cache.cc",
-      "cache/lru_cache.cc",
-      "cache/sharded_cache.cc",
-      "db/builder.cc",
-      "db/c.cc",
-      "db/column_family.cc",
-      "db/compacted_db_impl.cc",
-      "db/compaction.cc",
-      "db/compaction_iterator.cc",
-      "db/compaction_job.cc",
-      "db/compaction_picker.cc",
-      "db/compaction_picker_universal.cc",
-      "db/convenience.cc",
-      "db/db_filesnapshot.cc",
-      "db/db_impl.cc",
-      "db/db_impl_write.cc",
-      "db/db_impl_compaction_flush.cc",
-      "db/db_impl_files.cc",
-      "db/db_impl_open.cc",
-      "db/db_impl_debug.cc",
-      "db/db_impl_experimental.cc",
-      "db/db_impl_readonly.cc",
-      "db/db_info_dumper.cc",
-      "db/db_iter.cc",
-      "db/dbformat.cc",
-      "db/event_helpers.cc",
-      "db/experimental.cc",
-      "db/external_sst_file_ingestion_job.cc",
-      "db/file_indexer.cc",
-      "db/flush_job.cc",
-      "db/flush_scheduler.cc",
-      "db/forward_iterator.cc",
-      "db/internal_stats.cc",
-      "db/log_reader.cc",
-      "db/log_writer.cc",
-      "db/managed_iterator.cc",
-      "db/memtable.cc",
-      "db/memtable_list.cc",
-      "db/merge_helper.cc",
-      "db/merge_operator.cc",
-      "db/range_del_aggregator.cc",
-      "db/repair.cc",
-      "db/snapshot_impl.cc",
-      "db/table_cache.cc",
-      "db/table_properties_collector.cc",
-      "db/transaction_log_impl.cc",
-      "db/version_builder.cc",
-      "db/version_edit.cc",
-      "db/version_set.cc",
-      "db/wal_manager.cc",
-      "db/write_batch.cc",
-      "db/write_batch_base.cc",
-      "db/write_controller.cc",
-      "db/write_thread.cc",
-      "env/env.cc",
-      "env/env_chroot.cc",
-      "env/env_hdfs.cc",
-      "env/env_posix.cc",
-      "env/io_posix.cc",
-      "env/memenv.cc",
-      "memtable/hash_cuckoo_rep.cc",
-      "memtable/hash_linklist_rep.cc",
-      "memtable/hash_skiplist_rep.cc",
-      "memtable/memtable_allocator.cc",
-      "memtable/skiplistrep.cc",
-      "memtable/vectorrep.cc",
-      "monitoring/histogram.cc",
-      "monitoring/histogram_windowing.cc",
-      "monitoring/instrumented_mutex.cc",
-      "monitoring/iostats_context.cc",
-      "monitoring/perf_context.cc",
-      "monitoring/perf_level.cc",
-      "monitoring/statistics.cc",
-      "monitoring/thread_status_impl.cc",
-      "monitoring/thread_status_updater.cc",
-      "monitoring/thread_status_updater_debug.cc",
-      "monitoring/thread_status_util.cc",
-      "monitoring/thread_status_util_debug.cc",
-      "options/cf_options.cc",
-      "options/db_options.cc",
-      "options/options.cc",
-      "options/options_helper.cc",
-      "options/options_parser.cc",
-      "options/options_sanity_check.cc",
-      "port/port_posix.cc",
-      "port/stack_trace.cc",
-      "table/adaptive_table_factory.cc",
-      "table/block.cc",
-      "table/block_based_filter_block.cc",
-      "table/block_based_table_builder.cc",
-      "table/block_based_table_factory.cc",
-      "table/block_based_table_reader.cc",
-      "table/block_builder.cc",
-      "table/block_prefix_index.cc",
-      "table/bloom_block.cc",
-      "table/cuckoo_table_builder.cc",
-      "table/cuckoo_table_factory.cc",
-      "table/cuckoo_table_reader.cc",
-      "table/flush_block_policy.cc",
-      "table/format.cc",
-      "table/full_filter_block.cc",
-      "table/get_context.cc",
-      "table/index_builder.cc",
-      "table/iterator.cc",
-      "table/merging_iterator.cc",
-      "table/meta_blocks.cc",
-      "table/partitioned_filter_block.cc",
-      "table/persistent_cache_helper.cc",
-      "table/plain_table_builder.cc",
-      "table/plain_table_factory.cc",
-      "table/plain_table_index.cc",
-      "table/plain_table_key_coding.cc",
-      "table/plain_table_reader.cc",
-      "table/sst_file_writer.cc",
-      "table/table_properties.cc",
-      "table/two_level_iterator.cc",
-      "tools/dump/db_dump_tool.cc",
-      "util/arena.cc",
-      "util/auto_roll_logger.cc",
-      "util/bloom.cc",
-      "util/build_version.cc",
-      "util/coding.cc",
-      "util/compaction_job_stats_impl.cc",
-      "util/comparator.cc",
-      "util/concurrent_arena.cc",
-      "util/crc32c.cc",
-      "util/delete_scheduler.cc",
-      "util/dynamic_bloom.cc",
-      "util/event_logger.cc",
-      "util/file_reader_writer.cc",
-      "util/file_util.cc",
-      "util/filename.cc",
-      "util/filter_policy.cc",
-      "util/hash.cc",
-      "util/log_buffer.cc",
-      "util/murmurhash.cc",
-      "util/random.cc",
-      "util/rate_limiter.cc",
-      "util/slice.cc",
-      "util/sst_file_manager_impl.cc",
-      "util/status.cc",
-      "util/status_message.cc",
-      "util/string_util.cc",
-      "util/sync_point.cc",
-      "util/thread_local.cc",
-      "util/threadpool_imp.cc",
-      "util/transaction_test_util.cc",
-      "util/xxhash.cc",
-      "utilities/backupable/backupable_db.cc",
-      "utilities/blob_db/blob_db.cc",
-      "utilities/blob_db/blob_db_impl.cc",
-      "utilities/blob_db/blob_db_options_impl.cc",
-      "utilities/blob_db/blob_file.cc",
-      "utilities/blob_db/blob_log_reader.cc",
-      "utilities/blob_db/blob_log_writer.cc",
-      "utilities/blob_db/blob_log_format.cc",
-      "utilities/checkpoint/checkpoint_impl.cc",
-      "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
-      "utilities/convenience/info_log_finder.cc",
-      "utilities/date_tiered/date_tiered_db_impl.cc",
-      "utilities/debug.cc",
-      "utilities/document/document_db.cc",
-      "utilities/document/json_document.cc",
-      "utilities/document/json_document_builder.cc",
-      "utilities/env_mirror.cc",
-      "utilities/env_timed.cc",
-      "utilities/geodb/geodb_impl.cc",
-      "utilities/leveldb_options/leveldb_options.cc",
-      "utilities/lua/rocks_lua_compaction_filter.cc",
-      "utilities/memory/memory_util.cc",
-      "utilities/merge_operators/max.cc",
-      "utilities/merge_operators/put.cc",
-      "utilities/merge_operators/string_append/stringappend.cc",
-      "utilities/merge_operators/string_append/stringappend2.cc",
-      "utilities/merge_operators/uint64add.cc",
-      "utilities/option_change_migration/option_change_migration.cc",
-      "utilities/options/options_util.cc",
-      "utilities/persistent_cache/block_cache_tier.cc",
-      "utilities/persistent_cache/block_cache_tier_file.cc",
-      "utilities/persistent_cache/block_cache_tier_metadata.cc",
-      "utilities/persistent_cache/persistent_cache_tier.cc",
-      "utilities/persistent_cache/volatile_tier_impl.cc",
-      "utilities/redis/redis_lists.cc",
-      "utilities/simulator_cache/sim_cache.cc",
-      "utilities/spatialdb/spatial_db.cc",
-      "utilities/table_properties_collectors/compact_on_deletion_collector.cc",
-      "utilities/transactions/optimistic_transaction_db_impl.cc",
-      "utilities/transactions/optimistic_transaction_impl.cc",
-      "utilities/transactions/transaction_base.cc",
-      "utilities/transactions/transaction_db_impl.cc",
-      "utilities/transactions/transaction_db_mutex_impl.cc",
-      "utilities/transactions/transaction_impl.cc",
-      "utilities/transactions/transaction_lock_mgr.cc",
-      "utilities/transactions/transaction_util.cc",
-      "utilities/ttl/db_ttl_impl.cc",
-      "utilities/write_batch_with_index/write_batch_with_index.cc",
-      "utilities/write_batch_with_index/write_batch_with_index_internal.cc",
-      "tools/ldb_cmd.cc",
-      "tools/ldb_tool.cc",
-      "tools/sst_dump_tool.cc",
-      "utilities/blob_db/blob_dump_tool.cc",
-    ],
-    deps = [],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-
-cpp_library(
-    name = "rocksdb_test_lib",
-    headers = AutoHeaders.RECURSIVE_GLOB,
-    srcs = [
-      "env/mock_env.cc",
-      "table/mock_table.cc",
-      "util/fault_injection_test_env.cc",
-      "util/testharness.cc",
-      "util/testutil.cc",
-      "db/db_test_util.cc",
-      "utilities/col_buf_encoder.cc",
-      "utilities/col_buf_decoder.cc",
-      "utilities/column_aware_encoding_util.cc",
-    ],
-    deps = [":rocksdb_lib"],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-
-cpp_library(
-    name = "rocksdb_tools_lib",
-    headers = AutoHeaders.RECURSIVE_GLOB,
-    srcs = [
-      "tools/db_bench_tool.cc",
-      "util/testutil.cc",
-    ],
-    deps = [":rocksdb_lib"],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-
-cpp_library(
-    name = "env_basic_test_lib",
-    headers = AutoHeaders.RECURSIVE_GLOB,
-    srcs = ["env/env_basic_test.cc"],
-    deps = [":rocksdb_test_lib"],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-
-# [test_name, test_src, test_type]
-ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
- ['cache_test', 'cache/cache_test.cc', 'serial'],
- ['options_file_test', 'db/options_file_test.cc', 'serial'],
- ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
- ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
- ['thread_list_test', 'util/thread_list_test.cc', 'serial'],
- ['table_properties_collector_test',
-  'db/table_properties_collector_test.cc',
-  'serial'],
- ['document_db_test', 'utilities/document/document_db_test.cc', 'serial'],
- ['event_logger_test', 'util/event_logger_test.cc', 'serial'],
- ['coding_test', 'util/coding_test.cc', 'serial'],
- ['statistics_test', 'monitoring/statistics_test.cc', 'serial'],
- ['options_settable_test', 'options/options_settable_test.cc', 'serial'],
- ['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'],
- ['column_aware_encoding_test',
-  'utilities/column_aware_encoding_test.cc',
-  'serial'],
- ['db_iterator_test', 'db/db_iterator_test.cc', 'serial'],
- ['db_sst_test', 'db/db_sst_test.cc', 'parallel'],
- ['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'],
- ['listener_test', 'db/listener_test.cc', 'serial'],
- ['write_callback_test', 'db/write_callback_test.cc', 'serial'],
- ['version_set_test', 'db/version_set_test.cc', 'serial'],
- ['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'],
- ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
- ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
- ['compact_files_test', 'db/compact_files_test.cc', 'serial'],
- ['db_options_test', 'db/db_options_test.cc', 'serial'],
- ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
- ['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'],
- ['dbformat_test', 'db/dbformat_test.cc', 'serial'],
- ['write_batch_with_index_test',
-  'utilities/write_batch_with_index/write_batch_with_index_test.cc',
-  'serial'],
- ['json_document_test', 'utilities/document/json_document_test.cc', 'serial'],
- ['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'],
- ['repair_test', 'db/repair_test.cc', 'serial'],
- ['persistent_cache_test',
-  'utilities/persistent_cache/persistent_cache_test.cc',
-  'parallel'],
- ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
- ['external_sst_file_basic_test',
-  'db/external_sst_file_basic_test.cc',
-  'serial'],
- ['options_test', 'options/options_test.cc', 'serial'],
- ['perf_context_test', 'db/perf_context_test.cc', 'serial'],
- ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
- ['heap_test', 'util/heap_test.cc', 'serial'],
- ['db_test2', 'db/db_test2.cc', 'serial'],
- ['filelock_test', 'util/filelock_test.cc', 'serial'],
- ['write_controller_test', 'db/write_controller_test.cc', 'serial'],
- ['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'],
- ['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'],
- ['c_test', 'db/c_test.c', 'serial'],
- ['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'],
- ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
- ['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'],
- ['db_test', 'db/db_test.cc', 'parallel'],
- ['block_based_filter_block_test',
-  'table/block_based_filter_block_test.cc',
-  'serial'],
- ['merge_test', 'db/merge_test.cc', 'serial'],
- ['bloom_test', 'util/bloom_test.cc', 'serial'],
- ['block_test', 'table/block_test.cc', 'serial'],
- ['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'],
- ['backupable_db_test',
-  'utilities/backupable/backupable_db_test.cc',
-  'parallel'],
- ['db_flush_test', 'db/db_flush_test.cc', 'serial'],
- ['filename_test', 'db/filename_test.cc', 'serial'],
- ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
- ['slice_transform_test', 'util/slice_transform_test.cc', 'serial'],
- ['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'],
- ['inlineskiplist_test', 'memtable/inlineskiplist_test.cc', 'parallel'],
- ['optimistic_transaction_test',
-  'utilities/transactions/optimistic_transaction_test.cc',
-  'serial'],
- ['hash_table_test',
-  'utilities/persistent_cache/hash_table_test.cc',
-  'serial'],
- ['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'],
- ['option_change_migration_test',
-  'utilities/option_change_migration/option_change_migration_test.cc',
-  'serial'],
- ['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'],
- ['autovector_test', 'util/autovector_test.cc', 'serial'],
- ['db_iter_test', 'db/db_iter_test.cc', 'serial'],
- ['flush_job_test', 'db/flush_job_test.cc', 'serial'],
- ['wal_manager_test', 'db/wal_manager_test.cc', 'serial'],
- ['write_batch_test', 'db/write_batch_test.cc', 'serial'],
- ['crc32c_test', 'util/crc32c_test.cc', 'serial'],
- ['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'],
- ['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'],
- ['compaction_job_test', 'db/compaction_job_test.cc', 'serial'],
- ['mock_env_test', 'env/mock_env_test.cc', 'serial'],
- ['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'],
- ['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'],
- ['arena_test', 'util/arena_test.cc', 'serial'],
- ['stringappend_test',
-  'utilities/merge_operators/string_append/stringappend_test.cc',
-  'serial'],
- ['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'],
- ['prefix_test', 'db/prefix_test.cc', 'serial'],
- ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
- ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
- ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
- ['db_statistics_test', 'db/db_statistics_test.cc', 'serial'],
- ['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
- ['log_test', 'db/log_test.cc', 'serial'],
- ['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
- ['deletefile_test', 'db/deletefile_test.cc', 'serial'],
- ['partitioned_filter_block_test',
-  'table/partitioned_filter_block_test.cc',
-  'serial'],
- ['comparator_db_test', 'db/comparator_db_test.cc', 'serial'],
- ['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'],
- ['thread_local_test', 'util/thread_local_test.cc', 'serial'],
- ['version_builder_test', 'db/version_builder_test.cc', 'serial'],
- ['db_range_del_test', 'db/db_range_del_test.cc', 'serial'],
- ['table_test', 'table/table_test.cc', 'parallel'],
- ['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'],
- ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
- ['options_util_test', 'utilities/options/options_util_test.cc', 'serial'],
- ['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'],
- ['db_basic_test', 'db/db_basic_test.cc', 'serial'],
- ['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'],
- ['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'],
- ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
- ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
- ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
- ['corruption_test', 'db/corruption_test.cc', 'serial'],
- ['compact_on_deletion_collector_test',
-  'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc',
-  'serial'],
- ['env_test', 'env/env_test.cc', 'serial'],
- ['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
- ['timer_queue_test', 'util/timer_queue_test.cc', 'serial'],
- ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
- ['db_memtable_test', 'db/db_memtable_test.cc', 'serial'],
- ['db_universal_compaction_test',
-  'db/db_universal_compaction_test.cc',
-  'parallel'],
- ['histogram_test', 'monitoring/histogram_test.cc', 'serial'],
- ['util_merge_operators_test',
-  'utilities/util_merge_operators_test.cc',
-  'serial'],
- ['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'],
- ['env_basic_test', 'env/env_basic_test.cc', 'serial'],
- ['iostats_context_test', 'monitoring/iostats_context_test.cc', 'serial'],
- ['memtable_list_test', 'db/memtable_list_test.cc', 'serial'],
- ['column_family_test', 'db/column_family_test.cc', 'serial'],
- ['db_properties_test', 'db/db_properties_test.cc', 'serial'],
- ['version_edit_test', 'db/version_edit_test.cc', 'serial'],
- ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
- ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
- ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']]
-
-
-# Generate a test rule for each entry in ROCKS_TESTS
-for test_cfg in ROCKS_TESTS:
-    test_name = test_cfg[0]
-    test_cc = test_cfg[1]
-    ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
-    test_bin = test_name + "_bin"
-
-    cpp_binary (
-      name = test_bin,
-      srcs = [test_cc],
-      deps = [":rocksdb_test_lib"],
-      preprocessor_flags = rocksdb_preprocessor_flags,
-      compiler_flags = rocksdb_compiler_flags,
-      external_deps = rocksdb_external_deps,
-    )
-
-    custom_unittest(
-      name = test_name,
-      type = ttype,
-      deps = [":" + test_bin],
-      command = [TEST_RUNNER, BUCK_BINS + test_bin]
-    )

USERS.md

@@ -7,7 +7,7 @@ At Facebook, we use RocksDB as storage engines in multiple data management servi
 2. MongoRocks -- https://github.com/mongodb-partners/mongo-rocks
 3. ZippyDB --  Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[*] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
 4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[*]
-4. Dragon -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/
+4. Dragan -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/
 5. Stylus -- a low-level stream processing framework writtenin C++.[*]
 
 [*] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
@@ -64,7 +64,7 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub
 quasardb uses a heavily tuned RocksDB as its persistence layer.
 
 ## Netflix
-[Netflix](http://techblog.netflix.com/2016/05/application-data-caching-using-ssds.html) Netflix uses RocksDB on AWS EC2 instances with local SSD drives to cache application data.
+[Netflix](http://techblog.netflix.com/2016/05/application-data-caching-using-ssds.html) uses RocksDB on spinning disks to cache application data.
 
 ## TiKV
 [TiKV](https://github.com/pingcap/tikv) is a GEO-replicated, high-performance, distributed, transactional key-value database. TiKV is powered by Rust and Raft. TiKV uses RocksDB as its persistence layer.
@@ -74,9 +74,3 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 
 ## Dgraph
 [Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine.
-
-## Uber
-[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and and scalable task queue.
-
-## 360 Pika
-[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company

appveyor.yml

@@ -11,5 +11,7 @@ build:
   verbosity: minimal
 test:
 test_script:
-- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8
+- ps: build_tools\run_ci_db_test.ps1 -EnableRerun -Run db_test2 -Concurrency 8
+- ps: build_tools\run_ci_db_test.ps1 -EnableRerun -Run db_test -Exclude DBTest.GroupCommitTest -Concurrency 10
+- ps: build_tools\run_ci_db_test.ps1 -Run env_test -Concurrency 1
 

arcanist_util/config/FacebookArcanistConfiguration.php

@@ -9,13 +9,6 @@ require('RocksDBCommonHelper.php');
 define("DIFF_COMMAND", "diff");
 
 class FacebookArcanistConfiguration extends ArcanistConfiguration {
-  public function getCustomArgumentsForCommand($command) {
-    if ($command == "land") {
-      return array(
-          'async' => array('help' => 'Just to make tools happy'));
-    }
-    return array();
-  }
 
   public function didRunWorkflow($command,
                                  ArcanistWorkflow $workflow,

arcanist_util/config/FacebookOldArcanistConfiguration.php

@@ -9,13 +9,6 @@ require('RocksDBCommonHelper.php');
 define("DIFF_COMMAND", "diff");
 
 class FacebookArcanistConfiguration extends ArcanistConfiguration {
-  public function getCustomArgumentsForCommand($command) {
-    if ($command == "land") {
-      return array(
-          'async' => array('help' => 'Just to make tools happy'));
-    }
-    return array();
-  }
 
   public function didRunWorkflow($command,
                                  ArcanistBaseWorkflow $workflow,

arcanist_util/config/RocksDBCommonHelper.php

@@ -21,7 +21,7 @@ function postURL($diffID, $url) {
   assert(is_numeric($diffID));
   assert(strlen($url) > 0);
 
-  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+  $cmd = 'echo \'{"diff_id": "' . $diffID . '", '
          . '"name":"click here for sandcastle tests for D' . $diffID . '", '
          . '"link":"' . $url . '"}\' | '
          . 'arc call-conduit '
@@ -35,7 +35,7 @@ function buildUpdateTestStatusCmd($diffID, $test, $status) {
   assert(strlen($test) > 0);
   assert(strlen($status) > 0);
 
-  $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
+  $cmd = 'echo \'{"diff_id": "' . $diffID . '", '
          . '"name":"' . $test . '", '
          . '"result":"' . $status . '"}\' | '
          . 'arc call-conduit '

buckifier/buckify_rocksdb.py

@@ -1,172 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-from targets_builder import TARGETSBuilder
-from optparse import OptionParser
-import os
-import fnmatch
-import sys
-import tempfile
-
-from util import ColorString
-import util
-
-# tests to export as libraries for inclusion in other projects
-_EXPORTED_TEST_LIBS = ["env_basic_test"]
-
-# Parse src.mk files as a Dictionary of
-# VAR_NAME => list of files
-def parse_src_mk(repo_path):
-    src_mk = repo_path + "/src.mk"
-    src_files = {}
-    for line in open(src_mk):
-        line = line.strip()
-        if len(line) == 0 or line[0] == '#':
-            continue
-        if '=' in line:
-            current_src = line.split('=')[0].strip()
-            src_files[current_src] = []
-        elif '.cc' in line:
-            src_path = line.split('.cc')[0].strip() + '.cc'
-            src_files[current_src].append(src_path)
-    return src_files
-
-
-# get all .cc / .c files
-def get_cc_files(repo_path):
-    cc_files = []
-    for root, dirnames, filenames in os.walk(repo_path):
-        root = root[(len(repo_path) + 1):]
-        if "java" in root:
-            # Skip java
-            continue
-        for filename in fnmatch.filter(filenames, '*.cc'):
-            cc_files.append(os.path.join(root, filename))
-        for filename in fnmatch.filter(filenames, '*.c'):
-            cc_files.append(os.path.join(root, filename))
-    return cc_files
-
-
-# Get tests from Makefile
-def get_tests(repo_path):
-    Makefile = repo_path + "/Makefile"
-
-    # Dictionary TEST_NAME => IS_PARALLEL
-    tests = {}
-
-    found_tests = False
-    for line in open(Makefile):
-        line = line.strip()
-        if line.startswith("TESTS ="):
-            found_tests = True
-        elif found_tests:
-            if line.endswith("\\"):
-                # remove the trailing \
-                line = line[:-1]
-                line = line.strip()
-                tests[line] = False
-            else:
-                # we consumed all the tests
-                break
-
-    found_parallel_tests = False
-    for line in open(Makefile):
-        line = line.strip()
-        if line.startswith("PARALLEL_TEST ="):
-            found_parallel_tests = True
-        elif found_parallel_tests:
-            if line.endswith("\\"):
-                # remove the trailing \
-                line = line[:-1]
-                line = line.strip()
-                tests[line] = True
-            else:
-                # we consumed all the parallel tests
-                break
-
-    return tests
-
-
-# Prepare TARGETS file for buck
-def generate_targets(repo_path):
-    print(ColorString.info("Generating TARGETS"))
-    # parsed src.mk file
-    src_mk = parse_src_mk(repo_path)
-    # get all .cc files
-    cc_files = get_cc_files(repo_path)
-    # get tests from Makefile
-    tests = get_tests(repo_path)
-
-    if src_mk is None or cc_files is None or tests is None:
-        return False
-
-    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
-    # rocksdb_lib
-    TARGETS.add_library(
-        "rocksdb_lib",
-        src_mk["LIB_SOURCES"] +
-        src_mk["TOOL_LIB_SOURCES"])
-    # rocksdb_test_lib
-    TARGETS.add_library(
-        "rocksdb_test_lib",
-        src_mk.get("MOCK_LIB_SOURCES", []) +
-        src_mk.get("TEST_LIB_SOURCES", []) +
-        src_mk.get("EXP_LIB_SOURCES", []),
-        [":rocksdb_lib"])
-    # rocksdb_tools_lib
-    TARGETS.add_library(
-        "rocksdb_tools_lib",
-        src_mk.get("BENCH_LIB_SOURCES", []) +
-        ["util/testutil.cc"],
-        [":rocksdb_lib"])
-
-    # test for every test we found in the Makefile
-    for test in tests:
-        match_src = [src for src in cc_files if ("/%s.c" % test) in src]
-        if len(match_src) == 0:
-            print(ColorString.warning("Cannot find .cc file for %s" % test))
-            continue
-        elif len(match_src) > 1:
-            print(ColorString.warning("Found more than one .cc for %s" % test))
-            print(match_src)
-            continue
-
-        assert(len(match_src) == 1)
-        is_parallel = tests[test]
-        TARGETS.register_test(test, match_src[0], is_parallel)
-
-        if test in _EXPORTED_TEST_LIBS:
-            test_library = "%s_lib" % test
-            TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
-    TARGETS.flush_tests()
-
-    print(ColorString.info("Generated TARGETS Summary:"))
-    print(ColorString.info("- %d libs" % TARGETS.total_lib))
-    print(ColorString.info("- %d binarys" % TARGETS.total_bin))
-    print(ColorString.info("- %d tests" % TARGETS.total_test))
-    return True
-
-
-def get_rocksdb_path():
-    # rocksdb = {script_dir}/..
-    script_dir = os.path.dirname(sys.argv[0])
-    script_dir = os.path.abspath(script_dir)
-    rocksdb_path = os.path.abspath(
-        os.path.join(script_dir, "../"))
-
-    return rocksdb_path
-
-def exit_with_error(msg):
-    print(ColorString.error(msg))
-    sys.exit(1)
-
-
-def main():
-    # Generate TARGETS file for buck
-    ok = generate_targets(get_rocksdb_path())
-    if not ok:
-        exit_with_error("Failed to generate TARGETS files")
-
-if __name__ == "__main__":
-    main()

buckifier/rocks_test_runner.sh

@@ -1,4 +0,0 @@
-#!/bin/bash
-# Create a tmp directory for the test to use
-TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
-TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"

buckifier/targets_builder.py

@@ -1,65 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-import targets_cfg
-import pprint
-
-# TODO(tec): replace this with PrettyPrinter
-def pretty_list(lst, indent=6):
-    if lst is None or len(lst) == 0:
-        return ""
-
-    if len(lst) == 1:
-        return "\"%s\"" % lst[0]
-
-    separator = "\",\n%s\"" % (" " * indent)
-    res = separator.join(lst)
-    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2))
-    return res
-
-
-class TARGETSBuilder:
-    def __init__(self, path):
-        self.path = path
-        self.targets_file = open(path, 'w')
-        self.targets_file.write(targets_cfg.rocksdb_target_header)
-        self.total_lib = 0
-        self.total_bin = 0
-        self.total_test = 0
-        self.tests_cfg = []
-
-    def __del__(self):
-        self.targets_file.close()
-
-    def add_library(self, name, srcs, deps=None, headers=None):
-        if headers is None:
-            headers = "AutoHeaders.RECURSIVE_GLOB"
-        self.targets_file.write(targets_cfg.library_template % (
-            name,
-            headers,
-            pretty_list(srcs),
-            pretty_list(deps)))
-        self.total_lib = self.total_lib + 1
-
-    def add_binary(self, name, srcs, deps=None):
-        self.targets_file.write(targets_cfg.binary_template % (
-            name,
-            pretty_list(srcs),
-            pretty_list(deps)))
-        self.total_bin = self.total_bin + 1
-
-    def register_test(self, test_name, src, is_parallel):
-        exec_mode = "serial"
-        if is_parallel:
-            exec_mode = "parallel"
-        self.tests_cfg.append([test_name, str(src), str(exec_mode)])
-
-        self.total_test = self.total_test + 1
-
-    def flush_tests(self):
-        self.targets_file.write(targets_cfg.unittests_template % (
-            pprint.PrettyPrinter().pformat(self.tests_cfg)
-        ))
-
-        self.tests_cfg = []

buckifier/targets_cfg.py

@@ -1,115 +0,0 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-rocksdb_target_header = """REPO_PATH = "internal_repo_rocksdb/repo/"
-BUCK_BINS = "buck-out/gen/" + REPO_PATH
-TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
-rocksdb_compiler_flags = [
-  "-msse",
-  "-msse4.2",
-  "-fno-builtin-memcmp",
-  "-DROCKSDB_PLATFORM_POSIX",
-  "-DROCKSDB_LIB_IO_POSIX",
-  "-DROCKSDB_FALLOCATE_PRESENT",
-  "-DROCKSDB_MALLOC_USABLE_SIZE",
-  "-DROCKSDB_RANGESYNC_PRESENT",
-  "-DROCKSDB_SCHED_GETCPU_PRESENT",
-  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
-  "-DOS_LINUX",
-  # Flags to enable libs we include
-  "-DSNAPPY",
-  "-DZLIB",
-  "-DBZIP2",
-  "-DLZ4",
-  "-DZSTD",
-  "-DGFLAGS=gflags",
-  "-DNUMA",
-  "-DTBB",
-  # Needed to compile in fbcode
-  "-Wno-expansion-to-defined",
-]
-
-rocksdb_external_deps = [
-  ('bzip2', None, 'bz2'),
-  ('snappy', None, "snappy"),
-  ('zlib', None, 'z'),
-  ('gflags', None, 'gflags'),
-  ('lz4', None, 'lz4'),
-  ('zstd', None),
-  ('tbb', None),
-  ("numa", "2.0.8", "numa"),
-  ("googletest", None, "gtest"),
-]
-
-rocksdb_preprocessor_flags = [
-  # Directories with files for #include
-  "-I" + REPO_PATH + "include/",
-  "-I" + REPO_PATH,
-]
-"""
-
-
-library_template = """
-cpp_library(
-    name = "%s",
-    headers = %s,
-    srcs = [%s],
-    deps = [%s],
-    preprocessor_flags = rocksdb_preprocessor_flags,
-    compiler_flags = rocksdb_compiler_flags,
-    external_deps = rocksdb_external_deps,
-)
-"""
-
-binary_template = """
-cpp_binary(
-  name = "%s",
-  srcs = [%s],
-  deps = [%s],
-  preprocessor_flags = rocksdb_preprocessor_flags,
-  compiler_flags = rocksdb_compiler_flags,
-  external_deps = rocksdb_external_deps,
-)
-"""
-
-unittests_template = """
-# [test_name, test_src, test_type]
-ROCKS_TESTS = %s
-
-
-# Generate a test rule for each entry in ROCKS_TESTS
-for test_cfg in ROCKS_TESTS:
-    test_name = test_cfg[0]
-    test_cc = test_cfg[1]
-    ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
-    test_bin = test_name + "_bin"
-
-    cpp_binary (
-      name = test_bin,
-      srcs = [test_cc],
-      deps = [":rocksdb_test_lib"],
-      preprocessor_flags = rocksdb_preprocessor_flags,
-      compiler_flags = rocksdb_compiler_flags,
-      external_deps = rocksdb_external_deps,
-    )
-
-    custom_unittest(
-      name = test_name,
-      type = ttype,
-      deps = [":" + test_bin],
-      command = [TEST_RUNNER, BUCK_BINS + test_bin]
-    )
-
-custom_unittest(
-    name = "make_rocksdbjavastatic",
-    type = "simple",
-    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
-)
-
-custom_unittest(
-    name = "make_rocksdb_lite_release",
-    type = "simple",
-    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
-)
-"""

buckifier/util.py

@@ -1,107 +0,0 @@
-"""
-This module keeps commonly used components.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-import subprocess
-import os
-import time
-
-class ColorString:
-    """ Generate colorful strings on terminal """
-    HEADER = '\033[95m'
-    BLUE = '\033[94m'
-    GREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-
-    @staticmethod
-    def _make_color_str(text, color):
-        return "".join([color, text.encode('utf-8'), ColorString.ENDC])
-
-    @staticmethod
-    def ok(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.GREEN)
-
-    @staticmethod
-    def info(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.BLUE)
-
-    @staticmethod
-    def header(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.HEADER)
-
-    @staticmethod
-    def error(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.FAIL)
-
-    @staticmethod
-    def warning(text):
-        if ColorString.is_disabled:
-            return text
-        return ColorString._make_color_str(text, ColorString.WARNING)
-
-    is_disabled = False
-
-
-def run_shell_command(shell_cmd, cmd_dir=None):
-    """ Run a single shell command.
-        @returns a tuple of shell command return code, stdout, stderr """
-
-    if cmd_dir is not None and not os.path.exists(cmd_dir):
-        run_shell_command("mkdir -p %s" % cmd_dir)
-
-    start = time.time()
-    print("\t>>> Running: " + shell_cmd)
-    p = subprocess.Popen(shell_cmd,
-                         shell=True,
-                         stdout=subprocess.PIPE,
-                         stderr=subprocess.PIPE,
-                         cwd=cmd_dir)
-    stdout, stderr = p.communicate()
-    end = time.time()
-
-    # Report time if we spent more than 5 minutes executing a command
-    execution_time = end - start
-    if execution_time > (60 * 5):
-        mins = (execution_time / 60)
-        secs = (execution_time % 60)
-        print("\t>time spent: %d minutes %d seconds" % (mins, secs))
-
-
-    return p.returncode, stdout, stderr
-
-
-def run_shell_commands(shell_cmds, cmd_dir=None, verbose=False):
-    """ Execute a sequence of shell commands, which is equivalent to
-        running `cmd1 && cmd2 && cmd3`
-        @returns boolean indication if all commands succeeds.
-    """
-
-    if cmd_dir:
-        print("\t=== Set current working directory => %s" % cmd_dir)
-
-    for shell_cmd in shell_cmds:
-        ret_code, stdout, stderr = run_shell_command(shell_cmd, cmd_dir)
-        if stdout:
-            if verbose or ret_code != 0:
-                print(ColorString.info("stdout: \n"), stdout)
-        if stderr:
-            # contents in stderr is not necessarily to be error messages.
-            if verbose or ret_code != 0:
-                print(ColorString.error("stderr: \n"), stderr)
-        if ret_code != 0:
-            return False
-
-    return True

build_tools/build_detect_platform

@@ -51,12 +51,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
     FBCODE_BUILD="true"
     # If we're compiling with TSAN we need pic build
     PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      source "$PWD/build_tools/fbcode_config.sh"
-    else
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    fi
+    source "$PWD/build_tools/fbcode_config.sh"
 fi
 
 # Delete existing output, if it exists
@@ -128,17 +123,10 @@ case "$TARGET_OS" in
         ;;
     SunOS)
         PLATFORM=OS_SOLARIS
-        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64"
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/sunos/sunos_specific.cc
         ;;
-    AIX)
-        PLATFORM=OS_AIX
-        CC=gcc
-        COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc"
-        # PORT_FILES=port/aix/aix_specific.cc
-        ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
@@ -389,29 +377,6 @@ EOF
     if [ "$?" = 0 ]; then
         PROFILING_FLAGS=-pg
     fi
-
-    # Test whether sync_file_range is supported for compatibility with an old glibc
-    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-      #include <fcntl.h>
-      int main() {
-        int fd = open("/dev/null", 0);
-        sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
-      }
-EOF
-    if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
-    fi
-
-    # Test whether sched_getcpu is supported
-    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-      #include <sched.h>
-      int main() {
-        int cpuid = sched_getcpu();
-      }
-EOF
-    if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
-    fi
 fi
 
 # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
@@ -457,7 +422,7 @@ elif test -z "$PORTABLE"; then
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
   elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
     COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
-  elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
+  else
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
   fi
 fi

build_tools/cont_integration.sh

@@ -8,7 +8,7 @@
 # for a certain interval.
 #
 
-SRC_GIT_REPO=/data/git/rocksdb-public
+SRC_GIT_REPO=/data/git/rocksdb
 error=0
 
 function log {

build_tools/dependencies.sh

@@ -1,18 +1,19 @@
-GCC_BASE=/mnt/gvfs/third-party2/gcc/2928bb3ed95bf64f5b388ee88c30dc74710c3b35/5.x/centos6-native/f4950a1
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/a5fea028cb7ba43498976e1f8054b0b2e790c295/stable/centos6-native/6aaf4de
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/7a9099f6587ee4378c0b1fa32bb8934019d30ca4/5.x/gcc-5-glibc-2.23/339d858
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/3b7c6469854dfc7832a1c3cc5b86919a84e5f865/2.23/gcc-5-glibc-2.23/ca1d1c0
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-5-glibc-2.23/9bc6787
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/d7861abe6f0e27ab98c9303b95a662f0e4cdedb5/1.2.8/gcc-5-glibc-2.23/9bc6787
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0815d59804160c96caac5f27ca004f51af893dc6/r131/gcc-5-glibc-2.23/9bc6787
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/c15a4f5f619a2930478d01e2e34dc1e0652b0873/1.1.4/gcc-5-glibc-2.23/03859b5
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f905a5e1032fb30c05db3d3752319857388c0c49/2.2.0/gcc-5-glibc-2.23/9bc6787
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/8d60633d822a2a55849c73db24e74a25e52b71db/master/gcc-5-glibc-2.23/1c32b4b
-NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/8db74270cd6d0212ac92d69e7fc7beefe617d772/trunk/gcc-5-glibc-2.23/b1847cb
-TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-5-glibc-2.23/9bc6787
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/90c9734afc5579c9d1db529fa788d09f97763b85/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/9e829389ef61b92c62de8748c80169aaf25ce1f0/2.26.1/centos6-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.11.0/gcc-5-glibc-2.23/9bc6787
-LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-5-glibc-2.23/65372bd
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
+NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832

build_tools/fbcode_config.sh

@@ -13,7 +13,7 @@ source "$BASEDIR/dependencies.sh"
 CFLAGS=""
 
 # libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include"
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
 LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
 
 # glibc
@@ -43,12 +43,16 @@ if test -z $PIC_BUILD; then
   LZ4_INCLUDE=" -I $LZ4_BASE/include/"
   LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
   CFLAGS+=" -DLZ4"
-
-  ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-  CFLAGS+=" -DZSTD"
 fi
 
+ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+if test -z $PIC_BUILD; then
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+else
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+fi
+CFLAGS+=" -DZSTD"
+
 # location of gflags headers and libraries
 GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
 if test -z $PIC_BUILD; then
@@ -104,8 +108,8 @@ if [ -z "$USE_CLANG" ]; then
   CXX="$GCC_BASE/bin/g++"
 
   CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
   JEMALLOC=1
 else
   # clang
@@ -116,8 +120,8 @@ else
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
   CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
   CFLAGS+=" -isystem $CLANG_INCLUDE"
@@ -128,14 +132,14 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"
 
 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
 EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-5-glibc-2.23/lib/ld.so"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-5-glibc-2.23/lib"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
 # required by libtbb
 EXEC_LDFLAGS+=" -ldl"
 
@@ -145,12 +149,4 @@ EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GF
 
 VALGRIND_VER="$VALGRIND_BASE/bin/"
 
-LUA_PATH="$LUA_BASE"
-
-if test -z $PIC_BUILD; then
-  LUA_LIB=" $LUA_PATH/lib/liblua.a"
-else
-  LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
-fi
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD 

build_tools/fbcode_config4.8.1.sh

@@ -93,7 +93,7 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
 CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
 CXXFLAGS+=" $CFLAGS"
 

build_tools/format-diff.sh

@@ -9,12 +9,9 @@ fi
 # Check clang-format-diff.py
 if ! which $CLANG_FORMAT_DIFF &> /dev/null
 then
-  echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
-  echo "You can download clang-format-diff.py by running: "
-  echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
-  echo "You can download clang-format by running: "
-  echo "    brew install clang-format"
-  echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+  echo "You didn't have clang-format-diff.py available in your computer!"
+  echo "You can download it by running: "
+  echo "    curl http://goo.gl/iUW1u2"
   exit 128
 fi
 
@@ -53,15 +50,14 @@ fi
 set -e
 
 uncommitted_code=`git diff HEAD`
-LAST_MASTER=`git merge-base master HEAD`
 
 # If there's no uncommitted changes, we assume user are doing post-commit
-# format check, in which case we'll check the modified lines since last commit
-# from master. Otherwise, we'll check format of the uncommitted code only.
+# format check, in which case we'll check the modified lines from latest commit.
+# Otherwise, we'll check format of the uncommitted code only.
 if [ -z "$uncommitted_code" ]
 then
   # Check the format of last commit
-  diffs=$(git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -p 1)
+  diffs=$(git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -p 1)
 else
   # Check the format of uncommitted lines,
   diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
@@ -101,12 +97,7 @@ then
 fi
 
 # Do in-place format adjustment.
-if [ -z "$uncommitted_code" ]
-then
-  git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -i -p 1
-else
-  git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
-fi
+git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
 echo "Files reformatted!"
 
 # Amend to last commit if user do the post-commit format check

build_tools/make_new_version.sh

@@ -17,9 +17,7 @@ function title() {
 }
 
 usage="Create new RocksDB version and prepare it for the release process\n"
-usage+="USAGE: ./make_new_version.sh <version> [<remote>]\n"
-usage+="  version: specify a version without '.fb' suffix (e.g. 5.4).\n"
-usage+="  remote: name of the remote to push the branch to (default: origin)."
+usage+="USAGE: ./make_new_version.sh <version>"
 
 # -- Pre-check
 if [[ $# < 1 ]]; then
@@ -29,11 +27,6 @@ fi
 
 ROCKSDB_VERSION=$1
 
-REMOTE="origin"
-if [[ $# > 1 ]]; then
-  REMOTE=$2
-fi
-
 GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
 echo $GIT_BRANCH
 
@@ -48,6 +41,6 @@ $GIT checkout -b $BRANCH
 
 # Setting up the proxy for remote repo access
 title "Pushing new branch to remote repo ..."
-git push $REMOTE --set-upstream $BRANCH
+git push origin --set-upstream $BRANCH
 
 title "Branch $BRANCH is pushed to github;"

build_tools/make_package.sh

@@ -99,18 +99,6 @@ function main() {
 
   make static_lib
   make install INSTALL_PATH=package
-
-  cd package
-
-  LIB_DIR=lib
-  if [[ -z "$ARCH" ]]; then
-      ARCH=$(getconf LONG_BIT)
-  fi
-  if [[ ("$FPM_OUTPUT" = "rpm") && ($ARCH -eq 64) ]]; then
-      mv lib lib64
-      LIB_DIR=lib64
-  fi
-
   fpm \
     -s dir \
     -t $FPM_OUTPUT \
@@ -122,7 +110,7 @@ function main() {
     --license BSD \
     --vendor Facebook \
     --description "RocksDB is an embeddable persistent key-value store for fast storage." \
-    include $LIB_DIR
+    package
 }
 
 main $@

build_tools/precommit_checker.py

@@ -1,4 +1,4 @@
-#!/usr/local/fbcode/gcc-4.9-glibc-2.20-fb/bin/python2.7
+#!/usr/local/fbcode/gcc-4.8.1-glibc-2.17-fb/bin/python2.7
 
 from __future__ import absolute_import
 from __future__ import division

build_tools/regression_build_test.sh

@@ -55,6 +55,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0  > ${STAT_FILE}.fillseq
 
@@ -72,6 +73,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=8 > ${STAT_FILE}.overwrite
@@ -90,6 +92,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=1 > /dev/null
@@ -108,6 +111,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandom
@@ -127,6 +131,7 @@ make release
     --use_tailing_iterator=1 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandomtailing
@@ -145,6 +150,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandomsmallblockcache
@@ -165,6 +171,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandom_mem_sst
@@ -184,6 +191,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=1 > /dev/null
@@ -202,6 +210,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > /dev/null
@@ -221,6 +230,7 @@ make release
     --disable_auto_compactions=1 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readrandom_filluniquerandom
@@ -241,6 +251,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=16 > ${STAT_FILE}.readwhilewriting
@@ -259,6 +270,7 @@ make release
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --value_size=10 \
@@ -283,6 +295,7 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --disable_wal=0 \
     --wal_dir=/dev/shm/rocksdb \
     --sync=0 \
+    --disable_data_sync=1 \
     --verify_checksum=1 \
     --delete_obsolete_files_period_micros=314572800 \
     --max_grandparent_overlap_factor=10 \
@@ -343,6 +356,7 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0  > ${STAT_FILE}.fillseq_lots_column_families
 
@@ -358,6 +372,7 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --open_files=55000 \
     --statistics=1 \
     --histogram=1 \
+    --disable_data_sync=1 \
     --disable_wal=1 \
     --sync=0 \
     --threads=8 > ${STAT_FILE}.overwrite_lots_column_families

build_tools/rocksdb-lego-determinator

@@ -81,7 +81,6 @@ PARALLELISM="$PARALLEL_J $PARALLEL_j"
 
 DEBUG="OPT=-g"
 SHM="TEST_TMPDIR=/dev/shm/rocksdb"
-NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
@@ -90,7 +89,7 @@ TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
-SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
+SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-7u10-64/; export PATH=\$PATH:\$JAVA_HOME/bin"
 PARSER="'parser':'python build_tools/error_filter.py $1'"
 
 CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
@@ -192,7 +191,7 @@ UNIT_TEST_NON_SHM_COMMANDS="[
             {
                 'name':'Build and test RocksDB debug version',
                 'timeout': 86400,
-                'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
+                'shell':'$DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },

build_tools/run_ci_db_test.ps1

@@ -1,50 +1,24 @@
 # This script enables you running RocksDB tests by running
-# All the tests concurrently and utilizing all the cores
+# All the tests in parallel and utilizing all the cores
+# For db_test the script first lists and parses the tests
+# and then fires them up in parallel using async PS Job functionality
+# Run the script from the enlistment
 Param(
-  [switch]$EnableJE = $false,  # Look for and use _je executable, append _je to listed exclusions
-  [switch]$RunAll = $false,    # Will attempt discover all *_test[_je].exe binaries and run all
-                               # of them as Google suites. I.e. It will run test cases concurrently
-                               # except those mentioned as $Run, those will run as individual test cases
-                               # And any execlued with $ExcludeExes or $ExcludeCases
-                               # It will also not run any individual test cases
-                               # excluded but $ExcludeCasese
-  [string]$SuiteRun = "",      # Split test suites in test cases and run in parallel, not compatible with $RunAll
-  [string]$Run = "",           # Run specified executables in parallel but do not split to test cases
-  [string]$ExcludeCases = "",  # Exclude test cases, expects a comma separated list, no spaces
-                               # Takes effect when $RunAll or $SuiteRun is specified. Must have full
-                               # Test cases name including a group and a parameter if any
-  [string]$ExcludeExes = "",   # Exclude exes from consideration, expects a comma separated list,
-                               # no spaces. Takes effect only when $RunAll is specified
-  [string]$WorkFolder = "",    # Direct tests to use that folder. SSD or Ram drive are better options.
+  [switch]$EnableJE = $false,  # Use je executable
+  [switch]$EnableRerun = $false, # Rerun failed tests sequentially at the end
+  [string]$WorkFolder = "",  # Direct tests to use that folder
+  [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
+  [string]$Exclude = "", # Expect a comma separated list, no spaces
+  [string]$Run = "db_test",  # Run db_test|db_test2|tests|testname1,testname2...
    # Number of async tasks that would run concurrently. Recommend a number below 64.
    # However, CPU utlization really depends on the storage media. Recommend ram based disk.
    # a value of 1 will run everything serially
-  [int]$Concurrency = 8,
-  [int]$Limit = -1 # -1 means do not limit for test purposes
+  [int]$Concurrency = 16
 )
 
 # Folders and commands must be fullpath to run assuming
 # the current folder is at the root of the git enlistment
-$StartDate = (Get-Date)
-$StartDate
-
-
-$DebugPreference = "Continue"
-
-# These tests are not google test suites and we should guard
-# Against running them as suites
-$RunOnly = New-Object System.Collections.Generic.HashSet[string]
-$RunOnly.Add("c_test") | Out-Null
-$RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
-$RunOnly.Add("merge_test") | Out-Null
-$RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
-$RunOnly.Add("backupable_db_test") | Out-Null # Disabled
-
-
-if($RunAll -and $SuiteRun -ne "") {
-    Write-Error "$RunAll and $SuiteRun are not compatible"
-    exit 1
-}
+Get-Date
 
 # If running under Appveyor assume that root
 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
@@ -73,32 +47,39 @@ if($WorkFolder -eq "") {
 }
 
 Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-Write-Output "BinariesFolder: $BinariesFolder, LogFolder: $LogFolder"
+
+# Use JEMALLOC executables
+if($Run -ceq "db_test" -or
+   $Run -ceq "db_test2" ) {
+
+   $file_name = $Run
+
+   if($EnableJE) {
+     $file_name += "_je"
+   }
+
+   $file_name += ".exe"
+
+   $db_test = -Join ($BinariesFolder, $file_name)
+
+   Write-Output "Binaries: $BinariesFolder db_test: $db_test"
+}
+
+
+#Exclusions that we do not want to run
+$ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
+
+
+if($Exclude -ne "") {
+    Write-Host "Exclude: $Exclude"
+    $l = $Exclude -split ' '
+    ForEach($t in $l) { $ExcludeTests.Add($t) | Out-Null }
+}
 
 # Create test directories in the current folder
 md -Path $WorkFolder -ErrorAction Ignore | Out-Null
 md -Path $LogFolder -ErrorAction Ignore | Out-Null
 
-
-$ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
-if($ExcludeCases -ne "") {
-    Write-Host "ExcludeCases: $ExcludeCases"
-    $l = $ExcludeCases -split ' '
-    ForEach($t in $l) { 
-      $ExcludeCasesSet.Add($t) | Out-Null
-    }
-}
-
-$ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
-if($ExcludeExes -ne "") {
-    Write-Host "ExcludeExe: $ExcludeExes"
-    $l = $ExcludeExes -split ' '
-    ForEach($t in $l) { 
-      $ExcludeExesSet.Add($t) | Out-Null
-    }
-}
-
-
 # Extract the names of its tests by running db_test with --gtest_list_tests.
 # This filter removes the "#"-introduced comments, and expands to
 # fully-qualified names by changing input like this:
@@ -116,37 +97,28 @@ if($ExcludeExes -ne "") {
 #   DBTest.WriteEmptyBatch
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
-#
 # Output into the parameter in a form TestName -> Log File Name
-function ExtractTestCases([string]$GTestExe, $HashTable) {
+function Normalize-DbTests($HashTable) {
 
     $Tests = @()
 # Run db_test to get a list of tests and store it into $a array
-    &$GTestExe --gtest_list_tests | tee -Variable Tests | Out-Null
+    &$db_test --gtest_list_tests | tee -Variable Tests | Out-Null
 
     # Current group
     $Group=""
 
     ForEach( $l in $Tests) {
-
-      # Leading whitespace is fine
-      $l = $l -replace '^\s+',''
-      # but no whitespace any other place
-      if($l -match "\s+") {
-        continue
-      }
-      # Trailing dot is a test group but no whitespace
-      elseif ( $l -match "\.$" ) {
+      # Trailing dot is a test group
+      if( $l -match "\.$") {
         $Group = $l
       }  else {
         # Otherwise it is a test name, remove leading space
-        $test = $l
+        $test = $l -replace '^\s+',''
         # remove trailing comment if any and create a log name
         $test = $test -replace '\s+\#.*',''
         $test = "$Group$test"
 
-        if($ExcludeCasesSet.Contains($test)) {
-            Write-Warning "$test case is excluded"
+        if($ExcludeTests.Contains($test)) {
             continue
         }
 
@@ -162,130 +134,58 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
 
 # The function removes trailing .exe siffix if any,
 # creates a name for the log file
-# Then adds the test name if it was not excluded into
-# a HashTable in a form of test_name -> log_path
 function MakeAndAdd([string]$token, $HashTable) {
-
     $test_name = $token -replace '.exe$', ''
     $log_name =  -join ($test_name, ".log")
     $log_path = -join ($LogFolder, $log_name)
-    $HashTable.Add($test_name, $log_path)
-}
-
-# This function takes a list of Suites to run
-# Lists all the test cases in each of the suite
-# and populates HashOfHashes
-# Ordered by suite(exe) @{ Exe = @{ TestCase = LogName }}
-function ProcessSuites($ListOfSuites, $HashOfHashes) {
-
-  $suite_list = $ListOfSuites
-  # Problem: if you run --gtest_list_tests on
-  # a non Google Test executable then it will start executing
-  # and we will get nowhere
-  ForEach($suite in $suite_list) {
-
-    if($RunOnly.Contains($suite)) {
-      Write-Warning "$suite is excluded from running as Google test suite"
-      continue
-    }
-
-    if($EnableJE) {
-      $suite += "_je"
-    }
-
-    $Cases = [ordered]@{}
-    $Cases.Clear()
-    $suite_exe = -Join ($BinariesFolder, $suite)
-    ExtractTestCases -GTestExe $suite_exe -HashTable $Cases
-    if($Cases.Count -gt 0) {
-      $HashOfHashes.Add($suite, $Cases);
-    }
-  }
-
-  # Make logs and run
-  if($CasesToRun.Count -lt 1) {
-     Write-Error "Failed to extract tests from $SuiteRun"
-     exit 1
-  }
-
-}
-
-# This will contain all test executables to run
-
-# Hash table that contains all non suite
-# Test executable to run
-$TestExes = [ordered]@{}
-
-# Check for test exe that are not
-# Google Test Suites
-# Since this is explicitely mentioned it is not subject
-# for exclusions
-if($Run -ne "") {
-
-  $test_list = $Run -split ' '
-
-  ForEach($t in $test_list) {
-
-    if($EnableJE) {
-      $t += "_je"
-    }
-
-    MakeAndAdd -token $t -HashTable $TestExes
-  }
-
-  if($TestExes.Count -lt 1) {
-     Write-Error "Failed to extract tests from $Run"
-     exit 1
-  }
-}
-
-# Ordered by exe @{ Exe = @{ TestCase = LogName }}
-$CasesToRun = [ordered]@{}
-
-if($SuiteRun -ne "") {
-  $suite_list = $SuiteRun -split ' '
-  ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
-}
-
-if($RunAll) {
-# Discover all the test binaries
-  if($EnableJE) {
-    $pattern = "*_test_je.exe"
-  } else {
-    $pattern = "*_test.exe"
-  }
-
-
-  $search_path = -join ($BinariesFolder, $pattern)
-  Write-Host "Binaries Search Path: $search_path"
-
-  $ListOfExe = @()
-  dir -Path $search_path | ForEach-Object {
-     $ListOfExe += ($_.Name)     
-  }
-
-  # Exclude those in RunOnly from running as suites
-  $ListOfSuites = @()
-  ForEach($e in $ListOfExe) {
-
-    $e = $e -replace '.exe$', ''
-    $bare_name = $e -replace '_je$', ''
-
-    if($ExcludeExesSet.Contains($bare_name)) {
-      Write-Warning "Test $e is excluded"
-      continue
-    }
-
-    if($RunOnly.Contains($bare_name)) {
-      MakeAndAdd -token $e -HashTable $TestExes
+    if(!$ExcludeTests.Contains($test_name)) {
+        $HashTable.Add($test_name, $log_path)
     } else {
-      $ListOfSuites += $bare_name
+        Write-Warning "Test $test_name is excluded"
     }
-  }
-
-  ProcessSuites -ListOfSuites $ListOfSuites -HashOfHashes $CasesToRun
 }
 
+# The function scans build\Debug folder to discover
+# Test executables. It then populates a table with
+# Test executable name -> Log file
+function Discover-TestBinaries([string]$Pattern, $HashTable) {
+
+    $Exclusions = @("db_test*", "db_sanity_test*")
+
+    $p = -join ($BinariesFolder, $pattern)
+
+    Write-Host "Path: $p"
+
+    dir -Path $p -Exclude $Exclusions | ForEach-Object {
+       MakeAndAdd -token ($_.Name) -HashTable $HashTable
+    }
+}
+
+$TestsToRun = [ordered]@{}
+
+if($Run -ceq "db_test" -or
+   $Run -ceq "db_test2") {
+    Normalize-DbTests -HashTable $TestsToRun
+} elseif($Run -ceq "tests") {
+    if($EnableJE) {
+        $pattern = "*_test_je.exe"
+    } else {
+        $pattern = "*_test.exe"
+    }
+    Discover-TestBinaries -Pattern $pattern -HashTable $TestsToRun
+} else {
+
+    $test_list = $Run -split ' '
+
+    ForEach($t in $test_list) {
+       MakeAndAdd -token $t -HashTable $TestsToRun
+    }
+}
+
+$NumTestsToStart = $TestsToRun.Count
+if($Limit -ge 0 -and $NumTestsToStart -gt $Limit) {
+    $NumTestsToStart = $Limit
+}
 
 Write-Host "Attempting to start: $NumTestsToStart tests"
 
@@ -303,13 +203,13 @@ $InvokeTestAsync = {
 
 # Hash that contains tests to rerun if any failed
 # Those tests will be rerun sequentially
-# $Rerun = [ordered]@{}
+$Rerun = [ordered]@{}
 # Test limiting factor here
-[int]$count = 0
+$count = 0
 # Overall status
 [bool]$success = $true;
 
-function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
+function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
 {
     # Array to wait for any of the running jobs
     $jobs = @()
@@ -318,66 +218,37 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 
     # Wait for all to finish and get the results
     while(($JobToLog.Count -gt 0) -or
-          ($TestCmds.Count -gt 0) -or 
-           ($Suites.Count -gt 0)) {
+          ($TestToLog.Count -gt 0)) {
 
         # Make sure we have maximum concurrent jobs running if anything
         # and the $Limit either not set or allows to proceed
         while(($JobToLog.Count -lt $ConcurrencyVal) -and
-              ((($TestCmds.Count -gt 0) -or ($Suites.Count -gt 0)) -and
+              (($TestToLog.Count -gt 0) -and
               (($Limit -lt 0) -or ($count -lt $Limit)))) {
 
-            # We always favore suites to run if available
-            [string]$exe_name = ""
-            [string]$log_path = ""
-            $Cases = @{}
 
-            if($Suites.Count -gt 0) {
-              # Will teh first one
-              ForEach($e in $Suites.Keys) {
-                $exe_name = $e
-                $Cases = $Suites[$e]
+            # We only need the first key
+            foreach($key in $TestToLog.keys) {
+                $k = $key
                 break
-              }
-              [string]$test_case = ""
-              [string]$log_path = ""
-              ForEach($c in $Cases.Keys) {
-                 $test_case = $c
-                 $log_path = $Cases[$c]
-                 break
-              }
-
-              Write-Host "Starting $exe_name::$test_case"
-              [string]$Exe =  -Join ($BinariesFolder, $exe_name)
-              $job = Start-Job -Name "$exe_name::$test_case" -ArgumentList @($Exe,$test_case,$log_path) -ScriptBlock $InvokeTestCase
-              $JobToLog.Add($job, $log_path)
-
-              $Cases.Remove($test_case)
-              if($Cases.Count -lt 1) {
-                $Suites.Remove($exe_name)
-              }
-
-            } elseif ($TestCmds.Count -gt 0) {
-
-               ForEach($e in $TestCmds.Keys) {
-                 $exe_name = $e
-                 $log_path = $TestCmds[$e]
-                 break
-               }
-
-              [string]$Exe =  -Join ($BinariesFolder, $exe_name)
-              $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
-              $JobToLog.Add($job, $log_path)
-
-              $TestCmds.Remove($exe_name)
-
-            } else {
-                Write-Error "In the job loop but nothing to run"
-                exit 1
             }
 
+            Write-Host "Starting $k"
+            $log_path = ($TestToLog.$k)
+
+            if($Run -ceq "db_test" -or
+               $Run -ceq "db_test2") {
+              $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
+            } else {
+              [string]$Exe =  -Join ($BinariesFolder, $k)
+               $job = Start-Job -Name $k -ScriptBlock $InvokeTestAsync -ArgumentList @($exe,$log_path)
+            }
+
+            $JobToLog.Add($job, $log_path)
+            $TestToLog.Remove($k)
+
             ++$count
-        } # End of Job starting loop
+        }
 
         if($JobToLog.Count -lt 1) {
           break
@@ -422,6 +293,9 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
                 $success = $false;
                 Write-Warning $message
                 $log_content | Write-Warning
+                if($AddForRerun) {
+                    $Rerun.Add($completed.Name, $log)
+                }
             } else {
                 Write-Host $message
             }
@@ -433,14 +307,16 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
     }
 }
 
-RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
+RunJobs -TestToLog $TestsToRun -ConcurrencyVal $Concurrency -AddForRerun $EnableRerun
 
-$EndDate = (Get-Date)
+if($Rerun.Count -gt 0) {
+    Write-Host "Rerunning " ($Rerun.Count) " tests sequentially"
+    $success = $true
+    $count = 0
+    RunJobs -TestToLog $Rerun -ConcurrencyVal 1 -AddForRerun $false
+}
 
-New-TimeSpan -Start $StartDate -End $EndDate | 
-  ForEach-Object { 
-    "Elapsed time: {0:g}" -f $_
-  }
+Get-Date
 
 
 if(!$success) {
@@ -448,9 +324,7 @@ if(!$success) {
 # So we simply exit
 #    Remove-Job -Job $jobs -Force
 # indicate failure using this exit code
-    exit 1
+    exit 12345
  }
 
- exit 0
-
  

build_tools/update_dependencies.sh

@@ -53,7 +53,7 @@ function get_lib_base()
 }
 
 ###########################################################
-#                   5.x dependencies                      #
+#                   4.9.x dependencies                    #
 ###########################################################
 
 OUTPUT="$BASEDIR/dependencies.sh"
@@ -64,16 +64,16 @@ touch "$OUTPUT"
 echo "Writing dependencies to $OUTPUT"
 
 # Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos6-native/*/`
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
 CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
 log_variable GCC_BASE
 log_variable CLANG_BASE
 
 # Libraries locations
-get_lib_base libgcc     5.x
-get_lib_base glibc      2.23
-get_lib_base snappy     LATEST gcc-5-glibc-2.23
+get_lib_base libgcc     4.9.x
+get_lib_base glibc      2.20
+get_lib_base snappy     LATEST gcc-4.9-glibc-2.20
 get_lib_base zlib       LATEST
 get_lib_base bzip2      LATEST
 get_lib_base lz4        LATEST
@@ -82,12 +82,12 @@ get_lib_base gflags     LATEST
 get_lib_base jemalloc   LATEST
 get_lib_base numa       LATEST
 get_lib_base libunwind  LATEST
-get_lib_base tbb        4.0_update2 gcc-5-glibc-2.23
+get_lib_base tbb        4.0_update2 gcc-4.9-glibc-2.20
 
 get_lib_base kernel-headers LATEST 
 get_lib_base binutils   LATEST centos6-native 
-get_lib_base valgrind   3.10.0 gcc-5-glibc-2.23
-get_lib_base lua        5.2.3 gcc-5-glibc-2.23
+get_lib_base valgrind   3.10.0 gcc-4.9-glibc-2.20
+get_lib_base lua        5.2.3 gcc-4.9-glibc-2.20
 
 git diff $OUTPUT
 

build_tools/version.sh

@@ -1,9 +1,8 @@
 #!/bin/sh
 if [ "$#" = "0" ]; then
-  echo "Usage: $0 major|minor|patch|full"
+  echo "Usage: $0 major|minor|patch"
   exit 1
 fi
-
 if [ "$1" = "major" ]; then
   cat include/rocksdb/version.h  | grep MAJOR | head -n1 | awk '{print $3}'
 fi
@@ -13,10 +12,3 @@ fi
 if [ "$1" = "patch" ]; then
   cat include/rocksdb/version.h  | grep PATCH | head -n1 | awk '{print $3}'
 fi
-if [ "$1" = "full" ]; then
-  awk '/#define ROCKSDB/ { env[$2] = $3 }
-       END { printf "%s.%s.%s\n", env["ROCKSDB_MAJOR"],
-                                  env["ROCKSDB_MINOR"],
-                                  env["ROCKSDB_PATCH"] }'  \
-      include/rocksdb/version.h
-fi

cmake/modules/FindJeMalloc.cmake

@@ -1,21 +1,22 @@
 # - Find JeMalloc library
 # Find the native JeMalloc includes and library
+# This module defines
+#  JEMALLOC_INCLUDE_DIRS, where to find jemalloc.h, Set when
+#                        JEMALLOC_INCLUDE_DIR is found.
+#  JEMALLOC_LIBRARIES, libraries to link against to use JeMalloc.
+#  JEMALLOC_FOUND, If false, do not try to use JeMalloc.
 #
-# JEMALLOC_INCLUDE_DIR - where to find jemalloc.h, etc.
-# JEMALLOC_LIBRARIES - List of libraries when using jemalloc.
-# JEMALLOC_FOUND - True if jemalloc found.
-
 find_path(JEMALLOC_INCLUDE_DIR
-  NAMES jemalloc/jemalloc.h
-  HINTS ${JEMALLOC_ROOT_DIR}/include)
+  jemalloc/jemalloc.h)
 
 find_library(JEMALLOC_LIBRARIES
-  NAMES jemalloc
-  HINTS ${JEMALLOC_ROOT_DIR}/lib)
+  jemalloc)
 
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(jemalloc DEFAULT_MSG JEMALLOC_LIBRARIES JEMALLOC_INCLUDE_DIR)
+find_package_handle_standard_args(JeMalloc DEFAULT_MSG
+    JEMALLOC_LIBRARIES JEMALLOC_INCLUDE_DIR)
 
-mark_as_advanced(
+MARK_AS_ADVANCED(
+  JEMALLOC_INCLUDE_DIR
   JEMALLOC_LIBRARIES
-  JEMALLOC_INCLUDE_DIR)
+)

cmake/modules/Findsnappy.cmake

@@ -13,6 +13,8 @@ find_library(SNAPPY_LIBRARIES
   NAMES snappy
   HINTS ${SNAPPY_ROOT_DIR}/lib)
 
+# handle the QUIETLY and REQUIRED arguments and set UUID_FOUND to TRUE if
+# all listed variables are TRUE
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(snappy DEFAULT_MSG SNAPPY_LIBRARIES SNAPPY_INCLUDE_DIR)
 

db/auto_roll_logger.cc

@@ -2,15 +2,12 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
-#include "util/auto_roll_logger.h"
+#include "db/auto_roll_logger.h"
 #include "util/mutexlock.h"
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
 // -- AutoRollLogger
 Status AutoRollLogger::ResetLogger() {
   TEST_SYNC_POINT("AutoRollLogger::ResetLogger:BeforeNewLogger");
@@ -133,7 +130,6 @@ bool AutoRollLogger::LogExpired() {
   ++cached_now_access_count;
   return cached_now >= ctime_ + kLogFileTimeToRoll;
 }
-#endif  // !ROCKSDB_LITE
 
 Status CreateLoggerFromOptions(const std::string& dbname,
                                const DBOptions& options,
@@ -151,7 +147,6 @@ Status CreateLoggerFromOptions(const std::string& dbname,
 
   env->CreateDirIfMissing(dbname);  // In case it does not exist
   // Currently we only support roll by time-to-roll and log size
-#ifndef ROCKSDB_LITE
   if (options.log_file_time_to_roll > 0 || options.max_log_file_size > 0) {
     AutoRollLogger* result = new AutoRollLogger(
         env, dbname, options.db_log_dir, options.max_log_file_size,
@@ -163,17 +158,17 @@ Status CreateLoggerFromOptions(const std::string& dbname,
       logger->reset(result);
     }
     return s;
+  } else {
+    // Open a log file in the same directory as the db
+    env->RenameFile(
+        fname, OldInfoLogFileName(dbname, env->NowMicros(), db_absolute_path,
+                                  options.db_log_dir));
+    auto s = env->NewLogger(fname, logger);
+    if (logger->get() != nullptr) {
+      (*logger)->SetInfoLogLevel(options.info_log_level);
+    }
+    return s;
   }
-#endif  // !ROCKSDB_LITE
-  // Open a log file in the same directory as the db
-  env->RenameFile(fname,
-                  OldInfoLogFileName(dbname, env->NowMicros(), db_absolute_path,
-                                     options.db_log_dir));
-  auto s = env->NewLogger(fname, logger);
-  if (logger->get() != nullptr) {
-    (*logger)->SetInfoLogLevel(options.info_log_level);
-  }
-  return s;
 }
 
 }  // namespace rocksdb

db/auto_roll_logger.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Logger implementation that can be shared by all environments
 // where enough posix functionality is available.
@@ -12,15 +10,14 @@
 #include <list>
 #include <string>
 
+#include "db/filename.h"
 #include "port/port.h"
 #include "port/util_logger.h"
-#include "util/filename.h"
-#include "util/mutexlock.h"
 #include "util/sync_point.h"
+#include "util/mutexlock.h"
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
 // Rolls the log file by size and/or time
 class AutoRollLogger : public Logger {
  public:
@@ -93,8 +90,6 @@ class AutoRollLogger : public Logger {
     return log_fname_;
   }
 
-  uint64_t TEST_ctime() const { return ctime_; }
-
  private:
   bool LogExpired();
   Status ResetLogger();
@@ -125,7 +120,6 @@ class AutoRollLogger : public Logger {
   uint64_t call_NowMicros_every_N_records_;
   mutable port::Mutex mutex_;
 };
-#endif  // !ROCKSDB_LITE
 
 // Facade to craete logger automatically
 Status CreateLoggerFromOptions(const std::string& dbname,

db/auto_roll_logger_test.cc

@@ -2,46 +2,24 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
-
-#ifndef ROCKSDB_LITE
-
-#include "util/auto_roll_logger.h"
-#include <errno.h>
-#include <sys/stat.h>
-#include <algorithm>
-#include <cmath>
-#include <fstream>
-#include <iostream>
-#include <iterator>
 #include <string>
 #include <thread>
 #include <vector>
+#include <cmath>
+#include <iostream>
+#include <fstream>
+#include <iterator>
+#include <algorithm>
+#include "db/auto_roll_logger.h"
 #include "port/port.h"
-#include "rocksdb/db.h"
-#include "util/logging.h"
 #include "util/sync_point.h"
 #include "util/testharness.h"
+#include "rocksdb/db.h"
+#include <sys/stat.h>
+#include <errno.h>
 
 namespace rocksdb {
-namespace {
-class NoSleepEnv : public EnvWrapper {
- public:
-  NoSleepEnv(Env* base) : EnvWrapper(base) {}
-  virtual void SleepForMicroseconds(int micros) override {
-    fake_time_ += static_cast<uint64_t>(micros);
-  }
-
-  virtual uint64_t NowNanos() override { return fake_time_ * 1000; }
-
-  virtual uint64_t NowMicros() override { return fake_time_; }
-
- private:
-  uint64_t fake_time_ = 6666666666;
-};
-}  // namespace
 
 class AutoRollLoggerTest : public testing::Test {
  public:
@@ -62,13 +40,13 @@ class AutoRollLoggerTest : public testing::Test {
 
   void RollLogFileBySizeTest(AutoRollLogger* logger, size_t log_max_size,
                              const std::string& log_message);
-  void RollLogFileByTimeTest(Env*, AutoRollLogger* logger, size_t time,
-                             const std::string& log_message);
+  uint64_t RollLogFileByTimeTest(AutoRollLogger* logger, size_t time,
+                                 const std::string& log_message);
 
   static const std::string kSampleMessage;
   static const std::string kTestDir;
   static const std::string kLogFile;
-  static Env* default_env;
+  static Env* env;
 };
 
 const std::string AutoRollLoggerTest::kSampleMessage(
@@ -76,15 +54,15 @@ const std::string AutoRollLoggerTest::kSampleMessage(
 const std::string AutoRollLoggerTest::kTestDir(test::TmpDir() + "/db_log_test");
 const std::string AutoRollLoggerTest::kLogFile(test::TmpDir() +
                                                "/db_log_test/LOG");
-Env* AutoRollLoggerTest::default_env = Env::Default();
+Env* AutoRollLoggerTest::env = Env::Default();
 
 // In this test we only want to Log some simple log message with
 // no format. LogMessage() provides such a simple interface and
 // avoids the [format-security] warning which occurs when you
-// call ROCKS_LOG_INFO(logger, log_message) directly.
+// call Log(logger, log_message) directly.
 namespace {
 void LogMessage(Logger* logger, const char* message) {
-  ROCKS_LOG_INFO(logger, "%s", message);
+  Log(logger, "%s", message);
 }
 
 void LogMessage(const InfoLogLevel log_level, Logger* logger,
@@ -93,6 +71,16 @@ void LogMessage(const InfoLogLevel log_level, Logger* logger,
 }
 }  // namespace
 
+namespace {
+void GetFileCreateTime(const std::string& fname, uint64_t* file_ctime) {
+  struct stat s;
+  if (stat(fname.c_str(), &s) != 0) {
+    *file_ctime = (uint64_t)0;
+  }
+  *file_ctime = static_cast<uint64_t>(s.st_ctime);
+}
+}  // namespace
+
 void AutoRollLoggerTest::RollLogFileBySizeTest(AutoRollLogger* logger,
                                                size_t log_max_size,
                                                const std::string& log_message) {
@@ -119,42 +107,45 @@ void AutoRollLoggerTest::RollLogFileBySizeTest(AutoRollLogger* logger,
   ASSERT_TRUE(message_size == logger->GetLogFileSize());
 }
 
-void AutoRollLoggerTest::RollLogFileByTimeTest(Env* env, AutoRollLogger* logger,
-                                               size_t time,
-                                               const std::string& log_message) {
-  uint64_t expected_ctime;
-  uint64_t actual_ctime;
-
+uint64_t AutoRollLoggerTest::RollLogFileByTimeTest(
+    AutoRollLogger* logger, size_t time, const std::string& log_message) {
+  uint64_t expected_create_time;
+  uint64_t actual_create_time;
   uint64_t total_log_size;
   EXPECT_OK(env->GetFileSize(kLogFile, &total_log_size));
-  expected_ctime = logger->TEST_ctime();
+  GetFileCreateTime(kLogFile, &expected_create_time);
   logger->SetCallNowMicrosEveryNRecords(0);
 
   // -- Write to the log for several times, which is supposed
   // to be finished before time.
   for (int i = 0; i < 10; ++i) {
-    env->SleepForMicroseconds(50000);
-    LogMessage(logger, log_message.c_str());
-    EXPECT_OK(logger->GetStatus());
-    // Make sure we always write to the same log file (by
-    // checking the create time);
+     LogMessage(logger, log_message.c_str());
+     EXPECT_OK(logger->GetStatus());
+     // Make sure we always write to the same log file (by
+     // checking the create time);
+     GetFileCreateTime(kLogFile, &actual_create_time);
 
-    actual_ctime = logger->TEST_ctime();
-
-    // Also make sure the log size is increasing.
-    EXPECT_EQ(expected_ctime, actual_ctime);
-    EXPECT_GT(logger->GetLogFileSize(), total_log_size);
-    total_log_size = logger->GetLogFileSize();
+     // Also make sure the log size is increasing.
+     EXPECT_EQ(expected_create_time, actual_create_time);
+     EXPECT_GT(logger->GetLogFileSize(), total_log_size);
+     total_log_size = logger->GetLogFileSize();
   }
 
   // -- Make the log file expire
-  env->SleepForMicroseconds(static_cast<int>(time * 1000000));
+#ifdef OS_WIN
+  Sleep(static_cast<unsigned int>(time) * 1000);
+#else
+  sleep(static_cast<unsigned int>(time));
+#endif
   LogMessage(logger, log_message.c_str());
 
   // At this time, the new log file should be created.
-  actual_ctime = logger->TEST_ctime();
-  EXPECT_LT(expected_ctime, actual_ctime);
+  GetFileCreateTime(kLogFile, &actual_create_time);
+  EXPECT_GT(actual_create_time, expected_create_time);
   EXPECT_LT(logger->GetLogFileSize(), total_log_size);
+  expected_create_time = actual_create_time;
+
+  return expected_create_time;
 }
 
 TEST_F(AutoRollLoggerTest, RollLogFileBySize) {
@@ -168,19 +159,16 @@ TEST_F(AutoRollLoggerTest, RollLogFileBySize) {
 }
 
 TEST_F(AutoRollLoggerTest, RollLogFileByTime) {
-  NoSleepEnv nse(Env::Default());
+    size_t time = 2;
+    size_t log_size = 1024 * 5;
 
-  size_t time = 2;
-  size_t log_size = 1024 * 5;
+    InitTestDb();
+    // -- Test the existence of file during the server restart.
+    ASSERT_EQ(Status::NotFound(), env->FileExists(kLogFile));
+    AutoRollLogger logger(Env::Default(), kTestDir, "", log_size, time);
+    ASSERT_OK(env->FileExists(kLogFile));
 
-  InitTestDb();
-  // -- Test the existence of file during the server restart.
-  ASSERT_EQ(Status::NotFound(), default_env->FileExists(kLogFile));
-  AutoRollLogger logger(&nse, kTestDir, "", log_size, time);
-  ASSERT_OK(default_env->FileExists(kLogFile));
-
-  RollLogFileByTimeTest(&nse, &logger, time,
-                        kSampleMessage + ":RollLogFileByTime");
+    RollLogFileByTimeTest(&logger, time, kSampleMessage + ":RollLogFileByTime");
 }
 
 TEST_F(AutoRollLoggerTest, OpenLogFilesMultipleTimesWithOptionLog_max_size) {
@@ -213,16 +201,16 @@ TEST_F(AutoRollLoggerTest, CompositeRollByTimeAndSizeLogger) {
 
   InitTestDb();
 
-  NoSleepEnv nse(Env::Default());
-  AutoRollLogger logger(&nse, kTestDir, "", log_max_size, time);
+  AutoRollLogger logger(Env::Default(), kTestDir, "", log_max_size, time);
 
   // Test the ability to roll by size
-  RollLogFileBySizeTest(&logger, log_max_size,
-                        kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
+  RollLogFileBySizeTest(
+      &logger, log_max_size,
+      kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
 
   // Test the ability to roll by Time
-  RollLogFileByTimeTest(&nse, &logger, time,
-                        kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
+  RollLogFileByTimeTest( &logger, time,
+      kSampleMessage + ":CompositeRollByTimeAndSizeLogger");
 }
 
 #ifndef OS_WIN
@@ -230,7 +218,6 @@ TEST_F(AutoRollLoggerTest, CompositeRollByTimeAndSizeLogger) {
 // port
 TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
   DBOptions options;
-  NoSleepEnv nse(Env::Default());
   shared_ptr<Logger> logger;
 
   // Normal logger
@@ -249,15 +236,15 @@ TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
       kSampleMessage + ":CreateLoggerFromOptions - size");
 
   // Only roll by Time
-  options.env = &nse;
   InitTestDb();
   options.max_log_file_size = 0;
   options.log_file_time_to_roll = 2;
   ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
   auto_roll_logger =
     dynamic_cast<AutoRollLogger*>(logger.get());
-  RollLogFileByTimeTest(&nse, auto_roll_logger, options.log_file_time_to_roll,
-                        kSampleMessage + ":CreateLoggerFromOptions - time");
+  RollLogFileByTimeTest(
+      auto_roll_logger, options.log_file_time_to_roll,
+      kSampleMessage + ":CreateLoggerFromOptions - time");
 
   // roll by both Time and size
   InitTestDb();
@@ -266,10 +253,12 @@ TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
   ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
   auto_roll_logger =
     dynamic_cast<AutoRollLogger*>(logger.get());
-  RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size,
-                        kSampleMessage + ":CreateLoggerFromOptions - both");
-  RollLogFileByTimeTest(&nse, auto_roll_logger, options.log_file_time_to_roll,
-                        kSampleMessage + ":CreateLoggerFromOptions - both");
+  RollLogFileBySizeTest(
+      auto_roll_logger, options.max_log_file_size,
+      kSampleMessage + ":CreateLoggerFromOptions - both");
+  RollLogFileByTimeTest(
+      auto_roll_logger, options.log_file_time_to_roll,
+      kSampleMessage + ":CreateLoggerFromOptions - both");
 }
 
 TEST_F(AutoRollLoggerTest, LogFlushWhileRolling) {
@@ -282,7 +271,7 @@ TEST_F(AutoRollLoggerTest, LogFlushWhileRolling) {
   AutoRollLogger* auto_roll_logger =
       dynamic_cast<AutoRollLogger*>(logger.get());
   ASSERT_TRUE(auto_roll_logger);
-  rocksdb::port::Thread flush_thread;
+  std::thread flush_thread;
 
   // Notes:
   // (1) Need to pin the old logger before beginning the roll, as rolling grabs
@@ -304,7 +293,7 @@ TEST_F(AutoRollLoggerTest, LogFlushWhileRolling) {
        {"AutoRollLogger::Flush:PinnedLogger", "PosixLogger::Flush:Begin2"}});
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  flush_thread = port::Thread ([&]() { auto_roll_logger->Flush(); });
+  flush_thread = std::thread([&]() { auto_roll_logger->Flush(); });
   TEST_SYNC_POINT(
       "AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit");
   RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size,
@@ -340,12 +329,12 @@ TEST_F(AutoRollLoggerTest, InfoLogLevel) {
       logger.SetInfoLogLevel((InfoLogLevel)log_level);
 
       // again, messages with level smaller than log_level will not be logged.
-      ROCKS_LOG_HEADER(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_DEBUG(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_INFO(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_WARN(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_ERROR(&logger, "%s", kSampleMessage.c_str());
-      ROCKS_LOG_FATAL(&logger, "%s", kSampleMessage.c_str());
+      Log(InfoLogLevel::HEADER_LEVEL, &logger, "%s", kSampleMessage.c_str());
+      Debug(&logger, "%s", kSampleMessage.c_str());
+      Info(&logger, "%s", kSampleMessage.c_str());
+      Warn(&logger, "%s", kSampleMessage.c_str());
+      Error(&logger, "%s", kSampleMessage.c_str());
+      Fatal(&logger, "%s", kSampleMessage.c_str());
       log_lines += InfoLogLevel::HEADER_LEVEL - log_level + 1;
     }
   }
@@ -419,7 +408,8 @@ TEST_F(AutoRollLoggerTest, LogHeaderTest) {
     } else if (test_num == 1) {
       // HEADER_LEVEL should make this behave like calling Header()
       for (size_t i = 0; i < MAX_HEADERS; i++) {
-        ROCKS_LOG_HEADER(&logger, "%s %d", HEADER_STR.c_str(), i);
+        Log(InfoLogLevel::HEADER_LEVEL, &logger, "%s %d",
+            HEADER_STR.c_str(), i);
       }
     }
 
@@ -469,7 +459,7 @@ TEST_F(AutoRollLoggerTest, LogFileExistence) {
   options.max_log_file_size = 100 * 1024 * 1024;
   options.create_if_missing = true;
   ASSERT_OK(rocksdb::DB::Open(options, kTestDir, &db));
-  ASSERT_OK(default_env->FileExists(kLogFile));
+  ASSERT_OK(env->FileExists(kLogFile));
   delete db;
 }
 
@@ -479,14 +469,3 @@ int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
-
-#else
-#include <stdio.h>
-
-int main(int argc, char** argv) {
-  fprintf(stderr,
-          "SKIPPED as AutoRollLogger is not supported in ROCKSDB_LITE\n");
-  return 0;
-}
-
-#endif  // !ROCKSDB_LITE

db/builder.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -18,12 +16,11 @@
 #include "db/compaction_iterator.h"
 #include "db/dbformat.h"
 #include "db/event_helpers.h"
+#include "db/filename.h"
 #include "db/internal_stats.h"
 #include "db/merge_helper.h"
 #include "db/table_cache.h"
 #include "db/version_edit.h"
-#include "monitoring/iostats_context_imp.h"
-#include "monitoring/thread_status_util.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
@@ -32,9 +29,9 @@
 #include "table/block_based_table_builder.h"
 #include "table/internal_iterator.h"
 #include "util/file_reader_writer.h"
-#include "util/filename.h"
+#include "util/iostats_context_imp.h"
 #include "util/stop_watch.h"
-#include "util/sync_point.h"
+#include "util/thread_status_util.h"
 
 namespace rocksdb {
 
@@ -106,10 +103,6 @@ Status BuildTable(
     unique_ptr<WritableFileWriter> file_writer;
     {
       unique_ptr<WritableFile> file;
-#ifndef NDEBUG
-      bool use_direct_writes = env_options.use_direct_writes;
-      TEST_SYNC_POINT_CALLBACK("BuildTable:create_file", &use_direct_writes);
-#endif  // !NDEBUG
       s = NewWritableFile(env, fname, &file, env_options);
       if (!s.ok()) {
         EventHelpers::LogAndNotifyTableFileCreationFinished(
@@ -119,8 +112,7 @@ Status BuildTable(
       }
       file->SetIOPriority(io_priority);
 
-      file_writer.reset(new WritableFileWriter(std::move(file), env_options,
-                                               ioptions.statistics));
+      file_writer.reset(new WritableFileWriter(std::move(file), env_options));
 
       builder = NewTableBuilder(
           ioptions, internal_comparator, int_tbl_prop_collector_factories,
@@ -130,6 +122,7 @@ Status BuildTable(
 
     MergeHelper merge(env, internal_comparator.user_comparator(),
                       ioptions.merge_operator, nullptr, ioptions.info_log,
+                      mutable_cf_options.min_partial_merge_operands,
                       true /* internal key corruption is not ok */,
                       snapshots.empty() ? 0 : snapshots.back());
 
@@ -177,9 +170,9 @@ Status BuildTable(
     delete builder;
 
     // Finish and check for file errors
-    if (s.ok() && !empty) {
+    if (s.ok() && !empty && !ioptions.disable_data_sync) {
       StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
-      s = file_writer->Sync(ioptions.use_fsync);
+      file_writer->Sync(ioptions.use_fsync);
     }
     if (s.ok() && !empty) {
       s = file_writer->Close();
@@ -187,11 +180,6 @@ Status BuildTable(
 
     if (s.ok() && !empty) {
       // Verify that the table is usable
-      // We set for_compaction to false and don't OptimizeForCompactionTableRead
-      // here because this is a special case after we finish the table building
-      // No matter whether use_direct_io_for_flush_and_compaction is true,
-      // we will regrad this verification as user reads since the goal is
-      // to cache it here for further user reads
       std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
           ReadOptions(), env_options, internal_comparator, meta->fd,
           nullptr /* range_del_agg */, nullptr,

db/builder.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -12,7 +10,6 @@
 #include <utility>
 #include <vector>
 #include "db/table_properties_collector.h"
-#include "options/cf_options.h"
 #include "rocksdb/comparator.h"
 #include "rocksdb/env.h"
 #include "rocksdb/listener.h"
@@ -21,6 +18,7 @@
 #include "rocksdb/table_properties.h"
 #include "rocksdb/types.h"
 #include "table/scoped_arena_iterator.h"
+#include "util/cf_options.h"
 #include "util/event_logger.h"
 
 namespace rocksdb {

db/c_test.c

@@ -20,7 +20,7 @@
 // Can not use port/port.h macros as this is a c file
 #ifdef OS_WIN
 
-#include <windows.h>
+#include <Windows.h>
 
 #define snprintf _snprintf
 
@@ -40,7 +40,6 @@ const char* phase = "";
 static char dbname[200];
 static char sstfilename[200];
 static char dbbackupname[200];
-static char dbcheckpointname[200];
 
 static void StartPhase(const char* name) {
   fprintf(stderr, "=== Test %s\n", name);
@@ -88,16 +87,6 @@ static void Free(char** ptr) {
   }
 }
 
-static void CheckValue(
-    char* err,
-    const char* expected,
-    char** actual,
-    size_t actual_length) {
-  CheckNoError(err);
-  CheckEqual(expected, *actual, actual_length);
-  Free(actual);
-}
-
 static void CheckGet(
     rocksdb_t* db,
     const rocksdb_readoptions_t* options,
@@ -127,32 +116,6 @@ static void CheckGetCF(
   Free(&val);
 }
 
-static void CheckPinGet(rocksdb_t* db, const rocksdb_readoptions_t* options,
-                        const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  const char* val;
-  rocksdb_pinnableslice_t* p;
-  p = rocksdb_get_pinned(db, options, key, strlen(key), &err);
-  CheckNoError(err);
-  val = rocksdb_pinnableslice_value(p, &val_len);
-  CheckEqual(expected, val, val_len);
-  rocksdb_pinnableslice_destroy(p);
-}
-
-static void CheckPinGetCF(rocksdb_t* db, const rocksdb_readoptions_t* options,
-                          rocksdb_column_family_handle_t* handle,
-                          const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  const char* val;
-  rocksdb_pinnableslice_t* p;
-  p = rocksdb_get_pinned_cf(db, options, handle, key, strlen(key), &err);
-  CheckNoError(err);
-  val = rocksdb_pinnableslice_value(p, &val_len);
-  CheckEqual(expected, val, val_len);
-  rocksdb_pinnableslice_destroy(p);
-}
 
 static void CheckIter(rocksdb_iterator_t* iter,
                       const char* key, const char* val) {
@@ -319,49 +282,16 @@ static char* MergeOperatorPartialMerge(
   return result;
 }
 
-static void CheckTxnGet(
-        rocksdb_transaction_t* txn,
-        const rocksdb_readoptions_t* options,
-        const char* key,
-        const char* expected) {
-        char* err = NULL;
-        size_t val_len;
-        char* val;
-        val = rocksdb_transaction_get(txn, options, key, strlen(key), &val_len, &err);
-        CheckNoError(err);
-        CheckEqual(expected, val, val_len);
-        Free(&val);
-}
-
-static void CheckTxnDBGet(
-        rocksdb_transactiondb_t* txn_db,
-        const rocksdb_readoptions_t* options,
-        const char* key,
-        const char* expected) {
-        char* err = NULL;
-        size_t val_len;
-        char* val;
-        val = rocksdb_transactiondb_get(txn_db, options, key, strlen(key), &val_len, &err);
-        CheckNoError(err);
-        CheckEqual(expected, val, val_len);
-        Free(&val);
-}
-
 int main(int argc, char** argv) {
   rocksdb_t* db;
   rocksdb_comparator_t* cmp;
   rocksdb_cache_t* cache;
   rocksdb_env_t* env;
   rocksdb_options_t* options;
-  rocksdb_compactoptions_t* coptions;
   rocksdb_block_based_table_options_t* table_options;
   rocksdb_readoptions_t* roptions;
   rocksdb_writeoptions_t* woptions;
   rocksdb_ratelimiter_t* rate_limiter;
-  rocksdb_transactiondb_t* txn_db;
-  rocksdb_transactiondb_options_t* txn_db_options;
-  rocksdb_transaction_t* txn;
-  rocksdb_transaction_options_t* txn_options;
   char* err = NULL;
   int run = -1;
 
@@ -375,11 +305,6 @@ int main(int argc, char** argv) {
            GetTempDir(),
            ((int) geteuid()));
 
-  snprintf(dbcheckpointname, sizeof(dbcheckpointname),
-           "%s/rocksdb_c_test-%d-checkpoint",
-           GetTempDir(),
-           ((int) geteuid()));
-
   snprintf(sstfilename, sizeof(sstfilename),
            "%s/rocksdb_c_test-%d-sst",
            GetTempDir(),
@@ -414,14 +339,11 @@ int main(int argc, char** argv) {
 
   roptions = rocksdb_readoptions_create();
   rocksdb_readoptions_set_verify_checksums(roptions, 1);
-  rocksdb_readoptions_set_fill_cache(roptions, 1);
+  rocksdb_readoptions_set_fill_cache(roptions, 0);
 
   woptions = rocksdb_writeoptions_create();
   rocksdb_writeoptions_set_sync(woptions, 1);
 
-  coptions = rocksdb_compactoptions_create();
-  rocksdb_compactoptions_set_exclusive_manual_compaction(coptions, 1);
-
   StartPhase("destroy");
   rocksdb_destroy_db(options, dbname, &err);
   Free(&err);
@@ -495,36 +417,6 @@ int main(int argc, char** argv) {
     rocksdb_backup_engine_close(be);
   }
 
-  StartPhase("checkpoint");
-  {
-    rocksdb_destroy_db(options, dbcheckpointname, &err);
-    CheckNoError(err);
-
-    rocksdb_checkpoint_t* checkpoint = rocksdb_checkpoint_object_create(db, &err);
-    CheckNoError(err);
-
-    rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
-    CheckNoError(err);
-
-    // start a new database from the checkpoint
-    rocksdb_close(db);
-    rocksdb_options_set_error_if_exists(options, 0);
-    db = rocksdb_open(options, dbcheckpointname, &err);
-    CheckNoError(err);
-
-    CheckGet(db, roptions, "foo", "hello");
-
-    rocksdb_checkpoint_object_destroy(checkpoint);
-
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbcheckpointname, &err);
-    CheckNoError(err);
-
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-    rocksdb_options_set_error_if_exists(options, 1);
-  }
-
   StartPhase("compactall");
   rocksdb_compact_range(db, NULL, 0, NULL, 0);
   CheckGet(db, roptions, "foo", "hello");
@@ -533,30 +425,158 @@ int main(int argc, char** argv) {
   rocksdb_compact_range(db, "a", 1, "z", 1);
   CheckGet(db, roptions, "foo", "hello");
 
-  StartPhase("compactallopt");
-  rocksdb_compact_range_opt(db, coptions, NULL, 0, NULL, 0);
-  CheckGet(db, roptions, "foo", "hello");
-
-  StartPhase("compactrangeopt");
-  rocksdb_compact_range_opt(db, coptions, "a", 1, "z", 1);
-  CheckGet(db, roptions, "foo", "hello");
-
-  // Simple check cache usage
-  StartPhase("cache_usage");
+  StartPhase("writebatch");
+  {
+    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
+    rocksdb_writebatch_put(wb, "foo", 3, "a", 1);
+    rocksdb_writebatch_clear(wb);
+    rocksdb_writebatch_put(wb, "bar", 3, "b", 1);
+    rocksdb_writebatch_put(wb, "box", 3, "c", 1);
+    rocksdb_writebatch_delete(wb, "bar", 3);
+    rocksdb_write(db, woptions, wb, &err);
+    CheckNoError(err);
+    CheckGet(db, roptions, "foo", "hello");
+    CheckGet(db, roptions, "bar", NULL);
+    CheckGet(db, roptions, "box", "c");
+    int pos = 0;
+    rocksdb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
+    CheckCondition(pos == 3);
+    rocksdb_writebatch_destroy(wb);
+  }
+
+  StartPhase("writebatch_vectors");
+  {
+    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
+    const char* k_list[2] = { "z", "ap" };
+    const size_t k_sizes[2] = { 1, 2 };
+    const char* v_list[3] = { "x", "y", "z" };
+    const size_t v_sizes[3] = { 1, 1, 1 };
+    rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
+    rocksdb_write(db, woptions, wb, &err);
+    CheckNoError(err);
+    CheckGet(db, roptions, "zap", "xyz");
+    rocksdb_writebatch_delete(wb, "zap", 3);
+    rocksdb_write(db, woptions, wb, &err);
+    CheckNoError(err);
+    CheckGet(db, roptions, "zap", NULL);
+    rocksdb_writebatch_destroy(wb);
+  }
+
+  StartPhase("writebatch_rep");
+  {
+    rocksdb_writebatch_t* wb1 = rocksdb_writebatch_create();
+    rocksdb_writebatch_put(wb1, "baz", 3, "d", 1);
+    rocksdb_writebatch_put(wb1, "quux", 4, "e", 1);
+    rocksdb_writebatch_delete(wb1, "quux", 4);
+    size_t repsize1 = 0;
+    const char* rep = rocksdb_writebatch_data(wb1, &repsize1);
+    rocksdb_writebatch_t* wb2 = rocksdb_writebatch_create_from(rep, repsize1);
+    CheckCondition(rocksdb_writebatch_count(wb1) ==
+                   rocksdb_writebatch_count(wb2));
+    size_t repsize2 = 0;
+    CheckCondition(
+        memcmp(rep, rocksdb_writebatch_data(wb2, &repsize2), repsize1) == 0);
+    rocksdb_writebatch_destroy(wb1);
+    rocksdb_writebatch_destroy(wb2);
+  }
+
+  StartPhase("iter");
   {
-    rocksdb_readoptions_set_pin_data(roptions, 1);
     rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
-    rocksdb_iter_seek(iter, "foo", 3);
-
-    size_t usage = rocksdb_cache_get_usage(cache);
-    CheckCondition(usage > 0);
-
-    size_t pin_usage = rocksdb_cache_get_pinned_usage(cache);
-    CheckCondition(pin_usage > 0);
-
+    CheckCondition(!rocksdb_iter_valid(iter));
+    rocksdb_iter_seek_to_first(iter);
+    CheckCondition(rocksdb_iter_valid(iter));
+    CheckIter(iter, "box", "c");
     rocksdb_iter_next(iter);
+    CheckIter(iter, "foo", "hello");
+    rocksdb_iter_prev(iter);
+    CheckIter(iter, "box", "c");
+    rocksdb_iter_prev(iter);
+    CheckCondition(!rocksdb_iter_valid(iter));
+    rocksdb_iter_seek_to_last(iter);
+    CheckIter(iter, "foo", "hello");
+    rocksdb_iter_seek(iter, "b", 1);
+    CheckIter(iter, "box", "c");
+    rocksdb_iter_seek_for_prev(iter, "g", 1);
+    CheckIter(iter, "foo", "hello");
+    rocksdb_iter_seek_for_prev(iter, "box", 3);
+    CheckIter(iter, "box", "c");
+    rocksdb_iter_get_error(iter, &err);
+    CheckNoError(err);
     rocksdb_iter_destroy(iter);
-    rocksdb_readoptions_set_pin_data(roptions, 0);
+  }
+
+  StartPhase("multiget");
+  {
+    const char* keys[3] = { "box", "foo", "notfound" };
+    const size_t keys_sizes[3] = { 3, 3, 8 };
+    char* vals[3];
+    size_t vals_sizes[3];
+    char* errs[3];
+    rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes, errs);
+
+    int i;
+    for (i = 0; i < 3; i++) {
+      CheckEqual(NULL, errs[i], 0);
+      switch (i) {
+      case 0:
+        CheckEqual("c", vals[i], vals_sizes[i]);
+        break;
+      case 1:
+        CheckEqual("hello", vals[i], vals_sizes[i]);
+        break;
+      case 2:
+        CheckEqual(NULL, vals[i], vals_sizes[i]);
+        break;
+      }
+      Free(&vals[i]);
+    }
+  }
+
+  StartPhase("approximate_sizes");
+  {
+    int i;
+    int n = 20000;
+    char keybuf[100];
+    char valbuf[100];
+    uint64_t sizes[2];
+    const char* start[2] = { "a", "k00000000000000010000" };
+    size_t start_len[2] = { 1, 21 };
+    const char* limit[2] = { "k00000000000000010000", "z" };
+    size_t limit_len[2] = { 21, 1 };
+    rocksdb_writeoptions_set_sync(woptions, 0);
+    for (i = 0; i < n; i++) {
+      snprintf(keybuf, sizeof(keybuf), "k%020d", i);
+      snprintf(valbuf, sizeof(valbuf), "v%020d", i);
+      rocksdb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
+                  &err);
+      CheckNoError(err);
+    }
+    rocksdb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
+    CheckCondition(sizes[0] > 0);
+    CheckCondition(sizes[1] > 0);
+  }
+
+  StartPhase("property");
+  {
+    char* prop = rocksdb_property_value(db, "nosuchprop");
+    CheckCondition(prop == NULL);
+    prop = rocksdb_property_value(db, "rocksdb.stats");
+    CheckCondition(prop != NULL);
+    Free(&prop);
+  }
+
+  StartPhase("snapshot");
+  {
+    const rocksdb_snapshot_t* snap;
+    snap = rocksdb_create_snapshot(db);
+    rocksdb_delete(db, woptions, "foo", 3, &err);
+    CheckNoError(err);
+    rocksdb_readoptions_set_snapshot(roptions, snap);
+    CheckGet(db, roptions, "foo", "hello");
+    rocksdb_readoptions_set_snapshot(roptions, NULL);
+    CheckGet(db, roptions, "foo", NULL);
+    rocksdb_release_snapshot(db, snap);
   }
 
   StartPhase("addfile");
@@ -610,330 +630,6 @@ int main(int argc, char** argv) {
     rocksdb_sstfilewriter_destroy(writer);
     rocksdb_options_destroy(io_options);
     rocksdb_envoptions_destroy(env_opt);
-
-    // Delete all keys we just ingested
-    rocksdb_delete(db, woptions, "sstk1", 5, &err);
-    CheckNoError(err);
-    rocksdb_delete(db, woptions, "sstk2", 5, &err);
-    CheckNoError(err);
-    rocksdb_delete(db, woptions, "sstk22", 6, &err);
-    CheckNoError(err);
-    rocksdb_delete(db, woptions, "sstk3", 5, &err);
-    CheckNoError(err);
-  }
-
-  StartPhase("writebatch");
-  {
-    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
-    rocksdb_writebatch_put(wb, "foo", 3, "a", 1);
-    rocksdb_writebatch_clear(wb);
-    rocksdb_writebatch_put(wb, "bar", 3, "b", 1);
-    rocksdb_writebatch_put(wb, "box", 3, "c", 1);
-    rocksdb_writebatch_delete(wb, "bar", 3);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "foo", "hello");
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "box", "c");
-    int pos = 0;
-    rocksdb_writebatch_iterate(wb, &pos, CheckPut, CheckDel);
-    CheckCondition(pos == 3);
-    rocksdb_writebatch_clear(wb);
-    rocksdb_writebatch_put(wb, "bar", 3, "b", 1);
-    rocksdb_writebatch_put(wb, "bay", 3, "d", 1);
-    rocksdb_writebatch_delete_range(wb, "bar", 3, "bay", 3);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "bay", "d");
-    rocksdb_writebatch_clear(wb);
-    const char* start_list[1] = {"bay"};
-    const size_t start_sizes[1] = {3};
-    const char* end_list[1] = {"baz"};
-    const size_t end_sizes[1] = {3};
-    rocksdb_writebatch_delete_rangev(wb, 1, start_list, start_sizes, end_list,
-                                     end_sizes);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "bay", NULL);
-    rocksdb_writebatch_destroy(wb);
-  }
-
-  StartPhase("writebatch_vectors");
-  {
-    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
-    const char* k_list[2] = { "z", "ap" };
-    const size_t k_sizes[2] = { 1, 2 };
-    const char* v_list[3] = { "x", "y", "z" };
-    const size_t v_sizes[3] = { 1, 1, 1 };
-    rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", "xyz");
-    rocksdb_writebatch_delete(wb, "zap", 3);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", NULL);
-    rocksdb_writebatch_destroy(wb);
-  }
-
-  StartPhase("writebatch_savepoint");
-  {
-    rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
-    rocksdb_writebatch_set_save_point(wb);
-    rocksdb_writebatch_set_save_point(wb);
-    const char* k_list[2] = {"z", "ap"};
-    const size_t k_sizes[2] = {1, 2};
-    const char* v_list[3] = {"x", "y", "z"};
-    const size_t v_sizes[3] = {1, 1, 1};
-    rocksdb_writebatch_pop_save_point(wb, &err);
-    CheckNoError(err);
-    rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
-    rocksdb_writebatch_rollback_to_save_point(wb, &err);
-    CheckNoError(err);
-    rocksdb_write(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", NULL);
-    rocksdb_writebatch_destroy(wb);
-  }
-
-  StartPhase("writebatch_rep");
-  {
-    rocksdb_writebatch_t* wb1 = rocksdb_writebatch_create();
-    rocksdb_writebatch_put(wb1, "baz", 3, "d", 1);
-    rocksdb_writebatch_put(wb1, "quux", 4, "e", 1);
-    rocksdb_writebatch_delete(wb1, "quux", 4);
-    size_t repsize1 = 0;
-    const char* rep = rocksdb_writebatch_data(wb1, &repsize1);
-    rocksdb_writebatch_t* wb2 = rocksdb_writebatch_create_from(rep, repsize1);
-    CheckCondition(rocksdb_writebatch_count(wb1) ==
-                   rocksdb_writebatch_count(wb2));
-    size_t repsize2 = 0;
-    CheckCondition(
-        memcmp(rep, rocksdb_writebatch_data(wb2, &repsize2), repsize1) == 0);
-    rocksdb_writebatch_destroy(wb1);
-    rocksdb_writebatch_destroy(wb2);
-  }
-
-  StartPhase("writebatch_wi");
-  {
-    rocksdb_writebatch_wi_t* wbi = rocksdb_writebatch_wi_create(0, 1);
-    rocksdb_writebatch_wi_put(wbi, "foo", 3, "a", 1);
-    rocksdb_writebatch_wi_clear(wbi);
-    rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1);
-    rocksdb_writebatch_wi_put(wbi, "box", 3, "c", 1);
-    rocksdb_writebatch_wi_delete(wbi, "bar", 3);
-    int count = rocksdb_writebatch_wi_count(wbi);
-    CheckCondition(count == 3);
-    size_t size;
-    char* value;
-    value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "box", 3, &size, &err);
-    CheckValue(err, "c", &value, size);
-    value = rocksdb_writebatch_wi_get_from_batch(wbi, options, "bar", 3, &size, &err);
-    CheckValue(err, NULL, &value, size);
-    value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "foo", 3, &size, &err);
-    CheckValue(err, "hello", &value, size);
-    value = rocksdb_writebatch_wi_get_from_batch_and_db(wbi, db, roptions, "box", 3, &size, &err);
-    CheckValue(err, "c", &value, size);
-    rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "foo", "hello");
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "box", "c");
-    int pos = 0;
-    rocksdb_writebatch_wi_iterate(wbi, &pos, CheckPut, CheckDel);
-    CheckCondition(pos == 3);
-    rocksdb_writebatch_wi_clear(wbi);
-    rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1);
-    rocksdb_writebatch_wi_put(wbi, "bay", 3, "d", 1);
-    rocksdb_writebatch_wi_delete_range(wbi, "bar", 3, "bay", 3);
-    rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "bar", NULL);
-    CheckGet(db, roptions, "bay", "d");
-    rocksdb_writebatch_wi_clear(wbi);
-    const char* start_list[1] = {"bay"};
-    const size_t start_sizes[1] = {3};
-    const char* end_list[1] = {"baz"};
-    const size_t end_sizes[1] = {3};
-    rocksdb_writebatch_wi_delete_rangev(wbi, 1, start_list, start_sizes, end_list,
-                                     end_sizes);
-    rocksdb_write_writebatch_wi(db, woptions, wbi, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "bay", NULL);
-    rocksdb_writebatch_wi_destroy(wbi);
-  }
-
-  StartPhase("writebatch_wi_vectors");
-  {
-    rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1);
-    const char* k_list[2] = { "z", "ap" };
-    const size_t k_sizes[2] = { 1, 2 };
-    const char* v_list[3] = { "x", "y", "z" };
-    const size_t v_sizes[3] = { 1, 1, 1 };
-    rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
-    rocksdb_write_writebatch_wi(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", "xyz");
-    rocksdb_writebatch_wi_delete(wb, "zap", 3);
-    rocksdb_write_writebatch_wi(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", NULL);
-    rocksdb_writebatch_wi_destroy(wb);
-  }
-
-  StartPhase("writebatch_wi_savepoint");
-  {
-    rocksdb_writebatch_wi_t* wb = rocksdb_writebatch_wi_create(0, 1);
-    rocksdb_writebatch_wi_set_save_point(wb);
-    const char* k_list[2] = {"z", "ap"};
-    const size_t k_sizes[2] = {1, 2};
-    const char* v_list[3] = {"x", "y", "z"};
-    const size_t v_sizes[3] = {1, 1, 1};
-    rocksdb_writebatch_wi_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
-    rocksdb_writebatch_wi_rollback_to_save_point(wb, &err);
-    CheckNoError(err);
-    rocksdb_write_writebatch_wi(db, woptions, wb, &err);
-    CheckNoError(err);
-    CheckGet(db, roptions, "zap", NULL);
-    rocksdb_writebatch_wi_destroy(wb);
-  }
-
-  StartPhase("iter");
-  {
-    rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_first(iter);
-    CheckCondition(rocksdb_iter_valid(iter));
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_next(iter);
-    CheckIter(iter, "foo", "hello");
-    rocksdb_iter_prev(iter);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_prev(iter);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_last(iter);
-    CheckIter(iter, "foo", "hello");
-    rocksdb_iter_seek(iter, "b", 1);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_seek_for_prev(iter, "g", 1);
-    CheckIter(iter, "foo", "hello");
-    rocksdb_iter_seek_for_prev(iter, "box", 3);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    rocksdb_iter_destroy(iter);
-  }
-
-  StartPhase("wbwi_iter");
-  {
-    rocksdb_iterator_t* base_iter = rocksdb_create_iterator(db, roptions);
-    rocksdb_writebatch_wi_t* wbi = rocksdb_writebatch_wi_create(0, 1);
-    rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1);
-    rocksdb_writebatch_wi_delete(wbi, "foo", 3);
-    rocksdb_iterator_t* iter = rocksdb_writebatch_wi_create_iterator_with_base(wbi, base_iter);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_first(iter);
-    CheckCondition(rocksdb_iter_valid(iter));
-    CheckIter(iter, "bar", "b");
-    rocksdb_iter_next(iter);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_prev(iter);
-    CheckIter(iter, "bar", "b");
-    rocksdb_iter_prev(iter);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_last(iter);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_seek(iter, "b", 1);
-    CheckIter(iter, "bar", "b");
-    rocksdb_iter_seek_for_prev(iter, "c", 1);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_seek_for_prev(iter, "box", 3);
-    CheckIter(iter, "box", "c");
-    rocksdb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    rocksdb_iter_destroy(iter);
-    rocksdb_writebatch_wi_destroy(wbi);
-  }
-
-  StartPhase("multiget");
-  {
-    const char* keys[3] = { "box", "foo", "notfound" };
-    const size_t keys_sizes[3] = { 3, 3, 8 };
-    char* vals[3];
-    size_t vals_sizes[3];
-    char* errs[3];
-    rocksdb_multi_get(db, roptions, 3, keys, keys_sizes, vals, vals_sizes, errs);
-
-    int i;
-    for (i = 0; i < 3; i++) {
-      CheckEqual(NULL, errs[i], 0);
-      switch (i) {
-      case 0:
-        CheckEqual("c", vals[i], vals_sizes[i]);
-        break;
-      case 1:
-        CheckEqual("hello", vals[i], vals_sizes[i]);
-        break;
-      case 2:
-        CheckEqual(NULL, vals[i], vals_sizes[i]);
-        break;
-      }
-      Free(&vals[i]);
-    }
-  }
-
-  StartPhase("pin_get");
-  {
-    CheckPinGet(db, roptions, "box", "c");
-    CheckPinGet(db, roptions, "foo", "hello");
-    CheckPinGet(db, roptions, "notfound", NULL);
-  }
-
-  StartPhase("approximate_sizes");
-  {
-    int i;
-    int n = 20000;
-    char keybuf[100];
-    char valbuf[100];
-    uint64_t sizes[2];
-    const char* start[2] = { "a", "k00000000000000010000" };
-    size_t start_len[2] = { 1, 21 };
-    const char* limit[2] = { "k00000000000000010000", "z" };
-    size_t limit_len[2] = { 21, 1 };
-    rocksdb_writeoptions_set_sync(woptions, 0);
-    for (i = 0; i < n; i++) {
-      snprintf(keybuf, sizeof(keybuf), "k%020d", i);
-      snprintf(valbuf, sizeof(valbuf), "v%020d", i);
-      rocksdb_put(db, woptions, keybuf, strlen(keybuf), valbuf, strlen(valbuf),
-                  &err);
-      CheckNoError(err);
-    }
-    rocksdb_approximate_sizes(db, 2, start, start_len, limit, limit_len, sizes);
-    CheckCondition(sizes[0] > 0);
-    CheckCondition(sizes[1] > 0);
-  }
-
-  StartPhase("property");
-  {
-    char* prop = rocksdb_property_value(db, "nosuchprop");
-    CheckCondition(prop == NULL);
-    prop = rocksdb_property_value(db, "rocksdb.stats");
-    CheckCondition(prop != NULL);
-    Free(&prop);
-  }
-
-  StartPhase("snapshot");
-  {
-    const rocksdb_snapshot_t* snap;
-    snap = rocksdb_create_snapshot(db);
-    rocksdb_delete(db, woptions, "foo", 3, &err);
-    CheckNoError(err);
-    rocksdb_readoptions_set_snapshot(roptions, snap);
-    CheckGet(db, roptions, "foo", "hello");
-    rocksdb_readoptions_set_snapshot(roptions, NULL);
-    CheckGet(db, roptions, "foo", NULL);
-    rocksdb_release_snapshot(db, snap);
   }
 
   StartPhase("repair");
@@ -1069,9 +765,9 @@ int main(int argc, char** argv) {
   {
     rocksdb_close(db);
     rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
+    CheckNoError(err)
 
-    rocksdb_options_t* db_options = rocksdb_options_create();
+        rocksdb_options_t* db_options = rocksdb_options_create();
     rocksdb_options_set_create_if_missing(db_options, 1);
     db = rocksdb_open(db_options, dbname, &err);
     CheckNoError(err)
@@ -1101,13 +797,11 @@ int main(int argc, char** argv) {
     CheckNoError(err);
 
     CheckGetCF(db, roptions, handles[1], "foo", "hello");
-    CheckPinGetCF(db, roptions, handles[1], "foo", "hello");
 
     rocksdb_delete_cf(db, woptions, handles[1], "foo", 3, &err);
     CheckNoError(err);
 
     CheckGetCF(db, roptions, handles[1], "foo", NULL);
-    CheckPinGetCF(db, roptions, handles[1], "foo", NULL);
 
     rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
     rocksdb_writebatch_put_cf(wb, handles[1], "baz", 3, "a", 1);
@@ -1120,9 +814,6 @@ int main(int argc, char** argv) {
     CheckGetCF(db, roptions, handles[1], "baz", NULL);
     CheckGetCF(db, roptions, handles[1], "bar", NULL);
     CheckGetCF(db, roptions, handles[1], "box", "c");
-    CheckPinGetCF(db, roptions, handles[1], "baz", NULL);
-    CheckPinGetCF(db, roptions, handles[1], "bar", NULL);
-    CheckPinGetCF(db, roptions, handles[1], "box", "c");
     rocksdb_writebatch_destroy(wb);
 
     const char* keys[3] = { "box", "box", "barfooxx" };
@@ -1240,19 +931,6 @@ int main(int argc, char** argv) {
     CheckNoError(err);
     rocksdb_iter_destroy(iter);
 
-    rocksdb_readoptions_set_total_order_seek(roptions, 1);
-    iter = rocksdb_create_iterator(db, roptions);
-    CheckCondition(!rocksdb_iter_valid(iter));
-
-    rocksdb_iter_seek(iter, "ba", 2);
-    rocksdb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    CheckCondition(rocksdb_iter_valid(iter));
-    CheckIter(iter, "bar1", "bar");
-
-    rocksdb_iter_destroy(iter);
-    rocksdb_readoptions_set_total_order_seek(roptions, 0);
-
     rocksdb_close(db);
     rocksdb_destroy_db(options, dbname, &err);
   }
@@ -1334,98 +1012,14 @@ int main(int argc, char** argv) {
     }
   }
 
-  StartPhase("transactions");
-  {
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
-
-    // open a TransactionDB
-    txn_db_options = rocksdb_transactiondb_options_create();
-    txn_options = rocksdb_transaction_options_create();
-    rocksdb_options_set_create_if_missing(options, 1);
-    txn_db = rocksdb_transactiondb_open(options, txn_db_options, dbname, &err);
-    CheckNoError(err);
-
-    // put outside a transaction
-    rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hello", 5, &err);
-    CheckNoError(err);
-    CheckTxnDBGet(txn_db, roptions, "foo", "hello");
-
-    // delete from outside transaction
-    rocksdb_transactiondb_delete(txn_db, woptions, "foo", 3, &err);
-    CheckNoError(err);
-    CheckTxnDBGet(txn_db, roptions, "foo", NULL);
-
-    // begin a transaction
-    txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL);
-    // put
-    rocksdb_transaction_put(txn, "foo", 3, "hello", 5, &err);
-    CheckNoError(err);
-    CheckTxnGet(txn, roptions, "foo", "hello");
-    // delete
-    rocksdb_transaction_delete(txn, "foo", 3, &err);
-    CheckNoError(err);
-    CheckTxnGet(txn, roptions, "foo", NULL);
-
-    rocksdb_transaction_put(txn, "foo", 3, "hello", 5, &err);
-    CheckNoError(err);
-
-    // read from outside transaction, before commit
-    CheckTxnDBGet(txn_db, roptions, "foo", NULL);
-
-    // commit
-    rocksdb_transaction_commit(txn, &err);
-    CheckNoError(err);
-
-    // read from outside transaction, after commit
-    CheckTxnDBGet(txn_db, roptions, "foo", "hello");
-
-    // reuse old transaction
-    txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, txn);
-
-    // snapshot
-    const rocksdb_snapshot_t* snapshot;
-    snapshot = rocksdb_transactiondb_create_snapshot(txn_db);
-    rocksdb_readoptions_set_snapshot(roptions, snapshot);
-  
-    rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3,  &err);
-    CheckNoError(err);
-
-    CheckTxnDBGet(txn_db, roptions, "foo", "hello");
-    rocksdb_readoptions_set_snapshot(roptions, NULL);
-    rocksdb_transactiondb_release_snapshot(txn_db, snapshot);
-    CheckTxnDBGet(txn_db, roptions, "foo", "hey");
-
-    // iterate
-    rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err);
-    rocksdb_iterator_t* iter = rocksdb_transaction_create_iterator(txn, roptions);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_first(iter);
-    CheckCondition(rocksdb_iter_valid(iter));
-    CheckIter(iter, "bar", "hi");
-    rocksdb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    rocksdb_iter_destroy(iter);
-
-    // rollback
-    rocksdb_transaction_rollback(txn, &err);
-    CheckNoError(err);
-    CheckTxnDBGet(txn_db, roptions, "bar", NULL);
-
-    // close and destroy
-    rocksdb_transaction_destroy(txn);
-    rocksdb_transactiondb_close(txn_db);
-    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
-    rocksdb_transaction_options_destroy(txn_options);
-    rocksdb_transactiondb_options_destroy(txn_db_options);
-  }
-
   // Simple sanity check that setting memtable rep works.
   StartPhase("memtable_reps");
   {
     // Create database with vector memtable.
+    rocksdb_close(db);
+    rocksdb_destroy_db(options, dbname, &err);
+    CheckNoError(err);
+
     rocksdb_options_set_memtable_vector_rep(options);
     db = rocksdb_open(options, dbname, &err);
     CheckNoError(err);
@@ -1446,7 +1040,6 @@ int main(int argc, char** argv) {
   rocksdb_block_based_options_destroy(table_options);
   rocksdb_readoptions_destroy(roptions);
   rocksdb_writeoptions_destroy(woptions);
-  rocksdb_compactoptions_destroy(coptions);
   rocksdb_cache_destroy(cache);
   rocksdb_comparator_destroy(cmp);
   rocksdb_env_destroy(env);

db/column_family.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -22,7 +20,6 @@
 #include <limits>
 
 #include "db/compaction_picker.h"
-#include "db/compaction_picker_universal.h"
 #include "db/db_impl.h"
 #include "db/internal_stats.h"
 #include "db/job_context.h"
@@ -30,11 +27,12 @@
 #include "db/version_set.h"
 #include "db/write_controller.h"
 #include "memtable/hash_skiplist_rep.h"
-#include "monitoring/thread_status_util.h"
-#include "options/options_helper.h"
 #include "table/block_based_table_factory.h"
 #include "util/autovector.h"
 #include "util/compression.h"
+#include "util/options_helper.h"
+#include "util/thread_status_util.h"
+#include "util/xfunc.h"
 
 namespace rocksdb {
 
@@ -172,12 +170,6 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
       result.num_levels < 2) {
     result.num_levels = 2;
   }
-
-  if (result.compaction_style == kCompactionStyleUniversal &&
-      db_options.allow_ingest_behind && result.num_levels < 3) {
-    result.num_levels = 3;
-  }
-
   if (result.max_write_buffer_number < 2) {
     result.max_write_buffer_number = 2;
   }
@@ -190,6 +182,14 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
   } else if (result.memtable_prefix_bloom_size_ratio < 0) {
     result.memtable_prefix_bloom_size_ratio = 0;
   }
+  XFUNC_TEST("memtablelist_history", "transaction_xftest_SanitizeOptions",
+             xf_transaction_set_memtable_history1,
+             xf_transaction_set_memtable_history,
+             &result.max_write_buffer_number_to_maintain);
+  XFUNC_TEST("memtablelist_history_clear", "transaction_xftest_SanitizeOptions",
+             xf_transaction_clear_memtable_history1,
+             xf_transaction_clear_memtable_history,
+             &result.max_write_buffer_number_to_maintain);
 
   if (!result.prefix_extractor) {
     assert(result.memtable_factory);
@@ -204,6 +204,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
     result.num_levels = 1;
     // since we delete level0 files in FIFO compaction when there are too many
     // of them, these options don't really mean anything
+    result.level0_file_num_compaction_trigger = std::numeric_limits<int>::max();
     result.level0_slowdown_writes_trigger = std::numeric_limits<int>::max();
     result.level0_stop_writes_trigger = std::numeric_limits<int>::max();
   }
@@ -213,8 +214,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
   }
 
   if (result.level0_file_num_compaction_trigger == 0) {
-    ROCKS_LOG_WARN(db_options.info_log.get(),
-                   "level0_file_num_compaction_trigger cannot be 0");
+    Warn(db_options.info_log.get(),
+         "level0_file_num_compaction_trigger cannot be 0");
     result.level0_file_num_compaction_trigger = 1;
   }
 
@@ -222,14 +223,14 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
           result.level0_slowdown_writes_trigger ||
       result.level0_slowdown_writes_trigger <
           result.level0_file_num_compaction_trigger) {
-    ROCKS_LOG_WARN(db_options.info_log.get(),
-                   "This condition must be satisfied: "
-                   "level0_stop_writes_trigger(%d) >= "
-                   "level0_slowdown_writes_trigger(%d) >= "
-                   "level0_file_num_compaction_trigger(%d)",
-                   result.level0_stop_writes_trigger,
-                   result.level0_slowdown_writes_trigger,
-                   result.level0_file_num_compaction_trigger);
+    Warn(db_options.info_log.get(),
+         "This condition must be satisfied: "
+         "level0_stop_writes_trigger(%d) >= "
+         "level0_slowdown_writes_trigger(%d) >= "
+         "level0_file_num_compaction_trigger(%d)",
+         result.level0_stop_writes_trigger,
+         result.level0_slowdown_writes_trigger,
+         result.level0_file_num_compaction_trigger);
     if (result.level0_slowdown_writes_trigger <
         result.level0_file_num_compaction_trigger) {
       result.level0_slowdown_writes_trigger =
@@ -239,14 +240,14 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
         result.level0_slowdown_writes_trigger) {
       result.level0_stop_writes_trigger = result.level0_slowdown_writes_trigger;
     }
-    ROCKS_LOG_WARN(db_options.info_log.get(),
-                   "Adjust the value to "
-                   "level0_stop_writes_trigger(%d)"
-                   "level0_slowdown_writes_trigger(%d)"
-                   "level0_file_num_compaction_trigger(%d)",
-                   result.level0_stop_writes_trigger,
-                   result.level0_slowdown_writes_trigger,
-                   result.level0_file_num_compaction_trigger);
+    Warn(db_options.info_log.get(),
+         "Adjust the value to "
+         "level0_stop_writes_trigger(%d)"
+         "level0_slowdown_writes_trigger(%d)"
+         "level0_file_num_compaction_trigger(%d)",
+         result.level0_stop_writes_trigger,
+         result.level0_slowdown_writes_trigger,
+         result.level0_file_num_compaction_trigger);
   }
 
   if (result.soft_pending_compaction_bytes_limit == 0) {
@@ -394,27 +395,27 @@ ColumnFamilyData::ColumnFamilyData(
     } else if (ioptions_.compaction_style == kCompactionStyleNone) {
       compaction_picker_.reset(new NullCompactionPicker(
           ioptions_, &internal_comparator_));
-      ROCKS_LOG_WARN(ioptions_.info_log,
-                     "Column family %s does not use any background compaction. "
-                     "Compactions can only be done via CompactFiles\n",
-                     GetName().c_str());
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "Column family %s does not use any background compaction. "
+          "Compactions can only be done via CompactFiles\n",
+          GetName().c_str());
 #endif  // !ROCKSDB_LITE
     } else {
-      ROCKS_LOG_ERROR(ioptions_.info_log,
-                      "Unable to recognize the specified compaction style %d. "
-                      "Column family %s will use kCompactionStyleLevel.\n",
-                      ioptions_.compaction_style, GetName().c_str());
+      Log(InfoLogLevel::ERROR_LEVEL, ioptions_.info_log,
+          "Unable to recognize the specified compaction style %d. "
+          "Column family %s will use kCompactionStyleLevel.\n",
+          ioptions_.compaction_style, GetName().c_str());
       compaction_picker_.reset(
           new LevelCompactionPicker(ioptions_, &internal_comparator_));
     }
 
     if (column_family_set_->NumberOfColumnFamilies() < 10) {
-      ROCKS_LOG_INFO(ioptions_.info_log,
-                     "--------------- Options for column family [%s]:\n",
-                     name.c_str());
+      Log(InfoLogLevel::INFO_LEVEL, ioptions_.info_log,
+          "--------------- Options for column family [%s]:\n", name.c_str());
       initial_cf_options_.Dump(ioptions_.info_log);
     } else {
-      ROCKS_LOG_INFO(ioptions_.info_log, "\t(skipping printing options)\n");
+      Log(InfoLogLevel::INFO_LEVEL, ioptions_.info_log,
+          "\t(skipping printing options)\n");
     }
   }
 
@@ -628,8 +629,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
     if (imm()->NumNotFlushed() >= mutable_cf_options.max_write_buffer_number) {
       write_controller_token_ = write_controller->GetStopToken();
       internal_stats_->AddCFStats(InternalStats::MEMTABLE_COMPACTION, 1);
-      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
           "[%s] Stopping writes because we have %d immutable memtables "
           "(waiting for flush), max_write_buffer_number is set to %d",
           name_.c_str(), imm()->NumNotFlushed(),
@@ -643,9 +643,9 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
         internal_stats_->AddCFStats(
             InternalStats::LEVEL0_NUM_FILES_WITH_COMPACTION, 1);
       }
-      ROCKS_LOG_WARN(ioptions_.info_log,
-                     "[%s] Stopping writes because we have %d level-0 files",
-                     name_.c_str(), vstorage->l0_delay_trigger_count());
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "[%s] Stopping writes because we have %d level-0 files",
+          name_.c_str(), vstorage->l0_delay_trigger_count());
     } else if (!mutable_cf_options.disable_auto_compactions &&
                mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
                compaction_needed_bytes >=
@@ -653,8 +653,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
       write_controller_token_ = write_controller->GetStopToken();
       internal_stats_->AddCFStats(
           InternalStats::HARD_PENDING_COMPACTION_BYTES_LIMIT, 1);
-      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
           "[%s] Stopping writes because of estimated pending compaction "
           "bytes %" PRIu64,
           name_.c_str(), compaction_needed_bytes);
@@ -666,8 +665,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
                      prev_compaction_needed_bytes_, was_stopped,
                      mutable_cf_options.disable_auto_compactions);
       internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1);
-      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
           "[%s] Stalling writes because we have %d immutable memtables "
           "(waiting for flush), max_write_buffer_number is set to %d "
           "rate %" PRIu64,
@@ -690,11 +688,11 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
         internal_stats_->AddCFStats(
             InternalStats::LEVEL0_SLOWDOWN_WITH_COMPACTION, 1);
       }
-      ROCKS_LOG_WARN(ioptions_.info_log,
-                     "[%s] Stalling writes because we have %d level-0 files "
-                     "rate %" PRIu64,
-                     name_.c_str(), vstorage->l0_delay_trigger_count(),
-                     write_controller->delayed_write_rate());
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "[%s] Stalling writes because we have %d level-0 files "
+          "rate %" PRIu64,
+          name_.c_str(), vstorage->l0_delay_trigger_count(),
+          write_controller->delayed_write_rate());
     } else if (!mutable_cf_options.disable_auto_compactions &&
                mutable_cf_options.soft_pending_compaction_bytes_limit > 0 &&
                vstorage->estimated_compaction_needed_bytes() >=
@@ -716,8 +714,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
                      mutable_cf_options.disable_auto_compactions);
       internal_stats_->AddCFStats(
           InternalStats::SOFT_PENDING_COMPACTION_BYTES_LIMIT, 1);
-      ROCKS_LOG_WARN(
-          ioptions_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
           "[%s] Stalling writes because of estimated pending compaction "
           "bytes %" PRIu64 " rate %" PRIu64,
           name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
@@ -729,8 +726,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
               mutable_cf_options.level0_slowdown_writes_trigger)) {
         write_controller_token_ =
             write_controller->GetCompactionPressureToken();
-        ROCKS_LOG_WARN(
-            ioptions_.info_log,
+        Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
             "[%s] Increasing compaction threads because we have %d level-0 "
             "files ",
             name_.c_str(), vstorage->l0_delay_trigger_count());
@@ -743,8 +739,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
         write_controller_token_ =
             write_controller->GetCompactionPressureToken();
         if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
-          ROCKS_LOG_WARN(
-              ioptions_.info_log,
+          Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
               "[%s] Increasing compaction threads because of estimated pending "
               "compaction "
               "bytes %" PRIu64,
@@ -784,6 +779,7 @@ uint64_t ColumnFamilyData::GetTotalSstFilesSize() const {
 
 MemTable* ColumnFamilyData::ConstructNewMemtable(
     const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
+  assert(current_ != nullptr);
   return new MemTable(internal_comparator_, ioptions_, mutable_cf_options,
                       write_buffer_manager_, earliest_seq);
 }

db/column_family.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -21,11 +19,11 @@
 #include "db/table_properties_collector.h"
 #include "db/write_batch_internal.h"
 #include "db/write_controller.h"
-#include "options/cf_options.h"
 #include "rocksdb/compaction_job_stats.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
+#include "util/cf_options.h"
 #include "util/thread_local.h"
 
 namespace rocksdb {

db/column_family_test.cc

@@ -15,13 +15,12 @@
 
 #include "db/db_impl.h"
 #include "db/db_test_util.h"
-#include "options/options_parser.h"
-#include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
 #include "util/coding.h"
 #include "util/fault_injection_test_env.h"
+#include "util/options_parser.h"
 #include "util/string_util.h"
 #include "util/sync_point.h"
 #include "util/testharness.h"
@@ -55,7 +54,7 @@ class EnvCounter : public EnvWrapper {
   }
 
  private:
-  std::atomic<int> num_new_writable_file_;
+  int num_new_writable_file_;
 };
 
 class ColumnFamilyTest : public testing::Test {
@@ -213,30 +212,6 @@ class ColumnFamilyTest : public testing::Test {
 #endif
   }
 
-  bool IsDbWriteStopped() {
-#ifndef ROCKSDB_LITE
-    uint64_t v;
-    EXPECT_TRUE(dbfull()->GetIntProperty("rocksdb.is-write-stopped", &v));
-    return (v == 1);
-#else
-    return dbfull()->TEST_write_controler().IsStopped();
-#endif  // !ROCKSDB_LITE
-  }
-
-  uint64_t GetDbDelayedWriteRate() {
-#ifndef ROCKSDB_LITE
-    uint64_t v;
-    EXPECT_TRUE(
-        dbfull()->GetIntProperty("rocksdb.actual-delayed-write-rate", &v));
-    return v;
-#else
-    if (!dbfull()->TEST_write_controler().NeedsDelay()) {
-      return 0;
-    }
-    return dbfull()->TEST_write_controler().delayed_write_rate();
-#endif  // !ROCKSDB_LITE
-  }
-
   void Destroy() {
     Close();
     ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_)));
@@ -676,44 +651,6 @@ TEST_F(ColumnFamilyTest, AddDrop) {
               std::vector<std::string>({"default", "four", "three"}));
 }
 
-TEST_F(ColumnFamilyTest, BulkAddDrop) {
-  constexpr int kNumCF = 1000;
-  ColumnFamilyOptions cf_options;
-  WriteOptions write_options;
-  Open();
-  std::vector<std::string> cf_names;
-  std::vector<ColumnFamilyHandle*> cf_handles;
-  for (int i = 1; i <= kNumCF; i++) {
-    cf_names.push_back("cf1-" + ToString(i));
-  }
-  ASSERT_OK(db_->CreateColumnFamilies(cf_options, cf_names, &cf_handles));
-  for (int i = 1; i <= kNumCF; i++) {
-    ASSERT_OK(db_->Put(write_options, cf_handles[i - 1], "foo", "bar"));
-  }
-  ASSERT_OK(db_->DropColumnFamilies(cf_handles));
-  std::vector<ColumnFamilyDescriptor> cf_descriptors;
-  for (auto* handle : cf_handles) {
-    delete handle;
-  }
-  cf_handles.clear();
-  for (int i = 1; i <= kNumCF; i++) {
-    cf_descriptors.emplace_back("cf2-" + ToString(i), ColumnFamilyOptions());
-  }
-  ASSERT_OK(db_->CreateColumnFamilies(cf_descriptors, &cf_handles));
-  for (int i = 1; i <= kNumCF; i++) {
-    ASSERT_OK(db_->Put(write_options, cf_handles[i - 1], "foo", "bar"));
-  }
-  ASSERT_OK(db_->DropColumnFamilies(cf_handles));
-  for (auto* handle : cf_handles) {
-    delete handle;
-  }
-  Close();
-  std::vector<std::string> families;
-  ASSERT_OK(DB::ListColumnFamilies(db_options_, dbname_, &families));
-  std::sort(families.begin(), families.end());
-  ASSERT_TRUE(families == std::vector<std::string>({"default"}));
-}
-
 TEST_F(ColumnFamilyTest, DropTest) {
   // first iteration - dont reopen DB before dropping
   // second iteration - reopen DB before dropping
@@ -1210,6 +1147,7 @@ TEST_F(ColumnFamilyTest, DifferentCompactionStyles) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1282,6 +1220,7 @@ TEST_F(ColumnFamilyTest, MultipleManualCompactions) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
@@ -1329,7 +1268,7 @@ TEST_F(ColumnFamilyTest, MultipleManualCompactions) {
       });
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  std::vector<port::Thread> threads;
+  std::vector<std::thread> threads;
   threads.emplace_back([&] {
     CompactRangeOptions compact_options;
     compact_options.exclusive_manual_compaction = false;
@@ -1379,6 +1318,7 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1436,7 +1376,7 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
     WaitForFlush(2);
     AssertFilesPerLevel(ToString(i + 1), 2);
   }
-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
     CompactRangeOptions compact_options;
     compact_options.exclusive_manual_compaction = false;
     ASSERT_OK(
@@ -1471,6 +1411,7 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
   CreateColumnFamilies({"one", "two"});
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1523,7 +1464,7 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
       });
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
     CompactRangeOptions compact_options;
     compact_options.exclusive_manual_compaction = false;
     ASSERT_OK(
@@ -1566,6 +1507,7 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1615,7 +1557,7 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
       });
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
     CompactRangeOptions compact_options;
     compact_options.exclusive_manual_compaction = true;
     ASSERT_OK(
@@ -1635,7 +1577,7 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
                         1);
   }
 
-  rocksdb::port::Thread threads1([&] {
+  std::thread threads1([&] {
     CompactRangeOptions compact_options;
     compact_options.exclusive_manual_compaction = false;
     ASSERT_OK(
@@ -1664,6 +1606,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1712,7 +1655,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
       });
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
     CompactRangeOptions compact_options;
     compact_options.exclusive_manual_compaction = false;
     ASSERT_OK(
@@ -1753,6 +1696,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1769,7 +1713,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
 
   one.num_levels = 1;
   // trigger compaction if there are >= 4 files
-  one.level0_file_num_compaction_trigger = 3;
+  one.level0_file_num_compaction_trigger = 4;
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
@@ -1803,7 +1747,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
       });
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
     CompactRangeOptions compact_options;
     compact_options.exclusive_manual_compaction = false;
     ASSERT_OK(
@@ -1849,6 +1793,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -1913,7 +1858,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
       });
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
     CompactRangeOptions compact_options;
     compact_options.exclusive_manual_compaction = false;
     ASSERT_OK(
@@ -1968,6 +1913,7 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
   CreateColumnFamilies({"one"});
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
   db_options_.max_background_compactions = 3;
   db_options_.base_background_compactions = 3;
 
@@ -2377,7 +2323,7 @@ TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) {
   int kKeysNum = 10000;
   PutRandomData(1, kKeysNum, 100);
 
-  std::vector<port::Thread> threads;
+  std::vector<std::thread> threads;
   threads.emplace_back([&] { ASSERT_OK(db_->DropColumnFamily(handles_[1])); });
 
   sleeping_task.WakeUp();
@@ -2478,7 +2424,7 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
 
   // Start a thread that will drop the first column family
   // and its comparator
-  rocksdb::port::Thread drop_cf_thread(DropSingleColumnFamily, this, 1, &comparators);
+  std::thread drop_cf_thread(DropSingleColumnFamily, this, 1, &comparators);
 
   DropColumnFamilies({2});
 
@@ -2515,165 +2461,173 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(50);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(201);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(400);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(500);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(450);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(205);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(202);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(201);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(198);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(399);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(599);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(2001);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(IsDbWriteStopped());
+  ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(3001);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(IsDbWriteStopped());
+  ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(390);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(100);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->set_l0_delay_trigger_count(100);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(101);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->set_l0_delay_trigger_count(0);
   vstorage->TEST_set_estimated_compaction_needed_bytes(300);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->set_l0_delay_trigger_count(101);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25 / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25 / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(200);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->set_l0_delay_trigger_count(0);
   vstorage->TEST_set_estimated_compaction_needed_bytes(0);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   mutable_cf_options.disable_auto_compactions = true;
   dbfull()->TEST_write_controler().set_delayed_write_rate(kBaseRate);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage->set_l0_delay_trigger_count(50);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(0, GetDbDelayedWriteRate());
   ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->set_l0_delay_trigger_count(60);
   vstorage->TEST_set_estimated_compaction_needed_bytes(300);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(0, GetDbDelayedWriteRate());
   ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   mutable_cf_options.disable_auto_compactions = false;
   vstorage->set_l0_delay_trigger_count(70);
   vstorage->TEST_set_estimated_compaction_needed_bytes(500);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->set_l0_delay_trigger_count(71);
   vstorage->TEST_set_estimated_compaction_needed_bytes(501);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 }
 
 TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
@@ -2765,55 +2719,60 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(50);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(201);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(600);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(70);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(800);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(300);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(700);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(500);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25 / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25 / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(600);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_TRUE(!IsDbWriteStopped());
+  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
   ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
-  ASSERT_EQ(kBaseRate / 1.25, GetDbDelayedWriteRate());
+  ASSERT_EQ(kBaseRate / 1.25,
+            dbfull()->TEST_write_controler().delayed_write_rate());
 }
 
 TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
@@ -3119,7 +3078,7 @@ TEST_F(ColumnFamilyTest, LogSyncConflictFlush) {
 
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  rocksdb::port::Thread thread([&] { db_->SyncWAL(); });
+  std::thread thread([&] { db_->SyncWAL(); });
 
   TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:1");
   Flush(1);

db/compact_files_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #ifndef ROCKSDB_LITE
 
@@ -13,7 +11,6 @@
 #include <vector>
 
 #include "db/db_impl.h"
-#include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "util/string_util.h"
@@ -234,7 +231,7 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
   // Start compacting files.
-  rocksdb::port::Thread compaction_thread(
+  std::thread compaction_thread(
       [&] { EXPECT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1)); });
 
   // In the meantime flush another file.
@@ -256,61 +253,6 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
   delete db;
 }
 
-TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
-  class FilterWithGet : public CompactionFilter {
-   public:
-    virtual bool Filter(int level, const Slice& key, const Slice& value,
-                        std::string* new_value,
-                        bool* value_changed) const override {
-      if (db_ == nullptr) {
-        return true;
-      }
-      std::string res;
-      db_->Get(ReadOptions(), "", &res);
-      return true;
-    }
-
-    void SetDB(DB* db) {
-      db_ = db;
-    }
-
-    virtual const char* Name() const override { return "FilterWithGet"; }
-
-   private:
-    DB* db_;
-  };
-
-
-  std::shared_ptr<FilterWithGet> cf(new FilterWithGet());
-
-  Options options;
-  options.create_if_missing = true;
-  options.compaction_filter = cf.get();
-
-  DB* db = nullptr;
-  DestroyDB(db_name_, options);
-  Status s = DB::Open(options, db_name_, &db);
-  ASSERT_OK(s);
-
-  cf->SetDB(db);
-
-  // Write one L0 file
-  db->Put(WriteOptions(), "K1", "V1");
-  db->Flush(FlushOptions());
-
-  // Compact all L0 files using CompactFiles
-  rocksdb::ColumnFamilyMetaData meta;
-  db->GetColumnFamilyMetaData(&meta);
-  for (auto& file : meta.levels[0].files) {
-    std::string fname = file.db_path + "/" + file.name;
-    ASSERT_OK(
-        db->CompactFiles(rocksdb::CompactionOptions(), {fname}, 0));
-  }
-
-
-  delete db;
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compacted_db_impl.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #ifndef ROCKSDB_LITE
 #include "db/compacted_db_impl.h"
@@ -44,8 +42,8 @@ size_t CompactedDBImpl::FindFile(const Slice& key) {
   return right;
 }
 
-Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
-                            const Slice& key, PinnableSlice* value) {
+Status CompactedDBImpl::Get(const ReadOptions& options,
+     ColumnFamilyHandle*, const Slice& key, std::string* value) {
   GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
                          GetContext::kNotFound, key, value, nullptr, nullptr,
                          nullptr, nullptr);
@@ -77,14 +75,11 @@ std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
   int idx = 0;
   for (auto* r : reader_list) {
     if (r != nullptr) {
-      PinnableSlice pinnable_val;
-      std::string& value = (*values)[idx];
       GetContext get_context(user_comparator_, nullptr, nullptr, nullptr,
-                             GetContext::kNotFound, keys[idx], &pinnable_val,
+                             GetContext::kNotFound, keys[idx], &(*values)[idx],
                              nullptr, nullptr, nullptr, nullptr);
       LookupKey lkey(keys[idx], kMaxSequenceNumber);
       r->Get(options, lkey.internal_key(), &get_context);
-      value.assign(pinnable_val.data(), pinnable_val.size());
       if (get_context.State() == GetContext::kFound) {
         statuses[idx] = Status::OK();
       }
@@ -156,8 +151,8 @@ Status CompactedDBImpl::Open(const Options& options,
   std::unique_ptr<CompactedDBImpl> db(new CompactedDBImpl(db_options, dbname));
   Status s = db->Init(options);
   if (s.ok()) {
-    ROCKS_LOG_INFO(db->immutable_db_options_.info_log,
-                   "Opened the db as fully compacted mode");
+    Log(INFO_LEVEL, db->immutable_db_options_.info_log,
+        "Opened the db as fully compacted mode");
     LogFlush(db->immutable_db_options_.info_log);
     *dbptr = db.release();
   }

db/compacted_db_impl.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #pragma once
 #ifndef ROCKSDB_LITE
@@ -25,7 +23,7 @@ class CompactedDBImpl : public DBImpl {
   using DB::Get;
   virtual Status Get(const ReadOptions& options,
                      ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
+                     std::string* value) override;
   using DB::MultiGet;
   virtual std::vector<Status> MultiGet(
       const ReadOptions& options,
@@ -78,13 +76,6 @@ class CompactedDBImpl : public DBImpl {
                        ColumnFamilyHandle* column_family) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
-  using DB::IngestExternalFile;
-  virtual Status IngestExternalFile(
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& external_files,
-      const IngestExternalFileOptions& ingestion_options) override {
-    return Status::NotSupported("Not supported in compacted db mode.");
-  }
 
  private:
   friend class DB;

db/compaction.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -20,7 +18,7 @@
 
 #include "db/column_family.h"
 #include "rocksdb/compaction_filter.h"
-#include "util/string_util.h"
+#include "util/logging.h"
 #include "util/sync_point.h"
 
 namespace rocksdb {
@@ -152,8 +150,7 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
                        bool _manual_compaction, double _score,
                        bool _deletion_compaction,
                        CompactionReason _compaction_reason)
-    : input_vstorage_(vstorage),
-      start_level_(_inputs[0].level),
+    : start_level_(_inputs[0].level),
       output_level_(_output_level),
       max_output_file_size_(_target_file_size),
       max_compaction_bytes_(_max_compaction_bytes),
@@ -207,10 +204,11 @@ Compaction::~Compaction() {
 }
 
 bool Compaction::InputCompressionMatchesOutput() const {
-  int base_level = input_vstorage_->base_level();
-  bool matches = (GetCompressionType(immutable_cf_options_, input_vstorage_,
-                                     mutable_cf_options_, start_level_,
-                                     base_level) == output_compression_);
+  VersionStorageInfo* vstorage = input_version_->storage_info();
+  int base_level = vstorage->base_level();
+  bool matches =
+      (GetCompressionType(*cfd_->ioptions(), vstorage, mutable_cf_options_,
+                          start_level_, base_level) == output_compression_);
   if (matches) {
     TEST_SYNC_POINT("Compaction::InputCompressionMatchesOutput:Matches");
     return true;
@@ -227,14 +225,15 @@ bool Compaction::IsTrivialMove() const {
   // filter to be applied to that level, and thus cannot be a trivial move.
 
   // Check if start level have files with overlapping ranges
-  if (start_level_ == 0 && input_vstorage_->level0_non_overlapping() == false) {
+  if (start_level_ == 0 &&
+      input_version_->storage_info()->level0_non_overlapping() == false) {
     // We cannot move files from L0 to L1 if the files are overlapping
     return false;
   }
 
   if (is_manual_compaction_ &&
-      (immutable_cf_options_.compaction_filter != nullptr ||
-       immutable_cf_options_.compaction_filter_factory != nullptr)) {
+      (cfd_->ioptions()->compaction_filter != nullptr ||
+       cfd_->ioptions()->compaction_filter_factory != nullptr)) {
     // This is a manual compaction and we have a compaction filter that should
     // be executed, we cannot do a trivial move
     return false;
@@ -242,34 +241,15 @@ bool Compaction::IsTrivialMove() const {
 
   // Used in universal compaction, where trivial move can be done if the
   // input files are non overlapping
-  if ((immutable_cf_options_.compaction_options_universal.allow_trivial_move) &&
+  if ((cfd_->ioptions()->compaction_options_universal.allow_trivial_move) &&
       (output_level_ != 0)) {
     return is_trivial_move_;
   }
 
-  if (!(start_level_ != output_level_ && num_input_levels() == 1 &&
+  return (start_level_ != output_level_ && num_input_levels() == 1 &&
           input(0, 0)->fd.GetPathId() == output_path_id() &&
-          InputCompressionMatchesOutput())) {
-    return false;
-  }
-
-  // assert inputs_.size() == 1
-
-  for (const auto& file : inputs_.front().files) {
-    std::vector<FileMetaData*> file_grand_parents;
-    if (output_level_ + 1 >= number_levels_) {
-      continue;
-    }
-    input_vstorage_->GetOverlappingInputs(output_level_ + 1, &file->smallest,
-                                          &file->largest, &file_grand_parents);
-    const auto compaction_size =
-        file->fd.GetFileSize() + TotalFileSize(file_grand_parents);
-    if (compaction_size > max_compaction_bytes_) {
-      return false;
-    }
-  }
-
-  return true;
+          InputCompressionMatchesOutput() &&
+          TotalFileSize(grandparents_) <= max_compaction_bytes_);
 }
 
 void Compaction::AddInputDeletions(VersionEdit* out_edit) {
@@ -292,7 +272,8 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
   // Maybe use binary search to find right entry instead of linear search?
   const Comparator* user_cmp = cfd_->user_comparator();
   for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
-    const std::vector<FileMetaData*>& files = input_vstorage_->LevelFiles(lvl);
+    const std::vector<FileMetaData*>& files =
+        input_version_->storage_info()->LevelFiles(lvl);
     for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
       auto* f = files[level_ptrs->at(lvl)];
       if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
@@ -364,7 +345,7 @@ void Compaction::ReleaseCompactionFiles(Status status) {
 
 void Compaction::ResetNextCompactionIndex() {
   assert(input_version_ != nullptr);
-  input_vstorage_->ResetNextCompactionIndex(start_level_);
+  input_version_->storage_info()->ResetNextCompactionIndex(start_level_);
 }
 
 namespace {
@@ -415,15 +396,14 @@ void Compaction::Summary(char* output, int len) {
 uint64_t Compaction::OutputFilePreallocationSize() const {
   uint64_t preallocation_size = 0;
 
-  if (max_output_file_size_ != port::kMaxUint64 &&
-      (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
-       output_level() > 0)) {
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
+      output_level() > 0) {
     preallocation_size = max_output_file_size_;
   } else {
-    for (const auto& level_files : inputs_) {
-      for (const auto& file : level_files.files) {
-        preallocation_size += file->fd.GetFileSize();
-      }
+    // output_level() == 0
+    assert(num_input_levels() > 0);
+    for (const auto& f : inputs_[0].files) {
+      preallocation_size += f->fd.GetFileSize();
     }
   }
   // Over-estimate slightly so we don't end up just barely crossing
@@ -453,7 +433,7 @@ bool Compaction::ShouldFormSubcompactions() const {
     return false;
   }
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
-    return start_level_ == 0 && output_level_ > 0 && !IsOutputLevelEmpty();
+    return start_level_ == 0 && !IsOutputLevelEmpty();
   } else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
     return number_levels_ > 1 && output_level_ > 0;
   } else {

db/compaction.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -11,9 +9,9 @@
 
 #pragma once
 #include "db/version_set.h"
-#include "options/cf_options.h"
 #include "util/arena.h"
 #include "util/autovector.h"
+#include "util/cf_options.h"
 
 namespace rocksdb {
 
@@ -260,8 +258,6 @@ class Compaction {
   static bool IsFullCompaction(VersionStorageInfo* vstorage,
                                const std::vector<CompactionInputFiles>& inputs);
 
-  VersionStorageInfo* input_vstorage_;
-
   const int start_level_;    // the lowest level to be compacted
   const int output_level_;  // levels to which output files are stored
   uint64_t max_output_file_size_;

db/compaction_iteration_stats.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #pragma once
 

db/compaction_iterator.cc

@@ -4,53 +4,26 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #include "db/compaction_iterator.h"
-#include "rocksdb/listener.h"
 #include "table/internal_iterator.h"
 
 namespace rocksdb {
 
-#ifndef ROCKSDB_LITE
-CompactionEventListener::CompactionListenerValueType fromInternalValueType(
-    ValueType vt) {
-  switch (vt) {
-    case kTypeDeletion:
-      return CompactionEventListener::CompactionListenerValueType::kDelete;
-    case kTypeValue:
-      return CompactionEventListener::CompactionListenerValueType::kValue;
-    case kTypeMerge:
-      return CompactionEventListener::CompactionListenerValueType::
-          kMergeOperand;
-    case kTypeSingleDeletion:
-      return CompactionEventListener::CompactionListenerValueType::
-          kSingleDelete;
-    case kTypeRangeDeletion:
-      return CompactionEventListener::CompactionListenerValueType::kRangeDelete;
-    default:
-      assert(false);
-      return CompactionEventListener::CompactionListenerValueType::kInvalid;
-  }
-}
-#endif  // ROCKSDB_LITE
-
 CompactionIterator::CompactionIterator(
     InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
     SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
     SequenceNumber earliest_write_conflict_snapshot, Env* env,
     bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
     const Compaction* compaction, const CompactionFilter* compaction_filter,
-    CompactionEventListener* compaction_listener,
-    const std::atomic<bool>* shutting_down)
+    LogBuffer* log_buffer)
     : CompactionIterator(
           input, cmp, merge_helper, last_sequence, snapshots,
           earliest_write_conflict_snapshot, env, expect_valid_internal_key,
           range_del_agg,
           std::unique_ptr<CompactionProxy>(
               compaction ? new CompactionProxy(compaction) : nullptr),
-          compaction_filter, compaction_listener, shutting_down) {}
+          compaction_filter, log_buffer) {}
 
 CompactionIterator::CompactionIterator(
     InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
@@ -58,9 +31,7 @@ CompactionIterator::CompactionIterator(
     SequenceNumber earliest_write_conflict_snapshot, Env* env,
     bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
     std::unique_ptr<CompactionProxy> compaction,
-    const CompactionFilter* compaction_filter,
-    CompactionEventListener* compaction_listener,
-    const std::atomic<bool>* shutting_down)
+    const CompactionFilter* compaction_filter, LogBuffer* log_buffer)
     : input_(input),
       cmp_(cmp),
       merge_helper_(merge_helper),
@@ -71,9 +42,7 @@ CompactionIterator::CompactionIterator(
       range_del_agg_(range_del_agg),
       compaction_(std::move(compaction)),
       compaction_filter_(compaction_filter),
-      compaction_listener_(compaction_listener),
-      shutting_down_(shutting_down),
-      ignore_snapshots_(false),
+      log_buffer_(log_buffer),
       merge_out_iter_(merge_helper_) {
   assert(compaction_filter_ == nullptr || compaction_ != nullptr);
   bottommost_level_ =
@@ -85,17 +54,15 @@ CompactionIterator::CompactionIterator(
   if (snapshots_->size() == 0) {
     // optimize for fast path if there are no snapshots
     visible_at_tip_ = true;
-    earliest_snapshot_ = kMaxSequenceNumber;
+    earliest_snapshot_ = last_sequence;
     latest_snapshot_ = 0;
   } else {
     visible_at_tip_ = false;
     earliest_snapshot_ = snapshots_->at(0);
     latest_snapshot_ = snapshots_->back();
   }
-  if (compaction_filter_ != nullptr) {
-    if (compaction_filter_->IgnoreSnapshots()) {
-      ignore_snapshots_ = true;
-    }
+  if (compaction_filter_ != nullptr && compaction_filter_->IgnoreSnapshots()) {
+    ignore_snapshots_ = true;
   } else {
     ignore_snapshots_ = false;
   }
@@ -137,7 +104,7 @@ void CompactionIterator::Next() {
       assert(valid_key);
       // Keep current_key_ in sync.
       current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-      key_ = current_key_.GetInternalKey();
+      key_ = current_key_.GetKey();
       ikey_.user_key = current_key_.GetUserKey();
       valid_ = true;
     } else {
@@ -169,7 +136,7 @@ void CompactionIterator::NextFromInput() {
   at_next_ = false;
   valid_ = false;
 
-  while (!valid_ && input_->Valid() && !IsShuttingDown()) {
+  while (!valid_ && input_->Valid()) {
     key_ = input_->key();
     value_ = input_->value();
     iter_stats_.num_input_records++;
@@ -183,7 +150,7 @@ void CompactionIterator::NextFromInput() {
         status_ = Status::Corruption("Corrupted internal key not expected.");
         break;
       }
-      key_ = current_key_.SetInternalKey(key_);
+      key_ = current_key_.SetKey(key_);
       has_current_user_key_ = false;
       current_user_key_sequence_ = kMaxSequenceNumber;
       current_user_key_snapshot_ = 0;
@@ -213,21 +180,13 @@ void CompactionIterator::NextFromInput() {
         !cmp_->Equal(ikey_.user_key, current_user_key_)) {
       // First occurrence of this user key
       // Copy key for output
-      key_ = current_key_.SetInternalKey(key_, &ikey_);
+      key_ = current_key_.SetKey(key_, &ikey_);
       current_user_key_ = ikey_.user_key;
       has_current_user_key_ = true;
       has_outputted_key_ = false;
       current_user_key_sequence_ = kMaxSequenceNumber;
       current_user_key_snapshot_ = 0;
 
-#ifndef ROCKSDB_LITE
-      if (compaction_listener_) {
-        compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
-                                           fromInternalValueType(ikey_.type),
-                                           value_, ikey_.sequence, true);
-      }
-#endif  // ROCKSDB_LITE
-
       // apply the compaction filter to the first occurrence of the user key
       if (compaction_filter_ != nullptr && ikey_.type == kTypeValue &&
           (visible_at_tip_ || ikey_.sequence > latest_snapshot_ ||
@@ -258,8 +217,7 @@ void CompactionIterator::NextFromInput() {
         }
 
         if (filter == CompactionFilter::Decision::kRemove) {
-          // convert the current key to a delete; key_ is pointing into
-          // current_key_ at this point, so updating current_key_ updates key()
+          // convert the current key to a delete
           ikey_.type = kTypeDeletion;
           current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
           // no value associated with delete
@@ -275,21 +233,13 @@ void CompactionIterator::NextFromInput() {
         }
       }
     } else {
-#ifndef ROCKSDB_LITE
-      if (compaction_listener_) {
-        compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
-                                           fromInternalValueType(ikey_.type),
-                                           value_, ikey_.sequence, false);
-      }
-#endif  // ROCKSDB_LITE
-
       // Update the current key to reflect the new sequence number/type without
       // copying the user key.
       // TODO(rven): Compaction filter does not process keys in this path
       // Need to have the compaction filter process multiple versions
       // if we have versions on both sides of a snapshot
       current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-      key_ = current_key_.GetInternalKey();
+      key_ = current_key_.GetKey();
       ikey_.user_key = current_key_.GetUserKey();
     }
 
@@ -442,7 +392,7 @@ void CompactionIterator::NextFromInput() {
       // is the same as the visibility of a previous instance of the
       // same key, then this kv is not visible in any snapshot.
       // Hidden by an newer entry for same user key
-      // TODO(noetzli): why not > ?
+      // TODO: why not > ?
       //
       // Note: Dropping this key will not affect TransactionDB write-conflict
       // checking since there has already been a record returned for this key
@@ -472,6 +422,7 @@ void CompactionIterator::NextFromInput() {
       input_->Next();
     } else if (ikey_.type == kTypeMerge) {
       if (!merge_helper_->HasOperator()) {
+        LogToBuffer(log_buffer_, "Options::merge_operator is null.");
         status_ = Status::InvalidArgument(
             "merge_operator is not properly initialized.");
         return;
@@ -482,14 +433,11 @@ void CompactionIterator::NextFromInput() {
       // have hit (A)
       // We encapsulate the merge related state machine in a different
       // object to minimize change to the existing flow.
-      Status s = merge_helper_->MergeUntil(input_, range_del_agg_,
-                                           prev_snapshot, bottommost_level_);
+      merge_helper_->MergeUntil(input_, range_del_agg_, prev_snapshot,
+                                bottommost_level_);
       merge_out_iter_.SeekToFirst();
 
-      if (!s.ok() && !s.IsMergeInProgress()) {
-        status_ = s;
-        return;
-      } else if (merge_out_iter_.Valid()) {
+      if (merge_out_iter_.Valid()) {
         // NOTE: key, value, and ikey_ refer to old entries.
         //       These will be correctly set below.
         key_ = merge_out_iter_.key();
@@ -501,7 +449,7 @@ void CompactionIterator::NextFromInput() {
         assert(valid_key);
         // Keep current_key_ in sync.
         current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-        key_ = current_key_.GetInternalKey();
+        key_ = current_key_.GetKey();
         ikey_.user_key = current_key_.GetUserKey();
         valid_ = true;
       } else {
@@ -518,8 +466,7 @@ void CompactionIterator::NextFromInput() {
     } else {
       // 1. new user key -OR-
       // 2. different snapshot stripe
-      bool should_delete = range_del_agg_->ShouldDelete(
-          key_, RangeDelAggregator::RangePositioningMode::kForwardTraversal);
+      bool should_delete = range_del_agg_->ShouldDelete(key_);
       if (should_delete) {
         ++iter_stats_.num_record_drop_hidden;
         ++iter_stats_.num_record_drop_range_del;
@@ -533,10 +480,6 @@ void CompactionIterator::NextFromInput() {
       input_->Seek(skip_until);
     }
   }
-
-  if (!valid_ && IsShuttingDown()) {
-    status_ = Status::ShutdownInProgress();
-  }
 }
 
 void CompactionIterator::PrepareOutput() {
@@ -548,8 +491,7 @@ void CompactionIterator::PrepareOutput() {
 
   // This is safe for TransactionDB write-conflict checking since transactions
   // only care about sequence number larger than any active snapshots.
-  if ((compaction_ != nullptr && !compaction_->allow_ingest_behind()) &&
-      bottommost_level_ && valid_ && ikey_.sequence <= earliest_snapshot_ &&
+  if (bottommost_level_ && valid_ && ikey_.sequence < earliest_snapshot_ &&
       ikey_.type != kTypeMerge &&
       !cmp_->Equal(compaction_->GetLargestUserKey(), ikey_.user_key)) {
     assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);

db/compaction_iterator.h

@@ -4,8 +4,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 #pragma once
 
 #include <algorithm>
@@ -18,13 +16,11 @@
 #include "db/merge_helper.h"
 #include "db/pinned_iterators_manager.h"
 #include "db/range_del_aggregator.h"
-#include "options/cf_options.h"
 #include "rocksdb/compaction_filter.h"
+#include "util/log_buffer.h"
 
 namespace rocksdb {
 
-class CompactionEventListener;
-
 class CompactionIterator {
  public:
   // A wrapper around Compaction. Has a much smaller interface, only what
@@ -49,9 +45,6 @@ class CompactionIterator {
     virtual Slice GetLargestUserKey() const {
       return compaction_->GetLargestUserKey();
     }
-    virtual bool allow_ingest_behind() const {
-      return compaction_->immutable_cf_options()->allow_ingest_behind;
-    }
 
    protected:
     CompactionProxy() = default;
@@ -68,8 +61,7 @@ class CompactionIterator {
                      RangeDelAggregator* range_del_agg,
                      const Compaction* compaction = nullptr,
                      const CompactionFilter* compaction_filter = nullptr,
-                     CompactionEventListener* compaction_listener = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr);
+                     LogBuffer* log_buffer = nullptr);
 
   // Constructor with custom CompactionProxy, used for tests.
   CompactionIterator(InternalIterator* input, const Comparator* cmp,
@@ -80,8 +72,7 @@ class CompactionIterator {
                      RangeDelAggregator* range_del_agg,
                      std::unique_ptr<CompactionProxy> compaction,
                      const CompactionFilter* compaction_filter = nullptr,
-                     CompactionEventListener* compaction_listener = nullptr,
-                     const std::atomic<bool>* shutting_down = nullptr);
+                     LogBuffer* log_buffer = nullptr);
 
   ~CompactionIterator();
 
@@ -134,8 +125,7 @@ class CompactionIterator {
   RangeDelAggregator* range_del_agg_;
   std::unique_ptr<CompactionProxy> compaction_;
   const CompactionFilter* compaction_filter_;
-  CompactionEventListener* compaction_listener_;
-  const std::atomic<bool>* shutting_down_;
+  LogBuffer* log_buffer_;
   bool bottommost_level_;
   bool valid_ = false;
   bool visible_at_tip_;
@@ -190,10 +180,5 @@ class CompactionIterator {
   // is in or beyond the last file checked during the previous call
   std::vector<size_t> level_ptrs_;
   CompactionIterationStats iter_stats_;
-
-  bool IsShuttingDown() {
-    // This is a best-effort facility, so memory_order_relaxed is sufficient.
-    return shutting_down_ && shutting_down_->load(std::memory_order_relaxed);
-  }
 };
 }  // namespace rocksdb

db/compaction_iterator_test.cc

@@ -2,78 +2,17 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #include "db/compaction_iterator.h"
 
 #include <string>
 #include <vector>
 
-#include "port/port.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
 
 namespace rocksdb {
 
-// Expects no merging attempts.
-class NoMergingMergeOp : public MergeOperator {
- public:
-  bool FullMergeV2(const MergeOperationInput& merge_in,
-                   MergeOperationOutput* merge_out) const override {
-    ADD_FAILURE();
-    return false;
-  }
-  bool PartialMergeMulti(const Slice& key,
-                         const std::deque<Slice>& operand_list,
-                         std::string* new_value,
-                         Logger* logger) const override {
-    ADD_FAILURE();
-    return false;
-  }
-  const char* Name() const override {
-    return "CompactionIteratorTest NoMergingMergeOp";
-  }
-};
-
-// Compaction filter that gets stuck when it sees a particular key,
-// then gets unstuck when told to.
-// Always returns Decition::kRemove.
-class StallingFilter : public CompactionFilter {
- public:
-  virtual Decision FilterV2(int level, const Slice& key, ValueType t,
-                            const Slice& existing_value, std::string* new_value,
-                            std::string* skip_until) const override {
-    int k = std::atoi(key.ToString().c_str());
-    last_seen.store(k);
-    while (k >= stall_at.load()) {
-      std::this_thread::yield();
-    }
-    return Decision::kRemove;
-  }
-
-  const char* Name() const override {
-    return "CompactionIteratorTest StallingFilter";
-  }
-
-  // Wait until the filter sees a key >= k and stalls at that key.
-  // If `exact`, asserts that the seen key is equal to k.
-  void WaitForStall(int k, bool exact = true) {
-    stall_at.store(k);
-    while (last_seen.load() < k) {
-      std::this_thread::yield();
-    }
-    if (exact) {
-      EXPECT_EQ(k, last_seen.load());
-    }
-  }
-
-  // Filter will stall on key >= stall_at. Advance stall_at to unstall.
-  mutable std::atomic<int> stall_at{0};
-  // Last key the filter was called with.
-  mutable std::atomic<int> last_seen{0};
-};
-
 class LoggingForwardVectorIterator : public InternalIterator {
  public:
   struct Action {
@@ -149,16 +88,13 @@ class FakeCompaction : public CompactionIterator::CompactionProxy {
   virtual int level(size_t compaction_input_level) const { return 0; }
   virtual bool KeyNotExistsBeyondOutputLevel(
       const Slice& user_key, std::vector<size_t>* level_ptrs) const {
-    return key_not_exists_beyond_output_level;
+    return false;
   }
   virtual bool bottommost_level() const { return false; }
   virtual int number_levels() const { return 1; }
   virtual Slice GetLargestUserKey() const {
     return "\xff\xff\xff\xff\xff\xff\xff\xff\xff";
   }
-  virtual bool allow_ingest_behind() const { return false; }
-
-  bool key_not_exists_beyond_output_level = false;
 };
 
 class CompactionIteratorTest : public testing::Test {
@@ -180,19 +116,17 @@ class CompactionIteratorTest : public testing::Test {
 
     std::unique_ptr<CompactionIterator::CompactionProxy> compaction;
     if (filter) {
-      compaction_proxy_ = new FakeCompaction();
-      compaction.reset(compaction_proxy_);
+      compaction.reset(new FakeCompaction());
     }
 
     merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, merge_op, filter,
-                                        nullptr, false, 0, 0, nullptr,
-                                        &shutting_down_));
+                                        nullptr, 0U, false, 0));
     iter_.reset(new LoggingForwardVectorIterator(ks, vs));
     iter_->SeekToFirst();
     c_iter_.reset(new CompactionIterator(
         iter_.get(), cmp_, merge_helper_.get(), last_sequence, &snapshots_,
         kMaxSequenceNumber, Env::Default(), false, range_del_agg_.get(),
-        std::move(compaction), filter, nullptr, &shutting_down_));
+        std::move(compaction), filter));
   }
 
   void AddSnapshot(SequenceNumber snapshot) { snapshots_.push_back(snapshot); }
@@ -204,8 +138,6 @@ class CompactionIteratorTest : public testing::Test {
   std::unique_ptr<LoggingForwardVectorIterator> iter_;
   std::unique_ptr<CompactionIterator> c_iter_;
   std::unique_ptr<RangeDelAggregator> range_del_agg_;
-  std::atomic<bool> shutting_down_{false};
-  FakeCompaction* compaction_proxy_;
 };
 
 // It is possible that the output of the compaction iterator is empty even if
@@ -277,6 +209,26 @@ TEST_F(CompactionIteratorTest, RangeDeletionWithSnapshots) {
 }
 
 TEST_F(CompactionIteratorTest, CompactionFilterSkipUntil) {
+  // Expect no merging attempts.
+  class MergeOp : public MergeOperator {
+   public:
+    bool FullMergeV2(const MergeOperationInput& merge_in,
+                     MergeOperationOutput* merge_out) const override {
+      ADD_FAILURE();
+      return false;
+    }
+    bool PartialMergeMulti(const Slice& key,
+                           const std::deque<Slice>& operand_list,
+                           std::string* new_value,
+                           Logger* logger) const override {
+      ADD_FAILURE();
+      return false;
+    }
+    const char* Name() const override {
+      return "CompactionIteratorTest.CompactionFilterSkipUntil::MergeOp";
+    }
+  };
+
   class Filter : public CompactionFilter {
     virtual Decision FilterV2(int level, const Slice& key, ValueType t,
                               const Slice& existing_value,
@@ -334,7 +286,7 @@ TEST_F(CompactionIteratorTest, CompactionFilterSkipUntil) {
     }
   };
 
-  NoMergingMergeOp merge_op;
+  MergeOp merge_op;
   Filter filter;
   InitIterators(
       {test::KeyStr("a", 50, kTypeValue),  // keep
@@ -386,77 +338,6 @@ TEST_F(CompactionIteratorTest, CompactionFilterSkipUntil) {
   ASSERT_EQ(expected_actions, iter_->log);
 }
 
-TEST_F(CompactionIteratorTest, ShuttingDownInFilter) {
-  NoMergingMergeOp merge_op;
-  StallingFilter filter;
-  InitIterators(
-      {test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeValue),
-       test::KeyStr("3", 3, kTypeValue), test::KeyStr("4", 4, kTypeValue)},
-      {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
-  // Don't leave tombstones (kTypeDeletion) for filtered keys.
-  compaction_proxy_->key_not_exists_beyond_output_level = true;
-
-  std::atomic<bool> seek_done{false};
-  rocksdb::port::Thread compaction_thread([&] {
-    c_iter_->SeekToFirst();
-    EXPECT_FALSE(c_iter_->Valid());
-    EXPECT_TRUE(c_iter_->status().IsShutdownInProgress());
-    seek_done.store(true);
-  });
-
-  // Let key 1 through.
-  filter.WaitForStall(1);
-
-  // Shutdown during compaction filter call for key 2.
-  filter.WaitForStall(2);
-  shutting_down_.store(true);
-  EXPECT_FALSE(seek_done.load());
-
-  // Unstall filter and wait for SeekToFirst() to return.
-  filter.stall_at.store(3);
-  compaction_thread.join();
-  assert(seek_done.load());
-
-  // Check that filter was never called again.
-  EXPECT_EQ(2, filter.last_seen.load());
-}
-
-// Same as ShuttingDownInFilter, but shutdown happens during filter call for
-// a merge operand, not for a value.
-TEST_F(CompactionIteratorTest, ShuttingDownInMerge) {
-  NoMergingMergeOp merge_op;
-  StallingFilter filter;
-  InitIterators(
-      {test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeMerge),
-       test::KeyStr("3", 3, kTypeMerge), test::KeyStr("4", 4, kTypeValue)},
-      {"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
-  compaction_proxy_->key_not_exists_beyond_output_level = true;
-
-  std::atomic<bool> seek_done{false};
-  rocksdb::port::Thread compaction_thread([&] {
-    c_iter_->SeekToFirst();
-    ASSERT_FALSE(c_iter_->Valid());
-    ASSERT_TRUE(c_iter_->status().IsShutdownInProgress());
-    seek_done.store(true);
-  });
-
-  // Let key 1 through.
-  filter.WaitForStall(1);
-
-  // Shutdown during compaction filter call for key 2.
-  filter.WaitForStall(2);
-  shutting_down_.store(true);
-  EXPECT_FALSE(seek_done.load());
-
-  // Unstall filter and wait for SeekToFirst() to return.
-  filter.stall_at.store(3);
-  compaction_thread.join();
-  assert(seek_done.load());
-
-  // Check that filter was never called again.
-  EXPECT_EQ(2, filter.last_seen.load());
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compaction_job.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -30,6 +28,7 @@
 #include "db/db_iter.h"
 #include "db/dbformat.h"
 #include "db/event_helpers.h"
+#include "db/filename.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
 #include "db/memtable.h"
@@ -37,9 +36,6 @@
 #include "db/merge_context.h"
 #include "db/merge_helper.h"
 #include "db/version_set.h"
-#include "monitoring/iostats_context_imp.h"
-#include "monitoring/perf_context_imp.h"
-#include "monitoring/thread_status_util.h"
 #include "port/likely.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
@@ -49,19 +45,20 @@
 #include "rocksdb/table.h"
 #include "table/block.h"
 #include "table/block_based_table_factory.h"
-#include "table/merging_iterator.h"
+#include "table/merger.h"
 #include "table/table_builder.h"
 #include "util/coding.h"
 #include "util/file_reader_writer.h"
-#include "util/filename.h"
+#include "util/iostats_context_imp.h"
 #include "util/log_buffer.h"
 #include "util/logging.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
 #include "util/sst_file_manager_impl.h"
+#include "util/mutexlock.h"
+#include "util/perf_context_imp.h"
 #include "util/stop_watch.h"
 #include "util/string_util.h"
 #include "util/sync_point.h"
+#include "util/thread_status_util.h"
 
 namespace rocksdb {
 
@@ -267,7 +264,7 @@ void CompactionJob::AggregateStatistics() {
 CompactionJob::CompactionJob(
     int job_id, Compaction* compaction, const ImmutableDBOptions& db_options,
     const EnvOptions& env_options, VersionSet* versions,
-    const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
+    std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
     Directory* db_directory, Directory* output_directory, Statistics* stats,
     InstrumentedMutex* db_mutex, Status* db_bg_error,
     std::vector<SequenceNumber> existing_snapshots,
@@ -525,7 +522,7 @@ Status CompactionJob::Run() {
   const uint64_t start_micros = env_->NowMicros();
 
   // Launch a thread for each of subcompactions 1...num_threads-1
-  std::vector<port::Thread> thread_pool;
+  std::vector<std::thread> thread_pool;
   thread_pool.reserve(num_threads - 1);
   for (size_t i = 1; i < compact_->sub_compact_states.size(); i++) {
     thread_pool.emplace_back(&CompactionJob::ProcessKeyValueCompaction, this,
@@ -541,7 +538,7 @@ Status CompactionJob::Run() {
     thread.join();
   }
 
-  if (output_directory_) {
+  if (output_directory_ && !db_options_.disable_data_sync) {
     output_directory_->Fsync();
   }
 
@@ -603,7 +600,7 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
     write_amp = stats.bytes_written /
                 static_cast<double>(stats.bytes_read_non_output_levels);
   }
-  ROCKS_LOG_BUFFER(
+  LogToBuffer(
       log_buffer_,
       "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
       "files in(%d, %d) out(%d) "
@@ -724,10 +721,10 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   MergeHelper merge(
       env_, cfd->user_comparator(), cfd->ioptions()->merge_operator,
       compaction_filter, db_options_.info_log.get(),
+      mutable_cf_options->min_partial_merge_operands,
       false /* internal key corruption is expected */,
       existing_snapshots_.empty() ? 0 : existing_snapshots_.back(),
-      compact_->compaction->level(), db_options_.statistics.get(),
-      shutting_down_);
+      compact_->compaction->level(), db_options_.statistics.get());
 
   TEST_SYNC_POINT("CompactionJob::Run():Inprogress");
 
@@ -736,38 +733,18 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   if (start != nullptr) {
     IterKey start_iter;
     start_iter.SetInternalKey(*start, kMaxSequenceNumber, kValueTypeForSeek);
-    input->Seek(start_iter.GetInternalKey());
+    input->Seek(start_iter.GetKey());
   } else {
     input->SeekToFirst();
   }
 
-  // we allow only 1 compaction event listener. Used by blob storage
-  CompactionEventListener* comp_event_listener = nullptr;
-#ifndef ROCKSDB_LITE
-  for (auto& celitr : cfd->ioptions()->listeners) {
-    comp_event_listener = celitr->GetCompactionEventListener();
-    if (comp_event_listener != nullptr) {
-      break;
-    }
-  }
-#endif  // ROCKSDB_LITE
-
   Status status;
   sub_compact->c_iter.reset(new CompactionIterator(
       input.get(), cfd->user_comparator(), &merge, versions_->LastSequence(),
       &existing_snapshots_, earliest_write_conflict_snapshot_, env_, false,
-      range_del_agg.get(), sub_compact->compaction, compaction_filter,
-      comp_event_listener, shutting_down_));
+      range_del_agg.get(), sub_compact->compaction, compaction_filter));
   auto c_iter = sub_compact->c_iter.get();
   c_iter->SeekToFirst();
-  if (c_iter->Valid() &&
-      sub_compact->compaction->output_level() != 0) {
-    // ShouldStopBefore() maintains state based on keys processed so far. The
-    // compaction loop always calls it on the "next" key, thus won't tell it the
-    // first key. So we do that here.
-    sub_compact->ShouldStopBefore(
-      c_iter->key(), sub_compact->current_output_file_size);
-  }
   const auto& c_iter_stats = c_iter->iter_stats();
   auto sample_begin_offset_iter = sample_begin_offsets.cbegin();
   // data_begin_offset and compression_dict are only valid while generating
@@ -776,7 +753,10 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   std::string compression_dict;
   compression_dict.reserve(cfd->ioptions()->compression_opts.max_dict_bytes);
 
-  while (status.ok() && !cfd->IsDropped() && c_iter->Valid()) {
+  // TODO(noetzli): check whether we could check !shutting_down_->... only
+  // only occasionally (see diff D42687)
+  while (status.ok() && !shutting_down_->load(std::memory_order_acquire) &&
+         !cfd->IsDropped() && c_iter->Valid()) {
     // Invariant: c_iter.status() is guaranteed to be OK if c_iter->Valid()
     // returns true.
     const Slice& key = c_iter->key();
@@ -787,7 +767,21 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
     if (end != nullptr &&
         cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
       break;
+    } else if (sub_compact->compaction->output_level() != 0 &&
+               sub_compact->ShouldStopBefore(
+                   key, sub_compact->current_output_file_size) &&
+               sub_compact->builder != nullptr) {
+      CompactionIterationStats range_del_out_stats;
+      status = FinishCompactionOutputFile(input->status(), sub_compact,
+                                          range_del_agg.get(),
+                                          &range_del_out_stats, &key);
+      RecordDroppedKeys(range_del_out_stats,
+                        &sub_compact->compaction_job_stats);
+      if (!status.ok()) {
+        break;
+      }
     }
+
     if (c_iter_stats.num_input_records % kRecordStatsEvery ==
         kRecordStatsEvery - 1) {
       RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
@@ -859,37 +853,17 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
       }
     }
 
-    // Close output file if it is big enough. Two possibilities determine it's
-    // time to close it: (1) the current key should be this file's last key, (2)
-    // the next key should not be in this file.
-    //
+    // Close output file if it is big enough
     // TODO(aekmekji): determine if file should be closed earlier than this
     // during subcompactions (i.e. if output size, estimated by input size, is
     // going to be 1.2MB and max_output_file_size = 1MB, prefer to have 0.6MB
     // and 0.6MB instead of 1MB and 0.2MB)
-    bool output_file_ended = false;
-    Status input_status;
     if (sub_compact->compaction->output_level() != 0 &&
         sub_compact->current_output_file_size >=
             sub_compact->compaction->max_output_file_size()) {
-      // (1) this key terminates the file. For historical reasons, the iterator
-      // status before advancing will be given to FinishCompactionOutputFile().
-      input_status = input->status();
-      output_file_ended = true;
-    }
-    c_iter->Next();
-    if (!output_file_ended && c_iter->Valid() &&
-        sub_compact->compaction->output_level() != 0 &&
-        sub_compact->ShouldStopBefore(
-          c_iter->key(), sub_compact->current_output_file_size) &&
-        sub_compact->builder != nullptr) {
-      // (2) this key belongs to the next file. For historical reasons, the
-      // iterator status after advancing will be given to
-      // FinishCompactionOutputFile().
-      input_status = input->status();
-      output_file_ended = true;
-    }
-    if (output_file_ended) {
+      Status input_status = input->status();
+      c_iter->Next();
+
       const Slice* next_key = nullptr;
       if (c_iter->Valid()) {
         next_key = &c_iter->key();
@@ -905,6 +879,8 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
         // files.
         sub_compact->compression_dict = std::move(compression_dict);
       }
+    } else {
+      c_iter->Next();
     }
   }
 
@@ -927,36 +903,27 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
   RecordDroppedKeys(c_iter_stats, &sub_compact->compaction_job_stats);
   RecordCompactionIOStats();
 
-  if (status.ok() && (shutting_down_->load(std::memory_order_relaxed) ||
-                      cfd->IsDropped())) {
+  if (status.ok() &&
+      (shutting_down_->load(std::memory_order_acquire) || cfd->IsDropped())) {
     status = Status::ShutdownInProgress(
         "Database shutdown or Column family drop during compaction");
   }
-  if (status.ok()) {
-    status = input->status();
-  }
-  if (status.ok()) {
-    status = c_iter->status();
-  }
-
   if (status.ok() && sub_compact->builder == nullptr &&
       sub_compact->outputs.size() == 0 &&
       range_del_agg->ShouldAddTombstones(bottommost_level_)) {
     // handle subcompaction containing only range deletions
     status = OpenCompactionOutputFile(sub_compact);
   }
-
-  // Call FinishCompactionOutputFile() even if status is not ok: it needs to
-  // close the output file.
-  if (sub_compact->builder != nullptr) {
+  if (status.ok() && sub_compact->builder != nullptr) {
     CompactionIterationStats range_del_out_stats;
-    Status s = FinishCompactionOutputFile(
-        status, sub_compact, range_del_agg.get(), &range_del_out_stats);
-    if (status.ok()) {
-      status = s;
-    }
+    status =
+        FinishCompactionOutputFile(input->status(), sub_compact,
+                                   range_del_agg.get(), &range_del_out_stats);
     RecordDroppedKeys(range_del_out_stats, &sub_compact->compaction_job_stats);
   }
+  if (status.ok()) {
+    status = input->status();
+  }
 
   if (measure_io_stats_) {
     sub_compact->compaction_job_stats.file_write_nanos +=
@@ -1072,7 +1039,7 @@ Status CompactionJob::FinishCompactionOutputFile(
   sub_compact->total_bytes += current_bytes;
 
   // Finish and check for file errors
-  if (s.ok()) {
+  if (s.ok() && !db_options_.disable_data_sync) {
     StopWatch sw(env_, stats_, COMPACTION_OUTFILE_SYNC_MICROS);
     s = sub_compact->outfile->Sync(db_options_.use_fsync);
   }
@@ -1085,11 +1052,6 @@ Status CompactionJob::FinishCompactionOutputFile(
   TableProperties tp;
   if (s.ok() && current_entries > 0) {
     // Verify that the table is usable
-    // We set for_compaction to false and don't OptimizeForCompactionTableRead
-    // here because this is a special case after we finish the table building
-    // No matter whether use_direct_io_for_flush_and_compaction is true,
-    // we will regrad this verification as user reads since the goal is
-    // to cache it here for further user reads
     InternalIterator* iter = cfd->table_cache()->NewIterator(
         ReadOptions(), env_options_, cfd->internal_comparator(), meta->fd,
         nullptr /* range_del_agg */, nullptr,
@@ -1110,12 +1072,12 @@ Status CompactionJob::FinishCompactionOutputFile(
       tp = sub_compact->builder->GetTableProperties();
       sub_compact->current_output()->table_properties =
           std::make_shared<TableProperties>(tp);
-      ROCKS_LOG_INFO(db_options_.info_log,
-                     "[%s] [JOB %d] Generated table #%" PRIu64 ": %" PRIu64
-                     " keys, %" PRIu64 " bytes%s",
-                     cfd->GetName().c_str(), job_id_, output_number,
-                     current_entries, current_bytes,
-                     meta->marked_for_compaction ? " (need compaction)" : "");
+      Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+          "[%s] [JOB %d] Generated table #%" PRIu64 ": %" PRIu64
+          " keys, %" PRIu64 " bytes%s",
+          cfd->GetName().c_str(), job_id_, output_number, current_entries,
+          current_bytes,
+          meta->marked_for_compaction ? " (need compaction)" : "");
     }
   }
   std::string fname = TableFileName(db_options_.db_paths, meta->fd.GetNumber(),
@@ -1124,7 +1086,6 @@ Status CompactionJob::FinishCompactionOutputFile(
       event_logger_, cfd->ioptions()->listeners, dbname_, cfd->GetName(), fname,
       job_id_, meta->fd, tp, TableFileCreationReason::kCompaction, s);
 
-#ifndef ROCKSDB_LITE
   // Report new file to SstFileManagerImpl
   auto sfm =
       static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
@@ -1142,7 +1103,6 @@ Status CompactionJob::FinishCompactionOutputFile(
       }
     }
   }
-#endif
 
   sub_compact->builder.reset();
   sub_compact->current_output_file_size = 0;
@@ -1161,16 +1121,17 @@ Status CompactionJob::InstallCompactionResults(
   if (!versions_->VerifyCompactionFileConsistency(compaction)) {
     Compaction::InputLevelSummaryBuffer inputs_summary;
 
-    ROCKS_LOG_ERROR(db_options_.info_log, "[%s] [JOB %d] Compaction %s aborted",
-                    compaction->column_family_data()->GetName().c_str(),
-                    job_id_, compaction->InputLevelSummary(&inputs_summary));
+    Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
+        "[%s] [JOB %d] Compaction %s aborted",
+        compaction->column_family_data()->GetName().c_str(), job_id_,
+        compaction->InputLevelSummary(&inputs_summary));
     return Status::Corruption("Compaction input files inconsistent");
   }
 
   {
     Compaction::InputLevelSummaryBuffer inputs_summary;
-    ROCKS_LOG_INFO(
-        db_options_.info_log, "[%s] [JOB %d] Compacted %s => %" PRIu64 " bytes",
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "[%s] [JOB %d] Compacted %s => %" PRIu64 " bytes",
         compaction->column_family_data()->GetName().c_str(), job_id_,
         compaction->InputLevelSummary(&inputs_summary), compact_->total_bytes);
   }
@@ -1216,14 +1177,9 @@ Status CompactionJob::OpenCompactionOutputFile(
 #endif  // !ROCKSDB_LITE
   // Make the output file
   unique_ptr<WritableFile> writable_file;
-  EnvOptions opt_env_opts =
-      env_->OptimizeForCompactionTableWrite(env_options_, db_options_);
-  TEST_SYNC_POINT_CALLBACK("CompactionJob::OpenCompactionOutputFile",
-                           &opt_env_opts.use_direct_writes);
-  Status s = NewWritableFile(env_, fname, &writable_file, opt_env_opts);
+  Status s = NewWritableFile(env_, fname, &writable_file, env_options_);
   if (!s.ok()) {
-    ROCKS_LOG_ERROR(
-        db_options_.info_log,
+    Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
         "[%s] [JOB %d] OpenCompactionOutputFiles for table #%" PRIu64
         " fails at NewWritableFile with status %s",
         sub_compact->compaction->column_family_data()->GetName().c_str(),
@@ -1245,8 +1201,8 @@ Status CompactionJob::OpenCompactionOutputFile(
   writable_file->SetIOPriority(Env::IO_LOW);
   writable_file->SetPreallocationBlockSize(static_cast<size_t>(
       sub_compact->compaction->OutputFilePreallocationSize()));
-  sub_compact->outfile.reset(new WritableFileWriter(
-      std::move(writable_file), env_options_, db_options_.statistics.get()));
+  sub_compact->outfile.reset(
+      new WritableFileWriter(std::move(writable_file), env_options_));
 
   // If the Column family flag is to only optimize filters for hits,
   // we can skip creating filters if this is the bottommost_level where
@@ -1399,14 +1355,14 @@ void CompactionJob::LogCompaction() {
   // we're not logging
   if (db_options_.info_log_level <= InfoLogLevel::INFO_LEVEL) {
     Compaction::InputLevelSummaryBuffer inputs_summary;
-    ROCKS_LOG_INFO(
-        db_options_.info_log, "[%s] [JOB %d] Compacting %s, score %.2f",
-        cfd->GetName().c_str(), job_id_,
-        compaction->InputLevelSummary(&inputs_summary), compaction->score());
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "[%s] [JOB %d] Compacting %s, score %.2f", cfd->GetName().c_str(),
+        job_id_, compaction->InputLevelSummary(&inputs_summary),
+        compaction->score());
     char scratch[2345];
     compaction->Summary(scratch, sizeof(scratch));
-    ROCKS_LOG_INFO(db_options_.info_log, "[%s] Compaction start summary: %s\n",
-                   cfd->GetName().c_str(), scratch);
+    Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
+        "[%s] Compaction start summary: %s\n", cfd->GetName().c_str(), scratch);
     // build event logger report
     auto stream = event_logger_->Log();
     stream << "job" << job_id_ << "event"

db/compaction_job.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -31,7 +29,6 @@
 #include "db/version_edit.h"
 #include "db/write_controller.h"
 #include "db/write_thread.h"
-#include "options/db_options.h"
 #include "port/port.h"
 #include "rocksdb/compaction_filter.h"
 #include "rocksdb/compaction_job_stats.h"
@@ -41,6 +38,7 @@
 #include "rocksdb/transaction_log.h"
 #include "table/scoped_arena_iterator.h"
 #include "util/autovector.h"
+#include "util/db_options.h"
 #include "util/event_logger.h"
 #include "util/stop_watch.h"
 #include "util/thread_local.h"
@@ -59,7 +57,7 @@ class CompactionJob {
   CompactionJob(int job_id, Compaction* compaction,
                 const ImmutableDBOptions& db_options,
                 const EnvOptions& env_options, VersionSet* versions,
-                const std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
+                std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
                 Directory* db_directory, Directory* output_directory,
                 Statistics* stats, InstrumentedMutex* db_mutex,
                 Status* db_bg_error,
@@ -133,7 +131,7 @@ class CompactionJob {
 
   Env* env_;
   VersionSet* versions_;
-  const std::atomic<bool>* shutting_down_;
+  std::atomic<bool>* shutting_down_;
   LogBuffer* log_buffer_;
   Directory* db_directory_;
   Directory* output_directory_;

db/compaction_job_stats_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -25,13 +23,11 @@
 
 #include "db/db_impl.h"
 #include "db/dbformat.h"
+#include "db/filename.h"
 #include "db/job_context.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
-#include "env/mock_env.h"
 #include "memtable/hash_linklist_rep.h"
-#include "monitoring/statistics.h"
-#include "monitoring/thread_status_util.h"
 #include "port/stack_trace.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/compaction_filter.h"
@@ -54,15 +50,18 @@
 #include "table/plain_table_factory.h"
 #include "table/scoped_arena_iterator.h"
 #include "util/compression.h"
-#include "util/filename.h"
 #include "util/hash.h"
 #include "util/logging.h"
+#include "util/mock_env.h"
 #include "util/mutexlock.h"
 #include "util/rate_limiter.h"
+#include "util/statistics.h"
 #include "util/string_util.h"
 #include "util/sync_point.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
+#include "util/thread_status_util.h"
+#include "util/xfunc.h"
 #include "utilities/merge_operators.h"
 
 #if !defined(IOS_CROSS_COMPILE)

db/compaction_job_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #ifndef ROCKSDB_LITE
 

db/compaction_picker.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -19,10 +17,10 @@
 
 #include "db/compaction.h"
 #include "db/version_set.h"
-#include "options/cf_options.h"
 #include "rocksdb/env.h"
 #include "rocksdb/options.h"
 #include "rocksdb/status.h"
+#include "util/cf_options.h"
 
 namespace rocksdb {
 
@@ -86,15 +84,15 @@ class CompactionPicker {
   void ReleaseCompactionFiles(Compaction* c, Status status);
 
   // Returns true if any one of the specified files are being compacted
-  bool AreFilesInCompaction(const std::vector<FileMetaData*>& files);
+  bool FilesInCompaction(const std::vector<FileMetaData*>& files);
 
   // Takes a list of CompactionInputFiles and returns a (manual) Compaction
   // object.
-  Compaction* CompactFiles(const CompactionOptions& compact_options,
-                           const std::vector<CompactionInputFiles>& input_files,
-                           int output_level, VersionStorageInfo* vstorage,
-                           const MutableCFOptions& mutable_cf_options,
-                           uint32_t output_path_id);
+  Compaction* FormCompaction(
+      const CompactionOptions& compact_options,
+      const std::vector<CompactionInputFiles>& input_files, int output_level,
+      VersionStorageInfo* vstorage, const MutableCFOptions& mutable_cf_options,
+      uint32_t output_path_id);
 
   // Converts a set of compaction input file numbers into
   // a list of CompactionInputFiles.
@@ -104,6 +102,12 @@ class CompactionPicker {
       const VersionStorageInfo* vstorage,
       const CompactionOptions& compact_options) const;
 
+  // Used in universal compaction when the enabled_trivial_move
+  // option is set. Checks whether there are any overlapping files
+  // in the input. Returns true if the input files are non
+  // overlapping.
+  bool IsInputNonOverlapping(Compaction* c);
+
   // Is there currently a compaction involving level 0 taking place
   bool IsLevel0CompactionInProgress() const {
     return !level0_compactions_in_progress_.empty();
@@ -115,6 +119,9 @@ class CompactionPicker {
                                   const Slice& largest_user_key,
                                   int level) const;
 
+ protected:
+  int NumberLevels() const { return ioptions_.num_levels; }
+
   // Stores the minimal range that covers all entries in inputs in
   // *smallest, *largest.
   // REQUIRES: inputs is not empty
@@ -134,8 +141,6 @@ class CompactionPicker {
   void GetRange(const std::vector<CompactionInputFiles>& inputs,
                 InternalKey* smallest, InternalKey* largest) const;
 
-  int NumberLevels() const { return ioptions_.num_levels; }
-
   // Add more files to the inputs on "level" to make sure that
   // no newer version of a key is compacted to "level+1" while leaving an older
   // version in a "level". Otherwise, any Get() will search "level" first,
@@ -146,14 +151,14 @@ class CompactionPicker {
   // populated.
   //
   // Will return false if it is impossible to apply this compaction.
-  bool ExpandInputsToCleanCut(const std::string& cf_name,
+  bool ExpandWhileOverlapping(const std::string& cf_name,
                               VersionStorageInfo* vstorage,
                               CompactionInputFiles* inputs);
 
   // Returns true if any one of the parent files are being compacted
-  bool IsRangeInCompaction(VersionStorageInfo* vstorage,
-                           const InternalKey* smallest,
-                           const InternalKey* largest, int level, int* index);
+  bool RangeInCompaction(VersionStorageInfo* vstorage,
+                         const InternalKey* smallest,
+                         const InternalKey* largest, int level, int* index);
 
   // Returns true if the key range that `inputs` files cover overlap with the
   // key range of a currently running compaction.
@@ -172,20 +177,6 @@ class CompactionPicker {
                        const CompactionInputFiles& output_level_inputs,
                        std::vector<FileMetaData*>* grandparents);
 
-  // Register this compaction in the set of running compactions
-  void RegisterCompaction(Compaction* c);
-
-  // Remove this compaction from the set of running compactions
-  void UnregisterCompaction(Compaction* c);
-
-  std::set<Compaction*>* level0_compactions_in_progress() {
-    return &level0_compactions_in_progress_;
-  }
-  std::unordered_set<Compaction*>* compactions_in_progress() {
-    return &compactions_in_progress_;
-  }
-
- protected:
   const ImmutableCFOptions& ioptions_;
 
 // A helper function to SanitizeCompactionInputFiles() that
@@ -196,6 +187,12 @@ class CompactionPicker {
       const ColumnFamilyMetaData& cf_meta, const int output_level) const;
 #endif  // ROCKSDB_LITE
 
+  // Register this compaction in the set of running compactions
+  void RegisterCompaction(Compaction* c);
+
+  // Remove this compaction from the set of running compactions
+  void UnregisterCompaction(Compaction* c);
+
   // Keeps track of all compactions that are running on Level0.
   // Protected by DB mutex
   std::set<Compaction*> level0_compactions_in_progress_;
@@ -219,9 +216,101 @@ class LevelCompactionPicker : public CompactionPicker {
 
   virtual bool NeedsCompaction(
       const VersionStorageInfo* vstorage) const override;
+
+  // Pick a path ID to place a newly generated file, with its level
+  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
+                            const MutableCFOptions& mutable_cf_options,
+                            int level);
+
+ private:
+  // For the specfied level, pick a file that we want to compact.
+  // Returns false if there is no file to compact.
+  // If it returns true, inputs->files.size() will be exactly one.
+  // If level is 0 and there is already a compaction on that level, this
+  // function will return false.
+  bool PickCompactionBySize(VersionStorageInfo* vstorage, int level,
+                            int output_level, CompactionInputFiles* inputs,
+                            int* parent_index, int* base_index);
+
+  // If there is any file marked for compaction, put put it into inputs.
+  // This is still experimental. It will return meaningful results only if
+  // clients call experimental feature SuggestCompactRange()
+  void PickFilesMarkedForCompactionExperimental(const std::string& cf_name,
+                                                VersionStorageInfo* vstorage,
+                                                CompactionInputFiles* inputs,
+                                                int* level, int* output_level);
 };
 
 #ifndef ROCKSDB_LITE
+class UniversalCompactionPicker : public CompactionPicker {
+ public:
+  UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
+                            const InternalKeyComparator* icmp)
+      : CompactionPicker(ioptions, icmp) {}
+  virtual Compaction* PickCompaction(const std::string& cf_name,
+                                     const MutableCFOptions& mutable_cf_options,
+                                     VersionStorageInfo* vstorage,
+                                     LogBuffer* log_buffer) override;
+
+  virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
+
+  virtual bool NeedsCompaction(
+      const VersionStorageInfo* vstorage) const override;
+
+ private:
+  struct SortedRun {
+    SortedRun(int _level, FileMetaData* _file, uint64_t _size,
+              uint64_t _compensated_file_size, bool _being_compacted)
+        : level(_level),
+          file(_file),
+          size(_size),
+          compensated_file_size(_compensated_file_size),
+          being_compacted(_being_compacted) {
+      assert(compensated_file_size > 0);
+      assert(level != 0 || file != nullptr);
+    }
+
+    void Dump(char* out_buf, size_t out_buf_size,
+              bool print_path = false) const;
+
+    // sorted_run_count is added into the string to print
+    void DumpSizeInfo(char* out_buf, size_t out_buf_size,
+                      size_t sorted_run_count) const;
+
+    int level;
+    // `file` Will be null for level > 0. For level = 0, the sorted run is
+    // for this file.
+    FileMetaData* file;
+    // For level > 0, `size` and `compensated_file_size` are sum of sizes all
+    // files in the level. `being_compacted` should be the same for all files
+    // in a non-zero level. Use the value here.
+    uint64_t size;
+    uint64_t compensated_file_size;
+    bool being_compacted;
+  };
+
+  // Pick Universal compaction to limit read amplification
+  Compaction* PickCompactionUniversalReadAmp(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      VersionStorageInfo* vstorage, double score, unsigned int ratio,
+      unsigned int num_files, const std::vector<SortedRun>& sorted_runs,
+      LogBuffer* log_buffer);
+
+  // Pick Universal compaction to limit space amplification.
+  Compaction* PickCompactionUniversalSizeAmp(
+      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+      VersionStorageInfo* vstorage, double score,
+      const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer);
+
+  static std::vector<SortedRun> CalculateSortedRuns(
+      const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions);
+
+  // Pick a path ID to place a newly generated file, with its estimated file
+  // size.
+  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
+                            uint64_t file_size);
+};
+
 class FIFOCompactionPicker : public CompactionPicker {
  public:
   FIFOCompactionPicker(const ImmutableCFOptions& ioptions,

db/compaction_picker_test.cc

@@ -2,15 +2,12 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
+#include "db/compaction.h"
 #include "db/compaction_picker.h"
 #include <limits>
 #include <string>
 #include <utility>
-#include "db/compaction.h"
-#include "db/compaction_picker_universal.h"
 
 #include "util/logging.h"
 #include "util/string_util.h"
@@ -200,7 +197,7 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
   mutable_cf_options_.target_file_size_base = 10000000;
   mutable_cf_options_.target_file_size_multiplier = 10;
   mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
-  Add(0, 1U, "150", "200", 1000000U);
+  Add(0, 1U, "150", "200", 1000000000U);
   // Level 1 score 1.2
   Add(1, 66U, "150", "200", 6000000U);
   Add(1, 88U, "201", "300", 6000000U);
@@ -402,35 +399,6 @@ TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {
               vstorage_->CompactionScore(0) >= 1);
   }
 }
-
-TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
-  const uint64_t kFileSize = 100000;
-  NewVersionStorage(1, kCompactionStyleUniversal);
-  ioptions_.allow_ingest_behind = true;
-  ioptions_.num_levels = 3;
-  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
-  // must return false when there's no files.
-  ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
-            false);
-
-  NewVersionStorage(3, kCompactionStyleUniversal);
-
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
-  Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
-  Add(0, 4U, "260", "300", kFileSize, 0, 260, 300);
-  Add(1, 5U, "100", "151", kFileSize, 0, 200, 251);
-  Add(1, 3U, "301", "350", kFileSize, 0, 101, 150);
-  Add(2, 6U, "120", "200", kFileSize, 0, 20, 100);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(
-      universal_compaction_picker.PickCompaction(
-          cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-
-  // output level should be the one above the bottom-most
-  ASSERT_EQ(1, compaction->output_level());
-}
 // Tests if the files can be trivially moved in multi level
 // universal compaction when allow_trivial_move option is set
 // In this test as the input files overlaps, they cannot
@@ -790,70 +758,6 @@ TEST_F(CompactionPickerTest, OverlappingUserKeys7) {
   ASSERT_EQ(5U, compaction->inputs(1)->back()->fd.GetNumber());
 }
 
-TEST_F(CompactionPickerTest, OverlappingUserKeys8) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-  // grow the number of inputs in "level" without
-  // changing the number of "level+1" files we pick up
-  // Expand input level as much as possible
-  // no overlapping case
-  Add(1, 1U, "101", "150", 1U);
-  Add(1, 2U, "151", "200", 1U);
-  Add(1, 3U, "201", "300", 1000000000U);
-  Add(1, 4U, "301", "400", 1U);
-  Add(1, 5U, "401", "500", 1U);
-  Add(2, 6U, "150", "200", 1U);
-  Add(2, 7U, "200", "450", 1U, 0, 0);
-  Add(2, 8U, "500", "600", 1U);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(3U, compaction->num_input_files(0));
-  ASSERT_EQ(2U, compaction->num_input_files(1));
-  ASSERT_EQ(2U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(3U, compaction->input(0, 1)->fd.GetNumber());
-  ASSERT_EQ(4U, compaction->input(0, 2)->fd.GetNumber());
-  ASSERT_EQ(6U, compaction->input(1, 0)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 1)->fd.GetNumber());
-}
-
-TEST_F(CompactionPickerTest, OverlappingUserKeys9) {
-  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.max_compaction_bytes = 100000000000u;
-  // grow the number of inputs in "level" without
-  // changing the number of "level+1" files we pick up
-  // Expand input level as much as possible
-  // overlapping case
-  Add(1, 1U, "121", "150", 1U);
-  Add(1, 2U, "151", "200", 1U);
-  Add(1, 3U, "201", "300", 1000000000U);
-  Add(1, 4U, "301", "400", 1U);
-  Add(1, 5U, "401", "500", 1U);
-  Add(2, 6U, "100", "120", 1U);
-  Add(2, 7U, "150", "200", 1U);
-  Add(2, 8U, "200", "450", 1U, 0, 0);
-  Add(2, 9U, "501", "600", 1U);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_EQ(2U, compaction->num_input_levels());
-  ASSERT_EQ(5U, compaction->num_input_files(0));
-  ASSERT_EQ(2U, compaction->num_input_files(1));
-  ASSERT_EQ(1U, compaction->input(0, 0)->fd.GetNumber());
-  ASSERT_EQ(2U, compaction->input(0, 1)->fd.GetNumber());
-  ASSERT_EQ(3U, compaction->input(0, 2)->fd.GetNumber());
-  ASSERT_EQ(4U, compaction->input(0, 3)->fd.GetNumber());
-  ASSERT_EQ(7U, compaction->input(1, 0)->fd.GetNumber());
-  ASSERT_EQ(8U, compaction->input(1, 1)->fd.GetNumber());
-}
-
 TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri1) {
   NewVersionStorage(6, kCompactionStyleLevel);
   mutable_cf_options_.level0_file_num_compaction_trigger = 2;
@@ -1042,8 +946,7 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
 
   // Set Last level size 50000
   // num_levels - 1 target 5000
-  // num_levels - 2 is base level with target 1000 (rounded up to
-  // max_bytes_for_level_base).
+  // num_levels - 2 is base level with taret 500
   Add(num_levels - 1, 10U, "400", "500", 50000);
 
   Add(0, 1U, "150", "200", 200);
@@ -1052,16 +955,16 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
   Add(0, 5U, "150", "200", 200);
   Add(0, 6U, "150", "200", 200);
   // num_levels - 3 is over target by 100 + 1000
-  Add(num_levels - 3, 7U, "400", "500", 550);
-  Add(num_levels - 3, 8U, "600", "700", 550);
+  Add(num_levels - 3, 7U, "400", "500", 300);
+  Add(num_levels - 3, 8U, "600", "700", 300);
   // num_levels - 2 is over target by 1100 + 200
   Add(num_levels - 2, 9U, "150", "200", 5200);
 
   UpdateVersionStorageInfo();
 
-  // Merging to the second last level: (5200 / 2100 + 1) * 1100
+  // Merging to the second last level: (5200 / 1600 + 1) * 1100
   // Merging to the last level: (50000 / 6300 + 1) * 1300
-  ASSERT_EQ(2100u + 3823u + 11617u,
+  ASSERT_EQ(1600u + 4675u + 11617u,
             vstorage_->estimated_compaction_needed_bytes());
 }
 
@@ -1229,13 +1132,13 @@ TEST_F(CompactionPickerTest, MaxCompactionBytesHit) {
   ioptions_.level_compaction_dynamic_level_bytes = false;
   NewVersionStorage(6, kCompactionStyleLevel);
   // A compaction should be triggered and pick file 2 and 5.
-  // It can expand because adding file 1 and 3, the compaction size will
+  // It cannot expand because adding file 1 and 3, the compaction size will
   // exceed mutable_cf_options_.max_bytes_for_level_base.
   Add(1, 1U, "100", "150", 300000U);
   Add(1, 2U, "151", "200", 300001U, 0, 0);
   Add(1, 3U, "201", "250", 300000U, 0, 0);
   Add(1, 4U, "251", "300", 300000U, 0, 0);
-  Add(2, 5U, "100", "256", 1U);
+  Add(2, 5U, "160", "256", 1U);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
@@ -1249,7 +1152,7 @@ TEST_F(CompactionPickerTest, MaxCompactionBytesHit) {
 }
 
 TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) {
-  mutable_cf_options_.max_bytes_for_level_base = 800000u;
+  mutable_cf_options_.max_bytes_for_level_base = 1000000u;
   mutable_cf_options_.max_compaction_bytes = 1000000u;
   ioptions_.level_compaction_dynamic_level_bytes = false;
   NewVersionStorage(6, kCompactionStyleLevel);
@@ -1259,7 +1162,7 @@ TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) {
   Add(1, 2U, "151", "200", 300001U, 0, 0);
   Add(1, 3U, "201", "250", 300000U, 0, 0);
   Add(1, 4U, "251", "300", 300000U, 0, 0);
-  Add(2, 5U, "000", "251", 1U);
+  Add(2, 5U, "000", "233", 1U);
   UpdateVersionStorageInfo();
 
   std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
@@ -1274,50 +1177,6 @@ TEST_F(CompactionPickerTest, MaxCompactionBytesNotHit) {
   ASSERT_EQ(5U, compaction->input(1, 0)->fd.GetNumber());
 }
 
-TEST_F(CompactionPickerTest, IsTrivialMoveOn) {
-  mutable_cf_options_.max_bytes_for_level_base = 10000u;
-  mutable_cf_options_.max_compaction_bytes = 10001u;
-  ioptions_.level_compaction_dynamic_level_bytes = false;
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // A compaction should be triggered and pick file 2
-  Add(1, 1U, "100", "150", 3000U);
-  Add(1, 2U, "151", "200", 3001U);
-  Add(1, 3U, "201", "250", 3000U);
-  Add(1, 4U, "251", "300", 3000U);
-
-  Add(3, 5U, "120", "130", 7000U);
-  Add(3, 6U, "170", "180", 7000U);
-  Add(3, 5U, "220", "230", 7000U);
-  Add(3, 5U, "270", "280", 7000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-    cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_TRUE(compaction->IsTrivialMove());
-}
-
-TEST_F(CompactionPickerTest, IsTrivialMoveOff) {
-  mutable_cf_options_.max_bytes_for_level_base = 1000000u;
-  mutable_cf_options_.max_compaction_bytes = 10000u;
-  ioptions_.level_compaction_dynamic_level_bytes = false;
-  NewVersionStorage(6, kCompactionStyleLevel);
-  // A compaction should be triggered and pick all files from level 1
-  Add(1, 1U, "100", "150", 300000U, 0, 0);
-  Add(1, 2U, "150", "200", 300000U, 0, 0);
-  Add(1, 3U, "200", "250", 300000U, 0, 0);
-  Add(1, 4U, "250", "300", 300000U, 0, 0);
-
-  Add(3, 5U, "120", "130", 6000U);
-  Add(3, 6U, "140", "150", 6000U);
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
-    cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-  ASSERT_TRUE(compaction.get() != nullptr);
-  ASSERT_FALSE(compaction->IsTrivialMove());
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/compaction_picker_universal.cc

@@ -1,749 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/compaction_picker_universal.h"
-#ifndef ROCKSDB_LITE
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include <inttypes.h>
-#include <limits>
-#include <queue>
-#include <string>
-#include <utility>
-#include "db/column_family.h"
-#include "monitoring/statistics.h"
-#include "util/filename.h"
-#include "util/log_buffer.h"
-#include "util/random.h"
-#include "util/string_util.h"
-#include "util/sync_point.h"
-
-namespace rocksdb {
-namespace {
-// Used in universal compaction when trivial move is enabled.
-// This structure is used for the construction of min heap
-// that contains the file meta data, the level of the file
-// and the index of the file in that level
-
-struct InputFileInfo {
-  InputFileInfo() : f(nullptr) {}
-
-  FileMetaData* f;
-  size_t level;
-  size_t index;
-};
-
-// Used in universal compaction when trivial move is enabled.
-// This comparator is used for the construction of min heap
-// based on the smallest key of the file.
-struct SmallestKeyHeapComparator {
-  explicit SmallestKeyHeapComparator(const Comparator* ucmp) { ucmp_ = ucmp; }
-
-  bool operator()(InputFileInfo i1, InputFileInfo i2) const {
-    return (ucmp_->Compare(i1.f->smallest.user_key(),
-                           i2.f->smallest.user_key()) > 0);
-  }
-
- private:
-  const Comparator* ucmp_;
-};
-
-typedef std::priority_queue<InputFileInfo, std::vector<InputFileInfo>,
-                            SmallestKeyHeapComparator>
-    SmallestKeyHeap;
-
-// This function creates the heap that is used to find if the files are
-// overlapping during universal compaction when the allow_trivial_move
-// is set.
-SmallestKeyHeap create_level_heap(Compaction* c, const Comparator* ucmp) {
-  SmallestKeyHeap smallest_key_priority_q =
-      SmallestKeyHeap(SmallestKeyHeapComparator(ucmp));
-
-  InputFileInfo input_file;
-
-  for (size_t l = 0; l < c->num_input_levels(); l++) {
-    if (c->num_input_files(l) != 0) {
-      if (l == 0 && c->start_level() == 0) {
-        for (size_t i = 0; i < c->num_input_files(0); i++) {
-          input_file.f = c->input(0, i);
-          input_file.level = 0;
-          input_file.index = i;
-          smallest_key_priority_q.push(std::move(input_file));
-        }
-      } else {
-        input_file.f = c->input(l, 0);
-        input_file.level = l;
-        input_file.index = 0;
-        smallest_key_priority_q.push(std::move(input_file));
-      }
-    }
-  }
-  return smallest_key_priority_q;
-}
-
-#ifndef NDEBUG
-// smallest_seqno and largest_seqno are set iff. `files` is not empty.
-void GetSmallestLargestSeqno(const std::vector<FileMetaData*>& files,
-                             SequenceNumber* smallest_seqno,
-                             SequenceNumber* largest_seqno) {
-  bool is_first = true;
-  for (FileMetaData* f : files) {
-    assert(f->smallest_seqno <= f->largest_seqno);
-    if (is_first) {
-      is_first = false;
-      *smallest_seqno = f->smallest_seqno;
-      *largest_seqno = f->largest_seqno;
-    } else {
-      if (f->smallest_seqno < *smallest_seqno) {
-        *smallest_seqno = f->smallest_seqno;
-      }
-      if (f->largest_seqno > *largest_seqno) {
-        *largest_seqno = f->largest_seqno;
-      }
-    }
-  }
-}
-#endif
-}  // namespace
-
-// Algorithm that checks to see if there are any overlapping
-// files in the input
-bool UniversalCompactionPicker::IsInputFilesNonOverlapping(Compaction* c) {
-  auto comparator = icmp_->user_comparator();
-  int first_iter = 1;
-
-  InputFileInfo prev, curr, next;
-
-  SmallestKeyHeap smallest_key_priority_q =
-      create_level_heap(c, icmp_->user_comparator());
-
-  while (!smallest_key_priority_q.empty()) {
-    curr = smallest_key_priority_q.top();
-    smallest_key_priority_q.pop();
-
-    if (first_iter) {
-      prev = curr;
-      first_iter = 0;
-    } else {
-      if (comparator->Compare(prev.f->largest.user_key(),
-                              curr.f->smallest.user_key()) >= 0) {
-        // found overlapping files, return false
-        return false;
-      }
-      assert(comparator->Compare(curr.f->largest.user_key(),
-                                 prev.f->largest.user_key()) > 0);
-      prev = curr;
-    }
-
-    next.f = nullptr;
-
-    if (curr.level != 0 && curr.index < c->num_input_files(curr.level) - 1) {
-      next.f = c->input(curr.level, curr.index + 1);
-      next.level = curr.level;
-      next.index = curr.index + 1;
-    }
-
-    if (next.f) {
-      smallest_key_priority_q.push(std::move(next));
-    }
-  }
-  return true;
-}
-
-bool UniversalCompactionPicker::NeedsCompaction(
-    const VersionStorageInfo* vstorage) const {
-  const int kLevel0 = 0;
-  return vstorage->CompactionScore(kLevel0) >= 1;
-}
-
-void UniversalCompactionPicker::SortedRun::Dump(char* out_buf,
-                                                size_t out_buf_size,
-                                                bool print_path) const {
-  if (level == 0) {
-    assert(file != nullptr);
-    if (file->fd.GetPathId() == 0 || !print_path) {
-      snprintf(out_buf, out_buf_size, "file %" PRIu64, file->fd.GetNumber());
-    } else {
-      snprintf(out_buf, out_buf_size, "file %" PRIu64
-                                      "(path "
-                                      "%" PRIu32 ")",
-               file->fd.GetNumber(), file->fd.GetPathId());
-    }
-  } else {
-    snprintf(out_buf, out_buf_size, "level %d", level);
-  }
-}
-
-void UniversalCompactionPicker::SortedRun::DumpSizeInfo(
-    char* out_buf, size_t out_buf_size, size_t sorted_run_count) const {
-  if (level == 0) {
-    assert(file != nullptr);
-    snprintf(out_buf, out_buf_size,
-             "file %" PRIu64 "[%" ROCKSDB_PRIszt
-             "] "
-             "with size %" PRIu64 " (compensated size %" PRIu64 ")",
-             file->fd.GetNumber(), sorted_run_count, file->fd.GetFileSize(),
-             file->compensated_file_size);
-  } else {
-    snprintf(out_buf, out_buf_size,
-             "level %d[%" ROCKSDB_PRIszt
-             "] "
-             "with size %" PRIu64 " (compensated size %" PRIu64 ")",
-             level, sorted_run_count, size, compensated_file_size);
-  }
-}
-
-std::vector<UniversalCompactionPicker::SortedRun>
-UniversalCompactionPicker::CalculateSortedRuns(
-    const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions) {
-  std::vector<UniversalCompactionPicker::SortedRun> ret;
-  for (FileMetaData* f : vstorage.LevelFiles(0)) {
-    ret.emplace_back(0, f, f->fd.GetFileSize(), f->compensated_file_size,
-                     f->being_compacted);
-  }
-  for (int level = 1; level < vstorage.num_levels(); level++) {
-    uint64_t total_compensated_size = 0U;
-    uint64_t total_size = 0U;
-    bool being_compacted = false;
-    bool is_first = true;
-    for (FileMetaData* f : vstorage.LevelFiles(level)) {
-      total_compensated_size += f->compensated_file_size;
-      total_size += f->fd.GetFileSize();
-      if (ioptions.compaction_options_universal.allow_trivial_move == true) {
-        if (f->being_compacted) {
-          being_compacted = f->being_compacted;
-        }
-      } else {
-        // Compaction always includes all files for a non-zero level, so for a
-        // non-zero level, all the files should share the same being_compacted
-        // value.
-        // This assumption is only valid when
-        // ioptions.compaction_options_universal.allow_trivial_move is false
-        assert(is_first || f->being_compacted == being_compacted);
-      }
-      if (is_first) {
-        being_compacted = f->being_compacted;
-        is_first = false;
-      }
-    }
-    if (total_compensated_size > 0) {
-      ret.emplace_back(level, nullptr, total_size, total_compensated_size,
-                       being_compacted);
-    }
-  }
-  return ret;
-}
-
-// Universal style of compaction. Pick files that are contiguous in
-// time-range to compact.
-//
-Compaction* UniversalCompactionPicker::PickCompaction(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  const int kLevel0 = 0;
-  double score = vstorage->CompactionScore(kLevel0);
-  std::vector<SortedRun> sorted_runs =
-      CalculateSortedRuns(*vstorage, ioptions_);
-
-  if (sorted_runs.size() == 0 ||
-      sorted_runs.size() <
-          (unsigned int)mutable_cf_options.level0_file_num_compaction_trigger) {
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: nothing to do\n",
-                     cf_name.c_str());
-    TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
-                             nullptr);
-    return nullptr;
-  }
-  VersionStorageInfo::LevelSummaryStorage tmp;
-  ROCKS_LOG_BUFFER_MAX_SZ(
-      log_buffer, 3072,
-      "[%s] Universal: sorted runs files(%" ROCKSDB_PRIszt "): %s\n",
-      cf_name.c_str(), sorted_runs.size(), vstorage->LevelSummary(&tmp));
-
-  // Check for size amplification first.
-  Compaction* c;
-  if ((c = PickCompactionToReduceSizeAmp(cf_name, mutable_cf_options, vstorage,
-                                         score, sorted_runs, log_buffer)) !=
-      nullptr) {
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: compacting for size amp\n",
-                     cf_name.c_str());
-  } else {
-    // Size amplification is within limits. Try reducing read
-    // amplification while maintaining file size ratios.
-    unsigned int ratio = ioptions_.compaction_options_universal.size_ratio;
-
-    if ((c = PickCompactionToReduceSortedRuns(
-             cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
-             sorted_runs, log_buffer)) != nullptr) {
-      ROCKS_LOG_BUFFER(log_buffer,
-                       "[%s] Universal: compacting for size ratio\n",
-                       cf_name.c_str());
-    } else {
-      // Size amplification and file size ratios are within configured limits.
-      // If max read amplification is exceeding configured limits, then force
-      // compaction without looking at filesize ratios and try to reduce
-      // the number of files to fewer than level0_file_num_compaction_trigger.
-      // This is guaranteed by NeedsCompaction()
-      assert(sorted_runs.size() >=
-             static_cast<size_t>(
-                 mutable_cf_options.level0_file_num_compaction_trigger));
-      // Get the total number of sorted runs that are not being compacted
-      int num_sr_not_compacted = 0;
-      for (size_t i = 0; i < sorted_runs.size(); i++) {
-        if (sorted_runs[i].being_compacted == false) {
-          num_sr_not_compacted++;
-        }
-      }
-
-      // The number of sorted runs that are not being compacted is greater than
-      // the maximum allowed number of sorted runs
-      if (num_sr_not_compacted >
-          mutable_cf_options.level0_file_num_compaction_trigger) {
-        unsigned int num_files =
-            num_sr_not_compacted -
-            mutable_cf_options.level0_file_num_compaction_trigger + 1;
-        if ((c = PickCompactionToReduceSortedRuns(
-                 cf_name, mutable_cf_options, vstorage, score, UINT_MAX,
-                 num_files, sorted_runs, log_buffer)) != nullptr) {
-          ROCKS_LOG_BUFFER(log_buffer,
-                           "[%s] Universal: compacting for file num -- %u\n",
-                           cf_name.c_str(), num_files);
-        }
-      }
-    }
-  }
-  if (c == nullptr) {
-    TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
-                             nullptr);
-    return nullptr;
-  }
-
-  if (ioptions_.compaction_options_universal.allow_trivial_move == true) {
-    c->set_is_trivial_move(IsInputFilesNonOverlapping(c));
-  }
-
-// validate that all the chosen files of L0 are non overlapping in time
-#ifndef NDEBUG
-  SequenceNumber prev_smallest_seqno = 0U;
-  bool is_first = true;
-
-  size_t level_index = 0U;
-  if (c->start_level() == 0) {
-    for (auto f : *c->inputs(0)) {
-      assert(f->smallest_seqno <= f->largest_seqno);
-      if (is_first) {
-        is_first = false;
-      }
-      prev_smallest_seqno = f->smallest_seqno;
-    }
-    level_index = 1U;
-  }
-  for (; level_index < c->num_input_levels(); level_index++) {
-    if (c->num_input_files(level_index) != 0) {
-      SequenceNumber smallest_seqno = 0U;
-      SequenceNumber largest_seqno = 0U;
-      GetSmallestLargestSeqno(*(c->inputs(level_index)), &smallest_seqno,
-                              &largest_seqno);
-      if (is_first) {
-        is_first = false;
-      } else if (prev_smallest_seqno > 0) {
-        // A level is considered as the bottommost level if there are
-        // no files in higher levels or if files in higher levels do
-        // not overlap with the files being compacted. Sequence numbers
-        // of files in bottommost level can be set to 0 to help
-        // compression. As a result, the following assert may not hold
-        // if the prev_smallest_seqno is 0.
-        assert(prev_smallest_seqno > largest_seqno);
-      }
-      prev_smallest_seqno = smallest_seqno;
-    }
-  }
-#endif
-  // update statistics
-  MeasureTime(ioptions_.statistics, NUM_FILES_IN_SINGLE_COMPACTION,
-              c->inputs(0)->size());
-
-  RegisterCompaction(c);
-
-  TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
-                           c);
-  return c;
-}
-
-uint32_t UniversalCompactionPicker::GetPathId(
-    const ImmutableCFOptions& ioptions, uint64_t file_size) {
-  // Two conditions need to be satisfied:
-  // (1) the target path needs to be able to hold the file's size
-  // (2) Total size left in this and previous paths need to be not
-  //     smaller than expected future file size before this new file is
-  //     compacted, which is estimated based on size_ratio.
-  // For example, if now we are compacting files of size (1, 1, 2, 4, 8),
-  // we will make sure the target file, probably with size of 16, will be
-  // placed in a path so that eventually when new files are generated and
-  // compacted to (1, 1, 2, 4, 8, 16), all those files can be stored in or
-  // before the path we chose.
-  //
-  // TODO(sdong): now the case of multiple column families is not
-  // considered in this algorithm. So the target size can be violated in
-  // that case. We need to improve it.
-  uint64_t accumulated_size = 0;
-  uint64_t future_size =
-      file_size * (100 - ioptions.compaction_options_universal.size_ratio) /
-      100;
-  uint32_t p = 0;
-  assert(!ioptions.db_paths.empty());
-  for (; p < ioptions.db_paths.size() - 1; p++) {
-    uint64_t target_size = ioptions.db_paths[p].target_size;
-    if (target_size > file_size &&
-        accumulated_size + (target_size - file_size) > future_size) {
-      return p;
-    }
-    accumulated_size += target_size;
-  }
-  return p;
-}
-
-//
-// Consider compaction files based on their size differences with
-// the next file in time order.
-//
-Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, double score, unsigned int ratio,
-    unsigned int max_number_of_files_to_compact,
-    const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
-  unsigned int min_merge_width =
-      ioptions_.compaction_options_universal.min_merge_width;
-  unsigned int max_merge_width =
-      ioptions_.compaction_options_universal.max_merge_width;
-
-  const SortedRun* sr = nullptr;
-  bool done = false;
-  size_t start_index = 0;
-  unsigned int candidate_count = 0;
-
-  unsigned int max_files_to_compact =
-      std::min(max_merge_width, max_number_of_files_to_compact);
-  min_merge_width = std::max(min_merge_width, 2U);
-
-  // Caller checks the size before executing this function. This invariant is
-  // important because otherwise we may have a possible integer underflow when
-  // dealing with unsigned types.
-  assert(sorted_runs.size() > 0);
-
-  // Considers a candidate file only if it is smaller than the
-  // total size accumulated so far.
-  for (size_t loop = 0; loop < sorted_runs.size(); loop++) {
-    candidate_count = 0;
-
-    // Skip files that are already being compacted
-    for (sr = nullptr; loop < sorted_runs.size(); loop++) {
-      sr = &sorted_runs[loop];
-
-      if (!sr->being_compacted) {
-        candidate_count = 1;
-        break;
-      }
-      char file_num_buf[kFormatFileNumberBufSize];
-      sr->Dump(file_num_buf, sizeof(file_num_buf));
-      ROCKS_LOG_BUFFER(log_buffer,
-                       "[%s] Universal: %s"
-                       "[%d] being compacted, skipping",
-                       cf_name.c_str(), file_num_buf, loop);
-
-      sr = nullptr;
-    }
-
-    // This file is not being compacted. Consider it as the
-    // first candidate to be compacted.
-    uint64_t candidate_size = sr != nullptr ? sr->compensated_file_size : 0;
-    if (sr != nullptr) {
-      char file_num_buf[kFormatFileNumberBufSize];
-      sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-      ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Possible candidate %s[%d].",
-                       cf_name.c_str(), file_num_buf, loop);
-    }
-
-    // Check if the succeeding files need compaction.
-    for (size_t i = loop + 1;
-         candidate_count < max_files_to_compact && i < sorted_runs.size();
-         i++) {
-      const SortedRun* succeeding_sr = &sorted_runs[i];
-      if (succeeding_sr->being_compacted) {
-        break;
-      }
-      // Pick files if the total/last candidate file size (increased by the
-      // specified ratio) is still larger than the next candidate file.
-      // candidate_size is the total size of files picked so far with the
-      // default kCompactionStopStyleTotalSize; with
-      // kCompactionStopStyleSimilarSize, it's simply the size of the last
-      // picked file.
-      double sz = candidate_size * (100.0 + ratio) / 100.0;
-      if (sz < static_cast<double>(succeeding_sr->size)) {
-        break;
-      }
-      if (ioptions_.compaction_options_universal.stop_style ==
-          kCompactionStopStyleSimilarSize) {
-        // Similar-size stopping rule: also check the last picked file isn't
-        // far larger than the next candidate file.
-        sz = (succeeding_sr->size * (100.0 + ratio)) / 100.0;
-        if (sz < static_cast<double>(candidate_size)) {
-          // If the small file we've encountered begins a run of similar-size
-          // files, we'll pick them up on a future iteration of the outer
-          // loop. If it's some lonely straggler, it'll eventually get picked
-          // by the last-resort read amp strategy which disregards size ratios.
-          break;
-        }
-        candidate_size = succeeding_sr->compensated_file_size;
-      } else {  // default kCompactionStopStyleTotalSize
-        candidate_size += succeeding_sr->compensated_file_size;
-      }
-      candidate_count++;
-    }
-
-    // Found a series of consecutive files that need compaction.
-    if (candidate_count >= (unsigned int)min_merge_width) {
-      start_index = loop;
-      done = true;
-      break;
-    } else {
-      for (size_t i = loop;
-           i < loop + candidate_count && i < sorted_runs.size(); i++) {
-        const SortedRun* skipping_sr = &sorted_runs[i];
-        char file_num_buf[256];
-        skipping_sr->DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop);
-        ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Skipping %s",
-                         cf_name.c_str(), file_num_buf);
-      }
-    }
-  }
-  if (!done || candidate_count <= 1) {
-    return nullptr;
-  }
-  size_t first_index_after = start_index + candidate_count;
-  // Compression is enabled if files compacted earlier already reached
-  // size ratio of compression.
-  bool enable_compression = true;
-  int ratio_to_compress =
-      ioptions_.compaction_options_universal.compression_size_percent;
-  if (ratio_to_compress >= 0) {
-    uint64_t total_size = 0;
-    for (auto& sorted_run : sorted_runs) {
-      total_size += sorted_run.compensated_file_size;
-    }
-
-    uint64_t older_file_size = 0;
-    for (size_t i = sorted_runs.size() - 1; i >= first_index_after; i--) {
-      older_file_size += sorted_runs[i].size;
-      if (older_file_size * 100L >= total_size * (long)ratio_to_compress) {
-        enable_compression = false;
-        break;
-      }
-    }
-  }
-
-  uint64_t estimated_total_size = 0;
-  for (unsigned int i = 0; i < first_index_after; i++) {
-    estimated_total_size += sorted_runs[i].size;
-  }
-  uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
-  int start_level = sorted_runs[start_index].level;
-  int output_level;
-  if (first_index_after == sorted_runs.size()) {
-    output_level = vstorage->num_levels() - 1;
-  } else if (sorted_runs[first_index_after].level == 0) {
-    output_level = 0;
-  } else {
-    output_level = sorted_runs[first_index_after].level - 1;
-  }
-
-  // last level is reserved for the files ingested behind
-  if (ioptions_.allow_ingest_behind &&
-      (output_level == vstorage->num_levels() - 1)) {
-    assert(output_level > 1);
-    output_level--;
-  }
-
-  std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
-  for (size_t i = 0; i < inputs.size(); ++i) {
-    inputs[i].level = start_level + static_cast<int>(i);
-  }
-  for (size_t i = start_index; i < first_index_after; i++) {
-    auto& picking_sr = sorted_runs[i];
-    if (picking_sr.level == 0) {
-      FileMetaData* picking_file = picking_sr.file;
-      inputs[0].files.push_back(picking_file);
-    } else {
-      auto& files = inputs[picking_sr.level - start_level].files;
-      for (auto* f : vstorage->LevelFiles(picking_sr.level)) {
-        files.push_back(f);
-      }
-    }
-    char file_num_buf[256];
-    picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), i);
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: Picking %s", cf_name.c_str(),
-                     file_num_buf);
-  }
-
-  CompactionReason compaction_reason;
-  if (max_number_of_files_to_compact == UINT_MAX) {
-    compaction_reason = CompactionReason::kUniversalSortedRunNum;
-  } else {
-    compaction_reason = CompactionReason::kUniversalSizeRatio;
-  }
-  return new Compaction(
-      vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
-      mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
-      GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
-                         1, enable_compression),
-      /* grandparents */ {}, /* is manual */ false, score,
-      false /* deletion_compaction */, compaction_reason);
-}
-
-// Look at overall size amplification. If size amplification
-// exceeeds the configured value, then do a compaction
-// of the candidate files all the way upto the earliest
-// base file (overrides configured values of file-size ratios,
-// min_merge_width and max_merge_width).
-//
-Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, double score,
-    const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
-  // percentage flexibilty while reducing size amplification
-  uint64_t ratio =
-      ioptions_.compaction_options_universal.max_size_amplification_percent;
-
-  unsigned int candidate_count = 0;
-  uint64_t candidate_size = 0;
-  size_t start_index = 0;
-  const SortedRun* sr = nullptr;
-
-  // Skip files that are already being compacted
-  for (size_t loop = 0; loop < sorted_runs.size() - 1; loop++) {
-    sr = &sorted_runs[loop];
-    if (!sr->being_compacted) {
-      start_index = loop;  // Consider this as the first candidate.
-      break;
-    }
-    char file_num_buf[kFormatFileNumberBufSize];
-    sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: skipping %s[%d] compacted %s",
-                     cf_name.c_str(), file_num_buf, loop,
-                     " cannot be a candidate to reduce size amp.\n");
-    sr = nullptr;
-  }
-
-  if (sr == nullptr) {
-    return nullptr;  // no candidate files
-  }
-  {
-    char file_num_buf[kFormatFileNumberBufSize];
-    sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-    ROCKS_LOG_BUFFER(
-        log_buffer,
-        "[%s] Universal: First candidate %s[%" ROCKSDB_PRIszt "] %s",
-        cf_name.c_str(), file_num_buf, start_index, " to reduce size amp.\n");
-  }
-
-  // keep adding up all the remaining files
-  for (size_t loop = start_index; loop < sorted_runs.size() - 1; loop++) {
-    sr = &sorted_runs[loop];
-    if (sr->being_compacted) {
-      char file_num_buf[kFormatFileNumberBufSize];
-      sr->Dump(file_num_buf, sizeof(file_num_buf), true);
-      ROCKS_LOG_BUFFER(
-          log_buffer, "[%s] Universal: Possible candidate %s[%d] %s",
-          cf_name.c_str(), file_num_buf, start_index,
-          " is already being compacted. No size amp reduction possible.\n");
-      return nullptr;
-    }
-    candidate_size += sr->compensated_file_size;
-    candidate_count++;
-  }
-  if (candidate_count == 0) {
-    return nullptr;
-  }
-
-  // size of earliest file
-  uint64_t earliest_file_size = sorted_runs.back().size;
-
-  // size amplification = percentage of additional size
-  if (candidate_size * 100 < ratio * earliest_file_size) {
-    ROCKS_LOG_BUFFER(
-        log_buffer,
-        "[%s] Universal: size amp not needed. newer-files-total-size %" PRIu64
-        " earliest-file-size %" PRIu64,
-        cf_name.c_str(), candidate_size, earliest_file_size);
-    return nullptr;
-  } else {
-    ROCKS_LOG_BUFFER(
-        log_buffer,
-        "[%s] Universal: size amp needed. newer-files-total-size %" PRIu64
-        " earliest-file-size %" PRIu64,
-        cf_name.c_str(), candidate_size, earliest_file_size);
-  }
-  assert(start_index < sorted_runs.size() - 1);
-
-  // Estimate total file size
-  uint64_t estimated_total_size = 0;
-  for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
-    estimated_total_size += sorted_runs[loop].size;
-  }
-  uint32_t path_id = GetPathId(ioptions_, estimated_total_size);
-  int start_level = sorted_runs[start_index].level;
-
-  std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
-  for (size_t i = 0; i < inputs.size(); ++i) {
-    inputs[i].level = start_level + static_cast<int>(i);
-  }
-  // We always compact all the files, so always compress.
-  for (size_t loop = start_index; loop < sorted_runs.size(); loop++) {
-    auto& picking_sr = sorted_runs[loop];
-    if (picking_sr.level == 0) {
-      FileMetaData* f = picking_sr.file;
-      inputs[0].files.push_back(f);
-    } else {
-      auto& files = inputs[picking_sr.level - start_level].files;
-      for (auto* f : vstorage->LevelFiles(picking_sr.level)) {
-        files.push_back(f);
-      }
-    }
-    char file_num_buf[256];
-    picking_sr.DumpSizeInfo(file_num_buf, sizeof(file_num_buf), loop);
-    ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: size amp picking %s",
-                     cf_name.c_str(), file_num_buf);
-  }
-
-  // output files at the bottom most level, unless it's reserved
-  int output_level = vstorage->num_levels() - 1;
-  // last level is reserved for the files ingested behind
-  if (ioptions_.allow_ingest_behind) {
-    assert(output_level > 1);
-    output_level--;
-  }
-
-  return new Compaction(
-      vstorage, ioptions_, mutable_cf_options, std::move(inputs),
-      output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
-      /* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
-      GetCompressionType(ioptions_, vstorage, mutable_cf_options,
-                         output_level, 1),
-      /* grandparents */ {}, /* is manual */ false, score,
-      false /* deletion_compaction */,
-      CompactionReason::kUniversalSizeAmplification);
-}
-}  // namespace rocksdb
-
-#endif  // !ROCKSDB_LITE

db/compaction_picker_universal.h

@@ -1,93 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-#ifndef ROCKSDB_LITE
-
-#include "db/compaction_picker.h"
-
-namespace rocksdb {
-class UniversalCompactionPicker : public CompactionPicker {
- public:
-  UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
-                            const InternalKeyComparator* icmp)
-      : CompactionPicker(ioptions, icmp) {}
-  virtual Compaction* PickCompaction(const std::string& cf_name,
-                                     const MutableCFOptions& mutable_cf_options,
-                                     VersionStorageInfo* vstorage,
-                                     LogBuffer* log_buffer) override;
-
-  virtual int MaxOutputLevel() const override { return NumberLevels() - 1; }
-
-  virtual bool NeedsCompaction(
-      const VersionStorageInfo* vstorage) const override;
-
- private:
-  struct SortedRun {
-    SortedRun(int _level, FileMetaData* _file, uint64_t _size,
-              uint64_t _compensated_file_size, bool _being_compacted)
-        : level(_level),
-          file(_file),
-          size(_size),
-          compensated_file_size(_compensated_file_size),
-          being_compacted(_being_compacted) {
-      assert(compensated_file_size > 0);
-      assert(level != 0 || file != nullptr);
-    }
-
-    void Dump(char* out_buf, size_t out_buf_size,
-              bool print_path = false) const;
-
-    // sorted_run_count is added into the string to print
-    void DumpSizeInfo(char* out_buf, size_t out_buf_size,
-                      size_t sorted_run_count) const;
-
-    int level;
-    // `file` Will be null for level > 0. For level = 0, the sorted run is
-    // for this file.
-    FileMetaData* file;
-    // For level > 0, `size` and `compensated_file_size` are sum of sizes all
-    // files in the level. `being_compacted` should be the same for all files
-    // in a non-zero level. Use the value here.
-    uint64_t size;
-    uint64_t compensated_file_size;
-    bool being_compacted;
-  };
-
-  // Pick Universal compaction to limit read amplification
-  Compaction* PickCompactionToReduceSortedRuns(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, double score, unsigned int ratio,
-      unsigned int num_files, const std::vector<SortedRun>& sorted_runs,
-      LogBuffer* log_buffer);
-
-  // Pick Universal compaction to limit space amplification.
-  Compaction* PickCompactionToReduceSizeAmp(
-      const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-      VersionStorageInfo* vstorage, double score,
-      const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer);
-
-  // Used in universal compaction when the enabled_trivial_move
-  // option is set. Checks whether there are any overlapping files
-  // in the input. Returns true if the input files are non
-  // overlapping.
-  bool IsInputFilesNonOverlapping(Compaction* c);
-
-  static std::vector<SortedRun> CalculateSortedRuns(
-      const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions);
-
-  // Pick a path ID to place a newly generated file, with its estimated file
-  // size.
-  static uint32_t GetPathId(const ImmutableCFOptions& ioptions,
-                            uint64_t file_size);
-};
-}  // namespace rocksdb
-#endif  // !ROCKSDB_LITE

db/comparator_db_test.cc

@@ -4,8 +4,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 #include <map>
 #include <string>
 

db/convenience.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2012 Facebook.
 // Use of this source code is governed by a BSD-style license that can be
@@ -18,12 +16,12 @@
 namespace rocksdb {
 
 void CancelAllBackgroundWork(DB* db, bool wait) {
-  (dynamic_cast<DBImpl*>(db->GetRootDB()))->CancelAllBackgroundWork(wait);
+  (dynamic_cast<DBImpl*>(db))->CancelAllBackgroundWork(wait);
 }
 
 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
                           const Slice* begin, const Slice* end) {
-  return (dynamic_cast<DBImpl*>(db->GetRootDB()))
+  return (dynamic_cast<DBImpl*>(db))
       ->DeleteFilesInRange(column_family, begin, end);
 }
 

db/corruption_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -13,20 +11,20 @@
 
 #include "rocksdb/db.h"
 
+#include <inttypes.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <inttypes.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include "db/db_impl.h"
-#include "db/log_format.h"
-#include "db/version_set.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/env.h"
 #include "rocksdb/table.h"
 #include "rocksdb/write_batch.h"
-#include "util/filename.h"
-#include "util/string_util.h"
+#include "db/db_impl.h"
+#include "db/filename.h"
+#include "db/log_format.h"
+#include "db/version_set.h"
+#include "util/logging.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
 
@@ -43,10 +41,7 @@ class CorruptionTest : public testing::Test {
   DB* db_;
 
   CorruptionTest() {
-    // If LRU cache shard bit is smaller than 2 (or -1 which will automatically
-    // set it to 0), test SequenceNumberRecovery will fail, likely because of a
-    // bug in recovery code. Keep it 4 for now to make the test passes.
-    tiny_cache_ = NewLRUCache(100, 4);
+    tiny_cache_ = NewLRUCache(100);
     options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
     options_.env = &env_;
     dbname_ = test::TmpDir() + "/corruption_test";

db/cuckoo_table_db_test.cc

@@ -2,18 +2,15 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #ifndef ROCKSDB_LITE
 
 #include "db/db_impl.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
+#include "table/meta_blocks.h"
 #include "table/cuckoo_table_factory.h"
 #include "table/cuckoo_table_reader.h"
-#include "table/meta_blocks.h"
-#include "util/string_util.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
 
@@ -322,14 +319,8 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
-  if (rocksdb::port::kLittleEndian) {
-    ::testing::InitGoogleTest(&argc, argv);
-    return RUN_ALL_TESTS();
-  }
-  else {
-    fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
-    return 0;
-  }
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
 }
 
 #else

db/db_basic_test.cc

@@ -1,854 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "rocksdb/perf_context.h"
-#if !defined(ROCKSDB_LITE)
-#include "util/sync_point.h"
-#endif
-
-namespace rocksdb {
-
-class DBBasicTest : public DBTestBase {
- public:
-  DBBasicTest() : DBTestBase("/db_basic_test") {}
-};
-
-TEST_F(DBBasicTest, OpenWhenOpen) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  rocksdb::DB* db2 = nullptr;
-  rocksdb::Status s = DB::Open(options, dbname_, &db2);
-
-  ASSERT_EQ(Status::Code::kIOError, s.code());
-  ASSERT_EQ(Status::SubCode::kNone, s.subcode());
-  ASSERT_TRUE(strstr(s.getState(), "lock ") != nullptr);
-
-  delete db2;
-}
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBBasicTest, ReadOnlyDB) {
-  ASSERT_OK(Put("foo", "v1"));
-  ASSERT_OK(Put("bar", "v2"));
-  ASSERT_OK(Put("foo", "v3"));
-  Close();
-
-  auto options = CurrentOptions();
-  assert(options.env = env_);
-  ASSERT_OK(ReadOnlyReopen(options));
-  ASSERT_EQ("v3", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
-  Iterator* iter = db_->NewIterator(ReadOptions());
-  int count = 0;
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
-    ASSERT_OK(iter->status());
-    ++count;
-  }
-  ASSERT_EQ(count, 2);
-  delete iter;
-  Close();
-
-  // Reopen and flush memtable.
-  Reopen(options);
-  Flush();
-  Close();
-  // Now check keys in read only mode.
-  ASSERT_OK(ReadOnlyReopen(options));
-  ASSERT_EQ("v3", Get("foo"));
-  ASSERT_EQ("v2", Get("bar"));
-  ASSERT_TRUE(db_->SyncWAL().IsNotSupported());
-}
-
-TEST_F(DBBasicTest, CompactedDB) {
-  const uint64_t kFileSize = 1 << 20;
-  Options options = CurrentOptions();
-  options.disable_auto_compactions = true;
-  options.write_buffer_size = kFileSize;
-  options.target_file_size_base = kFileSize;
-  options.max_bytes_for_level_base = 1 << 30;
-  options.compression = kNoCompression;
-  Reopen(options);
-  // 1 L0 file, use CompactedDB if max_open_files = -1
-  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
-  Flush();
-  Close();
-  ASSERT_OK(ReadOnlyReopen(options));
-  Status s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
-  Close();
-  options.max_open_files = -1;
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported in compacted db mode.");
-  ASSERT_EQ(DummyString(kFileSize / 2, '1'), Get("aaa"));
-  Close();
-  Reopen(options);
-  // Add more L0 files
-  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, '2')));
-  Flush();
-  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, 'a')));
-  Flush();
-  ASSERT_OK(Put("bbb", DummyString(kFileSize / 2, 'b')));
-  ASSERT_OK(Put("eee", DummyString(kFileSize / 2, 'e')));
-  Flush();
-  Close();
-
-  ASSERT_OK(ReadOnlyReopen(options));
-  // Fallback to read-only DB
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-  Close();
-
-  // Full compaction
-  Reopen(options);
-  // Add more keys
-  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
-  ASSERT_OK(Put("hhh", DummyString(kFileSize / 2, 'h')));
-  ASSERT_OK(Put("iii", DummyString(kFileSize / 2, 'i')));
-  ASSERT_OK(Put("jjj", DummyString(kFileSize / 2, 'j')));
-  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-  ASSERT_EQ(3, NumTableFilesAtLevel(1));
-  Close();
-
-  // CompactedDB
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported in compacted db mode.");
-  ASSERT_EQ("NOT_FOUND", Get("abc"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), Get("aaa"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'b'), Get("bbb"));
-  ASSERT_EQ("NOT_FOUND", Get("ccc"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), Get("eee"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'f'), Get("fff"));
-  ASSERT_EQ("NOT_FOUND", Get("ggg"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'h'), Get("hhh"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), Get("iii"));
-  ASSERT_EQ(DummyString(kFileSize / 2, 'j'), Get("jjj"));
-  ASSERT_EQ("NOT_FOUND", Get("kkk"));
-
-  // MultiGet
-  std::vector<std::string> values;
-  std::vector<Status> status_list = dbfull()->MultiGet(
-      ReadOptions(),
-      std::vector<Slice>({Slice("aaa"), Slice("ccc"), Slice("eee"),
-                          Slice("ggg"), Slice("iii"), Slice("kkk")}),
-      &values);
-  ASSERT_EQ(status_list.size(), static_cast<uint64_t>(6));
-  ASSERT_EQ(values.size(), static_cast<uint64_t>(6));
-  ASSERT_OK(status_list[0]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'a'), values[0]);
-  ASSERT_TRUE(status_list[1].IsNotFound());
-  ASSERT_OK(status_list[2]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'e'), values[2]);
-  ASSERT_TRUE(status_list[3].IsNotFound());
-  ASSERT_OK(status_list[4]);
-  ASSERT_EQ(DummyString(kFileSize / 2, 'i'), values[4]);
-  ASSERT_TRUE(status_list[5].IsNotFound());
-
-  Reopen(options);
-  // Add a key
-  ASSERT_OK(Put("fff", DummyString(kFileSize / 2, 'f')));
-  Close();
-  ASSERT_OK(ReadOnlyReopen(options));
-  s = Put("new", "value");
-  ASSERT_EQ(s.ToString(),
-            "Not implemented: Not supported operation in read only mode.");
-}
-
-TEST_F(DBBasicTest, LevelLimitReopen) {
-  Options options = CurrentOptions();
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  const std::string value(1024 * 1024, ' ');
-  int i = 0;
-  while (NumTableFilesAtLevel(2, 1) == 0) {
-    ASSERT_OK(Put(1, Key(i++), value));
-  }
-
-  options.num_levels = 1;
-  options.max_bytes_for_level_multiplier_additional.resize(1, 1);
-  Status s = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ(s.IsInvalidArgument(), true);
-  ASSERT_EQ(s.ToString(),
-            "Invalid argument: db has more levels than options.num_levels");
-
-  options.num_levels = 10;
-  options.max_bytes_for_level_multiplier_additional.resize(10, 1);
-  ASSERT_OK(TryReopenWithColumnFamilies({"default", "pikachu"}, options));
-}
-#endif  // ROCKSDB_LITE
-
-TEST_F(DBBasicTest, PutDeleteGet) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_OK(Put(1, "foo", "v2"));
-    ASSERT_EQ("v2", Get(1, "foo"));
-    ASSERT_OK(Delete(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-  } while (ChangeOptions());
-}
-
-TEST_F(DBBasicTest, PutSingleDeleteGet) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_OK(Put(1, "foo2", "v2"));
-    ASSERT_EQ("v2", Get(1, "foo2"));
-    ASSERT_OK(SingleDelete(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because single delete does not get removed when it encounters a merge.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
-TEST_F(DBBasicTest, EmptyFlush) {
-  // It is possible to produce empty flushes when using single deletes. Tests
-  // whether empty flushes cause issues.
-  do {
-    Random rnd(301);
-
-    Options options = CurrentOptions();
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "a", Slice());
-    SingleDelete(1, "a");
-    ASSERT_OK(Flush(1));
-
-    ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
-    // Skip HashCuckooRep as it does not support single delete. FIFO and
-    // universal compaction do not apply to the test case. Skip MergePut
-    // because merges cannot be combined with single deletions.
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
-                         kSkipUniversalCompaction | kSkipMergePut));
-}
-
-TEST_F(DBBasicTest, GetFromVersions) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "foo", "v1"));
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
-  } while (ChangeOptions());
-}
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBBasicTest, GetSnapshot) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
-    // Try with both a short key and a long key
-    for (int i = 0; i < 2; i++) {
-      std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
-      ASSERT_OK(Put(1, key, "v1"));
-      const Snapshot* s1 = db_->GetSnapshot();
-      if (option_config_ == kHashCuckoo) {
-        // Unsupported case.
-        ASSERT_TRUE(s1 == nullptr);
-        break;
-      }
-      ASSERT_OK(Put(1, key, "v2"));
-      ASSERT_EQ("v2", Get(1, key));
-      ASSERT_EQ("v1", Get(1, key, s1));
-      ASSERT_OK(Flush(1));
-      ASSERT_EQ("v2", Get(1, key));
-      ASSERT_EQ("v1", Get(1, key, s1));
-      db_->ReleaseSnapshot(s1);
-    }
-  } while (ChangeOptions());
-}
-#endif  // ROCKSDB_LITE
-
-TEST_F(DBBasicTest, CheckLock) {
-  do {
-    DB* localdb;
-    Options options = CurrentOptions();
-    ASSERT_OK(TryReopen(options));
-
-    // second open should fail
-    ASSERT_TRUE(!(DB::Open(options, dbname_, &localdb)).ok());
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, FlushMultipleMemtable) {
-  do {
-    Options options = CurrentOptions();
-    WriteOptions writeOpt = WriteOptions();
-    writeOpt.disableWAL = true;
-    options.max_write_buffer_number = 4;
-    options.min_write_buffer_number_to_merge = 3;
-    options.max_write_buffer_number_to_maintain = -1;
-    CreateAndReopenWithCF({"pikachu"}, options);
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
-    ASSERT_OK(Flush(1));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
-
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_EQ("v1", Get(1, "bar"));
-    ASSERT_OK(Flush(1));
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, FlushEmptyColumnFamily) {
-  // Block flush thread and disable compaction thread
-  env_->SetBackgroundThreads(1, Env::HIGH);
-  env_->SetBackgroundThreads(1, Env::LOW);
-  test::SleepingBackgroundTask sleeping_task_low;
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
-                 Env::Priority::LOW);
-  test::SleepingBackgroundTask sleeping_task_high;
-  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
-                 &sleeping_task_high, Env::Priority::HIGH);
-
-  Options options = CurrentOptions();
-  // disable compaction
-  options.disable_auto_compactions = true;
-  WriteOptions writeOpt = WriteOptions();
-  writeOpt.disableWAL = true;
-  options.max_write_buffer_number = 2;
-  options.min_write_buffer_number_to_merge = 1;
-  options.max_write_buffer_number_to_maintain = 1;
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  // Compaction can still go through even if no thread can flush the
-  // mem table.
-  ASSERT_OK(Flush(0));
-  ASSERT_OK(Flush(1));
-
-  // Insert can go through
-  ASSERT_OK(dbfull()->Put(writeOpt, handles_[0], "foo", "v1"));
-  ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
-
-  ASSERT_EQ("v1", Get(0, "foo"));
-  ASSERT_EQ("v1", Get(1, "bar"));
-
-  sleeping_task_high.WakeUp();
-  sleeping_task_high.WaitUntilDone();
-
-  // Flush can still go through.
-  ASSERT_OK(Flush(0));
-  ASSERT_OK(Flush(1));
-
-  sleeping_task_low.WakeUp();
-  sleeping_task_low.WaitUntilDone();
-}
-
-TEST_F(DBBasicTest, FLUSH) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    WriteOptions writeOpt = WriteOptions();
-    writeOpt.disableWAL = true;
-    SetPerfLevel(kEnableTime);
-    ;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v1"));
-    // this will now also flush the last 2 writes
-    ASSERT_OK(Flush(1));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
-
-    perf_context.Reset();
-    Get(1, "foo");
-    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    ASSERT_EQ("v1", Get(1, "foo"));
-    ASSERT_EQ("v1", Get(1, "bar"));
-
-    writeOpt.disableWAL = true;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v2"));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v2"));
-    ASSERT_OK(Flush(1));
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    ASSERT_EQ("v2", Get(1, "bar"));
-    perf_context.Reset();
-    ASSERT_EQ("v2", Get(1, "foo"));
-    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
-
-    writeOpt.disableWAL = false;
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));
-    ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "foo", "v3"));
-    ASSERT_OK(Flush(1));
-
-    ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
-    // 'foo' should be there because its put
-    // has WAL enabled.
-    ASSERT_EQ("v3", Get(1, "foo"));
-    ASSERT_EQ("v3", Get(1, "bar"));
-
-    SetPerfLevel(kDisable);
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, ManifestRollOver) {
-  do {
-    Options options;
-    options.max_manifest_file_size = 10;  // 10 bytes
-    options = CurrentOptions(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-    {
-      ASSERT_OK(Put(1, "manifest_key1", std::string(1000, '1')));
-      ASSERT_OK(Put(1, "manifest_key2", std::string(1000, '2')));
-      ASSERT_OK(Put(1, "manifest_key3", std::string(1000, '3')));
-      uint64_t manifest_before_flush = dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_OK(Flush(1));  // This should trigger LogAndApply.
-      uint64_t manifest_after_flush = dbfull()->TEST_Current_Manifest_FileNo();
-      ASSERT_GT(manifest_after_flush, manifest_before_flush);
-      ReopenWithColumnFamilies({"default", "pikachu"}, options);
-      ASSERT_GT(dbfull()->TEST_Current_Manifest_FileNo(), manifest_after_flush);
-      // check if a new manifest file got inserted or not.
-      ASSERT_EQ(std::string(1000, '1'), Get(1, "manifest_key1"));
-      ASSERT_EQ(std::string(1000, '2'), Get(1, "manifest_key2"));
-      ASSERT_EQ(std::string(1000, '3'), Get(1, "manifest_key3"));
-    }
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, IdentityAcrossRestarts) {
-  do {
-    std::string id1;
-    ASSERT_OK(db_->GetDbIdentity(id1));
-
-    Options options = CurrentOptions();
-    Reopen(options);
-    std::string id2;
-    ASSERT_OK(db_->GetDbIdentity(id2));
-    // id1 should match id2 because identity was not regenerated
-    ASSERT_EQ(id1.compare(id2), 0);
-
-    std::string idfilename = IdentityFileName(dbname_);
-    ASSERT_OK(env_->DeleteFile(idfilename));
-    Reopen(options);
-    std::string id3;
-    ASSERT_OK(db_->GetDbIdentity(id3));
-    // id1 should NOT match id3 because identity was regenerated
-    ASSERT_NE(id1.compare(id3), 0);
-  } while (ChangeCompactOptions());
-}
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBBasicTest, Snapshot) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions(options_override));
-    Put(0, "foo", "0v1");
-    Put(1, "foo", "1v1");
-
-    const Snapshot* s1 = db_->GetSnapshot();
-    ASSERT_EQ(1U, GetNumSnapshots());
-    uint64_t time_snap1 = GetTimeOldestSnapshots();
-    ASSERT_GT(time_snap1, 0U);
-    Put(0, "foo", "0v2");
-    Put(1, "foo", "1v2");
-
-    env_->addon_time_.fetch_add(1);
-
-    const Snapshot* s2 = db_->GetSnapshot();
-    ASSERT_EQ(2U, GetNumSnapshots());
-    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
-    Put(0, "foo", "0v3");
-    Put(1, "foo", "1v3");
-
-    {
-      ManagedSnapshot s3(db_);
-      ASSERT_EQ(3U, GetNumSnapshots());
-      ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
-
-      Put(0, "foo", "0v4");
-      Put(1, "foo", "1v4");
-      ASSERT_EQ("0v1", Get(0, "foo", s1));
-      ASSERT_EQ("1v1", Get(1, "foo", s1));
-      ASSERT_EQ("0v2", Get(0, "foo", s2));
-      ASSERT_EQ("1v2", Get(1, "foo", s2));
-      ASSERT_EQ("0v3", Get(0, "foo", s3.snapshot()));
-      ASSERT_EQ("1v3", Get(1, "foo", s3.snapshot()));
-      ASSERT_EQ("0v4", Get(0, "foo"));
-      ASSERT_EQ("1v4", Get(1, "foo"));
-    }
-
-    ASSERT_EQ(2U, GetNumSnapshots());
-    ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
-    ASSERT_EQ("0v1", Get(0, "foo", s1));
-    ASSERT_EQ("1v1", Get(1, "foo", s1));
-    ASSERT_EQ("0v2", Get(0, "foo", s2));
-    ASSERT_EQ("1v2", Get(1, "foo", s2));
-    ASSERT_EQ("0v4", Get(0, "foo"));
-    ASSERT_EQ("1v4", Get(1, "foo"));
-
-    db_->ReleaseSnapshot(s1);
-    ASSERT_EQ("0v2", Get(0, "foo", s2));
-    ASSERT_EQ("1v2", Get(1, "foo", s2));
-    ASSERT_EQ("0v4", Get(0, "foo"));
-    ASSERT_EQ("1v4", Get(1, "foo"));
-    ASSERT_EQ(1U, GetNumSnapshots());
-    ASSERT_LT(time_snap1, GetTimeOldestSnapshots());
-
-    db_->ReleaseSnapshot(s2);
-    ASSERT_EQ(0U, GetNumSnapshots());
-    ASSERT_EQ("0v4", Get(0, "foo"));
-    ASSERT_EQ("1v4", Get(1, "foo"));
-  } while (ChangeOptions(kSkipHashCuckoo));
-}
-
-#endif  // ROCKSDB_LITE
-
-TEST_F(DBBasicTest, CompactBetweenSnapshots) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    Options options = CurrentOptions(options_override);
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-    Random rnd(301);
-    FillLevels("a", "z", 1);
-
-    Put(1, "foo", "first");
-    const Snapshot* snapshot1 = db_->GetSnapshot();
-    Put(1, "foo", "second");
-    Put(1, "foo", "third");
-    Put(1, "foo", "fourth");
-    const Snapshot* snapshot2 = db_->GetSnapshot();
-    Put(1, "foo", "fifth");
-    Put(1, "foo", "sixth");
-
-    // All entries (including duplicates) exist
-    // before any compaction or flush is triggered.
-    ASSERT_EQ(AllEntriesFor("foo", 1),
-              "[ sixth, fifth, fourth, third, second, first ]");
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
-    ASSERT_EQ("first", Get(1, "foo", snapshot1));
-
-    // After a flush, "second", "third" and "fifth" should
-    // be removed
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth, first ]");
-
-    // after we release the snapshot1, only two values left
-    db_->ReleaseSnapshot(snapshot1);
-    FillLevels("a", "z", 1);
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-
-    // We have only one valid snapshot snapshot2. Since snapshot1 is
-    // not valid anymore, "first" should be removed by a compaction.
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ("fourth", Get(1, "foo", snapshot2));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth, fourth ]");
-
-    // after we release the snapshot2, only one value should be left
-    db_->ReleaseSnapshot(snapshot2);
-    FillLevels("a", "z", 1);
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ("sixth", Get(1, "foo"));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]");
-    // skip HashCuckooRep as it does not support snapshot
-  } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction));
-}
-
-TEST_F(DBBasicTest, DBOpen_Options) {
-  Options options = CurrentOptions();
-  std::string dbname = test::TmpDir(env_) + "/db_options_test";
-  ASSERT_OK(DestroyDB(dbname, options));
-
-  // Does not exist, and create_if_missing == false: error
-  DB* db = nullptr;
-  options.create_if_missing = false;
-  Status s = DB::Open(options, dbname, &db);
-  ASSERT_TRUE(strstr(s.ToString().c_str(), "does not exist") != nullptr);
-  ASSERT_TRUE(db == nullptr);
-
-  // Does not exist, and create_if_missing == true: OK
-  options.create_if_missing = true;
-  s = DB::Open(options, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-
-  delete db;
-  db = nullptr;
-
-  // Does exist, and error_if_exists == true: error
-  options.create_if_missing = false;
-  options.error_if_exists = true;
-  s = DB::Open(options, dbname, &db);
-  ASSERT_TRUE(strstr(s.ToString().c_str(), "exists") != nullptr);
-  ASSERT_TRUE(db == nullptr);
-
-  // Does exist, and error_if_exists == false: OK
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  s = DB::Open(options, dbname, &db);
-  ASSERT_OK(s);
-  ASSERT_TRUE(db != nullptr);
-
-  delete db;
-  db = nullptr;
-}
-
-TEST_F(DBBasicTest, CompactOnFlush) {
-  anon::OptionsOverride options_override;
-  options_override.skip_policy = kSkipNoSnapshot;
-  do {
-    Options options = CurrentOptions(options_override);
-    options.disable_auto_compactions = true;
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    Put(1, "foo", "v1");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v1 ]");
-
-    // Write two new keys
-    Put(1, "a", "begin");
-    Put(1, "z", "end");
-    Flush(1);
-
-    // Case1: Delete followed by a put
-    Delete(1, "foo");
-    Put(1, "foo", "v2");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, DEL, v1 ]");
-
-    // After the current memtable is flushed, the DEL should
-    // have been removed
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2, v1 ]");
-
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v2 ]");
-
-    // Case 2: Delete followed by another delete
-    Delete(1, "foo");
-    Delete(1, "foo");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, DEL, v2 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v2 ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 3: Put followed by a delete
-    Put(1, "foo", "v3");
-    Delete(1, "foo");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL, v3 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ DEL ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 4: Put followed by another Put
-    Put(1, "foo", "v4");
-    Put(1, "foo", "v5");
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5, v4 ]");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v5 ]");
-
-    // clear database
-    Delete(1, "foo");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 5: Put followed by snapshot followed by another Put
-    // Both puts should remain.
-    Put(1, "foo", "v6");
-    const Snapshot* snapshot = db_->GetSnapshot();
-    Put(1, "foo", "v7");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v7, v6 ]");
-    db_->ReleaseSnapshot(snapshot);
-
-    // clear database
-    Delete(1, "foo");
-    dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr,
-                           nullptr);
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ ]");
-
-    // Case 5: snapshot followed by a put followed by another Put
-    // Only the last put should remain.
-    const Snapshot* snapshot1 = db_->GetSnapshot();
-    Put(1, "foo", "v8");
-    Put(1, "foo", "v9");
-    ASSERT_OK(Flush(1));
-    ASSERT_EQ(AllEntriesFor("foo", 1), "[ v9 ]");
-    db_->ReleaseSnapshot(snapshot1);
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, FlushOneColumnFamily) {
-  Options options = CurrentOptions();
-  CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich",
-                         "alyosha", "popovich"},
-                        options);
-
-  ASSERT_OK(Put(0, "Default", "Default"));
-  ASSERT_OK(Put(1, "pikachu", "pikachu"));
-  ASSERT_OK(Put(2, "ilya", "ilya"));
-  ASSERT_OK(Put(3, "muromec", "muromec"));
-  ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
-  ASSERT_OK(Put(5, "nikitich", "nikitich"));
-  ASSERT_OK(Put(6, "alyosha", "alyosha"));
-  ASSERT_OK(Put(7, "popovich", "popovich"));
-
-  for (int i = 0; i < 8; ++i) {
-    Flush(i);
-    auto tables = ListTableFiles(env_, dbname_);
-    ASSERT_EQ(tables.size(), i + 1U);
-  }
-}
-
-TEST_F(DBBasicTest, MultiGetSimple) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    ASSERT_OK(Put(1, "k1", "v1"));
-    ASSERT_OK(Put(1, "k2", "v2"));
-    ASSERT_OK(Put(1, "k3", "v3"));
-    ASSERT_OK(Put(1, "k4", "v4"));
-    ASSERT_OK(Delete(1, "k4"));
-    ASSERT_OK(Put(1, "k5", "v5"));
-    ASSERT_OK(Delete(1, "no_key"));
-
-    std::vector<Slice> keys({"k1", "k2", "k3", "k4", "k5", "no_key"});
-
-    std::vector<std::string> values(20, "Temporary data to be overwritten");
-    std::vector<ColumnFamilyHandle*> cfs(keys.size(), handles_[1]);
-
-    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(values.size(), keys.size());
-    ASSERT_EQ(values[0], "v1");
-    ASSERT_EQ(values[1], "v2");
-    ASSERT_EQ(values[2], "v3");
-    ASSERT_EQ(values[4], "v5");
-
-    ASSERT_OK(s[0]);
-    ASSERT_OK(s[1]);
-    ASSERT_OK(s[2]);
-    ASSERT_TRUE(s[3].IsNotFound());
-    ASSERT_OK(s[4]);
-    ASSERT_TRUE(s[5].IsNotFound());
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, MultiGetEmpty) {
-  do {
-    CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-    // Empty Key Set
-    std::vector<Slice> keys;
-    std::vector<std::string> values;
-    std::vector<ColumnFamilyHandle*> cfs;
-    std::vector<Status> s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(s.size(), 0U);
-
-    // Empty Database, Empty Key Set
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    DestroyAndReopen(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(s.size(), 0U);
-
-    // Empty Database, Search for Keys
-    keys.resize(2);
-    keys[0] = "a";
-    keys[1] = "b";
-    cfs.push_back(handles_[0]);
-    cfs.push_back(handles_[1]);
-    s = db_->MultiGet(ReadOptions(), cfs, keys, &values);
-    ASSERT_EQ(static_cast<int>(s.size()), 2);
-    ASSERT_TRUE(s[0].IsNotFound() && s[1].IsNotFound());
-  } while (ChangeCompactOptions());
-}
-
-TEST_F(DBBasicTest, ChecksumTest) {
-  BlockBasedTableOptions table_options;
-  Options options = CurrentOptions();
-
-  table_options.checksum = kCRC32c;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_OK(Put("a", "b"));
-  ASSERT_OK(Put("c", "d"));
-  ASSERT_OK(Flush());  // table with crc checksum
-
-  table_options.checksum = kxxHash;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_OK(Put("e", "f"));
-  ASSERT_OK(Put("g", "h"));
-  ASSERT_OK(Flush());  // table with xxhash checksum
-
-  table_options.checksum = kCRC32c;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_EQ("b", Get("a"));
-  ASSERT_EQ("d", Get("c"));
-  ASSERT_EQ("f", Get("e"));
-  ASSERT_EQ("h", Get("g"));
-
-  table_options.checksum = kCRC32c;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  Reopen(options);
-  ASSERT_EQ("b", Get("a"));
-  ASSERT_EQ("d", Get("c"));
-  ASSERT_EQ("f", Get("e"));
-  ASSERT_EQ("h", Get("g"));
-}
-
-// On Windows you can have either memory mapped file or a file
-// with unbuffered access. So this asserts and does not make
-// sense to run
-#ifndef OS_WIN
-TEST_F(DBBasicTest, MmapAndBufferOptions) {
-  Options options = CurrentOptions();
-
-  options.use_direct_reads = true;
-  options.allow_mmap_reads = true;
-  ASSERT_NOK(TryReopen(options));
-
-  // All other combinations are acceptable
-  options.use_direct_reads = false;
-  ASSERT_OK(TryReopen(options));
-
-  if (IsDirectIOSupported()) {
-    options.use_direct_reads = true;
-    options.allow_mmap_reads = false;
-    ASSERT_OK(TryReopen(options));
-  }
-
-  options.use_direct_reads = false;
-  ASSERT_OK(TryReopen(options));
-}
-#endif
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/db_block_cache_test.cc

@@ -2,16 +2,14 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #include <cstdlib>
-#include "cache/lru_cache.h"
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
+#include "util/lru_cache.h"
 
 namespace rocksdb {
 

db/db_bloom_filter_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -22,23 +20,17 @@ class DBBloomFilterTest : public DBTestBase {
   DBBloomFilterTest() : DBTestBase("/db_bloom_filter_test") {}
 };
 
-class DBBloomFilterTestWithParam
-    : public DBTestBase,
-      public testing::WithParamInterface<std::tuple<bool, bool>> {
-  //                             public testing::WithParamInterface<bool> {
+class DBBloomFilterTestWithParam : public DBTestBase,
+                                   public testing::WithParamInterface<bool> {
  protected:
   bool use_block_based_filter_;
-  bool partition_filters_;
 
  public:
   DBBloomFilterTestWithParam() : DBTestBase("/db_bloom_filter_tests") {}
 
   ~DBBloomFilterTestWithParam() {}
 
-  void SetUp() override {
-    use_block_based_filter_ = std::get<0>(GetParam());
-    partition_filters_ = std::get<1>(GetParam());
-  }
+  void SetUp() override { use_block_based_filter_ = GetParam(); }
 };
 
 // KeyMayExist can lead to a few false positives, but not false negatives.
@@ -51,17 +43,7 @@ TEST_P(DBBloomFilterTestWithParam, KeyMayExist) {
     anon::OptionsOverride options_override;
     options_override.filter_policy.reset(
         NewBloomFilterPolicy(20, use_block_based_filter_));
-    options_override.partition_filters = partition_filters_;
-    options_override.metadata_block_size = 32;
     Options options = CurrentOptions(options_override);
-    if (partition_filters_ &&
-        static_cast<BlockBasedTableOptions*>(
-            options.table_factory->GetOptions())
-                ->index_type != BlockBasedTableOptions::kTwoLevelIndexSearch) {
-      // In the current implementation partitioned filters depend on partitioned
-      // indexes
-      continue;
-    }
     options.statistics = rocksdb::CreateDBStatistics();
     CreateAndReopenWithCF({"pikachu"}, options);
 
@@ -120,204 +102,192 @@ TEST_P(DBBloomFilterTestWithParam, KeyMayExist) {
 }
 
 TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) {
-  for (bool partition_filters : {true, false}) {
-    Options options = last_options_;
-    options.prefix_extractor.reset(NewFixedPrefixTransform(8));
-    options.statistics = rocksdb::CreateDBStatistics();
-    BlockBasedTableOptions bbto;
-    bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
-    if (partition_filters) {
-      bbto.partition_filters = true;
-      bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-    }
-    bbto.whole_key_filtering = false;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    DestroyAndReopen(options);
+  Options options = last_options_;
+  options.prefix_extractor.reset(NewFixedPrefixTransform(8));
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions bbto;
+  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(options);
 
-    WriteOptions wo;
-    ReadOptions ro;
-    FlushOptions fo;
-    fo.wait = true;
-    std::string value;
+  WriteOptions wo;
+  ReadOptions ro;
+  FlushOptions fo;
+  fo.wait = true;
+  std::string value;
 
-    ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
-    ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
-    ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
+  ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
+  ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
+  ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
 
-    dbfull()->Flush(fo);
+  dbfull()->Flush(fo);
 
-    ASSERT_EQ("foo", Get("barbarbar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
-    ASSERT_EQ("foo2", Get("barbarbar2"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
-    ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  ASSERT_EQ("foo", Get("barbarbar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  ASSERT_EQ("foo2", Get("barbarbar2"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
 
-    ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
 
-    ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
+  ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
 
-    ro.total_order_seek = true;
-    ASSERT_TRUE(db_->Get(ro, "foobarbar", &value).IsNotFound());
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
-  }
+  ro.total_order_seek = true;
+  ASSERT_TRUE(db_->Get(ro, "foobarbar", &value).IsNotFound());
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
 }
 
 TEST_F(DBBloomFilterTest, WholeKeyFilterProp) {
-  for (bool partition_filters : {true, false}) {
-    Options options = last_options_;
-    options.prefix_extractor.reset(NewFixedPrefixTransform(3));
-    options.statistics = rocksdb::CreateDBStatistics();
+  Options options = last_options_;
+  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
+  options.statistics = rocksdb::CreateDBStatistics();
 
-    BlockBasedTableOptions bbto;
-    bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
-    bbto.whole_key_filtering = false;
-    if (partition_filters) {
-      bbto.partition_filters = true;
-      bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-    }
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    DestroyAndReopen(options);
+  BlockBasedTableOptions bbto;
+  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(options);
 
-    WriteOptions wo;
-    ReadOptions ro;
-    FlushOptions fo;
-    fo.wait = true;
-    std::string value;
+  WriteOptions wo;
+  ReadOptions ro;
+  FlushOptions fo;
+  fo.wait = true;
+  std::string value;
 
-    ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
-    // Needs insert some keys to make sure files are not filtered out by key
-    // ranges.
-    ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
-    ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
-    dbfull()->Flush(fo);
+  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
+  // Needs insert some keys to make sure files are not filtered out by key
+  // ranges.
+  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
+  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
+  dbfull()->Flush(fo);
 
-    Reopen(options);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  Reopen(options);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
 
-    // Reopen with whole key filtering enabled and prefix extractor
-    // NULL. Bloom filter should be off for both of whole key and
-    // prefix bloom.
-    bbto.whole_key_filtering = true;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    options.prefix_extractor.reset();
-    Reopen(options);
+  // Reopen with whole key filtering enabled and prefix extractor
+  // NULL. Bloom filter should be off for both of whole key and
+  // prefix bloom.
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  options.prefix_extractor.reset();
+  Reopen(options);
 
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    // Write DB with only full key filtering.
-    ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
-    // Needs insert some keys to make sure files are not filtered out by key
-    // ranges.
-    ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
-    ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
-    db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  // Write DB with only full key filtering.
+  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
+  // Needs insert some keys to make sure files are not filtered out by key
+  // ranges.
+  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
+  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
+  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
 
-    // Reopen with both of whole key off and prefix extractor enabled.
-    // Still no bloom filter should be used.
-    options.prefix_extractor.reset(NewFixedPrefixTransform(3));
-    bbto.whole_key_filtering = false;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    Reopen(options);
+  // Reopen with both of whole key off and prefix extractor enabled.
+  // Still no bloom filter should be used.
+  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
 
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
 
-    // Try to create a DB with mixed files:
-    ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
-    // Needs insert some keys to make sure files are not filtered out by key
-    // ranges.
-    ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
-    ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
-    db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
+  // Try to create a DB with mixed files:
+  ASSERT_OK(dbfull()->Put(wo, "foobar", "foo"));
+  // Needs insert some keys to make sure files are not filtered out by key
+  // ranges.
+  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
+  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
+  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
 
-    options.prefix_extractor.reset();
-    bbto.whole_key_filtering = true;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    Reopen(options);
+  options.prefix_extractor.reset();
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
 
-    // Try to create a DB with mixed files.
-    ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
-    // In this case needs insert some keys to make sure files are
-    // not filtered out by key ranges.
-    ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
-    ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
-    Flush();
+  // Try to create a DB with mixed files.
+  ASSERT_OK(dbfull()->Put(wo, "barfoo", "bar"));
+  // In this case needs insert some keys to make sure files are
+  // not filtered out by key ranges.
+  ASSERT_OK(dbfull()->Put(wo, "aaa", ""));
+  ASSERT_OK(dbfull()->Put(wo, "zzz", ""));
+  Flush();
 
-    // Now we have two files:
-    // File 1: An older file with prefix bloom.
-    // File 2: A newer file with whole bloom filter.
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 3);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
-    ASSERT_EQ("bar", Get("barfoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
+  // Now we have two files:
+  // File 1: An older file with prefix bloom.
+  // File 2: A newer file with whole bloom filter.
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 3);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
+  ASSERT_EQ("bar", Get("barfoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
 
-    // Reopen with the same setting: only whole key is used
-    Reopen(options);
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 5);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 6);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
-    ASSERT_EQ("bar", Get("barfoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
+  // Reopen with the same setting: only whole key is used
+  Reopen(options);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 4);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 5);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 6);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
+  ASSERT_EQ("bar", Get("barfoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
 
-    // Restart with both filters are allowed
-    options.prefix_extractor.reset(NewFixedPrefixTransform(3));
-    bbto.whole_key_filtering = true;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    Reopen(options);
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
-    // File 1 will has it filtered out.
-    // File 2 will not, as prefix `foo` exists in the file.
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 8);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 10);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
-    ASSERT_EQ("bar", Get("barfoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
+  // Restart with both filters are allowed
+  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 7);
+  // File 1 will has it filtered out.
+  // File 2 will not, as prefix `foo` exists in the file.
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 8);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 10);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
+  ASSERT_EQ("bar", Get("barfoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
 
-    // Restart with only prefix bloom is allowed.
-    options.prefix_extractor.reset(NewFixedPrefixTransform(3));
-    bbto.whole_key_filtering = false;
-    options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-    Reopen(options);
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
-    ASSERT_EQ("NOT_FOUND", Get("foo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
-    ASSERT_EQ("NOT_FOUND", Get("bar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
-    ASSERT_EQ("foo", Get("foobar"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
-    ASSERT_EQ("bar", Get("barfoo"));
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
-  }
+  // Restart with only prefix bloom is allowed.
+  options.prefix_extractor.reset(NewFixedPrefixTransform(3));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  Reopen(options);
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
+  ASSERT_EQ("NOT_FOUND", Get("foo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 11);
+  ASSERT_EQ("NOT_FOUND", Get("bar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
+  ASSERT_EQ("foo", Get("foobar"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
+  ASSERT_EQ("bar", Get("barfoo"));
+  ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
 }
 
 TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
@@ -331,12 +301,6 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
     table_options.no_block_cache = true;
     table_options.filter_policy.reset(
         NewBloomFilterPolicy(10, use_block_based_filter_));
-    table_options.partition_filters = partition_filters_;
-    if (partition_filters_) {
-      table_options.index_type =
-          BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-    }
-    table_options.metadata_block_size = 32;
     options.table_factory.reset(NewBlockBasedTableFactory(table_options));
 
     CreateAndReopenWithCF({"pikachu"}, options);
@@ -363,13 +327,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
     int reads = env_->random_read_counter_.Read();
     fprintf(stderr, "%d present => %d reads\n", N, reads);
     ASSERT_GE(reads, N);
-    if (partition_filters_) {
-      // Without block cache, we read an extra partition filter per each
-      // level*read and a partition index per each read
-      ASSERT_LE(reads, 4 * N + 2 * N / 100);
-    } else {
-      ASSERT_LE(reads, N + 2 * N / 100);
-    }
+    ASSERT_LE(reads, N + 2 * N / 100);
 
     // Lookup present keys.  Should rarely read from either sstable.
     env_->random_read_counter_.Reset();
@@ -378,13 +336,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
     }
     reads = env_->random_read_counter_.Read();
     fprintf(stderr, "%d missing => %d reads\n", N, reads);
-    if (partition_filters_) {
-      // With partitioned filter we read one extra filter per level per each
-      // missed read.
-      ASSERT_LE(reads, 2 * N + 3 * N / 100);
-    } else {
-      ASSERT_LE(reads, 3 * N / 100);
-    }
+    ASSERT_LE(reads, 3 * N / 100);
 
     env_->delay_sstable_sync_.store(false, std::memory_order_release);
     Close();
@@ -392,9 +344,7 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
 }
 
 INSTANTIATE_TEST_CASE_P(DBBloomFilterTestWithParam, DBBloomFilterTestWithParam,
-                        ::testing::Values(std::make_tuple(true, false),
-                                          std::make_tuple(false, true),
-                                          std::make_tuple(false, false)));
+                        ::testing::Bool());
 
 TEST_F(DBBloomFilterTest, BloomFilterRate) {
   while (ChangeFilterOptions()) {
@@ -451,13 +401,28 @@ TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
     ASSERT_EQ(Key(i), Get(1, Key(i)));
   }
   ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+}
 
-  // Check db with partitioned full filter
-  table_options.partition_filters = true;
-  table_options.index_type =
-      BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
+TEST_F(DBBloomFilterTest, BloomFilterReverseCompatibility) {
+  Options options = CurrentOptions();
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
   table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
   options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+
+  // Create with full filter
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  const int maxKey = 10000;
+  for (int i = 0; i < maxKey; i++) {
+    ASSERT_OK(Put(1, Key(i), Key(i)));
+  }
+  ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
+  Flush(1);
+
+  // Check db with block_based filter
+  table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
   ReopenWithColumnFamilies({"default", "pikachu"}, options);
 
   // Check if they can be found
@@ -467,43 +432,6 @@ TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
   ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
 }
 
-TEST_F(DBBloomFilterTest, BloomFilterReverseCompatibility) {
-  for (bool partition_filters : {true, false}) {
-    Options options = CurrentOptions();
-    options.statistics = rocksdb::CreateDBStatistics();
-    BlockBasedTableOptions table_options;
-    if (partition_filters) {
-      table_options.partition_filters = true;
-      table_options.index_type =
-          BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-    }
-    table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    DestroyAndReopen(options);
-
-    // Create with full filter
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    const int maxKey = 10000;
-    for (int i = 0; i < maxKey; i++) {
-      ASSERT_OK(Put(1, Key(i), Key(i)));
-    }
-    ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
-    Flush(1);
-
-    // Check db with block_based filter
-    table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    ReopenWithColumnFamilies({"default", "pikachu"}, options);
-
-    // Check if they can be found
-    for (int i = 0; i < maxKey; i++) {
-      ASSERT_EQ(Key(i), Get(1, Key(i)));
-    }
-    ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
-  }
-}
-
 namespace {
 // A wrapped bloom over default FilterPolicy
 class WrappedBloom : public FilterPolicy {
@@ -658,12 +586,11 @@ TEST_F(DBBloomFilterTest, PrefixExtractorBlockFilter) {
 #ifndef ROCKSDB_LITE
 class BloomStatsTestWithParam
     : public DBBloomFilterTest,
-      public testing::WithParamInterface<std::tuple<bool, bool, bool>> {
+      public testing::WithParamInterface<std::tuple<bool, bool>> {
  public:
   BloomStatsTestWithParam() {
     use_block_table_ = std::get<0>(GetParam());
     use_block_based_builder_ = std::get<1>(GetParam());
-    partition_filters_ = std::get<2>(GetParam());
 
     options_.create_if_missing = true;
     options_.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(4));
@@ -672,21 +599,13 @@ class BloomStatsTestWithParam
     if (use_block_table_) {
       BlockBasedTableOptions table_options;
       table_options.hash_index_allow_collision = false;
-      if (partition_filters_) {
-        assert(!use_block_based_builder_);
-        table_options.partition_filters = partition_filters_;
-        table_options.index_type =
-            BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
-      }
       table_options.filter_policy.reset(
           NewBloomFilterPolicy(10, use_block_based_builder_));
       options_.table_factory.reset(NewBlockBasedTableFactory(table_options));
     } else {
-      assert(!partition_filters_);  // not supported in plain table
       PlainTableOptions table_options;
       options_.table_factory.reset(NewPlainTableFactory(table_options));
     }
-    options_.env = env_;
 
     perf_context.Reset();
     DestroyAndReopen(options_);
@@ -703,7 +622,6 @@ class BloomStatsTestWithParam
 
   bool use_block_table_;
   bool use_block_based_builder_;
-  bool partition_filters_;
   Options options_;
 };
 
@@ -814,11 +732,9 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
 }
 
 INSTANTIATE_TEST_CASE_P(BloomStatsTestWithParam, BloomStatsTestWithParam,
-                        ::testing::Values(std::make_tuple(true, true, false),
-                                          std::make_tuple(true, false, false),
-                                          std::make_tuple(true, false, true),
-                                          std::make_tuple(false, false,
-                                                          false)));
+                        ::testing::Values(std::make_tuple(true, true),
+                                          std::make_tuple(true, false),
+                                          std::make_tuple(false, false)));
 
 namespace {
 void PrefixScanInit(DBBloomFilterTest* dbtest) {
@@ -872,6 +788,8 @@ void PrefixScanInit(DBBloomFilterTest* dbtest) {
 }  // namespace
 
 TEST_F(DBBloomFilterTest, PrefixScan) {
+  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip,
+             kSkipNoPrefix);
   while (ChangeFilterOptions()) {
     int count;
     Slice prefix;
@@ -918,6 +836,7 @@ TEST_F(DBBloomFilterTest, PrefixScan) {
     ASSERT_EQ(env_->random_read_counter_.Read(), 2);
     Close();
   }  // end of while
+  XFUNC_TEST("", "dbtest_prefix", prefix_skip1, XFuncPoint::SetSkip, 0);
 }
 
 TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {

db/db_compaction_filter_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -628,7 +626,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
       iter->Next();
     }
     ASSERT_EQ(total, 700);
-    ASSERT_EQ(count, 1);
+    ASSERT_EQ(count, 2);
   }
 }
 #endif  // ROCKSDB_LITE

db/db_compaction_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -11,7 +9,6 @@
 
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
-#include "port/port.h"
 #include "rocksdb/experimental.h"
 #include "rocksdb/utilities/convenience.h"
 #include "util/sync_point.h"
@@ -42,12 +39,6 @@ class DBCompactionTestWithParam
   bool exclusive_manual_compaction_;
 };
 
-class DBCompactionDirectIOTest : public DBCompactionTest,
-                                 public ::testing::WithParamInterface<bool> {
- public:
-  DBCompactionDirectIOTest() : DBCompactionTest() {}
-};
-
 namespace {
 
 class FlushedFileCollector : public EventListener {
@@ -780,41 +771,6 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
   ASSERT_OK(Put("", ""));
 }
 
-TEST_F(DBCompactionTest, ManualCompactionUnknownOutputSize) {
-  // github issue #2249
-  Options options = CurrentOptions();
-  options.compaction_style = kCompactionStyleLevel;
-  options.level0_file_num_compaction_trigger = 3;
-  DestroyAndReopen(options);
-
-  // create two files in l1 that we can compact
-  for (int i = 0; i < 2; ++i) {
-    for (int j = 0; j < options.level0_file_num_compaction_trigger; j++) {
-      // make l0 files' ranges overlap to avoid trivial move
-      Put(std::to_string(2 * i), std::string(1, 'A'));
-      Put(std::to_string(2 * i + 1), std::string(1, 'A'));
-      Flush();
-      dbfull()->TEST_WaitForFlushMemTable();
-    }
-    dbfull()->TEST_WaitForCompact();
-    ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0);
-    ASSERT_EQ(NumTableFilesAtLevel(1, 0), i + 1);
-  }
-
-  ColumnFamilyMetaData cf_meta;
-  dbfull()->GetColumnFamilyMetaData(dbfull()->DefaultColumnFamily(), &cf_meta);
-  ASSERT_EQ(2, cf_meta.levels[1].files.size());
-  std::vector<std::string> input_filenames;
-  for (const auto& sst_file : cf_meta.levels[1].files) {
-    input_filenames.push_back(sst_file.name);
-  }
-
-  // note CompactionOptions::output_file_size_limit is unset.
-  CompactionOptions compact_opt;
-  compact_opt.compression = kNoCompression;
-  dbfull()->CompactFiles(compact_opt, input_filenames, 1);
-}
-
 // Check that writes done during a memtable compaction are recovered
 // if the database is shutdown during the memtable compaction.
 TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
@@ -1145,7 +1101,7 @@ TEST_P(DBCompactionTestWithParam, ManualCompactionPartial) {
   ASSERT_EQ(trivial_move, 6);
   ASSERT_EQ(non_trivial_move, 0);
 
-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
     compact_options.change_level = false;
     compact_options.exclusive_manual_compaction = false;
     std::string begin_string = Key(0);
@@ -1195,8 +1151,7 @@ TEST_P(DBCompactionTestWithParam, ManualCompactionPartial) {
   }
 }
 
-// Disable as the test is flaky.
-TEST_F(DBCompactionTest, DISABLED_ManualPartialFill) {
+TEST_F(DBCompactionTest, ManualPartialFill) {
   int32_t trivial_move = 0;
   int32_t non_trivial_move = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
@@ -1278,7 +1233,7 @@ TEST_F(DBCompactionTest, DISABLED_ManualPartialFill) {
   ASSERT_EQ(trivial_move, 2);
   ASSERT_EQ(non_trivial_move, 0);
 
-  rocksdb::port::Thread threads([&] {
+  std::thread threads([&] {
     compact_options.change_level = false;
     compact_options.exclusive_manual_compaction = false;
     std::string begin_string = Key(0);
@@ -1887,47 +1842,6 @@ TEST_F(DBCompactionTest, L0_CompactionBug_Issue44_b) {
   } while (ChangeCompactOptions());
 }
 
-TEST_F(DBCompactionTest, ManualAutoRace) {
-  CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::BGWorkCompaction", "DBCompactionTest::ManualAutoRace:1"},
-       {"DBImpl::RunManualCompaction:WaitScheduled",
-        "BackgroundCallCompaction:0"}});
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Put(1, "foo", "");
-  Put(1, "bar", "");
-  Flush(1);
-  Put(1, "foo", "");
-  Put(1, "bar", "");
-  // Generate four files in CF 0, which should trigger an auto compaction
-  Put("foo", "");
-  Put("bar", "");
-  Flush();
-  Put("foo", "");
-  Put("bar", "");
-  Flush();
-  Put("foo", "");
-  Put("bar", "");
-  Flush();
-  Put("foo", "");
-  Put("bar", "");
-  Flush();
-
-  // The auto compaction is scheduled but waited until here
-  TEST_SYNC_POINT("DBCompactionTest::ManualAutoRace:1");
-  // The auto compaction will wait until the the manual compaction is registerd
-  // before processing so that it will be cancelled.
-  dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr);
-  ASSERT_EQ("0,1", FilesPerLevel(1));
-
-  // Eventually the cancelled compaction will be rescheduled and executed.
-  dbfull()->TEST_WaitForCompact();
-  ASSERT_EQ("0,1", FilesPerLevel(0));
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
 TEST_P(DBCompactionTestWithParam, ManualCompaction) {
   Options options = CurrentOptions();
   options.max_subcompactions = max_subcompactions_;
@@ -2577,107 +2491,12 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
   rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }
 
-TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
-  Options options = CurrentOptions();
-  options.compression = kNoCompression;
-  options.level0_file_num_compaction_trigger = 5;
-  options.max_background_compactions = 2;
-  options.max_subcompactions = max_subcompactions_;
-  DestroyAndReopen(options);
-
-  const size_t kValueSize = 1 << 20;
-  Random rnd(301);
-  std::string value(RandomString(&rnd, kValueSize));
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"LevelCompactionPicker::PickCompactionBySize:0",
-        "CompactionJob::Run():Start"}});
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  // index:   0   1   2   3   4   5   6   7   8   9
-  // size:  1MB 1MB 1MB 1MB 1MB 2MB 1MB 1MB 1MB 1MB
-  // score:                     1.5 1.3 1.5 2.0 inf
-  //
-  // Files 0-4 will be included in an L0->L1 compaction.
-  //
-  // L0->L0 will be triggered since the sync points guarantee compaction to base
-  // level is still blocked when files 5-9 trigger another compaction.
-  //
-  // Files 6-9 are the longest span of available files for which
-  // work-per-deleted-file decreases (see "score" row above).
-  for (int i = 0; i < 10; ++i) {
-    for (int j = 0; j < 2; ++j) {
-      ASSERT_OK(Put(Key(0), ""));  // prevents trivial move
-      if (i == 5) {
-        ASSERT_OK(Put(Key(i + 1), value + value));
-      } else {
-        ASSERT_OK(Put(Key(i + 1), value));
-      }
-    }
-    ASSERT_OK(Flush());
-  }
-  dbfull()->TEST_WaitForCompact();
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  std::vector<std::vector<FileMetaData>> level_to_files;
-  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
-                                  &level_to_files);
-  ASSERT_GE(level_to_files.size(), 2);  // at least L0 and L1
-  // L0 has the 2MB file (not compacted) and 4MB file (output of L0->L0)
-  ASSERT_EQ(2, level_to_files[0].size());
-  ASSERT_GT(level_to_files[1].size(), 0);
-  for (int i = 0; i < 2; ++i) {
-    ASSERT_GE(level_to_files[0][0].fd.file_size, 1 << 21);
-  }
-}
-
 INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
                         ::testing::Values(std::make_tuple(1, true),
                                           std::make_tuple(1, false),
                                           std::make_tuple(4, true),
                                           std::make_tuple(4, false)));
 
-TEST_P(DBCompactionDirectIOTest, DirectIO) {
-  Options options = CurrentOptions();
-  Destroy(options);
-  options.create_if_missing = true;
-  options.disable_auto_compactions = true;
-  options.use_direct_io_for_flush_and_compaction = GetParam();
-  options.env = new MockEnv(Env::Default());
-  Reopen(options);
-  bool readahead = false;
-  SyncPoint::GetInstance()->SetCallBack(
-      "TableCache::NewIterator:for_compaction", [&](void* arg) {
-        bool* use_direct_reads = static_cast<bool*>(arg);
-        ASSERT_EQ(*use_direct_reads,
-                  options.use_direct_io_for_flush_and_compaction);
-      });
-  SyncPoint::GetInstance()->SetCallBack(
-      "CompactionJob::OpenCompactionOutputFile", [&](void* arg) {
-        bool* use_direct_writes = static_cast<bool*>(arg);
-        ASSERT_EQ(*use_direct_writes,
-                  options.use_direct_io_for_flush_and_compaction);
-      });
-  if (options.use_direct_io_for_flush_and_compaction) {
-    SyncPoint::GetInstance()->SetCallBack(
-        "SanitizeOptions:direct_io", [&](void* arg) {
-          readahead = true;
-        });
-  }
-  SyncPoint::GetInstance()->EnableProcessing();
-  CreateAndReopenWithCF({"pikachu"}, options);
-  MakeTables(3, "p", "q", 1);
-  ASSERT_EQ("1,1,1", FilesPerLevel(1));
-  Compact(1, "p1", "p9");
-  ASSERT_FALSE(readahead ^ options.use_direct_io_for_flush_and_compaction);
-  ASSERT_EQ("0,0,1", FilesPerLevel(1));
-  Destroy(options);
-  delete options.env;
-}
-
-INSTANTIATE_TEST_CASE_P(DBCompactionDirectIOTest, DBCompactionDirectIOTest,
-                        testing::Bool());
-
 class CompactionPriTest : public DBTestBase,
                           public testing::WithParamInterface<uint32_t> {
  public:

db/db_dynamic_level_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -15,7 +13,6 @@
 #if !defined(ROCKSDB_LITE)
 
 #include "db/db_test_util.h"
-#include "port/port.h"
 #include "port/stack_trace.h"
 
 namespace rocksdb {
@@ -77,7 +74,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
       options.compression_per_level[0] = kNoCompression;
       options.compression_per_level[1] = kLZ4Compression;
       options.compression_per_level[2] = kSnappyCompression;
-      options.env = env_;
 
       DestroyAndReopen(options);
 
@@ -255,7 +251,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
   });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  rocksdb::port::Thread thread([this] {
+  std::thread thread([this] {
     TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_start");
     ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
     TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_finish");
@@ -410,7 +406,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
   env_->SetBackgroundThreads(1, Env::HIGH);
 }
 
-TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
+TEST_F(DBTestDynamicLevel, MigrateToDynamicLevelMaxBytesBase) {
   Random rnd(301);
   const int kMaxKey = 2000;
 
@@ -466,7 +462,7 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
   compaction_finished = false;
   // Issue manual compaction in one thread and still verify DB state
   // in main thread.
-  rocksdb::port::Thread t([&]() {
+  std::thread t([&]() {
     CompactRangeOptions compact_options;
     compact_options.change_level = true;
     compact_options.target_level = options.num_levels - 1;

db/db_filesnapshot.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2012 Facebook.
 // Use of this source code is governed by a BSD-style license that can be
@@ -20,15 +18,16 @@
 #include <algorithm>
 #include <string>
 #include "db/db_impl.h"
+#include "db/filename.h"
 #include "db/job_context.h"
 #include "db/version_set.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "util/file_util.h"
-#include "util/filename.h"
 #include "util/mutexlock.h"
 #include "util/sync_point.h"
+#include "util/testharness.h"
 
 namespace rocksdb {
 
@@ -36,11 +35,12 @@ Status DBImpl::DisableFileDeletions() {
   InstrumentedMutexLock l(&mutex_);
   ++disable_delete_obsolete_files_;
   if (disable_delete_obsolete_files_ == 1) {
-    ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Disabled");
+    Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log,
+        "File Deletions Disabled");
   } else {
-    ROCKS_LOG_WARN(immutable_db_options_.info_log,
-                   "File Deletions Disabled, but already disabled. Counter: %d",
-                   disable_delete_obsolete_files_);
+    Log(InfoLogLevel::WARN_LEVEL, immutable_db_options_.info_log,
+        "File Deletions Disabled, but already disabled. Counter: %d",
+        disable_delete_obsolete_files_);
   }
   return Status::OK();
 }
@@ -59,12 +59,12 @@ Status DBImpl::EnableFileDeletions(bool force) {
       --disable_delete_obsolete_files_;
     }
     if (disable_delete_obsolete_files_ == 0)  {
-      ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled");
+      Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log,
+          "File Deletions Enabled");
       should_purge_files = true;
       FindObsoleteFiles(&job_context, true);
     } else {
-      ROCKS_LOG_WARN(
-          immutable_db_options_.info_log,
+      Log(InfoLogLevel::WARN_LEVEL, immutable_db_options_.info_log,
           "File Deletions Enable, but not really enabled. Counter: %d",
           disable_delete_obsolete_files_);
     }
@@ -110,8 +110,8 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
 
     if (!status.ok()) {
       mutex_.Unlock();
-      ROCKS_LOG_ERROR(immutable_db_options_.info_log, "Cannot Flush data %s\n",
-                      status.ToString().c_str());
+      Log(InfoLogLevel::ERROR_LEVEL, immutable_db_options_.info_log,
+          "Cannot Flush data %s\n", status.ToString().c_str());
       return status;
     }
   }

db/db_flush_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -11,7 +9,6 @@
 
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
-#include "util/fault_injection_test_env.h"
 #include "util/sync_point.h"
 
 namespace rocksdb {
@@ -21,19 +18,12 @@ class DBFlushTest : public DBTestBase {
   DBFlushTest() : DBTestBase("/db_flush_test") {}
 };
 
-class DBFlushDirectIOTest : public DBFlushTest,
-                            public ::testing::WithParamInterface<bool> {
- public:
-  DBFlushDirectIOTest() : DBFlushTest() {}
-};
-
 // We had issue when two background threads trying to flush at the same time,
 // only one of them get committed. The test verifies the issue is fixed.
 TEST_F(DBFlushTest, FlushWhileWritingManifest) {
   Options options;
   options.disable_auto_compactions = true;
   options.max_background_flushes = 2;
-  options.env = env_;
   Reopen(options);
   FlushOptions no_wait;
   no_wait.wait = false;
@@ -57,67 +47,6 @@ TEST_F(DBFlushTest, FlushWhileWritingManifest) {
 #endif  // ROCKSDB_LITE
 }
 
-TEST_F(DBFlushTest, SyncFail) {
-  std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
-      new FaultInjectionTestEnv(env_));
-  Options options;
-  options.disable_auto_compactions = true;
-  options.env = fault_injection_env.get();
-
-  SyncPoint::GetInstance()->LoadDependency(
-      {{"DBFlushTest::SyncFail:1", "DBImpl::SyncClosedLogs:Start"},
-       {"DBImpl::SyncClosedLogs:Failed", "DBFlushTest::SyncFail:2"}});
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  Reopen(options);
-  Put("key", "value");
-  auto* cfd =
-      reinterpret_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())
-          ->cfd();
-  int refs_before = cfd->current()->TEST_refs();
-  FlushOptions flush_options;
-  flush_options.wait = false;
-  ASSERT_OK(dbfull()->Flush(flush_options));
-  fault_injection_env->SetFilesystemActive(false);
-  TEST_SYNC_POINT("DBFlushTest::SyncFail:1");
-  TEST_SYNC_POINT("DBFlushTest::SyncFail:2");
-  fault_injection_env->SetFilesystemActive(true);
-  dbfull()->TEST_WaitForFlushMemTable();
-#ifndef ROCKSDB_LITE
-  ASSERT_EQ("", FilesPerLevel());  // flush failed.
-#endif                             // ROCKSDB_LITE
-  // Flush job should release ref count to current version.
-  ASSERT_EQ(refs_before, cfd->current()->TEST_refs());
-  Destroy(options);
-}
-
-TEST_P(DBFlushDirectIOTest, DirectIO) {
-  Options options;
-  options.create_if_missing = true;
-  options.disable_auto_compactions = true;
-  options.max_background_flushes = 2;
-  options.use_direct_io_for_flush_and_compaction = GetParam();
-  options.env = new MockEnv(Env::Default());
-  SyncPoint::GetInstance()->SetCallBack(
-      "BuildTable:create_file", [&](void* arg) {
-        bool* use_direct_writes = static_cast<bool*>(arg);
-        ASSERT_EQ(*use_direct_writes,
-                  options.use_direct_io_for_flush_and_compaction);
-      });
-
-  SyncPoint::GetInstance()->EnableProcessing();
-  Reopen(options);
-  ASSERT_OK(Put("foo", "v"));
-  FlushOptions flush_options;
-  flush_options.wait = true;
-  ASSERT_OK(dbfull()->Flush(flush_options));
-  Destroy(options);
-  delete options.env;
-}
-
-INSTANTIATE_TEST_CASE_P(DBFlushDirectIOTest, DBFlushDirectIOTest,
-                        testing::Bool());
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_impl.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -36,19 +34,18 @@
 #include "db/write_controller.h"
 #include "db/write_thread.h"
 #include "memtable_list.h"
-#include "monitoring/instrumented_mutex.h"
-#include "options/db_options.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/memtablerep.h"
-#include "rocksdb/status.h"
 #include "rocksdb/transaction_log.h"
 #include "rocksdb/write_buffer_manager.h"
 #include "table/scoped_arena_iterator.h"
 #include "util/autovector.h"
+#include "util/db_options.h"
 #include "util/event_logger.h"
 #include "util/hash.h"
+#include "util/instrumented_mutex.h"
 #include "util/stop_watch.h"
 #include "util/thread_local.h"
 
@@ -94,7 +91,7 @@ class DBImpl : public DB {
   using DB::Get;
   virtual Status Get(const ReadOptions& options,
                      ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
+                     std::string* value) override;
   using DB::MultiGet;
   virtual std::vector<Status> MultiGet(
       const ReadOptions& options,
@@ -102,19 +99,10 @@ class DBImpl : public DB {
       const std::vector<Slice>& keys,
       std::vector<std::string>* values) override;
 
-  virtual Status CreateColumnFamily(const ColumnFamilyOptions& cf_options,
+  virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
                                     const std::string& column_family,
                                     ColumnFamilyHandle** handle) override;
-  virtual Status CreateColumnFamilies(
-      const ColumnFamilyOptions& cf_options,
-      const std::vector<std::string>& column_family_names,
-      std::vector<ColumnFamilyHandle*>* handles) override;
-  virtual Status CreateColumnFamilies(
-      const std::vector<ColumnFamilyDescriptor>& column_families,
-      std::vector<ColumnFamilyHandle*>* handles) override;
   virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) override;
-  virtual Status DropColumnFamilies(
-      const std::vector<ColumnFamilyHandle*>& column_families) override;
 
   // Returns false if key doesn't exist in the database and true if it may.
   // If value_found is not passed in as null, then return the value if found in
@@ -150,13 +138,7 @@ class DBImpl : public DB {
   using DB::GetApproximateSizes;
   virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
                                    const Range* range, int n, uint64_t* sizes,
-                                   uint8_t include_flags
-                                   = INCLUDE_FILES) override;
-  using DB::GetApproximateMemTableStats;
-  virtual void GetApproximateMemTableStats(ColumnFamilyHandle* column_family,
-                                           const Range& range,
-                                           uint64_t* const count,
-                                           uint64_t* const size) override;
+                                   bool include_memtable = false) override;
   using DB::CompactRange;
   virtual Status CompactRange(const CompactRangeOptions& options,
                               ColumnFamilyHandle* column_family,
@@ -203,11 +185,7 @@ class DBImpl : public DB {
 
   virtual SequenceNumber GetLatestSequenceNumber() const override;
 
-  bool HasActiveSnapshotLaterThanSN(SequenceNumber sn);
-
 #ifndef ROCKSDB_LITE
-  using DB::ResetStats;
-  virtual Status ResetStats() override;
   virtual Status DisableFileDeletions() override;
   virtual Status EnableFileDeletions(bool force) override;
   virtual int IsFileDeletionsEnabled() const;
@@ -330,7 +308,7 @@ class DBImpl : public DB {
                            ColumnFamilyHandle* column_family = nullptr,
                            bool disallow_trivial_move = false);
 
-  void TEST_HandleWALFull();
+  void TEST_MaybeFlushColumnFamilies();
 
   bool TEST_UnableToFlushOldestLog() {
     return unable_to_flush_oldest_log_;
@@ -389,9 +367,9 @@ class DBImpl : public DB {
   Status TEST_GetAllImmutableCFOptions(
       std::unordered_map<std::string, const ImmutableCFOptions*>* iopts_map);
 
-  // Return the lastest MutableCFOptions of a column family
+  // Return the lastest MutableCFOptions of of a column family
   Status TEST_GetLatestMutableCFOptions(ColumnFamilyHandle* column_family,
-                                        MutableCFOptions* mutable_cf_options);
+                                        MutableCFOptions* mutable_cf_opitons);
 
   Cache* TEST_table_cache() { return table_cache_.get(); }
 
@@ -452,6 +430,9 @@ class DBImpl : public DB {
   // mutex is held.
   SuperVersion* GetAndRefSuperVersion(uint32_t column_family_id);
 
+  // Same as above, should called without mutex held and not on write thread.
+  SuperVersion* GetAndRefSuperVersionUnlocked(uint32_t column_family_id);
+
   // Un-reference the super version and return it to thread local cache if
   // needed. If it is the last reference of the super version. Clean it up
   // after un-referencing it.
@@ -462,6 +443,10 @@ class DBImpl : public DB {
   // REQUIRED: this function should only be called on the write thread.
   void ReturnAndCleanupSuperVersion(uint32_t colun_family_id, SuperVersion* sv);
 
+  // Same as above, should called without mutex held and not on write thread.
+  void ReturnAndCleanupSuperVersionUnlocked(uint32_t colun_family_id,
+                                            SuperVersion* sv);
+
   // REQUIRED: this function should only be called on the write thread or if the
   // mutex is held.  Return value only valid until next call to this function or
   // mutex is released.
@@ -484,8 +469,6 @@ class DBImpl : public DB {
     return num_running_compactions_;
   }
 
-  const WriteController& write_controller() { return write_controller_; }
-
   // hollow transactions shell used for recovery.
   // these will then be passed to TransactionDB so that
   // locks can be reacquired before writing can resume.
@@ -565,10 +548,9 @@ class DBImpl : public DB {
                                         RangeDelAggregator* range_del_agg);
 
   // Except in DB::Open(), WriteOptionsFile can only be called when:
-  // Persist options to options file.
-  // If need_mutex_lock = false, the method will lock DB mutex.
-  // If need_enter_write_thread = false, the method will enter write thread.
-  Status WriteOptionsFile(bool need_mutex_lock, bool need_enter_write_thread);
+  // 1. WriteThread::Writer::EnterUnbatched() is used.
+  // 2. db_mutex is held
+  Status WriteOptionsFile();
 
   // The following two functions can only be called when:
   // 1. WriteThread::Writer::EnterUnbatched() is used.
@@ -576,10 +558,6 @@ class DBImpl : public DB {
   Status RenameTempFileToOptionsFile(const std::string& file_name);
   Status DeleteObsoleteOptionsFiles();
 
-  void NotifyOnFlushBegin(ColumnFamilyData* cfd, FileMetaData* file_meta,
-                          const MutableCFOptions& mutable_cf_options,
-                          int job_id, TableProperties prop);
-
   void NotifyOnFlushCompleted(ColumnFamilyData* cfd, FileMetaData* file_meta,
                               const MutableCFOptions& mutable_cf_options,
                               int job_id, TableProperties prop);
@@ -591,10 +569,8 @@ class DBImpl : public DB {
   void NotifyOnMemTableSealed(ColumnFamilyData* cfd,
                               const MemTableInfo& mem_table_info);
 
-#ifndef ROCKSDB_LITE
   void NotifyOnExternalFileIngested(
       ColumnFamilyData* cfd, const ExternalSstFileIngestionJob& ingestion_job);
-#endif  // !ROCKSDB_LITE
 
   void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const;
 
@@ -607,11 +583,6 @@ class DBImpl : public DB {
                    uint64_t* log_used = nullptr, uint64_t log_ref = 0,
                    bool disable_memtable = false);
 
-  Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
-                            WriteCallback* callback = nullptr,
-                            uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                            bool disable_memtable = false);
-
   uint64_t FindMinLogContainingOutstandingPrep();
   uint64_t FindMinPrepLogReferencedByMemTable();
 
@@ -629,19 +600,7 @@ class DBImpl : public DB {
 #endif
   struct CompactionState;
 
-  struct WriteContext {
-    autovector<SuperVersion*> superversions_to_free_;
-    autovector<MemTable*> memtables_to_free_;
-
-    ~WriteContext() {
-      for (auto& sv : superversions_to_free_) {
-        delete sv;
-      }
-      for (auto& m : memtables_to_free_) {
-        delete m;
-      }
-    }
-  };
+  struct WriteContext;
 
   struct PurgeFileInfo;
 
@@ -656,12 +615,6 @@ class DBImpl : public DB {
 
   const Status CreateArchivalDirectory();
 
-  Status CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
-                                const std::string& cf_name,
-                                ColumnFamilyHandle** handle);
-
-  Status DropColumnFamilyImpl(ColumnFamilyHandle* column_family);
-
   // Delete any unneeded files and stale in-memory entries.
   void DeleteObsoleteFiles();
   // Delete obsolete files and log status and information of file deletion
@@ -724,26 +677,6 @@ class DBImpl : public DB {
   // Wait for memtable flushed
   Status WaitForFlushMemTable(ColumnFamilyData* cfd);
 
-  // REQUIRES: mutex locked
-  Status HandleWALFull(WriteContext* write_context);
-
-  // REQUIRES: mutex locked
-  Status HandleWriteBufferFull(WriteContext* write_context);
-
-  // REQUIRES: mutex locked
-  Status PreprocessWrite(const WriteOptions& write_options, bool* need_log_sync,
-                         WriteContext* write_context);
-
-  Status WriteToWAL(const WriteThread::WriteGroup& write_group,
-                    log::Writer* log_writer, bool need_log_sync,
-                    bool need_log_dir_sync, SequenceNumber sequence);
-
-  // Used by WriteImpl to update bg_error_ if paranoid check is enabled.
-  void ParanoidCheck(const Status& status);
-
-  // Used by WriteImpl to update bg_error_ in case of memtable insert error.
-  void MemTableInsertStatusCheck(const Status& memtable_insert_status);
-
 #ifndef ROCKSDB_LITE
 
   Status CompactFilesImpl(const CompactionOptions& compact_options,
@@ -807,6 +740,12 @@ class DBImpl : public DB {
 
   const Snapshot* GetSnapshotImpl(bool is_write_conflict_boundary);
 
+  // Persist RocksDB options under the single write thread
+  // REQUIRES: mutex locked
+  Status PersistOptions();
+
+  void MaybeFlushColumnFamilies();
+
   uint64_t GetMaxTotalWalSize() const;
 
   // table_cache_ provides its own synchronization
@@ -882,7 +821,7 @@ class DBImpl : public DB {
   std::deque<LogWriterNumber> logs_;
   // Signaled when getting_synced becomes false for some of the logs_.
   InstrumentedCondVar log_sync_cv_;
-  std::atomic<uint64_t> total_log_size_;
+  uint64_t total_log_size_;
   // only used for dynamically adjusting max_total_wal_size. it is a sum of
   // [write_buffer_size * max_write_buffer_number] over all column families
   uint64_t max_total_in_memory_state_;
@@ -1049,9 +988,8 @@ class DBImpl : public DB {
   // without any synchronization
   int disable_delete_obsolete_files_;
 
-  // last time when DeleteObsoleteFiles with full scan was executed. Originaly
-  // initialized with startup time.
-  uint64_t delete_obsolete_files_last_run_;
+  // next time when we should run DeleteObsoleteFiles with full scan
+  uint64_t delete_obsolete_files_next_run_;
 
   // last time stats were dumped to LOG
   std::atomic<uint64_t> last_stats_dump_time_microsec_;
@@ -1063,7 +1001,8 @@ class DBImpl : public DB {
   // A flag indicating whether the current rocksdb database has any
   // data that is not yet persisted into either WAL or SST file.
   // Used when disableWAL is true.
-  std::atomic<bool> has_unpersisted_data_;
+  bool has_unpersisted_data_;
+
 
   // if an attempt was made to flush all column families that
   // the oldest log depends on but uncommited data in the oldest
@@ -1105,7 +1044,7 @@ class DBImpl : public DB {
   // Indicate DB was opened successfully
   bool opened_successfully_;
 
-  // minimum log number still containing prepared data.
+  // minmum log number still containing prepared data.
   // this is used by FindObsoleteFiles to determine which
   // flushed logs we must keep around because they still
   // contain prepared data which has not been flushed or rolled back
@@ -1118,7 +1057,7 @@ class DBImpl : public DB {
   // to prepared_section_completed_ which maps LOG -> instance_count
   // since a log could contain multiple prepared sections
   //
-  // when trying to determine the minimum log still active we first
+  // when trying to determine the minmum log still active we first
   // consult min_log_with_prep_. while that root value maps to
   // a value > 0 in prepared_section_completed_ we decrement the
   // instance_count for that log and pop the root value in
@@ -1131,6 +1070,13 @@ class DBImpl : public DB {
   DBImpl(const DBImpl&);
   void operator=(const DBImpl&);
 
+  // Return the earliest snapshot where seqno is visible.
+  // Store the snapshot right before that, if any, in prev_snapshot
+  inline SequenceNumber findEarliestVisibleSnapshot(
+    SequenceNumber in,
+    std::vector<SequenceNumber>& snapshots,
+    SequenceNumber* prev_snapshot);
+
   // Background threads call this function, which is just a wrapper around
   // the InstallSuperVersion() function. Background threads carry
   // job_context which can have new_superversion already
@@ -1160,7 +1106,7 @@ class DBImpl : public DB {
   // Function that Get and KeyMayExist call with no_io true or false
   // Note: 'value_found' from KeyMayExist propagates here
   Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
-                 const Slice& key, PinnableSlice* value,
+                 const Slice& key, std::string* value,
                  bool* value_found = nullptr);
 
   bool GetIntPropertyInternal(ColumnFamilyData* cfd,
@@ -1183,10 +1129,6 @@ extern Options SanitizeOptions(const std::string& db,
 
 extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src);
 
-extern CompressionType GetCompressionFlush(
-    const ImmutableCFOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options);
- 
 // Fix user-supplied options to be reasonable
 template <class T, class V>
 static void ClipToRange(T* ptr, V minvalue, V maxvalue) {

db/db_impl_debug.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -12,7 +10,7 @@
 #ifndef NDEBUG
 
 #include "db/db_impl.h"
-#include "monitoring/thread_status_updater.h"
+#include "util/thread_status_updater.h"
 
 namespace rocksdb {
 
@@ -21,10 +19,9 @@ uint64_t DBImpl::TEST_GetLevel0TotalSize() {
   return default_cf_handle_->cfd()->current()->storage_info()->NumLevelBytes(0);
 }
 
-void DBImpl::TEST_HandleWALFull() {
-  WriteContext write_context;
+void DBImpl::TEST_MaybeFlushColumnFamilies() {
   InstrumentedMutexLock l(&mutex_);
-  HandleWALFull(&write_context);
+  MaybeFlushColumnFamilies();
 }
 
 int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(

db/db_impl_experimental.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -63,8 +61,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
   assert(column_family);
 
   if (target_level < 1) {
-    ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                   "PromoteL0 FAILED. Invalid target level %d\n", target_level);
+    Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log,
+        "PromoteL0 FAILED. Invalid target level %d\n", target_level);
     return Status::InvalidArgument("Invalid target level");
   }
 
@@ -77,9 +75,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
     const auto* vstorage = cfd->current()->storage_info();
 
     if (target_level >= vstorage->num_levels()) {
-      ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                     "PromoteL0 FAILED. Target level %d does not exist\n",
-                     target_level);
+      Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log,
+          "PromoteL0 FAILED. Target level %d does not exist\n", target_level);
       job_context.Clean();
       return Status::InvalidArgument("Target level does not exist");
     }
@@ -97,9 +94,9 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
     for (size_t i = 0; i < l0_files.size(); ++i) {
       auto f = l0_files[i];
       if (f->being_compacted) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "PromoteL0 FAILED. File %" PRIu64 " being compacted\n",
-                       f->fd.GetNumber());
+        Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log,
+            "PromoteL0 FAILED. File %" PRIu64 " being compacted\n",
+            f->fd.GetNumber());
         job_context.Clean();
         return Status::InvalidArgument("PromoteL0 called during L0 compaction");
       }
@@ -107,10 +104,10 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
       if (i == 0) continue;
       auto prev_f = l0_files[i - 1];
       if (icmp->Compare(prev_f->largest, f->smallest) >= 0) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "PromoteL0 FAILED. Files %" PRIu64 " and %" PRIu64
-                       " have overlapping ranges\n",
-                       prev_f->fd.GetNumber(), f->fd.GetNumber());
+        Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log,
+            "PromoteL0 FAILED. Files %" PRIu64 " and %" PRIu64
+            " have overlapping ranges\n",
+            prev_f->fd.GetNumber(), f->fd.GetNumber());
         job_context.Clean();
         return Status::InvalidArgument("L0 has overlapping files");
       }
@@ -119,8 +116,8 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
     // Check that all levels up to target_level are empty.
     for (int level = 1; level <= target_level; ++level) {
       if (vstorage->NumLevelFiles(level) > 0) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "PromoteL0 FAILED. Level %d not empty\n", level);
+        Log(InfoLogLevel::INFO_LEVEL, immutable_db_options_.info_log,
+            "PromoteL0 FAILED. Level %d not empty\n", level);
         job_context.Clean();
         return Status::InvalidArgument(
             "All levels up to target_level "

db/db_impl_files.cc

@@ -1,538 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include "db/db_impl.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-#include <inttypes.h>
-#include "db/event_helpers.h"
-#include "util/file_util.h"
-#include "util/sst_file_manager_impl.h"
-
-
-namespace rocksdb {
-uint64_t DBImpl::FindMinPrepLogReferencedByMemTable() {
-  if (!allow_2pc()) {
-    return 0;
-  }
-
-  uint64_t min_log = 0;
-
-  // we must look through the memtables for two phase transactions
-  // that have been committed but not yet flushed
-  for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
-    if (loop_cfd->IsDropped()) {
-      continue;
-    }
-
-    auto log = loop_cfd->imm()->GetMinLogContainingPrepSection();
-
-    if (log > 0 && (min_log == 0 || log < min_log)) {
-      min_log = log;
-    }
-
-    log = loop_cfd->mem()->GetMinLogContainingPrepSection();
-
-    if (log > 0 && (min_log == 0 || log < min_log)) {
-      min_log = log;
-    }
-  }
-
-  return min_log;
-}
-
-void DBImpl::MarkLogAsHavingPrepSectionFlushed(uint64_t log) {
-  assert(log != 0);
-  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
-  auto it = prepared_section_completed_.find(log);
-  assert(it != prepared_section_completed_.end());
-  it->second += 1;
-}
-
-void DBImpl::MarkLogAsContainingPrepSection(uint64_t log) {
-  assert(log != 0);
-  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
-  min_log_with_prep_.push(log);
-  auto it = prepared_section_completed_.find(log);
-  if (it == prepared_section_completed_.end()) {
-    prepared_section_completed_[log] = 0;
-  }
-}
-
-uint64_t DBImpl::FindMinLogContainingOutstandingPrep() {
-
-  if (!allow_2pc()) {
-    return 0;
-  }
-
-  std::lock_guard<std::mutex> lock(prep_heap_mutex_);
-  uint64_t min_log = 0;
-
-  // first we look in the prepared heap where we keep
-  // track of transactions that have been prepared (written to WAL)
-  // but not yet committed.
-  while (!min_log_with_prep_.empty()) {
-    min_log = min_log_with_prep_.top();
-
-    auto it = prepared_section_completed_.find(min_log);
-
-    // value was marked as 'deleted' from heap
-    if (it != prepared_section_completed_.end() && it->second > 0) {
-      it->second -= 1;
-      min_log_with_prep_.pop();
-
-      // back to squere one...
-      min_log = 0;
-      continue;
-    } else {
-      // found a valid value
-      break;
-    }
-  }
-
-  return min_log;
-}
-
-uint64_t DBImpl::MinLogNumberToKeep() {
-  uint64_t log_number = versions_->MinLogNumber();
-
-  if (allow_2pc()) {
-    // if are 2pc we must consider logs containing prepared
-    // sections of outstanding transactions.
-    //
-    // We must check min logs with outstanding prep before we check
-    // logs referneces by memtables because a log referenced by the
-    // first data structure could transition to the second under us.
-    //
-    // TODO(horuff): iterating over all column families under db mutex.
-    // should find more optimial solution
-    auto min_log_in_prep_heap = FindMinLogContainingOutstandingPrep();
-
-    if (min_log_in_prep_heap != 0 && min_log_in_prep_heap < log_number) {
-      log_number = min_log_in_prep_heap;
-    }
-
-    auto min_log_refed_by_mem = FindMinPrepLogReferencedByMemTable();
-
-    if (min_log_refed_by_mem != 0 && min_log_refed_by_mem < log_number) {
-      log_number = min_log_refed_by_mem;
-    }
-  }
-  return log_number;
-}
-
-// * Returns the list of live files in 'sst_live'
-// If it's doing full scan:
-// * Returns the list of all files in the filesystem in
-// 'full_scan_candidate_files'.
-// Otherwise, gets obsolete files from VersionSet.
-// no_full_scan = true -- never do the full scan using GetChildren()
-// force = false -- don't force the full scan, except every
-//  mutable_db_options_.delete_obsolete_files_period_micros
-// force = true -- force the full scan
-void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
-                               bool no_full_scan) {
-  mutex_.AssertHeld();
-
-  // if deletion is disabled, do nothing
-  if (disable_delete_obsolete_files_ > 0) {
-    return;
-  }
-
-  bool doing_the_full_scan = false;
-
-  // logic for figurint out if we're doing the full scan
-  if (no_full_scan) {
-    doing_the_full_scan = false;
-  } else if (force ||
-             mutable_db_options_.delete_obsolete_files_period_micros == 0) {
-    doing_the_full_scan = true;
-  } else {
-    const uint64_t now_micros = env_->NowMicros();
-    if ((delete_obsolete_files_last_run_ +
-         mutable_db_options_.delete_obsolete_files_period_micros) <
-        now_micros) {
-      doing_the_full_scan = true;
-      delete_obsolete_files_last_run_ = now_micros;
-    }
-  }
-
-  // don't delete files that might be currently written to from compaction
-  // threads
-  // Since job_context->min_pending_output is set, until file scan finishes,
-  // mutex_ cannot be released. Otherwise, we might see no min_pending_output
-  // here but later find newer generated unfinalized files while scannint.
-  if (!pending_outputs_.empty()) {
-    job_context->min_pending_output = *pending_outputs_.begin();
-  } else {
-    // delete all of them
-    job_context->min_pending_output = std::numeric_limits<uint64_t>::max();
-  }
-
-  // Get obsolete files.  This function will also update the list of
-  // pending files in VersionSet().
-  versions_->GetObsoleteFiles(&job_context->sst_delete_files,
-                              &job_context->manifest_delete_files,
-                              job_context->min_pending_output);
-
-  // store the current filenum, lognum, etc
-  job_context->manifest_file_number = versions_->manifest_file_number();
-  job_context->pending_manifest_file_number =
-      versions_->pending_manifest_file_number();
-  job_context->log_number = MinLogNumberToKeep();
-
-  job_context->prev_log_number = versions_->prev_log_number();
-
-  versions_->AddLiveFiles(&job_context->sst_live);
-  if (doing_the_full_scan) {
-    for (size_t path_id = 0; path_id < immutable_db_options_.db_paths.size();
-         path_id++) {
-      // set of all files in the directory. We'll exclude files that are still
-      // alive in the subsequent processings.
-      std::vector<std::string> files;
-      env_->GetChildren(immutable_db_options_.db_paths[path_id].path,
-                        &files);  // Ignore errors
-      for (std::string file : files) {
-        // TODO(icanadi) clean up this mess to avoid having one-off "/" prefixes
-        job_context->full_scan_candidate_files.emplace_back(
-            "/" + file, static_cast<uint32_t>(path_id));
-      }
-    }
-
-    // Add log files in wal_dir
-    if (immutable_db_options_.wal_dir != dbname_) {
-      std::vector<std::string> log_files;
-      env_->GetChildren(immutable_db_options_.wal_dir,
-                        &log_files);  // Ignore errors
-      for (std::string log_file : log_files) {
-        job_context->full_scan_candidate_files.emplace_back(log_file, 0);
-      }
-    }
-    // Add info log files in db_log_dir
-    if (!immutable_db_options_.db_log_dir.empty() &&
-        immutable_db_options_.db_log_dir != dbname_) {
-      std::vector<std::string> info_log_files;
-      // Ignore errors
-      env_->GetChildren(immutable_db_options_.db_log_dir, &info_log_files);
-      for (std::string log_file : info_log_files) {
-        job_context->full_scan_candidate_files.emplace_back(log_file, 0);
-      }
-    }
-  }
-
-  // logs_ is empty when called during recovery, in which case there can't yet
-  // be any tracked obsolete logs
-  if (!alive_log_files_.empty() && !logs_.empty()) {
-    uint64_t min_log_number = job_context->log_number;
-    size_t num_alive_log_files = alive_log_files_.size();
-    // find newly obsoleted log files
-    while (alive_log_files_.begin()->number < min_log_number) {
-      auto& earliest = *alive_log_files_.begin();
-      if (immutable_db_options_.recycle_log_file_num >
-          log_recycle_files.size()) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "adding log %" PRIu64 " to recycle list\n",
-                       earliest.number);
-        log_recycle_files.push_back(earliest.number);
-      } else {
-        job_context->log_delete_files.push_back(earliest.number);
-      }
-      if (job_context->size_log_to_delete == 0) {
-        job_context->prev_total_log_size = total_log_size_;
-        job_context->num_alive_log_files = num_alive_log_files;
-      }
-      job_context->size_log_to_delete += earliest.size;
-      total_log_size_ -= earliest.size;
-      alive_log_files_.pop_front();
-      // Current log should always stay alive since it can't have
-      // number < MinLogNumber().
-      assert(alive_log_files_.size());
-    }
-    while (!logs_.empty() && logs_.front().number < min_log_number) {
-      auto& log = logs_.front();
-      if (log.getting_synced) {
-        log_sync_cv_.Wait();
-        // logs_ could have changed while we were waiting.
-        continue;
-      }
-      logs_to_free_.push_back(log.ReleaseWriter());
-      logs_.pop_front();
-    }
-    // Current log cannot be obsolete.
-    assert(!logs_.empty());
-  }
-
-  // We're just cleaning up for DB::Write().
-  assert(job_context->logs_to_free.empty());
-  job_context->logs_to_free = logs_to_free_;
-  job_context->log_recycle_files.assign(log_recycle_files.begin(),
-                                        log_recycle_files.end());
-  logs_to_free_.clear();
-}
-
-namespace {
-bool CompareCandidateFile(const JobContext::CandidateFileInfo& first,
-                          const JobContext::CandidateFileInfo& second) {
-  if (first.file_name > second.file_name) {
-    return true;
-  } else if (first.file_name < second.file_name) {
-    return false;
-  } else {
-    return (first.path_id > second.path_id);
-  }
-}
-};  // namespace
-
-// Delete obsolete files and log status and information of file deletion
-void DBImpl::DeleteObsoleteFileImpl(Status file_deletion_status, int job_id,
-                                    const std::string& fname, FileType type,
-                                    uint64_t number, uint32_t path_id) {
-  if (type == kTableFile) {
-    file_deletion_status =
-        DeleteSSTFile(&immutable_db_options_, fname, path_id);
-  } else {
-    file_deletion_status = env_->DeleteFile(fname);
-  }
-  if (file_deletion_status.ok()) {
-    ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
-                    "[JOB %d] Delete %s type=%d #%" PRIu64 " -- %s\n", job_id,
-                    fname.c_str(), type, number,
-                    file_deletion_status.ToString().c_str());
-  } else if (env_->FileExists(fname).IsNotFound()) {
-    ROCKS_LOG_INFO(
-        immutable_db_options_.info_log,
-        "[JOB %d] Tried to delete a non-existing file %s type=%d #%" PRIu64
-        " -- %s\n",
-        job_id, fname.c_str(), type, number,
-        file_deletion_status.ToString().c_str());
-  } else {
-    ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                    "[JOB %d] Failed to delete %s type=%d #%" PRIu64 " -- %s\n",
-                    job_id, fname.c_str(), type, number,
-                    file_deletion_status.ToString().c_str());
-  }
-  if (type == kTableFile) {
-    EventHelpers::LogAndNotifyTableFileDeletion(
-        &event_logger_, job_id, number, fname, file_deletion_status, GetName(),
-        immutable_db_options_.listeners);
-  }
-}
-
-// Diffs the files listed in filenames and those that do not
-// belong to live files are posibly removed. Also, removes all the
-// files in sst_delete_files and log_delete_files.
-// It is not necessary to hold the mutex when invoking this method.
-void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
-  // we'd better have sth to delete
-  assert(state.HaveSomethingToDelete());
-
-  // this checks if FindObsoleteFiles() was run before. If not, don't do
-  // PurgeObsoleteFiles(). If FindObsoleteFiles() was run, we need to also
-  // run PurgeObsoleteFiles(), even if disable_delete_obsolete_files_ is true
-  if (state.manifest_file_number == 0) {
-    return;
-  }
-
-  // Now, convert live list to an unordered map, WITHOUT mutex held;
-  // set is slow.
-  std::unordered_map<uint64_t, const FileDescriptor*> sst_live_map;
-  for (const FileDescriptor& fd : state.sst_live) {
-    sst_live_map[fd.GetNumber()] = &fd;
-  }
-  std::unordered_set<uint64_t> log_recycle_files_set(
-      state.log_recycle_files.begin(), state.log_recycle_files.end());
-
-  auto candidate_files = state.full_scan_candidate_files;
-  candidate_files.reserve(
-      candidate_files.size() + state.sst_delete_files.size() +
-      state.log_delete_files.size() + state.manifest_delete_files.size());
-  // We may ignore the dbname when generating the file names.
-  const char* kDumbDbName = "";
-  for (auto file : state.sst_delete_files) {
-    candidate_files.emplace_back(
-        MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
-        file->fd.GetPathId());
-    delete file;
-  }
-
-  for (auto file_num : state.log_delete_files) {
-    if (file_num > 0) {
-      candidate_files.emplace_back(LogFileName(kDumbDbName, file_num), 0);
-    }
-  }
-  for (const auto& filename : state.manifest_delete_files) {
-    candidate_files.emplace_back(filename, 0);
-  }
-
-  // dedup state.candidate_files so we don't try to delete the same
-  // file twice
-  std::sort(candidate_files.begin(), candidate_files.end(),
-            CompareCandidateFile);
-  candidate_files.erase(
-      std::unique(candidate_files.begin(), candidate_files.end()),
-      candidate_files.end());
-
-  if (state.prev_total_log_size > 0) {
-    ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                   "[JOB %d] Try to delete WAL files size %" PRIu64
-                   ", prev total WAL file size %" PRIu64
-                   ", number of live WAL files %" ROCKSDB_PRIszt ".\n",
-                   state.job_id, state.size_log_to_delete,
-                   state.prev_total_log_size, state.num_alive_log_files);
-  }
-
-  std::vector<std::string> old_info_log_files;
-  InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(),
-                                dbname_);
-  for (const auto& candidate_file : candidate_files) {
-    std::string to_delete = candidate_file.file_name;
-    uint32_t path_id = candidate_file.path_id;
-    uint64_t number;
-    FileType type;
-    // Ignore file if we cannot recognize it.
-    if (!ParseFileName(to_delete, &number, info_log_prefix.prefix, &type)) {
-      continue;
-    }
-
-    bool keep = true;
-    switch (type) {
-      case kLogFile:
-        keep = ((number >= state.log_number) ||
-                (number == state.prev_log_number) ||
-                (log_recycle_files_set.find(number) !=
-                 log_recycle_files_set.end()));
-        break;
-      case kDescriptorFile:
-        // Keep my manifest file, and any newer incarnations'
-        // (can happen during manifest roll)
-        keep = (number >= state.manifest_file_number);
-        break;
-      case kTableFile:
-        // If the second condition is not there, this makes
-        // DontDeletePendingOutputs fail
-        keep = (sst_live_map.find(number) != sst_live_map.end()) ||
-               number >= state.min_pending_output;
-        break;
-      case kTempFile:
-        // Any temp files that are currently being written to must
-        // be recorded in pending_outputs_, which is inserted into "live".
-        // Also, SetCurrentFile creates a temp file when writing out new
-        // manifest, which is equal to state.pending_manifest_file_number. We
-        // should not delete that file
-        //
-        // TODO(yhchiang): carefully modify the third condition to safely
-        //                 remove the temp options files.
-        keep = (sst_live_map.find(number) != sst_live_map.end()) ||
-               (number == state.pending_manifest_file_number) ||
-               (to_delete.find(kOptionsFileNamePrefix) != std::string::npos);
-        break;
-      case kInfoLogFile:
-        keep = true;
-        if (number != 0) {
-          old_info_log_files.push_back(to_delete);
-        }
-        break;
-      case kCurrentFile:
-      case kDBLockFile:
-      case kIdentityFile:
-      case kMetaDatabase:
-      case kOptionsFile:
-      case kBlobFile:
-        keep = true;
-        break;
-    }
-
-    if (keep) {
-      continue;
-    }
-
-    std::string fname;
-    if (type == kTableFile) {
-      // evict from cache
-      TableCache::Evict(table_cache_.get(), number);
-      fname = TableFileName(immutable_db_options_.db_paths, number, path_id);
-    } else {
-      fname = ((type == kLogFile) ? immutable_db_options_.wal_dir : dbname_) +
-              "/" + to_delete;
-    }
-
-#ifndef ROCKSDB_LITE
-    if (type == kLogFile && (immutable_db_options_.wal_ttl_seconds > 0 ||
-                             immutable_db_options_.wal_size_limit_mb > 0)) {
-      wal_manager_.ArchiveWALFile(fname, number);
-      continue;
-    }
-#endif  // !ROCKSDB_LITE
-
-    Status file_deletion_status;
-    if (schedule_only) {
-      InstrumentedMutexLock guard_lock(&mutex_);
-      SchedulePendingPurge(fname, type, number, path_id, state.job_id);
-    } else {
-      DeleteObsoleteFileImpl(file_deletion_status, state.job_id, fname, type,
-                             number, path_id);
-    }
-  }
-
-  // Delete old info log files.
-  size_t old_info_log_file_count = old_info_log_files.size();
-  if (old_info_log_file_count != 0 &&
-      old_info_log_file_count >= immutable_db_options_.keep_log_file_num) {
-    std::sort(old_info_log_files.begin(), old_info_log_files.end());
-    size_t end =
-        old_info_log_file_count - immutable_db_options_.keep_log_file_num;
-    for (unsigned int i = 0; i <= end; i++) {
-      std::string& to_delete = old_info_log_files.at(i);
-      std::string full_path_to_delete =
-          (immutable_db_options_.db_log_dir.empty()
-               ? dbname_
-               : immutable_db_options_.db_log_dir) +
-          "/" + to_delete;
-      ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                     "[JOB %d] Delete info log file %s\n", state.job_id,
-                     full_path_to_delete.c_str());
-      Status s = env_->DeleteFile(full_path_to_delete);
-      if (!s.ok()) {
-        if (env_->FileExists(full_path_to_delete).IsNotFound()) {
-          ROCKS_LOG_INFO(
-              immutable_db_options_.info_log,
-              "[JOB %d] Tried to delete non-existing info log file %s FAILED "
-              "-- %s\n",
-              state.job_id, to_delete.c_str(), s.ToString().c_str());
-        } else {
-          ROCKS_LOG_ERROR(immutable_db_options_.info_log,
-                          "[JOB %d] Delete info log file %s FAILED -- %s\n",
-                          state.job_id, to_delete.c_str(),
-                          s.ToString().c_str());
-        }
-      }
-    }
-  }
-#ifndef ROCKSDB_LITE
-  wal_manager_.PurgeObsoleteWALFiles();
-#endif  // ROCKSDB_LITE
-  LogFlush(immutable_db_options_.info_log);
-}
-
-void DBImpl::DeleteObsoleteFiles() {
-  mutex_.AssertHeld();
-  JobContext job_context(next_job_id_.fetch_add(1));
-  FindObsoleteFiles(&job_context, true);
-
-  mutex_.Unlock();
-  if (job_context.HaveSomethingToDelete()) {
-    PurgeObsoleteFiles(job_context);
-  }
-  job_context.Clean();
-  mutex_.Lock();
-}
-}  // namespace rocksdb

db/db_impl_readonly.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 
 #include "db/db_impl_readonly.h"
@@ -13,7 +11,7 @@
 #include "db/db_iter.h"
 #include "db/merge_context.h"
 #include "db/range_del_aggregator.h"
-#include "monitoring/perf_context_imp.h"
+#include "util/perf_context_imp.h"
 
 namespace rocksdb {
 
@@ -22,8 +20,8 @@ namespace rocksdb {
 DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
                                const std::string& dbname)
     : DBImpl(db_options, dbname) {
-  ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                 "Opening the db in read only mode");
+  Log(INFO_LEVEL, immutable_db_options_.info_log,
+      "Opening the db in read only mode");
   LogFlush(immutable_db_options_.info_log);
 }
 
@@ -33,8 +31,7 @@ DBImplReadOnly::~DBImplReadOnly() {
 // Implementations of the DB interface
 Status DBImplReadOnly::Get(const ReadOptions& read_options,
                            ColumnFamilyHandle* column_family, const Slice& key,
-                           PinnableSlice* pinnable_val) {
-  assert(pinnable_val != nullptr);
+                           std::string* value) {
   Status s;
   SequenceNumber snapshot = versions_->LastSequence();
   auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
@@ -43,13 +40,12 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options,
   MergeContext merge_context;
   RangeDelAggregator range_del_agg(cfd->internal_comparator(), snapshot);
   LookupKey lkey(key, snapshot);
-  if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
-                              &range_del_agg, read_options)) {
-    pinnable_val->PinSelf();
+  if (super_version->mem->Get(lkey, value, &s, &merge_context, &range_del_agg,
+                              read_options)) {
   } else {
     PERF_TIMER_GUARD(get_from_output_files_time);
-    super_version->current->Get(read_options, lkey, pinnable_val, &s,
-                                &merge_context, &range_del_agg);
+    super_version->current->Get(read_options, lkey, value, &s, &merge_context,
+                                &range_del_agg);
   }
   return s;
 }
@@ -61,7 +57,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
   SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
   SequenceNumber latest_snapshot = versions_->LastSequence();
   auto db_iter = NewArenaWrappedDbIterator(
-      env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
+      env_, *cfd->ioptions(), cfd->user_comparator(),
       (read_options.snapshot != nullptr
            ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                  ->number_
@@ -90,7 +86,7 @@ Status DBImplReadOnly::NewIterators(
     auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
     auto* sv = cfd->GetSuperVersion()->Ref();
     auto* db_iter = NewArenaWrappedDbIterator(
-        env_, read_options, *cfd->ioptions(), cfd->user_comparator(),
+        env_, *cfd->ioptions(), cfd->user_comparator(),
         (read_options.snapshot != nullptr
              ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
                    ->number_

db/db_impl_readonly.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #pragma once
 
@@ -24,7 +22,7 @@ class DBImplReadOnly : public DBImpl {
   using DB::Get;
   virtual Status Get(const ReadOptions& options,
                      ColumnFamilyHandle* column_family, const Slice& key,
-                     PinnableSlice* value) override;
+                     std::string* value) override;
 
   // TODO: Implement ReadOnly MultiGet?
 
@@ -81,6 +79,7 @@ class DBImplReadOnly : public DBImpl {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
+#ifndef ROCKSDB_LITE
   virtual Status DisableFileDeletions() override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
@@ -93,6 +92,7 @@ class DBImplReadOnly : public DBImpl {
                               bool flush_memtable = true) override {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
+#endif  // ROCKSDB_LITE
 
   using DBImpl::Flush;
   virtual Status Flush(const FlushOptions& options,
@@ -105,14 +105,6 @@ class DBImplReadOnly : public DBImpl {
     return Status::NotSupported("Not supported operation in read only mode.");
   }
 
-  using DB::IngestExternalFile;
-  virtual Status IngestExternalFile(
-      ColumnFamilyHandle* column_family,
-      const std::vector<std::string>& external_files,
-      const IngestExternalFileOptions& ingestion_options) override {
-    return Status::NotSupported("Not supported operation in read only mode.");
-  }
-
  private:
   friend class DB;
 

db/db_impl_write.cc

@@ -1,973 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#include "db/db_impl.h"
-
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-#include <inttypes.h>
-#include "monitoring/perf_context_imp.h"
-#include "options/options_helper.h"
-#include "util/sync_point.h"
-
-namespace rocksdb {
-// Convenience methods
-Status DBImpl::Put(const WriteOptions& o, ColumnFamilyHandle* column_family,
-                   const Slice& key, const Slice& val) {
-  return DB::Put(o, column_family, key, val);
-}
-
-Status DBImpl::Merge(const WriteOptions& o, ColumnFamilyHandle* column_family,
-                     const Slice& key, const Slice& val) {
-  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
-  if (!cfh->cfd()->ioptions()->merge_operator) {
-    return Status::NotSupported("Provide a merge_operator when opening DB");
-  } else {
-    return DB::Merge(o, column_family, key, val);
-  }
-}
-
-Status DBImpl::Delete(const WriteOptions& write_options,
-                      ColumnFamilyHandle* column_family, const Slice& key) {
-  return DB::Delete(write_options, column_family, key);
-}
-
-Status DBImpl::SingleDelete(const WriteOptions& write_options,
-                            ColumnFamilyHandle* column_family,
-                            const Slice& key) {
-  return DB::SingleDelete(write_options, column_family, key);
-}
-
-Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
-  return WriteImpl(write_options, my_batch, nullptr, nullptr);
-}
-
-#ifndef ROCKSDB_LITE
-Status DBImpl::WriteWithCallback(const WriteOptions& write_options,
-                                 WriteBatch* my_batch,
-                                 WriteCallback* callback) {
-  return WriteImpl(write_options, my_batch, callback, nullptr);
-}
-#endif  // ROCKSDB_LITE
-
-Status DBImpl::WriteImpl(const WriteOptions& write_options,
-                         WriteBatch* my_batch, WriteCallback* callback,
-                         uint64_t* log_used, uint64_t log_ref,
-                         bool disable_memtable) {
-  if (my_batch == nullptr) {
-    return Status::Corruption("Batch is nullptr!");
-  }
-
-  if (immutable_db_options_.enable_pipelined_write) {
-    return PipelinedWriteImpl(write_options, my_batch, callback, log_used,
-                              log_ref, disable_memtable);
-  }
-
-  Status status;
-
-  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        disable_memtable);
-
-  if (!write_options.disableWAL) {
-    RecordTick(stats_, WRITE_WITH_WAL);
-  }
-
-  StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
-
-  write_thread_.JoinBatchGroup(&w);
-  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
-    // we are a non-leader in a parallel group
-    PERF_TIMER_GUARD(write_memtable_time);
-
-    if (w.ShouldWriteToMemtable()) {
-      ColumnFamilyMemTablesImpl column_family_memtables(
-          versions_->GetColumnFamilySet());
-      WriteBatchInternal::SetSequence(w.batch, w.sequence);
-      w.status = WriteBatchInternal::InsertInto(
-          &w, &column_family_memtables, &flush_scheduler_,
-          write_options.ignore_missing_column_families, 0 /*log_number*/, this,
-          true /*concurrent_memtable_writes*/);
-    }
-
-    if (write_thread_.CompleteParallelMemTableWriter(&w)) {
-      // we're responsible for exit batch group
-      auto last_sequence = w.write_group->last_sequence;
-      versions_->SetLastSequence(last_sequence);
-      MemTableInsertStatusCheck(w.status);
-      write_thread_.ExitAsBatchGroupFollower(&w);
-    }
-    assert(w.state == WriteThread::STATE_COMPLETED);
-    // STATE_COMPLETED conditional below handles exit
-
-    status = w.FinalStatus();
-  }
-  if (w.state == WriteThread::STATE_COMPLETED) {
-    if (log_used != nullptr) {
-      *log_used = w.log_used;
-    }
-    // write is complete and leader has updated sequence
-    return w.FinalStatus();
-  }
-  // else we are the leader of the write batch group
-  assert(w.state == WriteThread::STATE_GROUP_LEADER);
-
-  // Once reaches this point, the current writer "w" will try to do its write
-  // job.  It may also pick up some of the remaining writers in the "writers_"
-  // when it finds suitable, and finish them in the same write batch.
-  // This is how a write job could be done by the other writer.
-  WriteContext write_context;
-  WriteThread::WriteGroup write_group;
-  bool in_parallel_group = false;
-  uint64_t last_sequence = versions_->LastSequence();
-
-  mutex_.Lock();
-
-  bool need_log_sync = !write_options.disableWAL && write_options.sync;
-  bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
-  status = PreprocessWrite(write_options, &need_log_sync, &write_context);
-  log::Writer* cur_log_writer = logs_.back().writer;
-
-  mutex_.Unlock();
-
-  // Add to log and apply to memtable.  We can release the lock
-  // during this phase since &w is currently responsible for logging
-  // and protects against concurrent loggers and concurrent writes
-  // into memtables
-
-  last_batch_group_size_ =
-      write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
-
-  if (status.ok()) {
-    // Rules for when we can update the memtable concurrently
-    // 1. supported by memtable
-    // 2. Puts are not okay if inplace_update_support
-    // 3. Merges are not okay
-    //
-    // Rules 1..2 are enforced by checking the options
-    // during startup (CheckConcurrentWritesSupported), so if
-    // options.allow_concurrent_memtable_write is true then they can be
-    // assumed to be true.  Rule 3 is checked for each batch.  We could
-    // relax rules 2 if we could prevent write batches from referring
-    // more than once to a particular key.
-    bool parallel = immutable_db_options_.allow_concurrent_memtable_write &&
-                    write_group.size > 1;
-    int total_count = 0;
-    uint64_t total_byte_size = 0;
-    for (auto* writer : write_group) {
-      if (writer->CheckCallback(this)) {
-        if (writer->ShouldWriteToMemtable()) {
-          total_count += WriteBatchInternal::Count(writer->batch);
-          parallel = parallel && !writer->batch->HasMerge();
-        }
-
-        total_byte_size = WriteBatchInternal::AppendedByteSize(
-            total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
-      }
-    }
-
-    const SequenceNumber current_sequence = last_sequence + 1;
-    last_sequence += total_count;
-
-    // Update stats while we are an exclusive group leader, so we know
-    // that nobody else can be writing to these particular stats.
-    // We're optimistic, updating the stats before we successfully
-    // commit.  That lets us release our leader status early.
-    auto stats = default_cf_internal_stats_;
-    stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count);
-    RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);
-    stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size);
-    RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
-    stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1);
-    RecordTick(stats_, WRITE_DONE_BY_SELF);
-    auto write_done_by_other = write_group.size - 1;
-    if (write_done_by_other > 0) {
-      stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER,
-                        write_done_by_other);
-      RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other);
-    }
-    MeasureTime(stats_, BYTES_PER_WRITE, total_byte_size);
-
-    if (write_options.disableWAL) {
-      has_unpersisted_data_.store(true, std::memory_order_relaxed);
-    }
-
-    PERF_TIMER_STOP(write_pre_and_post_process_time);
-
-    if (status.ok() && !write_options.disableWAL) {
-      PERF_TIMER_GUARD(write_wal_time);
-      status = WriteToWAL(write_group, cur_log_writer, need_log_sync,
-                          need_log_dir_sync, current_sequence);
-      if (log_used != nullptr) {
-        *log_used = logfile_number_;
-      }
-    }
-
-    if (status.ok()) {
-      PERF_TIMER_GUARD(write_memtable_time);
-
-      if (!parallel) {
-        w.status = WriteBatchInternal::InsertInto(
-            write_group, current_sequence, column_family_memtables_.get(),
-            &flush_scheduler_, write_options.ignore_missing_column_families,
-            0 /*recovery_log_number*/, this);
-      } else {
-        SequenceNumber next_sequence = current_sequence;
-        for (auto* writer : write_group) {
-          if (writer->ShouldWriteToMemtable()) {
-            writer->sequence = next_sequence;
-            next_sequence += WriteBatchInternal::Count(writer->batch);
-          }
-        }
-        write_group.last_sequence = last_sequence;
-        write_group.running.store(static_cast<uint32_t>(write_group.size),
-                                  std::memory_order_relaxed);
-        write_thread_.LaunchParallelMemTableWriters(&write_group);
-        in_parallel_group = true;
-
-        // Each parallel follower is doing each own writes. The leader should
-        // also do its own.
-        if (w.ShouldWriteToMemtable()) {
-          ColumnFamilyMemTablesImpl column_family_memtables(
-              versions_->GetColumnFamilySet());
-          assert(w.sequence == current_sequence);
-          WriteBatchInternal::SetSequence(w.batch, w.sequence);
-          w.status = WriteBatchInternal::InsertInto(
-              &w, &column_family_memtables, &flush_scheduler_,
-              write_options.ignore_missing_column_families, 0 /*log_number*/,
-              this, true /*concurrent_memtable_writes*/);
-        }
-      }
-    }
-  }
-  PERF_TIMER_START(write_pre_and_post_process_time);
-
-  if (!w.CallbackFailed()) {
-    ParanoidCheck(status);
-  }
-
-  if (need_log_sync) {
-    mutex_.Lock();
-    MarkLogsSynced(logfile_number_, need_log_dir_sync, status);
-    mutex_.Unlock();
-  }
-
-  bool should_exit_batch_group = true;
-  if (in_parallel_group) {
-    // CompleteParallelWorker returns true if this thread should
-    // handle exit, false means somebody else did
-    should_exit_batch_group = write_thread_.CompleteParallelMemTableWriter(&w);
-  }
-  if (should_exit_batch_group) {
-    versions_->SetLastSequence(last_sequence);
-    MemTableInsertStatusCheck(w.status);
-    write_thread_.ExitAsBatchGroupLeader(write_group, w.status);
-  }
-
-  if (status.ok()) {
-    status = w.FinalStatus();
-  }
-  return status;
-}
-
-Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
-                                  WriteBatch* my_batch, WriteCallback* callback,
-                                  uint64_t* log_used, uint64_t log_ref,
-                                  bool disable_memtable) {
-  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
-
-  WriteContext write_context;
-
-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        disable_memtable);
-  write_thread_.JoinBatchGroup(&w);
-  if (w.state == WriteThread::STATE_GROUP_LEADER) {
-    WriteThread::WriteGroup wal_write_group;
-    if (w.callback && !w.callback->AllowWriteBatching()) {
-      write_thread_.WaitForMemTableWriters();
-    }
-    mutex_.Lock();
-    bool need_log_sync = !write_options.disableWAL && write_options.sync;
-    bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
-    w.status = PreprocessWrite(write_options, &need_log_sync, &write_context);
-    log::Writer* cur_log_writer = logs_.back().writer;
-    mutex_.Unlock();
-
-    // This can set non-OK status if callback fail.
-    last_batch_group_size_ =
-        write_thread_.EnterAsBatchGroupLeader(&w, &wal_write_group);
-    const SequenceNumber current_sequence =
-        write_thread_.UpdateLastSequence(versions_->LastSequence()) + 1;
-    size_t total_count = 0;
-    size_t total_byte_size = 0;
-
-    if (w.status.ok()) {
-      SequenceNumber next_sequence = current_sequence;
-      for (auto writer : wal_write_group) {
-        if (writer->CheckCallback(this)) {
-          if (writer->ShouldWriteToMemtable()) {
-            writer->sequence = next_sequence;
-            size_t count = WriteBatchInternal::Count(writer->batch);
-            next_sequence += count;
-            total_count += count;
-          }
-          total_byte_size = WriteBatchInternal::AppendedByteSize(
-              total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
-        }
-      }
-      if (w.disable_wal) {
-        has_unpersisted_data_.store(true, std::memory_order_relaxed);
-      }
-      write_thread_.UpdateLastSequence(current_sequence + total_count - 1);
-    }
-
-    auto stats = default_cf_internal_stats_;
-    stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count);
-    RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);
-    stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size);
-    RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
-
-    PERF_TIMER_STOP(write_pre_and_post_process_time);
-
-    if (w.ShouldWriteToWAL()) {
-      PERF_TIMER_GUARD(write_wal_time);
-      stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1);
-      RecordTick(stats_, WRITE_DONE_BY_SELF, 1);
-      if (wal_write_group.size > 1) {
-        stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER,
-                          wal_write_group.size - 1);
-        RecordTick(stats_, WRITE_DONE_BY_OTHER, wal_write_group.size - 1);
-      }
-      w.status = WriteToWAL(wal_write_group, cur_log_writer, need_log_sync,
-                            need_log_dir_sync, current_sequence);
-    }
-
-    if (!w.CallbackFailed()) {
-      ParanoidCheck(w.status);
-    }
-
-    if (need_log_sync) {
-      mutex_.Lock();
-      MarkLogsSynced(logfile_number_, need_log_dir_sync, w.status);
-      mutex_.Unlock();
-    }
-
-    write_thread_.ExitAsBatchGroupLeader(wal_write_group, w.status);
-  }
-
-  WriteThread::WriteGroup memtable_write_group;
-  if (w.state == WriteThread::STATE_MEMTABLE_WRITER_LEADER) {
-    PERF_TIMER_GUARD(write_memtable_time);
-    assert(w.status.ok());
-    write_thread_.EnterAsMemTableWriter(&w, &memtable_write_group);
-    if (memtable_write_group.size > 1 &&
-        immutable_db_options_.allow_concurrent_memtable_write) {
-      write_thread_.LaunchParallelMemTableWriters(&memtable_write_group);
-    } else {
-      memtable_write_group.status = WriteBatchInternal::InsertInto(
-          memtable_write_group, w.sequence, column_family_memtables_.get(),
-          &flush_scheduler_, write_options.ignore_missing_column_families,
-          0 /*log_number*/, this);
-      versions_->SetLastSequence(memtable_write_group.last_sequence);
-      write_thread_.ExitAsMemTableWriter(&w, memtable_write_group);
-    }
-  }
-
-  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
-    assert(w.ShouldWriteToMemtable());
-    WriteBatchInternal::SetSequence(w.batch, w.sequence);
-    ColumnFamilyMemTablesImpl column_family_memtables(
-        versions_->GetColumnFamilySet());
-    w.status = WriteBatchInternal::InsertInto(
-        &w, &column_family_memtables, &flush_scheduler_,
-        write_options.ignore_missing_column_families, 0 /*log_number*/, this,
-        true /*concurrent_memtable_writes*/);
-    if (write_thread_.CompleteParallelMemTableWriter(&w)) {
-      MemTableInsertStatusCheck(w.status);
-      versions_->SetLastSequence(w.write_group->last_sequence);
-      write_thread_.ExitAsMemTableWriter(&w, *w.write_group);
-    }
-  }
-
-  assert(w.state == WriteThread::STATE_COMPLETED);
-  if (log_used != nullptr) {
-    *log_used = w.log_used;
-  }
-
-  return w.FinalStatus();
-}
-
-void DBImpl::ParanoidCheck(const Status& status) {
-  // Is setting bg_error_ enough here?  This will at least stop
-  // compaction and fail any further writes.
-  if (immutable_db_options_.paranoid_checks && !status.ok() &&
-      !status.IsBusy() && !status.IsIncomplete()) {
-    mutex_.Lock();
-    if (bg_error_.ok()) {
-      bg_error_ = status;  // stop compaction & fail any further writes
-    }
-    mutex_.Unlock();
-  }
-}
-
-void DBImpl::MemTableInsertStatusCheck(const Status& status) {
-  // A non-OK status here indicates that the state implied by the
-  // WAL has diverged from the in-memory state.  This could be
-  // because of a corrupt write_batch (very bad), or because the
-  // client specified an invalid column family and didn't specify
-  // ignore_missing_column_families.
-  if (!status.ok()) {
-    mutex_.Lock();
-    assert(bg_error_.ok());
-    bg_error_ = status;
-    mutex_.Unlock();
-  }
-}
-
-Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
-                               bool* need_log_sync,
-                               WriteContext* write_context) {
-  mutex_.AssertHeld();
-  assert(write_context != nullptr && need_log_sync != nullptr);
-  Status status;
-
-  assert(!single_column_family_mode_ ||
-         versions_->GetColumnFamilySet()->NumberOfColumnFamilies() == 1);
-  if (UNLIKELY(status.ok() && !single_column_family_mode_ &&
-               total_log_size_ > GetMaxTotalWalSize())) {
-    status = HandleWALFull(write_context);
-  }
-
-  if (UNLIKELY(status.ok() && write_buffer_manager_->ShouldFlush())) {
-    // Before a new memtable is added in SwitchMemtable(),
-    // write_buffer_manager_->ShouldFlush() will keep returning true. If another
-    // thread is writing to another DB with the same write buffer, they may also
-    // be flushed. We may end up with flushing much more DBs than needed. It's
-    // suboptimal but still correct.
-    status = HandleWriteBufferFull(write_context);
-  }
-
-  if (UNLIKELY(status.ok() && !bg_error_.ok())) {
-    return bg_error_;
-  }
-
-  if (UNLIKELY(status.ok() && !flush_scheduler_.Empty())) {
-    status = ScheduleFlushes(write_context);
-  }
-
-  if (UNLIKELY(status.ok() && (write_controller_.IsStopped() ||
-                               write_controller_.NeedsDelay()))) {
-    PERF_TIMER_GUARD(write_delay_time);
-    // We don't know size of curent batch so that we always use the size
-    // for previous one. It might create a fairness issue that expiration
-    // might happen for smaller writes but larger writes can go through.
-    // Can optimize it if it is an issue.
-    status = DelayWrite(last_batch_group_size_, write_options);
-  }
-
-  if (status.ok() && *need_log_sync) {
-    // Wait until the parallel syncs are finished. Any sync process has to sync
-    // the front log too so it is enough to check the status of front()
-    // We do a while loop since log_sync_cv_ is signalled when any sync is
-    // finished
-    // Note: there does not seem to be a reason to wait for parallel sync at
-    // this early step but it is not important since parallel sync (SyncWAL) and
-    // need_log_sync are usually not used together.
-    while (logs_.front().getting_synced) {
-      log_sync_cv_.Wait();
-    }
-    for (auto& log : logs_) {
-      assert(!log.getting_synced);
-      // This is just to prevent the logs to be synced by a parallel SyncWAL
-      // call. We will do the actual syncing later after we will write to the
-      // WAL.
-      // Note: there does not seem to be a reason to set this early before we
-      // actually write to the WAL
-      log.getting_synced = true;
-    }
-  } else {
-    *need_log_sync = false;
-  }
-
-  return status;
-}
-
-Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
-                          log::Writer* log_writer, bool need_log_sync,
-                          bool need_log_dir_sync, SequenceNumber sequence) {
-  Status status;
-
-  WriteBatch* merged_batch = nullptr;
-  size_t write_with_wal = 0;
-  auto* leader = write_group.leader;
-  if (write_group.size == 1 && leader->ShouldWriteToWAL() &&
-      leader->batch->GetWalTerminationPoint().is_cleared()) {
-    // we simply write the first WriteBatch to WAL if the group only
-    // contains one batch, that batch should be written to the WAL,
-    // and the batch is not wanting to be truncated
-    merged_batch = leader->batch;
-    leader->log_used = logfile_number_;
-    write_with_wal = 1;
-  } else {
-    // WAL needs all of the batches flattened into a single batch.
-    // We could avoid copying here with an iov-like AddRecord
-    // interface
-    merged_batch = &tmp_batch_;
-    for (auto writer : write_group) {
-      if (writer->ShouldWriteToWAL()) {
-        WriteBatchInternal::Append(merged_batch, writer->batch,
-                                   /*WAL_only*/ true);
-        write_with_wal++;
-      }
-      writer->log_used = logfile_number_;
-    }
-  }
-
-  WriteBatchInternal::SetSequence(merged_batch, sequence);
-
-  Slice log_entry = WriteBatchInternal::Contents(merged_batch);
-  status = log_writer->AddRecord(log_entry);
-  total_log_size_ += log_entry.size();
-  alive_log_files_.back().AddSize(log_entry.size());
-  log_empty_ = false;
-  uint64_t log_size = log_entry.size();
-
-  if (status.ok() && need_log_sync) {
-    StopWatch sw(env_, stats_, WAL_FILE_SYNC_MICROS);
-    // It's safe to access logs_ with unlocked mutex_ here because:
-    //  - we've set getting_synced=true for all logs,
-    //    so other threads won't pop from logs_ while we're here,
-    //  - only writer thread can push to logs_, and we're in
-    //    writer thread, so no one will push to logs_,
-    //  - as long as other threads don't modify it, it's safe to read
-    //    from std::deque from multiple threads concurrently.
-    for (auto& log : logs_) {
-      status = log.writer->file()->Sync(immutable_db_options_.use_fsync);
-      if (!status.ok()) {
-        break;
-      }
-    }
-    if (status.ok() && need_log_dir_sync) {
-      // We only sync WAL directory the first time WAL syncing is
-      // requested, so that in case users never turn on WAL sync,
-      // we can avoid the disk I/O in the write code path.
-      status = directories_.GetWalDir()->Fsync();
-    }
-  }
-
-  if (merged_batch == &tmp_batch_) {
-    tmp_batch_.Clear();
-  }
-  if (status.ok()) {
-    auto stats = default_cf_internal_stats_;
-    if (need_log_sync) {
-      stats->AddDBStats(InternalStats::WAL_FILE_SYNCED, 1);
-      RecordTick(stats_, WAL_FILE_SYNCED);
-    }
-    stats->AddDBStats(InternalStats::WAL_FILE_BYTES, log_size);
-    RecordTick(stats_, WAL_FILE_BYTES, log_size);
-    stats->AddDBStats(InternalStats::WRITE_WITH_WAL, write_with_wal);
-    RecordTick(stats_, WRITE_WITH_WAL, write_with_wal);
-  }
-  return status;
-}
-
-Status DBImpl::HandleWALFull(WriteContext* write_context) {
-  mutex_.AssertHeld();
-  assert(write_context != nullptr);
-  Status status;
-
-  if (alive_log_files_.begin()->getting_flushed) {
-    return status;
-  }
-
-  auto oldest_alive_log = alive_log_files_.begin()->number;
-  auto oldest_log_with_uncommited_prep = FindMinLogContainingOutstandingPrep();
-
-  if (allow_2pc() &&
-      oldest_log_with_uncommited_prep > 0 &&
-      oldest_log_with_uncommited_prep <= oldest_alive_log) {
-    if (unable_to_flush_oldest_log_) {
-        // we already attempted to flush all column families dependent on
-        // the oldest alive log but the log still contained uncommited transactions.
-        // the oldest alive log STILL contains uncommited transaction so there
-        // is still nothing that we can do.
-        return status;
-    } else {
-      ROCKS_LOG_WARN(
-          immutable_db_options_.info_log,
-          "Unable to release oldest log due to uncommited transaction");
-      unable_to_flush_oldest_log_ = true;
-    }
-  } else {
-    // we only mark this log as getting flushed if we have successfully
-    // flushed all data in this log. If this log contains outstanding prepared
-    // transactions then we cannot flush this log until those transactions are commited.
-    unable_to_flush_oldest_log_ = false;
-    alive_log_files_.begin()->getting_flushed = true;
-  }
-
-  ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                 "Flushing all column families with data in WAL number %" PRIu64
-                 ". Total log size is %" PRIu64
-                 " while max_total_wal_size is %" PRIu64,
-                 oldest_alive_log, total_log_size_.load(), GetMaxTotalWalSize());
-  // no need to refcount because drop is happening in write thread, so can't
-  // happen while we're in the write thread
-  for (auto cfd : *versions_->GetColumnFamilySet()) {
-    if (cfd->IsDropped()) {
-      continue;
-    }
-    if (cfd->OldestLogToKeep() <= oldest_alive_log) {
-      status = SwitchMemtable(cfd, write_context);
-      if (!status.ok()) {
-        break;
-      }
-      cfd->imm()->FlushRequested();
-      SchedulePendingFlush(cfd);
-    }
-  }
-  MaybeScheduleFlushOrCompaction();
-  return status;
-}
-
-Status DBImpl::HandleWriteBufferFull(WriteContext* write_context) {
-  mutex_.AssertHeld();
-  assert(write_context != nullptr);
-  Status status;
-
-  // Before a new memtable is added in SwitchMemtable(),
-  // write_buffer_manager_->ShouldFlush() will keep returning true. If another
-  // thread is writing to another DB with the same write buffer, they may also
-  // be flushed. We may end up with flushing much more DBs than needed. It's
-  // suboptimal but still correct.
-  ROCKS_LOG_INFO(
-      immutable_db_options_.info_log,
-      "Flushing column family with largest mem table size. Write buffer is "
-      "using %" PRIu64 " bytes out of a total of %" PRIu64 ".",
-      write_buffer_manager_->memory_usage(),
-      write_buffer_manager_->buffer_size());
-  // no need to refcount because drop is happening in write thread, so can't
-  // happen while we're in the write thread
-  ColumnFamilyData* cfd_picked = nullptr;
-  SequenceNumber seq_num_for_cf_picked = kMaxSequenceNumber;
-
-  for (auto cfd : *versions_->GetColumnFamilySet()) {
-    if (cfd->IsDropped()) {
-      continue;
-    }
-    if (!cfd->mem()->IsEmpty()) {
-      // We only consider active mem table, hoping immutable memtable is
-      // already in the process of flushing.
-      uint64_t seq = cfd->mem()->GetCreationSeq();
-      if (cfd_picked == nullptr || seq < seq_num_for_cf_picked) {
-        cfd_picked = cfd;
-        seq_num_for_cf_picked = seq;
-      }
-    }
-  }
-  if (cfd_picked != nullptr) {
-    status = SwitchMemtable(cfd_picked, write_context);
-    if (status.ok()) {
-      cfd_picked->imm()->FlushRequested();
-      SchedulePendingFlush(cfd_picked);
-      MaybeScheduleFlushOrCompaction();
-    }
-  }
-  return status;
-}
-
-uint64_t DBImpl::GetMaxTotalWalSize() const {
-  mutex_.AssertHeld();
-  return mutable_db_options_.max_total_wal_size == 0
-             ? 4 * max_total_in_memory_state_
-             : mutable_db_options_.max_total_wal_size;
-}
-
-// REQUIRES: mutex_ is held
-// REQUIRES: this thread is currently at the front of the writer queue
-Status DBImpl::DelayWrite(uint64_t num_bytes,
-                          const WriteOptions& write_options) {
-  uint64_t time_delayed = 0;
-  bool delayed = false;
-  {
-    StopWatch sw(env_, stats_, WRITE_STALL, &time_delayed);
-    uint64_t delay = write_controller_.GetDelay(env_, num_bytes);
-    if (delay > 0) {
-      if (write_options.no_slowdown) {
-        return Status::Incomplete();
-      }
-      TEST_SYNC_POINT("DBImpl::DelayWrite:Sleep");
-
-      mutex_.Unlock();
-      // We will delay the write until we have slept for delay ms or
-      // we don't need a delay anymore
-      const uint64_t kDelayInterval = 1000;
-      uint64_t stall_end = sw.start_time() + delay;
-      while (write_controller_.NeedsDelay()) {
-        if (env_->NowMicros() >= stall_end) {
-          // We already delayed this write `delay` microseconds
-          break;
-        }
-
-        delayed = true;
-        // Sleep for 0.001 seconds
-        env_->SleepForMicroseconds(kDelayInterval);
-      }
-      mutex_.Lock();
-    }
-
-    while (bg_error_.ok() && write_controller_.IsStopped()) {
-      if (write_options.no_slowdown) {
-        return Status::Incomplete();
-      }
-      delayed = true;
-      TEST_SYNC_POINT("DBImpl::DelayWrite:Wait");
-      bg_cv_.Wait();
-    }
-  }
-  assert(!delayed || !write_options.no_slowdown);
-  if (delayed) {
-    default_cf_internal_stats_->AddDBStats(InternalStats::WRITE_STALL_MICROS,
-                                           time_delayed);
-    RecordTick(stats_, STALL_MICROS, time_delayed);
-  }
-
-  return bg_error_;
-}
-
-Status DBImpl::ScheduleFlushes(WriteContext* context) {
-  ColumnFamilyData* cfd;
-  while ((cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {
-    auto status = SwitchMemtable(cfd, context);
-    if (cfd->Unref()) {
-      delete cfd;
-    }
-    if (!status.ok()) {
-      return status;
-    }
-  }
-  return Status::OK();
-}
-
-#ifndef ROCKSDB_LITE
-void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* cfd,
-                                    const MemTableInfo& mem_table_info) {
-  if (immutable_db_options_.listeners.size() == 0U) {
-    return;
-  }
-  if (shutting_down_.load(std::memory_order_acquire)) {
-    return;
-  }
-
-  for (auto listener : immutable_db_options_.listeners) {
-    listener->OnMemTableSealed(mem_table_info);
-  }
-}
-#endif  // ROCKSDB_LITE
-
-// REQUIRES: mutex_ is held
-// REQUIRES: this thread is currently at the front of the writer queue
-Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
-  mutex_.AssertHeld();
-  unique_ptr<WritableFile> lfile;
-  log::Writer* new_log = nullptr;
-  MemTable* new_mem = nullptr;
-
-  // In case of pipelined write is enabled, wait for all pending memtable
-  // writers.
-  if (immutable_db_options_.enable_pipelined_write) {
-    write_thread_.WaitForMemTableWriters();
-  }
-
-  // Attempt to switch to a new memtable and trigger flush of old.
-  // Do this without holding the dbmutex lock.
-  assert(versions_->prev_log_number() == 0);
-  bool creating_new_log = !log_empty_;
-  uint64_t recycle_log_number = 0;
-  if (creating_new_log && immutable_db_options_.recycle_log_file_num &&
-      !log_recycle_files.empty()) {
-    recycle_log_number = log_recycle_files.front();
-    log_recycle_files.pop_front();
-  }
-  uint64_t new_log_number =
-      creating_new_log ? versions_->NewFileNumber() : logfile_number_;
-  SuperVersion* new_superversion = nullptr;
-  const MutableCFOptions mutable_cf_options = *cfd->GetLatestMutableCFOptions();
-
-  // Set current_memtble_info for memtable sealed callback
-#ifndef ROCKSDB_LITE
-  MemTableInfo memtable_info;
-  memtable_info.cf_name = cfd->GetName();
-  memtable_info.first_seqno = cfd->mem()->GetFirstSequenceNumber();
-  memtable_info.earliest_seqno = cfd->mem()->GetEarliestSequenceNumber();
-  memtable_info.num_entries = cfd->mem()->num_entries();
-  memtable_info.num_deletes = cfd->mem()->num_deletes();
-#endif  // ROCKSDB_LITE
-  // Log this later after lock release. It may be outdated, e.g., if background
-  // flush happens before logging, but that should be ok.
-  int num_imm_unflushed = cfd->imm()->NumNotFlushed();
-  DBOptions db_options =
-      BuildDBOptions(immutable_db_options_, mutable_db_options_);
-  const auto preallocate_block_size =
-    GetWalPreallocateBlockSize(mutable_cf_options.write_buffer_size);
-  mutex_.Unlock();
-  Status s;
-  {
-    if (creating_new_log) {
-      EnvOptions opt_env_opt =
-          env_->OptimizeForLogWrite(env_options_, db_options);
-      if (recycle_log_number) {
-        ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                       "reusing log %" PRIu64 " from recycle list\n",
-                       recycle_log_number);
-        s = env_->ReuseWritableFile(
-            LogFileName(immutable_db_options_.wal_dir, new_log_number),
-            LogFileName(immutable_db_options_.wal_dir, recycle_log_number),
-            &lfile, opt_env_opt);
-      } else {
-        s = NewWritableFile(
-            env_, LogFileName(immutable_db_options_.wal_dir, new_log_number),
-            &lfile, opt_env_opt);
-      }
-      if (s.ok()) {
-        // Our final size should be less than write_buffer_size
-        // (compression, etc) but err on the side of caution.
-
-        // use preallocate_block_size instead
-        // of calling GetWalPreallocateBlockSize()
-        lfile->SetPreallocationBlockSize(preallocate_block_size);
-        unique_ptr<WritableFileWriter> file_writer(
-            new WritableFileWriter(std::move(lfile), opt_env_opt));
-        new_log =
-            new log::Writer(std::move(file_writer), new_log_number,
-                            immutable_db_options_.recycle_log_file_num > 0);
-      }
-    }
-
-    if (s.ok()) {
-      SequenceNumber seq = versions_->LastSequence();
-      new_mem = cfd->ConstructNewMemtable(mutable_cf_options, seq);
-      new_superversion = new SuperVersion();
-    }
-
-#ifndef ROCKSDB_LITE
-    // PLEASE NOTE: We assume that there are no failable operations
-    // after lock is acquired below since we are already notifying
-    // client about mem table becoming immutable.
-    NotifyOnMemTableSealed(cfd, memtable_info);
-#endif //ROCKSDB_LITE
-  }
-  ROCKS_LOG_INFO(immutable_db_options_.info_log,
-                 "[%s] New memtable created with log file: #%" PRIu64
-                 ". Immutable memtables: %d.\n",
-                 cfd->GetName().c_str(), new_log_number, num_imm_unflushed);
-  mutex_.Lock();
-  if (!s.ok()) {
-    // how do we fail if we're not creating new log?
-    assert(creating_new_log);
-    assert(!new_mem);
-    assert(!new_log);
-    return s;
-  }
-  if (creating_new_log) {
-    logfile_number_ = new_log_number;
-    assert(new_log != nullptr);
-    log_empty_ = true;
-    log_dir_synced_ = false;
-    logs_.emplace_back(logfile_number_, new_log);
-    alive_log_files_.push_back(LogFileNumberSize(logfile_number_));
-  }
-  for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
-    // all this is just optimization to delete logs that
-    // are no longer needed -- if CF is empty, that means it
-    // doesn't need that particular log to stay alive, so we just
-    // advance the log number. no need to persist this in the manifest
-    if (loop_cfd->mem()->GetFirstSequenceNumber() == 0 &&
-        loop_cfd->imm()->NumNotFlushed() == 0) {
-      if (creating_new_log) {
-        loop_cfd->SetLogNumber(logfile_number_);
-      }
-      loop_cfd->mem()->SetCreationSeq(versions_->LastSequence());
-    }
-  }
-
-  cfd->mem()->SetNextLogNumber(logfile_number_);
-  cfd->imm()->Add(cfd->mem(), &context->memtables_to_free_);
-  new_mem->Ref();
-  cfd->SetMemtable(new_mem);
-  context->superversions_to_free_.push_back(InstallSuperVersionAndScheduleWork(
-      cfd, new_superversion, mutable_cf_options));
-  return s;
-}
-
-size_t DBImpl::GetWalPreallocateBlockSize(uint64_t write_buffer_size) const {
-  mutex_.AssertHeld();
-  size_t bsize = write_buffer_size / 10 + write_buffer_size;
-  // Some users might set very high write_buffer_size and rely on
-  // max_total_wal_size or other parameters to control the WAL size.
-  if (mutable_db_options_.max_total_wal_size > 0) {
-    bsize = std::min<size_t>(bsize, mutable_db_options_.max_total_wal_size);
-  }
-  if (immutable_db_options_.db_write_buffer_size > 0) {
-    bsize = std::min<size_t>(bsize, immutable_db_options_.db_write_buffer_size);
-  }
-  if (immutable_db_options_.write_buffer_manager &&
-      immutable_db_options_.write_buffer_manager->enabled()) {
-    bsize = std::min<size_t>(
-        bsize, immutable_db_options_.write_buffer_manager->buffer_size());
-  }
-
-  return bsize;
-}
-
-// Default implementations of convenience methods that subclasses of DB
-// can call if they wish
-Status DB::Put(const WriteOptions& opt, ColumnFamilyHandle* column_family,
-               const Slice& key, const Slice& value) {
-  // Pre-allocate size of write batch conservatively.
-  // 8 bytes are taken by header, 4 bytes for count, 1 byte for type,
-  // and we allocate 11 extra bytes for key length, as well as value length.
-  WriteBatch batch(key.size() + value.size() + 24);
-  batch.Put(column_family, key, value);
-  return Write(opt, &batch);
-}
-
-Status DB::Delete(const WriteOptions& opt, ColumnFamilyHandle* column_family,
-                  const Slice& key) {
-  WriteBatch batch;
-  batch.Delete(column_family, key);
-  return Write(opt, &batch);
-}
-
-Status DB::SingleDelete(const WriteOptions& opt,
-                        ColumnFamilyHandle* column_family, const Slice& key) {
-  WriteBatch batch;
-  batch.SingleDelete(column_family, key);
-  return Write(opt, &batch);
-}
-
-Status DB::DeleteRange(const WriteOptions& opt,
-                       ColumnFamilyHandle* column_family,
-                       const Slice& begin_key, const Slice& end_key) {
-  WriteBatch batch;
-  batch.DeleteRange(column_family, begin_key, end_key);
-  return Write(opt, &batch);
-}
-
-Status DB::Merge(const WriteOptions& opt, ColumnFamilyHandle* column_family,
-                 const Slice& key, const Slice& value) {
-  WriteBatch batch;
-  batch.Merge(column_family, key, value);
-  return Write(opt, &batch);
-}
-}  // namespace rocksdb

db/db_info_dumper.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #ifndef __STDC_FORMAT_MACROS
 #define __STDC_FORMAT_MACROS
@@ -17,8 +15,8 @@
 #include <algorithm>
 #include <vector>
 
+#include "db/filename.h"
 #include "rocksdb/env.h"
-#include "util/filename.h"
 
 namespace rocksdb {
 

db/db_info_dumper.h

@@ -2,13 +2,11 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 #pragma once
 
 #include <string>
 
-#include "options/db_options.h"
+#include "util/db_options.h"
 
 namespace rocksdb {
 void DumpDBFileSummary(const ImmutableDBOptions& options,

db/db_inplace_update_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/db_io_failure_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/db_iter.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -16,10 +14,10 @@
 #include <limits>
 
 #include "db/dbformat.h"
+#include "db/filename.h"
 #include "db/merge_context.h"
 #include "db/merge_helper.h"
 #include "db/pinned_iterators_manager.h"
-#include "monitoring/perf_context_imp.h"
 #include "port/port.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
@@ -27,9 +25,9 @@
 #include "rocksdb/options.h"
 #include "table/internal_iterator.h"
 #include "util/arena.h"
-#include "util/filename.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
+#include "util/perf_context_imp.h"
 #include "util/string_util.h"
 
 namespace rocksdb {
@@ -103,32 +101,32 @@ class DBIter: public Iterator {
     uint64_t bytes_read_;
   };
 
-  DBIter(Env* env, const ReadOptions& read_options,
-         const ImmutableCFOptions& cf_options, const Comparator* cmp,
+  DBIter(Env* env, const ImmutableCFOptions& ioptions, const Comparator* cmp,
          InternalIterator* iter, SequenceNumber s, bool arena_mode,
-         uint64_t max_sequential_skip_in_iterations, uint64_t version_number)
+         uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+         const Slice* iterate_upper_bound = nullptr,
+         bool prefix_same_as_start = false, bool pin_data = false,
+         bool total_order_seek = false)
       : arena_mode_(arena_mode),
         env_(env),
-        logger_(cf_options.info_log),
+        logger_(ioptions.info_log),
         user_comparator_(cmp),
-        merge_operator_(cf_options.merge_operator),
+        merge_operator_(ioptions.merge_operator),
         iter_(iter),
         sequence_(s),
         direction_(kForward),
         valid_(false),
         current_entry_is_merged_(false),
-        statistics_(cf_options.statistics),
+        statistics_(ioptions.statistics),
         version_number_(version_number),
-        iterate_upper_bound_(read_options.iterate_upper_bound),
-        prefix_same_as_start_(read_options.prefix_same_as_start),
-        pin_thru_lifetime_(read_options.pin_data),
-        total_order_seek_(read_options.total_order_seek),
-        range_del_agg_(cf_options.internal_comparator, s,
-                       true /* collapse_deletions */) {
+        iterate_upper_bound_(iterate_upper_bound),
+        prefix_same_as_start_(prefix_same_as_start),
+        pin_thru_lifetime_(pin_data),
+        total_order_seek_(total_order_seek),
+        range_del_agg_(ioptions.internal_comparator, s) {
     RecordTick(statistics_, NO_ITERATORS);
-    prefix_extractor_ = cf_options.prefix_extractor;
+    prefix_extractor_ = ioptions.prefix_extractor;
     max_skip_ = max_sequential_skip_in_iterations;
-    max_skippable_internal_keys_ = read_options.max_skippable_internal_keys;
     if (pin_thru_lifetime_) {
       pinned_iters_mgr_.StartPinning();
     }
@@ -161,7 +159,7 @@ class DBIter: public Iterator {
   virtual bool Valid() const override { return valid_; }
   virtual Slice key() const override {
     assert(valid_);
-    return saved_key_.GetUserKey();
+    return saved_key_.GetKey();
   }
   virtual Slice value() const override {
     assert(valid_);
@@ -225,7 +223,6 @@ class DBIter: public Iterator {
   void FindNextUserEntryInternal(bool skipping, bool prefix_check);
   bool ParseKey(ParsedInternalKey* key);
   void MergeValuesNewToOld();
-  bool TooManyInternalKeysSkipped(bool increment = true);
 
   // Temporarily pin the blocks that we encounter until ReleaseTempPinnedData()
   // is called
@@ -251,10 +248,6 @@ class DBIter: public Iterator {
     }
   }
 
-  inline void ResetInternalKeysSkippedCounter() {
-    num_internal_keys_skipped_ = 0;
-  }
-
   const SliceTransform* prefix_extractor_;
   bool arena_mode_;
   Env* const env_;
@@ -274,8 +267,6 @@ class DBIter: public Iterator {
   // for prefix seek mode to support prev()
   Statistics* statistics_;
   uint64_t max_skip_;
-  uint64_t max_skippable_internal_keys_;
-  uint64_t num_internal_keys_skipped_;
   uint64_t version_number_;
   const Slice* iterate_upper_bound_;
   IterKey prefix_start_buf_;
@@ -299,8 +290,9 @@ class DBIter: public Iterator {
 inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
   if (!ParseInternalKey(iter_->key(), ikey)) {
     status_ = Status::Corruption("corrupted internal key in DBIter");
-    ROCKS_LOG_ERROR(logger_, "corrupted internal key in DBIter: %s",
-                    iter_->key().ToString(true).c_str());
+    Log(InfoLogLevel::ERROR_LEVEL,
+        logger_, "corrupted internal key in DBIter: %s",
+        iter_->key().ToString(true).c_str());
     return false;
   } else {
     return true;
@@ -312,7 +304,6 @@ void DBIter::Next() {
 
   // Release temporarily pinned blocks from last operation
   ReleaseTempPinnedData();
-  ResetInternalKeysSkippedCounter();
   if (direction_ == kReverse) {
     ReverseToForward();
   } else if (iter_->Valid() && !current_entry_is_merged_) {
@@ -349,7 +340,6 @@ void DBIter::Next() {
 //
 // NOTE: In between, saved_key_ can point to a user key that has
 //       a delete marker or a sequence number higher than sequence_
-//       saved_key_ MUST have a proper user_key before calling this function
 //
 // The prefix_check parameter controls whether we check the iterated
 // keys against the prefix of the seeked key. Set to false when
@@ -399,14 +389,9 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
       break;
     }
 
-    if (TooManyInternalKeysSkipped()) {
-      return;
-    }
-
     if (ikey.sequence <= sequence_) {
       if (skipping &&
-          user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) <=
-              0) {
+          user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) <= 0) {
         num_skipped++;  // skip this entry
         PERF_COUNTER_ADD(internal_key_skipped_count, 1);
       } else {
@@ -416,19 +401,17 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
           case kTypeSingleDeletion:
             // Arrange to skip all upcoming entries for this key since
             // they are hidden by this deletion.
-            saved_key_.SetUserKey(
+            saved_key_.SetKey(
                 ikey.user_key,
                 !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
             skipping = true;
             PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
             break;
           case kTypeValue:
-            saved_key_.SetUserKey(
+            saved_key_.SetKey(
                 ikey.user_key,
                 !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
-            if (range_del_agg_.ShouldDelete(
-                    ikey, RangeDelAggregator::RangePositioningMode::
-                              kForwardTraversal)) {
+            if (range_del_agg_.ShouldDelete(ikey)) {
               // Arrange to skip all upcoming entries for this key since
               // they are hidden by this deletion.
               skipping = true;
@@ -440,12 +423,10 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
             }
             break;
           case kTypeMerge:
-            saved_key_.SetUserKey(
+            saved_key_.SetKey(
                 ikey.user_key,
                 !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
-            if (range_del_agg_.ShouldDelete(
-                    ikey, RangeDelAggregator::RangePositioningMode::
-                              kForwardTraversal)) {
+            if (range_del_agg_.ShouldDelete(ikey)) {
               // Arrange to skip all upcoming entries for this key since
               // they are hidden by this deletion.
               skipping = true;
@@ -471,13 +452,12 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
 
       // Here saved_key_ may contain some old key, or the default empty key, or
       // key assigned by some random other method. We don't care.
-      if (user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) <=
-          0) {
+      if (user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) <= 0) {
         num_skipped++;
       } else {
-        saved_key_.SetUserKey(
-            ikey.user_key,
-            !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
+        saved_key_.SetKey(
+                ikey.user_key,
+                !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
         skipping = false;
         num_skipped = 0;
       }
@@ -492,8 +472,8 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
         // We're looking for the next user-key but all we see are the same
         // user-key with decreasing sequence numbers. Fast forward to
         // sequence number 0 and type deletion (the smallest type).
-        AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetUserKey(),
-                                                       0, kTypeDeletion));
+        AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetKey(), 0,
+                                                       kTypeDeletion));
         // Don't set skipping = false because we may still see more user-keys
         // equal to saved_key_.
       } else {
@@ -503,7 +483,7 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
         // many times since our snapshot was taken, not the case when a lot of
         // different keys were inserted after our snapshot was taken.
         AppendInternalKey(&last_key,
-                          ParsedInternalKey(saved_key_.GetUserKey(), sequence_,
+                          ParsedInternalKey(saved_key_.GetKey(), sequence_,
                                             kValueTypeForSeek));
       }
       iter_->Seek(last_key);
@@ -523,7 +503,8 @@ void DBIter::FindNextUserEntryInternal(bool skipping, bool prefix_check) {
 //       iter_ points to the next entry (or invalid)
 void DBIter::MergeValuesNewToOld() {
   if (!merge_operator_) {
-    ROCKS_LOG_ERROR(logger_, "Options::merge_operator is null.");
+    Log(InfoLogLevel::ERROR_LEVEL,
+        logger_, "Options::merge_operator is null.");
     status_ = Status::InvalidArgument("merge_operator_ must be set.");
     valid_ = false;
     return;
@@ -544,13 +525,11 @@ void DBIter::MergeValuesNewToOld() {
       continue;
     }
 
-    if (!user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+    if (!user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
       // hit the next user key, stop right here
       break;
     } else if (kTypeDeletion == ikey.type || kTypeSingleDeletion == ikey.type ||
-               range_del_agg_.ShouldDelete(
-                   ikey, RangeDelAggregator::RangePositioningMode::
-                             kForwardTraversal)) {
+               range_del_agg_.ShouldDelete(ikey)) {
       // hit a delete with the same user key, stop right here
       // iter_ is positioned after delete
       iter_->Next();
@@ -584,10 +563,9 @@ void DBIter::MergeValuesNewToOld() {
   // a deletion marker.
   // feed null as the existing value to the merge operator, such that
   // client can differentiate this scenario and do things accordingly.
-  s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
-                                  nullptr, merge_context_.GetOperands(),
-                                  &saved_value_, logger_, statistics_, env_,
-                                  &pinned_value_);
+  s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetKey(), nullptr,
+                                  merge_context_.GetOperands(), &saved_value_,
+                                  logger_, statistics_, env_, &pinned_value_);
   if (!s.ok()) {
     status_ = s;
   }
@@ -596,7 +574,6 @@ void DBIter::MergeValuesNewToOld() {
 void DBIter::Prev() {
   assert(valid_);
   ReleaseTempPinnedData();
-  ResetInternalKeysSkippedCounter();
   if (direction_ == kForward) {
     ReverseToBackward();
   }
@@ -614,36 +591,33 @@ void DBIter::ReverseToForward() {
   if (prefix_extractor_ != nullptr && !total_order_seek_) {
     IterKey last_key;
     last_key.SetInternalKey(ParsedInternalKey(
-        saved_key_.GetUserKey(), kMaxSequenceNumber, kValueTypeForSeek));
-    iter_->Seek(last_key.GetInternalKey());
+        saved_key_.GetKey(), kMaxSequenceNumber, kValueTypeForSeek));
+    iter_->Seek(last_key.GetKey());
   }
   FindNextUserKey();
   direction_ = kForward;
   if (!iter_->Valid()) {
     iter_->SeekToFirst();
-    range_del_agg_.InvalidateTombstoneMapPositions();
   }
 }
 
 void DBIter::ReverseToBackward() {
   if (prefix_extractor_ != nullptr && !total_order_seek_) {
     IterKey last_key;
-    last_key.SetInternalKey(ParsedInternalKey(saved_key_.GetUserKey(), 0,
-                                              kValueTypeForSeekForPrev));
-    iter_->SeekForPrev(last_key.GetInternalKey());
+    last_key.SetInternalKey(
+        ParsedInternalKey(saved_key_.GetKey(), 0, kValueTypeForSeekForPrev));
+    iter_->SeekForPrev(last_key.GetKey());
   }
   if (current_entry_is_merged_) {
     // Not placed in the same key. Need to call Prev() until finding the
     // previous key.
     if (!iter_->Valid()) {
       iter_->SeekToLast();
-      range_del_agg_.InvalidateTombstoneMapPositions();
     }
     ParsedInternalKey ikey;
     FindParseableKey(&ikey, kReverse);
     while (iter_->Valid() &&
-           user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) >
-               0) {
+           user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) > 0) {
       if (ikey.sequence > sequence_) {
         PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
       } else {
@@ -657,8 +631,7 @@ void DBIter::ReverseToBackward() {
   if (iter_->Valid()) {
     ParsedInternalKey ikey;
     assert(ParseKey(&ikey));
-    assert(user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) <=
-           0);
+    assert(user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) <= 0);
   }
 #endif
 
@@ -675,36 +648,29 @@ void DBIter::PrevInternal() {
   ParsedInternalKey ikey;
 
   while (iter_->Valid()) {
-    saved_key_.SetUserKey(
-        ExtractUserKey(iter_->key()),
-        !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
-
+    saved_key_.SetKey(ExtractUserKey(iter_->key()),
+                      !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
     if (FindValueForCurrentKey()) {
       valid_ = true;
       if (!iter_->Valid()) {
         return;
       }
       FindParseableKey(&ikey, kReverse);
-      if (user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+      if (user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
         FindPrevUserKey();
       }
       if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
-          prefix_extractor_->Transform(saved_key_.GetUserKey())
+          prefix_extractor_->Transform(saved_key_.GetKey())
                   .compare(prefix_start_key_) != 0) {
         valid_ = false;
       }
       return;
     }
-
-    if (TooManyInternalKeysSkipped(false)) {
-      return;
-    }
-
     if (!iter_->Valid()) {
       break;
     }
     FindParseableKey(&ikey, kReverse);
-    if (user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+    if (user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
       FindPrevUserKey();
     }
   }
@@ -734,11 +700,7 @@ bool DBIter::FindValueForCurrentKey() {
   TempPinData();
   size_t num_skipped = 0;
   while (iter_->Valid() && ikey.sequence <= sequence_ &&
-         user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
-    if (TooManyInternalKeysSkipped()) {
-      return false;
-    }
-
+         user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
     // We iterate too much: let's use Seek() to avoid too much key comparisons
     if (num_skipped >= max_skip_) {
       return FindValueForCurrentKeyUsingSeek();
@@ -747,9 +709,7 @@ bool DBIter::FindValueForCurrentKey() {
     last_key_entry_type = ikey.type;
     switch (last_key_entry_type) {
       case kTypeValue:
-        if (range_del_agg_.ShouldDelete(
-                ikey,
-                RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+        if (range_del_agg_.ShouldDelete(ikey)) {
           last_key_entry_type = kTypeRangeDeletion;
           PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
         } else {
@@ -766,9 +726,7 @@ bool DBIter::FindValueForCurrentKey() {
         PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
         break;
       case kTypeMerge:
-        if (range_del_agg_.ShouldDelete(
-                ikey,
-                RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+        if (range_del_agg_.ShouldDelete(ikey)) {
           merge_context_.Clear();
           last_key_entry_type = kTypeRangeDeletion;
           last_not_merge_type = last_key_entry_type;
@@ -785,7 +743,7 @@ bool DBIter::FindValueForCurrentKey() {
     }
 
     PERF_COUNTER_ADD(internal_key_skipped_count, 1);
-    assert(user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey()));
+    assert(user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()));
     iter_->Prev();
     ++num_skipped;
     FindParseableKey(&ikey, kReverse);
@@ -803,14 +761,14 @@ bool DBIter::FindValueForCurrentKey() {
       if (last_not_merge_type == kTypeDeletion ||
           last_not_merge_type == kTypeSingleDeletion ||
           last_not_merge_type == kTypeRangeDeletion) {
-        s = MergeHelper::TimedFullMerge(
-            merge_operator_, saved_key_.GetUserKey(), nullptr,
-            merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
-            env_, &pinned_value_);
+        s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetKey(),
+                                        nullptr, merge_context_.GetOperands(),
+                                        &saved_value_, logger_, statistics_,
+                                        env_, &pinned_value_);
       } else {
         assert(last_not_merge_type == kTypeValue);
         s = MergeHelper::TimedFullMerge(
-            merge_operator_, saved_key_.GetUserKey(), &pinned_value_,
+            merge_operator_, saved_key_.GetKey(), &pinned_value_,
             merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
             env_, &pinned_value_);
       }
@@ -836,8 +794,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   // FindValueForCurrentKeyUsingSeek()
   assert(pinned_iters_mgr_.PinningEnabled());
   std::string last_key;
-  AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetUserKey(),
-                                                 sequence_, kValueTypeForSeek));
+  AppendInternalKey(&last_key, ParsedInternalKey(saved_key_.GetKey(), sequence_,
+                                                 kValueTypeForSeek));
   iter_->Seek(last_key);
   RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
 
@@ -846,8 +804,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   FindParseableKey(&ikey, kForward);
 
   if (ikey.type == kTypeDeletion || ikey.type == kTypeSingleDeletion ||
-      range_del_agg_.ShouldDelete(
-          ikey, RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+      range_del_agg_.ShouldDelete(ikey)) {
     valid_ = false;
     return false;
   }
@@ -862,12 +819,9 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   // in operands
   current_entry_is_merged_ = true;
   merge_context_.Clear();
-  while (
-      iter_->Valid() &&
-      user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey()) &&
-      ikey.type == kTypeMerge &&
-      !range_del_agg_.ShouldDelete(
-          ikey, RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
+  while (iter_->Valid() &&
+         user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) &&
+         ikey.type == kTypeMerge && !range_del_agg_.ShouldDelete(ikey)) {
     merge_context_.PushOperand(iter_->value(),
                                iter_->IsValuePinned() /* operand_pinned */);
     PERF_COUNTER_ADD(internal_merge_count, 1);
@@ -877,17 +831,16 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
 
   Status s;
   if (!iter_->Valid() ||
-      !user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey()) ||
+      !user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) ||
       ikey.type == kTypeDeletion || ikey.type == kTypeSingleDeletion ||
-      range_del_agg_.ShouldDelete(
-          ikey, RangeDelAggregator::RangePositioningMode::kBackwardTraversal)) {
-    s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
+      range_del_agg_.ShouldDelete(ikey)) {
+    s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetKey(),
                                     nullptr, merge_context_.GetOperands(),
                                     &saved_value_, logger_, statistics_, env_,
                                     &pinned_value_);
     // Make iter_ valid and point to saved_key_
     if (!iter_->Valid() ||
-        !user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+        !user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
       iter_->Seek(last_key);
       RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
     }
@@ -899,10 +852,9 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   }
 
   const Slice& val = iter_->value();
-  s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
-                                  &val, merge_context_.GetOperands(),
-                                  &saved_value_, logger_, statistics_, env_,
-                                  &pinned_value_);
+  s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetKey(), &val,
+                                  merge_context_.GetOperands(), &saved_value_,
+                                  logger_, statistics_, env_, &pinned_value_);
   valid_ = true;
   if (!s.ok()) {
     status_ = s;
@@ -921,7 +873,7 @@ void DBIter::FindNextUserKey() {
   ParsedInternalKey ikey;
   FindParseableKey(&ikey, kForward);
   while (iter_->Valid() &&
-         !user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
+         !user_comparator_->Equal(ikey.user_key, saved_key_.GetKey())) {
     iter_->Next();
     FindParseableKey(&ikey, kForward);
   }
@@ -936,21 +888,16 @@ void DBIter::FindPrevUserKey() {
   ParsedInternalKey ikey;
   FindParseableKey(&ikey, kReverse);
   int cmp;
-  while (iter_->Valid() &&
-         ((cmp = user_comparator_->Compare(ikey.user_key,
-                                           saved_key_.GetUserKey())) == 0 ||
-          (cmp > 0 && ikey.sequence > sequence_))) {
-    if (TooManyInternalKeysSkipped()) {
-      return;
-    }
-
+  while (iter_->Valid() && ((cmp = user_comparator_->Compare(
+                                 ikey.user_key, saved_key_.GetKey())) == 0 ||
+                            (cmp > 0 && ikey.sequence > sequence_))) {
     if (cmp == 0) {
       if (num_skipped >= max_skip_) {
         num_skipped = 0;
         IterKey last_key;
         last_key.SetInternalKey(ParsedInternalKey(
-            saved_key_.GetUserKey(), kMaxSequenceNumber, kValueTypeForSeek));
-        iter_->Seek(last_key.GetInternalKey());
+            saved_key_.GetKey(), kMaxSequenceNumber, kValueTypeForSeek));
+        iter_->Seek(last_key.GetKey());
         RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
       } else {
         ++num_skipped;
@@ -966,18 +913,6 @@ void DBIter::FindPrevUserKey() {
   }
 }
 
-bool DBIter::TooManyInternalKeysSkipped(bool increment) {
-  if ((max_skippable_internal_keys_ > 0) &&
-      (num_internal_keys_skipped_ > max_skippable_internal_keys_)) {
-    valid_ = false;
-    status_ = Status::Incomplete("Too many internal keys skipped.");
-    return true;
-  } else if (increment) {
-    num_internal_keys_skipped_++;
-  }
-  return false;
-}
-
 // Skip all unparseable keys
 void DBIter::FindParseableKey(ParsedInternalKey* ikey, Direction direction) {
   while (iter_->Valid() && !ParseKey(ikey)) {
@@ -992,14 +927,13 @@ void DBIter::FindParseableKey(ParsedInternalKey* ikey, Direction direction) {
 void DBIter::Seek(const Slice& target) {
   StopWatch sw(env_, statistics_, DB_SEEK);
   ReleaseTempPinnedData();
-  ResetInternalKeysSkippedCounter();
   saved_key_.Clear();
+  // now savved_key is used to store internal key.
   saved_key_.SetInternalKey(target, sequence_);
 
   {
     PERF_TIMER_GUARD(seek_internal_seek_time);
-    iter_->Seek(saved_key_.GetInternalKey());
-    range_del_agg_.InvalidateTombstoneMapPositions();
+    iter_->Seek(saved_key_.GetKey());
   }
   RecordTick(statistics_, NUMBER_DB_SEEK);
   if (iter_->Valid()) {
@@ -1023,15 +957,14 @@ void DBIter::Seek(const Slice& target) {
   }
 
   if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
-    prefix_start_buf_.SetUserKey(prefix_start_key_);
-    prefix_start_key_ = prefix_start_buf_.GetUserKey();
+    prefix_start_buf_.SetKey(prefix_start_key_);
+    prefix_start_key_ = prefix_start_buf_.GetKey();
   }
 }
 
 void DBIter::SeekForPrev(const Slice& target) {
   StopWatch sw(env_, statistics_, DB_SEEK);
   ReleaseTempPinnedData();
-  ResetInternalKeysSkippedCounter();
   saved_key_.Clear();
   // now saved_key is used to store internal key.
   saved_key_.SetInternalKey(target, 0 /* sequence_number */,
@@ -1039,8 +972,7 @@ void DBIter::SeekForPrev(const Slice& target) {
 
   {
     PERF_TIMER_GUARD(seek_internal_seek_time);
-    iter_->SeekForPrev(saved_key_.GetInternalKey());
-    range_del_agg_.InvalidateTombstoneMapPositions();
+    iter_->SeekForPrev(saved_key_.GetKey());
   }
 
   RecordTick(statistics_, NUMBER_DB_SEEK);
@@ -1064,8 +996,8 @@ void DBIter::SeekForPrev(const Slice& target) {
     valid_ = false;
   }
   if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
-    prefix_start_buf_.SetUserKey(prefix_start_key_);
-    prefix_start_key_ = prefix_start_buf_.GetUserKey();
+    prefix_start_buf_.SetKey(prefix_start_key_);
+    prefix_start_key_ = prefix_start_buf_.GetKey();
   }
 }
 
@@ -1077,20 +1009,15 @@ void DBIter::SeekToFirst() {
   }
   direction_ = kForward;
   ReleaseTempPinnedData();
-  ResetInternalKeysSkippedCounter();
   ClearSavedValue();
 
   {
     PERF_TIMER_GUARD(seek_internal_seek_time);
     iter_->SeekToFirst();
-    range_del_agg_.InvalidateTombstoneMapPositions();
   }
 
   RecordTick(statistics_, NUMBER_DB_SEEK);
   if (iter_->Valid()) {
-    saved_key_.SetUserKey(
-        ExtractUserKey(iter_->key()),
-        !iter_->IsKeyPinned() || !pin_thru_lifetime_ /* copy */);
     FindNextUserEntry(false /* not skipping */, false /* no prefix check */);
     if (statistics_ != nullptr) {
       if (valid_) {
@@ -1102,9 +1029,8 @@ void DBIter::SeekToFirst() {
     valid_ = false;
   }
   if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
-    prefix_start_buf_.SetUserKey(
-        prefix_extractor_->Transform(saved_key_.GetUserKey()));
-    prefix_start_key_ = prefix_start_buf_.GetUserKey();
+    prefix_start_buf_.SetKey(prefix_extractor_->Transform(saved_key_.GetKey()));
+    prefix_start_key_ = prefix_start_buf_.GetKey();
   }
 }
 
@@ -1116,20 +1042,17 @@ void DBIter::SeekToLast() {
   }
   direction_ = kReverse;
   ReleaseTempPinnedData();
-  ResetInternalKeysSkippedCounter();
   ClearSavedValue();
 
   {
     PERF_TIMER_GUARD(seek_internal_seek_time);
     iter_->SeekToLast();
-    range_del_agg_.InvalidateTombstoneMapPositions();
   }
   // When the iterate_upper_bound is set to a value,
   // it will seek to the last key before the
   // ReadOptions.iterate_upper_bound
   if (iter_->Valid() && iterate_upper_bound_ != nullptr) {
     SeekForPrev(*iterate_upper_bound_);
-    range_del_agg_.InvalidateTombstoneMapPositions();
     if (!Valid()) {
       return;
     } else if (user_comparator_->Equal(*iterate_upper_bound_, key())) {
@@ -1146,22 +1069,21 @@ void DBIter::SeekToLast() {
     }
   }
   if (valid_ && prefix_extractor_ && prefix_same_as_start_) {
-    prefix_start_buf_.SetUserKey(
-        prefix_extractor_->Transform(saved_key_.GetUserKey()));
-    prefix_start_key_ = prefix_start_buf_.GetUserKey();
+    prefix_start_buf_.SetKey(prefix_extractor_->Transform(saved_key_.GetKey()));
+    prefix_start_key_ = prefix_start_buf_.GetKey();
   }
 }
 
-Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
-                        const ImmutableCFOptions& cf_options,
-                        const Comparator* user_key_comparator,
-                        InternalIterator* internal_iter,
-                        const SequenceNumber& sequence,
-                        uint64_t max_sequential_skip_in_iterations,
-                        uint64_t version_number) {
+Iterator* NewDBIterator(
+    Env* env, const ImmutableCFOptions& ioptions,
+    const Comparator* user_key_comparator, InternalIterator* internal_iter,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number, const Slice* iterate_upper_bound,
+    bool prefix_same_as_start, bool pin_data, bool total_order_seek) {
   DBIter* db_iter = new DBIter(
-      env, read_options, cf_options, user_key_comparator, internal_iter,
-      sequence, false, max_sequential_skip_in_iterations, version_number);
+      env, ioptions, user_key_comparator, internal_iter, sequence, false,
+      max_sequential_skip_in_iterations, version_number, iterate_upper_bound,
+      prefix_same_as_start, pin_data, total_order_seek);
   return db_iter;
 }
 
@@ -1201,16 +1123,18 @@ void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
 }
 
 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number) {
+    Env* env, const ImmutableCFOptions& ioptions,
+    const Comparator* user_key_comparator, const SequenceNumber& sequence,
+    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+    const Slice* iterate_upper_bound, bool prefix_same_as_start, bool pin_data,
+    bool total_order_seek) {
   ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
   Arena* arena = iter->GetArena();
   auto mem = arena->AllocateAligned(sizeof(DBIter));
-  DBIter* db_iter = new (mem)
-      DBIter(env, read_options, cf_options, user_key_comparator, nullptr,
-             sequence, true, max_sequential_skip_in_iterations, version_number);
+  DBIter* db_iter = new (mem) DBIter(
+      env, ioptions, user_key_comparator, nullptr, sequence, true,
+      max_sequential_skip_in_iterations, version_number, iterate_upper_bound,
+      prefix_same_as_start, pin_data, total_order_seek);
 
   iter->SetDBIter(db_iter);
 

db/db_iter.h

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -14,11 +12,11 @@
 #include <string>
 #include "db/dbformat.h"
 #include "db/range_del_aggregator.h"
-#include "options/cf_options.h"
 #include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "util/arena.h"
 #include "util/autovector.h"
+#include "util/cf_options.h"
 
 namespace rocksdb {
 
@@ -29,13 +27,13 @@ class InternalIterator;
 // Return a new iterator that converts internal keys (yielded by
 // "*internal_iter") that were live at the specified "sequence" number
 // into appropriate user keys.
-extern Iterator* NewDBIterator(Env* env, const ReadOptions& read_options,
-                               const ImmutableCFOptions& cf_options,
-                               const Comparator* user_key_comparator,
-                               InternalIterator* internal_iter,
-                               const SequenceNumber& sequence,
-                               uint64_t max_sequential_skip_in_iterations,
-                               uint64_t version_number);
+extern Iterator* NewDBIterator(
+    Env* env, const ImmutableCFOptions& options,
+    const Comparator* user_key_comparator, InternalIterator* internal_iter,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number, const Slice* iterate_upper_bound = nullptr,
+    bool prefix_same_as_start = false, bool pin_data = false,
+    bool total_order_seek = false);
 
 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed be allocated. This class is used as an entry point of
@@ -79,9 +77,11 @@ class ArenaWrappedDBIter : public Iterator {
 
 // Generate the arena wrapped iterator class.
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options, const Comparator* user_key_comparator,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number);
+    Env* env, const ImmutableCFOptions& options,
+    const Comparator* user_key_comparator, const SequenceNumber& sequence,
+    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+    const Slice* iterate_upper_bound = nullptr,
+    bool prefix_same_as_start = false, bool pin_data = false,
+    bool total_order_seek = false);
 
 }  // namespace rocksdb

db/db_iterator_test.cc

@@ -2,15 +2,11 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include <functional>
-
 #include "db/db_test_util.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -24,34 +20,6 @@ class DBIteratorTest : public DBTestBase {
   DBIteratorTest() : DBTestBase("/db_iterator_test") {}
 };
 
-class FlushBlockEveryKeyPolicy : public FlushBlockPolicy {
- public:
-  virtual bool Update(const Slice& key, const Slice& value) override {
-    if (!start_) {
-      start_ = true;
-      return false;
-    }
-    return true;
-  }
- private:
-  bool start_ = false;
-};
-
-class FlushBlockEveryKeyPolicyFactory : public FlushBlockPolicyFactory {
- public:
-  explicit FlushBlockEveryKeyPolicyFactory() {}
-
-  const char* Name() const override {
-    return "FlushBlockEveryKeyPolicyFactory";
-  }
-
-  FlushBlockPolicy* NewFlushBlockPolicy(
-    const BlockBasedTableOptions& table_options,
-    const BlockBuilder& data_block_builder) const override {
-    return new FlushBlockEveryKeyPolicy;
-  }
-};
-
 TEST_F(DBIteratorTest, IteratorProperty) {
   // The test needs to be changed if kPersistedTier is supported in iterator.
   Options options = CurrentOptions();
@@ -1005,57 +973,12 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
   }
 }
 
-TEST_F(DBIteratorTest, DBIteratorBoundOptimizationTest) {
-  int upper_bound_hits = 0;
-  Options options = CurrentOptions();
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "BlockBasedTable::BlockEntryIteratorState::KeyReachedUpperBound",
-      [&upper_bound_hits](void* arg) {
-        assert(arg != nullptr);
-        upper_bound_hits += (*static_cast<bool*>(arg) ? 1 : 0);
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.prefix_extractor = nullptr;
-  BlockBasedTableOptions table_options;
-  table_options.flush_block_policy_factory =
-    std::make_shared<FlushBlockEveryKeyPolicyFactory>();
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-
-  DestroyAndReopen(options);
-  ASSERT_OK(Put("foo1", "bar1"));
-  ASSERT_OK(Put("foo2", "bar2"));
-  ASSERT_OK(Put("foo4", "bar4"));
-  ASSERT_OK(Flush());
-
-  Slice ub("foo3");
-  ReadOptions ro;
-  ro.iterate_upper_bound = &ub;
-
-  std::unique_ptr<Iterator> iter(db_->NewIterator(ro));
-
-  iter->Seek("foo");
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("foo1")), 0);
-  ASSERT_EQ(upper_bound_hits, 0);
-
-  iter->Next();
-  ASSERT_TRUE(iter->Valid());
-  ASSERT_EQ(iter->key().compare(Slice("foo2")), 0);
-  ASSERT_EQ(upper_bound_hits, 0);
-
-  iter->Next();
-  ASSERT_FALSE(iter->Valid());
-  ASSERT_EQ(upper_bound_hits, 1);
-}
 // TODO(3.13): fix the issue of Seek() + Prev() which might not necessary
 //             return the biggest key which is smaller than the seek key.
 TEST_F(DBIteratorTest, PrevAfterAndNextAfterMerge) {
   Options options;
   options.create_if_missing = true;
   options.merge_operator = MergeOperators::CreatePutOperator();
-  options.env = env_;
   DestroyAndReopen(options);
 
   // write three entries with different keys using Merge()
@@ -1719,7 +1642,7 @@ TEST_F(DBIteratorTest, IteratorWithLocalStatistics) {
   std::atomic<uint64_t> total_prev_found(0);
   std::atomic<uint64_t> total_bytes(0);
 
-  std::vector<port::Thread> threads;
+  std::vector<std::thread> threads;
   std::function<void()> reader_func_next = [&]() {
     Iterator* iter = db_->NewIterator(ReadOptions());
 

db/db_log_iter_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/db_memtable_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 
 #include <memory>
 #include <string>
@@ -125,7 +123,6 @@ TEST_F(DBMemTableTest, InsertWithHint) {
   options.memtable_factory.reset(new MockMemTableRepFactory());
   options.memtable_insert_with_hint_prefix_extractor.reset(
       new TestPrefixExtractor());
-  options.env = env_;
   Reopen(options);
   MockMemTableRep* rep =
       reinterpret_cast<MockMemTableRepFactory*>(options.memtable_factory.get())

db/db_merge_operator_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 #include <string>
 #include <vector>
 
@@ -46,7 +44,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnRead) {
   Options options;
   options.create_if_missing = true;
   options.merge_operator.reset(new TestPutOperator());
-  options.env = env_;
   Reopen(options);
   ASSERT_OK(Merge("k1", "v1"));
   ASSERT_OK(Merge("k1", "corrupted"));
@@ -60,7 +57,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnWrite) {
   options.create_if_missing = true;
   options.merge_operator.reset(new TestPutOperator());
   options.max_successive_merges = 3;
-  options.env = env_;
   Reopen(options);
   ASSERT_OK(Merge("k1", "v1"));
   ASSERT_OK(Merge("k1", "v2"));
@@ -75,7 +71,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) {
   Options options;
   options.create_if_missing = true;
   options.merge_operator.reset(new TestPutOperator());
-  options.env = env_;
 
   DestroyAndReopen(options);
   ASSERT_OK(Merge("k1", "v1"));
@@ -94,294 +89,6 @@ TEST_F(DBMergeOperatorTest, MergeErrorOnIteration) {
   VerifyDBInternal({{"k1", "v1"}, {"k2", "corrupted"}, {"k2", "v2"}});
 }
 
-
-class MergeOperatorPinningTest : public DBMergeOperatorTest,
-                                 public testing::WithParamInterface<bool> {
- public:
-  MergeOperatorPinningTest() { disable_block_cache_ = GetParam(); }
-
-  bool disable_block_cache_;
-};
-
-INSTANTIATE_TEST_CASE_P(MergeOperatorPinningTest, MergeOperatorPinningTest,
-                        ::testing::Bool());
-
-#ifndef ROCKSDB_LITE
-TEST_P(MergeOperatorPinningTest, OperandsMultiBlocks) {
-  Options options = CurrentOptions();
-  BlockBasedTableOptions table_options;
-  table_options.block_size = 1;  // every block will contain one entry
-  table_options.no_block_cache = disable_block_cache_;
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  options.merge_operator = MergeOperators::CreateStringAppendTESTOperator();
-  options.level0_slowdown_writes_trigger = (1 << 30);
-  options.level0_stop_writes_trigger = (1 << 30);
-  options.disable_auto_compactions = true;
-  DestroyAndReopen(options);
-
-  const int kKeysPerFile = 10;
-  const int kOperandsPerKeyPerFile = 7;
-  const int kOperandSize = 100;
-  // Filse to write in L0 before compacting to lower level
-  const int kFilesPerLevel = 3;
-
-  Random rnd(301);
-  std::map<std::string, std::string> true_data;
-  int batch_num = 1;
-  int lvl_to_fill = 4;
-  int key_id = 0;
-  while (true) {
-    for (int j = 0; j < kKeysPerFile; j++) {
-      std::string key = Key(key_id % 35);
-      key_id++;
-      for (int k = 0; k < kOperandsPerKeyPerFile; k++) {
-        std::string val = RandomString(&rnd, kOperandSize);
-        ASSERT_OK(db_->Merge(WriteOptions(), key, val));
-        if (true_data[key].size() == 0) {
-          true_data[key] = val;
-        } else {
-          true_data[key] += "," + val;
-        }
-      }
-    }
-
-    if (lvl_to_fill == -1) {
-      // Keep last batch in memtable and stop
-      break;
-    }
-
-    ASSERT_OK(Flush());
-    if (batch_num % kFilesPerLevel == 0) {
-      if (lvl_to_fill != 0) {
-        MoveFilesToLevel(lvl_to_fill);
-      }
-      lvl_to_fill--;
-    }
-    batch_num++;
-  }
-
-  // 3 L0 files
-  // 1 L1 file
-  // 3 L2 files
-  // 1 L3 file
-  // 3 L4 Files
-  ASSERT_EQ(FilesPerLevel(), "3,1,3,1,3");
-
-  VerifyDBFromMap(true_data);
-}
-
-TEST_P(MergeOperatorPinningTest, Randomized) {
-  do {
-    Options options = CurrentOptions();
-    options.merge_operator = MergeOperators::CreateMaxOperator();
-    BlockBasedTableOptions table_options;
-    table_options.no_block_cache = disable_block_cache_;
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    DestroyAndReopen(options);
-
-    Random rnd(301);
-    std::map<std::string, std::string> true_data;
-
-    const int kTotalMerges = 10000;
-    // Every key gets ~10 operands
-    const int kKeyRange = kTotalMerges / 10;
-    const int kOperandSize = 20;
-    const int kNumPutBefore = kKeyRange / 10;  // 10% value
-    const int kNumPutAfter = kKeyRange / 10;   // 10% overwrite
-    const int kNumDelete = kKeyRange / 10;     // 10% delete
-
-    // kNumPutBefore keys will have base values
-    for (int i = 0; i < kNumPutBefore; i++) {
-      std::string key = Key(rnd.Next() % kKeyRange);
-      std::string value = RandomString(&rnd, kOperandSize);
-      ASSERT_OK(db_->Put(WriteOptions(), key, value));
-
-      true_data[key] = value;
-    }
-
-    // Do kTotalMerges merges
-    for (int i = 0; i < kTotalMerges; i++) {
-      std::string key = Key(rnd.Next() % kKeyRange);
-      std::string value = RandomString(&rnd, kOperandSize);
-      ASSERT_OK(db_->Merge(WriteOptions(), key, value));
-
-      if (true_data[key] < value) {
-        true_data[key] = value;
-      }
-    }
-
-    // Overwrite random kNumPutAfter keys
-    for (int i = 0; i < kNumPutAfter; i++) {
-      std::string key = Key(rnd.Next() % kKeyRange);
-      std::string value = RandomString(&rnd, kOperandSize);
-      ASSERT_OK(db_->Put(WriteOptions(), key, value));
-
-      true_data[key] = value;
-    }
-
-    // Delete random kNumDelete keys
-    for (int i = 0; i < kNumDelete; i++) {
-      std::string key = Key(rnd.Next() % kKeyRange);
-      ASSERT_OK(db_->Delete(WriteOptions(), key));
-
-      true_data.erase(key);
-    }
-
-    VerifyDBFromMap(true_data);
-
-    // Skip HashCuckoo since it does not support merge operators
-  } while (ChangeOptions(kSkipMergePut | kSkipHashCuckoo));
-}
-
-class MergeOperatorHook : public MergeOperator {
- public:
-  explicit MergeOperatorHook(std::shared_ptr<MergeOperator> _merge_op)
-      : merge_op_(_merge_op) {}
-
-  virtual bool FullMergeV2(const MergeOperationInput& merge_in,
-                           MergeOperationOutput* merge_out) const override {
-    before_merge_();
-    bool res = merge_op_->FullMergeV2(merge_in, merge_out);
-    after_merge_();
-    return res;
-  }
-
-  virtual const char* Name() const override { return merge_op_->Name(); }
-
-  std::shared_ptr<MergeOperator> merge_op_;
-  std::function<void()> before_merge_ = []() {};
-  std::function<void()> after_merge_ = []() {};
-};
-
-TEST_P(MergeOperatorPinningTest, EvictCacheBeforeMerge) {
-  Options options = CurrentOptions();
-
-  auto merge_hook =
-      std::make_shared<MergeOperatorHook>(MergeOperators::CreateMaxOperator());
-  options.merge_operator = merge_hook;
-  options.disable_auto_compactions = true;
-  options.level0_slowdown_writes_trigger = (1 << 30);
-  options.level0_stop_writes_trigger = (1 << 30);
-  options.max_open_files = 20;
-  BlockBasedTableOptions bbto;
-  bbto.no_block_cache = disable_block_cache_;
-  if (bbto.no_block_cache == false) {
-    bbto.block_cache = NewLRUCache(64 * 1024 * 1024);
-  } else {
-    bbto.block_cache = nullptr;
-  }
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-  DestroyAndReopen(options);
-
-  const int kNumOperands = 30;
-  const int kNumKeys = 1000;
-  const int kOperandSize = 100;
-  Random rnd(301);
-
-  // 1000 keys every key have 30 operands, every operand is in a different file
-  std::map<std::string, std::string> true_data;
-  for (int i = 0; i < kNumOperands; i++) {
-    for (int j = 0; j < kNumKeys; j++) {
-      std::string k = Key(j);
-      std::string v = RandomString(&rnd, kOperandSize);
-      ASSERT_OK(db_->Merge(WriteOptions(), k, v));
-
-      true_data[k] = std::max(true_data[k], v);
-    }
-    ASSERT_OK(Flush());
-  }
-
-  std::vector<uint64_t> file_numbers = ListTableFiles(env_, dbname_);
-  ASSERT_EQ(file_numbers.size(), kNumOperands);
-  int merge_cnt = 0;
-
-  // Code executed before merge operation
-  merge_hook->before_merge_ = [&]() {
-    // Evict all tables from cache before every merge operation
-    for (uint64_t num : file_numbers) {
-      TableCache::Evict(dbfull()->TEST_table_cache(), num);
-    }
-    // Decrease cache capacity to force all unrefed blocks to be evicted
-    if (bbto.block_cache) {
-      bbto.block_cache->SetCapacity(1);
-    }
-    merge_cnt++;
-  };
-
-  // Code executed after merge operation
-  merge_hook->after_merge_ = [&]() {
-    // Increase capacity again after doing the merge
-    if (bbto.block_cache) {
-      bbto.block_cache->SetCapacity(64 * 1024 * 1024);
-    }
-  };
-
-  size_t total_reads;
-  VerifyDBFromMap(true_data, &total_reads);
-  ASSERT_EQ(merge_cnt, total_reads);
-
-  db_->CompactRange(CompactRangeOptions(), nullptr, nullptr);
-
-  VerifyDBFromMap(true_data, &total_reads);
-}
-
-TEST_P(MergeOperatorPinningTest, TailingIterator) {
-  Options options = CurrentOptions();
-  options.merge_operator = MergeOperators::CreateMaxOperator();
-  BlockBasedTableOptions bbto;
-  bbto.no_block_cache = disable_block_cache_;
-  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
-  DestroyAndReopen(options);
-
-  const int kNumOperands = 100;
-  const int kNumWrites = 100000;
-
-  std::function<void()> writer_func = [&]() {
-    int k = 0;
-    for (int i = 0; i < kNumWrites; i++) {
-      db_->Merge(WriteOptions(), Key(k), Key(k));
-
-      if (i && i % kNumOperands == 0) {
-        k++;
-      }
-      if (i && i % 127 == 0) {
-        ASSERT_OK(Flush());
-      }
-      if (i && i % 317 == 0) {
-        ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-      }
-    }
-  };
-
-  std::function<void()> reader_func = [&]() {
-    ReadOptions ro;
-    ro.tailing = true;
-    Iterator* iter = db_->NewIterator(ro);
-
-    iter->SeekToFirst();
-    for (int i = 0; i < (kNumWrites / kNumOperands); i++) {
-      while (!iter->Valid()) {
-        // wait for the key to be written
-        env_->SleepForMicroseconds(100);
-        iter->Seek(Key(i));
-      }
-      ASSERT_EQ(iter->key(), Key(i));
-      ASSERT_EQ(iter->value(), Key(i));
-
-      iter->Next();
-    }
-
-    delete iter;
-  };
-
-  rocksdb::port::Thread writer_thread(writer_func);
-  rocksdb::port::Thread reader_thread(reader_func);
-
-  writer_thread.join();
-  reader_thread.join();
-}
-#endif  // ROCKSDB_LITE
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_options_test.cc

@@ -2,8 +2,6 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -15,10 +13,9 @@
 #include "db/column_family.h"
 #include "db/db_impl.h"
 #include "db/db_test_util.h"
-#include "options/options_helper.h"
 #include "port/stack_trace.h"
-#include "rocksdb/cache.h"
 #include "rocksdb/convenience.h"
+#include "util/options_helper.h"
 #include "util/random.h"
 #include "util/sync_point.h"
 #include "util/testutil.h"
@@ -65,7 +62,6 @@ class DBOptionsTest : public DBTestBase {
   std::unordered_map<std::string, std::string> GetRandomizedMutableCFOptionsMap(
       Random* rnd) {
     Options options;
-    options.env = env_;
     ImmutableDBOptions db_options(options);
     test::RandomInitCFOptions(&options, rnd);
     auto sanitized_options = SanitizeOptions(db_options, options);
@@ -91,7 +87,6 @@ TEST_F(DBOptionsTest, GetLatestDBOptions) {
   // GetOptions should be able to get latest option changed by SetOptions.
   Options options;
   options.create_if_missing = true;
-  options.env = env_;
   Random rnd(228);
   Reopen(options);
   auto new_options = GetRandomizedMutableDBOptionsMap(&rnd);
@@ -103,7 +98,6 @@ TEST_F(DBOptionsTest, GetLatestCFOptions) {
   // GetOptions should be able to get latest option changed by SetOptions.
   Options options;
   options.create_if_missing = true;
-  options.env = env_;
   Random rnd(228);
   Reopen(options);
   CreateColumnFamilies({"foo"}, options);
@@ -124,7 +118,6 @@ TEST_F(DBOptionsTest, SetOptionsAndReopen) {
   ASSERT_OK(dbfull()->SetOptions(rand_opts));
   // Verify if DB can be reopen after setting options.
   Options options;
-  options.env = env_;
   ASSERT_OK(TryReopen(options));
 }
 
@@ -144,7 +137,6 @@ TEST_F(DBOptionsTest, EnableAutoCompactionAndTriggerStall) {
           std::numeric_limits<uint64_t>::max();
       options.soft_pending_compaction_bytes_limit =
           std::numeric_limits<uint64_t>::max();
-      options.env = env_;
 
       DestroyAndReopen(options);
       int i = 0;
@@ -236,7 +228,6 @@ TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) {
   Options options;
   options.create_if_missing = true;
   options.level0_file_num_compaction_trigger = 1000;
-  options.env = env_;
   Reopen(options);
   for (int i = 0; i < 3; i++) {
     // Need to insert two keys to avoid trivial move.
@@ -256,7 +247,6 @@ TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
   options.create_if_missing = true;
   options.base_background_compactions = 1;  // default value
   options.max_background_compactions = 1;   // default value
-  options.env = env_;
   Reopen(options);
   ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
   ASSERT_OK(dbfull()->SetDBOptions({{"base_background_compactions", "2"},
@@ -270,7 +260,6 @@ TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) {
   Options options;
   options.create_if_missing = true;
   options.disable_auto_compactions = true;
-  options.env = env_;
   WriteOptions write_without_wal;
   write_without_wal.disableWAL = true;
 
@@ -293,7 +282,6 @@ TEST_F(DBOptionsTest, SetDelayedWriteRateOption) {
   Options options;
   options.create_if_missing = true;
   options.delayed_write_rate = 2 * 1024U * 1024U;
-  options.env = env_;
   Reopen(options);
   ASSERT_EQ(2 * 1024U * 1024U, dbfull()->TEST_write_controler().max_delayed_write_rate());
 
@@ -309,7 +297,6 @@ TEST_F(DBOptionsTest, MaxTotalWalSizeChange) {
 
   Options options;
   options.create_if_missing = true;
-  options.env = env_;
   CreateColumnFamilies({"1", "2", "3"}, options);
   ReopenWithColumnFamilies({"default", "1", "2", "3"}, options);
 
@@ -329,81 +316,6 @@ TEST_F(DBOptionsTest, MaxTotalWalSizeChange) {
   }
 }
 
-TEST_F(DBOptionsTest, SetStatsDumpPeriodSec) {
-  Options options;
-  options.create_if_missing = true;
-  options.stats_dump_period_sec = 5;
-  options.env = env_;
-  Reopen(options);
-  ASSERT_EQ(5, dbfull()->GetDBOptions().stats_dump_period_sec);
-
-  for (int i = 0; i < 20; i++) {
-    int num = rand() % 5000 + 1;
-    ASSERT_OK(dbfull()->SetDBOptions(
-        {{"stats_dump_period_sec", std::to_string(num)}}));
-    ASSERT_EQ(num, dbfull()->GetDBOptions().stats_dump_period_sec);
-  }
-}
-
-static void assert_candidate_files_empty(DBImpl* dbfull, const bool empty) {
-  dbfull->TEST_LockMutex();
-  JobContext job_context(0);
-  dbfull->FindObsoleteFiles(&job_context, false);
-  ASSERT_EQ(empty, job_context.full_scan_candidate_files.empty());
-  job_context.Clean();
-  dbfull->TEST_UnlockMutex();
-}
-
-TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
-  SpecialEnv env(env_);
-  env.time_elapse_only_sleep_ = true;
-  Options options;
-  options.env = &env;
-  options.create_if_missing = true;
-  ASSERT_OK(TryReopen(options));
-
-  // Verify that candidate files set is empty when no full scan requested.
-  assert_candidate_files_empty(dbfull(), true);
-
-  ASSERT_OK(
-      dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "0"}}));
-
-  // After delete_obsolete_files_period_micros updated to 0, the next call
-  // to FindObsoleteFiles should make a full scan
-  assert_candidate_files_empty(dbfull(), false);
-
-  ASSERT_OK(
-      dbfull()->SetDBOptions({{"delete_obsolete_files_period_micros", "20"}}));
-
-  assert_candidate_files_empty(dbfull(), true);
-
-  env.addon_time_.store(20);
-  assert_candidate_files_empty(dbfull(), true);
-
-  env.addon_time_.store(21);
-  assert_candidate_files_empty(dbfull(), false);
-
-  Close();
-}
-
-TEST_F(DBOptionsTest, MaxOpenFilesChange) {
-  SpecialEnv env(env_);
-  Options options;
-  options.max_open_files = -1;
-
-  Reopen(options);
-
-  Cache* tc = dbfull()->TEST_table_cache();
-
-  ASSERT_EQ(-1, dbfull()->GetDBOptions().max_open_files);
-  ASSERT_LT(2000, tc->GetCapacity());
-  ASSERT_OK(dbfull()->SetDBOptions({{"max_open_files", "1024"}}));
-  ASSERT_EQ(1024, dbfull()->GetDBOptions().max_open_files);
-  // examine the table cache (actual size should be 1014)
-  ASSERT_GT(1500, tc->GetCapacity());
-  Close();
-}
-
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

db/db_properties_test.cc

@@ -342,7 +342,7 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
   BlockBasedTableOptions table_options;
   table_options.no_block_cache = true;
 
-  CreateAndReopenWithCF({"pikachu"}, options);
+  DestroyAndReopen(options);
   int key_index = 0;
   Random rnd(301);
   for (int num = 0; num < 8; num++) {
@@ -359,26 +359,25 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
   for (int key = 0; key < key_index; key++) {
     Get(Key(key));
   }
-  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
   ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
   ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
   ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
 
   // Reopen and issue Get(). See thee latency tracked
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
+  Reopen(options);
   dbfull()->TEST_WaitForCompact();
   for (int key = 0; key < key_index; key++) {
     Get(Key(key));
   }
-  ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
-                                    "rocksdb.cf-file-histogram", &prop));
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
   ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
   ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
   ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
 
   // Reopen and issue iterating. See thee latency tracked
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop));
+  Reopen(options);
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
   ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
   ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
   ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
@@ -387,51 +386,25 @@ TEST_F(DBPropertiesTest, ReadLatencyHistogramByLevel) {
     for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
     }
   }
-  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cf-file-histogram", &prop));
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
   ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
   ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
   ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
 
-  // CF 1 should show no histogram.
-  ASSERT_TRUE(
-      dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop));
-  ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
-  ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
-  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
-  // put something and read it back , CF 1 should show histogram.
-  Put(1, "foo", "bar");
-  Flush(1);
-  dbfull()->TEST_WaitForCompact();
-  ASSERT_EQ("bar", Get(1, "foo"));
-
-  ASSERT_TRUE(
-      dbfull()->GetProperty(handles_[1], "rocksdb.cf-file-histogram", &prop));
-  ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
-  ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
-  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
-
   // options.max_open_files preloads table readers.
   options.max_open_files = -1;
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_TRUE(dbfull()->GetProperty(dbfull()->DefaultColumnFamily(),
-                                    "rocksdb.cf-file-histogram", &prop));
+  Reopen(options);
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
   ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
   ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
   ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
   for (int key = 0; key < key_index; key++) {
     Get(Key(key));
   }
-  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.dbstats", &prop));
   ASSERT_NE(std::string::npos, prop.find("** Level 0 read latency histogram"));
   ASSERT_NE(std::string::npos, prop.find("** Level 1 read latency histogram"));
   ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
-
-  // Clear internal stats
-  dbfull()->ResetStats();
-  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.cfstats", &prop));
-  ASSERT_EQ(std::string::npos, prop.find("** Level 0 read latency histogram"));
-  ASSERT_EQ(std::string::npos, prop.find("** Level 1 read latency histogram"));
-  ASSERT_EQ(std::string::npos, prop.find("** Level 2 read latency histogram"));
 }
 
 TEST_F(DBPropertiesTest, AggregatedTablePropertiesAtLevel) {
@@ -1183,7 +1156,6 @@ TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) {
   options.max_bytes_for_level_multiplier = 4;
   options.soft_pending_compaction_bytes_limit = 1024 * 1024;
   options.num_levels = 8;
-  options.env = env_;
 
   std::shared_ptr<TablePropertiesCollectorFactory> collector_factory =
       std::make_shared<CountingDeleteTabPropCollectorFactory>();
@@ -1255,7 +1227,6 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
   options.level0_slowdown_writes_trigger = 10;
   options.level0_stop_writes_trigger = 10;
   options.disable_auto_compactions = true;
-  options.env = env_;
 
   std::shared_ptr<TablePropertiesCollectorFactory> collector_factory =
       std::make_shared<CountingDeleteTabPropCollectorFactory>();
@@ -1300,18 +1271,6 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
     SetPerfLevel(kDisable);
   }
 }
-
-TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
-  Options options;
-  Reopen(options);
-  Put("foo", "bar");
-  Delete("foo");
-  Delete("foo");
-  uint64_t num_keys = 0;
-  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &num_keys));
-  ASSERT_EQ(0, num_keys);
-}
-
 #endif  // ROCKSDB_LITE
 }  // namespace rocksdb