Bump version to 5.4.6 and update HISTORY.md

Summary:
Lite build does not recognize FLAGS_use_blob_db. Fixing it.
Closes https://github.com/facebook/rocksdb/pull/2372

Reviewed By: anirbanr-fb

Differential Revision: D5130773

Pulled By: yiwu-arbug

fbshipit-source-id: 43131d9d0be5811f2129af562be72cca26369cb3

.gitignore

@@ -31,7 +31,6 @@ build/
 ldb
 manifest_dump
 sst_dump
-blob_dump
 column_aware_encoding_exp
 util/build_version.cc
 build_tools/VALGRIND_LOGS/
@@ -42,7 +41,6 @@ package/
 .phutil_module_cache
 unity.a
 tags
-etags
 rocksdb_dump
 rocksdb_undump
 db_test2
@@ -61,7 +59,7 @@ rocksdb.h
 unity.cc
 java/crossbuild/.vagrant
 .vagrant/
-java/**/*.asc
+java/**.asc
 java/javadoc
 
 scan_build_report/

.travis.yml

@@ -7,7 +7,6 @@ os:
 compiler:
   - clang
   - gcc
-osx_image: xcode8.3
 jdk:
   - oraclejdk7
 cache:
@@ -16,55 +15,47 @@ cache:
 
 addons:
    apt:
-      packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
+      sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.6']
+      packages: ['clang-3.6' , 'g++-6', 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev']
 env:
-  - TEST_GROUP=platform_dependent # 16-18 minutes
-  - TEST_GROUP=1 # 33-35 minutes
-  - TEST_GROUP=2 # 30-32 minutes
+  - TEST_GROUP=platform_dependent1
+  - TEST_GROUP=platform_dependent2
+  - TEST_GROUP=platform_dependent3
+  - TEST_GROUP=1
+  - TEST_GROUP=2
   # Run java tests
-  - JOB_NAME=java_test # 4-11 minutes
+  - JOB_NAME=java_test
   # Build ROCKSDB_LITE
-  - JOB_NAME=lite_build # 3-4 minutes
+  - JOB_NAME=lite_build
   # Build examples
-  - JOB_NAME=examples # 5-7 minutes
-  - JOB_NAME=cmake # 3-5 minutes
-  - JOB_NAME=cmake-mingw # 3 minutes
+  - JOB_NAME=examples
 
 matrix:
   exclude:
+  - os: osx
+    compiler: gcc
   - os: osx
     env: TEST_GROUP=1
   - os: osx
     env: TEST_GROUP=2
-  - os : osx
-    env: JOB_NAME=cmake-mingw
-  - os : linux
-    compiler: clang
-  - os : osx
-    compiler: gcc
-
-# https://docs.travis-ci.com/user/caching/#ccache-cache
-install:
-  - if [ "${TRAVIS_OS_NAME}" == osx ]; then
-      brew install ccache;
-      PATH=$PATH:/usr/local/opt/ccache/libexec;
-    fi
 
 before_script:
-  # Increase the maximum number of open file descriptors, since some tests use
-  # more FDs than the default limit.
-  - ulimit -n 8192
+  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'clang++' ]]; then CXX=clang++-3.6; fi
+  # test one linux g++ build with g++-6
+  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'g++' && "${JOB_NAME}" == 'unittests' ]]; then CXX=g++-6; fi
+  # Limit the maximum number of open file descriptors to 8192
+  - ulimit -n 8192 || true
 
 script:
   - ${CXX} --version
-  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 make -j4 all; OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
-  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
-  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
-  - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
-  - if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
-  - if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
-  - if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
-  - if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi
+  - if [[ "${TEST_GROUP}" == 'platform_dependent1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=external_sst_file_basic_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == 'platform_dependent2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=external_sst_file_basic_test ROCKSDBTESTS_END=checkpoint_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == 'platform_dependent3' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=checkpoint_test ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == '1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == '2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
+  - if [[ "${JOB_NAME}" == 'java_test' ]]; then OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest; fi
+  - if [[ "${JOB_NAME}" == 'lite_build' ]]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
+  - if [[ "${JOB_NAME}" == 'examples' ]]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
 notifications:
     email:
       - leveldb@fb.com

CMakeLists.txt

@@ -55,7 +55,7 @@ else()
       add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
       include_directories(${JEMALLOC_INCLUDE_DIR})
     endif()
-  endif()
+  endif() 
   option(WITH_SNAPPY "build with SNAPPY" OFF)
   if(WITH_SNAPPY)
     find_package(snappy REQUIRED)
@@ -66,8 +66,8 @@ else()
 endif()
 
 if(WIN32)
-  execute_process(COMMAND powershell -noprofile -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
-  execute_process(COMMAND powershell -noprofile -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
+  execute_process(COMMAND powershell -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
+  execute_process(COMMAND powershell -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
   string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE "${DATE}")
   string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME "${TIME}")
   set(GIT_DATE_TIME "${DATE} ${TIME}")
@@ -106,58 +106,17 @@ if(NOT WIN32)
   string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
 endif()
 
-option(WITH_MD_LIBRARY "build with MD" ON)
-if(WIN32 AND MSVC)
-  if(WITH_MD_LIBRARY)
-    set(RUNTIME_LIBRARY "MD")
-  else()
-    set(RUNTIME_LIBRARY "MT")
+if(WIN32)
+  option(WITH_AVX2 "build with AVX2" ON)
+  if(WITH_AVX2)
+    if(MSVC)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
+    else()
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
+    endif()
   endif()
 endif()
 
-option(PORTABLE "build a portable binary" OFF)
-option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
-if(PORTABLE)
-  # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
-  # is available, it is available by default.
-  if(FORCE_SSE42 AND NOT MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
-  endif()
-else()
-  if(MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
-  else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
-  endif()
-endif()
-
-set(CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS})
-include(CheckCXXSourceCompiles)
-CHECK_CXX_SOURCE_COMPILES("
-#include <cstdint>
-#include <nmmintrin.h>
-int main() {
-  volatile uint32_t x = _mm_crc32_u32(0, 0);
-}
-" HAVE_SSE42)
-if(HAVE_SSE42)
-  add_definitions(-DHAVE_SSE42)
-elseif(FORCE_SSE42)
-  message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
-endif()
-
-CHECK_CXX_SOURCE_COMPILES("
-#if defined(_MSC_VER) && !defined(__thread)
-#define __thread __declspec(thread)
-#endif
-int main() {
-  static __thread int tls;
-}
-" HAVE_THREAD_LOCAL)
-if(HAVE_THREAD_LOCAL)
-  add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
-endif()
-
 set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
 configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
 add_library(build_version OBJECT ${BUILD_VERSION_CC})
@@ -174,8 +133,7 @@ else()
   endif()
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
   if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
-    add_definitions(-DNDEBUG)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
     include(CheckCXXCompilerFlag)
     CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
     if(HAVE_OMIT_LEAF_FRAME_POINTER)
@@ -195,6 +153,7 @@ endif()
 
 option(WITH_ASAN "build with ASAN" OFF)
 if(WITH_ASAN)
+  add_definitions(-DROCKSDB_TSAN_RUN)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
@@ -206,6 +165,7 @@ endif()
 option(WITH_TSAN "build with TSAN" OFF)
 if(WITH_TSAN)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie")
+  add_definitions(-DROCKSDB_TSAN_RUN)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC")
   if(WITH_JEMALLOC)
@@ -215,7 +175,6 @@ endif()
 
 option(WITH_UBSAN "build with UBSAN" OFF)
 if(WITH_UBSAN)
-  add_definitions(-DROCKSDB_UBSAN_RUN)
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
@@ -236,11 +195,11 @@ endif()
 if(MSVC)
   if((${OPTIMIZE_DEBUG} EQUAL 1))
     message(STATUS "Debug optimization is enabled")
-    set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d")
+    set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /MDd")
   else()
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d")
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /MDd")
   endif()
-  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
+  set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /MD")
 
   set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /DEBUG")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /DEBUG")
@@ -358,7 +317,6 @@ set(SOURCES
         db/internal_stats.cc
         db/log_reader.cc
         db/log_writer.cc
-        db/malloc_stats.cc
         db/managed_iterator.cc
         db/memtable.cc
         db/memtable_list.cc
@@ -380,16 +338,14 @@ set(SOURCES
         db/write_thread.cc
         env/env.cc
         env/env_chroot.cc
-        env/env_encryption.cc
         env/env_hdfs.cc
-        env/mock_env.cc
-        memtable/alloc_tracker.cc
+        env/memenv.cc
         memtable/hash_cuckoo_rep.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc
+        memtable/memtable_allocator.cc
         memtable/skiplistrep.cc
         memtable/vectorrep.cc
-        memtable/write_buffer_manager.cc
         monitoring/histogram.cc
         monitoring/histogram_windowing.cc
         monitoring/instrumented_mutex.cc
@@ -477,19 +433,17 @@ set(SOURCES
         utilities/backupable/backupable_db.cc
         utilities/blob_db/blob_db.cc
         utilities/blob_db/blob_db_impl.cc
-        utilities/blob_db/blob_dump_tool.cc
+        utilities/blob_db/blob_db_options_impl.cc
         utilities/blob_db/blob_file.cc
         utilities/blob_db/blob_log_reader.cc
         utilities/blob_db/blob_log_writer.cc
         utilities/blob_db/blob_log_format.cc
-        utilities/blob_db/ttl_extractor.cc
-        utilities/checkpoint/checkpoint_impl.cc
+        utilities/checkpoint/checkpoint.cc
         utilities/col_buf_decoder.cc
         utilities/col_buf_encoder.cc
         utilities/column_aware_encoding_util.cc
         utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
         utilities/date_tiered/date_tiered_db_impl.cc
-        utilities/debug.cc
         utilities/document/document_db.cc
         utilities/document/json_document.cc
         utilities/document/json_document_builder.cc
@@ -501,8 +455,6 @@ set(SOURCES
         utilities/memory/memory_util.cc
         utilities/merge_operators/max.cc
         utilities/merge_operators/put.cc
-        utilities/merge_operators/cassandra/format.cc
-        utilities/merge_operators/cassandra/merge_operator.cc
         utilities/merge_operators/string_append/stringappend.cc
         utilities/merge_operators/string_append/stringappend2.cc
         utilities/merge_operators/uint64add.cc
@@ -546,14 +498,16 @@ else()
     env/io_posix.cc)
 endif()
 
-set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
-set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
-set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
 if(WIN32)
   set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
+  set(ROCKSDB_STATIC_LIB rocksdblib${ARTIFACT_SUFFIX})
+  set(ROCKSDB_IMPORT_LIB rocksdb${ARTIFACT_SUFFIX})
   set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 else()
-  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
+  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT} rt)
+  set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
+  set(ROCKSDB_SHARED_LIB rocksdb-shared)
+  set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
   set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
   add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
@@ -630,14 +584,12 @@ set(TESTS
         db/db_properties_test.cc
         db/db_range_del_test.cc
         db/db_sst_test.cc
-        db/db_statistics_test.cc
         db/db_table_properties_test.cc
         db/db_tailing_iter_test.cc
         db/db_test.cc
         db/db_test2.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc
-        db/db_write_test.cc
         db/dbformat_test.cc
         db/deletefile_test.cc
         db/external_sst_file_basic_test.cc
@@ -670,7 +622,6 @@ set(TESTS
         env/mock_env_test.cc
         memtable/inlineskiplist_test.cc
         memtable/skiplist_test.cc
-        memtable/write_buffer_manager_test.cc
         monitoring/histogram_test.cc
         monitoring/iostats_context_test.cc
         monitoring/statistics_test.cc
@@ -697,7 +648,6 @@ set(TESTS
         util/event_logger_test.cc
         util/file_reader_writer_test.cc
         util/filelock_test.cc
-        util/hash_test.cc
         util/heap_test.cc
         util/rate_limiter_test.cc
         util/slice_transform_test.cc
@@ -705,7 +655,6 @@ set(TESTS
         util/thread_list_test.cc
         util/thread_local_test.cc
         utilities/backupable/backupable_db_test.cc
-        utilities/blob_db/blob_db_test.cc
         utilities/checkpoint/checkpoint_test.cc
         utilities/column_aware_encoding_test.cc
         utilities/date_tiered/date_tiered_test.cc
@@ -714,10 +663,6 @@ set(TESTS
         utilities/geodb/geodb_test.cc
         utilities/lua/rocks_lua_test.cc
         utilities/memory/memory_test.cc
-        utilities/merge_operators/cassandra/cassandra_merge_test.cc
-        utilities/merge_operators/cassandra/cassandra_format_test.cc
-        utilities/merge_operators/cassandra/cassandra_row_merge_test.cc
-        utilities/merge_operators/cassandra/cassandra_serialize_test.cc
         utilities/merge_operators/string_append/stringappend_test.cc
         utilities/object_registry_test.cc
         utilities/option_change_migration/option_change_migration_test.cc
@@ -755,10 +700,10 @@ endforeach(sourcefile ${BENCHMARKS})
 # and linked to tests. Add test only code that is not #ifdefed for Release here.
 set(TESTUTIL_SOURCE
     db/db_test_util.cc
+    env/mock_env.cc
     monitoring/thread_status_updater_debug.cc
     table/mock_table.cc
     util/fault_injection_test_env.cc
-    utilities/merge_operators/cassandra/test_utils.cc
 )
 # test utilities are only build in debug
 enable_testing()

CONTRIBUTING.md

@@ -12,3 +12,8 @@ Complete your CLA here: <https://code.facebook.com/cla>
 
 If you prefer to sign a paper copy, we can send you a PDF.  Send us an 
 e-mail or create a new github issue to request the CLA in PDF format.
+
+## License
+
+By contributing to RocksDB, you agree that your contributions will be
+licensed under the [BSD License](LICENSE).

COPYING

@@ -1,339 +0,0 @@
-                    GNU GENERAL PUBLIC LICENSE
-                       Version 2, June 1991
-
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
- Everyone is permitted to copy and distribute verbatim copies
- of this license document, but changing it is not allowed.
-
-                            Preamble
-
-  The licenses for most software are designed to take away your
-freedom to share and change it.  By contrast, the GNU General Public
-License is intended to guarantee your freedom to share and change free
-software--to make sure the software is free for all its users.  This
-General Public License applies to most of the Free Software
-Foundation's software and to any other program whose authors commit to
-using it.  (Some other Free Software Foundation software is covered by
-the GNU Lesser General Public License instead.)  You can apply it to
-your programs, too.
-
-  When we speak of free software, we are referring to freedom, not
-price.  Our General Public Licenses are designed to make sure that you
-have the freedom to distribute copies of free software (and charge for
-this service if you wish), that you receive source code or can get it
-if you want it, that you can change the software or use pieces of it
-in new free programs; and that you know you can do these things.
-
-  To protect your rights, we need to make restrictions that forbid
-anyone to deny you these rights or to ask you to surrender the rights.
-These restrictions translate to certain responsibilities for you if you
-distribute copies of the software, or if you modify it.
-
-  For example, if you distribute copies of such a program, whether
-gratis or for a fee, you must give the recipients all the rights that
-you have.  You must make sure that they, too, receive or can get the
-source code.  And you must show them these terms so they know their
-rights.
-
-  We protect your rights with two steps: (1) copyright the software, and
-(2) offer you this license which gives you legal permission to copy,
-distribute and/or modify the software.
-
-  Also, for each author's protection and ours, we want to make certain
-that everyone understands that there is no warranty for this free
-software.  If the software is modified by someone else and passed on, we
-want its recipients to know that what they have is not the original, so
-that any problems introduced by others will not reflect on the original
-authors' reputations.
-
-  Finally, any free program is threatened constantly by software
-patents.  We wish to avoid the danger that redistributors of a free
-program will individually obtain patent licenses, in effect making the
-program proprietary.  To prevent this, we have made it clear that any
-patent must be licensed for everyone's free use or not licensed at all.
-
-  The precise terms and conditions for copying, distribution and
-modification follow.
-
-                    GNU GENERAL PUBLIC LICENSE
-   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
-
-  0. This License applies to any program or other work which contains
-a notice placed by the copyright holder saying it may be distributed
-under the terms of this General Public License.  The "Program", below,
-refers to any such program or work, and a "work based on the Program"
-means either the Program or any derivative work under copyright law:
-that is to say, a work containing the Program or a portion of it,
-either verbatim or with modifications and/or translated into another
-language.  (Hereinafter, translation is included without limitation in
-the term "modification".)  Each licensee is addressed as "you".
-
-Activities other than copying, distribution and modification are not
-covered by this License; they are outside its scope.  The act of
-running the Program is not restricted, and the output from the Program
-is covered only if its contents constitute a work based on the
-Program (independent of having been made by running the Program).
-Whether that is true depends on what the Program does.
-
-  1. You may copy and distribute verbatim copies of the Program's
-source code as you receive it, in any medium, provided that you
-conspicuously and appropriately publish on each copy an appropriate
-copyright notice and disclaimer of warranty; keep intact all the
-notices that refer to this License and to the absence of any warranty;
-and give any other recipients of the Program a copy of this License
-along with the Program.
-
-You may charge a fee for the physical act of transferring a copy, and
-you may at your option offer warranty protection in exchange for a fee.
-
-  2. You may modify your copy or copies of the Program or any portion
-of it, thus forming a work based on the Program, and copy and
-distribute such modifications or work under the terms of Section 1
-above, provided that you also meet all of these conditions:
-
-    a) You must cause the modified files to carry prominent notices
-    stating that you changed the files and the date of any change.
-
-    b) You must cause any work that you distribute or publish, that in
-    whole or in part contains or is derived from the Program or any
-    part thereof, to be licensed as a whole at no charge to all third
-    parties under the terms of this License.
-
-    c) If the modified program normally reads commands interactively
-    when run, you must cause it, when started running for such
-    interactive use in the most ordinary way, to print or display an
-    announcement including an appropriate copyright notice and a
-    notice that there is no warranty (or else, saying that you provide
-    a warranty) and that users may redistribute the program under
-    these conditions, and telling the user how to view a copy of this
-    License.  (Exception: if the Program itself is interactive but
-    does not normally print such an announcement, your work based on
-    the Program is not required to print an announcement.)
-
-These requirements apply to the modified work as a whole.  If
-identifiable sections of that work are not derived from the Program,
-and can be reasonably considered independent and separate works in
-themselves, then this License, and its terms, do not apply to those
-sections when you distribute them as separate works.  But when you
-distribute the same sections as part of a whole which is a work based
-on the Program, the distribution of the whole must be on the terms of
-this License, whose permissions for other licensees extend to the
-entire whole, and thus to each and every part regardless of who wrote it.
-
-Thus, it is not the intent of this section to claim rights or contest
-your rights to work written entirely by you; rather, the intent is to
-exercise the right to control the distribution of derivative or
-collective works based on the Program.
-
-In addition, mere aggregation of another work not based on the Program
-with the Program (or with a work based on the Program) on a volume of
-a storage or distribution medium does not bring the other work under
-the scope of this License.
-
-  3. You may copy and distribute the Program (or a work based on it,
-under Section 2) in object code or executable form under the terms of
-Sections 1 and 2 above provided that you also do one of the following:
-
-    a) Accompany it with the complete corresponding machine-readable
-    source code, which must be distributed under the terms of Sections
-    1 and 2 above on a medium customarily used for software interchange; or,
-
-    b) Accompany it with a written offer, valid for at least three
-    years, to give any third party, for a charge no more than your
-    cost of physically performing source distribution, a complete
-    machine-readable copy of the corresponding source code, to be
-    distributed under the terms of Sections 1 and 2 above on a medium
-    customarily used for software interchange; or,
-
-    c) Accompany it with the information you received as to the offer
-    to distribute corresponding source code.  (This alternative is
-    allowed only for noncommercial distribution and only if you
-    received the program in object code or executable form with such
-    an offer, in accord with Subsection b above.)
-
-The source code for a work means the preferred form of the work for
-making modifications to it.  For an executable work, complete source
-code means all the source code for all modules it contains, plus any
-associated interface definition files, plus the scripts used to
-control compilation and installation of the executable.  However, as a
-special exception, the source code distributed need not include
-anything that is normally distributed (in either source or binary
-form) with the major components (compiler, kernel, and so on) of the
-operating system on which the executable runs, unless that component
-itself accompanies the executable.
-
-If distribution of executable or object code is made by offering
-access to copy from a designated place, then offering equivalent
-access to copy the source code from the same place counts as
-distribution of the source code, even though third parties are not
-compelled to copy the source along with the object code.
-
-  4. You may not copy, modify, sublicense, or distribute the Program
-except as expressly provided under this License.  Any attempt
-otherwise to copy, modify, sublicense or distribute the Program is
-void, and will automatically terminate your rights under this License.
-However, parties who have received copies, or rights, from you under
-this License will not have their licenses terminated so long as such
-parties remain in full compliance.
-
-  5. You are not required to accept this License, since you have not
-signed it.  However, nothing else grants you permission to modify or
-distribute the Program or its derivative works.  These actions are
-prohibited by law if you do not accept this License.  Therefore, by
-modifying or distributing the Program (or any work based on the
-Program), you indicate your acceptance of this License to do so, and
-all its terms and conditions for copying, distributing or modifying
-the Program or works based on it.
-
-  6. Each time you redistribute the Program (or any work based on the
-Program), the recipient automatically receives a license from the
-original licensor to copy, distribute or modify the Program subject to
-these terms and conditions.  You may not impose any further
-restrictions on the recipients' exercise of the rights granted herein.
-You are not responsible for enforcing compliance by third parties to
-this License.
-
-  7. If, as a consequence of a court judgment or allegation of patent
-infringement or for any other reason (not limited to patent issues),
-conditions are imposed on you (whether by court order, agreement or
-otherwise) that contradict the conditions of this License, they do not
-excuse you from the conditions of this License.  If you cannot
-distribute so as to satisfy simultaneously your obligations under this
-License and any other pertinent obligations, then as a consequence you
-may not distribute the Program at all.  For example, if a patent
-license would not permit royalty-free redistribution of the Program by
-all those who receive copies directly or indirectly through you, then
-the only way you could satisfy both it and this License would be to
-refrain entirely from distribution of the Program.
-
-If any portion of this section is held invalid or unenforceable under
-any particular circumstance, the balance of the section is intended to
-apply and the section as a whole is intended to apply in other
-circumstances.
-
-It is not the purpose of this section to induce you to infringe any
-patents or other property right claims or to contest validity of any
-such claims; this section has the sole purpose of protecting the
-integrity of the free software distribution system, which is
-implemented by public license practices.  Many people have made
-generous contributions to the wide range of software distributed
-through that system in reliance on consistent application of that
-system; it is up to the author/donor to decide if he or she is willing
-to distribute software through any other system and a licensee cannot
-impose that choice.
-
-This section is intended to make thoroughly clear what is believed to
-be a consequence of the rest of this License.
-
-  8. If the distribution and/or use of the Program is restricted in
-certain countries either by patents or by copyrighted interfaces, the
-original copyright holder who places the Program under this License
-may add an explicit geographical distribution limitation excluding
-those countries, so that distribution is permitted only in or among
-countries not thus excluded.  In such case, this License incorporates
-the limitation as if written in the body of this License.
-
-  9. The Free Software Foundation may publish revised and/or new versions
-of the General Public License from time to time.  Such new versions will
-be similar in spirit to the present version, but may differ in detail to
-address new problems or concerns.
-
-Each version is given a distinguishing version number.  If the Program
-specifies a version number of this License which applies to it and "any
-later version", you have the option of following the terms and conditions
-either of that version or of any later version published by the Free
-Software Foundation.  If the Program does not specify a version number of
-this License, you may choose any version ever published by the Free Software
-Foundation.
-
-  10. If you wish to incorporate parts of the Program into other free
-programs whose distribution conditions are different, write to the author
-to ask for permission.  For software which is copyrighted by the Free
-Software Foundation, write to the Free Software Foundation; we sometimes
-make exceptions for this.  Our decision will be guided by the two goals
-of preserving the free status of all derivatives of our free software and
-of promoting the sharing and reuse of software generally.
-
-                            NO WARRANTY
-
-  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
-FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
-OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
-PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
-OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
-TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
-PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
-REPAIR OR CORRECTION.
-
-  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
-WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
-REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
-INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
-OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
-TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
-YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
-PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGES.
-
-                     END OF TERMS AND CONDITIONS
-
-            How to Apply These Terms to Your New Programs
-
-  If you develop a new program, and you want it to be of the greatest
-possible use to the public, the best way to achieve this is to make it
-free software which everyone can redistribute and change under these terms.
-
-  To do so, attach the following notices to the program.  It is safest
-to attach them to the start of each source file to most effectively
-convey the exclusion of warranty; and each file should have at least
-the "copyright" line and a pointer to where the full notice is found.
-
-    <one line to give the program's name and a brief idea of what it does.>
-    Copyright (C) <year>  <name of author>
-
-    This program is free software; you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation; either version 2 of the License, or
-    (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
-
-    You should have received a copy of the GNU General Public License along
-    with this program; if not, write to the Free Software Foundation, Inc.,
-    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-Also add information on how to contact you by electronic and paper mail.
-
-If the program is interactive, make it output a short notice like this
-when it starts in an interactive mode:
-
-    Gnomovision version 69, Copyright (C) year name of author
-    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
-    This is free software, and you are welcome to redistribute it
-    under certain conditions; type `show c' for details.
-
-The hypothetical commands `show w' and `show c' should show the appropriate
-parts of the General Public License.  Of course, the commands you use may
-be called something other than `show w' and `show c'; they could even be
-mouse-clicks or menu items--whatever suits your program.
-
-You should also get your employer (if you work as a programmer) or your
-school, if any, to sign a "copyright disclaimer" for the program, if
-necessary.  Here is a sample; alter the names:
-
-  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
-  `Gnomovision' (which makes passes at compilers) written by James Hacker.
-
-  <signature of Ty Coon>, 1 April 1989
-  Ty Coon, President of Vice
-
-This General Public License does not permit incorporating your program into
-proprietary programs.  If your program is a subroutine library, you may
-consider it more useful to permit linking proprietary applications with the
-library.  If this is what you want to do, use the GNU Lesser General
-Public License instead of this License.

DEFAULT_OPTIONS_HISTORY.md

@@ -1,8 +1,5 @@
 # RocksDB default options change log
 ## Unreleased
-* delayed_write_rate takes the rate given by rate_limiter if not specified.
-
-## 5.2
 * Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
 
 ## 5.0 (11/17/2016)

HISTORY.md

@@ -1,78 +1,43 @@
 # Rocksdb Change Log
-## 5.7.5 (11/28/2017)
+## 5.4.6 (05/30/2017)
 ### Bug Fixes
-* Fix IOError on WAL write doesn't propagate to write group follower
+* Fix db_bench with lite build.
 
-## 5.7.4 (08/31/2017)
-No significant changes.
-
-## 5.7.3 (08/29/2017)
+## 5.4.5 (05/19/2017)
 ### Bug Fixes
-* Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled.
-* Fix potentially wrong file smallest key when range deletions separated by snapshot are written together.
-
-## 5.7.2 (08/15/2017)
-### Bug Fixes
-* Fix incorrect dropping of deletions issue with FIFO compaction.
-* Fix LITE build compiler error with missing abort().
-
-## 5.7.1 (08/13/2017)
-### Bug Fixes
-* Fix incorrect dropping of deletions during intra-L0 compaction.
-
-## 5.7.0 (07/13/2017)
-### Public API Change
-* DB property "rocksdb.sstables" now prints keys in hex form.
+* Facility for cross-building RocksJava using Docker.
+* Build and link with ZStd when creating the static RocksJava build.
+* Fix snprintf's usage to be cross-platform.
+* Fix build error with blob DB.
+* Fix readamp test type inconsistency.
 
+## 5.4.4 (05/11/2017)
 ### New Features
-* Measure estimated number of reads per file. The information can be accessed through DB::GetColumnFamilyMetaData or "rocksdb.sstables" DB property.
-* RateLimiter support for throttling background reads, or throttling the sum of background reads and writes. This can give more predictable I/O usage when compaction reads more data than it writes, e.g., due to lots of deletions.
-* [Experimental] FIFO compaction with TTL support. It can be enabled by setting CompactionOptionsFIFO.ttl > 0.
-* Introduce `EventListener::OnBackgroundError()` callback. Users can implement it to be notified of errors causing the DB to enter read-only mode, and optionally override them.
-* Partitioned Index/Filters exiting the experimental mode. To enable partitioned indexes set index_type to kTwoLevelIndexSearch and to further enable partitioned filters set partition_filters to true. To configure the partition size set metadata_block_size.
-
-
-### Bug Fixes
-* Fix discarding empty compaction output files when `DeleteRange()` is used together with subcompactions.
-
-## 5.6.0 (06/06/2017)
-### Public API Change
-* Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads.
-* Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
-* options.delayed_write_rate by default take the value of options.rate_limiter rate.
-* Replace global variable `IOStatsContext iostats_context` with `IOStatsContext* get_iostats_context()`; replace global variable `PerfContext perf_context` with `PerfContext* get_perf_context()`.
-
-### New Features
-* Change ticker/histogram statistics implementations to use core-local storage. This improves aggregation speed compared to our previous thread-local approach, particularly for applications with many threads.
-* Users can pass a cache object to write buffer manager, so that they can cap memory usage for memtable and block cache using one single limit.
-* Flush will be triggered when 7/8 of the limit introduced by write_buffer_manager or db_write_buffer_size is triggered, so that the hard threshold is hard to hit.
-* Introduce WriteOptions.low_pri. If it is true, low priority writes will be throttled if the compaction is behind.
-* `DB::IngestExternalFile()` now supports ingesting files into a database containing range deletions.
-
-### Bug Fixes
-* Shouldn't ignore return value of fsync() in flush.
-
-## 5.5.0 (05/17/2017)
-### New Features
-* FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true.
-* DB::ResetStats() to reset internal stats.
-* Statistics::Reset() to reset user stats.
-* ldb add option --try_load_options, which will open DB with its own option file.
-* Introduce WriteBatch::PopSavePoint to pop the most recent save point explicitly.
-* Support dynamically change `max_open_files` option via SetDBOptions()
 * Added DB::CreateColumnFamilie() and DB::DropColumnFamilies() to bulk create/drop column families.
-* Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys.
-* Support file ingestion with universal compaction style
-* Support file ingestion behind with option `allow_ingest_behind`
-* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled.
 
 ### Bug Fixes
-* Fix the bug that Direct I/O uses direct reads for non-SST file
+* Fix a manual / auto compaction data race.
+* Fix CentOS 5 cross-building of RocksJava.
+
+## 5.4.3 (05/10/2017)
+### Bug Fixes
+* Fix memory alignment with logical sector size.
+* Fix alignment in ReadaheadRandomAccessFile.
+* Fix bias with read amplification stats (READ_AMP_ESTIMATE_USEFUL_BYTES and READ_AMP_TOTAL_READ_BYTES).
+
+## 5.4.2 (05/08/2017)
+### Bug Fixes
+* Fix huge fallocate calls fail and make XFS unhappy.
+
+## 5.4.1 (05/02/2017)
+### Bug Fxies
+* Fix WriteBatchWithIndex address use after scope error.
+* Fix WritableFile buffer size in direct IO.
+* Add prefetch to PosixRandomAccessFile in buffered io.
+* Fix PinnableSlice access invalid address when row cache is enabled.
 
 ## 5.4.0 (04/11/2017)
 ### Public API Change
-* random_access_max_buffer_size no longer has any effect
-* Removed Env::EnableReadAhead(), Env::ShouldForwardRawRequest()
 * Support dynamically change `stats_dump_period_sec` option via SetDBOptions().
 * Added ReadOptions::max_skippable_internal_keys to set a threshold to fail a request as incomplete when too many keys are being skipped when using iterators.
 * DB::Get in place of std::string accepts PinnableSlice, which avoids the extra memcpy of value to std::string in most of cases.
@@ -85,6 +50,7 @@ No significant changes.
 * Memtable flush can be avoided during checkpoint creation if total log file size is smaller than a threshold specified by the user.
 * Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
 * (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.
+* DB::ResetStats() to reset internal stats.
 
 ## 5.3.0 (03/08/2017)
 ### Public API Change
@@ -311,7 +277,7 @@ No significant changes.
 * Added a new way to report QPS from db_bench (check out --report_file and --report_interval_seconds)
 * Added a cache for individual rows. See DBOptions::row_cache for more info.
 * Several new features on EventListener (see include/rocksdb/listener.h):
- - OnCompationCompleted() now returns per-compaction job statistics, defined in include/rocksdb/compaction_job_stats.h.
+ - OnCompationCompleted() now returns per-compaciton job statistics, defined in include/rocksdb/compaction_job_stats.h.
  - Added OnTableFileCreated() and OnTableFileDeleted().
 * Add compaction_options_universal.enable_trivial_move to true, to allow trivial move while performing universal compaction. Trivial move will happen only when all the input files are non overlapping.
 

INSTALL.md

@@ -21,18 +21,15 @@ depend on gflags. You will need to have gflags installed to run `make all`. This
 use binaries compiled by `make all` in production.
 
 * By default the binary we produce is optimized for the platform you're compiling on
-(`-march=native` or the equivalent). SSE4.2 will thus be enabled automatically if your
-CPU supports it. To print a warning if your CPU does not support SSE4.2, build with
-`USE_SSE=1 make static_lib` or, if using CMake, `cmake -DFORCE_SSE42=ON`. If you want
-to build a portable binary, add `PORTABLE=1` before your make commands, like this:
-`PORTABLE=1 make static_lib`.
+(-march=native or the equivalent). If you want to build a portable binary, add 'PORTABLE=1' before
+your make commands, like this: `PORTABLE=1 make static_lib`. If you want to build a binary that
+makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SSE=1 make static_lib`.
 
 ## Dependencies
 
 * You can link RocksDB with following compression libraries:
   - [zlib](http://www.zlib.net/) - a library for data compression.
   - [bzip2](http://www.bzip.org/) - a library for data compression.
-  - [lz4](https://github.com/lz4/lz4) - a library for extremely fast data compression.
   - [snappy](http://google.github.io/snappy/) - a library for fast
       data compression.
   - [zstandard](http://www.zstd.net) - Fast real-time compression
@@ -54,7 +51,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
       `sudo apt-get install libsnappy-dev`.
     * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
     * Install bzip2: `sudo apt-get install libbz2-dev`.
-    * Install lz4: `sudo apt-get install liblz4-dev`.
     * Install zstandard: `sudo apt-get install libzstd-dev`.
 
 * **Linux - CentOS / RHEL**
@@ -67,29 +63,26 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
               git checkout v2.0
               ./configure && make && sudo make install
 
-      **Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the
-      lib path to `LIBRARY_PATH`. If installed with default settings, the include path will be `/usr/local/include`
-      and the lib path will be `/usr/local/lib`.
+      **Notice**: Once installed, please add the include path for gflags to your CPATH env var and the
+      lib path to LIBRARY_PATH. If installed with default settings, the lib will be /usr/local/lib
+      and the include path will be /usr/local/include.
 
     * Install snappy:
 
-              sudo yum install snappy snappy-devel
+              wget https://github.com/google/snappy/releases/download/1.1.4/snappy-1.1.4.tar.gz
+              tar -xzvf snappy-1.1.4.tar.gz
+              cd snappy-1.1.4
+              ./configure && make && sudo make install
 
     * Install zlib:
 
-              sudo yum install zlib zlib-devel
+              sudo yum install zlib
+              sudo yum install zlib-devel
 
     * Install bzip2:
 
-              sudo yum install bzip2 bzip2-devel
-
-    * Install lz4:
-
-              sudo yum install lz4-devel
-
-    * Install ASAN (optional for debugging):
-
-              sudo yum install libasan
+              sudo yum install bzip2
+              sudo yum install bzip2-devel
 
     * Install zstandard:
 
@@ -113,30 +106,3 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 * **Windows**:
   * For building with MS Visual Studio 13 you will need Update 4 installed.
   * Read and follow the instructions at CMakeLists.txt
-
-* **AIX 6.1**
-    * Install AIX Toolbox rpms with gcc
-    * Use these environment variables:
-  
-             export PORTABLE=1
-             export CC=gcc
-             export AR="ar -X64"
-             export EXTRA_ARFLAGS=-X64
-             export EXTRA_CFLAGS=-maix64
-             export EXTRA_CXXFLAGS=-maix64
-             export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
-             export LIBPATH=/opt/freeware/lib
-             export JAVA_HOME=/usr/java8_64
-             export PATH=/opt/freeware/bin:$PATH
-  
-* **Solaris Sparc**
-    * Install GCC 4.8.2 and higher.
-    * Use these environment variables:
-
-             export CC=gcc
-             export EXTRA_CFLAGS=-m64
-             export EXTRA_CXXFLAGS=-m64
-             export EXTRA_LDFLAGS=-m64
-             export PORTABLE=1
-             export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
-

LICENSE

@@ -1,4 +1,10 @@
-This contains code that is from LevelDB, and that code is under the following license:
+BSD License
+
+For rocksdb software
+
+Copyright (c) 2011-present, Facebook, Inc.
+All rights reserved.
+---------------------------------------------------------------------
 
 Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 

LICENSE.Apache

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

Makefile

@@ -14,8 +14,7 @@ CFLAGS += ${EXTRA_CFLAGS}
 CXXFLAGS += ${EXTRA_CXXFLAGS}
 LDFLAGS += $(EXTRA_LDFLAGS)
 MACHINE ?= $(shell uname -m)
-ARFLAGS = ${EXTRA_ARFLAGS} rs
-STRIPFLAGS = -S -x
+ARFLAGS = rs
 
 # Transform parallel LOG output into something more readable.
 perl_command = perl -n \
@@ -155,9 +154,7 @@ missing_make_config_paths := $(shell				\
 $(foreach path, $(missing_make_config_paths), \
 	$(warning Warning: $(path) dont exist))
 
-ifeq ($(PLATFORM), OS_AIX)
-# no debug info
-else ifneq ($(PLATFORM), IOS)
+ifneq ($(PLATFORM), IOS)
 CFLAGS += -g
 CXXFLAGS += -g
 else
@@ -165,11 +162,6 @@ else
 OPT += -DNDEBUG
 endif
 
-ifeq ($(PLATFORM), OS_AIX)
-ARFLAGS = -X64 rs
-STRIPFLAGS = -X64 -x
-endif
-
 ifeq ($(PLATFORM), OS_SOLARIS)
 	PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99
 endif
@@ -192,9 +184,9 @@ endif
 # TSAN doesn't work well with jemalloc. If we're compiling with TSAN, we should use regular malloc.
 ifdef COMPILE_WITH_TSAN
 	DISABLE_JEMALLOC=1
-	EXEC_LDFLAGS += -fsanitize=thread
-	PLATFORM_CCFLAGS += -fsanitize=thread -fPIC
-	PLATFORM_CXXFLAGS += -fsanitize=thread -fPIC
+	EXEC_LDFLAGS += -fsanitize=thread -pie
+	PLATFORM_CCFLAGS += -fsanitize=thread -fPIC -DROCKSDB_TSAN_RUN
+	PLATFORM_CXXFLAGS += -fsanitize=thread -fPIC -DROCKSDB_TSAN_RUN
         # Turn off -pg when enabling TSAN testing, because that induces
         # a link failure.  TODO: find the root cause
 	PROFILING_FLAGS =
@@ -202,17 +194,12 @@ ifdef COMPILE_WITH_TSAN
 	LUA_PATH =
 endif
 
-# AIX doesn't work with -pg
-ifeq ($(PLATFORM), OS_AIX)
-	PROFILING_FLAGS =
-endif
-
 # USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
 ifdef COMPILE_WITH_UBSAN
 	DISABLE_JEMALLOC=1
 	EXEC_LDFLAGS += -fsanitize=undefined
-	PLATFORM_CCFLAGS += -fsanitize=undefined -DROCKSDB_UBSAN_RUN
-	PLATFORM_CXXFLAGS += -fsanitize=undefined -DROCKSDB_UBSAN_RUN
+	PLATFORM_CCFLAGS += -fsanitize=undefined
+	PLATFORM_CXXFLAGS += -fsanitize=undefined
 endif
 
 ifndef DISABLE_JEMALLOC
@@ -228,14 +215,8 @@ endif
 export GTEST_THROW_ON_FAILURE=1
 export GTEST_HAS_EXCEPTIONS=1
 GTEST_DIR = ./third-party/gtest-1.7.0/fused-src
-# AIX: pre-defined system headers are surrounded by an extern "C" block
-ifeq ($(PLATFORM), OS_AIX)
-	PLATFORM_CCFLAGS += -I$(GTEST_DIR)
-	PLATFORM_CXXFLAGS += -I$(GTEST_DIR)
-else
-	PLATFORM_CCFLAGS += -isystem $(GTEST_DIR)
-	PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
-endif
+PLATFORM_CCFLAGS += -isystem $(GTEST_DIR)
+PLATFORM_CXXFLAGS += -isystem $(GTEST_DIR)
 
 # This (the first rule) must depend on "all".
 default: all
@@ -280,9 +261,6 @@ CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverl
 
 LDFLAGS += $(PLATFORM_LDFLAGS)
 
-# If NO_UPDATE_BUILD_VERSION is set we don't update util/build_version.cc, but
-# the file needs to already exist or else the build will fail
-ifndef NO_UPDATE_BUILD_VERSION
 date := $(shell date +%F)
 ifdef FORCE_GIT_SHA
 	git_sha := $(FORCE_GIT_SHA)
@@ -296,6 +274,7 @@ gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(da
 # as a regular source file as part of the compilation process.
 # One can run "strings executable_filename | grep _build_" to find
 # the version of the source that we used to build the executable file.
+CLEAN_FILES += util/build_version.cc:
 FORCE:
 util/build_version.cc: FORCE
 	$(AM_V_GEN)rm -f $@-t
@@ -303,7 +282,6 @@ util/build_version.cc: FORCE
 	$(AM_V_at)if test -f $@; then					\
 	  cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@;		\
 	else mv -f $@-t $@; fi
-endif
 
 LIBOBJECTS = $(LIB_SOURCES:.cc=.o)
 LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o)
@@ -323,7 +301,6 @@ EXPOBJECTS = $(EXP_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
 
 TESTS = \
 	db_basic_test \
-	db_encryption_test \
 	db_test2 \
 	external_sst_file_basic_test \
 	auto_roll_logger_test \
@@ -336,7 +313,6 @@ TESTS = \
 	inlineskiplist_test \
 	env_basic_test \
 	env_test \
-	hash_test \
 	thread_local_test \
 	rate_limiter_test \
 	perf_context_test \
@@ -363,10 +339,7 @@ TESTS = \
 	db_io_failure_test \
 	db_properties_test \
 	db_table_properties_test \
-	db_statistics_test \
-	db_write_test \
 	autovector_test \
-	blob_db_test \
 	cleanable_test \
 	column_family_test \
 	table_properties_collector_test \
@@ -402,12 +375,7 @@ TESTS = \
 	external_sst_file_test \
 	prefix_test \
 	skiplist_test \
-	write_buffer_manager_test \
 	stringappend_test \
-	cassandra_format_test \
-	cassandra_merge_test \
-	cassandra_row_merge_test \
-	cassandra_serialize_test \
 	ttl_test \
 	date_tiered_test \
 	backupable_db_test \
@@ -494,8 +462,7 @@ TOOLS = \
 	ldb \
 	db_repl_stress \
 	rocksdb_dump \
-	rocksdb_undump \
-	blob_dump \
+	rocksdb_undump
 
 TEST_LIBS = \
 	librocksdb_env_basic_test.a
@@ -556,14 +523,9 @@ $(SHARED3): $(SHARED4)
 	ln -fs $(SHARED4) $(SHARED3)
 endif
 
-shared_libobjects = $(patsubst %,shared-objects/%,$(LIBOBJECTS))
-CLEAN_FILES += shared-objects
-
-$(shared_libobjects): shared-objects/%.o: %.cc
-	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
-
-$(SHARED4): $(shared_libobjects)
-	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(shared_libobjects) $(LDFLAGS) -o $@
+$(SHARED4):
+	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LIB_SOURCES) $(TOOL_LIB_SOURCES) \
+		$(LDFLAGS) -o $@
 
 endif  # PLATFORM_SHARED_EXT
 
@@ -604,8 +566,7 @@ ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
 # and create a randomly-named rocksdb.XXXX directory therein.
 # We'll use that directory in the "make check" rules.
 ifeq ($(TMPD),)
-TMPDIR := $(shell echo $${TMPDIR:-/tmp})
-TMPD := $(shell f=/dev/shm; test -k $$f || f=$(TMPDIR);     \
+TMPD := $(shell f=/dev/shm; test -k $$f || f=/tmp;			\
   perl -le 'use File::Temp "tempdir";'					\
     -e 'print tempdir("'$$f'/rocksdb.XXXX", CLEANUP => 0)')
 endif
@@ -760,12 +721,10 @@ check: all
 	      echo "===== Running $$t"; ./$$t || exit 1; done;          \
 	fi
 	rm -rf $(TMPD)
-ifneq ($(PLATFORM), OS_AIX)
 ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
 	python tools/ldb_test.py
 	sh tools/rocksdb_dump_test.sh
 endif
-endif
 
 # TODO add ldb_tests
 check_some: $(SUBSET)
@@ -911,13 +870,12 @@ clean:
 	rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
 	find . -name "*.[oda]" -exec rm -f {} \;
 	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
-	rm -rf bzip2* snappy* zlib* lz4* zstd*
+	rm -rf bzip2* snappy* zlib* lz4*
 	cd java; $(MAKE) clean
 
 tags:
 	ctags * -R
 	cscope -b `find . -name '*.cc'` `find . -name '*.h'` `find . -name '*.c'`
-	ctags -e -R -o etags *
 
 format:
 	build_tools/format-diff.sh
@@ -991,27 +949,12 @@ cache_test: cache/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
 coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 option_change_migration_test: utilities/option_change_migration/option_change_migration_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-cassandra_format_test: utilities/merge_operators/cassandra/cassandra_format_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-cassandra_merge_test: utilities/merge_operators/cassandra/cassandra_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-cassandra_row_merge_test: utilities/merge_operators/cassandra/cassandra_row_merge_test.o utilities/merge_operators/cassandra/test_utils.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-cassandra_serialize_test: utilities/merge_operators/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1036,9 +979,6 @@ slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 db_basic_test: db/db_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_encryption_test: db/db_encryption_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1087,12 +1027,6 @@ db_range_del_test: db/db_range_del_test.o db/db_test_util.o $(LIBOBJECTS) $(TEST
 db_sst_test: db/db_sst_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-db_statistics_test: db/db_statistics_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
-db_write_test: db/db_write_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 external_sst_file_basic_test: db/external_sst_file_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1248,9 +1182,6 @@ inlineskiplist_test: memtable/inlineskiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 skiplist_test: memtable/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-write_buffer_manager_test: memtable/write_buffer_manager_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1377,9 +1308,6 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)
 
-blob_dump: tools/blob_dump.o $(LIBOBJECTS)
-	$(AM_LINK)
-
 column_aware_encoding_exp: utilities/column_aware_encoding_exp.o $(EXPOBJECTS)
 	$(AM_LINK)
 
@@ -1410,9 +1338,6 @@ lua_test: utilities/lua/rocks_lua_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTH
 range_del_aggregator_test: db/range_del_aggregator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
-blob_db_test: utilities/blob_db/blob_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(AM_LINK)
-
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -1472,18 +1397,14 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PA
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar
 SHA256_CMD = sha256sum
 
-ZLIB_VER ?= 1.2.11
-ZLIB_SHA256 ?= c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
-ZLIB_DOWNLOAD_BASE ?= http://zlib.net
-BZIP2_VER ?= 1.0.6
-BZIP2_SHA256 ?= a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
-BZIP2_DOWNLOAD_BASE ?= http://www.bzip.org
-SNAPPY_VER ?= 1.1.4
-SNAPPY_SHA256 ?= 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
-SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/releases/download
-LZ4_VER ?= 1.7.5
-LZ4_SHA256 ?= 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
-LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
+ZLIB_VER = 1.2.11
+ZLIB_SHA256 = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
+BZIP2_VER = 1.0.6
+BZIP2_SHA256 = a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
+SNAPPY_VER = 1.1.4
+SNAPPY_SHA256 = 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
+LZ4_VER = 1.7.5
+LZ4_SHA256 = 0190cacd63022ccb86f44fa5041dc6c3804407ad61550ca21c382827319e7e7e
 ZSTD_VER ?= 1.2.0
 ZSTD_SHA256 ?= 4a7e4593a3638276ca7f2a09dc4f38e674d8317bbea51626393ca73fc047cbfb
 ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
@@ -1509,53 +1430,47 @@ ifeq ($(PLATFORM), OS_SOLARIS)
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris
 	SHA256_CMD = digest -a sha256
 endif
-ifeq ($(PLATFORM), OS_AIX)
-	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/aix
-	ROCKSDBJNILIB = librocksdbjni-aix.so
-	EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf -
-	SNAPPY_MAKE_TARGET = libsnappy.la
-endif
 
 libz.a:
 	-rm -rf zlib-$(ZLIB_VER)
-	curl -O -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
+	curl -O -L http://zlib.net/zlib-$(ZLIB_VER).tar.gz
 	ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \
 		echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \
 		exit 1; \
 	fi
 	tar xvzf zlib-$(ZLIB_VER).tar.gz
-	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make
+	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC' ./configure --static && make
 	cp zlib-$(ZLIB_VER)/libz.a .
 
 libbz2.a:
 	-rm -rf bzip2-$(BZIP2_VER)
-	curl -O -L ${BZIP2_DOWNLOAD_BASE}/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
+	curl -O -L http://www.bzip.org/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
 	BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \
 		echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \
 		exit 1; \
 	fi
 	tar xvzf bzip2-$(BZIP2_VER).tar.gz
-	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
+	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64'
 	cp bzip2-$(BZIP2_VER)/libbz2.a .
 
 libsnappy.a:
 	-rm -rf snappy-$(SNAPPY_VER)
-	curl -O -L ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
+	curl -O -L https://github.com/google/snappy/releases/download/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
 	SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
 		echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \
 		exit 1; \
 	fi
 	tar xvzf snappy-$(SNAPPY_VER).tar.gz
-	cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
-	cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET}
+	cd snappy-$(SNAPPY_VER) && ./configure --with-pic --enable-static
+	cd snappy-$(SNAPPY_VER) && make
 	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
 
 liblz4.a:
 	-rm -rf lz4-$(LZ4_VER)
-	curl -O -L ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
+	curl -O -L https://github.com/lz4/lz4/archive/v$(LZ4_VER).tar.gz
 	mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz
 	LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
@@ -1563,7 +1478,7 @@ liblz4.a:
 		exit 1; \
 	fi
 	tar xvzf lz4-$(LZ4_VER).tar.gz
-	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
+	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC' all
 	cp lz4-$(LZ4_VER)/lib/liblz4.a .
 
 libzstd.a:
@@ -1600,7 +1515,7 @@ rocksdbjavastatic: $(java_static_libobjects)
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
 	  $(java_static_libobjects) $(COVERAGEFLAGS) \
 	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
-	cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
+	cd java/target;strip -S -x $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
@@ -1616,12 +1531,12 @@ rocksdbjavastaticrelease: rocksdbjavastatic
 rocksdbjavastaticreleasedocker: rocksdbjavastatic
 	DOCKER_LINUX_X64_CONTAINER=`docker ps -aqf name=rocksdb_linux_x64-be`; \
 	if [ -z "$$DOCKER_LINUX_X64_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x64-be evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
+		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x64-be evolvedbinary/rocksjava:centos5_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
 	fi
 	docker start -a rocksdb_linux_x64-be
 	DOCKER_LINUX_X86_CONTAINER=`docker ps -aqf name=rocksdb_linux_x86-be`; \
 	if [ -z "$$DOCKER_LINUX_X86_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
+		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos5_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
 	fi
 	docker start -a rocksdb_linux_x86-be
 	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md

PATENTS

@@ -0,0 +1,33 @@
+Additional Grant of Patent Rights Version 2
+
+"Software" means the RocksDB software distributed by Facebook, Inc.
+
+Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software
+("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable
+(subject to the termination provision below) license under any Necessary
+Claims, to make, have made, use, sell, offer to sell, import, and otherwise
+transfer the Software. For avoidance of doubt, no license is granted under
+Facebook’s rights in any patent claims that are infringed by (i) modifications
+to the Software made by you or any third party or (ii) the Software in
+combination with any software or other technology.
+
+The license granted hereunder will terminate, automatically and without notice,
+if you (or any of your subsidiaries, corporate affiliates or agents) initiate
+directly or indirectly, or take a direct financial interest in, any Patent
+Assertion: (i) against Facebook or any of its subsidiaries or corporate
+affiliates, (ii) against any party if such Patent Assertion arises in whole or
+in part from any software, technology, product or service of Facebook or any of
+its subsidiaries or corporate affiliates, or (iii) against any party relating
+to the Software. Notwithstanding the foregoing, if Facebook or any of its
+subsidiaries or corporate affiliates files a lawsuit alleging patent
+infringement against you in the first instance, and you respond by filing a
+patent infringement counterclaim in that lawsuit against that party that is
+unrelated to the Software, the license granted hereunder will not terminate
+under section (i) of this paragraph due to such counterclaim.
+
+A "Necessary Claim" is a claim of a patent owned by Facebook that is
+necessarily infringed by the Software standing alone.
+
+A "Patent Assertion" is any lawsuit or other action alleging direct, indirect,
+or contributory infringement or inducement to infringe any patent, including a
+cross-claim or counterclaim.

TARGETS

@@ -1,8 +1,4 @@
-
-import os
-
-TARGETS_PATH = os.path.dirname(__file__)
-REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
+REPO_PATH = "internal_repo_rocksdb/repo/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
@@ -13,10 +9,6 @@ rocksdb_compiler_flags = [
   "-DROCKSDB_LIB_IO_POSIX",
   "-DROCKSDB_FALLOCATE_PRESENT",
   "-DROCKSDB_MALLOC_USABLE_SIZE",
-  "-DROCKSDB_RANGESYNC_PRESENT",
-  "-DROCKSDB_SCHED_GETCPU_PRESENT",
-  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
-  "-DHAVE_SSE42",
   "-DOS_LINUX",
   # Flags to enable libs we include
   "-DSNAPPY",
@@ -39,7 +31,7 @@ rocksdb_external_deps = [
   ('lz4', None, 'lz4'),
   ('zstd', None),
   ('tbb', None),
-  ("numa", None, "numa"),
+  ("numa", "2.0.8", "numa"),
   ("googletest", None, "gtest"),
 ]
 
@@ -88,7 +80,6 @@ cpp_library(
       "db/internal_stats.cc",
       "db/log_reader.cc",
       "db/log_writer.cc",
-      "db/malloc_stats.cc",
       "db/managed_iterator.cc",
       "db/memtable.cc",
       "db/memtable_list.cc",
@@ -110,18 +101,16 @@ cpp_library(
       "db/write_thread.cc",
       "env/env.cc",
       "env/env_chroot.cc",
-      "env/env_encryption.cc",
       "env/env_hdfs.cc",
       "env/env_posix.cc",
       "env/io_posix.cc",
-      "env/mock_env.cc",
-      "memtable/alloc_tracker.cc",
+      "env/memenv.cc",
       "memtable/hash_cuckoo_rep.cc",
       "memtable/hash_linklist_rep.cc",
       "memtable/hash_skiplist_rep.cc",
+      "memtable/memtable_allocator.cc",
       "memtable/skiplistrep.cc",
       "memtable/vectorrep.cc",
-      "memtable/write_buffer_manager.cc",
       "monitoring/histogram.cc",
       "monitoring/histogram_windowing.cc",
       "monitoring/instrumented_mutex.cc",
@@ -207,16 +196,15 @@ cpp_library(
       "utilities/backupable/backupable_db.cc",
       "utilities/blob_db/blob_db.cc",
       "utilities/blob_db/blob_db_impl.cc",
+      "utilities/blob_db/blob_db_options_impl.cc",
       "utilities/blob_db/blob_file.cc",
       "utilities/blob_db/blob_log_reader.cc",
       "utilities/blob_db/blob_log_writer.cc",
       "utilities/blob_db/blob_log_format.cc",
-      "utilities/blob_db/ttl_extractor.cc",
-      "utilities/checkpoint/checkpoint_impl.cc",
+      "utilities/checkpoint/checkpoint.cc",
       "utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
       "utilities/convenience/info_log_finder.cc",
       "utilities/date_tiered/date_tiered_db_impl.cc",
-      "utilities/debug.cc",
       "utilities/document/document_db.cc",
       "utilities/document/json_document.cc",
       "utilities/document/json_document_builder.cc",
@@ -226,8 +214,6 @@ cpp_library(
       "utilities/leveldb_options/leveldb_options.cc",
       "utilities/lua/rocks_lua_compaction_filter.cc",
       "utilities/memory/memory_util.cc",
-      "utilities/merge_operators/cassandra/format.cc",
-      "utilities/merge_operators/cassandra/merge_operator.cc",
       "utilities/merge_operators/max.cc",
       "utilities/merge_operators/put.cc",
       "utilities/merge_operators/string_append/stringappend.cc",
@@ -258,7 +244,6 @@ cpp_library(
       "tools/ldb_cmd.cc",
       "tools/ldb_tool.cc",
       "tools/sst_dump_tool.cc",
-      "utilities/blob_db/blob_dump_tool.cc",
     ],
     deps = [],
     preprocessor_flags = rocksdb_preprocessor_flags,
@@ -270,12 +255,12 @@ cpp_library(
     name = "rocksdb_test_lib",
     headers = AutoHeaders.RECURSIVE_GLOB,
     srcs = [
+      "env/mock_env.cc",
       "table/mock_table.cc",
       "util/fault_injection_test_env.cc",
       "util/testharness.cc",
       "util/testutil.cc",
       "db/db_test_util.cc",
-      "utilities/merge_operators/cassandra/test_utils.cc",
       "utilities/col_buf_encoder.cc",
       "utilities/col_buf_decoder.cc",
       "utilities/column_aware_encoding_util.cc",
@@ -310,182 +295,161 @@ cpp_library(
 )
 
 # [test_name, test_src, test_type]
-ROCKS_TESTS = [['arena_test', 'util/arena_test.cc', 'serial'],
- ['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'],
- ['autovector_test', 'util/autovector_test.cc', 'serial'],
- ['backupable_db_test',
-  'utilities/backupable/backupable_db_test.cc',
-  'parallel'],
- ['blob_db_test', 'utilities/blob_db/blob_db_test.cc', 'serial'],
- ['block_based_filter_block_test',
-  'table/block_based_filter_block_test.cc',
-  'serial'],
- ['block_test', 'table/block_test.cc', 'serial'],
- ['bloom_test', 'util/bloom_test.cc', 'serial'],
- ['c_test', 'db/c_test.c', 'serial'],
+ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
  ['cache_test', 'cache/cache_test.cc', 'serial'],
- ['cassandra_format_test',
-  'utilities/merge_operators/cassandra/cassandra_format_test.cc',
-  'serial'],
- ['cassandra_merge_test',
-  'utilities/merge_operators/cassandra/cassandra_merge_test.cc',
-  'serial'],
- ['cassandra_row_merge_test',
-  'utilities/merge_operators/cassandra/cassandra_row_merge_test.cc',
-  'serial'],
- ['cassandra_serialize_test',
-  'utilities/merge_operators/cassandra/cassandra_serialize_test.cc',
-  'serial'],
- ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
- ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
- ['coding_test', 'util/coding_test.cc', 'serial'],
- ['column_aware_encoding_test',
-  'utilities/column_aware_encoding_test.cc',
-  'serial'],
- ['column_family_test', 'db/column_family_test.cc', 'serial'],
- ['compact_files_test', 'db/compact_files_test.cc', 'serial'],
- ['compact_on_deletion_collector_test',
-  'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc',
-  'serial'],
- ['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'],
- ['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'],
- ['compaction_job_test', 'db/compaction_job_test.cc', 'serial'],
- ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
- ['comparator_db_test', 'db/comparator_db_test.cc', 'serial'],
- ['corruption_test', 'db/corruption_test.cc', 'serial'],
- ['crc32c_test', 'util/crc32c_test.cc', 'serial'],
- ['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'],
- ['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'],
- ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
- ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
- ['db_basic_test', 'db/db_basic_test.cc', 'serial'],
- ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
- ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
- ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
- ['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'],
- ['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'],
- ['db_encryption_test', 'db/db_encryption_test.cc', 'serial'],
- ['db_flush_test', 'db/db_flush_test.cc', 'serial'],
- ['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'],
- ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
- ['db_iter_test', 'db/db_iter_test.cc', 'serial'],
- ['db_iterator_test', 'db/db_iterator_test.cc', 'serial'],
- ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
- ['db_memtable_test', 'db/db_memtable_test.cc', 'serial'],
- ['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'],
- ['db_options_test', 'db/db_options_test.cc', 'serial'],
- ['db_properties_test', 'db/db_properties_test.cc', 'serial'],
- ['db_range_del_test', 'db/db_range_del_test.cc', 'serial'],
- ['db_sst_test', 'db/db_sst_test.cc', 'parallel'],
- ['db_statistics_test', 'db/db_statistics_test.cc', 'serial'],
- ['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'],
- ['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'],
- ['db_test', 'db/db_test.cc', 'parallel'],
- ['db_test2', 'db/db_test2.cc', 'serial'],
- ['db_universal_compaction_test',
-  'db/db_universal_compaction_test.cc',
-  'parallel'],
- ['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
- ['db_write_test', 'db/db_write_test.cc', 'serial'],
- ['dbformat_test', 'db/dbformat_test.cc', 'serial'],
- ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
- ['deletefile_test', 'db/deletefile_test.cc', 'serial'],
- ['document_db_test', 'utilities/document/document_db_test.cc', 'serial'],
- ['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'],
- ['env_basic_test', 'env/env_basic_test.cc', 'serial'],
- ['env_test', 'env/env_test.cc', 'serial'],
- ['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
- ['event_logger_test', 'util/event_logger_test.cc', 'serial'],
- ['external_sst_file_basic_test',
-  'db/external_sst_file_basic_test.cc',
-  'serial'],
- ['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'],
- ['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'],
- ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
- ['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'],
- ['filelock_test', 'util/filelock_test.cc', 'serial'],
- ['filename_test', 'db/filename_test.cc', 'serial'],
- ['flush_job_test', 'db/flush_job_test.cc', 'serial'],
- ['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'],
- ['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'],
- ['hash_table_test',
-  'utilities/persistent_cache/hash_table_test.cc',
-  'serial'],
- ['hash_test', 'util/hash_test.cc', 'serial'],
- ['heap_test', 'util/heap_test.cc', 'serial'],
- ['histogram_test', 'monitoring/histogram_test.cc', 'serial'],
- ['inlineskiplist_test', 'memtable/inlineskiplist_test.cc', 'parallel'],
- ['iostats_context_test', 'monitoring/iostats_context_test.cc', 'serial'],
- ['json_document_test', 'utilities/document/json_document_test.cc', 'serial'],
- ['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'],
- ['listener_test', 'db/listener_test.cc', 'serial'],
- ['log_test', 'db/log_test.cc', 'serial'],
- ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
- ['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'],
- ['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
- ['memtable_list_test', 'db/memtable_list_test.cc', 'serial'],
- ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
- ['merge_test', 'db/merge_test.cc', 'serial'],
- ['merger_test', 'table/merger_test.cc', 'serial'],
- ['mock_env_test', 'env/mock_env_test.cc', 'serial'],
- ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
- ['optimistic_transaction_test',
-  'utilities/transactions/optimistic_transaction_test.cc',
-  'serial'],
- ['option_change_migration_test',
-  'utilities/option_change_migration/option_change_migration_test.cc',
-  'serial'],
  ['options_file_test', 'db/options_file_test.cc', 'serial'],
- ['options_settable_test', 'options/options_settable_test.cc', 'serial'],
- ['options_test', 'options/options_test.cc', 'serial'],
- ['options_util_test', 'utilities/options/options_util_test.cc', 'serial'],
- ['partitioned_filter_block_test',
-  'table/partitioned_filter_block_test.cc',
-  'serial'],
- ['perf_context_test', 'db/perf_context_test.cc', 'serial'],
- ['persistent_cache_test',
-  'utilities/persistent_cache/persistent_cache_test.cc',
-  'parallel'],
- ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'],
- ['prefix_test', 'db/prefix_test.cc', 'serial'],
- ['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'],
- ['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'],
- ['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'],
- ['repair_test', 'db/repair_test.cc', 'serial'],
- ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
- ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
- ['slice_transform_test', 'util/slice_transform_test.cc', 'serial'],
- ['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'],
- ['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'],
- ['statistics_test', 'monitoring/statistics_test.cc', 'serial'],
- ['stringappend_test',
-  'utilities/merge_operators/string_append/stringappend_test.cc',
-  'serial'],
+ ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
+ ['corruption_test', 'db/corruption_test.cc', 'serial'],
+ ['thread_list_test', 'util/thread_list_test.cc', 'serial'],
  ['table_properties_collector_test',
   'db/table_properties_collector_test.cc',
   'serial'],
- ['table_test', 'table/table_test.cc', 'parallel'],
- ['thread_list_test', 'util/thread_list_test.cc', 'serial'],
- ['thread_local_test', 'util/thread_local_test.cc', 'serial'],
- ['timer_queue_test', 'util/timer_queue_test.cc', 'serial'],
- ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
- ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
- ['util_merge_operators_test',
-  'utilities/util_merge_operators_test.cc',
+ ['document_db_test', 'utilities/document/document_db_test.cc', 'serial'],
+ ['event_logger_test', 'util/event_logger_test.cc', 'serial'],
+ ['coding_test', 'util/coding_test.cc', 'serial'],
+ ['statistics_test', 'monitoring/statistics_test.cc', 'serial'],
+ ['options_settable_test', 'options/options_settable_test.cc', 'serial'],
+ ['sst_dump_test', 'tools/sst_dump_test.cc', 'serial'],
+ ['column_aware_encoding_test',
+  'utilities/column_aware_encoding_test.cc',
   'serial'],
- ['version_builder_test', 'db/version_builder_test.cc', 'serial'],
- ['version_edit_test', 'db/version_edit_test.cc', 'serial'],
+ ['db_iterator_test', 'db/db_iterator_test.cc', 'serial'],
+ ['db_sst_test', 'db/db_sst_test.cc', 'parallel'],
+ ['geodb_test', 'utilities/geodb/geodb_test.cc', 'serial'],
+ ['listener_test', 'db/listener_test.cc', 'serial'],
+ ['write_callback_test', 'db/write_callback_test.cc', 'serial'],
  ['version_set_test', 'db/version_set_test.cc', 'serial'],
- ['wal_manager_test', 'db/wal_manager_test.cc', 'serial'],
- ['write_batch_test', 'db/write_batch_test.cc', 'serial'],
+ ['full_filter_block_test', 'table/full_filter_block_test.cc', 'serial'],
+ ['cleanable_test', 'table/cleanable_test.cc', 'serial'],
+ ['checkpoint_test', 'utilities/checkpoint/checkpoint_test.cc', 'serial'],
+ ['compact_files_test', 'db/compact_files_test.cc', 'serial'],
+ ['db_options_test', 'db/db_options_test.cc', 'serial'],
+ ['object_registry_test', 'utilities/object_registry_test.cc', 'serial'],
+ ['auto_roll_logger_test', 'util/auto_roll_logger_test.cc', 'serial'],
+ ['dbformat_test', 'db/dbformat_test.cc', 'serial'],
  ['write_batch_with_index_test',
   'utilities/write_batch_with_index/write_batch_with_index_test.cc',
   'serial'],
- ['write_buffer_manager_test',
-  'memtable/write_buffer_manager_test.cc',
+ ['json_document_test', 'utilities/document/json_document_test.cc', 'serial'],
+ ['file_reader_writer_test', 'util/file_reader_writer_test.cc', 'serial'],
+ ['repair_test', 'db/repair_test.cc', 'serial'],
+ ['persistent_cache_test',
+  'utilities/persistent_cache/persistent_cache_test.cc',
+  'parallel'],
+ ['db_bloom_filter_test', 'db/db_bloom_filter_test.cc', 'serial'],
+ ['external_sst_file_basic_test',
+  'db/external_sst_file_basic_test.cc',
   'serial'],
- ['write_callback_test', 'db/write_callback_test.cc', 'serial'],
- ['write_controller_test', 'db/write_controller_test.cc', 'serial']]
+ ['options_test', 'options/options_test.cc', 'serial'],
+ ['perf_context_test', 'db/perf_context_test.cc', 'serial'],
+ ['db_block_cache_test', 'db/db_block_cache_test.cc', 'serial'],
+ ['heap_test', 'util/heap_test.cc', 'serial'],
+ ['db_test2', 'db/db_test2.cc', 'serial'],
+ ['filelock_test', 'util/filelock_test.cc', 'serial'],
+ ['write_controller_test', 'db/write_controller_test.cc', 'serial'],
+ ['compaction_iterator_test', 'db/compaction_iterator_test.cc', 'serial'],
+ ['spatial_db_test', 'utilities/spatialdb/spatial_db_test.cc', 'serial'],
+ ['c_test', 'db/c_test.c', 'serial'],
+ ['range_del_aggregator_test', 'db/range_del_aggregator_test.cc', 'serial'],
+ ['date_tiered_test', 'utilities/date_tiered/date_tiered_test.cc', 'serial'],
+ ['ldb_cmd_test', 'tools/ldb_cmd_test.cc', 'serial'],
+ ['db_test', 'db/db_test.cc', 'parallel'],
+ ['block_based_filter_block_test',
+  'table/block_based_filter_block_test.cc',
+  'serial'],
+ ['merge_test', 'db/merge_test.cc', 'serial'],
+ ['bloom_test', 'util/bloom_test.cc', 'serial'],
+ ['block_test', 'table/block_test.cc', 'serial'],
+ ['cuckoo_table_builder_test', 'table/cuckoo_table_builder_test.cc', 'serial'],
+ ['backupable_db_test',
+  'utilities/backupable/backupable_db_test.cc',
+  'parallel'],
+ ['db_flush_test', 'db/db_flush_test.cc', 'serial'],
+ ['filename_test', 'db/filename_test.cc', 'serial'],
+ ['cuckoo_table_reader_test', 'table/cuckoo_table_reader_test.cc', 'serial'],
+ ['slice_transform_test', 'util/slice_transform_test.cc', 'serial'],
+ ['cuckoo_table_db_test', 'db/cuckoo_table_db_test.cc', 'serial'],
+ ['inlineskiplist_test', 'memtable/inlineskiplist_test.cc', 'parallel'],
+ ['optimistic_transaction_test',
+  'utilities/transactions/optimistic_transaction_test.cc',
+  'serial'],
+ ['hash_table_test',
+  'utilities/persistent_cache/hash_table_test.cc',
+  'serial'],
+ ['db_dynamic_level_test', 'db/db_dynamic_level_test.cc', 'serial'],
+ ['option_change_migration_test',
+  'utilities/option_change_migration/option_change_migration_test.cc',
+  'serial'],
+ ['db_inplace_update_test', 'db/db_inplace_update_test.cc', 'serial'],
+ ['autovector_test', 'util/autovector_test.cc', 'serial'],
+ ['db_iter_test', 'db/db_iter_test.cc', 'serial'],
+ ['flush_job_test', 'db/flush_job_test.cc', 'serial'],
+ ['wal_manager_test', 'db/wal_manager_test.cc', 'serial'],
+ ['write_batch_test', 'db/write_batch_test.cc', 'serial'],
+ ['crc32c_test', 'util/crc32c_test.cc', 'serial'],
+ ['rate_limiter_test', 'util/rate_limiter_test.cc', 'serial'],
+ ['external_sst_file_test', 'db/external_sst_file_test.cc', 'parallel'],
+ ['compaction_job_test', 'db/compaction_job_test.cc', 'serial'],
+ ['mock_env_test', 'env/mock_env_test.cc', 'serial'],
+ ['db_table_properties_test', 'db/db_table_properties_test.cc', 'serial'],
+ ['db_compaction_test', 'db/db_compaction_test.cc', 'parallel'],
+ ['arena_test', 'util/arena_test.cc', 'serial'],
+ ['stringappend_test',
+  'utilities/merge_operators/string_append/stringappend_test.cc',
+  'serial'],
+ ['reduce_levels_test', 'tools/reduce_levels_test.cc', 'serial'],
+ ['prefix_test', 'db/prefix_test.cc', 'serial'],
+ ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
+ ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
+ ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
+ ['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
+ ['log_test', 'db/log_test.cc', 'serial'],
+ ['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
+ ['deletefile_test', 'db/deletefile_test.cc', 'serial'],
+ ['partitioned_filter_block_test',
+  'table/partitioned_filter_block_test.cc',
+  'serial'],
+ ['comparator_db_test', 'db/comparator_db_test.cc', 'serial'],
+ ['compaction_job_stats_test', 'db/compaction_job_stats_test.cc', 'serial'],
+ ['thread_local_test', 'util/thread_local_test.cc', 'serial'],
+ ['version_builder_test', 'db/version_builder_test.cc', 'serial'],
+ ['db_range_del_test', 'db/db_range_del_test.cc', 'serial'],
+ ['table_test', 'table/table_test.cc', 'parallel'],
+ ['db_tailing_iter_test', 'db/db_tailing_iter_test.cc', 'serial'],
+ ['db_compaction_filter_test', 'db/db_compaction_filter_test.cc', 'parallel'],
+ ['options_util_test', 'utilities/options/options_util_test.cc', 'serial'],
+ ['dynamic_bloom_test', 'util/dynamic_bloom_test.cc', 'serial'],
+ ['db_basic_test', 'db/db_basic_test.cc', 'serial'],
+ ['db_merge_operator_test', 'db/db_merge_operator_test.cc', 'serial'],
+ ['manual_compaction_test', 'db/manual_compaction_test.cc', 'parallel'],
+ ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
+ ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
+ ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
+ ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
+ ['compact_on_deletion_collector_test',
+  'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc',
+  'serial'],
+ ['env_test', 'env/env_test.cc', 'serial'],
+ ['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
+ ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
+ ['db_memtable_test', 'db/db_memtable_test.cc', 'serial'],
+ ['db_universal_compaction_test',
+  'db/db_universal_compaction_test.cc',
+  'parallel'],
+ ['histogram_test', 'monitoring/histogram_test.cc', 'serial'],
+ ['util_merge_operators_test',
+  'utilities/util_merge_operators_test.cc',
+  'serial'],
+ ['fault_injection_test', 'db/fault_injection_test.cc', 'parallel'],
+ ['env_basic_test', 'env/env_basic_test.cc', 'serial'],
+ ['iostats_context_test', 'monitoring/iostats_context_test.cc', 'serial'],
+ ['memtable_list_test', 'db/memtable_list_test.cc', 'serial'],
+ ['column_family_test', 'db/column_family_test.cc', 'serial'],
+ ['db_properties_test', 'db/db_properties_test.cc', 'serial'],
+ ['version_edit_test', 'db/version_edit_test.cc', 'serial'],
+ ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
+ ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
+ ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']]
 
 
 # Generate a test rule for each entry in ROCKS_TESTS
@@ -510,15 +474,3 @@ for test_cfg in ROCKS_TESTS:
       deps = [":" + test_bin],
       command = [TEST_RUNNER, BUCK_BINS + test_bin]
     )
-
-custom_unittest(
-    name = "make_rocksdbjavastatic",
-    type = "simple",
-    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
-)
-
-custom_unittest(
-    name = "make_rocksdb_lite_release",
-    type = "simple",
-    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
-)

USERS.md

@@ -76,7 +76,7 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 [Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine.
 
 ## Uber
-[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue.
+[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and and scalable task queue.
 
 ## 360 Pika
 [360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company

buckifier/buckify_rocksdb.py

@@ -84,7 +84,7 @@ def get_tests(repo_path):
             else:
                 # we consumed all the parallel tests
                 break
-    
+
     return tests
 
 
@@ -122,7 +122,7 @@ def generate_targets(repo_path):
         [":rocksdb_lib"])
 
     # test for every test we found in the Makefile
-    for test in sorted(tests):
+    for test in tests:
         match_src = [src for src in cc_files if ("/%s.c" % test) in src]
         if len(match_src) == 0:
             print(ColorString.warning("Cannot find .cc file for %s" % test))

buckifier/targets_cfg.py

@@ -2,11 +2,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-rocksdb_target_header = """
-import os
-
-TARGETS_PATH = os.path.dirname(__file__)
-REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
+rocksdb_target_header = """REPO_PATH = "internal_repo_rocksdb/repo/"
 BUCK_BINS = "buck-out/gen/" + REPO_PATH
 TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
 rocksdb_compiler_flags = [
@@ -17,10 +13,6 @@ rocksdb_compiler_flags = [
   "-DROCKSDB_LIB_IO_POSIX",
   "-DROCKSDB_FALLOCATE_PRESENT",
   "-DROCKSDB_MALLOC_USABLE_SIZE",
-  "-DROCKSDB_RANGESYNC_PRESENT",
-  "-DROCKSDB_SCHED_GETCPU_PRESENT",
-  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
-  "-DHAVE_SSE42",
   "-DOS_LINUX",
   # Flags to enable libs we include
   "-DSNAPPY",
@@ -43,7 +35,7 @@ rocksdb_external_deps = [
   ('lz4', None, 'lz4'),
   ('zstd', None),
   ('tbb', None),
-  ("numa", None, "numa"),
+  ("numa", "2.0.8", "numa"),
   ("googletest", None, "gtest"),
 ]
 

build_tools/build_detect_platform

@@ -51,13 +51,11 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
     FBCODE_BUILD="true"
     # If we're compiling with TSAN we need pic build
     PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
+    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
       source "$PWD/build_tools/fbcode_config.sh"
     else
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
+      # we need this to build with MySQL. Don't use for other purposes.
+      source "$PWD/build_tools/fbcode_config4.8.1.sh"
     fi
 fi
 
@@ -130,17 +128,10 @@ case "$TARGET_OS" in
         ;;
     SunOS)
         PLATFORM=OS_SOLARIS
-        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64"
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/sunos/sunos_specific.cc
         ;;
-    AIX)
-        PLATFORM=OS_AIX
-        CC=gcc
-        COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc"
-        # PORT_FILES=port/aix/aix_specific.cc
-        ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
@@ -391,29 +382,6 @@ EOF
     if [ "$?" = 0 ]; then
         PROFILING_FLAGS=-pg
     fi
-
-    # Test whether sync_file_range is supported for compatibility with an old glibc
-    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-      #include <fcntl.h>
-      int main() {
-        int fd = open("/dev/null", 0);
-        sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
-      }
-EOF
-    if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
-    fi
-
-    # Test whether sched_getcpu is supported
-    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-      #include <sched.h>
-      int main() {
-        int cpuid = sched_getcpu();
-      }
-EOF
-    if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
-    fi
 fi
 
 # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
@@ -444,8 +412,14 @@ if test "$USE_HDFS"; then
   JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
 fi
 
+if [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
+  # Intel SSE instructions breaks compilation on FreeBSD i386
+  unset USE_SSE
+fi
+
 if test "$USE_SSE"; then
-  COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
+  # if Intel SSE instruction set is supported, set USE_SSE=1
+  COMMON_FLAGS="$COMMON_FLAGS -msse -msse4.2 "
 elif test -z "$PORTABLE"; then
   if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
     # Tune for this POWER processor, treating '+' models as base models
@@ -453,41 +427,11 @@ elif test -z "$PORTABLE"; then
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
   elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
     COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
-  elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
+  else
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
   fi
 fi
 
-$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <nmmintrin.h>
-  int main() {
-    volatile uint32_t x = _mm_crc32_u32(0, 0);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
-fi
-
-# iOS doesn't support thread-local storage, but this check would erroneously
-# succeed because the cross-compiler flags are added by the Makefile, not this
-# script.
-if [ "$PLATFORM" != IOS ]; then
-  $CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #if defined(_MSC_VER) && !defined(__thread)
-  #define __thread __declspec(thread)
-  #endif
-  int main() {
-    static __thread int tls;
-  }
-EOF
-  if [ "$?" = 0 ]; then
-    COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL"
-  fi
-fi
-
 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
 

build_tools/dependencies.sh

@@ -1,18 +1,18 @@
-GCC_BASE=/mnt/gvfs/third-party2/gcc/2928bb3ed95bf64f5b388ee88c30dc74710c3b35/5.x/centos6-native/f4950a1
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/a5fea028cb7ba43498976e1f8054b0b2e790c295/stable/centos6-native/6aaf4de
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/7a9099f6587ee4378c0b1fa32bb8934019d30ca4/5.x/gcc-5-glibc-2.23/339d858
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/3b7c6469854dfc7832a1c3cc5b86919a84e5f865/2.23/gcc-5-glibc-2.23/ca1d1c0
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-5-glibc-2.23/9bc6787
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/d7861abe6f0e27ab98c9303b95a662f0e4cdedb5/1.2.8/gcc-5-glibc-2.23/9bc6787
+GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.9.x/centos6-native/108cf83
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.9.x/gcc-4.9-glibc-2.20/024dbc3
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.20/gcc-4.9-glibc-2.20/500e281
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.9-glibc-2.20/e9936bf
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-5-glibc-2.23/9bc6787
 BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0815d59804160c96caac5f27ca004f51af893dc6/r131/gcc-5-glibc-2.23/9bc6787
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/c15a4f5f619a2930478d01e2e34dc1e0652b0873/1.1.4/gcc-5-glibc-2.23/03859b5
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f905a5e1032fb30c05db3d3752319857388c0c49/2.2.0/gcc-5-glibc-2.23/9bc6787
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/8d60633d822a2a55849c73db24e74a25e52b71db/master/gcc-5-glibc-2.23/1c32b4b
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-5-glibc-2.23/9bc6787
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-5-glibc-2.23/9bc6787
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-5-glibc-2.23/1c32b4b
 NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/8db74270cd6d0212ac92d69e7fc7beefe617d772/trunk/gcc-5-glibc-2.23/b1847cb
-TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-5-glibc-2.23/9bc6787
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/90c9734afc5579c9d1db529fa788d09f97763b85/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/9e829389ef61b92c62de8748c80169aaf25ce1f0/2.26.1/centos6-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.11.0/gcc-5-glibc-2.23/9bc6787
-LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-5-glibc-2.23/65372bd
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-5-glibc-2.23/b1847cb
+TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.9-glibc-2.20/e9936bf
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.10.0/gcc-4.9-glibc-2.20/e9936bf
+LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-4.9-glibc-2.20/690f0d7

build_tools/dependencies_platform007.sh

@@ -1,18 +0,0 @@
-GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
-NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
-TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
-LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832

build_tools/error_filter.py

@@ -1,7 +1,7 @@
-#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-#  This source code is licensed under both the GPLv2 (found in the
-#  COPYING file in the root directory) and Apache 2.0 License
-#  (found in the LICENSE.Apache file in the root directory).
+# Copyright (c) 2016-present, Facebook, Inc.  All rights reserved.
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree. An additional grant
+# of patent rights can be found in the PATENTS file in the same directory.
 
 '''Filter for error messages in test output:
     - Receives merged stdout/stderr from test on stdin

build_tools/fbcode_config.sh

@@ -56,7 +56,7 @@ if test -z $PIC_BUILD; then
 else
   GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
 fi
-CFLAGS+=" -DGFLAGS=gflags"
+CFLAGS+=" -DGFLAGS=google"
 
 # location of jemalloc
 JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
@@ -116,8 +116,8 @@ else
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
   CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux "
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
   CFLAGS+=" -isystem $CLANG_INCLUDE"
@@ -128,14 +128,13 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
 CXXFLAGS+=" $CFLAGS"
 
 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-5-glibc-2.23/lib/ld.so"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-5-glibc-2.23/lib"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.9-glibc-2.20/lib"
 # required by libtbb
 EXEC_LDFLAGS+=" -ldl"
 

build_tools/fbcode_config4.8.1.sh

@@ -93,7 +93,7 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
 CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
 CXXFLAGS+=" $CFLAGS"
 

build_tools/fbcode_config_platform007.sh

@@ -1,157 +0,0 @@
-#!/bin/sh
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform007.sh"
-
-CFLAGS=""
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-
-# snappy
-SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-else
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-fi
-CFLAGS+=" -DSNAPPY"
-
-if test -z $PIC_BUILD; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-  CFLAGS+=" -DZLIB"
-
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-fi
-CFLAGS+=" -DZSTD"
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
-else
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
-fi
-CFLAGS+=" -DGFLAGS=gflags"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
-
-if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
-  # location of libunwind
-  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
-fi
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-if test -z $PIC_BUILD; then
-  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-else
-  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
-fi
-CFLAGS+=" -DTBB"
-
-# use Intel SSE support for checksum calculations
-export USE_SSE=1
-export PORTABLE=1
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-
-  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-
-  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-
-  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-# lua not supported because it's on track for deprecation, I think
-LUA_PATH=
-LUA_LIB=
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/make_new_version.sh

@@ -17,9 +17,7 @@ function title() {
 }
 
 usage="Create new RocksDB version and prepare it for the release process\n"
-usage+="USAGE: ./make_new_version.sh <version> [<remote>]\n"
-usage+="  version: specify a version without '.fb' suffix (e.g. 5.4).\n"
-usage+="  remote: name of the remote to push the branch to (default: origin)."
+usage+="USAGE: ./make_new_version.sh <version>"
 
 # -- Pre-check
 if [[ $# < 1 ]]; then
@@ -29,11 +27,6 @@ fi
 
 ROCKSDB_VERSION=$1
 
-REMOTE="origin"
-if [[ $# > 1 ]]; then
-  REMOTE=$2
-fi
-
 GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
 echo $GIT_BRANCH
 
@@ -48,6 +41,6 @@ $GIT checkout -b $BRANCH
 
 # Setting up the proxy for remote repo access
 title "Pushing new branch to remote repo ..."
-git push $REMOTE --set-upstream $BRANCH
+git push origin --set-upstream $BRANCH
 
 title "Branch $BRANCH is pushed to github;"

build_tools/make_package.sh

@@ -99,18 +99,6 @@ function main() {
 
   make static_lib
   make install INSTALL_PATH=package
-
-  cd package
-
-  LIB_DIR=lib
-  if [[ -z "$ARCH" ]]; then
-      ARCH=$(getconf LONG_BIT)
-  fi
-  if [[ ("$FPM_OUTPUT" = "rpm") && ($ARCH -eq 64) ]]; then
-      mv lib lib64
-      LIB_DIR=lib64
-  fi
-
   fpm \
     -s dir \
     -t $FPM_OUTPUT \
@@ -122,7 +110,7 @@ function main() {
     --license BSD \
     --vendor Facebook \
     --description "RocksDB is an embeddable persistent key-value store for fast storage." \
-    include $LIB_DIR
+    package
 }
 
 main $@

build_tools/rocksdb-lego-determinator

@@ -85,9 +85,8 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
-# using platform007 gives us gcc-8 or higher which has that bug fixed.
-TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
+LITE="OPT=\"-DROCKSDB_LITE -g\""
+TSAN="COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
@@ -131,6 +130,35 @@ DISABLE_COMMANDS="[
     }
 ]"
 
+#
+# RocksDB unit test in parallel
+# Currently we always have noise in our parallel runs. This job is to help
+# manage the noise
+#
+PARALLEL_UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Parallel Unit Test',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version',
+                'shell':'$DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check $TASK_CREATION_TOOL || cat t/log-*',
+                'user':'root',
+                $PARSER
+            },
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version under gcc-4.8.1',
+                'shell':'$GCC_481 $DEBUG make -j$(nproc) all && $SHM make check > /dev/null 2>&1 || $CONTRUN_NAME=punit_check_gcc481 $TASK_CREATION_TOOL || cat t/log-*',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
 #
 # RocksDB unit test
 #
@@ -748,6 +776,9 @@ JAVA_BUILD_TEST_COMMANDS="[
 
 
 case $1 in
+  punit)
+    echo $PARALLEL_UNIT_TEST_COMMANDS
+    ;;
   unit)
     echo $UNIT_TEST_COMMANDS
     ;;

build_tools/run_ci_db_test.ps1

@@ -276,7 +276,7 @@ if($RunAll) {
       continue
     }
 
-    if($RunOnly.Contains($bare_name)) {
+    if($RunOnly.Contains($e)) {
       MakeAndAdd -token $e -HashTable $TestExes
     } else {
       $ListOfSuites += $bare_name
@@ -333,7 +333,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
             $Cases = @{}
 
             if($Suites.Count -gt 0) {
-              # Will the first one
+              # Will teh first one
               ForEach($e in $Suites.Keys) {
                 $exe_name = $e
                 $Cases = $Suites[$e]

build_tools/update_dependencies.sh

@@ -53,46 +53,7 @@ function get_lib_base()
 }
 
 ###########################################################
-#                platform007 dependencies                 #
-###########################################################
-
-OUTPUT="$BASEDIR/dependencies_platform007.sh"
-
-rm -f "$OUTPUT"
-touch "$OUTPUT"
-
-echo "Writing dependencies to $OUTPUT"
-
-# Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
-
-log_variable GCC_BASE
-log_variable CLANG_BASE
-
-# Libraries locations
-get_lib_base libgcc     7.x     platform007
-get_lib_base glibc      2.26    platform007
-get_lib_base snappy     LATEST  platform007
-get_lib_base zlib       LATEST  platform007
-get_lib_base bzip2      LATEST  platform007
-get_lib_base lz4        LATEST  platform007
-get_lib_base zstd       LATEST  platform007
-get_lib_base gflags     LATEST  platform007
-get_lib_base jemalloc   LATEST  platform007
-get_lib_base numa       LATEST  platform007
-get_lib_base libunwind  LATEST  platform007
-get_lib_base tbb        LATEST  platform007
-
-get_lib_base kernel-headers fb platform007
-get_lib_base binutils   LATEST centos7-native
-get_lib_base valgrind   LATEST platform007
-get_lib_base lua        5.3.4  platform007
-
-git diff $OUTPUT
-
-###########################################################
-#                   5.x dependencies                      #
+#                   4.9.x dependencies                    #
 ###########################################################
 
 OUTPUT="$BASEDIR/dependencies.sh"
@@ -103,16 +64,16 @@ touch "$OUTPUT"
 echo "Writing dependencies to $OUTPUT"
 
 # Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos6-native/*/`
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
 CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
 log_variable GCC_BASE
 log_variable CLANG_BASE
 
 # Libraries locations
-get_lib_base libgcc     5.x
-get_lib_base glibc      2.23
-get_lib_base snappy     LATEST gcc-5-glibc-2.23
+get_lib_base libgcc     4.9.x
+get_lib_base glibc      2.20
+get_lib_base snappy     LATEST gcc-4.9-glibc-2.20
 get_lib_base zlib       LATEST
 get_lib_base bzip2      LATEST
 get_lib_base lz4        LATEST
@@ -121,12 +82,12 @@ get_lib_base gflags     LATEST
 get_lib_base jemalloc   LATEST
 get_lib_base numa       LATEST
 get_lib_base libunwind  LATEST
-get_lib_base tbb        4.0_update2 gcc-5-glibc-2.23
+get_lib_base tbb        4.0_update2 gcc-4.9-glibc-2.20
 
 get_lib_base kernel-headers LATEST 
 get_lib_base binutils   LATEST centos6-native 
-get_lib_base valgrind   3.10.0 gcc-5-glibc-2.23
-get_lib_base lua        5.2.3 gcc-5-glibc-2.23
+get_lib_base valgrind   3.10.0 gcc-4.9-glibc-2.20
+get_lib_base lua        5.2.3 gcc-4.9-glibc-2.20
 
 git diff $OUTPUT
 

cache/cache_bench.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #ifndef __STDC_FORMAT_MACROS
 #define __STDC_FORMAT_MACROS

cache/cache_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -475,37 +475,6 @@ void deleter(const Slice& key, void* value) {
 }
 }  // namespace
 
-TEST_P(CacheTest, ReleaseAndErase) {
-  std::shared_ptr<Cache> cache = NewCache(5, 0, false);
-  Cache::Handle* handle;
-  Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1,
-                           &CacheTest::Deleter, &handle);
-  ASSERT_TRUE(s.ok());
-  ASSERT_EQ(5U, cache->GetCapacity());
-  ASSERT_EQ(1U, cache->GetUsage());
-  ASSERT_EQ(0U, deleted_keys_.size());
-  auto erased = cache->Release(handle, true);
-  ASSERT_TRUE(erased);
-  // This tests that deleter has been called
-  ASSERT_EQ(1U, deleted_keys_.size());
-}
-
-TEST_P(CacheTest, ReleaseWithoutErase) {
-  std::shared_ptr<Cache> cache = NewCache(5, 0, false);
-  Cache::Handle* handle;
-  Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1,
-                           &CacheTest::Deleter, &handle);
-  ASSERT_TRUE(s.ok());
-  ASSERT_EQ(5U, cache->GetCapacity());
-  ASSERT_EQ(1U, cache->GetUsage());
-  ASSERT_EQ(0U, deleted_keys_.size());
-  auto erased = cache->Release(handle);
-  ASSERT_FALSE(erased);
-  // This tests that deleter is not called. When cache has free capacity it is
-  // not expected to immediately erase the released items.
-  ASSERT_EQ(0U, deleted_keys_.size());
-}
-
 TEST_P(CacheTest, SetCapacity) {
   // test1: increase capacity
   // lets create a cache with capacity 5,

cache/clock_cache.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -26,6 +26,7 @@ std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
 #include <assert.h>
 #include <atomic>
 #include <deque>
+#include <limits>
 
 #include "tbb/concurrent_hash_map.h"
 
@@ -251,11 +252,8 @@ class ClockCacheShard : public CacheShard {
   //
   // Not necessary to hold mutex_ before being called.
   virtual bool Ref(Cache::Handle* handle) override;
-  virtual bool Release(Cache::Handle* handle,
-                       bool force_erase = false) override;
+  virtual void Release(Cache::Handle* handle) override;
   virtual void Erase(const Slice& key, uint32_t hash) override;
-  bool EraseAndConfirm(const Slice& key, uint32_t hash,
-                       CleanupContext* context);
   virtual size_t GetUsage() const override;
   virtual size_t GetPinnedUsage() const override;
   virtual void EraseUnRefEntries() override;
@@ -276,17 +274,13 @@ class ClockCacheShard : public CacheShard {
   // Decrease reference count of the entry. If this decreases the count to 0,
   // recycle the entry. If set_usage is true, also set the usage bit.
   //
-  // returns true if a value is erased.
-  //
   // Not necessary to hold mutex_ before being called.
-  bool Unref(CacheHandle* handle, bool set_usage, CleanupContext* context);
+  void Unref(CacheHandle* handle, bool set_usage, CleanupContext* context);
 
   // Unset in-cache bit of the entry. Recycle the handle if necessary.
   //
-  // returns true if a value is erased.
-  //
   // Has to hold mutex_ before being called.
-  bool UnsetInCache(CacheHandle* handle, CleanupContext* context);
+  void UnsetInCache(CacheHandle* handle, CleanupContext* context);
 
   // Put the handle back to recycle_ list, and put the value associated with
   // it into to-be-deleted list. It doesn't cleanup the key as it might be
@@ -438,7 +432,7 @@ bool ClockCacheShard::Ref(Cache::Handle* h) {
   return false;
 }
 
-bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
+void ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
                             CleanupContext* context) {
   if (set_usage) {
     handle->flags.fetch_or(kUsageBit, std::memory_order_relaxed);
@@ -457,10 +451,9 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
       RecycleHandle(handle, context);
     }
   }
-  return context->to_delete_value.size();
 }
 
-bool ClockCacheShard::UnsetInCache(CacheHandle* handle,
+void ClockCacheShard::UnsetInCache(CacheHandle* handle,
                                    CleanupContext* context) {
   mutex_.AssertHeld();
   // Use acquire-release semantics as previous operations on the cache entry
@@ -472,7 +465,6 @@ bool ClockCacheShard::UnsetInCache(CacheHandle* handle,
   if (InCache(flags) && CountRefs(flags) == 0) {
     RecycleHandle(handle, context);
   }
-  return context->to_delete_value.size();
 }
 
 bool ClockCacheShard::TryEvict(CacheHandle* handle, CleanupContext* context) {
@@ -626,34 +618,25 @@ Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
   return reinterpret_cast<Cache::Handle*>(handle);
 }
 
-bool ClockCacheShard::Release(Cache::Handle* h, bool force_erase) {
+void ClockCacheShard::Release(Cache::Handle* h) {
   CleanupContext context;
   CacheHandle* handle = reinterpret_cast<CacheHandle*>(h);
-  bool erased = Unref(handle, true, &context);
-  if (force_erase && !erased) {
-    erased = EraseAndConfirm(handle->key, handle->hash, &context);
-  }
+  Unref(handle, true, &context);
   Cleanup(context);
-  return erased;
 }
 
 void ClockCacheShard::Erase(const Slice& key, uint32_t hash) {
   CleanupContext context;
-  EraseAndConfirm(key, hash, &context);
-  Cleanup(context);
-}
-
-bool ClockCacheShard::EraseAndConfirm(const Slice& key, uint32_t hash,
-                                      CleanupContext* context) {
-  MutexLock l(&mutex_);
-  HashTable::accessor accessor;
-  bool erased = false;
-  if (table_.find(accessor, CacheKey(key, hash))) {
-    CacheHandle* handle = accessor->second;
-    table_.erase(accessor);
-    erased = UnsetInCache(handle, context);
+  {
+    MutexLock l(&mutex_);
+    HashTable::accessor accessor;
+    if (table_.find(accessor, CacheKey(key, hash))) {
+      CacheHandle* handle = accessor->second;
+      table_.erase(accessor);
+      UnsetInCache(handle, &context);
+    }
   }
-  return erased;
+  Cleanup(context);
 }
 
 void ClockCacheShard::EraseUnRefEntries() {

cache/clock_cache.h

@@ -1,7 +1,7 @@
 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

cache/lru_cache.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -273,9 +273,9 @@ void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) {
   MaintainPoolSize();
 }
 
-bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
+void LRUCacheShard::Release(Cache::Handle* handle) {
   if (handle == nullptr) {
-    return false;
+    return;
   }
   LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
   bool last_reference = false;
@@ -287,10 +287,10 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
     }
     if (e->refs == 1 && e->InCache()) {
       // The item is still in cache, and nobody else holds a reference to it
-      if (usage_ > capacity_ || force_erase) {
+      if (usage_ > capacity_) {
         // the cache is full
         // The LRU list must be empty since the cache is full
-        assert(!(usage_ > capacity_) || lru_.next == &lru_);
+        assert(lru_.next == &lru_);
         // take this opportunity and remove the item
         table_.Remove(e->key(), e->hash);
         e->SetInCache(false);
@@ -308,7 +308,6 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
   if (last_reference) {
     e->Free();
   }
-  return last_reference;
 }
 
 Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,

cache/lru_cache.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -178,8 +178,7 @@ class LRUCacheShard : public CacheShard {
                         Cache::Priority priority) override;
   virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
   virtual bool Ref(Cache::Handle* handle) override;
-  virtual bool Release(Cache::Handle* handle,
-                       bool force_erase = false) override;
+  virtual void Release(Cache::Handle* handle) override;
   virtual void Erase(const Slice& key, uint32_t hash) override;
 
   // Although in some platforms the update of size_t is atomic, to make sure

cache/lru_cache_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #include "cache/lru_cache.h"
 

cache/sharded_cache.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -63,9 +63,9 @@ bool ShardedCache::Ref(Handle* handle) {
   return GetShard(Shard(hash))->Ref(handle);
 }
 
-bool ShardedCache::Release(Handle* handle, bool force_erase) {
+void ShardedCache::Release(Handle* handle) {
   uint32_t hash = GetHash(handle);
-  return GetShard(Shard(hash))->Release(handle, force_erase);
+  GetShard(Shard(hash))->Release(handle);
 }
 
 void ShardedCache::Erase(const Slice& key) {

cache/sharded_cache.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -30,7 +30,7 @@ class CacheShard {
                         Cache::Handle** handle, Cache::Priority priority) = 0;
   virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0;
   virtual bool Ref(Cache::Handle* handle) = 0;
-  virtual bool Release(Cache::Handle* handle, bool force_erase = false) = 0;
+  virtual void Release(Cache::Handle* handle) = 0;
   virtual void Erase(const Slice& key, uint32_t hash) = 0;
   virtual void SetCapacity(size_t capacity) = 0;
   virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
@@ -65,7 +65,7 @@ class ShardedCache : public Cache {
                         Handle** handle, Priority priority) override;
   virtual Handle* Lookup(const Slice& key, Statistics* stats) override;
   virtual bool Ref(Handle* handle) override;
-  virtual bool Release(Handle* handle, bool force_erase = false) override;
+  virtual void Release(Handle* handle) override;
   virtual void Erase(const Slice& key) override;
   virtual uint64_t NewId() override;
   virtual size_t GetCapacity() const override;

db/builder.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -45,9 +45,9 @@ TableBuilder* NewTableBuilder(
         int_tbl_prop_collector_factories,
     uint32_t column_family_id, const std::string& column_family_name,
     WritableFileWriter* file, const CompressionType compression_type,
-    const CompressionOptions& compression_opts, int level,
-    const std::string* compression_dict, const bool skip_filters,
-    const uint64_t creation_time) {
+    const CompressionOptions& compression_opts,
+    int level,
+    const std::string* compression_dict, const bool skip_filters) {
   assert((column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
          column_family_name.empty());
@@ -55,7 +55,7 @@ TableBuilder* NewTableBuilder(
       TableBuilderOptions(ioptions, internal_comparator,
                           int_tbl_prop_collector_factories, compression_type,
                           compression_opts, compression_dict, skip_filters,
-                          column_family_name, level, creation_time),
+                          column_family_name, level),
       column_family_id, file);
 }
 
@@ -74,8 +74,7 @@ Status BuildTable(
     const CompressionOptions& compression_opts, bool paranoid_file_checks,
     InternalStats* internal_stats, TableFileCreationReason reason,
     EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
-    TableProperties* table_properties, int level,
-    const uint64_t creation_time) {
+    TableProperties* table_properties, int level) {
   assert((column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
          column_family_name.empty());
@@ -124,8 +123,7 @@ Status BuildTable(
       builder = NewTableBuilder(
           ioptions, internal_comparator, int_tbl_prop_collector_factories,
           column_family_id, column_family_name, file_writer.get(), compression,
-          compression_opts, level, nullptr /* compression_dict */,
-          false /* skip_filters */, creation_time);
+          compression_opts, level);
     }
 
     MergeHelper merge(env, internal_comparator.user_comparator(),
@@ -179,7 +177,7 @@ Status BuildTable(
     // Finish and check for file errors
     if (s.ok() && !empty) {
       StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
-      s = file_writer->Sync(ioptions.use_fsync);
+      file_writer->Sync(ioptions.use_fsync);
     }
     if (s.ok() && !empty) {
       s = file_writer->Close();

db/builder.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
@@ -48,9 +48,10 @@ TableBuilder* NewTableBuilder(
         int_tbl_prop_collector_factories,
     uint32_t column_family_id, const std::string& column_family_name,
     WritableFileWriter* file, const CompressionType compression_type,
-    const CompressionOptions& compression_opts, int level,
+    const CompressionOptions& compression_opts,
+    int level,
     const std::string* compression_dict = nullptr,
-    const bool skip_filters = false, const uint64_t creation_time = 0);
+    const bool skip_filters = false);
 
 // Build a Table file from the contents of *iter.  The generated file
 // will be named according to number specified in meta. On success, the rest of
@@ -76,7 +77,6 @@ extern Status BuildTable(
     InternalStats* internal_stats, TableFileCreationReason reason,
     EventLogger* event_logger = nullptr, int job_id = 0,
     const Env::IOPriority io_priority = Env::IO_HIGH,
-    TableProperties* table_properties = nullptr, int level = -1,
-    const uint64_t creation_time = 0);
+    TableProperties* table_properties = nullptr, int level = -1);
 
 }  // namespace rocksdb

db/c.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -34,9 +34,6 @@
 #include "rocksdb/utilities/backupable_db.h"
 #include "rocksdb/utilities/write_batch_with_index.h"
 #include "utilities/merge_operators.h"
-#include "rocksdb/utilities/transaction.h"
-#include "rocksdb/utilities/transaction_db.h"
-#include "rocksdb/utilities/checkpoint.h"
 
 using rocksdb::BytewiseComparator;
 using rocksdb::Cache;
@@ -90,12 +87,6 @@ using rocksdb::RestoreOptions;
 using rocksdb::CompactRangeOptions;
 using rocksdb::RateLimiter;
 using rocksdb::NewGenericRateLimiter;
-using rocksdb::PinnableSlice;
-using rocksdb::TransactionDBOptions;
-using rocksdb::TransactionDB;
-using rocksdb::TransactionOptions;
-using rocksdb::Transaction;
-using rocksdb::Checkpoint;
 
 using std::shared_ptr;
 
@@ -134,24 +125,6 @@ struct rocksdb_envoptions_t      { EnvOptions        rep; };
 struct rocksdb_ingestexternalfileoptions_t  { IngestExternalFileOptions rep; };
 struct rocksdb_sstfilewriter_t   { SstFileWriter*    rep; };
 struct rocksdb_ratelimiter_t     { RateLimiter*      rep; };
-struct rocksdb_pinnableslice_t {
-  PinnableSlice rep;
-};
-struct rocksdb_transactiondb_options_t {
-  TransactionDBOptions rep;
-};
-struct rocksdb_transactiondb_t {
-  TransactionDB* rep;
-};
-struct rocksdb_transaction_options_t {
-  TransactionOptions rep;
-};
-struct rocksdb_transaction_t {
-  Transaction* rep;
-};
-struct rocksdb_checkpoint_t {
-  Checkpoint* rep;
-};
 
 struct rocksdb_compactionfiltercontext_t {
   CompactionFilter::Context rep;
@@ -564,29 +537,6 @@ void rocksdb_backup_engine_close(rocksdb_backup_engine_t* be) {
   delete be;
 }
 
-rocksdb_checkpoint_t* rocksdb_checkpoint_object_create(rocksdb_t* db,
-                                                       char** errptr) {
-  Checkpoint* checkpoint;
-  if (SaveError(errptr, Checkpoint::Create(db->rep, &checkpoint))) {
-    return nullptr;
-  }
-  rocksdb_checkpoint_t* result = new rocksdb_checkpoint_t;
-  result->rep = checkpoint;
-  return result;
-}
-
-void rocksdb_checkpoint_create(rocksdb_checkpoint_t* checkpoint,
-                               const char* checkpoint_dir,
-                               uint64_t log_size_for_flush, char** errptr) {
-  SaveError(errptr, checkpoint->rep->CreateCheckpoint(
-                        std::string(checkpoint_dir), log_size_for_flush));
-}
-
-void rocksdb_checkpoint_object_destroy(rocksdb_checkpoint_t* checkpoint) {
-  delete checkpoint->rep;
-  delete checkpoint;
-}
-
 void rocksdb_close(rocksdb_t* db) {
   delete db->rep;
   delete db;
@@ -1372,24 +1322,23 @@ void rocksdb_writebatch_put_log_data(
   b->rep.PutLogData(Slice(blob, len));
 }
 
-class H : public WriteBatch::Handler {
- public:
-  void* state_;
-  void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
-  void (*deleted_)(void*, const char* k, size_t klen);
-  virtual void Put(const Slice& key, const Slice& value) override {
-    (*put_)(state_, key.data(), key.size(), value.data(), value.size());
-  }
-  virtual void Delete(const Slice& key) override {
-    (*deleted_)(state_, key.data(), key.size());
-  }
-};
-
 void rocksdb_writebatch_iterate(
     rocksdb_writebatch_t* b,
     void* state,
     void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
     void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
   H handler;
   handler.state_ = state;
   handler.put_ = put;
@@ -1411,10 +1360,6 @@ void rocksdb_writebatch_rollback_to_save_point(rocksdb_writebatch_t* b,
   SaveError(errptr, b->rep.RollbackToSavePoint());
 }
 
-void rocksdb_writebatch_pop_save_point(rocksdb_writebatch_t* b, char** errptr) {
-  SaveError(errptr, b->rep.PopSavePoint());
-}
-
 rocksdb_writebatch_wi_t* rocksdb_writebatch_wi_create(size_t reserved_bytes, unsigned char overwrite_key) {
   rocksdb_writebatch_wi_t* b = new rocksdb_writebatch_wi_t;
   b->rep = new WriteBatchWithIndex(BytewiseComparator(), reserved_bytes, overwrite_key);
@@ -1634,6 +1579,18 @@ void rocksdb_writebatch_wi_iterate(
     void* state,
     void (*put)(void*, const char* k, size_t klen, const char* v, size_t vlen),
     void (*deleted)(void*, const char* k, size_t klen)) {
+  class H : public WriteBatch::Handler {
+   public:
+    void* state_;
+    void (*put_)(void*, const char* k, size_t klen, const char* v, size_t vlen);
+    void (*deleted_)(void*, const char* k, size_t klen);
+    virtual void Put(const Slice& key, const Slice& value) override {
+      (*put_)(state_, key.data(), key.size(), value.data(), value.size());
+    }
+    virtual void Delete(const Slice& key) override {
+      (*deleted_)(state_, key.data(), key.size());
+    }
+  };
   H handler;
   handler.state_ = state;
   handler.put_ = put;
@@ -2026,10 +1983,6 @@ void rocksdb_options_set_max_open_files(rocksdb_options_t* opt, int n) {
   opt->rep.max_open_files = n;
 }
 
-void rocksdb_options_set_max_file_opening_threads(rocksdb_options_t* opt, int n) {
-  opt->rep.max_file_opening_threads = n;
-}
-
 void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n) {
   opt->rep.max_total_wal_size = n;
 }
@@ -2839,26 +2792,7 @@ void rocksdb_sstfilewriter_open(rocksdb_sstfilewriter_t* writer,
 void rocksdb_sstfilewriter_add(rocksdb_sstfilewriter_t* writer, const char* key,
                                size_t keylen, const char* val, size_t vallen,
                                char** errptr) {
-  SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(val, vallen)));
-}
-
-void rocksdb_sstfilewriter_put(rocksdb_sstfilewriter_t* writer, const char* key,
-                               size_t keylen, const char* val, size_t vallen,
-                               char** errptr) {
-  SaveError(errptr, writer->rep->Put(Slice(key, keylen), Slice(val, vallen)));
-}
-
-void rocksdb_sstfilewriter_merge(rocksdb_sstfilewriter_t* writer,
-                                 const char* key, size_t keylen,
-                                 const char* val, size_t vallen,
-                                 char** errptr) {
-  SaveError(errptr, writer->rep->Merge(Slice(key, keylen), Slice(val, vallen)));
-}
-
-void rocksdb_sstfilewriter_delete(rocksdb_sstfilewriter_t* writer,
-                                  const char* key, size_t keylen,
-                                  char** errptr) {
-  SaveError(errptr, writer->rep->Delete(Slice(key, keylen)));
+  SaveError(errptr, writer->rep->Add(Slice(key, keylen), Slice(val, vallen)));
 }
 
 void rocksdb_sstfilewriter_finish(rocksdb_sstfilewriter_t* writer,
@@ -2955,21 +2889,20 @@ void rocksdb_slicetransform_destroy(rocksdb_slicetransform_t* st) {
   delete st;
 }
 
-struct Wrapper : public rocksdb_slicetransform_t {
-  const SliceTransform* rep_;
-  ~Wrapper() { delete rep_; }
-  const char* Name() const override { return rep_->Name(); }
-  Slice Transform(const Slice& src) const override {
-    return rep_->Transform(src);
-  }
-  bool InDomain(const Slice& src) const override {
-    return rep_->InDomain(src);
-  }
-  bool InRange(const Slice& src) const override { return rep_->InRange(src); }
-  static void DoNothing(void*) { }
-};
-
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t prefixLen) {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = rocksdb::NewFixedPrefixTransform(prefixLen);
   wrapper->state_ = nullptr;
@@ -2978,6 +2911,19 @@ rocksdb_slicetransform_t* rocksdb_slicetransform_create_fixed_prefix(size_t pref
 }
 
 rocksdb_slicetransform_t* rocksdb_slicetransform_create_noop() {
+  struct Wrapper : public rocksdb_slicetransform_t {
+    const SliceTransform* rep_;
+    ~Wrapper() { delete rep_; }
+    const char* Name() const override { return rep_->Name(); }
+    Slice Transform(const Slice& src) const override {
+      return rep_->Transform(src);
+    }
+    bool InDomain(const Slice& src) const override {
+      return rep_->InDomain(src);
+    }
+    bool InRange(const Slice& src) const override { return rep_->InRange(src); }
+    static void DoNothing(void*) { }
+  };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = rocksdb::NewNoopTransform();
   wrapper->state_ = nullptr;
@@ -3134,270 +3080,8 @@ void rocksdb_delete_file_in_range_cf(
           (limit_key ? (b = Slice(limit_key, limit_key_len), &b) : nullptr)));
 }
 
-rocksdb_transactiondb_options_t* rocksdb_transactiondb_options_create() {
-  return new rocksdb_transactiondb_options_t;
-}
-
-void rocksdb_transactiondb_options_destroy(rocksdb_transactiondb_options_t* opt){
-  delete opt;
-}
-
-void rocksdb_transactiondb_options_set_max_num_locks(
-    rocksdb_transactiondb_options_t* opt, int64_t max_num_locks) {
-  opt->rep.max_num_locks = max_num_locks;
-}
-
-void rocksdb_transactiondb_options_set_num_stripes(
-    rocksdb_transactiondb_options_t* opt, size_t num_stripes) {
-  opt->rep.num_stripes = num_stripes;
-}
-
-void rocksdb_transactiondb_options_set_transaction_lock_timeout(
-    rocksdb_transactiondb_options_t* opt, int64_t txn_lock_timeout) {
-  opt->rep.transaction_lock_timeout = txn_lock_timeout;
-}
-
-void rocksdb_transactiondb_options_set_default_lock_timeout(
-    rocksdb_transactiondb_options_t* opt, int64_t default_lock_timeout) {
-  opt->rep.default_lock_timeout = default_lock_timeout;
-}
-
-rocksdb_transaction_options_t* rocksdb_transaction_options_create() {
-  return new rocksdb_transaction_options_t;
-}
-
-void rocksdb_transaction_options_destroy(rocksdb_transaction_options_t* opt) {
-  delete opt;
-}
-
-void rocksdb_transaction_options_set_set_snapshot(
-    rocksdb_transaction_options_t* opt, unsigned char v) {
-  opt->rep.set_snapshot = v;
-}
-
-void rocksdb_transaction_options_set_deadlock_detect(
-    rocksdb_transaction_options_t* opt, unsigned char v) {
-  opt->rep.deadlock_detect = v;
-}
-
-void rocksdb_transaction_options_set_lock_timeout(
-    rocksdb_transaction_options_t* opt, int64_t lock_timeout) {
-  opt->rep.lock_timeout = lock_timeout;
-}
-
-void rocksdb_transaction_options_set_expiration(
-    rocksdb_transaction_options_t* opt, int64_t expiration) {
-  opt->rep.expiration = expiration;
-}
-
-void rocksdb_transaction_options_set_deadlock_detect_depth(
-    rocksdb_transaction_options_t* opt, int64_t depth) {
-  opt->rep.deadlock_detect_depth = depth;
-}
-
-void rocksdb_transaction_options_set_max_write_batch_size(
-    rocksdb_transaction_options_t* opt, size_t size) {
-  opt->rep.max_write_batch_size = size;
-}
-
-rocksdb_transactiondb_t* rocksdb_transactiondb_open(
-    const rocksdb_options_t* options,
-    const rocksdb_transactiondb_options_t* txn_db_options, const char* name,
-    char** errptr) {
-  TransactionDB* txn_db;
-  if (SaveError(errptr, TransactionDB::Open(options->rep, txn_db_options->rep,
-                                            std::string(name), &txn_db))) {
-    return nullptr;
-  }
-  rocksdb_transactiondb_t* result = new rocksdb_transactiondb_t;
-  result->rep = txn_db;
-  return result;
-}
-
-const rocksdb_snapshot_t* rocksdb_transactiondb_create_snapshot(
-    rocksdb_transactiondb_t* txn_db) {
-  rocksdb_snapshot_t* result = new rocksdb_snapshot_t;
-  result->rep = txn_db->rep->GetSnapshot();
-  return result;
-}
-
-void rocksdb_transactiondb_release_snapshot(
-    rocksdb_transactiondb_t* txn_db, const rocksdb_snapshot_t* snapshot) {
-  txn_db->rep->ReleaseSnapshot(snapshot->rep);
-  delete snapshot;
-}
-
-rocksdb_transaction_t* rocksdb_transaction_begin(
-    rocksdb_transactiondb_t* txn_db,
-    const rocksdb_writeoptions_t* write_options,
-    const rocksdb_transaction_options_t* txn_options,
-    rocksdb_transaction_t* old_txn) {
-  if (old_txn == nullptr) {
-    rocksdb_transaction_t* result = new rocksdb_transaction_t;
-    result->rep = txn_db->rep->BeginTransaction(write_options->rep,
-                                                txn_options->rep, nullptr);
-    return result;
-  }
-  old_txn->rep = txn_db->rep->BeginTransaction(write_options->rep,
-                                                txn_options->rep, old_txn->rep);
-  return old_txn;
-}
-
-void rocksdb_transaction_commit(rocksdb_transaction_t* txn, char** errptr) {
-  SaveError(errptr, txn->rep->Commit());
-}
-
-void rocksdb_transaction_rollback(rocksdb_transaction_t* txn, char** errptr) {
-  SaveError(errptr, txn->rep->Rollback());
-}
-
-void rocksdb_transaction_destroy(rocksdb_transaction_t* txn) {
-  delete txn->rep;
-  delete txn;
-}
-
-//Read a key inside a transaction
-char* rocksdb_transaction_get(rocksdb_transaction_t* txn,
-                              const rocksdb_readoptions_t* options,
-                              const char* key, size_t klen, size_t* vlen,
-                              char** errptr) {
-  char* result = nullptr;
-  std::string tmp;
-  Status s = txn->rep->Get(options->rep, Slice(key, klen), &tmp);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
-// Read a key outside a transaction
-char* rocksdb_transactiondb_get(
-    rocksdb_transactiondb_t* txn_db,
-    const rocksdb_readoptions_t* options,
-    const char* key, size_t klen,
-    size_t* vlen,
-    char** errptr){
-  char* result = nullptr;
-  std::string tmp;
-  Status s = txn_db->rep->Get(options->rep, Slice(key, klen), &tmp);
-  if (s.ok()) {
-    *vlen = tmp.size();
-    result = CopyString(tmp);
-  } else {
-    *vlen = 0;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-  }
-  return result;
-}
-
-// Put a key inside a transaction
-void rocksdb_transaction_put(rocksdb_transaction_t* txn, const char* key,
-                             size_t klen, const char* val, size_t vlen,
-                             char** errptr) {
-  SaveError(errptr, txn->rep->Put(Slice(key, klen), Slice(val, vlen)));
-}
-
-//Put a key outside a transaction
-void rocksdb_transactiondb_put(rocksdb_transactiondb_t* txn_db,
-                               const rocksdb_writeoptions_t* options,
-                               const char* key, size_t klen, const char* val,
-                               size_t vlen, char** errptr) {
-  SaveError(errptr,
-            txn_db->rep->Put(options->rep, Slice(key, klen), Slice(val, vlen)));
-}
-
-// Delete a key inside a transaction
-void rocksdb_transaction_delete(rocksdb_transaction_t* txn, const char* key,
-                                size_t klen, char** errptr) {
-  SaveError(errptr, txn->rep->Delete(Slice(key, klen)));
-}
-
-// Delete a key outside a transaction
-void rocksdb_transactiondb_delete(rocksdb_transactiondb_t* txn_db,
-                                  const rocksdb_writeoptions_t* options,
-                                  const char* key, size_t klen, char** errptr) {
-  SaveError(errptr, txn_db->rep->Delete(options->rep, Slice(key, klen)));
-}
-
-// Create an iterator inside a transaction
-rocksdb_iterator_t* rocksdb_transaction_create_iterator(
-    rocksdb_transaction_t* txn, const rocksdb_readoptions_t* options) {
-  rocksdb_iterator_t* result = new rocksdb_iterator_t;
-  result->rep = txn->rep->GetIterator(options->rep);
-  return result;
-}
-
-void rocksdb_transactiondb_close(rocksdb_transactiondb_t* txn_db) {
-  delete txn_db->rep;
-  delete txn_db;
-}
-
-rocksdb_checkpoint_t* rocksdb_transactiondb_checkpoint_object_create(
-    rocksdb_transactiondb_t* txn_db, char** errptr) {
-  Checkpoint* checkpoint;
-  if (SaveError(errptr, Checkpoint::Create(txn_db->rep, &checkpoint))) {
-    return nullptr;
-  }
-  rocksdb_checkpoint_t* result = new rocksdb_checkpoint_t;
-  result->rep = checkpoint;
-  return result;
-}
-
 void rocksdb_free(void* ptr) { free(ptr); }
 
-rocksdb_pinnableslice_t* rocksdb_get_pinned(
-    rocksdb_t* db, const rocksdb_readoptions_t* options, const char* key,
-    size_t keylen, char** errptr) {
-  rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t);
-  Status s = db->rep->Get(options->rep, db->rep->DefaultColumnFamily(),
-                          Slice(key, keylen), &v->rep);
-  if (!s.ok()) {
-    delete (v);
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-    return NULL;
-  }
-  return v;
-}
-
-rocksdb_pinnableslice_t* rocksdb_get_pinned_cf(
-    rocksdb_t* db, const rocksdb_readoptions_t* options,
-    rocksdb_column_family_handle_t* column_family, const char* key,
-    size_t keylen, char** errptr) {
-  rocksdb_pinnableslice_t* v = new (rocksdb_pinnableslice_t);
-  Status s = db->rep->Get(options->rep, column_family->rep, Slice(key, keylen),
-                          &v->rep);
-  if (!s.ok()) {
-    delete v;
-    if (!s.IsNotFound()) {
-      SaveError(errptr, s);
-    }
-    return NULL;
-  }
-  return v;
-}
-
-void rocksdb_pinnableslice_destroy(rocksdb_pinnableslice_t* v) { delete v; }
-
-const char* rocksdb_pinnableslice_value(const rocksdb_pinnableslice_t* v,
-                                        size_t* vlen) {
-  if (!v) {
-    *vlen = 0;
-    return NULL;
-  }
-
-  *vlen = v->rep.size();
-  return v->rep.data();
-}
 }  // end extern "C"
 
 #endif  // !ROCKSDB_LITE

db/c_test.c

@@ -20,7 +20,7 @@
 // Can not use port/port.h macros as this is a c file
 #ifdef OS_WIN
 
-#include <windows.h>
+#include <Windows.h>
 
 #define snprintf _snprintf
 
@@ -40,7 +40,6 @@ const char* phase = "";
 static char dbname[200];
 static char sstfilename[200];
 static char dbbackupname[200];
-static char dbcheckpointname[200];
 
 static void StartPhase(const char* name) {
   fprintf(stderr, "=== Test %s\n", name);
@@ -127,32 +126,6 @@ static void CheckGetCF(
   Free(&val);
 }
 
-static void CheckPinGet(rocksdb_t* db, const rocksdb_readoptions_t* options,
-                        const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  const char* val;
-  rocksdb_pinnableslice_t* p;
-  p = rocksdb_get_pinned(db, options, key, strlen(key), &err);
-  CheckNoError(err);
-  val = rocksdb_pinnableslice_value(p, &val_len);
-  CheckEqual(expected, val, val_len);
-  rocksdb_pinnableslice_destroy(p);
-}
-
-static void CheckPinGetCF(rocksdb_t* db, const rocksdb_readoptions_t* options,
-                          rocksdb_column_family_handle_t* handle,
-                          const char* key, const char* expected) {
-  char* err = NULL;
-  size_t val_len;
-  const char* val;
-  rocksdb_pinnableslice_t* p;
-  p = rocksdb_get_pinned_cf(db, options, handle, key, strlen(key), &err);
-  CheckNoError(err);
-  val = rocksdb_pinnableslice_value(p, &val_len);
-  CheckEqual(expected, val, val_len);
-  rocksdb_pinnableslice_destroy(p);
-}
 
 static void CheckIter(rocksdb_iterator_t* iter,
                       const char* key, const char* val) {
@@ -319,34 +292,6 @@ static char* MergeOperatorPartialMerge(
   return result;
 }
 
-static void CheckTxnGet(
-        rocksdb_transaction_t* txn,
-        const rocksdb_readoptions_t* options,
-        const char* key,
-        const char* expected) {
-        char* err = NULL;
-        size_t val_len;
-        char* val;
-        val = rocksdb_transaction_get(txn, options, key, strlen(key), &val_len, &err);
-        CheckNoError(err);
-        CheckEqual(expected, val, val_len);
-        Free(&val);
-}
-
-static void CheckTxnDBGet(
-        rocksdb_transactiondb_t* txn_db,
-        const rocksdb_readoptions_t* options,
-        const char* key,
-        const char* expected) {
-        char* err = NULL;
-        size_t val_len;
-        char* val;
-        val = rocksdb_transactiondb_get(txn_db, options, key, strlen(key), &val_len, &err);
-        CheckNoError(err);
-        CheckEqual(expected, val, val_len);
-        Free(&val);
-}
-
 int main(int argc, char** argv) {
   rocksdb_t* db;
   rocksdb_comparator_t* cmp;
@@ -358,10 +303,6 @@ int main(int argc, char** argv) {
   rocksdb_readoptions_t* roptions;
   rocksdb_writeoptions_t* woptions;
   rocksdb_ratelimiter_t* rate_limiter;
-  rocksdb_transactiondb_t* txn_db;
-  rocksdb_transactiondb_options_t* txn_db_options;
-  rocksdb_transaction_t* txn;
-  rocksdb_transaction_options_t* txn_options;
   char* err = NULL;
   int run = -1;
 
@@ -375,11 +316,6 @@ int main(int argc, char** argv) {
            GetTempDir(),
            ((int) geteuid()));
 
-  snprintf(dbcheckpointname, sizeof(dbcheckpointname),
-           "%s/rocksdb_c_test-%d-checkpoint",
-           GetTempDir(),
-           ((int) geteuid()));
-
   snprintf(sstfilename, sizeof(sstfilename),
            "%s/rocksdb_c_test-%d-sst",
            GetTempDir(),
@@ -495,36 +431,6 @@ int main(int argc, char** argv) {
     rocksdb_backup_engine_close(be);
   }
 
-  StartPhase("checkpoint");
-  {
-    rocksdb_destroy_db(options, dbcheckpointname, &err);
-    CheckNoError(err);
-
-    rocksdb_checkpoint_t* checkpoint = rocksdb_checkpoint_object_create(db, &err);
-    CheckNoError(err);
-
-    rocksdb_checkpoint_create(checkpoint, dbcheckpointname, 0, &err);
-    CheckNoError(err);
-
-    // start a new database from the checkpoint
-    rocksdb_close(db);
-    rocksdb_options_set_error_if_exists(options, 0);
-    db = rocksdb_open(options, dbcheckpointname, &err);
-    CheckNoError(err);
-
-    CheckGet(db, roptions, "foo", "hello");
-
-    rocksdb_checkpoint_object_destroy(checkpoint);
-
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbcheckpointname, &err);
-    CheckNoError(err);
-
-    db = rocksdb_open(options, dbname, &err);
-    CheckNoError(err);
-    rocksdb_options_set_error_if_exists(options, 1);
-  }
-
   StartPhase("compactall");
   rocksdb_compact_range(db, NULL, 0, NULL, 0);
   CheckGet(db, roptions, "foo", "hello");
@@ -569,11 +475,11 @@ int main(int argc, char** argv) {
     unlink(sstfilename);
     rocksdb_sstfilewriter_open(writer, sstfilename, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_put(writer, "sstk1", 5, "v1", 2, &err);
+    rocksdb_sstfilewriter_add(writer, "sstk1", 5, "v1", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v2", 2, &err);
+    rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v2", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_put(writer, "sstk3", 5, "v3", 2, &err);
+    rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v3", 2, &err);
     CheckNoError(err);
     rocksdb_sstfilewriter_finish(writer, &err);
     CheckNoError(err);
@@ -590,11 +496,11 @@ int main(int argc, char** argv) {
     unlink(sstfilename);
     rocksdb_sstfilewriter_open(writer, sstfilename, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v4", 2, &err);
+    rocksdb_sstfilewriter_add(writer, "sstk2", 5, "v4", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_put(writer, "sstk22", 6, "v5", 2, &err);
+    rocksdb_sstfilewriter_add(writer, "sstk22", 6, "v5", 2, &err);
     CheckNoError(err);
-    rocksdb_sstfilewriter_put(writer, "sstk3", 5, "v6", 2, &err);
+    rocksdb_sstfilewriter_add(writer, "sstk3", 5, "v6", 2, &err);
     CheckNoError(err);
     rocksdb_sstfilewriter_finish(writer, &err);
     CheckNoError(err);
@@ -681,13 +587,10 @@ int main(int argc, char** argv) {
   {
     rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
     rocksdb_writebatch_set_save_point(wb);
-    rocksdb_writebatch_set_save_point(wb);
     const char* k_list[2] = {"z", "ap"};
     const size_t k_sizes[2] = {1, 2};
     const char* v_list[3] = {"x", "y", "z"};
     const size_t v_sizes[3] = {1, 1, 1};
-    rocksdb_writebatch_pop_save_point(wb, &err);
-    CheckNoError(err);
     rocksdb_writebatch_putv(wb, 2, k_list, k_sizes, 3, v_list, v_sizes);
     rocksdb_writebatch_rollback_to_save_point(wb, &err);
     CheckNoError(err);
@@ -883,13 +786,6 @@ int main(int argc, char** argv) {
     }
   }
 
-  StartPhase("pin_get");
-  {
-    CheckPinGet(db, roptions, "box", "c");
-    CheckPinGet(db, roptions, "foo", "hello");
-    CheckPinGet(db, roptions, "notfound", NULL);
-  }
-
   StartPhase("approximate_sizes");
   {
     int i;
@@ -1101,13 +997,11 @@ int main(int argc, char** argv) {
     CheckNoError(err);
 
     CheckGetCF(db, roptions, handles[1], "foo", "hello");
-    CheckPinGetCF(db, roptions, handles[1], "foo", "hello");
 
     rocksdb_delete_cf(db, woptions, handles[1], "foo", 3, &err);
     CheckNoError(err);
 
     CheckGetCF(db, roptions, handles[1], "foo", NULL);
-    CheckPinGetCF(db, roptions, handles[1], "foo", NULL);
 
     rocksdb_writebatch_t* wb = rocksdb_writebatch_create();
     rocksdb_writebatch_put_cf(wb, handles[1], "baz", 3, "a", 1);
@@ -1120,9 +1014,6 @@ int main(int argc, char** argv) {
     CheckGetCF(db, roptions, handles[1], "baz", NULL);
     CheckGetCF(db, roptions, handles[1], "bar", NULL);
     CheckGetCF(db, roptions, handles[1], "box", "c");
-    CheckPinGetCF(db, roptions, handles[1], "baz", NULL);
-    CheckPinGetCF(db, roptions, handles[1], "bar", NULL);
-    CheckPinGetCF(db, roptions, handles[1], "box", "c");
     rocksdb_writebatch_destroy(wb);
 
     const char* keys[3] = { "box", "box", "barfooxx" };
@@ -1334,98 +1225,14 @@ int main(int argc, char** argv) {
     }
   }
 
-  StartPhase("transactions");
-  {
-    rocksdb_close(db);
-    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
-
-    // open a TransactionDB
-    txn_db_options = rocksdb_transactiondb_options_create();
-    txn_options = rocksdb_transaction_options_create();
-    rocksdb_options_set_create_if_missing(options, 1);
-    txn_db = rocksdb_transactiondb_open(options, txn_db_options, dbname, &err);
-    CheckNoError(err);
-
-    // put outside a transaction
-    rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hello", 5, &err);
-    CheckNoError(err);
-    CheckTxnDBGet(txn_db, roptions, "foo", "hello");
-
-    // delete from outside transaction
-    rocksdb_transactiondb_delete(txn_db, woptions, "foo", 3, &err);
-    CheckNoError(err);
-    CheckTxnDBGet(txn_db, roptions, "foo", NULL);
-
-    // begin a transaction
-    txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, NULL);
-    // put
-    rocksdb_transaction_put(txn, "foo", 3, "hello", 5, &err);
-    CheckNoError(err);
-    CheckTxnGet(txn, roptions, "foo", "hello");
-    // delete
-    rocksdb_transaction_delete(txn, "foo", 3, &err);
-    CheckNoError(err);
-    CheckTxnGet(txn, roptions, "foo", NULL);
-
-    rocksdb_transaction_put(txn, "foo", 3, "hello", 5, &err);
-    CheckNoError(err);
-
-    // read from outside transaction, before commit
-    CheckTxnDBGet(txn_db, roptions, "foo", NULL);
-
-    // commit
-    rocksdb_transaction_commit(txn, &err);
-    CheckNoError(err);
-
-    // read from outside transaction, after commit
-    CheckTxnDBGet(txn_db, roptions, "foo", "hello");
-
-    // reuse old transaction
-    txn = rocksdb_transaction_begin(txn_db, woptions, txn_options, txn);
-
-    // snapshot
-    const rocksdb_snapshot_t* snapshot;
-    snapshot = rocksdb_transactiondb_create_snapshot(txn_db);
-    rocksdb_readoptions_set_snapshot(roptions, snapshot);
-  
-    rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3,  &err);
-    CheckNoError(err);
-
-    CheckTxnDBGet(txn_db, roptions, "foo", "hello");
-    rocksdb_readoptions_set_snapshot(roptions, NULL);
-    rocksdb_transactiondb_release_snapshot(txn_db, snapshot);
-    CheckTxnDBGet(txn_db, roptions, "foo", "hey");
-
-    // iterate
-    rocksdb_transaction_put(txn, "bar", 3, "hi", 2, &err);
-    rocksdb_iterator_t* iter = rocksdb_transaction_create_iterator(txn, roptions);
-    CheckCondition(!rocksdb_iter_valid(iter));
-    rocksdb_iter_seek_to_first(iter);
-    CheckCondition(rocksdb_iter_valid(iter));
-    CheckIter(iter, "bar", "hi");
-    rocksdb_iter_get_error(iter, &err);
-    CheckNoError(err);
-    rocksdb_iter_destroy(iter);
-
-    // rollback
-    rocksdb_transaction_rollback(txn, &err);
-    CheckNoError(err);
-    CheckTxnDBGet(txn_db, roptions, "bar", NULL);
-
-    // close and destroy
-    rocksdb_transaction_destroy(txn);
-    rocksdb_transactiondb_close(txn_db);
-    rocksdb_destroy_db(options, dbname, &err);
-    CheckNoError(err);
-    rocksdb_transaction_options_destroy(txn_options);
-    rocksdb_transactiondb_options_destroy(txn_db_options);
-  }
-
   // Simple sanity check that setting memtable rep works.
   StartPhase("memtable_reps");
   {
     // Create database with vector memtable.
+    rocksdb_close(db);
+    rocksdb_destroy_db(options, dbname, &err);
+    CheckNoError(err);
+
     rocksdb_options_set_memtable_vector_rep(options);
     db = rocksdb_open(options, dbname, &err);
     CheckNoError(err);

db/column_family.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -170,12 +170,6 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
       result.num_levels < 2) {
     result.num_levels = 2;
   }
-
-  if (result.compaction_style == kCompactionStyleUniversal &&
-      db_options.allow_ingest_behind && result.num_levels < 3) {
-    result.num_levels = 3;
-  }
-
   if (result.max_write_buffer_number < 2) {
     result.max_write_buffer_number = 2;
   }
@@ -202,6 +196,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
     result.num_levels = 1;
     // since we delete level0 files in FIFO compaction when there are too many
     // of them, these options don't really mean anything
+    result.level0_file_num_compaction_trigger = std::numeric_limits<int>::max();
     result.level0_slowdown_writes_trigger = std::numeric_limits<int>::max();
     result.level0_stop_writes_trigger = std::numeric_limits<int>::max();
   }
@@ -347,14 +342,13 @@ ColumnFamilyData::ColumnFamilyData(
       dummy_versions_(_dummy_versions),
       current_(nullptr),
       refs_(0),
-      initialized_(false),
       dropped_(false),
       internal_comparator_(cf_options.comparator),
       initial_cf_options_(SanitizeOptions(db_options, cf_options)),
       ioptions_(db_options, initial_cf_options_),
       mutable_cf_options_(initial_cf_options_),
-      is_delete_range_supported_(
-          cf_options.table_factory->IsDeleteRangeSupported()),
+      is_delete_range_supported_(strcmp(cf_options.table_factory->Name(),
+                                        BlockBasedTableFactory().Name()) == 0),
       write_buffer_manager_(write_buffer_manager),
       mem_(nullptr),
       imm_(ioptions_.min_write_buffer_number_to_merge,
@@ -550,7 +544,7 @@ std::unique_ptr<WriteControllerToken> SetupDelay(
     // If DB just falled into the stop condition, we need to further reduce
     // the write rate to avoid the stop condition.
     if (penalize_stop) {
-      // Penalize the near stop or stop condition by more aggressive slowdown.
+      // Penalize the near stop or stop condition by more agressive slowdown.
       // This is to provide the long term slowdown increase signal.
       // The penalty is more than the reward of recovering to the normal
       // condition.
@@ -728,7 +722,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
               mutable_cf_options.level0_slowdown_writes_trigger)) {
         write_controller_token_ =
             write_controller->GetCompactionPressureToken();
-        ROCKS_LOG_INFO(
+        ROCKS_LOG_WARN(
             ioptions_.info_log,
             "[%s] Increasing compaction threads because we have %d level-0 "
             "files ",
@@ -742,7 +736,7 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
         write_controller_token_ =
             write_controller->GetCompactionPressureToken();
         if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
-          ROCKS_LOG_INFO(
+          ROCKS_LOG_WARN(
               ioptions_.info_log,
               "[%s] Increasing compaction threads because of estimated pending "
               "compaction "
@@ -759,12 +753,6 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
         uint64_t write_rate = write_controller->delayed_write_rate();
         write_controller->set_delayed_write_rate(static_cast<uint64_t>(
             static_cast<double>(write_rate) * kDelayRecoverSlowdownRatio));
-        // Set the low pri limit to be 1/4 the delayed write rate.
-        // Note we don't reset this value even after delay condition is relased.
-        // Low-pri rate will continue to apply if there is a compaction
-        // pressure.
-        write_controller->low_pri_rate_limiter()->SetBytesPerSecond(write_rate /
-                                                                    4);
       }
     }
     prev_compaction_needed_bytes_ = compaction_needed_bytes;
@@ -789,8 +777,9 @@ uint64_t ColumnFamilyData::GetTotalSstFilesSize() const {
 
 MemTable* ColumnFamilyData::ConstructNewMemtable(
     const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
+  assert(current_ != nullptr);
   return new MemTable(internal_comparator_, ioptions_, mutable_cf_options,
-                      write_buffer_manager_, earliest_seq, id_);
+                      write_buffer_manager_, earliest_seq);
 }
 
 void ColumnFamilyData::CreateNewMemtable(

db/column_family.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -333,10 +333,6 @@ class ColumnFamilyData {
   void RecalculateWriteStallConditions(
       const MutableCFOptions& mutable_cf_options);
 
-  void set_initialized() { initialized_.store(true); }
-
-  bool initialized() const { return initialized_.load(); }
-
  private:
   friend class ColumnFamilySet;
   ColumnFamilyData(uint32_t id, const std::string& name,
@@ -353,7 +349,6 @@ class ColumnFamilyData {
   Version* current_;         // == dummy_versions->prev_
 
   std::atomic<int> refs_;      // outstanding references to ColumnFamilyData
-  std::atomic<bool> initialized_;
   bool dropped_;               // true if client dropped it
 
   const InternalKeyComparator internal_comparator_;

db/column_family_test.cc

@@ -1,7 +1,8 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in
+//  the LICENSE file in the root directory of this source tree. An
+//  additional grant of patent rights can be found in the PATENTS file
+//  in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -54,7 +55,7 @@ class EnvCounter : public EnvWrapper {
   }
 
  private:
-  std::atomic<int> num_new_writable_file_;
+  int num_new_writable_file_;
 };
 
 class ColumnFamilyTest : public testing::Test {
@@ -302,7 +303,6 @@ class ColumnFamilyTest : public testing::Test {
         ASSERT_OK(Put(cf, key, RandomString(&rnd_, key_value_size - 10)));
       }
     }
-    db_->FlushWAL(false);
   }
 
 #ifndef ROCKSDB_LITE  // TEST functions in DB are not supported in lite
@@ -521,28 +521,6 @@ TEST_F(ColumnFamilyTest, DontReuseColumnFamilyID) {
   }
 }
 
-TEST_F(ColumnFamilyTest, CreateCFRaceWithGetAggProperty) {
-  Open();
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"DBImpl::WriteOptionsFile:1",
-        "ColumnFamilyTest.CreateCFRaceWithGetAggProperty:1"},
-       {"ColumnFamilyTest.CreateCFRaceWithGetAggProperty:2",
-        "DBImpl::WriteOptionsFile:2"}});
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  rocksdb::port::Thread thread([&] { CreateColumnFamilies({"one"}); });
-
-  TEST_SYNC_POINT("ColumnFamilyTest.CreateCFRaceWithGetAggProperty:1");
-  uint64_t pv;
-  db_->GetAggregatedIntProperty(DB::Properties::kEstimateTableReadersMem, &pv);
-  TEST_SYNC_POINT("ColumnFamilyTest.CreateCFRaceWithGetAggProperty:2");
-
-  thread.join();
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-}
-
 class FlushEmptyCFTestWithParam : public ColumnFamilyTest,
                                   public testing::WithParamInterface<bool> {
  public:
@@ -580,7 +558,6 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest) {
   Flush(0);
   ASSERT_OK(Put(1, "bar", "v3"));  // seqID 4
   ASSERT_OK(Put(1, "foo", "v4"));  // seqID 5
-  db_->FlushWAL(false);
 
   // Preserve file system state up to here to simulate a crash condition.
   fault_env->SetFilesystemActive(false);
@@ -643,7 +620,6 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest2) {
   // Write to log file D
   ASSERT_OK(Put(1, "bar", "v4"));  // seqID 7
   ASSERT_OK(Put(1, "bar", "v5"));  // seqID 8
-  db_->FlushWAL(false);
   // Preserve file system state up to here to simulate a crash condition.
   fault_env->SetFilesystemActive(false);
   std::vector<std::string> names;
@@ -1404,6 +1380,7 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
+  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1427,9 +1404,6 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
   two.write_buffer_size = 100000;
 
   Reopen({default_cf, one, two});
-  // make sure all background compaction jobs can be scheduled
-  auto stop_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   bool cf_1_1 = true;
   rocksdb::SyncPoint::GetInstance()->LoadDependency(
@@ -1489,6 +1463,7 @@ TEST_F(ColumnFamilyTest, AutomaticAndManualCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
+  Close();
 }
 
 TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
@@ -1497,6 +1472,7 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
   ColumnFamilyOptions default_cf, one, two;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
+  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1520,9 +1496,6 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
   two.write_buffer_size = 100000;
 
   Reopen({default_cf, one, two});
-  // make sure all background compaction jobs can be scheduled
-  auto stop_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1585,6 +1558,7 @@ TEST_F(ColumnFamilyTest, ManualAndAutomaticCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
+  Close();
 }
 
 TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
@@ -1593,6 +1567,7 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
+  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1611,9 +1586,6 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
-  // make sure all background compaction jobs can be scheduled
-  auto stop_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1684,6 +1656,7 @@ TEST_F(ColumnFamilyTest, SameCFManualManualCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
+  Close();
 }
 
 TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
@@ -1692,6 +1665,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
+  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1710,9 +1684,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
-  // make sure all background compaction jobs can be scheduled
-  auto stop_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1774,6 +1745,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
+  Close();
 }
 
 TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
@@ -1782,6 +1754,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
+  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1800,9 +1773,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
-  // make sure all background compaction jobs can be scheduled
-  auto stop_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- level style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1864,6 +1834,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticCompactionsLevel) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
+  Close();
 }
 
 // This test checks for automatic getting a conflict if there is a
@@ -1879,6 +1850,7 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
+  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -1897,9 +1869,6 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
-  // make sure all background compaction jobs can be scheduled
-  auto stop_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   // SETUP column family "one" -- universal style
   for (int i = 0; i < one.level0_file_num_compaction_trigger - 2; ++i) {
@@ -1984,6 +1953,8 @@ TEST_F(ColumnFamilyTest, SameCFManualAutomaticConflict) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
+
+  Close();
 }
 
 // In this test, we generate enough files to trigger automatic compactions.
@@ -1998,6 +1969,7 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
   ColumnFamilyOptions default_cf, one;
   db_options_.max_open_files = 20;  // only 10 files in file cache
   db_options_.max_background_compactions = 3;
+  db_options_.base_background_compactions = 3;
 
   default_cf.compaction_style = kCompactionStyleLevel;
   default_cf.num_levels = 3;
@@ -2016,9 +1988,6 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
   one.write_buffer_size = 120000;
 
   Reopen({default_cf, one});
-  // make sure all background compaction jobs can be scheduled
-  auto stop_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
 
   bool cf_1_1 = true;
   bool cf_1_2 = true;
@@ -2073,6 +2042,8 @@ TEST_F(ColumnFamilyTest, SameCFAutomaticManualCompactions) {
     ASSERT_NE("NOT_FOUND", Get(1, *key_iter));
     key_iter++;
   }
+
+  Close();
 }
 #endif  // !ROCKSDB_LITE
 
@@ -2525,6 +2496,7 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
 TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
   const uint64_t kBaseRate = 800000u;
   db_options_.delayed_write_rate = kBaseRate;
+  db_options_.base_background_compactions = 2;
   db_options_.max_background_compactions = 6;
 
   Open({"default"});
@@ -2705,6 +2677,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
 }
 
 TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
+  db_options_.base_background_compactions = 2;
   db_options_.max_background_compactions = 6;
   Open({"default"});
   ColumnFamilyData* cfd =
@@ -2724,7 +2697,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(40);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(50);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2736,11 +2709,11 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(45);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(7);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(9);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2748,7 +2721,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->set_l0_delay_trigger_count(6);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 
   // Speed up threshold = min(4 * 2, 4 + (12 - 4)/4) = 6
   mutable_cf_options.level0_file_num_compaction_trigger = 4;
@@ -2757,7 +2730,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->set_l0_delay_trigger_count(5);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(7);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2765,7 +2738,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
 
   vstorage->set_l0_delay_trigger_count(3);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 }
 
 TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
@@ -2844,6 +2817,7 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
 }
 
 TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
+  db_options_.base_background_compactions = 2;
   db_options_.max_background_compactions = 6;
   column_family_options_.soft_pending_compaction_bytes_limit = 200;
   column_family_options_.hard_pending_compaction_bytes_limit = 2000;
@@ -2871,11 +2845,11 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(40);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->TEST_set_estimated_compaction_needed_bytes(60);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
   ASSERT_EQ(6, dbfull()->TEST_BGCompactionsAllowed());
 
@@ -2893,7 +2867,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
 
   vstorage1->TEST_set_estimated_compaction_needed_bytes(3);
   cfd1->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 
   vstorage->set_l0_delay_trigger_count(9);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2905,7 +2879,7 @@ TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
 
   vstorage->set_l0_delay_trigger_count(0);
   cfd->RecalculateWriteStallConditions(mutable_cf_options);
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
 }
 
 #ifndef ROCKSDB_LITE

db/compact_files_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #ifndef ROCKSDB_LITE
 

db/compacted_db_impl.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #ifndef ROCKSDB_LITE
 #include "db/compacted_db_impl.h"

db/compacted_db_impl.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #pragma once
 #ifndef ROCKSDB_LITE

db/compaction.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -283,32 +283,28 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
   assert(input_version_ != nullptr);
   assert(level_ptrs != nullptr);
   assert(level_ptrs->size() == static_cast<size_t>(number_levels_));
-  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
-    if (output_level_ == 0) {
-      return false;
-    }
-    // Maybe use binary search to find right entry instead of linear search?
-    const Comparator* user_cmp = cfd_->user_comparator();
-    for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
-      const std::vector<FileMetaData*>& files =
-          input_vstorage_->LevelFiles(lvl);
-      for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
-        auto* f = files[level_ptrs->at(lvl)];
-        if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
-          // We've advanced far enough
-          if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
-            // Key falls in this file's range, so definitely
-            // exists beyond output level
-            return false;
-          }
-          break;
-        }
-      }
-    }
-    return true;
-  } else {
+  assert(cfd_->ioptions()->compaction_style != kCompactionStyleFIFO);
+  if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
     return bottommost_level_;
   }
+  // Maybe use binary search to find right entry instead of linear search?
+  const Comparator* user_cmp = cfd_->user_comparator();
+  for (int lvl = output_level_ + 1; lvl < number_levels_; lvl++) {
+    const std::vector<FileMetaData*>& files = input_vstorage_->LevelFiles(lvl);
+    for (; level_ptrs->at(lvl) < files.size(); level_ptrs->at(lvl)++) {
+      auto* f = files[level_ptrs->at(lvl)];
+      if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
+        // We've advanced far enough
+        if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
+          // Key falls in this file's range, so definitely
+          // exists beyond output level
+          return false;
+        }
+        break;
+      }
+    }
+  }
+  return true;
 }
 
 // Mark (or clear) each file that is being compacted
@@ -463,17 +459,4 @@ bool Compaction::ShouldFormSubcompactions() const {
   }
 }
 
-uint64_t Compaction::MaxInputFileCreationTime() const {
-  uint64_t max_creation_time = 0;
-  for (const auto& file : inputs_[0].files) {
-    if (file->fd.table_reader != nullptr &&
-        file->fd.table_reader->GetTableProperties() != nullptr) {
-      uint64_t creation_time =
-          file->fd.table_reader->GetTableProperties()->creation_time;
-      max_creation_time = std::max(max_creation_time, creation_time);
-    }
-  }
-  return max_creation_time;
-}
-
 }  // namespace rocksdb

db/compaction.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -99,8 +99,7 @@ class Compaction {
   // input level.
   // REQUIREMENT: "compaction_input_level" must be >= 0 and
   //              < "input_levels()"
-  const std::vector<FileMetaData*>* inputs(
-      size_t compaction_input_level) const {
+  const std::vector<FileMetaData*>* inputs(size_t compaction_input_level) {
     assert(compaction_input_level < inputs_.size());
     return &inputs_[compaction_input_level].files;
   }
@@ -241,8 +240,6 @@ class Compaction {
 
   uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
 
-  uint64_t MaxInputFileCreationTime() const;
-
  private:
   // mark (or clear) all files that are being compacted
   void MarkFilesBeingCompacted(bool mark_as_compacted);

db/compaction_iteration_stats.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2016-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #pragma once
 

db/compaction_iterator.cc

@@ -1,10 +1,11 @@
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #include "db/compaction_iterator.h"
-#include "rocksdb/listener.h"
 #include "table/internal_iterator.h"
 
 namespace rocksdb {
@@ -67,9 +68,7 @@ CompactionIterator::CompactionIterator(
       range_del_agg_(range_del_agg),
       compaction_(std::move(compaction)),
       compaction_filter_(compaction_filter),
-#ifndef ROCKSDB_LITE
       compaction_listener_(compaction_listener),
-#endif  // ROCKSDB_LITE
       shutting_down_(shutting_down),
       ignore_snapshots_(false),
       merge_out_iter_(merge_helper_) {
@@ -83,7 +82,7 @@ CompactionIterator::CompactionIterator(
   if (snapshots_->size() == 0) {
     // optimize for fast path if there are no snapshots
     visible_at_tip_ = true;
-    earliest_snapshot_ = kMaxSequenceNumber;
+    earliest_snapshot_ = last_sequence;
     latest_snapshot_ = 0;
   } else {
     visible_at_tip_ = false;
@@ -546,8 +545,7 @@ void CompactionIterator::PrepareOutput() {
 
   // This is safe for TransactionDB write-conflict checking since transactions
   // only care about sequence number larger than any active snapshots.
-  if ((compaction_ != nullptr && !compaction_->allow_ingest_behind()) &&
-      bottommost_level_ && valid_ && ikey_.sequence <= earliest_snapshot_ &&
+  if (bottommost_level_ && valid_ && ikey_.sequence < earliest_snapshot_ &&
       ikey_.type != kTypeMerge &&
       !cmp_->Equal(compaction_->GetLargestUserKey(), ikey_.user_key)) {
     assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);

db/compaction_iterator.h

@@ -1,7 +1,9 @@
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 #pragma once
 
 #include <algorithm>
@@ -14,7 +16,6 @@
 #include "db/merge_helper.h"
 #include "db/pinned_iterators_manager.h"
 #include "db/range_del_aggregator.h"
-#include "options/cf_options.h"
 #include "rocksdb/compaction_filter.h"
 
 namespace rocksdb {
@@ -45,9 +46,6 @@ class CompactionIterator {
     virtual Slice GetLargestUserKey() const {
       return compaction_->GetLargestUserKey();
     }
-    virtual bool allow_ingest_behind() const {
-      return compaction_->immutable_cf_options()->allow_ingest_behind;
-    }
 
    protected:
     CompactionProxy() = default;
@@ -130,9 +128,7 @@ class CompactionIterator {
   RangeDelAggregator* range_del_agg_;
   std::unique_ptr<CompactionProxy> compaction_;
   const CompactionFilter* compaction_filter_;
-#ifndef ROCKSDB_LITE
   CompactionEventListener* compaction_listener_;
-#endif  // ROCKSDB_LITE
   const std::atomic<bool>* shutting_down_;
   bool bottommost_level_;
   bool valid_ = false;

db/compaction_iterator_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #include "db/compaction_iterator.h"
 
@@ -154,7 +154,6 @@ class FakeCompaction : public CompactionIterator::CompactionProxy {
   virtual Slice GetLargestUserKey() const {
     return "\xff\xff\xff\xff\xff\xff\xff\xff\xff";
   }
-  virtual bool allow_ingest_behind() const { return false; }
 
   bool key_not_exists_beyond_output_level = false;
 };

db/compaction_job.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -1023,6 +1023,7 @@ Status CompactionJob::FinishCompactionOutputFile(
   uint64_t output_number = sub_compact->current_output()->meta.fd.GetNumber();
   assert(output_number != 0);
 
+  TableProperties table_properties;
   // Check for iterator errors
   Status s = input_status;
   auto meta = &sub_compact->current_output()->meta;
@@ -1078,23 +1079,6 @@ Status CompactionJob::FinishCompactionOutputFile(
   }
   sub_compact->outfile.reset();
 
-  if (s.ok() && current_entries == 0) {
-    // If there is nothing to output, no necessary to generate a sst file.
-    // This happens when the output level is bottom level, at the same time
-    // the sub_compact output nothing.
-    std::string fname = TableFileName(
-        db_options_.db_paths, meta->fd.GetNumber(), meta->fd.GetPathId());
-    env_->DeleteFile(fname);
-
-    // Also need to remove the file from outputs, or it will be added to the
-    // VersionEdit.
-    assert(!sub_compact->outputs.empty());
-    sub_compact->outputs.pop_back();
-    sub_compact->builder.reset();
-    sub_compact->current_output_file_size = 0;
-    return s;
-  }
-
   ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
   TableProperties tp;
   if (s.ok() && current_entries > 0) {
@@ -1147,22 +1131,12 @@ Status CompactionJob::FinishCompactionOutputFile(
                             meta->fd.GetPathId());
     sfm->OnAddFile(fn);
     if (sfm->IsMaxAllowedSpaceReached()) {
-      // TODO(ajkr): should we return OK() if max space was reached by the final
-      // compaction output file (similarly to how flush works when full)?
-      s = Status::IOError("Max allowed space was reached");
-      TEST_SYNC_POINT(
-          "CompactionJob::FinishCompactionOutputFile:"
-          "MaxAllowedSpaceReached");
       InstrumentedMutexLock l(db_mutex_);
       if (db_bg_error_->ok()) {
-        Status new_bg_error = s;
-        // may temporarily unlock and lock the mutex.
-        EventHelpers::NotifyOnBackgroundError(
-            cfd->ioptions()->listeners, BackgroundErrorReason::kCompaction,
-            &new_bg_error, db_mutex_);
-        if (!new_bg_error.ok()) {
-          *db_bg_error_ = new_bg_error;
-        }
+        s = Status::IOError("Max allowed space was reached");
+        *db_bg_error_ = s;
+        TEST_SYNC_POINT(
+            "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached");
       }
     }
   }
@@ -1277,22 +1251,14 @@ Status CompactionJob::OpenCompactionOutputFile(
   // data is going to be found
   bool skip_filters =
       cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
-
-  uint64_t output_file_creation_time =
-      sub_compact->compaction->MaxInputFileCreationTime();
-  if (output_file_creation_time == 0) {
-    int64_t _current_time = 0;
-    db_options_.env->GetCurrentTime(&_current_time);  // ignore error
-    output_file_creation_time = static_cast<uint64_t>(_current_time);
-  }
-
   sub_compact->builder.reset(NewTableBuilder(
       *cfd->ioptions(), cfd->internal_comparator(),
       cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
       sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
       cfd->ioptions()->compression_opts,
-      sub_compact->compaction->output_level(), &sub_compact->compression_dict,
-      skip_filters, output_file_creation_time));
+      sub_compact->compaction->output_level(),
+      &sub_compact->compression_dict,
+      skip_filters));
   LogFlush(db_options_.info_log);
   return s;
 }

db/compaction_job.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/compaction_job_stats_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -654,6 +654,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
   Options options;
   options.listeners.emplace_back(stats_checker);
   options.create_if_missing = true;
+  options.max_background_flushes = 0;
   // just enough setting to hold off auto-compaction.
   options.level0_file_num_compaction_trigger = kTestScale + 1;
   options.num_levels = 3;
@@ -875,6 +876,7 @@ TEST_P(CompactionJobStatsTest, DeletionStatsTest) {
   Options options;
   options.listeners.emplace_back(stats_checker);
   options.create_if_missing = true;
+  options.max_background_flushes = 0;
   options.level0_file_num_compaction_trigger = kTestScale+1;
   options.num_levels = 3;
   options.compression = kNoCompression;

db/compaction_job_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #ifndef ROCKSDB_LITE
 
@@ -142,7 +142,6 @@ class CompactionJobTest : public testing::Test {
   }
 
   void SetLastSequence(const SequenceNumber sequence_number) {
-    versions_->SetLastToBeWrittenSequence(sequence_number + 1);
     versions_->SetLastSequence(sequence_number + 1);
   }
 
@@ -338,24 +337,6 @@ TEST_F(CompactionJobTest, SimpleDeletion) {
   RunCompaction({files}, expected_results);
 }
 
-TEST_F(CompactionJobTest, OutputNothing) {
-  NewDB();
-
-  auto file1 = mock::MakeMockFile({{KeyStr("a", 1U, kTypeValue), "val"}});
-
-  AddMockFile(file1);
-
-  auto file2 = mock::MakeMockFile({{KeyStr("a", 2U, kTypeDeletion), ""}});
-
-  AddMockFile(file2);
-
-  auto expected_results = mock::MakeMockFile();
-
-  SetLastSequence(4U);
-  auto files = cfd_->current()->storage_info()->LevelFiles(0);
-  RunCompaction({files}, expected_results);
-}
-
 TEST_F(CompactionJobTest, SimpleOverwrite) {
   NewDB();
 

db/compaction_picker.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -36,39 +36,6 @@ uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files) {
   }
   return sum;
 }
-
-bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
-                           size_t min_files_to_compact,
-                           uint64_t max_compact_bytes_per_del_file,
-                           CompactionInputFiles* comp_inputs) {
-  size_t compact_bytes = level_files[0]->fd.file_size;
-  size_t compact_bytes_per_del_file = port::kMaxSizet;
-  // compaction range will be [0, span_len).
-  size_t span_len;
-  // pull in files until the amount of compaction work per deleted file begins
-  // increasing.
-  size_t new_compact_bytes_per_del_file = 0;
-  for (span_len = 1; span_len < level_files.size(); ++span_len) {
-    compact_bytes += level_files[span_len]->fd.file_size;
-    new_compact_bytes_per_del_file = compact_bytes / span_len;
-    if (level_files[span_len]->being_compacted ||
-        new_compact_bytes_per_del_file > compact_bytes_per_del_file) {
-      break;
-    }
-    compact_bytes_per_del_file = new_compact_bytes_per_del_file;
-  }
-
-  if (span_len >= min_files_to_compact &&
-      new_compact_bytes_per_del_file < max_compact_bytes_per_del_file) {
-    assert(comp_inputs != nullptr);
-    comp_inputs->level = 0;
-    for (size_t i = 0; i < span_len; ++i) {
-      comp_inputs->files.push_back(level_files[i]);
-    }
-    return true;
-  }
-  return false;
-}
 }  // anonymous namespace
 
 // Determine compression type, based on user options, level of the output
@@ -527,11 +494,7 @@ Compaction* CompactionPicker::CompactRange(
     // files together to the last level.
     assert(vstorage->num_levels() > 1);
     // DBImpl::CompactRange() set output level to be the last level
-    if (ioptions_.allow_ingest_behind) {
-      assert(output_level == vstorage->num_levels() - 2);
-    } else {
-      assert(output_level == vstorage->num_levels() - 1);
-    }
+    assert(output_level == vstorage->num_levels() - 1);
     // DBImpl::RunManualCompaction will make full range for universal compaction
     assert(begin == nullptr);
     assert(end == nullptr);
@@ -1383,8 +1346,31 @@ bool LevelCompactionBuilder::PickIntraL0Compaction() {
     // resort to L0->L0 compaction yet.
     return false;
   }
-  return FindIntraL0Compaction(level_files, kMinFilesForIntraL0Compaction,
-                               port::kMaxUint64, &start_level_inputs_);
+
+  size_t compact_bytes = level_files[0]->fd.file_size;
+  size_t compact_bytes_per_del_file = port::kMaxSizet;
+  // compaction range will be [0, span_len).
+  size_t span_len;
+  // pull in files until the amount of compaction work per deleted file begins
+  // increasing.
+  for (span_len = 1; span_len < level_files.size(); ++span_len) {
+    compact_bytes += level_files[span_len]->fd.file_size;
+    size_t new_compact_bytes_per_del_file = compact_bytes / span_len;
+    if (level_files[span_len]->being_compacted ||
+        new_compact_bytes_per_del_file > compact_bytes_per_del_file) {
+      break;
+    }
+    compact_bytes_per_del_file = new_compact_bytes_per_del_file;
+  }
+
+  if (span_len >= kMinFilesForIntraL0Compaction) {
+    start_level_inputs_.level = 0;
+    for (size_t i = 0; i < span_len; ++i) {
+      start_level_inputs_.files.push_back(level_files[i]);
+    }
+    return true;
+  }
+  return false;
 }
 }  // namespace
 
@@ -1403,111 +1389,20 @@ bool FIFOCompactionPicker::NeedsCompaction(
   return vstorage->CompactionScore(kLevel0) >= 1;
 }
 
-namespace {
-uint64_t GetTotalFilesSize(
-    const std::vector<FileMetaData*>& files) {
+Compaction* FIFOCompactionPicker::PickCompaction(
+    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
+    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
+  assert(vstorage->num_levels() == 1);
+  const int kLevel0 = 0;
+  const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
   uint64_t total_size = 0;
-  for (const auto& f : files) {
-    total_size += f->fd.file_size;
+  for (const auto& file : level_files) {
+    total_size += file->fd.file_size;
   }
-  return total_size;
-}
-}  // anonymous namespace
-
-Compaction* FIFOCompactionPicker::PickTTLCompaction(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  assert(ioptions_.compaction_options_fifo.ttl > 0);
-
-  const int kLevel0 = 0;
-  const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
-  uint64_t total_size = GetTotalFilesSize(level_files);
-
-  int64_t _current_time;
-  auto status = ioptions_.env->GetCurrentTime(&_current_time);
-  if (!status.ok()) {
-    ROCKS_LOG_BUFFER(log_buffer,
-                     "[%s] FIFO compaction: Couldn't get current time: %s. "
-                     "Not doing compactions based on TTL. ",
-                     cf_name.c_str(), status.ToString().c_str());
-    return nullptr;
-  }
-  const uint64_t current_time = static_cast<uint64_t>(_current_time);
-
-  std::vector<CompactionInputFiles> inputs;
-  inputs.emplace_back();
-  inputs[0].level = 0;
-
-  for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
-    auto f = *ritr;
-    if (f->fd.table_reader != nullptr &&
-        f->fd.table_reader->GetTableProperties() != nullptr) {
-      auto creation_time =
-          f->fd.table_reader->GetTableProperties()->creation_time;
-      if (creation_time == 0 ||
-          creation_time >=
-              (current_time - ioptions_.compaction_options_fifo.ttl)) {
-        break;
-      }
-      total_size -= f->compensated_file_size;
-      inputs[0].files.push_back(f);
-    }
-  }
-
-  // Return a nullptr and proceed to size-based FIFO compaction if:
-  // 1. there are no files older than ttl OR
-  // 2. there are a few files older than ttl, but deleting them will not bring
-  //    the total size to be less than max_table_files_size threshold.
-  if (inputs[0].files.empty() ||
-      total_size > ioptions_.compaction_options_fifo.max_table_files_size) {
-    return nullptr;
-  }
-
-  for (const auto& f : inputs[0].files) {
-    ROCKS_LOG_BUFFER(log_buffer,
-                     "[%s] FIFO compaction: picking file %" PRIu64
-                     " with creation time %" PRIu64 " for deletion",
-                     cf_name.c_str(), f->fd.GetNumber(),
-                     f->fd.table_reader->GetTableProperties()->creation_time);
-  }
-
-  Compaction* c = new Compaction(
-      vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
-      kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
-      /* is deletion compaction */ true, CompactionReason::kFIFOTtl);
-  return c;
-}
-
-Compaction* FIFOCompactionPicker::PickSizeCompaction(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  const int kLevel0 = 0;
-  const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
-  uint64_t total_size = GetTotalFilesSize(level_files);
 
   if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size ||
       level_files.size() == 0) {
     // total size not exceeded
-    if (ioptions_.compaction_options_fifo.allow_compaction &&
-        level_files.size() > 0) {
-      CompactionInputFiles comp_inputs;
-      if (FindIntraL0Compaction(
-              level_files,
-              mutable_cf_options
-                  .level0_file_num_compaction_trigger /* min_files_to_compact */,
-              mutable_cf_options.write_buffer_size, &comp_inputs)) {
-        Compaction* c = new Compaction(
-            vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0,
-            16 * 1024 * 1024 /* output file size limit */,
-            0 /* max compaction bytes, not applicable */,
-            0 /* output path ID */, mutable_cf_options.compression, {},
-            /* is manual */ false, vstorage->CompactionScore(0),
-            /* is deletion compaction */ false,
-            CompactionReason::kFIFOReduceNumFiles);
-        return c;
-      }
-    }
-
     ROCKS_LOG_BUFFER(log_buffer,
                      "[%s] FIFO compaction: nothing to do. Total size %" PRIu64
                      ", max size %" PRIu64 "\n",
@@ -1528,41 +1423,24 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction(
   std::vector<CompactionInputFiles> inputs;
   inputs.emplace_back();
   inputs[0].level = 0;
-
+  // delete old files (FIFO)
   for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
     auto f = *ritr;
     total_size -= f->compensated_file_size;
     inputs[0].files.push_back(f);
     char tmp_fsize[16];
     AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize));
-    ROCKS_LOG_BUFFER(log_buffer,
-                     "[%s] FIFO compaction: picking file %" PRIu64
-                     " with size %s for deletion",
+    ROCKS_LOG_BUFFER(log_buffer, "[%s] FIFO compaction: picking file %" PRIu64
+                                 " with size %s for deletion",
                      cf_name.c_str(), f->fd.GetNumber(), tmp_fsize);
     if (total_size <= ioptions_.compaction_options_fifo.max_table_files_size) {
       break;
     }
   }
-
   Compaction* c = new Compaction(
       vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
       kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
       /* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
-  return c;
-}
-
-Compaction* FIFOCompactionPicker::PickCompaction(
-    const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
-    VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
-  assert(vstorage->num_levels() == 1);
-
-  Compaction* c = nullptr;
-  if (ioptions_.compaction_options_fifo.ttl > 0) {
-    c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
-  }
-  if (c == nullptr) {
-    c = PickSizeCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
-  }
   RegisterCompaction(c);
   return c;
 }

db/compaction_picker.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -242,17 +242,6 @@ class FIFOCompactionPicker : public CompactionPicker {
 
   virtual bool NeedsCompaction(
       const VersionStorageInfo* vstorage) const override;
-
- private:
-  Compaction* PickTTLCompaction(const std::string& cf_name,
-                                const MutableCFOptions& mutable_cf_options,
-                                VersionStorageInfo* version,
-                                LogBuffer* log_buffer);
-
-  Compaction* PickSizeCompaction(const std::string& cf_name,
-                                 const MutableCFOptions& mutable_cf_options,
-                                 VersionStorageInfo* version,
-                                 LogBuffer* log_buffer);
 };
 
 class NullCompactionPicker : public CompactionPicker {

db/compaction_picker_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #include "db/compaction_picker.h"
 #include <limits>
@@ -400,35 +400,6 @@ TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {
               vstorage_->CompactionScore(0) >= 1);
   }
 }
-
-TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
-  const uint64_t kFileSize = 100000;
-  NewVersionStorage(1, kCompactionStyleUniversal);
-  ioptions_.allow_ingest_behind = true;
-  ioptions_.num_levels = 3;
-  UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
-  // must return false when there's no files.
-  ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
-            false);
-
-  NewVersionStorage(3, kCompactionStyleUniversal);
-
-  Add(0, 1U, "150", "200", kFileSize, 0, 500, 550);
-  Add(0, 2U, "201", "250", kFileSize, 0, 401, 450);
-  Add(0, 4U, "260", "300", kFileSize, 0, 260, 300);
-  Add(1, 5U, "100", "151", kFileSize, 0, 200, 251);
-  Add(1, 3U, "301", "350", kFileSize, 0, 101, 150);
-  Add(2, 6U, "120", "200", kFileSize, 0, 20, 100);
-
-  UpdateVersionStorageInfo();
-
-  std::unique_ptr<Compaction> compaction(
-      universal_compaction_picker.PickCompaction(
-          cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
-
-  // output level should be the one above the bottom-most
-  ASSERT_EQ(1, compaction->output_level());
-}
 // Tests if the files can be trivially moved in multi level
 // universal compaction when allow_trivial_move option is set
 // In this test as the input files overlaps, they cannot
@@ -950,7 +921,7 @@ TEST_F(CompactionPickerTest, NotScheduleL1IfL0WithHigherPri3) {
 TEST_F(CompactionPickerTest, EstimateCompactionBytesNeeded1) {
   int num_levels = ioptions_.num_levels;
   ioptions_.level_compaction_dynamic_level_bytes = false;
-  mutable_cf_options_.level0_file_num_compaction_trigger = 4;
+  mutable_cf_options_.level0_file_num_compaction_trigger = 3;
   mutable_cf_options_.max_bytes_for_level_base = 1000;
   mutable_cf_options_.max_bytes_for_level_multiplier = 10;
   NewVersionStorage(num_levels, kCompactionStyleLevel);
@@ -1040,8 +1011,7 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
 
   // Set Last level size 50000
   // num_levels - 1 target 5000
-  // num_levels - 2 is base level with target 1000 (rounded up to
-  // max_bytes_for_level_base).
+  // num_levels - 2 is base level with taret 500
   Add(num_levels - 1, 10U, "400", "500", 50000);
 
   Add(0, 1U, "150", "200", 200);
@@ -1050,16 +1020,16 @@ TEST_F(CompactionPickerTest, EstimateCompactionBytesNeededDynamicLevel) {
   Add(0, 5U, "150", "200", 200);
   Add(0, 6U, "150", "200", 200);
   // num_levels - 3 is over target by 100 + 1000
-  Add(num_levels - 3, 7U, "400", "500", 550);
-  Add(num_levels - 3, 8U, "600", "700", 550);
+  Add(num_levels - 3, 7U, "400", "500", 300);
+  Add(num_levels - 3, 8U, "600", "700", 300);
   // num_levels - 2 is over target by 1100 + 200
   Add(num_levels - 2, 9U, "150", "200", 5200);
 
   UpdateVersionStorageInfo();
 
-  // Merging to the second last level: (5200 / 2100 + 1) * 1100
+  // Merging to the second last level: (5200 / 1600 + 1) * 1100
   // Merging to the last level: (50000 / 6300 + 1) * 1300
-  ASSERT_EQ(2100u + 3823u + 11617u,
+  ASSERT_EQ(1600u + 4675u + 11617u,
             vstorage_->estimated_compaction_needed_bytes());
 }
 

db/compaction_picker_universal.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -342,6 +342,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
       assert(f->smallest_seqno <= f->largest_seqno);
       if (is_first) {
         is_first = false;
+      } else {
+        assert(prev_smallest_seqno > f->largest_seqno);
       }
       prev_smallest_seqno = f->smallest_seqno;
     }
@@ -566,13 +568,6 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
     output_level = sorted_runs[first_index_after].level - 1;
   }
 
-  // last level is reserved for the files ingested behind
-  if (ioptions_.allow_ingest_behind &&
-      (output_level == vstorage->num_levels() - 1)) {
-    assert(output_level > 1);
-    output_level--;
-  }
-
   std::vector<CompactionInputFiles> inputs(vstorage->num_levels());
   for (size_t i = 0; i < inputs.size(); ++i) {
     inputs[i].level = start_level + static_cast<int>(i);
@@ -619,7 +614,7 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
     const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
     VersionStorageInfo* vstorage, double score,
     const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer) {
-  // percentage flexibility while reducing size amplification
+  // percentage flexibilty while reducing size amplification
   uint64_t ratio =
       ioptions_.compaction_options_universal.max_size_amplification_percent;
 
@@ -724,20 +719,13 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
                      cf_name.c_str(), file_num_buf);
   }
 
-  // output files at the bottom most level, unless it's reserved
-  int output_level = vstorage->num_levels() - 1;
-  // last level is reserved for the files ingested behind
-  if (ioptions_.allow_ingest_behind) {
-    assert(output_level > 1);
-    output_level--;
-  }
-
   return new Compaction(
       vstorage, ioptions_, mutable_cf_options, std::move(inputs),
-      output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
+      vstorage->num_levels() - 1,
+      mutable_cf_options.MaxFileSizeForLevel(vstorage->num_levels() - 1),
       /* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
       GetCompressionType(ioptions_, vstorage, mutable_cf_options,
-                         output_level, 1),
+                         vstorage->num_levels() - 1, 1),
       /* grandparents */ {}, /* is manual */ false, score,
       false /* deletion_compaction */,
       CompactionReason::kUniversalSizeAmplification);

db/compaction_picker_universal.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/comparator_db_test.cc

@@ -1,7 +1,9 @@
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 #include <map>
 #include <string>
 

db/convenience.cc

@@ -1,8 +1,11 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
+// Copyright (c) 2012 Facebook.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
 
 #ifndef ROCKSDB_LITE
 
@@ -13,12 +16,12 @@
 namespace rocksdb {
 
 void CancelAllBackgroundWork(DB* db, bool wait) {
-  (dynamic_cast<DBImpl*>(db->GetRootDB()))->CancelAllBackgroundWork(wait);
+  (dynamic_cast<DBImpl*>(db))->CancelAllBackgroundWork(wait);
 }
 
 Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
                           const Slice* begin, const Slice* end) {
-  return (dynamic_cast<DBImpl*>(db->GetRootDB()))
+  return (dynamic_cast<DBImpl*>(db))
       ->DeleteFilesInRange(column_family, begin, end);
 }
 

db/corruption_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/cuckoo_table_db_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #ifndef ROCKSDB_LITE
 
@@ -320,14 +320,8 @@ TEST_F(CuckooTableDBTest, AdaptiveTable) {
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {
-  if (rocksdb::port::kLittleEndian) {
-    ::testing::InitGoogleTest(&argc, argv);
-    return RUN_ALL_TESTS();
-  }
-  else {
-    fprintf(stderr, "SKIPPED as Cuckoo table doesn't support Big Endian\n");
-    return 0;
-  }
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
 }
 
 #else

db/db_basic_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -364,9 +364,9 @@ TEST_F(DBBasicTest, FLUSH) {
     ASSERT_OK(Flush(1));
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v1"));
 
-    get_perf_context()->Reset();
+    perf_context.Reset();
     Get(1, "foo");
-    ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
+    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
 
     ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
     ASSERT_EQ("v1", Get(1, "foo"));
@@ -379,9 +379,9 @@ TEST_F(DBBasicTest, FLUSH) {
 
     ReopenWithColumnFamilies({"default", "pikachu"}, CurrentOptions());
     ASSERT_EQ("v2", Get(1, "bar"));
-    get_perf_context()->Reset();
+    perf_context.Reset();
     ASSERT_EQ("v2", Get(1, "foo"));
-    ASSERT_TRUE((int)get_perf_context()->get_from_output_files_time > 0);
+    ASSERT_TRUE((int)perf_context.get_from_output_files_time > 0);
 
     writeOpt.disableWAL = false;
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "bar", "v3"));
@@ -822,9 +822,6 @@ TEST_F(DBBasicTest, ChecksumTest) {
 // sense to run
 #ifndef OS_WIN
 TEST_F(DBBasicTest, MmapAndBufferOptions) {
-  if (!IsMemoryMappedAccessSupported()) {
-    return;
-  }
   Options options = CurrentOptions();
 
   options.use_direct_reads = true;

db/db_block_cache_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/db_bloom_filter_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -686,12 +686,12 @@ class BloomStatsTestWithParam
     }
     options_.env = env_;
 
-    get_perf_context()->Reset();
+    perf_context.Reset();
     DestroyAndReopen(options_);
   }
 
   ~BloomStatsTestWithParam() {
-    get_perf_context()->Reset();
+    perf_context.Reset();
     Destroy(options_);
   }
 
@@ -724,33 +724,33 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTest) {
 
   // check memtable bloom stats
   ASSERT_EQ(value1, Get(key1));
-  ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
+  ASSERT_EQ(1, perf_context.bloom_memtable_hit_count);
   ASSERT_EQ(value3, Get(key3));
-  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
-  ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
+  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
+  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
 
   ASSERT_EQ("NOT_FOUND", Get(key2));
-  ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
-  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
+  ASSERT_EQ(1, perf_context.bloom_memtable_miss_count);
+  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
 
   // sanity checks
-  ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
-  ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count);
+  ASSERT_EQ(0, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);
 
   Flush();
 
   // sanity checks
-  ASSERT_EQ(0, get_perf_context()->bloom_sst_hit_count);
-  ASSERT_EQ(0, get_perf_context()->bloom_sst_miss_count);
+  ASSERT_EQ(0, perf_context.bloom_sst_hit_count);
+  ASSERT_EQ(0, perf_context.bloom_sst_miss_count);
 
   // check SST bloom stats
   ASSERT_EQ(value1, Get(key1));
-  ASSERT_EQ(1, get_perf_context()->bloom_sst_hit_count);
+  ASSERT_EQ(1, perf_context.bloom_sst_hit_count);
   ASSERT_EQ(value3, Get(key3));
-  ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
+  ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
 
   ASSERT_EQ("NOT_FOUND", Get(key2));
-  ASSERT_EQ(1, get_perf_context()->bloom_sst_miss_count);
+  ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
 }
 
 // Same scenario as in BloomStatsTest but using an iterator
@@ -771,21 +771,21 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
   ASSERT_OK(iter->status());
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ(value1, iter->value().ToString());
-  ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
-  ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
+  ASSERT_EQ(1, perf_context.bloom_memtable_hit_count);
+  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
 
   iter->Seek(key3);
   ASSERT_OK(iter->status());
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ(value3, iter->value().ToString());
-  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
-  ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
+  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
+  ASSERT_EQ(0, perf_context.bloom_memtable_miss_count);
 
   iter->Seek(key2);
   ASSERT_OK(iter->status());
   ASSERT_TRUE(!iter->Valid());
-  ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
-  ASSERT_EQ(2, get_perf_context()->bloom_memtable_hit_count);
+  ASSERT_EQ(1, perf_context.bloom_memtable_miss_count);
+  ASSERT_EQ(2, perf_context.bloom_memtable_hit_count);
 
   Flush();
 
@@ -796,19 +796,36 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
   ASSERT_OK(iter->status());
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ(value1, iter->value().ToString());
-  ASSERT_EQ(1, get_perf_context()->bloom_sst_hit_count);
+  if (partition_filters_) {
+    ASSERT_EQ(0, perf_context.bloom_sst_hit_count);   // no_io
+    ASSERT_EQ(0, perf_context.bloom_sst_miss_count);  // no_io
+  } else {
+    ASSERT_EQ(1, perf_context.bloom_sst_hit_count);
+  }
 
   iter->Seek(key3);
   ASSERT_OK(iter->status());
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ(value3, iter->value().ToString());
-  ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
+  if (partition_filters_) {
+    ASSERT_EQ(0, perf_context.bloom_sst_hit_count);   // no_io
+    ASSERT_EQ(0, perf_context.bloom_sst_miss_count);  // no_io
+  } else {
+    ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
+  }
 
   iter->Seek(key2);
   ASSERT_OK(iter->status());
-  ASSERT_TRUE(!iter->Valid());
-  ASSERT_EQ(1, get_perf_context()->bloom_sst_miss_count);
-  ASSERT_EQ(2, get_perf_context()->bloom_sst_hit_count);
+  if (partition_filters_) {
+    // iter is still valid since filter did not reject the key2
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(0, perf_context.bloom_sst_hit_count);   // no_io
+    ASSERT_EQ(0, perf_context.bloom_sst_miss_count);  // no_io
+  } else {
+    ASSERT_TRUE(!iter->Valid());
+    ASSERT_EQ(1, perf_context.bloom_sst_miss_count);
+    ASSERT_EQ(2, perf_context.bloom_sst_hit_count);
+  }
 }
 
 INSTANTIATE_TEST_CASE_P(BloomStatsTestWithParam, BloomStatsTestWithParam,

db/db_compaction_filter_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -626,7 +626,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
       iter->Next();
     }
     ASSERT_EQ(total, 700);
-    ASSERT_EQ(count, 1);
+    ASSERT_EQ(count, 2);
   }
 }
 #endif  // ROCKSDB_LITE
@@ -778,7 +778,7 @@ TEST_F(DBTestCompactionFilter, SkipUntil) {
 
   cfilter_skips = 0;
   ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  // Number of skips in tables: 2, 3, 3, 3.
+  // Numberof skips in tables: 2, 3, 3, 3.
   ASSERT_EQ(11, cfilter_skips);
 
   for (int table = 0; table < 4; ++table) {
@@ -799,43 +799,6 @@ TEST_F(DBTestCompactionFilter, SkipUntil) {
   }
 }
 
-TEST_F(DBTestCompactionFilter, SkipUntilWithBloomFilter) {
-  BlockBasedTableOptions table_options;
-  table_options.whole_key_filtering = false;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(100, false));
-
-  Options options = CurrentOptions();
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  options.prefix_extractor.reset(NewCappedPrefixTransform(9));
-  options.compaction_filter_factory = std::make_shared<SkipEvenFilterFactory>();
-  options.disable_auto_compactions = true;
-  options.create_if_missing = true;
-  DestroyAndReopen(options);
-
-  Put("0000000010", "v10");
-  Put("0000000020", "v20"); // skipped
-  Put("0000000050", "v50");
-  Flush();
-
-  cfilter_skips = 0;
-  EXPECT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
-  EXPECT_EQ(1, cfilter_skips);
-
-  Status s;
-  std::string val;
-
-  s = db_->Get(ReadOptions(), "0000000010", &val);
-  ASSERT_OK(s);
-  EXPECT_EQ("v10", val);
-
-  s = db_->Get(ReadOptions(), "0000000020", &val);
-  EXPECT_TRUE(s.IsNotFound());
-
-  s = db_->Get(ReadOptions(), "0000000050", &val);
-  ASSERT_OK(s);
-  EXPECT_EQ("v50", val);
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_compaction_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -519,6 +519,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
   options.level0_file_num_compaction_trigger = 2;
   options.level0_slowdown_writes_trigger = 20;
   options.soft_pending_compaction_bytes_limit = 1 << 30;  // Infinitely large
+  options.base_background_compactions = 1;
   options.max_background_compactions = 3;
   options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
 
@@ -1192,8 +1193,7 @@ TEST_P(DBCompactionTestWithParam, ManualCompactionPartial) {
   }
 }
 
-// Disable as the test is flaky.
-TEST_F(DBCompactionTest, DISABLED_ManualPartialFill) {
+TEST_F(DBCompactionTest, ManualPartialFill) {
   int32_t trivial_move = 0;
   int32_t non_trivial_move = 0;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
@@ -1227,9 +1227,6 @@ TEST_F(DBCompactionTest, DISABLED_ManualPartialFill) {
   options.max_background_compactions = 3;
 
   DestroyAndReopen(options);
-  // make sure all background compaction jobs can be scheduled
-  auto stop_token =
-      dbfull()->TEST_write_controler().GetCompactionPressureToken();
   int32_t value_size = 10 * 1024;  // 10 KB
 
   // Add 2 non-overlapping files
@@ -1917,7 +1914,7 @@ TEST_F(DBCompactionTest, ManualAutoRace) {
 
   // The auto compaction is scheduled but waited until here
   TEST_SYNC_POINT("DBCompactionTest::ManualAutoRace:1");
-  // The auto compaction will wait until the manual compaction is registerd
+  // The auto compaction will wait until the the manual compaction is registerd
   // before processing so that it will be cancelled.
   dbfull()->CompactRange(CompactRangeOptions(), handles_[1], nullptr, nullptr);
   ASSERT_EQ("0,1", FilesPerLevel(1));
@@ -1977,6 +1974,7 @@ TEST_P(DBCompactionTestWithParam, ManualCompaction) {
 
     if (iter == 0) {
       options = CurrentOptions();
+      options.max_background_flushes = 0;
       options.num_levels = 3;
       options.create_if_missing = true;
       options.statistics = rocksdb::CreateDBStatistics();
@@ -2422,12 +2420,15 @@ TEST_F(DBCompactionTest, SanitizeCompactionOptionsTest) {
   options.hard_pending_compaction_bytes_limit = 100;
   options.create_if_missing = true;
   DestroyAndReopen(options);
+  ASSERT_EQ(5, db_->GetOptions().base_background_compactions);
   ASSERT_EQ(100, db_->GetOptions().soft_pending_compaction_bytes_limit);
 
+  options.base_background_compactions = 4;
   options.max_background_compactions = 3;
   options.soft_pending_compaction_bytes_limit = 200;
   options.hard_pending_compaction_bytes_limit = 150;
   DestroyAndReopen(options);
+  ASSERT_EQ(3, db_->GetOptions().base_background_compactions);
   ASSERT_EQ(150, db_->GetOptions().soft_pending_compaction_bytes_limit);
 }
 
@@ -2602,11 +2603,13 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
   // Files 6-9 are the longest span of available files for which
   // work-per-deleted-file decreases (see "score" row above).
   for (int i = 0; i < 10; ++i) {
-    ASSERT_OK(Put(Key(0), ""));  // prevents trivial move
-    if (i == 5) {
-      ASSERT_OK(Put(Key(i + 1), value + value));
-    } else {
-      ASSERT_OK(Put(Key(i + 1), value));
+    for (int j = 0; j < 2; ++j) {
+      ASSERT_OK(Put(Key(0), ""));  // prevents trivial move
+      if (i == 5) {
+        ASSERT_OK(Put(Key(i + 1), value + value));
+      } else {
+        ASSERT_OK(Put(Key(i + 1), value));
+      }
     }
     ASSERT_OK(Flush());
   }
@@ -2621,69 +2624,10 @@ TEST_P(DBCompactionTestWithParam, IntraL0Compaction) {
   ASSERT_EQ(2, level_to_files[0].size());
   ASSERT_GT(level_to_files[1].size(), 0);
   for (int i = 0; i < 2; ++i) {
-    ASSERT_GE(level_to_files[0][i].fd.file_size, 1 << 21);
+    ASSERT_GE(level_to_files[0][0].fd.file_size, 1 << 21);
   }
 }
 
-TEST_P(DBCompactionTestWithParam, IntraL0CompactionDoesNotObsoleteDeletions) {
-  // regression test for issue #2722: L0->L0 compaction can resurrect deleted
-  // keys from older L0 files if L1+ files' key-ranges do not include the key.
-  Options options = CurrentOptions();
-  options.compression = kNoCompression;
-  options.level0_file_num_compaction_trigger = 5;
-  options.max_background_compactions = 2;
-  options.max_subcompactions = max_subcompactions_;
-  DestroyAndReopen(options);
-
-  const size_t kValueSize = 1 << 20;
-  Random rnd(301);
-  std::string value(RandomString(&rnd, kValueSize));
-
-  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"LevelCompactionPicker::PickCompactionBySize:0",
-        "CompactionJob::Run():Start"}});
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  // index:   0   1   2   3   4    5    6   7   8   9
-  // size:  1MB 1MB 1MB 1MB 1MB  1MB  1MB 1MB 1MB 1MB
-  // score:                     1.25 1.33 1.5 2.0 inf
-  //
-  // Files 0-4 will be included in an L0->L1 compaction.
-  //
-  // L0->L0 will be triggered since the sync points guarantee compaction to base
-  // level is still blocked when files 5-9 trigger another compaction. All files
-  // 5-9 are included in the L0->L0 due to work-per-deleted file decreasing.
-  //
-  // Put a key-value in files 0-4. Delete that key in files 5-9. Verify the
-  // L0->L0 preserves the deletion such that the key remains deleted.
-  for (int i = 0; i < 10; ++i) {
-    // key 0 serves both to prevent trivial move and as the key we want to
-    // verify is not resurrected by L0->L0 compaction.
-    if (i < 5) {
-      ASSERT_OK(Put(Key(0), ""));
-    } else {
-      ASSERT_OK(Delete(Key(0)));
-    }
-    ASSERT_OK(Put(Key(i + 1), value));
-    ASSERT_OK(Flush());
-  }
-  dbfull()->TEST_WaitForCompact();
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  std::vector<std::vector<FileMetaData>> level_to_files;
-  dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
-                                  &level_to_files);
-  ASSERT_GE(level_to_files.size(), 2);  // at least L0 and L1
-  // L0 has a single output file from L0->L0
-  ASSERT_EQ(1, level_to_files[0].size());
-  ASSERT_GT(level_to_files[1].size(), 0);
-  ASSERT_GE(level_to_files[0][0].fd.file_size, 1 << 22);
-
-  ReadOptions roptions;
-  std::string result;
-  ASSERT_TRUE(db_->Get(roptions, Key(0), &result).IsNotFound());
-}
-
 INSTANTIATE_TEST_CASE_P(DBCompactionTestWithParam, DBCompactionTestWithParam,
                         ::testing::Values(std::make_tuple(1, true),
                                           std::make_tuple(1, false),

db/db_dynamic_level_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -57,6 +57,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
       Options options;
       options.env = env.get();
       options.create_if_missing = true;
+      options.db_write_buffer_size = 2048;
       options.write_buffer_size = 2048;
       options.max_write_buffer_number = 2;
       options.level0_file_num_compaction_trigger = 2;
@@ -126,6 +127,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
 
   Options options = CurrentOptions();
   options.create_if_missing = true;
+  options.db_write_buffer_size = 204800;
   options.write_buffer_size = 20480;
   options.max_write_buffer_number = 2;
   options.level0_file_num_compaction_trigger = 2;
@@ -282,6 +284,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
 
   Options options = CurrentOptions();
   options.create_if_missing = true;
+  options.db_write_buffer_size = 2048;
   options.write_buffer_size = 2048;
   options.max_write_buffer_number = 2;
   options.level0_file_num_compaction_trigger = 2;
@@ -359,6 +362,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesCompactRange) {
 TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
   Options options = CurrentOptions();
   options.create_if_missing = true;
+  options.db_write_buffer_size = 2048;
   options.write_buffer_size = 2048;
   options.max_write_buffer_number = 2;
   options.level0_file_num_compaction_trigger = 2;
@@ -371,7 +375,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
   options.soft_rate_limit = 1.1;
   options.max_background_compactions = 2;
   options.num_levels = 5;
-  options.max_compaction_bytes = 100000000;
 
   DestroyAndReopen(options);
 
@@ -411,6 +414,7 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
 
   Options options;
   options.create_if_missing = true;
+  options.db_write_buffer_size = 2048;
   options.write_buffer_size = 2048;
   options.max_write_buffer_number = 8;
   options.level0_file_num_compaction_trigger = 4;

db/db_encryption_test.cc

@@ -1,96 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//  This source code is also licensed under the GPLv2 license found in the
-//  COPYING file in the root directory of this source tree.
-//
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "rocksdb/perf_context.h"
-#if !defined(ROCKSDB_LITE)
-#include "util/sync_point.h"
-#endif
-#include <iostream>
-#include <string>
-
-namespace rocksdb {
-
-class DBEncryptionTest : public DBTestBase {
- public:
-  DBEncryptionTest() : DBTestBase("/db_encryption_test") {}
-};
-
-#ifndef ROCKSDB_LITE
-
-TEST_F(DBEncryptionTest, CheckEncrypted) {
-  ASSERT_OK(Put("foo567", "v1.fetdq"));
-  ASSERT_OK(Put("bar123", "v2.dfgkjdfghsd"));
-  Close();
-
-  // Open all files and look for the values we've put in there.
-  // They should not be found if encrypted, otherwise
-  // they should be found.
-  std::vector<std::string> fileNames;
-  auto status = env_->GetChildren(dbname_, &fileNames);
-  ASSERT_OK(status);
-
-  auto defaultEnv = Env::Default();
-  int hits = 0;
-  for (auto it = fileNames.begin() ; it != fileNames.end(); ++it) {
-    if ((*it == "..") || (*it == ".")) {
-      continue;
-    }
-    auto filePath = dbname_ + "/" + *it;
-    unique_ptr<SequentialFile> seqFile;
-    auto envOptions = EnvOptions(CurrentOptions());
-    status = defaultEnv->NewSequentialFile(filePath, &seqFile, envOptions);
-    ASSERT_OK(status);
-
-    uint64_t fileSize;
-    status = defaultEnv->GetFileSize(filePath, &fileSize);
-    ASSERT_OK(status);
-
-    std::string scratch;
-    scratch.reserve(fileSize);
-    Slice data;
-    status = seqFile->Read(fileSize, &data, (char*)scratch.data());
-    ASSERT_OK(status);
-
-    if (data.ToString().find("foo567") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-    if (data.ToString().find("v1.fetdq") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-    if (data.ToString().find("bar123") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-    if (data.ToString().find("v2.dfgkjdfghsd") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-    if (data.ToString().find("dfgk") != std::string::npos) {
-      hits++; 
-      //std::cout << "Hit in " << filePath << "\n";
-    }
-  }
-  if (encrypted_env_) {
-    ASSERT_EQ(hits, 0);
-  } else {
-    ASSERT_GE(hits, 4);
-  }
-}
-
-#endif // ROCKSDB_LITE
-
-}  // namespace rocksdb
-
-int main(int argc, char** argv) {
-  rocksdb::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/db_filesnapshot.cc

@@ -1,8 +1,11 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
+// Copyright (c) 2012 Facebook.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
 
 #ifndef ROCKSDB_LITE
 

db/db_flush_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -89,43 +89,6 @@ TEST_F(DBFlushTest, SyncFail) {
   Destroy(options);
 }
 
-TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
-  // Verify setting an empty high-pri (flush) thread pool causes flushes to be
-  // scheduled in the low-pri (compaction) thread pool.
-  Options options = CurrentOptions();
-  options.level0_file_num_compaction_trigger = 4;
-  options.memtable_factory.reset(new SpecialSkipListFactory(1));
-  Reopen(options);
-  env_->SetBackgroundThreads(0, Env::HIGH);
-
-  std::thread::id tid;
-  int num_flushes = 0, num_compactions = 0;
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BGWorkFlush", [&](void* arg) {
-        if (tid == std::thread::id()) {
-          tid = std::this_thread::get_id();
-        } else {
-          ASSERT_EQ(tid, std::this_thread::get_id());
-        }
-        ++num_flushes;
-      });
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::BGWorkCompaction", [&](void* arg) {
-        ASSERT_EQ(tid, std::this_thread::get_id());
-        ++num_compactions;
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put("key", "val"));
-  for (int i = 0; i < 4; ++i) {
-    ASSERT_OK(Put("key", "val"));
-    dbfull()->TEST_WaitForFlushMemTable();
-  }
-  dbfull()->TEST_WaitForCompact();
-  ASSERT_EQ(4, num_flushes);
-  ASSERT_EQ(1, num_compactions);
-}
-
 TEST_P(DBFlushDirectIOTest, DirectIO) {
   Options options;
   options.create_if_missing = true;

db/db_impl.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -16,6 +16,9 @@
 #ifdef OS_SOLARIS
 #include <alloca.h>
 #endif
+#ifdef ROCKSDB_JEMALLOC
+#include "jemalloc/jemalloc.h"
+#endif
 
 #include <algorithm>
 #include <climits>
@@ -41,7 +44,6 @@
 #include "db/job_context.h"
 #include "db/log_reader.h"
 #include "db/log_writer.h"
-#include "db/malloc_stats.h"
 #include "db/managed_iterator.h"
 #include "db/memtable.h"
 #include "db/memtable_list.h"
@@ -132,8 +134,6 @@ void DumpSupportInfo(Logger* logger) {
   ROCKS_LOG_HEADER(logger, "Fast CRC32 supported: %d",
                    crc32c::IsFastCrc32Supported());
 }
-
-int64_t kDefaultLowPriThrottledRate = 2 * 1024 * 1024;
 } // namespace
 
 DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
@@ -157,14 +157,11 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
       max_total_in_memory_state_(0),
       is_snapshot_supported_(true),
       write_buffer_manager_(immutable_db_options_.write_buffer_manager.get()),
-      write_thread_(immutable_db_options_),
-      nonmem_write_thread_(immutable_db_options_),
+      write_thread_(immutable_db_options_.enable_write_thread_adaptive_yield
+                        ? immutable_db_options_.write_thread_max_yield_usec
+                        : 0,
+                    immutable_db_options_.write_thread_slow_yield_usec),
       write_controller_(mutable_db_options_.delayed_write_rate),
-      // Use delayed_write_rate as a base line to determine the initial
-      // low pri write rate limit. It may be adjusted later.
-      low_pri_write_rate_limiter_(NewGenericRateLimiter(std::min(
-          static_cast<int64_t>(mutable_db_options_.delayed_write_rate / 8),
-          kDefaultLowPriThrottledRate))),
       last_batch_group_size_(0),
       unscheduled_flushes_(0),
       unscheduled_compactions_(0),
@@ -188,16 +185,14 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
       bg_work_paused_(0),
       bg_compaction_paused_(0),
       refitting_level_(false),
-      opened_successfully_(false),
-      concurrent_prepare_(options.concurrent_prepare),
-      manual_wal_flush_(options.manual_wal_flush) {
+      opened_successfully_(false) {
   env_->GetAbsolutePath(dbname, &db_absolute_path_);
 
   // Reserve ten files or so for other uses and give the rest to TableCache.
   // Give a large number for setting of "infinite" open files.
-  const int table_cache_size = (mutable_db_options_.max_open_files == -1)
-                                   ? TableCache::kInfiniteCapacity
-                                   : mutable_db_options_.max_open_files - 10;
+  const int table_cache_size = (immutable_db_options_.max_open_files == -1)
+                                   ? 4194304
+                                   : immutable_db_options_.max_open_files - 10;
   table_cache_ = NewLRUCache(table_cache_size,
                              immutable_db_options_.table_cache_numshardbits);
 
@@ -225,7 +220,7 @@ void DBImpl::CancelAllBackgroundWork(bool wait) {
       has_unpersisted_data_.load(std::memory_order_relaxed) &&
       !mutable_db_options_.avoid_flush_during_shutdown) {
     for (auto cfd : *versions_->GetColumnFamilySet()) {
-      if (!cfd->IsDropped() && cfd->initialized() && !cfd->mem()->IsEmpty()) {
+      if (!cfd->IsDropped() && !cfd->mem()->IsEmpty()) {
         cfd->Ref();
         mutex_.Unlock();
         FlushMemTable(cfd, FlushOptions());
@@ -376,6 +371,39 @@ void DBImpl::PrintStatistics() {
   }
 }
 
+#ifndef ROCKSDB_LITE
+#ifdef ROCKSDB_JEMALLOC
+typedef struct {
+  char* cur;
+  char* end;
+} MallocStatus;
+
+static void GetJemallocStatus(void* mstat_arg, const char* status) {
+  MallocStatus* mstat = reinterpret_cast<MallocStatus*>(mstat_arg);
+  size_t status_len = status ? strlen(status) : 0;
+  size_t buf_size = (size_t)(mstat->end - mstat->cur);
+  if (!status_len || status_len > buf_size) {
+    return;
+  }
+
+  snprintf(mstat->cur, buf_size, "%s", status);
+  mstat->cur += status_len;
+}
+#endif  // ROCKSDB_JEMALLOC
+
+static void DumpMallocStats(std::string* stats) {
+#ifdef ROCKSDB_JEMALLOC
+  MallocStatus mstat;
+  const unsigned int kMallocStatusLen = 1000000;
+  std::unique_ptr<char[]> buf{new char[kMallocStatusLen + 1]};
+  mstat.cur = buf.get();
+  mstat.end = buf.get() + kMallocStatusLen;
+  je_malloc_stats_print(GetJemallocStatus, &mstat, "");
+  stats->append(buf.get());
+#endif  // ROCKSDB_JEMALLOC
+}
+#endif  // !ROCKSDB_LITE
+
 void DBImpl::MaybeDumpStats() {
   mutex_.Lock();
   unsigned int stats_dump_period_sec =
@@ -407,17 +435,12 @@ void DBImpl::MaybeDumpStats() {
       default_cf_internal_stats_->GetStringProperty(
           *db_property_info, DB::Properties::kDBStats, &stats);
       for (auto cfd : *versions_->GetColumnFamilySet()) {
-        if (cfd->initialized()) {
-          cfd->internal_stats()->GetStringProperty(
-              *cf_property_info, DB::Properties::kCFStatsNoFileHistogram,
-              &stats);
-        }
+        cfd->internal_stats()->GetStringProperty(
+            *cf_property_info, DB::Properties::kCFStatsNoFileHistogram, &stats);
       }
       for (auto cfd : *versions_->GetColumnFamilySet()) {
-        if (cfd->initialized()) {
-          cfd->internal_stats()->GetStringProperty(
-              *cf_property_info, DB::Properties::kCFFileHistogram, &stats);
-        }
+        cfd->internal_stats()->GetStringProperty(
+            *cf_property_info, DB::Properties::kCFFileHistogram, &stats);
       }
     }
     ROCKS_LOG_WARN(immutable_db_options_.info_log,
@@ -548,9 +571,6 @@ Status DBImpl::SetDBOptions(
       }
 
       write_controller_.set_max_delayed_write_rate(new_options.delayed_write_rate);
-      table_cache_.get()->SetCapacity(new_options.max_open_files == -1
-                                          ? TableCache::kInfiniteCapacity
-                                          : new_options.max_open_files - 10);
 
       mutable_db_options_ = new_options;
 
@@ -613,26 +633,6 @@ int DBImpl::FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd,
   return minimum_level;
 }
 
-Status DBImpl::FlushWAL(bool sync) {
-  {
-    // We need to lock log_write_mutex_ since logs_ might change concurrently
-    InstrumentedMutexLock wl(&log_write_mutex_);
-    log::Writer* cur_log_writer = logs_.back().writer;
-    auto s = cur_log_writer->WriteBuffer();
-    if (!s.ok()) {
-      ROCKS_LOG_ERROR(immutable_db_options_.info_log, "WAL flush error %s",
-                      s.ToString().c_str());
-    }
-    if (!sync) {
-      ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "FlushWAL sync=false");
-      return s;
-    }
-  }
-  // sync = true
-  ROCKS_LOG_DEBUG(immutable_db_options_.info_log, "FlushWAL sync=true");
-  return SyncWAL();
-}
-
 Status DBImpl::SyncWAL() {
   autovector<log::Writer*, 1> logs_to_sync;
   bool need_log_dir_sync;
@@ -671,7 +671,6 @@ Status DBImpl::SyncWAL() {
     need_log_dir_sync = !log_dir_synced_;
   }
 
-  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:1");
   RecordTick(stats_, WAL_FILE_SYNCED);
   Status status;
   for (log::Writer* log : logs_to_sync) {
@@ -683,7 +682,6 @@ Status DBImpl::SyncWAL() {
   if (status.ok() && need_log_dir_sync) {
     status = directories_.GetWalDir()->Fsync();
   }
-  TEST_SYNC_POINT("DBWALTest::SyncWALNotWaitWrite:2");
 
   TEST_SYNC_POINT("DBImpl::SyncWAL:BeforeMarkLogsSynced:1");
   {
@@ -1236,8 +1234,6 @@ Status DBImpl::CreateColumnFamilyImpl(const ColumnFamilyOptions& cf_options,
         is_snapshot_supported_ = false;
       }
 
-      cfd->set_initialized();
-
       *handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
       ROCKS_LOG_INFO(immutable_db_options_.info_log,
                      "Created column family [%s] (ID %u)",
@@ -1739,9 +1735,7 @@ bool DBImpl::GetIntPropertyInternal(ColumnFamilyData* cfd,
 Status DBImpl::ResetStats() {
   InstrumentedMutexLock l(&mutex_);
   for (auto* cfd : *versions_->GetColumnFamilySet()) {
-    if (cfd->initialized()) {
-      cfd->internal_stats()->Clear();
-    }
+    cfd->internal_stats()->Clear();
   }
   return Status::OK();
 }
@@ -1760,9 +1754,6 @@ bool DBImpl::GetAggregatedIntProperty(const Slice& property,
     InstrumentedMutexLock l(&mutex_);
     uint64_t value;
     for (auto* cfd : *versions_->GetColumnFamilySet()) {
-      if (!cfd->initialized()) {
-        continue;
-      }
       if (GetIntPropertyInternal(cfd, *property_info, true, &value)) {
         sum += value;
       } else {
@@ -2355,9 +2346,6 @@ Status DBImpl::WriteOptionsFile(bool need_mutex_lock,
       BuildDBOptions(immutable_db_options_, mutable_db_options_);
   mutex_.Unlock();
 
-  TEST_SYNC_POINT("DBImpl::WriteOptionsFile:1");
-  TEST_SYNC_POINT("DBImpl::WriteOptionsFile:2");
-
   std::string file_name =
       TempOptionsFileName(GetName(), versions_->NewFileNumber());
   Status s =
@@ -2493,7 +2481,7 @@ void DBImpl::EraseThreadStatusDbInfo() const {
 // A global method that can dump out the build version
 void DumpRocksDBBuildVersion(Logger * log) {
 #if !defined(IOS_CROSS_COMPILE)
-  // if we compile with Xcode, we don't run build_detect_version, so we don't
+  // if we compile with Xcode, we don't run build_detect_vesion, so we don't
   // generate util/build_version.cc
   ROCKS_LOG_HEADER(log, "RocksDB version: %d.%d.%d\n", ROCKSDB_MAJOR,
                    ROCKSDB_MINOR, ROCKSDB_PATCH);
@@ -2621,15 +2609,6 @@ Status DBImpl::IngestExternalFile(
   auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
   auto cfd = cfh->cfd();
 
-  // Ingest should immediately fail if ingest_behind is requested,
-  // but the DB doesn't support it.
-  if (ingestion_options.ingest_behind) {
-    if (!immutable_db_options_.allow_ingest_behind) {
-      return Status::InvalidArgument(
-        "Can't ingest_behind file in DB with allow_ingest_behind=false");
-    }
-  }
-
   ExternalSstFileIngestionJob ingestion_job(env_, versions_.get(), cfd,
                                             immutable_db_options_, env_options_,
                                             &snapshots_, ingestion_options);
@@ -2657,13 +2636,9 @@ Status DBImpl::IngestExternalFile(
     InstrumentedMutexLock l(&mutex_);
     TEST_SYNC_POINT("DBImpl::AddFile:MutexLock");
 
-    // Stop writes to the DB by entering both write threads
+    // Stop writes to the DB
     WriteThread::Writer w;
     write_thread_.EnterUnbatched(&w, &mutex_);
-    WriteThread::Writer nonmem_w;
-    if (concurrent_prepare_) {
-      nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
-    }
 
     num_running_ingest_file_++;
 
@@ -2677,8 +2652,7 @@ Status DBImpl::IngestExternalFile(
     if (status.ok()) {
       bool need_flush = false;
       status = ingestion_job.NeedsFlush(&need_flush);
-      TEST_SYNC_POINT_CALLBACK("DBImpl::IngestExternalFile:NeedFlush",
-                               &need_flush);
+
       if (status.ok() && need_flush) {
         mutex_.Unlock();
         status = FlushMemTable(cfd, FlushOptions(), true /* writes_stopped */);
@@ -2704,9 +2678,6 @@ Status DBImpl::IngestExternalFile(
     }
 
     // Resume writes to the DB
-    if (concurrent_prepare_) {
-      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
-    }
     write_thread_.ExitUnbatched(&w);
 
     // Update stats

db/db_impl.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -197,7 +197,6 @@ class DBImpl : public DB {
   using DB::Flush;
   virtual Status Flush(const FlushOptions& options,
                        ColumnFamilyHandle* column_family) override;
-  virtual Status FlushWAL(bool sync) override;
   virtual Status SyncWAL() override;
 
   virtual SequenceNumber GetLatestSequenceNumber() const override;
@@ -388,9 +387,9 @@ class DBImpl : public DB {
   Status TEST_GetAllImmutableCFOptions(
       std::unordered_map<std::string, const ImmutableCFOptions*>* iopts_map);
 
-  // Return the lastest MutableCFOptions of a column family
+  // Return the lastest MutableCFOptions of of a column family
   Status TEST_GetLatestMutableCFOptions(ColumnFamilyHandle* column_family,
-                                        MutableCFOptions* mutable_cf_options);
+                                        MutableCFOptions* mutable_cf_opitons);
 
   Cache* TEST_table_cache() { return table_cache_.get(); }
 
@@ -400,21 +399,12 @@ class DBImpl : public DB {
   uint64_t TEST_FindMinPrepLogReferencedByMemTable();
 
   int TEST_BGCompactionsAllowed() const;
-  int TEST_BGFlushesAllowed() const;
 
 #endif  // NDEBUG
 
-  struct BGJobLimits {
-    int max_flushes;
-    int max_compactions;
-  };
-  // Returns maximum background flushes and compactions allowed to be scheduled
-  BGJobLimits GetBGJobLimits() const;
-  // Need a static version that can be called during SanitizeOptions().
-  static BGJobLimits GetBGJobLimits(int max_background_flushes,
-                                    int max_background_compactions,
-                                    int max_background_jobs,
-                                    bool parallelize_compactions);
+  // Return maximum background compaction allowed to be scheduled based on
+  // compaction status.
+  int BGCompactionsAllowed() const;
 
   // move logs pending closing from job_context to the DB queue and
   // schedule a purge
@@ -615,15 +605,6 @@ class DBImpl : public DB {
                    uint64_t* log_used = nullptr, uint64_t log_ref = 0,
                    bool disable_memtable = false);
 
-  Status PipelinedWriteImpl(const WriteOptions& options, WriteBatch* updates,
-                            WriteCallback* callback = nullptr,
-                            uint64_t* log_used = nullptr, uint64_t log_ref = 0,
-                            bool disable_memtable = false);
-
-  Status WriteImplWALOnly(const WriteOptions& options, WriteBatch* updates,
-                          WriteCallback* callback = nullptr,
-                          uint64_t* log_used = nullptr, uint64_t log_ref = 0);
-
   uint64_t FindMinLogContainingOutstandingPrep();
   uint64_t FindMinPrepLogReferencedByMemTable();
 
@@ -725,9 +706,6 @@ class DBImpl : public DB {
   //            `num_bytes` going through.
   Status DelayWrite(uint64_t num_bytes, const WriteOptions& write_options);
 
-  Status ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
-                                      WriteBatch* my_batch);
-
   Status ScheduleFlushes(WriteContext* context);
 
   Status SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context);
@@ -746,29 +724,12 @@ class DBImpl : public DB {
   Status HandleWriteBufferFull(WriteContext* write_context);
 
   // REQUIRES: mutex locked
-  Status PreprocessWrite(const WriteOptions& write_options, bool* need_log_sync,
-                         WriteContext* write_context);
+  Status PreprocessWrite(const WriteOptions& write_options, bool need_log_sync,
+                         bool* logs_getting_syned, WriteContext* write_context);
 
-  WriteBatch* MergeBatch(const WriteThread::WriteGroup& write_group,
-                         WriteBatch* tmp_batch, size_t* write_with_wal);
-
-  Status WriteToWAL(const WriteBatch& merged_batch, log::Writer* log_writer,
-                    uint64_t* log_used, uint64_t* log_size);
-
-  Status WriteToWAL(const WriteThread::WriteGroup& write_group,
-                    log::Writer* log_writer, uint64_t* log_used,
-                    bool need_log_sync, bool need_log_dir_sync,
-                    SequenceNumber sequence);
-
-  Status ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
-                              uint64_t* log_used, SequenceNumber* last_sequence,
-                              int total_count);
-
-  // Used by WriteImpl to update bg_error_ if paranoid check is enabled.
-  void WriteCallbackStatusCheck(const Status& status);
-
-  // Used by WriteImpl to update bg_error_ in case of memtable insert error.
-  void MemTableInsertStatusCheck(const Status& memtable_insert_status);
+  Status WriteToWAL(const autovector<WriteThread::Writer*>& write_group,
+                    log::Writer* log_writer, bool need_log_sync,
+                    bool need_log_dir_sync, SequenceNumber sequence);
 
 #ifndef ROCKSDB_LITE
 
@@ -841,16 +802,11 @@ class DBImpl : public DB {
   // Lock over the persistent DB state.  Non-nullptr iff successfully acquired.
   FileLock* db_lock_;
 
-  // In addition to mutex_, log_write_mutex_ protected writes to logs_ and
-  // logfile_number_. With concurrent_prepare it also protects alive_log_files_,
-  // and log_empty_. Refer to the definition of each variable below for more
-  // details.
-  InstrumentedMutex log_write_mutex_;
+  // The mutex for options file related operations.
+  // NOTE: should never acquire options_file_mutex_ and mutex_ at the
+  //       same time.
+  InstrumentedMutex options_files_mutex_;
   // State below is protected by mutex_
-  // With concurrent_prepare enabled, some of the variables that accessed during
-  // WriteToWAL need different synchronization: log_empty_, alive_log_files_,
-  // logs_, logfile_number_. Refer to the definition of each variable below for
-  // more description.
   mutable InstrumentedMutex mutex_;
 
   std::atomic<bool> shutting_down_;
@@ -864,20 +820,10 @@ class DBImpl : public DB {
   // * whenever there is an error in background purge, flush or compaction
   // * whenever num_running_ingest_file_ goes to 0.
   InstrumentedCondVar bg_cv_;
-  // Writes are protected by locking both mutex_ and log_write_mutex_, and reads
-  // must be under either mutex_ or log_write_mutex_. Since after ::Open,
-  // logfile_number_ is currently updated only in write_thread_, it can be read
-  // from the same write_thread_ without any locks.
   uint64_t logfile_number_;
   std::deque<uint64_t>
       log_recycle_files;  // a list of log files that we can recycle
   bool log_dir_synced_;
-  // Without concurrent_prepare, read and writes to log_empty_ are protected by
-  // mutex_. Since it is currently updated/read only in write_thread_, it can be
-  // accessed from the same write_thread_ without any locks. With
-  // concurrent_prepare writes, where it can be updated in different threads,
-  // read and writes are protected by log_write_mutex_ instead. This is to avoid
-  // expesnive mutex_ lock during WAL write, which update log_empty_.
   bool log_empty_;
   ColumnFamilyHandleImpl* default_cf_handle_;
   InternalStats* default_cf_internal_stats_;
@@ -912,26 +858,14 @@ class DBImpl : public DB {
     // true for some prefix of logs_
     bool getting_synced = false;
   };
-  // Without concurrent_prepare, read and writes to alive_log_files_ are
-  // protected by mutex_. However since back() is never popped, and push_back()
-  // is done only from write_thread_, the same thread can access the item
-  // reffered by back() without mutex_. With concurrent_prepare_, writes
-  // are protected by locking both mutex_ and log_write_mutex_, and reads must
-  // be under either mutex_ or log_write_mutex_.
   std::deque<LogFileNumberSize> alive_log_files_;
   // Log files that aren't fully synced, and the current log file.
   // Synchronization:
-  //  - push_back() is done from write_thread_ with locked mutex_ and
-  //  log_write_mutex_
-  //  - pop_front() is done from any thread with locked mutex_ and
-  //  log_write_mutex_
-  //  - reads are done with either locked mutex_ or log_write_mutex_
+  //  - push_back() is done from write thread with locked mutex_,
+  //  - pop_front() is done from any thread with locked mutex_,
   //  - back() and items with getting_synced=true are not popped,
-  //  - The same thread that sets getting_synced=true will reset it.
-  //  - it follows that the object referred by back() can be safely read from
-  //  the write_thread_ without using mutex
-  //  - it follows that the items with getting_synced=true can be safely read
-  //  from the same thread that has set getting_synced=true
+  //  - it follows that write thread with unlocked mutex_ can safely access
+  //    back() and items with getting_synced=true.
   std::deque<LogWriterNumber> logs_;
   // Signaled when getting_synced becomes false for some of the logs_.
   InstrumentedCondVar log_sync_cv_;
@@ -980,19 +914,13 @@ class DBImpl : public DB {
   WriteBufferManager* write_buffer_manager_;
 
   WriteThread write_thread_;
+
   WriteBatch tmp_batch_;
-  // The write thread when the writers have no memtable write. This will be used
-  // in 2PC to batch the prepares separately from the serial commit.
-  WriteThread nonmem_write_thread_;
 
   WriteController write_controller_;
 
-  unique_ptr<RateLimiter> low_pri_write_rate_limiter_;
-
   // Size of the last batch group. In slowdown mode, next write needs to
   // sleep if it uses up the quota.
-  // Note: This is to protect memtable and compaction. If the batch only writes
-  // to the WAL its size need not to be included in this.
   uint64_t last_batch_group_size_;
 
   FlushScheduler flush_scheduler_;
@@ -1164,7 +1092,7 @@ class DBImpl : public DB {
   // Indicate DB was opened successfully
   bool opened_successfully_;
 
-  // minimum log number still containing prepared data.
+  // minmum log number still containing prepared data.
   // this is used by FindObsoleteFiles to determine which
   // flushed logs we must keep around because they still
   // contain prepared data which has not been flushed or rolled back
@@ -1177,7 +1105,7 @@ class DBImpl : public DB {
   // to prepared_section_completed_ which maps LOG -> instance_count
   // since a log could contain multiple prepared sections
   //
-  // when trying to determine the minimum log still active we first
+  // when trying to determine the minmum log still active we first
   // consult min_log_with_prep_. while that root value maps to
   // a value > 0 in prepared_section_completed_ we decrement the
   // instance_count for that log and pop the root value in
@@ -1235,11 +1163,6 @@ class DBImpl : public DB {
   bool MCOverlap(ManualCompaction* m, ManualCompaction* m1);
 
   size_t GetWalPreallocateBlockSize(uint64_t write_buffer_size) const;
-
-  // When set, we use a seprate queue for writes that dont write to memtable. In
-  // 2PC these are the writes at Prepare phase.
-  const bool concurrent_prepare_;
-  const bool manual_wal_flush_;
 };
 
 extern Options SanitizeOptions(const std::string& db,
@@ -1250,7 +1173,7 @@ extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src);
 extern CompressionType GetCompressionFlush(
     const ImmutableCFOptions& ioptions,
     const MutableCFOptions& mutable_cf_options);
-
+ 
 // Fix user-supplied options to be reasonable
 template <class T, class V>
 static void ClipToRange(T* ptr, V minvalue, V maxvalue) {

db/db_impl_compaction_flush.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -14,7 +14,6 @@
 #include <inttypes.h>
 
 #include "db/builder.h"
-#include "db/event_helpers.h"
 #include "monitoring/iostats_context_imp.h"
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/thread_status_updater.h"
@@ -60,14 +59,7 @@ Status DBImpl::SyncClosedLogs(JobContext* job_context) {
     // "number < current_log_number".
     MarkLogsSynced(current_log_number - 1, true, s);
     if (!s.ok()) {
-      Status new_bg_error = s;
-      // may temporarily unlock and lock the mutex.
-      EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                            BackgroundErrorReason::kFlush,
-                                            &new_bg_error, &mutex_);
-      if (!new_bg_error.ok()) {
-        bg_error_ = new_bg_error;
-      }
+      bg_error_ = s;
       TEST_SYNC_POINT("DBImpl::SyncClosedLogs:Failed");
       return s;
     }
@@ -142,16 +134,9 @@ Status DBImpl::FlushMemTableToOutputFile(
 
   if (!s.ok() && !s.IsShutdownInProgress() &&
       immutable_db_options_.paranoid_checks && bg_error_.ok()) {
-    Status new_bg_error = s;
-    // may temporarily unlock and lock the mutex.
-    EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                          BackgroundErrorReason::kFlush,
-                                          &new_bg_error, &mutex_);
-    if (!new_bg_error.ok()) {
-      // if a bad error happened (not ShutdownInProgress), paranoid_checks is
-      // true, and the error isn't handled by callback, mark DB read-only
-      bg_error_ = new_bg_error;
-    }
+    // if a bad error happened (not ShutdownInProgress) and paranoid_checks is
+    // true, mark DB read-only
+    bg_error_ = s;
   }
   if (s.ok()) {
 #ifndef ROCKSDB_LITE
@@ -166,17 +151,10 @@ Status DBImpl::FlushMemTableToOutputFile(
           immutable_db_options_.db_paths[0].path, file_meta.fd.GetNumber());
       sfm->OnAddFile(file_path);
       if (sfm->IsMaxAllowedSpaceReached() && bg_error_.ok()) {
-        Status new_bg_error = Status::IOError("Max allowed space was reached");
+        bg_error_ = Status::IOError("Max allowed space was reached");
         TEST_SYNC_POINT_CALLBACK(
             "DBImpl::FlushMemTableToOutputFile:MaxAllowedSpaceReached",
-            &new_bg_error);
-        // may temporarily unlock and lock the mutex.
-        EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                              BackgroundErrorReason::kFlush,
-                                              &new_bg_error, &mutex_);
-        if (!new_bg_error.ok()) {
-          bg_error_ = new_bg_error;
-        }
+            &bg_error_);
       }
     }
 #endif  // ROCKSDB_LITE
@@ -304,14 +282,10 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
   if (cfd->ioptions()->compaction_style == kCompactionStyleUniversal &&
       cfd->NumberLevels() > 1) {
     // Always compact all files together.
-    final_output_level = cfd->NumberLevels() - 1;
-    // if bottom most level is reserved
-    if (immutable_db_options_.allow_ingest_behind) {
-      final_output_level--;
-    }
     s = RunManualCompaction(cfd, ColumnFamilyData::kCompactAllLevels,
-                            final_output_level, options.target_path_id,
+                            cfd->NumberLevels() - 1, options.target_path_id,
                             begin, end, exclusive);
+    final_output_level = cfd->NumberLevels() - 1;
   } else {
     for (int level = 0; level <= max_level_with_files; level++) {
       int output_level;
@@ -587,14 +561,7 @@ Status DBImpl::CompactFilesImpl(
                    c->column_family_data()->GetName().c_str(),
                    job_context->job_id, status.ToString().c_str());
     if (immutable_db_options_.paranoid_checks && bg_error_.ok()) {
-      Status new_bg_error = status;
-      // may temporarily unlock and lock the mutex.
-      EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                            BackgroundErrorReason::kCompaction,
-                                            &new_bg_error, &mutex_);
-      if (!new_bg_error.ok()) {
-        bg_error_ = new_bg_error;
-      }
+      bg_error_ = status;
     }
   }
 
@@ -1010,22 +977,22 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
     // DB is being deleted; no more background compactions
     return;
   }
-  auto bg_job_limits = GetBGJobLimits();
-  bool is_flush_pool_empty =
-    env_->GetBackgroundThreads(Env::Priority::HIGH) == 0;
-  while (!is_flush_pool_empty && unscheduled_flushes_ > 0 &&
-         bg_flush_scheduled_ < bg_job_limits.max_flushes) {
+
+  while (unscheduled_flushes_ > 0 &&
+         bg_flush_scheduled_ < immutable_db_options_.max_background_flushes) {
     unscheduled_flushes_--;
     bg_flush_scheduled_++;
     env_->Schedule(&DBImpl::BGWorkFlush, this, Env::Priority::HIGH, this);
   }
 
-  // special case -- if high-pri (flush) thread pool is empty, then schedule
-  // flushes in low-pri (compaction) thread pool.
-  if (is_flush_pool_empty) {
+  auto bg_compactions_allowed = BGCompactionsAllowed();
+
+  // special case -- if max_background_flushes == 0, then schedule flush on a
+  // compaction thread
+  if (immutable_db_options_.max_background_flushes == 0) {
     while (unscheduled_flushes_ > 0 &&
            bg_flush_scheduled_ + bg_compaction_scheduled_ <
-               bg_job_limits.max_flushes) {
+               bg_compactions_allowed) {
       unscheduled_flushes_--;
       bg_flush_scheduled_++;
       env_->Schedule(&DBImpl::BGWorkFlush, this, Env::Priority::LOW, this);
@@ -1043,7 +1010,7 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
     return;
   }
 
-  while (bg_compaction_scheduled_ < bg_job_limits.max_compactions &&
+  while (bg_compaction_scheduled_ < bg_compactions_allowed &&
          unscheduled_compactions_ > 0) {
     CompactionArg* ca = new CompactionArg;
     ca->db = this;
@@ -1055,35 +1022,13 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
   }
 }
 
-DBImpl::BGJobLimits DBImpl::GetBGJobLimits() const {
+int DBImpl::BGCompactionsAllowed() const {
   mutex_.AssertHeld();
-  return GetBGJobLimits(immutable_db_options_.max_background_flushes,
-                        mutable_db_options_.max_background_compactions,
-                        mutable_db_options_.max_background_jobs,
-                        write_controller_.NeedSpeedupCompaction());
-}
-
-DBImpl::BGJobLimits DBImpl::GetBGJobLimits(int max_background_flushes,
-                                           int max_background_compactions,
-                                           int max_background_jobs,
-                                           bool parallelize_compactions) {
-  BGJobLimits res;
-  if (max_background_flushes == -1 && max_background_compactions == -1) {
-    // for our first stab implementing max_background_jobs, simply allocate a
-    // quarter of the threads to flushes.
-    res.max_flushes = std::max(1, max_background_jobs / 4);
-    res.max_compactions = std::max(1, max_background_jobs - res.max_flushes);
+  if (write_controller_.NeedSpeedupCompaction()) {
+    return mutable_db_options_.max_background_compactions;
   } else {
-    // compatibility code in case users haven't migrated to max_background_jobs,
-    // which automatically computes flush/compaction limits
-    res.max_flushes = std::max(1, max_background_flushes);
-    res.max_compactions = std::max(1, max_background_compactions);
+    return mutable_db_options_.base_background_compactions;
   }
-  if (!parallelize_compactions) {
-    // throttle background compactions until we deem necessary
-    res.max_compactions = 1;
-  }
-  return res;
 }
 
 void DBImpl::AddToCompactionQueue(ColumnFamilyData* cfd) {
@@ -1205,15 +1150,13 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
   if (cfd != nullptr) {
     const MutableCFOptions mutable_cf_options =
         *cfd->GetLatestMutableCFOptions();
-    auto bg_job_limits = GetBGJobLimits();
     ROCKS_LOG_BUFFER(
         log_buffer,
         "Calling FlushMemTableToOutputFile with column "
-        "family [%s], flush slots available %d, compaction slots available %d, "
-        "flush slots scheduled %d, compaction slots scheduled %d",
-        cfd->GetName().c_str(), bg_job_limits.max_flushes,
-        bg_job_limits.max_compactions, bg_flush_scheduled_,
-        bg_compaction_scheduled_);
+        "family [%s], flush slots available %d, compaction slots allowed %d, "
+        "compaction slots scheduled %d",
+        cfd->GetName().c_str(), immutable_db_options_.max_background_flushes,
+        bg_flush_scheduled_, BGCompactionsAllowed() - bg_compaction_scheduled_);
     status = FlushMemTableToOutputFile(cfd, mutable_cf_options, made_progress,
                                        job_context, log_buffer);
     if (cfd->Unref()) {
@@ -1226,6 +1169,7 @@ Status DBImpl::BackgroundFlush(bool* made_progress, JobContext* job_context,
 void DBImpl::BackgroundCallFlush() {
   bool made_progress = false;
   JobContext job_context(next_job_id_.fetch_add(1), true);
+  assert(bg_flush_scheduled_);
 
   TEST_SYNC_POINT("DBImpl::BackgroundCallFlush:start");
 
@@ -1233,7 +1177,6 @@ void DBImpl::BackgroundCallFlush() {
                        immutable_db_options_.info_log.get());
   {
     InstrumentedMutexLock l(&mutex_);
-    assert(bg_flush_scheduled_);
     num_running_flushes_++;
 
     auto pending_outputs_inserted_elem =
@@ -1454,7 +1397,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
       // Can't compact right now, but try again later
       TEST_SYNC_POINT("DBImpl::BackgroundCompaction()::Conflict");
 
-      // Stay in the compaction queue.
+      // Stay in the compaciton queue.
       unscheduled_compactions_++;
 
       return Status::OK();
@@ -1652,14 +1595,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
     ROCKS_LOG_WARN(immutable_db_options_.info_log, "Compaction error: %s",
                    status.ToString().c_str());
     if (immutable_db_options_.paranoid_checks && bg_error_.ok()) {
-      Status new_bg_error = status;
-      // may temporarily unlock and lock the mutex.
-      EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                            BackgroundErrorReason::kCompaction,
-                                            &new_bg_error, &mutex_);
-      if (!new_bg_error.ok()) {
-        bg_error_ = new_bg_error;
-      }
+      bg_error_ = status;
     }
   }
 
@@ -1764,7 +1700,7 @@ bool DBImpl::HaveManualCompaction(ColumnFamilyData* cfd) {
     }
     if ((cfd == (*it)->cfd) && (!((*it)->in_progress || (*it)->done))) {
       // Allow automatic compaction if manual compaction is
-      // in progress
+      // is in progress
       return true;
     }
     it++;

db/db_impl_debug.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -186,12 +186,7 @@ Status DBImpl::TEST_GetLatestMutableCFOptions(
 
 int DBImpl::TEST_BGCompactionsAllowed() const {
   InstrumentedMutexLock l(&mutex_);
-  return GetBGJobLimits().max_compactions;
-}
-
-int DBImpl::TEST_BGFlushesAllowed() const {
-  InstrumentedMutexLock l(&mutex_);
-  return GetBGJobLimits().max_flushes;
+  return BGCompactionsAllowed();
 }
 
 }  // namespace rocksdb

db/db_impl_experimental.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/db_impl_files.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -250,13 +250,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
       }
       job_context->size_log_to_delete += earliest.size;
       total_log_size_ -= earliest.size;
-      if (concurrent_prepare_) {
-        log_write_mutex_.Lock();
-      }
       alive_log_files_.pop_front();
-      if (concurrent_prepare_) {
-        log_write_mutex_.Unlock();
-      }
       // Current log should always stay alive since it can't have
       // number < MinLogNumber().
       assert(alive_log_files_.size());
@@ -269,10 +263,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
         continue;
       }
       logs_to_free_.push_back(log.ReleaseWriter());
-      {
-        InstrumentedMutexLock wl(&log_write_mutex_);
-        logs_.pop_front();
-      }
+      logs_.pop_front();
     }
     // Current log cannot be obsolete.
     assert(!logs_.empty());

db/db_impl_open.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -16,8 +16,6 @@
 #include "db/builder.h"
 #include "options/options_helper.h"
 #include "rocksdb/wal_filter.h"
-#include "table/block_based_table_factory.h"
-#include "util/rate_limiter.h"
 #include "util/sst_file_manager_impl.h"
 #include "util/sync_point.h"
 
@@ -38,7 +36,7 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
   if (result.max_open_files != -1) {
     int max_max_open_files = port::GetMaxOpenFiles();
     if (max_max_open_files == -1) {
-      max_max_open_files = 0x400000;
+      max_max_open_files = 1000000;
     }
     ClipToRange(&result.max_open_files, 20, max_max_open_files);
   }
@@ -55,13 +53,15 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
     result.write_buffer_manager.reset(
         new WriteBufferManager(result.db_write_buffer_size));
   }
-  auto bg_job_limits = DBImpl::GetBGJobLimits(result.max_background_flushes,
-                                              result.max_background_compactions,
-                                              result.max_background_jobs,
-                                              true /* parallelize_compactions */);
-  result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_compactions,
+  if (result.base_background_compactions == -1) {
+    result.base_background_compactions = result.max_background_compactions;
+  }
+  if (result.base_background_compactions > result.max_background_compactions) {
+    result.base_background_compactions = result.max_background_compactions;
+  }
+  result.env->IncBackgroundThreadsIfNeeded(src.max_background_compactions,
                                            Env::Priority::LOW);
-  result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_flushes,
+  result.env->IncBackgroundThreadsIfNeeded(src.max_background_flushes,
                                            Env::Priority::HIGH);
 
   if (result.rate_limiter.get() != nullptr) {
@@ -70,15 +70,6 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
     }
   }
 
-  if (result.delayed_write_rate == 0) {
-    if (result.rate_limiter.get() != nullptr) {
-      result.delayed_write_rate = result.rate_limiter->GetBytesPerSecond();
-    }
-    if (result.delayed_write_rate == 0) {
-      result.delayed_write_rate = 16 * 1024 * 1024;
-    }
-  }
-
   if (result.WAL_ttl_seconds > 0 || result.WAL_size_limit_MB > 0) {
     result.recycle_log_file_num = false;
   }
@@ -163,19 +154,6 @@ static Status ValidateOptions(
             "universal and level compaction styles. ");
       }
     }
-    if (cfd.options.compaction_options_fifo.ttl > 0) {
-      if (db_options.max_open_files != -1) {
-        return Status::NotSupported(
-            "FIFO Compaction with TTL is only supported when files are always "
-            "kept open (set max_open_files = -1). ");
-      }
-      if (cfd.options.table_factory->Name() !=
-          BlockBasedTableFactory().Name()) {
-        return Status::NotSupported(
-            "FIFO Compaction with TTL is only supported in "
-            "Block-Based Table format. ");
-      }
-    }
   }
 
   if (db_options.db_paths.size() > 4) {
@@ -520,8 +498,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     unique_ptr<SequentialFileReader> file_reader;
     {
       unique_ptr<SequentialFile> file;
-      status = env_->NewSequentialFile(fname, &file,
-                                       env_->OptimizeForLogRead(env_options_));
+      status = env_->NewSequentialFile(fname, &file, env_options_);
       if (!status.ok()) {
         MaybeIgnoreError(&status);
         if (!status.ok()) {
@@ -737,7 +714,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
     auto last_sequence = *next_sequence - 1;
     if ((*next_sequence != kMaxSequenceNumber) &&
         (versions_->LastSequence() <= last_sequence)) {
-      versions_->SetLastToBeWrittenSequence(last_sequence);
       versions_->SetLastSequence(last_sequence);
     }
   }
@@ -809,15 +785,9 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
   if (data_seen && !flushed) {
     // Mark these as alive so they'll be considered for deletion later by
     // FindObsoleteFiles()
-    if (concurrent_prepare_) {
-      log_write_mutex_.Lock();
-    }
     for (auto log_number : log_numbers) {
       alive_log_files_.push_back(LogFileNumberSize(log_number));
     }
-    if (concurrent_prepare_) {
-      log_write_mutex_.Unlock();
-    }
   }
 
   event_logger_.Log() << "job" << job_id << "event"
@@ -851,11 +821,6 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
         *cfd->GetLatestMutableCFOptions();
     bool paranoid_file_checks =
         cfd->GetLatestMutableCFOptions()->paranoid_file_checks;
-
-    int64_t _current_time = 0;
-    env_->GetCurrentTime(&_current_time);  // ignore error
-    const uint64_t current_time = static_cast<uint64_t>(_current_time);
-
     {
       mutex_.Unlock();
 
@@ -875,8 +840,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
           GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
           cfd->ioptions()->compression_opts, paranoid_file_checks,
           cfd->internal_stats(), TableFileCreationReason::kRecovery,
-          &event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */,
-          -1 /* level */, current_time);
+          &event_logger_, job_id);
       LogFlush(immutable_db_options_.info_log);
       ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
                       "[%s] [WriteLevel0TableForRecovery]"
@@ -925,7 +889,7 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
   }
   return s;
 }
-
+  
 Status DB::Open(const DBOptions& db_options, const std::string& dbname,
                 const std::vector<ColumnFamilyDescriptor>& column_families,
                 std::vector<ColumnFamilyHandle*>* handles, DB** dbptr) {
@@ -987,17 +951,14 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
     if (s.ok()) {
       lfile->SetPreallocationBlockSize(
           impl->GetWalPreallocateBlockSize(max_write_buffer_size));
-      {
-        InstrumentedMutexLock wl(&impl->log_write_mutex_);
-        impl->logfile_number_ = new_log_number;
-        unique_ptr<WritableFileWriter> file_writer(
-            new WritableFileWriter(std::move(lfile), opt_env_options));
-        impl->logs_.emplace_back(
-            new_log_number,
-            new log::Writer(
-                std::move(file_writer), new_log_number,
-                impl->immutable_db_options_.recycle_log_file_num > 0));
-      }
+      impl->logfile_number_ = new_log_number;
+      unique_ptr<WritableFileWriter> file_writer(
+          new WritableFileWriter(std::move(lfile), opt_env_options));
+      impl->logs_.emplace_back(
+          new_log_number,
+          new log::Writer(
+              std::move(file_writer), new_log_number,
+              impl->immutable_db_options_.recycle_log_file_num > 0));
 
       // set column family handles
       for (auto cf : column_families) {
@@ -1031,14 +992,8 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
         delete impl->InstallSuperVersionAndScheduleWork(
             cfd, nullptr, *cfd->GetLatestMutableCFOptions());
       }
-      if (impl->concurrent_prepare_) {
-        impl->log_write_mutex_.Lock();
-      }
       impl->alive_log_files_.push_back(
           DBImpl::LogFileNumberSize(impl->logfile_number_));
-      if (impl->concurrent_prepare_) {
-        impl->log_write_mutex_.Unlock();
-      }
       impl->DeleteObsoleteFiles();
       s = impl->directories_.GetDbDir()->Fsync();
     }

db/db_impl_readonly.cc

@@ -1,7 +1,8 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
 
 #include "db/db_impl_readonly.h"
 

db/db_impl_readonly.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #pragma once
 

db/db_impl_write.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -12,7 +12,6 @@
 #define __STDC_FORMAT_MACROS
 #endif
 #include <inttypes.h>
-#include "db/event_helpers.h"
 #include "monitoring/perf_context_imp.h"
 #include "options/options_helper.h"
 #include "util/sync_point.h"
@@ -64,28 +63,8 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   if (my_batch == nullptr) {
     return Status::Corruption("Batch is nullptr!");
   }
-  if (concurrent_prepare_ && immutable_db_options_.enable_pipelined_write) {
-    return Status::NotSupported(
-        "pipelined_writes is not compatible with concurrent prepares");
-  }
 
   Status status;
-  if (write_options.low_pri) {
-    status = ThrottleLowPriWritesIfNeeded(write_options, my_batch);
-    if (!status.ok()) {
-      return status;
-    }
-  }
-
-  if (concurrent_prepare_ && disable_memtable) {
-    return WriteImplWALOnly(write_options, my_batch, callback, log_used,
-                            log_ref);
-  }
-
-  if (immutable_db_options_.enable_pipelined_write) {
-    return PipelinedWriteImpl(write_options, my_batch, callback, log_used,
-                              log_ref, disable_memtable);
-  }
 
   PERF_TIMER_GUARD(write_pre_and_post_process_time);
   WriteThread::Writer w(write_options, my_batch, callback, log_ref,
@@ -98,25 +77,25 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
 
   write_thread_.JoinBatchGroup(&w);
-  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
+  if (w.state == WriteThread::STATE_PARALLEL_FOLLOWER) {
     // we are a non-leader in a parallel group
     PERF_TIMER_GUARD(write_memtable_time);
 
     if (w.ShouldWriteToMemtable()) {
       ColumnFamilyMemTablesImpl column_family_memtables(
           versions_->GetColumnFamilySet());
+      WriteBatchInternal::SetSequence(w.batch, w.sequence);
       w.status = WriteBatchInternal::InsertInto(
-          &w, w.sequence, &column_family_memtables, &flush_scheduler_,
+          &w, &column_family_memtables, &flush_scheduler_,
           write_options.ignore_missing_column_families, 0 /*log_number*/, this,
           true /*concurrent_memtable_writes*/);
     }
 
-    if (write_thread_.CompleteParallelMemTableWriter(&w)) {
-      // we're responsible for exit batch group
-      auto last_sequence = w.write_group->last_sequence;
+    if (write_thread_.CompleteParallelWorker(&w)) {
+      // we're responsible for early exit
+      auto last_sequence = w.parallel_group->last_sequence;
       versions_->SetLastSequence(last_sequence);
-      MemTableInsertStatusCheck(w.status);
-      write_thread_.ExitAsBatchGroupFollower(&w);
+      write_thread_.EarlyExitParallelGroup(&w);
     }
     assert(w.state == WriteThread::STATE_COMPLETED);
     // STATE_COMPLETED conditional below handles exit
@@ -138,24 +117,17 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   // when it finds suitable, and finish them in the same write batch.
   // This is how a write job could be done by the other writer.
   WriteContext write_context;
-  WriteThread::WriteGroup write_group;
-  bool in_parallel_group = false;
-  uint64_t last_sequence = kMaxSequenceNumber;
-  if (!concurrent_prepare_) {
-    last_sequence = versions_->LastSequence();
-  }
+  WriteThread::Writer* last_writer = &w;  // Dummy intial value
+  autovector<WriteThread::Writer*> write_group;
+  bool logs_getting_synced = false;
 
   mutex_.Lock();
 
   bool need_log_sync = !write_options.disableWAL && write_options.sync;
   bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
-  if (!concurrent_prepare_ || !disable_memtable) {
-    // With concurrent writes we do preprocess only in the write thread that
-    // also does write to memtable to avoid sync issue on shared data structure
-    // with the other thread
-    status = PreprocessWrite(write_options, &need_log_sync, &write_context);
-  }
-  log::Writer* log_writer = logs_.back().writer;
+  status = PreprocessWrite(write_options, need_log_sync, &logs_getting_synced,
+                           &write_context);
+  log::Writer* cur_log_writer = logs_.back().writer;
 
   mutex_.Unlock();
 
@@ -164,8 +136,9 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
   // and protects against concurrent loggers and concurrent writes
   // into memtables
 
+  bool exit_completed_early = false;
   last_batch_group_size_ =
-      write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
+      write_thread_.EnterAsBatchGroupLeader(&w, &last_writer, &write_group);
 
   if (status.ok()) {
     // Rules for when we can update the memtable concurrently
@@ -180,39 +153,43 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
     // relax rules 2 if we could prevent write batches from referring
     // more than once to a particular key.
     bool parallel = immutable_db_options_.allow_concurrent_memtable_write &&
-                    write_group.size > 1;
+                    write_group.size() > 1;
     int total_count = 0;
     uint64_t total_byte_size = 0;
-    for (auto* writer : write_group) {
+    for (auto writer : write_group) {
       if (writer->CheckCallback(this)) {
         if (writer->ShouldWriteToMemtable()) {
           total_count += WriteBatchInternal::Count(writer->batch);
           parallel = parallel && !writer->batch->HasMerge();
         }
 
-        total_byte_size = WriteBatchInternal::AppendedByteSize(
-            total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
+        if (writer->ShouldWriteToWAL()) {
+          total_byte_size = WriteBatchInternal::AppendedByteSize(
+              total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
+        }
       }
     }
 
-    const bool concurrent_update = concurrent_prepare_;
+    uint64_t last_sequence = versions_->LastSequence();
+    const SequenceNumber current_sequence = last_sequence + 1;
+    last_sequence += total_count;
+
     // Update stats while we are an exclusive group leader, so we know
     // that nobody else can be writing to these particular stats.
     // We're optimistic, updating the stats before we successfully
-    // commit.  That lets us release our leader status early.
+    // commit.  That lets us release our leader status early in
+    // some cases.
     auto stats = default_cf_internal_stats_;
-    stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count,
-                      concurrent_update);
+    stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count);
     RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);
-    stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size,
-                      concurrent_update);
+    stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size);
     RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
-    stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1, concurrent_update);
+    stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1);
     RecordTick(stats_, WRITE_DONE_BY_SELF);
-    auto write_done_by_other = write_group.size - 1;
+    auto write_done_by_other = write_group.size() - 1;
     if (write_done_by_other > 0) {
-      stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER, write_done_by_other,
-                        concurrent_update);
+      stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER,
+                        write_done_by_other);
       RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other);
     }
     MeasureTime(stats_, BYTES_PER_WRITE, total_byte_size);
@@ -223,49 +200,39 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
 
     PERF_TIMER_STOP(write_pre_and_post_process_time);
 
-    if (!concurrent_prepare_) {
-      if (status.ok() && !write_options.disableWAL) {
-        PERF_TIMER_GUARD(write_wal_time);
-        status = WriteToWAL(write_group, log_writer, log_used, need_log_sync,
-                            need_log_dir_sync, last_sequence + 1);
-      }
-    } else {
-      if (status.ok() && !write_options.disableWAL) {
-        PERF_TIMER_GUARD(write_wal_time);
-        // LastToBeWrittenSequence is increased inside WriteToWAL under
-        // wal_write_mutex_ to ensure ordered events in WAL
-        status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
-                                      total_count);
-      } else {
-        // Otherwise we inc seq number for memtable writes
-        last_sequence = versions_->FetchAddLastToBeWrittenSequence(total_count);
+    if (status.ok() && !write_options.disableWAL) {
+      PERF_TIMER_GUARD(write_wal_time);
+      status = WriteToWAL(write_group, cur_log_writer, need_log_sync,
+                          need_log_dir_sync, current_sequence);
+      if (log_used != nullptr) {
+        *log_used = logfile_number_;
       }
     }
-    assert(last_sequence != kMaxSequenceNumber);
-    const SequenceNumber current_sequence = last_sequence + 1;
-    last_sequence += total_count;
 
     if (status.ok()) {
       PERF_TIMER_GUARD(write_memtable_time);
 
       if (!parallel) {
-        w.status = WriteBatchInternal::InsertInto(
+        status = WriteBatchInternal::InsertInto(
             write_group, current_sequence, column_family_memtables_.get(),
             &flush_scheduler_, write_options.ignore_missing_column_families,
             0 /*recovery_log_number*/, this);
-      } else {
-        SequenceNumber next_sequence = current_sequence;
-        for (auto* writer : write_group) {
-          if (writer->ShouldWriteToMemtable()) {
-            writer->sequence = next_sequence;
-            next_sequence += WriteBatchInternal::Count(writer->batch);
-          }
+
+        if (status.ok()) {
+          // There were no write failures. Set leader's status
+          // in case the write callback returned a non-ok status.
+          status = w.FinalStatus();
         }
-        write_group.last_sequence = last_sequence;
-        write_group.running.store(static_cast<uint32_t>(write_group.size),
-                                  std::memory_order_relaxed);
-        write_thread_.LaunchParallelMemTableWriters(&write_group);
-        in_parallel_group = true;
+
+      } else {
+        WriteThread::ParallelGroup pg;
+        pg.leader = &w;
+        pg.last_writer = last_writer;
+        pg.last_sequence = last_sequence;
+        pg.early_exit_allowed = !need_log_sync;
+        pg.running.store(static_cast<uint32_t>(write_group.size()),
+                         std::memory_order_relaxed);
+        write_thread_.LaunchParallelFollowers(&pg, current_sequence);
 
         // Each parallel follower is doing each own writes. The leader should
         // also do its own.
@@ -273,309 +240,73 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
           ColumnFamilyMemTablesImpl column_family_memtables(
               versions_->GetColumnFamilySet());
           assert(w.sequence == current_sequence);
+          WriteBatchInternal::SetSequence(w.batch, w.sequence);
           w.status = WriteBatchInternal::InsertInto(
-              &w, w.sequence, &column_family_memtables, &flush_scheduler_,
+              &w, &column_family_memtables, &flush_scheduler_,
               write_options.ignore_missing_column_families, 0 /*log_number*/,
               this, true /*concurrent_memtable_writes*/);
         }
+
+        // CompleteParallelWorker returns true if this thread should
+        // handle exit, false means somebody else did
+        exit_completed_early = !write_thread_.CompleteParallelWorker(&w);
+        status = w.FinalStatus();
+      }
+
+      if (!exit_completed_early && w.status.ok()) {
+        versions_->SetLastSequence(last_sequence);
+        if (!need_log_sync) {
+          write_thread_.ExitAsBatchGroupLeader(&w, last_writer, w.status);
+          exit_completed_early = true;
+        }
+      }
+
+      // A non-OK status here indicates that the state implied by the
+      // WAL has diverged from the in-memory state.  This could be
+      // because of a corrupt write_batch (very bad), or because the
+      // client specified an invalid column family and didn't specify
+      // ignore_missing_column_families.
+      //
+      // Is setting bg_error_ enough here?  This will at least stop
+      // compaction and fail any further writes.
+      if (!status.ok() && !w.CallbackFailed()) {
+        mutex_.Lock();
+        if (bg_error_.ok()) {
+          bg_error_ = status;  // stop compaction & fail any further writes
+        }
+        mutex_.Unlock();
       }
     }
   }
   PERF_TIMER_START(write_pre_and_post_process_time);
 
-  if (!w.CallbackFailed()) {
-    WriteCallbackStatusCheck(status);
+  if (immutable_db_options_.paranoid_checks && !status.ok() &&
+      !w.CallbackFailed() && !status.IsBusy() && !status.IsIncomplete()) {
+    mutex_.Lock();
+    if (bg_error_.ok()) {
+      bg_error_ = status;  // stop compaction & fail any further writes
+    }
+    mutex_.Unlock();
   }
 
-  if (need_log_sync) {
+  if (logs_getting_synced) {
     mutex_.Lock();
     MarkLogsSynced(logfile_number_, need_log_dir_sync, status);
     mutex_.Unlock();
-    // Requesting sync with concurrent_prepare_ is expected to be very rare. We
-    // hance provide a simple implementation that is not necessarily efficient.
-    if (concurrent_prepare_) {
-      if (manual_wal_flush_) {
-        status = FlushWAL(true);
-      } else {
-        status = SyncWAL();
-      }
-    }
   }
 
-  bool should_exit_batch_group = true;
-  if (in_parallel_group) {
-    // CompleteParallelWorker returns true if this thread should
-    // handle exit, false means somebody else did
-    should_exit_batch_group = write_thread_.CompleteParallelMemTableWriter(&w);
-  }
-  if (should_exit_batch_group) {
-    if (status.ok()) {
-      versions_->SetLastSequence(last_sequence);
-    }
-    MemTableInsertStatusCheck(w.status);
-    write_thread_.ExitAsBatchGroupLeader(write_group, status);
+  if (!exit_completed_early) {
+    write_thread_.ExitAsBatchGroupLeader(&w, last_writer, w.status);
   }
 
-  if (status.ok()) {
-    status = w.FinalStatus();
-  }
   return status;
 }
 
-Status DBImpl::PipelinedWriteImpl(const WriteOptions& write_options,
-                                  WriteBatch* my_batch, WriteCallback* callback,
-                                  uint64_t* log_used, uint64_t log_ref,
-                                  bool disable_memtable) {
-  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
-
-  WriteContext write_context;
-
-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        disable_memtable);
-  write_thread_.JoinBatchGroup(&w);
-  if (w.state == WriteThread::STATE_GROUP_LEADER) {
-    WriteThread::WriteGroup wal_write_group;
-    if (w.callback && !w.callback->AllowWriteBatching()) {
-      write_thread_.WaitForMemTableWriters();
-    }
-    mutex_.Lock();
-    bool need_log_sync = !write_options.disableWAL && write_options.sync;
-    bool need_log_dir_sync = need_log_sync && !log_dir_synced_;
-    w.status = PreprocessWrite(write_options, &need_log_sync, &write_context);
-    log::Writer* log_writer = logs_.back().writer;
-    mutex_.Unlock();
-
-    // This can set non-OK status if callback fail.
-    last_batch_group_size_ =
-        write_thread_.EnterAsBatchGroupLeader(&w, &wal_write_group);
-    const SequenceNumber current_sequence =
-        write_thread_.UpdateLastSequence(versions_->LastSequence()) + 1;
-    size_t total_count = 0;
-    size_t total_byte_size = 0;
-
-    if (w.status.ok()) {
-      SequenceNumber next_sequence = current_sequence;
-      for (auto writer : wal_write_group) {
-        if (writer->CheckCallback(this)) {
-          if (writer->ShouldWriteToMemtable()) {
-            writer->sequence = next_sequence;
-            size_t count = WriteBatchInternal::Count(writer->batch);
-            next_sequence += count;
-            total_count += count;
-          }
-          total_byte_size = WriteBatchInternal::AppendedByteSize(
-              total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
-        }
-      }
-      if (w.disable_wal) {
-        has_unpersisted_data_.store(true, std::memory_order_relaxed);
-      }
-      write_thread_.UpdateLastSequence(current_sequence + total_count - 1);
-    }
-
-    auto stats = default_cf_internal_stats_;
-    stats->AddDBStats(InternalStats::NUMBER_KEYS_WRITTEN, total_count);
-    RecordTick(stats_, NUMBER_KEYS_WRITTEN, total_count);
-    stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size);
-    RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
-
-    PERF_TIMER_STOP(write_pre_and_post_process_time);
-
-    if (w.ShouldWriteToWAL()) {
-      PERF_TIMER_GUARD(write_wal_time);
-      stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1);
-      RecordTick(stats_, WRITE_DONE_BY_SELF, 1);
-      if (wal_write_group.size > 1) {
-        stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER,
-                          wal_write_group.size - 1);
-        RecordTick(stats_, WRITE_DONE_BY_OTHER, wal_write_group.size - 1);
-      }
-      w.status = WriteToWAL(wal_write_group, log_writer, log_used,
-                            need_log_sync, need_log_dir_sync, current_sequence);
-    }
-
-    if (!w.CallbackFailed()) {
-      WriteCallbackStatusCheck(w.status);
-    }
-
-    if (need_log_sync) {
-      mutex_.Lock();
-      MarkLogsSynced(logfile_number_, need_log_dir_sync, w.status);
-      mutex_.Unlock();
-    }
-
-    write_thread_.ExitAsBatchGroupLeader(wal_write_group, w.status);
-  }
-
-  WriteThread::WriteGroup memtable_write_group;
-  if (w.state == WriteThread::STATE_MEMTABLE_WRITER_LEADER) {
-    PERF_TIMER_GUARD(write_memtable_time);
-    assert(w.status.ok());
-    write_thread_.EnterAsMemTableWriter(&w, &memtable_write_group);
-    if (memtable_write_group.size > 1 &&
-        immutable_db_options_.allow_concurrent_memtable_write) {
-      write_thread_.LaunchParallelMemTableWriters(&memtable_write_group);
-    } else {
-      memtable_write_group.status = WriteBatchInternal::InsertInto(
-          memtable_write_group, w.sequence, column_family_memtables_.get(),
-          &flush_scheduler_, write_options.ignore_missing_column_families,
-          0 /*log_number*/, this);
-      versions_->SetLastSequence(memtable_write_group.last_sequence);
-      write_thread_.ExitAsMemTableWriter(&w, memtable_write_group);
-    }
-  }
-
-  if (w.state == WriteThread::STATE_PARALLEL_MEMTABLE_WRITER) {
-    assert(w.ShouldWriteToMemtable());
-    ColumnFamilyMemTablesImpl column_family_memtables(
-        versions_->GetColumnFamilySet());
-    w.status = WriteBatchInternal::InsertInto(
-        &w, w.sequence, &column_family_memtables, &flush_scheduler_,
-        write_options.ignore_missing_column_families, 0 /*log_number*/, this,
-        true /*concurrent_memtable_writes*/);
-    if (write_thread_.CompleteParallelMemTableWriter(&w)) {
-      MemTableInsertStatusCheck(w.status);
-      versions_->SetLastSequence(w.write_group->last_sequence);
-      write_thread_.ExitAsMemTableWriter(&w, *w.write_group);
-    }
-  }
-
-  assert(w.state == WriteThread::STATE_COMPLETED);
-  return w.FinalStatus();
-}
-
-Status DBImpl::WriteImplWALOnly(const WriteOptions& write_options,
-                                WriteBatch* my_batch, WriteCallback* callback,
-                                uint64_t* log_used, uint64_t log_ref) {
-  Status status;
-  PERF_TIMER_GUARD(write_pre_and_post_process_time);
-  WriteThread::Writer w(write_options, my_batch, callback, log_ref,
-                        true /* disable_memtable */);
-  if (write_options.disableWAL) {
-    return status;
-  }
-  RecordTick(stats_, WRITE_WITH_WAL);
-  StopWatch write_sw(env_, immutable_db_options_.statistics.get(), DB_WRITE);
-
-  nonmem_write_thread_.JoinBatchGroup(&w);
-  assert(w.state != WriteThread::STATE_PARALLEL_MEMTABLE_WRITER);
-  if (w.state == WriteThread::STATE_COMPLETED) {
-    if (log_used != nullptr) {
-      *log_used = w.log_used;
-    }
-    return w.FinalStatus();
-  }
-  // else we are the leader of the write batch group
-  assert(w.state == WriteThread::STATE_GROUP_LEADER);
-  WriteContext write_context;
-  WriteThread::WriteGroup write_group;
-  uint64_t last_sequence;
-  nonmem_write_thread_.EnterAsBatchGroupLeader(&w, &write_group);
-  // Note: no need to update last_batch_group_size_ here since the batch writes
-  // to WAL only
-
-  uint64_t total_byte_size = 0;
-  for (auto* writer : write_group) {
-    if (writer->CheckCallback(this)) {
-      total_byte_size = WriteBatchInternal::AppendedByteSize(
-          total_byte_size, WriteBatchInternal::ByteSize(writer->batch));
-    }
-  }
-
-  const bool concurrent_update = true;
-  // Update stats while we are an exclusive group leader, so we know
-  // that nobody else can be writing to these particular stats.
-  // We're optimistic, updating the stats before we successfully
-  // commit.  That lets us release our leader status early.
-  auto stats = default_cf_internal_stats_;
-  stats->AddDBStats(InternalStats::BYTES_WRITTEN, total_byte_size,
-                    concurrent_update);
-  RecordTick(stats_, BYTES_WRITTEN, total_byte_size);
-  stats->AddDBStats(InternalStats::WRITE_DONE_BY_SELF, 1, concurrent_update);
-  RecordTick(stats_, WRITE_DONE_BY_SELF);
-  auto write_done_by_other = write_group.size - 1;
-  if (write_done_by_other > 0) {
-    stats->AddDBStats(InternalStats::WRITE_DONE_BY_OTHER, write_done_by_other,
-                      concurrent_update);
-    RecordTick(stats_, WRITE_DONE_BY_OTHER, write_done_by_other);
-  }
-  MeasureTime(stats_, BYTES_PER_WRITE, total_byte_size);
-
-  PERF_TIMER_STOP(write_pre_and_post_process_time);
-
-  PERF_TIMER_GUARD(write_wal_time);
-  // LastToBeWrittenSequence is increased inside WriteToWAL under
-  // wal_write_mutex_ to ensure ordered events in WAL
-  status = ConcurrentWriteToWAL(write_group, log_used, &last_sequence,
-                                0 /*total_count*/);
-  if (status.ok() && write_options.sync) {
-    // Requesting sync with concurrent_prepare_ is expected to be very rare. We
-    // hance provide a simple implementation that is not necessarily efficient.
-    if (manual_wal_flush_) {
-      status = FlushWAL(true);
-    } else {
-      status = SyncWAL();
-    }
-  }
-  PERF_TIMER_START(write_pre_and_post_process_time);
-
-  if (!w.CallbackFailed()) {
-    WriteCallbackStatusCheck(status);
-  }
-  nonmem_write_thread_.ExitAsBatchGroupLeader(write_group, status);
-  if (status.ok()) {
-    status = w.FinalStatus();
-  }
-  return status;
-}
-
-void DBImpl::WriteCallbackStatusCheck(const Status& status) {
-  // Is setting bg_error_ enough here?  This will at least stop
-  // compaction and fail any further writes.
-  if (immutable_db_options_.paranoid_checks && !status.ok() &&
-      !status.IsBusy() && !status.IsIncomplete()) {
-    mutex_.Lock();
-    if (bg_error_.ok()) {
-      Status new_bg_error = status;
-      // may temporarily unlock and lock the mutex.
-      EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                            BackgroundErrorReason::kWriteCallback,
-                                            &new_bg_error, &mutex_);
-      if (!new_bg_error.ok()) {
-        bg_error_ = new_bg_error;  // stop compaction & fail any further writes
-      }
-    }
-    mutex_.Unlock();
-  }
-}
-
-void DBImpl::MemTableInsertStatusCheck(const Status& status) {
-  // A non-OK status here indicates that the state implied by the
-  // WAL has diverged from the in-memory state.  This could be
-  // because of a corrupt write_batch (very bad), or because the
-  // client specified an invalid column family and didn't specify
-  // ignore_missing_column_families.
-  if (!status.ok()) {
-    mutex_.Lock();
-    assert(bg_error_.ok());
-    Status new_bg_error = status;
-    // may temporarily unlock and lock the mutex.
-    EventHelpers::NotifyOnBackgroundError(immutable_db_options_.listeners,
-                                          BackgroundErrorReason::kMemTable,
-                                          &new_bg_error, &mutex_);
-    if (!new_bg_error.ok()) {
-      bg_error_ = new_bg_error;  // stop compaction & fail any further writes
-    }
-    mutex_.Unlock();
-  }
-}
-
 Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
-                               bool* need_log_sync,
+                               bool need_log_sync, bool* logs_getting_synced,
                                WriteContext* write_context) {
   mutex_.AssertHeld();
-  assert(write_context != nullptr && need_log_sync != nullptr);
+  assert(write_context != nullptr && logs_getting_synced != nullptr);
   Status status;
 
   assert(!single_column_family_mode_ ||
@@ -595,7 +326,7 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
   }
 
   if (UNLIKELY(status.ok() && !bg_error_.ok())) {
-    return bg_error_;
+    status = bg_error_;
   }
 
   if (UNLIKELY(status.ok() && !flush_scheduler_.Empty())) {
@@ -612,7 +343,7 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
     status = DelayWrite(last_batch_group_size_, write_options);
   }
 
-  if (status.ok() && *need_log_sync) {
+  if (status.ok() && need_log_sync) {
     // Wait until the parallel syncs are finished. Any sync process has to sync
     // the front log too so it is enough to check the status of front()
     // We do a while loop since log_sync_cv_ is signalled when any sync is
@@ -632,84 +363,50 @@ Status DBImpl::PreprocessWrite(const WriteOptions& write_options,
       // actually write to the WAL
       log.getting_synced = true;
     }
-  } else {
-    *need_log_sync = false;
+    *logs_getting_synced = true;
   }
 
   return status;
 }
 
-WriteBatch* DBImpl::MergeBatch(const WriteThread::WriteGroup& write_group,
-                               WriteBatch* tmp_batch, size_t* write_with_wal) {
-  assert(write_with_wal != nullptr);
-  assert(tmp_batch != nullptr);
+Status DBImpl::WriteToWAL(const autovector<WriteThread::Writer*>& write_group,
+                          log::Writer* log_writer, bool need_log_sync,
+                          bool need_log_dir_sync, SequenceNumber sequence) {
+  Status status;
+
   WriteBatch* merged_batch = nullptr;
-  *write_with_wal = 0;
-  auto* leader = write_group.leader;
-  if (write_group.size == 1 && leader->ShouldWriteToWAL() &&
-      leader->batch->GetWalTerminationPoint().is_cleared()) {
+  size_t write_with_wal = 0;
+  if (write_group.size() == 1 && write_group[0]->ShouldWriteToWAL() &&
+      write_group[0]->batch->GetWalTerminationPoint().is_cleared()) {
     // we simply write the first WriteBatch to WAL if the group only
     // contains one batch, that batch should be written to the WAL,
     // and the batch is not wanting to be truncated
-    merged_batch = leader->batch;
-    *write_with_wal = 1;
+    merged_batch = write_group[0]->batch;
+    write_group[0]->log_used = logfile_number_;
+    write_with_wal = 1;
   } else {
     // WAL needs all of the batches flattened into a single batch.
     // We could avoid copying here with an iov-like AddRecord
     // interface
-    merged_batch = tmp_batch;
+    merged_batch = &tmp_batch_;
     for (auto writer : write_group) {
       if (writer->ShouldWriteToWAL()) {
         WriteBatchInternal::Append(merged_batch, writer->batch,
                                    /*WAL_only*/ true);
-        (*write_with_wal)++;
+        write_with_wal++;
       }
-    }
-  }
-  return merged_batch;
-}
-
-// When concurrent_prepare_ is disabled, this function is called from the only
-// write thread. Otherwise this must be called holding log_write_mutex_.
-Status DBImpl::WriteToWAL(const WriteBatch& merged_batch,
-                          log::Writer* log_writer, uint64_t* log_used,
-                          uint64_t* log_size) {
-  assert(log_size != nullptr);
-  Slice log_entry = WriteBatchInternal::Contents(&merged_batch);
-  *log_size = log_entry.size();
-  Status status = log_writer->AddRecord(log_entry);
-  if (log_used != nullptr) {
-    *log_used = logfile_number_;
-  }
-  total_log_size_ += log_entry.size();
-  // TODO(myabandeh): it might be unsafe to access alive_log_files_.back() here
-  // since alive_log_files_ might be modified concurrently
-  alive_log_files_.back().AddSize(log_entry.size());
-  log_empty_ = false;
-  return status;
-}
-
-Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
-                          log::Writer* log_writer, uint64_t* log_used,
-                          bool need_log_sync, bool need_log_dir_sync,
-                          SequenceNumber sequence) {
-  Status status;
-
-  size_t write_with_wal = 0;
-  WriteBatch* merged_batch =
-      MergeBatch(write_group, &tmp_batch_, &write_with_wal);
-  if (merged_batch == write_group.leader->batch) {
-    write_group.leader->log_used = logfile_number_;
-  } else if (write_with_wal > 1) {
-    for (auto writer : write_group) {
       writer->log_used = logfile_number_;
     }
   }
 
   WriteBatchInternal::SetSequence(merged_batch, sequence);
 
-  uint64_t log_size;
-  status = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
+  Slice log_entry = WriteBatchInternal::Contents(merged_batch);
+  status = log_writer->AddRecord(log_entry);
+  total_log_size_ += log_entry.size();
+  alive_log_files_.back().AddSize(log_entry.size());
+  log_empty_ = false;
+  uint64_t log_size = log_entry.size();
 
   if (status.ok() && need_log_sync) {
     StopWatch sw(env_, stats_, WAL_FILE_SYNC_MICROS);
@@ -751,48 +448,6 @@ Status DBImpl::WriteToWAL(const WriteThread::WriteGroup& write_group,
   return status;
 }
 
-Status DBImpl::ConcurrentWriteToWAL(const WriteThread::WriteGroup& write_group,
-                                    uint64_t* log_used,
-                                    SequenceNumber* last_sequence,
-                                    int total_count) {
-  Status status;
-
-  WriteBatch tmp_batch;
-  size_t write_with_wal = 0;
-  WriteBatch* merged_batch =
-      MergeBatch(write_group, &tmp_batch, &write_with_wal);
-
-  // We need to lock log_write_mutex_ since logs_ and alive_log_files might be
-  // pushed back concurrently
-  log_write_mutex_.Lock();
-  if (merged_batch == write_group.leader->batch) {
-    write_group.leader->log_used = logfile_number_;
-  } else if (write_with_wal > 1) {
-    for (auto writer : write_group) {
-      writer->log_used = logfile_number_;
-    }
-  }
-  *last_sequence = versions_->FetchAddLastToBeWrittenSequence(total_count);
-  auto sequence = *last_sequence + 1;
-  WriteBatchInternal::SetSequence(merged_batch, sequence);
-
-  log::Writer* log_writer = logs_.back().writer;
-  uint64_t log_size;
-  status = WriteToWAL(*merged_batch, log_writer, log_used, &log_size);
-  log_write_mutex_.Unlock();
-
-  if (status.ok()) {
-    const bool concurrent = true;
-    auto stats = default_cf_internal_stats_;
-    stats->AddDBStats(InternalStats::WAL_FILE_BYTES, log_size, concurrent);
-    RecordTick(stats_, WAL_FILE_BYTES, log_size);
-    stats->AddDBStats(InternalStats::WRITE_WITH_WAL, write_with_wal,
-                      concurrent);
-    RecordTick(stats_, WRITE_WITH_WAL, write_with_wal);
-  }
-  return status;
-}
-
 Status DBImpl::HandleWALFull(WriteContext* write_context) {
   mutex_.AssertHeld();
   assert(write_context != nullptr);
@@ -957,35 +612,6 @@ Status DBImpl::DelayWrite(uint64_t num_bytes,
   return bg_error_;
 }
 
-Status DBImpl::ThrottleLowPriWritesIfNeeded(const WriteOptions& write_options,
-                                            WriteBatch* my_batch) {
-  assert(write_options.low_pri);
-  // This is called outside the DB mutex. Although it is safe to make the call,
-  // the consistency condition is not guaranteed to hold. It's OK to live with
-  // it in this case.
-  // If we need to speed compaction, it means the compaction is left behind
-  // and we start to limit low pri writes to a limit.
-  if (write_controller_.NeedSpeedupCompaction()) {
-    if (allow_2pc() && (my_batch->HasCommit() || my_batch->HasRollback())) {
-      // For 2PC, we only rate limit prepare, not commit.
-      return Status::OK();
-    }
-    if (write_options.no_slowdown) {
-      return Status::Incomplete();
-    } else {
-      assert(my_batch != nullptr);
-      // Rate limit those writes. The reason that we don't completely wait
-      // is that in case the write is heavy, low pri writes may never have
-      // a chance to run. Now we guarantee we are still slowly making
-      // progress.
-      write_controller_.low_pri_rate_limiter()->Request(
-          my_batch->GetDataSize(), Env::IO_HIGH, nullptr /* stats */,
-          RateLimiter::OpType::kWrite);
-    }
-  }
-  return Status::OK();
-}
-
 Status DBImpl::ScheduleFlushes(WriteContext* context) {
   ColumnFamilyData* cfd;
   while ((cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {
@@ -1020,33 +646,14 @@ void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* cfd,
 // REQUIRES: this thread is currently at the front of the writer queue
 Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   mutex_.AssertHeld();
-  WriteThread::Writer nonmem_w;
-  if (concurrent_prepare_) {
-    // SwitchMemtable is a rare event. To simply the reasoning, we make sure
-    // that there is no concurrent thread writing to WAL.
-    nonmem_write_thread_.EnterUnbatched(&nonmem_w, &mutex_);
-  }
-
   unique_ptr<WritableFile> lfile;
   log::Writer* new_log = nullptr;
   MemTable* new_mem = nullptr;
 
-  // In case of pipelined write is enabled, wait for all pending memtable
-  // writers.
-  if (immutable_db_options_.enable_pipelined_write) {
-    write_thread_.WaitForMemTableWriters();
-  }
-
   // Attempt to switch to a new memtable and trigger flush of old.
   // Do this without holding the dbmutex lock.
   assert(versions_->prev_log_number() == 0);
-  if (concurrent_prepare_) {
-    log_write_mutex_.Lock();
-  }
   bool creating_new_log = !log_empty_;
-  if (concurrent_prepare_) {
-    log_write_mutex_.Unlock();
-  }
   uint64_t recycle_log_number = 0;
   if (creating_new_log && immutable_db_options_.recycle_log_file_num &&
       !log_recycle_files.empty()) {
@@ -1102,9 +709,9 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
         lfile->SetPreallocationBlockSize(preallocate_block_size);
         unique_ptr<WritableFileWriter> file_writer(
             new WritableFileWriter(std::move(lfile), opt_env_opt));
-        new_log = new log::Writer(
-            std::move(file_writer), new_log_number,
-            immutable_db_options_.recycle_log_file_num > 0, manual_wal_flush_);
+        new_log =
+            new log::Writer(std::move(file_writer), new_log_number,
+                            immutable_db_options_.recycle_log_file_num > 0);
       }
     }
 
@@ -1131,25 +738,15 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
     assert(creating_new_log);
     assert(!new_mem);
     assert(!new_log);
-    if (concurrent_prepare_) {
-      nonmem_write_thread_.ExitUnbatched(&nonmem_w);
-    }
     return s;
   }
   if (creating_new_log) {
-    log_write_mutex_.Lock();
     logfile_number_ = new_log_number;
     assert(new_log != nullptr);
     log_empty_ = true;
     log_dir_synced_ = false;
-    if (!logs_.empty()) {
-      // Alway flush the buffer of the last log before switching to a new one
-      log::Writer* cur_log_writer = logs_.back().writer;
-      cur_log_writer->WriteBuffer();
-    }
     logs_.emplace_back(logfile_number_, new_log);
     alive_log_files_.push_back(LogFileNumberSize(logfile_number_));
-    log_write_mutex_.Unlock();
   }
   for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
     // all this is just optimization to delete logs that
@@ -1171,9 +768,6 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
   cfd->SetMemtable(new_mem);
   context->superversions_to_free_.push_back(InstallSuperVersionAndScheduleWork(
       cfd, new_superversion, mutable_cf_options));
-  if (concurrent_prepare_) {
-    nonmem_write_thread_.ExitUnbatched(&nonmem_w);
-  }
   return s;
 }
 

db/db_info_dumper.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #ifndef __STDC_FORMAT_MACROS
 #define __STDC_FORMAT_MACROS

db/db_info_dumper.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 #pragma once
 
 #include <string>

db/db_inplace_update_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/db_io_failure_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -250,315 +250,7 @@ TEST_F(DBIOFailureTest, PutFailsParanoid) {
   // the next put should NOT fail
   ASSERT_TRUE(s.ok());
 }
-#if !(defined NDEBUG) || !defined(OS_WIN)
-TEST_F(DBIOFailureTest, FlushSstRangeSyncError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.write_buffer_size = 256 * 1024 * 1024;
-  options.writable_file_max_buffer_size = 128 * 1024;
-  options.bytes_per_sync = 128 * 1024;
-  options.level0_file_num_compaction_trigger = 4;
-  options.memtable_factory.reset(new SpecialSkipListFactory(10));
-  BlockBasedTableOptions table_options;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(10));
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
 
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-
-  std::atomic<int> range_sync_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::RangeSync", [&](void* arg) {
-        if (range_sync_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("range sync dummy error");
-        }
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  Random rnd(301);
-  std::string rnd_str =
-      RandomString(&rnd, static_cast<int>(options.bytes_per_sync / 2));
-  std::string rnd_str_512kb = RandomString(&rnd, 512 * 1024);
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  // First 1MB doesn't get range synced
-  ASSERT_OK(Put(1, "foo0_0", rnd_str_512kb));
-  ASSERT_OK(Put(1, "foo0_1", rnd_str_512kb));
-  ASSERT_OK(Put(1, "foo1_1", rnd_str));
-  ASSERT_OK(Put(1, "foo1_2", rnd_str));
-  ASSERT_OK(Put(1, "foo1_3", rnd_str));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  ASSERT_OK(Put(1, "foo3_1", rnd_str));
-  ASSERT_OK(Put(1, "foo3_2", rnd_str));
-  ASSERT_OK(Put(1, "foo3_3", rnd_str));
-  ASSERT_OK(Put(1, "foo4", "bar"));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-
-  // Following writes should fail as flush failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar", Get(1, "foo"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  ASSERT_GE(1, range_sync_called.load());
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar", Get(1, "foo"));
-}
-
-TEST_F(DBIOFailureTest, CompactSstRangeSyncError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.write_buffer_size = 256 * 1024 * 1024;
-  options.writable_file_max_buffer_size = 128 * 1024;
-  options.bytes_per_sync = 128 * 1024;
-  options.level0_file_num_compaction_trigger = 2;
-  options.target_file_size_base = 256 * 1024 * 1024;
-  options.disable_auto_compactions = true;
-  BlockBasedTableOptions table_options;
-  table_options.filter_policy.reset(NewBloomFilterPolicy(10));
-  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-
-  Random rnd(301);
-  std::string rnd_str =
-      RandomString(&rnd, static_cast<int>(options.bytes_per_sync / 2));
-  std::string rnd_str_512kb = RandomString(&rnd, 512 * 1024);
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  // First 1MB doesn't get range synced
-  ASSERT_OK(Put(1, "foo0_0", rnd_str_512kb));
-  ASSERT_OK(Put(1, "foo0_1", rnd_str_512kb));
-  ASSERT_OK(Put(1, "foo1_1", rnd_str));
-  ASSERT_OK(Put(1, "foo1_2", rnd_str));
-  ASSERT_OK(Put(1, "foo1_3", rnd_str));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo3_1", rnd_str));
-  ASSERT_OK(Put(1, "foo3_2", rnd_str));
-  ASSERT_OK(Put(1, "foo3_3", rnd_str));
-  ASSERT_OK(Put(1, "foo4", "bar"));
-  Flush(1);
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-
-  std::atomic<int> range_sync_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::RangeSync", [&](void* arg) {
-        if (range_sync_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("range sync dummy error");
-        }
-      });
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(dbfull()->SetOptions(handles_[1],
-                                 {
-                                     {"disable_auto_compactions", "false"},
-                                 }));
-  dbfull()->TEST_WaitForCompact();
-
-  // Following writes should fail as flush failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar", Get(1, "foo"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-  ASSERT_GE(1, range_sync_called.load());
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar", Get(1, "foo"));
-}
-
-TEST_F(DBIOFailureTest, FlushSstCloseError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.level0_file_num_compaction_trigger = 4;
-  options.memtable_factory.reset(new SpecialSkipListFactory(2));
-
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-  std::atomic<int> close_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::Close", [&](void* arg) {
-        if (close_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("close dummy error");
-        }
-      });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo1", "bar1"));
-  ASSERT_OK(Put(1, "foo", "bar2"));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-
-  // Following writes should fail as flush failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar2", Get(1, "foo"));
-  ASSERT_EQ("bar1", Get(1, "foo1"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar2", Get(1, "foo"));
-  ASSERT_EQ("bar1", Get(1, "foo1"));
-}
-
-TEST_F(DBIOFailureTest, CompactionSstCloseError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.level0_file_num_compaction_trigger = 2;
-  options.disable_auto_compactions = true;
-
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar2"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar3"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  dbfull()->TEST_WaitForCompact();
-
-  std::atomic<int> close_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::Close", [&](void* arg) {
-        if (close_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("close dummy error");
-        }
-      });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  ASSERT_OK(dbfull()->SetOptions(handles_[1],
-                                 {
-                                     {"disable_auto_compactions", "false"},
-                                 }));
-  dbfull()->TEST_WaitForCompact();
-
-  // Following writes should fail as compaction failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar3", Get(1, "foo"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar3", Get(1, "foo"));
-}
-
-TEST_F(DBIOFailureTest, FlushSstSyncError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.use_fsync = false;
-  options.level0_file_num_compaction_trigger = 4;
-  options.memtable_factory.reset(new SpecialSkipListFactory(2));
-
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-  std::atomic<int> sync_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::Sync", [&](void* arg) {
-        if (sync_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("sync dummy error");
-        }
-      });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo1", "bar1"));
-  ASSERT_OK(Put(1, "foo", "bar2"));
-  dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
-
-  // Following writes should fail as flush failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar2", Get(1, "foo"));
-  ASSERT_EQ("bar1", Get(1, "foo1"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar2", Get(1, "foo"));
-  ASSERT_EQ("bar1", Get(1, "foo1"));
-}
-
-TEST_F(DBIOFailureTest, CompactionSstSyncError) {
-  Options options = CurrentOptions();
-  options.env = env_;
-  options.create_if_missing = true;
-  options.error_if_exists = false;
-  options.paranoid_checks = true;
-  options.level0_file_num_compaction_trigger = 2;
-  options.disable_auto_compactions = true;
-  options.use_fsync = false;
-
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-  Status s;
-
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar2"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  ASSERT_OK(Put(1, "foo", "bar3"));
-  ASSERT_OK(Put(1, "foo2", "bar"));
-  Flush(1);
-  dbfull()->TEST_WaitForCompact();
-
-  std::atomic<int> sync_called(0);
-  rocksdb::SyncPoint::GetInstance()->SetCallBack(
-      "SpecialEnv::SStableFile::Sync", [&](void* arg) {
-        if (sync_called.fetch_add(1) == 0) {
-          Status* st = static_cast<Status*>(arg);
-          *st = Status::IOError("close dummy error");
-        }
-      });
-
-  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
-  ASSERT_OK(dbfull()->SetOptions(handles_[1],
-                                 {
-                                     {"disable_auto_compactions", "false"},
-                                 }));
-  dbfull()->TEST_WaitForCompact();
-
-  // Following writes should fail as compaction failed.
-  ASSERT_NOK(Put(1, "foo2", "bar3"));
-  ASSERT_EQ("bar3", Get(1, "foo"));
-
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
-
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_EQ("bar3", Get(1, "foo"));
-}
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_iter.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -560,7 +560,7 @@ void DBIter::MergeValuesNewToOld() {
       const Slice val = iter_->value();
       s = MergeHelper::TimedFullMerge(
           merge_operator_, ikey.user_key, &val, merge_context_.GetOperands(),
-          &saved_value_, logger_, statistics_, env_, &pinned_value_, true);
+          &saved_value_, logger_, statistics_, env_, &pinned_value_);
       if (!s.ok()) {
         status_ = s;
       }
@@ -585,7 +585,7 @@ void DBIter::MergeValuesNewToOld() {
   s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
                                   nullptr, merge_context_.GetOperands(),
                                   &saved_value_, logger_, statistics_, env_,
-                                  &pinned_value_, true);
+                                  &pinned_value_);
   if (!s.ok()) {
     status_ = s;
   }
@@ -642,7 +642,6 @@ void DBIter::ReverseToBackward() {
     while (iter_->Valid() &&
            user_comparator_->Compare(ikey.user_key, saved_key_.GetUserKey()) >
                0) {
-      assert(ikey.sequence != kMaxSequenceNumber);
       if (ikey.sequence > sequence_) {
         PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
       } else {
@@ -805,13 +804,13 @@ bool DBIter::FindValueForCurrentKey() {
         s = MergeHelper::TimedFullMerge(
             merge_operator_, saved_key_.GetUserKey(), nullptr,
             merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
-            env_, &pinned_value_, true);
+            env_, &pinned_value_);
       } else {
         assert(last_not_merge_type == kTypeValue);
         s = MergeHelper::TimedFullMerge(
             merge_operator_, saved_key_.GetUserKey(), &pinned_value_,
             merge_context_.GetOperands(), &saved_value_, logger_, statistics_,
-            env_, &pinned_value_, true);
+            env_, &pinned_value_);
       }
       break;
     case kTypeValue:
@@ -883,7 +882,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
     s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
                                     nullptr, merge_context_.GetOperands(),
                                     &saved_value_, logger_, statistics_, env_,
-                                    &pinned_value_, true);
+                                    &pinned_value_);
     // Make iter_ valid and point to saved_key_
     if (!iter_->Valid() ||
         !user_comparator_->Equal(ikey.user_key, saved_key_.GetUserKey())) {
@@ -901,7 +900,7 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
   s = MergeHelper::TimedFullMerge(merge_operator_, saved_key_.GetUserKey(),
                                   &val, merge_context_.GetOperands(),
                                   &saved_value_, logger_, statistics_, env_,
-                                  &pinned_value_, true);
+                                  &pinned_value_);
   valid_ = true;
   if (!s.ok()) {
     status_ = s;
@@ -955,7 +954,6 @@ void DBIter::FindPrevUserKey() {
         ++num_skipped;
       }
     }
-    assert(ikey.sequence != kMaxSequenceNumber);
     if (ikey.sequence > sequence_) {
       PERF_COUNTER_ADD(internal_recent_skipped_count, 1);
     } else {

db/db_iter.h

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be

db/db_iter_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #include <string>
 #include <vector>
@@ -352,11 +352,11 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
 
-    get_perf_context()->Reset();
+    perf_context.Reset();
     db_iter->SeekToLast();
 
     ASSERT_TRUE(db_iter->Valid());
-    ASSERT_EQ(static_cast<int>(get_perf_context()->internal_key_skipped_count), 7);
+    ASSERT_EQ(static_cast<int>(perf_context.internal_key_skipped_count), 7);
     ASSERT_EQ(db_iter->key().ToString(), "b");
 
     SetPerfLevel(kDisable);
@@ -471,11 +471,11 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
     SetPerfLevel(kEnableCount);
     ASSERT_TRUE(GetPerfLevel() == kEnableCount);
 
-    get_perf_context()->Reset();
+    perf_context.Reset();
     db_iter->SeekToLast();
 
     ASSERT_TRUE(db_iter->Valid());
-    ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 1);
+    ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 1);
     ASSERT_EQ(db_iter->key().ToString(), "b");
 
     SetPerfLevel(kDisable);

db/db_iterator_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -972,11 +972,11 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(iter->key().compare(("b1")), 0);
 
-    get_perf_context()->Reset();
+    perf_context.Reset();
     iter->Next();
 
     ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 2);
+    ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 2);
 
     // now testing with iterate_bound
     Slice prefix("c");
@@ -984,7 +984,7 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
 
     iter.reset(db_->NewIterator(ro));
 
-    get_perf_context()->Reset();
+    perf_context.Reset();
 
     iter->Seek("b");
     ASSERT_TRUE(iter->Valid());
@@ -999,13 +999,12 @@ TEST_F(DBIteratorTest, DBIteratorBoundTest) {
     // even though the key is deleted
     // hence internal_delete_skipped_count should be 0
     ASSERT_TRUE(!iter->Valid());
-    ASSERT_EQ(static_cast<int>(get_perf_context()->internal_delete_skipped_count), 0);
+    ASSERT_EQ(static_cast<int>(perf_context.internal_delete_skipped_count), 0);
   }
 }
 
 TEST_F(DBIteratorTest, DBIteratorBoundOptimizationTest) {
   int upper_bound_hits = 0;
-  Options options = CurrentOptions();
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "BlockBasedTable::BlockEntryIteratorState::KeyReachedUpperBound",
       [&upper_bound_hits](void* arg) {
@@ -1013,6 +1012,7 @@ TEST_F(DBIteratorTest, DBIteratorBoundOptimizationTest) {
         upper_bound_hits += (*static_cast<bool*>(arg) ? 1 : 0);
       });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+  Options options = CurrentOptions();
   options.env = env_;
   options.create_if_missing = true;
   options.prefix_extractor = nullptr;
@@ -1886,7 +1886,7 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
 #endif
 
   // Seek iterator to a smaller key.
-  get_perf_context()->Reset();
+  perf_context.Reset();
   iter->Seek("a");
   ASSERT_TRUE(iter->Valid());
   EXPECT_EQ("b", iter->key().ToString());
@@ -1894,17 +1894,17 @@ TEST_F(DBIteratorTest, DBIteratorSkipRecentDuplicatesTest) {
 
   // Check that the seek didn't do too much work.
   // Checks are not tight, just make sure that everything is well below 100.
-  EXPECT_LT(get_perf_context()->internal_key_skipped_count, 4);
-  EXPECT_LT(get_perf_context()->internal_recent_skipped_count, 8);
-  EXPECT_LT(get_perf_context()->seek_on_memtable_count, 10);
-  EXPECT_LT(get_perf_context()->next_on_memtable_count, 10);
-  EXPECT_LT(get_perf_context()->prev_on_memtable_count, 10);
+  EXPECT_LT(perf_context.internal_key_skipped_count, 4);
+  EXPECT_LT(perf_context.internal_recent_skipped_count, 8);
+  EXPECT_LT(perf_context.seek_on_memtable_count, 10);
+  EXPECT_LT(perf_context.next_on_memtable_count, 10);
+  EXPECT_LT(perf_context.prev_on_memtable_count, 10);
 
   // Check that iterator did something like what we expect.
-  EXPECT_EQ(get_perf_context()->internal_delete_skipped_count, 0);
-  EXPECT_EQ(get_perf_context()->internal_merge_count, 0);
-  EXPECT_GE(get_perf_context()->internal_recent_skipped_count, 2);
-  EXPECT_GE(get_perf_context()->seek_on_memtable_count, 2);
+  EXPECT_EQ(perf_context.internal_delete_skipped_count, 0);
+  EXPECT_EQ(perf_context.internal_merge_count, 0);
+  EXPECT_GE(perf_context.internal_recent_skipped_count, 2);
+  EXPECT_GE(perf_context.seek_on_memtable_count, 2);
   EXPECT_EQ(1, options.statistics->getTickerCount(
                  NUMBER_OF_RESEEKS_IN_ITERATION));
 }

db/db_log_iter_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -116,7 +116,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
       dbfull()->Flush(FlushOptions());
       Put("key4", DummyString(1024));
       ASSERT_EQ(dbfull()->GetLatestSequenceNumber(), 4U);
-      dbfull()->FlushWAL(false);
 
       {
         auto iter = OpenTransactionLogIter(0);
@@ -133,7 +132,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorRace) {
 
       // "key5" would be written in a new memtable and log
       Put("key5", DummyString(1024));
-      dbfull()->FlushWAL(false);
       {
         // this iter would miss "key4" if not fixed
         auto iter = OpenTransactionLogIter(0);
@@ -183,7 +181,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) {
       Put("key"+ToString(i), DummyString(10));
     }
     dbfull()->Flush(FlushOptions());
-    dbfull()->FlushWAL(false);
     // Corrupt this log to create a gap
     rocksdb::VectorLogPtr wal_files;
     ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
@@ -197,7 +194,6 @@ TEST_F(DBTestXactLogIterator, TransactionLogIteratorCorruptedLog) {
 
     // Insert a new entry to a new log file
     Put("key1025", DummyString(10));
-    dbfull()->FlushWAL(false);
     // Try to read from the beginning. Should stop before the gap and read less
     // than 1025 entries
     auto iter = OpenTransactionLogIter(0);

db/db_memtable_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #include <memory>
 #include <string>
@@ -21,7 +21,7 @@ class DBMemTableTest : public DBTestBase {
 
 class MockMemTableRep : public MemTableRep {
  public:
-  explicit MockMemTableRep(Allocator* allocator, MemTableRep* rep)
+  explicit MockMemTableRep(MemTableAllocator* allocator, MemTableRep* rep)
       : MemTableRep(allocator), rep_(rep), num_insert_with_hint_(0) {}
 
   virtual KeyHandle Allocate(const size_t len, char** buf) override {
@@ -72,7 +72,7 @@ class MockMemTableRep : public MemTableRep {
 class MockMemTableRepFactory : public MemTableRepFactory {
  public:
   virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator& cmp,
-                                         Allocator* allocator,
+                                         MemTableAllocator* allocator,
                                          const SliceTransform* transform,
                                          Logger* logger) override {
     SkipListFactory factory;
@@ -82,27 +82,14 @@ class MockMemTableRepFactory : public MemTableRepFactory {
     return mock_rep_;
   }
 
-  virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator& cmp,
-                                         Allocator* allocator,
-                                         const SliceTransform* transform,
-                                         Logger* logger,
-                                         uint32_t column_family_id) override {
-    last_column_family_id_ = column_family_id;
-    return CreateMemTableRep(cmp, allocator, transform, logger);
-  }
-
   virtual const char* Name() const override { return "MockMemTableRepFactory"; }
 
   MockMemTableRep* rep() { return mock_rep_; }
 
   bool IsInsertConcurrentlySupported() const override { return false; }
 
-  uint32_t GetLastColumnFamilyId() { return last_column_family_id_; }
-
  private:
   MockMemTableRep* mock_rep_;
-  // workaround since there's no port::kMaxUint32 yet.
-  uint32_t last_column_family_id_ = static_cast<uint32_t>(-1);
 };
 
 class TestPrefixExtractor : public SliceTransform {
@@ -168,24 +155,6 @@ TEST_F(DBMemTableTest, InsertWithHint) {
   ASSERT_EQ("vvv", Get("whitelisted"));
 }
 
-TEST_F(DBMemTableTest, ColumnFamilyId) {
-  // Verifies MemTableRepFactory is told the right column family id.
-  Options options;
-  options.allow_concurrent_memtable_write = false;
-  options.create_if_missing = true;
-  options.memtable_factory.reset(new MockMemTableRepFactory());
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  for (int cf = 0; cf < 2; ++cf) {
-    ASSERT_OK(Put(cf, "key", "val"));
-    ASSERT_OK(Flush(cf));
-    ASSERT_EQ(
-        cf, static_cast<MockMemTableRepFactory*>(options.memtable_factory.get())
-                ->GetLastColumnFamilyId());
-  }
-}
-
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_merge_operator_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 #include <string>
 #include <vector>
 
@@ -18,6 +18,28 @@ class DBMergeOperatorTest : public DBTestBase {
   DBMergeOperatorTest() : DBTestBase("/db_merge_operator_test") {}
 };
 
+// A test merge operator mimics put but also fails if one of merge operands is
+// "corrupted".
+class TestPutOperator : public MergeOperator {
+ public:
+  virtual bool FullMergeV2(const MergeOperationInput& merge_in,
+                           MergeOperationOutput* merge_out) const override {
+    if (merge_in.existing_value != nullptr &&
+        *(merge_in.existing_value) == "corrupted") {
+      return false;
+    }
+    for (auto value : merge_in.operand_list) {
+      if (value == "corrupted") {
+        return false;
+      }
+    }
+    merge_out->existing_operand = merge_in.operand_list.back();
+    return true;
+  }
+
+  virtual const char* Name() const override { return "TestPutOperator"; }
+};
+
 TEST_F(DBMergeOperatorTest, MergeErrorOnRead) {
   Options options;
   options.create_if_missing = true;

db/db_options_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -15,9 +15,7 @@
 #include "db/db_test_util.h"
 #include "options/options_helper.h"
 #include "port/stack_trace.h"
-#include "rocksdb/cache.h"
 #include "rocksdb/convenience.h"
-#include "rocksdb/rate_limiter.h"
 #include "util/random.h"
 #include "util/sync_point.h"
 #include "util/testutil.h"
@@ -253,44 +251,18 @@ TEST_F(DBOptionsTest, SetOptionsMayTriggerCompaction) {
 TEST_F(DBOptionsTest, SetBackgroundCompactionThreads) {
   Options options;
   options.create_if_missing = true;
+  options.base_background_compactions = 1;  // default value
   options.max_background_compactions = 1;   // default value
   options.env = env_;
   Reopen(options);
   ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
-  ASSERT_OK(dbfull()->SetDBOptions({{"max_background_compactions", "3"}}));
-  ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
+  ASSERT_OK(dbfull()->SetDBOptions({{"base_background_compactions", "2"},
+                                    {"max_background_compactions", "3"}}));
+  ASSERT_EQ(2, dbfull()->TEST_BGCompactionsAllowed());
   auto stop_token = dbfull()->TEST_write_controler().GetStopToken();
   ASSERT_EQ(3, dbfull()->TEST_BGCompactionsAllowed());
 }
 
-TEST_F(DBOptionsTest, SetBackgroundJobs) {
-  Options options;
-  options.create_if_missing = true;
-  options.max_background_jobs = 8;
-  options.env = env_;
-  Reopen(options);
-
-  for (int i = 0; i < 2; ++i) {
-    if (i > 0) {
-      options.max_background_jobs = 12;
-      ASSERT_OK(dbfull()->SetDBOptions(
-          {{"max_background_jobs",
-            std::to_string(options.max_background_jobs)}}));
-    }
-
-    ASSERT_EQ(options.max_background_jobs / 4,
-              dbfull()->TEST_BGFlushesAllowed());
-    ASSERT_EQ(1, dbfull()->TEST_BGCompactionsAllowed());
-
-    auto stop_token = dbfull()->TEST_write_controler().GetStopToken();
-
-    ASSERT_EQ(options.max_background_jobs / 4,
-              dbfull()->TEST_BGFlushesAllowed());
-    ASSERT_EQ(3 * options.max_background_jobs / 4,
-              dbfull()->TEST_BGCompactionsAllowed());
-  }
-}
-
 TEST_F(DBOptionsTest, AvoidFlushDuringShutdown) {
   Options options;
   options.create_if_missing = true;
@@ -411,36 +383,6 @@ TEST_F(DBOptionsTest, DeleteObsoleteFilesPeriodChange) {
   Close();
 }
 
-TEST_F(DBOptionsTest, MaxOpenFilesChange) {
-  SpecialEnv env(env_);
-  Options options;
-  options.env = CurrentOptions().env;
-  options.max_open_files = -1;
-
-  Reopen(options);
-
-  Cache* tc = dbfull()->TEST_table_cache();
-
-  ASSERT_EQ(-1, dbfull()->GetDBOptions().max_open_files);
-  ASSERT_LT(2000, tc->GetCapacity());
-  ASSERT_OK(dbfull()->SetDBOptions({{"max_open_files", "1024"}}));
-  ASSERT_EQ(1024, dbfull()->GetDBOptions().max_open_files);
-  // examine the table cache (actual size should be 1014)
-  ASSERT_GT(1500, tc->GetCapacity());
-  Close();
-}
-
-TEST_F(DBOptionsTest, SanitizeDelayedWriteRate) {
-  Options options;
-  options.delayed_write_rate = 0;
-  Reopen(options);
-  ASSERT_EQ(16 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
-
-  options.rate_limiter.reset(NewGenericRateLimiter(31 * 1024 * 1024));
-  Reopen(options);
-  ASSERT_EQ(31 * 1024 * 1024, dbfull()->GetDBOptions().delayed_write_rate);
-}
-
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

db/db_properties_test.cc

@@ -1,7 +1,7 @@
 // Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -533,9 +533,9 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
     ASSERT_TRUE(dbfull()->GetProperty(
         handles_[1], "rocksdb.num-entries-active-mem-table", &num));
     ASSERT_EQ(num, "1");
-    get_perf_context()->Reset();
+    perf_context.Reset();
     Get(1, "k1");
-    ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
+    ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count));
 
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k2", big_value));
     ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
@@ -548,12 +548,12 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
         handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
     ASSERT_EQ(num, "1");
 
-    get_perf_context()->Reset();
+    perf_context.Reset();
     Get(1, "k1");
-    ASSERT_EQ(2, static_cast<int>(get_perf_context()->get_from_memtable_count));
-    get_perf_context()->Reset();
+    ASSERT_EQ(2, static_cast<int>(perf_context.get_from_memtable_count));
+    perf_context.Reset();
     Get(1, "k2");
-    ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
+    ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count));
 
     ASSERT_OK(dbfull()->Put(writeOpt, handles_[1], "k3", big_value));
     ASSERT_TRUE(dbfull()->GetProperty(
@@ -567,15 +567,15 @@ TEST_F(DBPropertiesTest, NumImmutableMemTable) {
     ASSERT_TRUE(dbfull()->GetProperty(
         handles_[1], "rocksdb.num-entries-imm-mem-tables", &num));
     ASSERT_EQ(num, "2");
-    get_perf_context()->Reset();
+    perf_context.Reset();
     Get(1, "k2");
-    ASSERT_EQ(2, static_cast<int>(get_perf_context()->get_from_memtable_count));
-    get_perf_context()->Reset();
+    ASSERT_EQ(2, static_cast<int>(perf_context.get_from_memtable_count));
+    perf_context.Reset();
     Get(1, "k3");
-    ASSERT_EQ(1, static_cast<int>(get_perf_context()->get_from_memtable_count));
-    get_perf_context()->Reset();
+    ASSERT_EQ(1, static_cast<int>(perf_context.get_from_memtable_count));
+    perf_context.Reset();
     Get(1, "k1");
-    ASSERT_EQ(3, static_cast<int>(get_perf_context()->get_from_memtable_count));
+    ASSERT_EQ(3, static_cast<int>(perf_context.get_from_memtable_count));
 
     ASSERT_OK(Flush(1));
     ASSERT_TRUE(dbfull()->GetProperty(handles_[1],
@@ -670,7 +670,7 @@ TEST_F(DBPropertiesTest, DISABLED_GetProperty) {
   ASSERT_EQ(num, "0");
   ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
   ASSERT_EQ(num, "1");
-  get_perf_context()->Reset();
+  perf_context.Reset();
 
   ASSERT_OK(dbfull()->Put(writeOpt, "k2", big_value));
   ASSERT_TRUE(dbfull()->GetProperty("rocksdb.num-immutable-mem-table", &num));
@@ -913,7 +913,7 @@ TEST_F(DBPropertiesTest, EstimatePendingCompBytes) {
   Flush();
   ASSERT_TRUE(dbfull()->GetIntProperty(
       "rocksdb.estimate-pending-compaction-bytes", &int_num));
-  ASSERT_GT(int_num, 0U);
+  ASSERT_EQ(int_num, 0U);
 
   ASSERT_OK(dbfull()->Put(writeOpt, "k3", big_value));
   Flush();
@@ -940,6 +940,7 @@ TEST_F(DBPropertiesTest, EstimateCompressionRatio) {
   Options options = CurrentOptions();
   options.compression_per_level = {kNoCompression, kSnappyCompression};
   options.disable_auto_compactions = true;
+  options.max_background_flushes = 0;
   options.num_levels = 2;
   Reopen(options);
 
@@ -1067,6 +1068,7 @@ class CountingDeleteTabPropCollectorFactory
 TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) {
   Options options = CurrentOptions();
   options.level0_file_num_compaction_trigger = (1 << 30);
+  options.max_background_flushes = 0;
   options.table_properties_collector_factories.resize(1);
   std::shared_ptr<CountingUserTblPropCollectorFactory> collector_factory =
       std::make_shared<CountingUserTblPropCollectorFactory>(0);
@@ -1107,6 +1109,7 @@ TEST_F(DBPropertiesTest, GetUserDefinedTableProperties) {
 TEST_F(DBPropertiesTest, UserDefinedTablePropertiesContext) {
   Options options = CurrentOptions();
   options.level0_file_num_compaction_trigger = 3;
+  options.max_background_flushes = 0;
   options.table_properties_collector_factories.resize(1);
   std::shared_ptr<CountingUserTblPropCollectorFactory> collector_factory =
       std::make_shared<CountingUserTblPropCollectorFactory>(1);
@@ -1228,7 +1231,7 @@ TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) {
 
   {
     SetPerfLevel(kEnableCount);
-    get_perf_context()->Reset();
+    perf_context.Reset();
     int c = 0;
     std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
     iter->Seek(Key(kMaxKey - 100));
@@ -1236,8 +1239,8 @@ TEST_F(DBPropertiesTest, TablePropertiesNeedCompactTest) {
       iter->Next();
     }
     ASSERT_EQ(c, 0);
-    ASSERT_LT(get_perf_context()->internal_delete_skipped_count, 30u);
-    ASSERT_LT(get_perf_context()->internal_key_skipped_count, 30u);
+    ASSERT_LT(perf_context.internal_delete_skipped_count, 30u);
+    ASSERT_LT(perf_context.internal_key_skipped_count, 30u);
     SetPerfLevel(kDisable);
   }
 }
@@ -1284,16 +1287,16 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
   ASSERT_EQ(NumTableFilesAtLevel(0), 0);
   {
     SetPerfLevel(kEnableCount);
-    get_perf_context()->Reset();
+    perf_context.Reset();
     int c = 0;
     std::unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
     for (iter->Seek(Key(0)); iter->Valid(); iter->Next()) {
       c++;
     }
     ASSERT_EQ(c, 2);
-    ASSERT_EQ(get_perf_context()->internal_delete_skipped_count, 0);
+    ASSERT_EQ(perf_context.internal_delete_skipped_count, 0);
     // We iterate every key twice. Is it a bug?
-    ASSERT_LE(get_perf_context()->internal_key_skipped_count, 2);
+    ASSERT_LE(perf_context.internal_key_skipped_count, 2);
     SetPerfLevel(kDisable);
   }
 }

db/db_range_del_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2016-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
@@ -27,9 +27,6 @@ class DBRangeDelTest : public DBTestBase {
 // ROCKSDB_LITE
 #ifndef ROCKSDB_LITE
 TEST_F(DBRangeDelTest, NonBlockBasedTableNotSupported) {
-  if (!IsMemoryMappedAccessSupported()) {
-    return;
-  }
   Options opts = CurrentOptions();
   opts.table_factory.reset(new PlainTableFactory());
   opts.prefix_extractor.reset(NewNoopTransform());
@@ -792,7 +789,6 @@ TEST_F(DBRangeDelTest, IteratorIgnoresRangeDeletions) {
   db_->ReleaseSnapshot(snapshot);
 }
 
-#ifndef ROCKSDB_UBSAN_RUN
 TEST_F(DBRangeDelTest, TailingIteratorRangeTombstoneUnsupported) {
   db_->Put(WriteOptions(), "key", "val");
   // snapshot prevents key from being deleted during flush
@@ -819,115 +815,6 @@ TEST_F(DBRangeDelTest, TailingIteratorRangeTombstoneUnsupported) {
   }
   db_->ReleaseSnapshot(snapshot);
 }
-
-#endif  // !ROCKSDB_UBSAN_RUN
-
-TEST_F(DBRangeDelTest, SubcompactionHasEmptyDedicatedRangeDelFile) {
-  const int kNumFiles = 2, kNumKeysPerFile = 4;
-  Options options = CurrentOptions();
-  options.compression = kNoCompression;
-  options.disable_auto_compactions = true;
-  options.level0_file_num_compaction_trigger = kNumFiles;
-  options.max_subcompactions = 2;
-  options.num_levels = 2;
-  options.target_file_size_base = 4096;
-  Reopen(options);
-
-  // need a L1 file for subcompaction to be triggered
-  ASSERT_OK(
-      db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(0), "val"));
-  ASSERT_OK(db_->Flush(FlushOptions()));
-  MoveFilesToLevel(1);
-
-  // put enough keys to fill up the first subcompaction, and later range-delete
-  // them so that the first subcompaction outputs no key-values. In that case
-  // it'll consider making an SST file dedicated to range deletions.
-  for (int i = 0; i < kNumKeysPerFile; ++i) {
-    ASSERT_OK(db_->Put(WriteOptions(), db_->DefaultColumnFamily(), Key(i),
-                       std::string(1024, 'a')));
-  }
-  ASSERT_OK(db_->Flush(FlushOptions()));
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
-                             Key(kNumKeysPerFile)));
-
-  // the above range tombstone can be dropped, so that one alone won't cause a
-  // dedicated file to be opened. We can make one protected by snapshot that
-  // must be considered. Make its range outside the first subcompaction's range
-  // to exercise the tricky part of the code.
-  const Snapshot* snapshot = db_->GetSnapshot();
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(),
-                             Key(kNumKeysPerFile + 1),
-                             Key(kNumKeysPerFile + 2)));
-  ASSERT_OK(db_->Flush(FlushOptions()));
-
-  ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(0));
-  ASSERT_EQ(1, NumTableFilesAtLevel(1));
-
-  db_->EnableAutoCompaction({db_->DefaultColumnFamily()});
-  dbfull()->TEST_WaitForCompact();
-  db_->ReleaseSnapshot(snapshot);
-}
-
-TEST_F(DBRangeDelTest, MemtableBloomFilter) {
-  // regression test for #2743. the range delete tombstones in memtable should
-  // be added even when Get() skips searching due to its prefix bloom filter
-  const int kMemtableSize = 1 << 20;              // 1MB
-  const int kMemtablePrefixFilterSize = 1 << 13;  // 8KB
-  const int kNumKeys = 1000;
-  const int kPrefixLen = 8;
-  Options options = CurrentOptions();
-  options.memtable_prefix_bloom_size_ratio =
-      static_cast<double>(kMemtablePrefixFilterSize) / kMemtableSize;
-  options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(kPrefixLen));
-  options.write_buffer_size = kMemtableSize;
-  Reopen(options);
-
-  for (int i = 0; i < kNumKeys; ++i) {
-    ASSERT_OK(Put(Key(i), "val"));
-  }
-  Flush();
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
-                             Key(kNumKeys)));
-  for (int i = 0; i < kNumKeys; ++i) {
-    std::string value;
-    ASSERT_TRUE(db_->Get(ReadOptions(), Key(i), &value).IsNotFound());
-  }
-}
-
-TEST_F(DBRangeDelTest, UnorderedTombstones) {
-  // Regression test for #2752. Range delete tombstones between
-  // different snapshot stripes are not stored in order, so the first
-  // tombstone of each snapshot stripe should be checked as a smallest
-  // candidate.
-  Options options = CurrentOptions();
-  DestroyAndReopen(options);
-
-  auto cf = db_->DefaultColumnFamily();
-
-  ASSERT_OK(db_->Put(WriteOptions(), cf, "a", "a"));
-  ASSERT_OK(db_->Flush(FlushOptions(), cf));
-  ASSERT_EQ(1, NumTableFilesAtLevel(0));
-  ASSERT_OK(dbfull()->TEST_CompactRange(0, nullptr, nullptr));
-  ASSERT_EQ(1, NumTableFilesAtLevel(1));
-
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "b", "c"));
-  // Hold a snapshot to separate these two delete ranges.
-  auto snapshot = db_->GetSnapshot();
-  ASSERT_OK(db_->DeleteRange(WriteOptions(), cf, "a", "b"));
-  ASSERT_OK(db_->Flush(FlushOptions(), cf));
-  db_->ReleaseSnapshot(snapshot);
-
-  std::vector<std::vector<FileMetaData>> files;
-  dbfull()->TEST_GetFilesMetaData(cf, &files);
-  ASSERT_EQ(1, files[0].size());
-  ASSERT_EQ("a", files[0][0].smallest.user_key());
-  ASSERT_EQ("c", files[0][0].largest.user_key());
-
-  std::string v;
-  auto s = db_->Get(ReadOptions(), "a", &v);
-  ASSERT_TRUE(s.IsNotFound());
-}
-
 #endif  // ROCKSDB_LITE
 
 }  // namespace rocksdb

db/db_sst_test.cc

@@ -1,7 +1,7 @@
 //  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
 //
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
@@ -297,7 +297,7 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   std::vector<uint64_t> penalties;
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "DeleteScheduler::BackgroundEmptyTrash:Wait",
-      [&](void* arg) { penalties.push_back(*(static_cast<uint64_t*>(arg))); });
+      [&](void* arg) { penalties.push_back(*(static_cast<int*>(arg))); });
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
       "InstrumentedCondVar::TimedWaitInternal", [&](void* arg) {
         // Turn timed wait into a simulated sleep
@@ -335,7 +335,6 @@ TEST_F(DBSSTTest, RateLimitedDelete) {
   ASSERT_OK(s);
   options.sst_file_manager->SetDeleteRateBytesPerSecond(rate_bytes_per_sec);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
 
   ASSERT_OK(TryReopen(options));
   // Create 4 files in L0
@@ -401,7 +400,6 @@ TEST_F(DBSSTTest, DeleteSchedulerMultipleDBPaths) {
       env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
   ASSERT_OK(s);
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
 
   DestroyAndReopen(options);
 
@@ -456,14 +454,9 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
       [&](void* arg) { bg_delete_file++; });
   rocksdb::SyncPoint::GetInstance()->EnableProcessing();
 
-  Status s;
   Options options = CurrentOptions();
   options.disable_auto_compactions = true;
   options.env = env_;
-  std::string trash_dir = test::TmpDir(env_) + "/trash";
-  options.sst_file_manager.reset(
-      NewSstFileManager(env_, nullptr, trash_dir, 0, false, &s));
-  ASSERT_OK(s);
   DestroyAndReopen(options);
 
   // Create 4 files in L0
@@ -476,12 +469,15 @@ TEST_F(DBSSTTest, DestroyDBWithRateLimitedDelete) {
 
   // Close DB and destroy it using DeleteScheduler
   Close();
+  std::string trash_dir = test::TmpDir(env_) + "/trash";
+  int64_t rate_bytes_per_sec = 1024 * 1024;  // 1 Mb / Sec
+  Status s;
+  options.sst_file_manager.reset(NewSstFileManager(
+      env_, nullptr, trash_dir, rate_bytes_per_sec, false, &s));
+  ASSERT_OK(s);
+  ASSERT_OK(DestroyDB(dbname_, options));
 
   auto sfm = static_cast<SstFileManagerImpl*>(options.sst_file_manager.get());
-
-  sfm->SetDeleteRateBytesPerSecond(1024 * 1024);
-  sfm->delete_scheduler()->TEST_SetMaxTrashDBRatio(1.1);
-  ASSERT_OK(DestroyDB(dbname_, options));
   sfm->WaitForEmptyTrash();
   // We have deleted the 4 sst files in the delete_scheduler
   ASSERT_EQ(bg_delete_file, 4);