Add Visual Studio 2015 to AppVeyor (#5446)

Summary:
This is required to compile on Windows with Visual Studio 2015, which is used for creating the RocksJava releases.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5446

Differential Revision: D18924811

fbshipit-source-id: a183a62e79a2af5aaf59cd08235458a172fe7dcb

.travis.yml

@@ -70,7 +70,7 @@ install:
       CC=gcc-8 && CXX=g++-8;
     fi
   - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
-      mkdir cmake-dist && curl -sfSL https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH;
+      mkdir cmake-dist && curl --silent --fail --show-error --location https://github.com/Kitware/CMake/releases/download/v3.14.5/cmake-3.14.5-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH;
     fi
   - if [[ "${JOB_NAME}" == java_test ]]; then
       java -version && echo "JAVA_HOME=${JAVA_HOME}";

CMakeLists.txt

@@ -636,6 +636,7 @@ set(SOURCES
         test_util/testutil.cc
         test_util/transaction_test_util.cc
         tools/block_cache_analyzer/block_cache_trace_analyzer.cc
+        tools/db_bench_tool.cc
         tools/dump/db_dump_tool.cc
         tools/ldb_cmd.cc
         tools/ldb_tool.cc
@@ -771,8 +772,7 @@ endif()
 
 set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
 set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
-
-option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" ON)
+set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
 
 option(WITH_LIBRADOS "Build with librados" OFF)
 if(WITH_LIBRADOS)
@@ -783,44 +783,40 @@ endif()
 
 if(WIN32)
   set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
+  set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 else()
   set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
+  set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+
+  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
+  target_link_libraries(${ROCKSDB_SHARED_LIB}
+    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+  set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
+                        LINKER_LANGUAGE CXX
+                        VERSION ${rocksdb_VERSION}
+                        SOVERSION ${rocksdb_VERSION_MAJOR}
+                        CXX_STANDARD 11
+                        OUTPUT_NAME "rocksdb")
 endif()
 
 add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
 target_link_libraries(${ROCKSDB_STATIC_LIB}
   ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
-if(ROCKSDB_BUILD_SHARED)
-  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
-  target_link_libraries(${ROCKSDB_SHARED_LIB}
+if(WIN32)
+  add_library(${ROCKSDB_IMPORT_LIB} SHARED ${SOURCES})
+  target_link_libraries(${ROCKSDB_IMPORT_LIB}
     ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-
-  if(WIN32)
-    set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
-      COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS")
-    if(MSVC)
-      set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES
-        COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb")
-      set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
-        COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_SHARED_LIB}.pdb")
-    endif()
-  else()
-    set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
-                          LINKER_LANGUAGE CXX
-                          VERSION ${rocksdb_VERSION}
-                          SOVERSION ${rocksdb_VERSION_MAJOR}
-                          CXX_STANDARD 11
-                          OUTPUT_NAME "rocksdb")
+  set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
+    COMPILE_DEFINITIONS "ROCKSDB_DLL;ROCKSDB_LIBRARY_EXPORTS")
+  if(MSVC)
+    set_target_properties(${ROCKSDB_STATIC_LIB} PROPERTIES
+      COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_STATIC_LIB}.pdb")
+    set_target_properties(${ROCKSDB_IMPORT_LIB} PROPERTIES
+      COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/${ROCKSDB_IMPORT_LIB}.pdb")
   endif()
 endif()
 
-if(ROCKSDB_BUILD_SHARED AND NOT WIN32)
-  set(ROCKSDB_LIB ${ROCKSDB_SHARED_LIB})
-else()
-  set(ROCKSDB_LIB ${ROCKSDB_STATIC_LIB})
-endif()
-
 option(WITH_JNI "build with JNI" OFF)
 if(WITH_JNI OR JNI)
   message(STATUS "JNI library is enabled")
@@ -867,17 +863,15 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
     INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
   )
 
-  if(ROCKSDB_BUILD_SHARED)
-    install(
-      TARGETS ${ROCKSDB_SHARED_LIB}
-      EXPORT RocksDBTargets
-      COMPONENT runtime
-      ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-      RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-      LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-      INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-    )
-  endif()
+  install(
+    TARGETS ${ROCKSDB_SHARED_LIB}
+    EXPORT RocksDBTargets
+    COMPONENT runtime
+    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
+    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+  )
 
   install(
     EXPORT RocksDBTargets
@@ -1064,7 +1058,7 @@ if(WITH_TESTS AND CMAKE_BUILD_TYPE STREQUAL "Debug")
   add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND})
   set(TESTUTILLIB testutillib${ARTIFACT_SUFFIX})
   add_library(${TESTUTILLIB} STATIC ${TESTUTIL_SOURCE})
-  target_link_libraries(${TESTUTILLIB} ${ROCKSDB_LIB})
+  target_link_libraries(${TESTUTILLIB} ${LIBS})
   if(MSVC)
     set_target_properties(${TESTUTILLIB} PROPERTIES COMPILE_FLAGS "/Fd${CMAKE_CFG_INTDIR}/testutillib${ARTIFACT_SUFFIX}.pdb")
   endif()
@@ -1085,42 +1079,30 @@ if(WITH_TESTS AND CMAKE_BUILD_TYPE STREQUAL "Debug")
         EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
         OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
         )
-      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${ROCKSDB_LIB})
+      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} gtest ${LIBS})
       if(NOT "${exename}" MATCHES "db_sanity_test")
         add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
         add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
       endif()
   endforeach(sourcefile ${TEST_EXES})
 
-  if(WIN32)
-    # C executables must link to a shared object
-    if(ROCKSDB_BUILD_SHARED)
-      set(ROCKSDB_LIB_FOR_C ${ROCKSDB_SHARED_LIB})
-    else()
-      set(ROCKSDB_LIB_FOR_C OFF)
-    endif()
-  else()
-    set(ROCKSDB_LIB_FOR_C ${ROCKSDB_LIB})
-  endif()
+  # C executables must link to a shared object
+  set(C_TESTS db/c_test.c)
+  set(C_TEST_EXES ${C_TESTS})
 
-  if(ROCKSDB_LIB_FOR_C)
-    set(C_TESTS db/c_test.c)
-    set(C_TEST_EXES ${C_TESTS})
-
-    foreach(sourcefile ${C_TEST_EXES})
-        string(REPLACE ".c" "" exename ${sourcefile})
-        string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
-        add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-        set_target_properties(${exename}${ARTIFACT_SUFFIX}
-          PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
-          EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
-          EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-          )
-        target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_LIB_FOR_C} testutillib${ARTIFACT_SUFFIX})
-        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
-        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-    endforeach(sourcefile ${C_TEST_EXES})
-  endif()
+  foreach(sourcefile ${C_TEST_EXES})
+      string(REPLACE ".c" "" exename ${sourcefile})
+      string(REGEX REPLACE "^((.+)/)+" "" exename ${exename})
+      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+      set_target_properties(${exename}${ARTIFACT_SUFFIX}
+        PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
+        EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
+        EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
+        )
+      target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_IMPORT_LIB} testutillib${ARTIFACT_SUFFIX})
+      add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
+      add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
+  endforeach(sourcefile ${C_TEST_EXES})
 endif()
 
 option(WITH_BENCHMARK_TOOLS "build with benchmarks" ON)
@@ -1132,6 +1114,7 @@ if(WITH_BENCHMARK_TOOLS)
         cache/cache_bench.cc
         memtable/memtablerep_bench.cc
         db/range_del_aggregator_bench.cc
+        tools/db_bench.cc
         table/table_reader_bench.cc
         util/filter_bench.cc
         utilities/persistent_cache/hash_table_bench.cc
@@ -1141,20 +1124,11 @@ if(WITH_BENCHMARK_TOOLS)
     get_filename_component(exename ${sourcefile} NAME_WE)
     add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile}
       $<TARGET_OBJECTS:testharness>)
-    target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${ROCKSDB_LIB})
+    target_link_libraries(${exename}${ARTIFACT_SUFFIX} gtest ${LIBS})
   endforeach(sourcefile ${BENCHMARKS})
-
-  add_executable(db_bench${ARTIFACT_SUFFIX}
-    tools/db_bench.cc
-    tools/db_bench_tool.cc
-    $<TARGET_OBJECTS:testharness>)
-  target_link_libraries(db_bench${ARTIFACT_SUFFIX} gtest ${ROCKSDB_LIB})
 endif()
 
 option(WITH_TOOLS "build with tools" ON)
 if(WITH_TOOLS)
   add_subdirectory(tools)
-  add_subdirectory(db_stress_tool)
-  add_custom_target(tools
-    DEPENDS ${tool_deps})
 endif()

HISTORY.md

@@ -1,13 +1,34 @@
 # Rocksdb Change Log
 ## Unreleased
-### Buf Fixes
-* Fix a bug that can cause unnecessary bg thread to be scheduled(#6104).
+
+## 6.6.4 (1/31/2020)
+### Bug Fixes
+* Fixed issue #6316 that can cause a corruption of the MANIFEST file in the middle when writing to it fails due to no disk space.
+
+## 6.6.3 (01/24/2020)
+### Bug Fixes
+* Fix a bug that can cause write threads to hang when a slowdown/stall happens and there is a mix of writers with WriteOptions::no_slowdown set/unset.
+
+## 6.6.2 (01/13/2020)
+### Bug Fixes
+* Fixed a bug where non-L0 compaction input files were not considered to compute the `creation_time` of new compaction outputs.
+
+## 6.6.1 (01/02/2020)
+### Bug Fixes
+* Fix a bug in WriteBatchWithIndex::MultiGetFromBatchAndDB, which is called by Transaction::MultiGet, that causes due to stale pointer access when the number of keys is > 32
+* Fixed two performance issues related to memtable history trimming. First, a new SuperVersion is now created only if some memtables were actually trimmed. Second, trimming is only scheduled if there is at least one flushed memtable that is kept in memory for the purposes of transaction conflict checking.
+* BlobDB no longer updates the SST to blob file mapping upon failed compactions.
+* Fix a bug in which a snapshot read through an iterator could be affected by a DeleteRange after the snapshot (#6062).
+* Fixed a bug where BlobDB was comparing the `ColumnFamilyHandle` pointers themselves instead of only the column family IDs when checking whether an API call uses the default column family or not.
+* Delete superversions in BackgroundCallPurge.
+* Fix use-after-free and double-deleting files in BackgroundCallPurge().
+
 ## 6.6.0 (11/25/2019)
 ### Bug Fixes
-* Fix data corruption caused by output of intra-L0 compaction on ingested file not being placed in correct order in L0.
+* Fix data corruption casued by output of intra-L0 compaction on ingested file not being placed in correct order in L0.
 * Fix a data race between Version::GetColumnFamilyMetaData() and Compaction::MarkFilesBeingCompacted() for access to being_compacted (#6056). The current fix acquires the db mutex during Version::GetColumnFamilyMetaData(), which may cause regression.
 * Fix a bug in DBIter that is_blob_ state isn't updated when iterating backward using seek.
-* Fix a bug when format_version=3, partitioned filters, and prefix search are used in conjunction. The bug could result into Seek::(prefix) returning NotFound for an existing prefix.
+* Fix a bug when format_version=3, partitioned fitlers, and prefix search are used in conjunction. The bug could result into Seek::(prefix) returning NotFound for an existing prefix.
 * Revert the feature "Merging iterator to avoid child iterator reseek for some cases (#5286)" since it might cause strong results when reseek happens with a different iterator upper bound.
 * Fix a bug causing a crash during ingest external file when background compaction cause severe error (file not found).
 * Fix a bug when partitioned filters and prefix search are used in conjunction, ::SeekForPrev could return invalid for an existing prefix. ::SeekForPrev might be called by the user, or internally on ::Prev, or within ::Seek if the return value involves Delete or a Merge operand.
@@ -20,13 +41,13 @@
 * Universal compaction to support options.periodic_compaction_seconds. A full compaction will be triggered if any file is over the threshold.
 * `GetLiveFilesMetaData` and `GetColumnFamilyMetaData` now expose the file number of SST files as well as the oldest blob file referenced by each SST.
 * A batched MultiGet API (DB::MultiGet()) that supports retrieving keys from multiple column families.
-* Full and partitioned filters in the block-based table use an improved Bloom filter implementation, enabled with format_version 5 (or above) because previous releases cannot read this filter. This replacement is faster and more accurate, especially for high bits per key or millions of keys in a single (full) filter. For example, the new Bloom filter has the same false positive rate at 9.55 bits per key as the old one at 10 bits per key, and a lower false positive rate at 16 bits per key than the old one at 100 bits per key.
+* Full and partitioned filters in the block-based table use an improved Bloom filter implementation, enabled with format_version 5 (or above) because previous releases cannot read this filter. This replacement is faster and more accurate, especially for high bits per key or millions of keys in a single (full) filter. For example, the new Bloom filter has the same false postive rate at 9.55 bits per key as the old one at 10 bits per key, and a lower false positive rate at 16 bits per key than the old one at 100 bits per key.
 * Added AVX2 instructions to USE_SSE builds to accelerate the new Bloom filter and XXH3-based hash function on compatible x86_64 platforms (Haswell and later, ~2014).
 * Support options.ttl or options.periodic_compaction_seconds with options.max_open_files = -1. File's oldest ancester time and file creation time will be written to manifest. If it is availalbe, this information will be used instead of creation_time and file_creation_time in table properties.
 * Setting options.ttl for universal compaction now has the same meaning as setting periodic_compaction_seconds.
 * SstFileMetaData also returns file creation time and oldest ancester time.
 * The `sst_dump` command line tool `recompress` command now displays how many blocks were compressed and how many were not, in particular how many were not compressed because the compression ratio was not met (12.5% threshold for GoodCompressionRatio), as seen in the `number.block.not_compressed` counter stat since version 6.0.0.
-* The block cache usage is now takes into account the overhead of metadata per each entry. This results into more accurate management of memory. A side-effect of this feature is that less items are fit into the block cache of the same size, which would result to higher cache miss rates. This can be remedied by increasing the block cache size or passing kDontChargeCacheMetadata to its constuctor to restore the old behavior.
+* The block cache usage is now takes into account the overhead of metadata per each entry. This results into more accurate managment of memory. A side-effect of this feature is that less items are fit into the block cache of the same size, which would result to higher cache miss rates. This can be remedied by increasing the block cache size or passing kDontChargeCacheMetadata to its constuctor to restore the old behavior.
 * When using BlobDB, a mapping is maintained and persisted in the MANIFEST between each SST file and the oldest non-TTL blob file it references.
 * `db_bench` now supports and by default issues non-TTL Puts to BlobDB. TTL Puts can be enabled by specifying a non-zero value for the `blob_db_max_ttl_range` command line parameter explicitly.
 * `sst_dump` now supports printing BlobDB blob indexes in a human-readable format. This can be enabled by specifying the `decode_blob_index` flag on the command line.
@@ -49,7 +70,7 @@
 
 ### Default Option Changes
 * Changed the default value of periodic_compaction_seconds to `UINT64_MAX - 1` which allows RocksDB to auto-tune periodic compaction scheduling. When using the default value, periodic compactions are now auto-enabled if a compaction filter is used. A value of `0` will turn off the feature completely.
-* Changed the default value of ttl to `UINT64_MAX - 1` which allows RocksDB to auto-tune ttl value. When using the default value, TTL will be auto-enabled to 30 days, when the feature is supported. To revert the old behavior, you can explicitly set it to 0.
+* Changed the default value of ttl to `UINT64_MAX - 1` which allows RocksDB to auto-tune ttl value. When using the default value, TTL will be auto-enabled to 30 days, when the feature is supported. To revert the old behavior, you can explictly set it to 0.
 
 ### Performance Improvements
 * For 64-bit hashing, RocksDB is standardizing on a slightly modified preview version of XXH3. This function is now used for many non-persisted hashes, along with fastrange64() in place of the modulus operator, and some benchmarks show a slight improvement.
@@ -90,7 +111,7 @@
 
 ## 6.4.0 (7/30/2019)
 ### Default Option Change
-* LRUCacheOptions.high_pri_pool_ratio is set to 0.5 (previously 0.0) by default, which means that by default midpoint insertion is enabled. The same change is made for the default value of high_pri_pool_ratio argument in NewLRUCache(). When block cache is not explicitly created, the small block cache created by BlockBasedTable will still has this option to be 0.0.
+* LRUCacheOptions.high_pri_pool_ratio is set to 0.5 (previously 0.0) by default, which means that by default midpoint insertion is enabled. The same change is made for the default value of high_pri_pool_ratio argument in NewLRUCache(). When block cache is not explictly created, the small block cache created by BlockBasedTable will still has this option to be 0.0.
 * Change BlockBasedTableOptions.cache_index_and_filter_blocks_with_high_priority's default value from false to true.
 
 ### Public API Change
@@ -114,7 +135,7 @@
 * Support loading custom objects in unit tests. In the affected unit tests, RocksDB will create custom Env objects based on environment variable TEST_ENV_URI. Users need to make sure custom object types are properly registered. For example, a static library should expose a `RegisterCustomObjects` function. By linking the unit test binary with the static library, the unit test can execute this function.
 
 ### Performance Improvements
-* Reduce iterator key comparison for upper/lower bound check.
+* Reduce iterator key comparision for upper/lower bound check.
 * Improve performance of row_cache: make reads with newer snapshots than data in an SST file share the same cache key, except in some transaction cases.
 * The compression dictionary is no longer copied to a new object upon retrieval.
 
@@ -210,7 +231,7 @@
 * Introduce two more stats levels, kExceptHistogramOrTimers and kExceptTimers.
 * Added a feature to perform data-block sampling for compressibility, and report stats to user.
 * Add support for trace filtering.
-* Add DBOptions.avoid_unnecessary_blocking_io. If true, we avoid file deletion when destroying ColumnFamilyHandle and Iterator. Instead, a job is scheduled to delete the files in background.
+* Add DBOptions.avoid_unnecessary_blocking_io. If true, we avoid file deletion when destorying ColumnFamilyHandle and Iterator. Instead, a job is scheduled to delete the files in background.
 
 ### Public API Change
 * Remove bundled fbson library.
@@ -589,7 +610,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 
 ## 5.2.0 (02/08/2017)
 ### Public API Change
-* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explicit provide one.
+* NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explict provide one.
 * Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
 * Options::use_direct_writes and Options::use_direct_reads are now ready to use.
 * (Experimental) Two-level indexing that partition the index and creates a 2nd level index on the partitions. The feature can be enabled by setting kTwoLevelIndexSearch as IndexType and configuring index_per_partition.
@@ -660,7 +681,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 
 ### New Features
 * A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h.
-* Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destroying iterators.
+* Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destorying iterators.
 
 ## 4.10.0 (7/5/2016)
 ### Public API Change

Makefile

@@ -743,8 +743,7 @@ endif  # PLATFORM_SHARED_EXT
 	release tags tags0 valgrind_check whitebox_crash_test format static_lib shared_lib all \
 	dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
 	analyze tools tools_lib \
-	blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush  \
-	blackbox_crash_test_with_txn whitebox_crash_test_with_txn 
+	blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush
 
 
 all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
@@ -956,8 +955,6 @@ crash_test: whitebox_crash_test blackbox_crash_test
 
 crash_test_with_atomic_flush: whitebox_crash_test_with_atomic_flush blackbox_crash_test_with_atomic_flush
 
-crash_test_with_txn: whitebox_crash_test_with_txn blackbox_crash_test_with_txn
-
 blackbox_crash_test: db_stress
 	python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
 	python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
@@ -965,9 +962,6 @@ blackbox_crash_test: db_stress
 blackbox_crash_test_with_atomic_flush: db_stress
 	python -u tools/db_crashtest.py --cf_consistency blackbox $(CRASH_TEST_EXT_ARGS)
 
-blackbox_crash_test_with_txn: db_stress
-	python -u tools/db_crashtest.py --txn blackbox $(CRASH_TEST_EXT_ARGS)
-
 ifeq ($(CRASH_TEST_KILL_ODD),)
   CRASH_TEST_KILL_ODD=888887
 endif
@@ -982,10 +976,6 @@ whitebox_crash_test_with_atomic_flush: db_stress
 	python -u tools/db_crashtest.py --cf_consistency whitebox  --random_kill_odd \
       $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
 
-whitebox_crash_test_with_txn: db_stress
-	python -u tools/db_crashtest.py --txn whitebox --random_kill_odd \
-      $(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
-
 asan_check:
 	$(MAKE) clean
 	COMPILE_WITH_ASAN=1 $(MAKE) check -j32
@@ -1001,11 +991,6 @@ asan_crash_test_with_atomic_flush:
 	COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_atomic_flush
 	$(MAKE) clean
 
-asan_crash_test_with_txn:
-	$(MAKE) clean
-	COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_txn
-	$(MAKE) clean
-
 ubsan_check:
 	$(MAKE) clean
 	COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
@@ -1021,11 +1006,6 @@ ubsan_crash_test_with_atomic_flush:
 	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_atomic_flush
 	$(MAKE) clean
 
-ubsan_crash_test_with_txn:
-	$(MAKE) clean
-	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_txn
-	$(MAKE) clean
-
 valgrind_test:
 	ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check
 
@@ -1123,16 +1103,21 @@ unity_test: db/db_test.o db/db_test_util.o $(TESTHARNESS) $(TOOLLIBOBJECTS) unit
 rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc
 	build_tools/amalgamate.py -I. -i./include unity.cc -x include/rocksdb/c.h -H rocksdb.h -o rocksdb.cc
 
-clean: clean-ext-libraries-all clean-rocks
+clean: clean-ext-libraries-all clean-rocks clean-rocksjava
 
-clean-not-downloaded: clean-ext-libraries-bin clean-rocks
+clean-not-downloaded: clean-ext-libraries-bin clean-rocks clean-not-downloaded-rocksjava
 
 clean-rocks:
 	rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(LIBRARY) $(SHARED)
 	rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
 	$(FIND) . -name "*.[oda]" -exec rm -f {} \;
 	$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
-	cd java; $(MAKE) clean
+
+clean-rocksjava:
+	cd java && $(MAKE) clean
+
+clean-not-downloaded-rocksjava:
+	cd java && $(MAKE) clean-not-downloaded
 
 clean-ext-libraries-all:
 	rm -rf bzip2* snappy* zlib* lz4* zstd*
@@ -1203,7 +1188,7 @@ memtablerep_bench: memtable/memtablerep_bench.o $(LIBOBJECTS) $(TESTUTIL)
 filter_bench: util/filter_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(AM_LINK)
 
-db_stress: db_stress_tool/db_stress.o $(STRESSTOOLOBJECTS)
+db_stress: tools/db_stress.o $(STRESSTOOLOBJECTS)
 	$(AM_LINK)
 
 write_stress: tools/write_stress.o $(LIBOBJECTS) $(TESTUTIL)
@@ -1826,7 +1811,7 @@ endif
 libz.a:
 	-rm -rf zlib-$(ZLIB_VER)
 ifeq (,$(wildcard ./zlib-$(ZLIB_VER).tar.gz))
-	curl --output zlib-$(ZLIB_VER).tar.gz -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
+	curl --fail --output zlib-$(ZLIB_VER).tar.gz --location ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
 endif
 	ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \
@@ -1840,7 +1825,7 @@ endif
 libbz2.a:
 	-rm -rf bzip2-$(BZIP2_VER)
 ifeq (,$(wildcard ./bzip2-$(BZIP2_VER).tar.gz))
-	curl --output bzip2-$(BZIP2_VER).tar.gz -L ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz
+	curl --fail --output bzip2-$(BZIP2_VER).tar.gz --location ${CURL_SSL_OPTS} ${BZIP2_DOWNLOAD_BASE}/bzip2-$(BZIP2_VER).tar.gz
 endif
 	BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \
@@ -1854,7 +1839,7 @@ endif
 libsnappy.a:
 	-rm -rf snappy-$(SNAPPY_VER)
 ifeq (,$(wildcard ./snappy-$(SNAPPY_VER).tar.gz))
-	curl --output snappy-$(SNAPPY_VER).tar.gz -L ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz
+	curl --fail --output snappy-$(SNAPPY_VER).tar.gz --location ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz
 endif
 	SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
@@ -1869,7 +1854,7 @@ endif
 liblz4.a:
 	-rm -rf lz4-$(LZ4_VER)
 ifeq (,$(wildcard ./lz4-$(LZ4_VER).tar.gz))
-	curl --output lz4-$(LZ4_VER).tar.gz -L ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
+	curl --fail --output lz4-$(LZ4_VER).tar.gz --location ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
 endif
 	LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
@@ -1883,7 +1868,7 @@ endif
 libzstd.a:
 	-rm -rf zstd-$(ZSTD_VER)
 ifeq (,$(wildcard ./zstd-$(ZSTD_VER).tar.gz))
-	curl --output zstd-$(ZSTD_VER).tar.gz -L ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
+	curl --fail --output zstd-$(ZSTD_VER).tar.gz --location ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
 endif
 	ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \
 	if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \
@@ -1953,35 +1938,35 @@ rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 roc
 
 rocksdbjavastaticdockerx86:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x86-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_x86-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
 
 rocksdbjavastaticdockerx86_64:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_x64-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
 
 rocksdbjavastaticdockerppc64le:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_ppc64le-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
 
 rocksdbjavastaticdockerarm64v8:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_arm64v8-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:centos7_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
 
 rocksdbjavastaticdockerx86musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x86-musl-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_x86-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
 
 rocksdbjavastaticdockerx86_64musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_x64-musl-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_x64-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
 
 rocksdbjavastaticdockerppc64lemusl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_ppc64le-musl-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_ppc64le-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_ppc64le-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
 
 rocksdbjavastaticdockerarm64v8musl:
 	mkdir -p java/target
-	docker run --rm --name rocksdb_linux_arm64v8-musl-be --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
+	docker run --rm --name rocksdb_linux_arm64v8-musl-be --attach stdin --attach stdout --attach stderr --volume $(HOME)/.m2:/root/.m2:ro --volume `pwd`:/rocksdb-host:ro --volume /rocksdb-local-build --volume `pwd`/java/target:/rocksdb-java-target --env DEBUG_LEVEL=$(DEBUG_LEVEL) evolvedbinary/rocksjava:alpine3_arm64v8-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh
 
 rocksdbjavastaticpublish: rocksdbjavastaticrelease rocksdbjavastaticpublishcentral
 
@@ -2094,9 +2079,6 @@ endif
 .cc.o:
 	$(AM_V_CC)$(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS)
 
-.cpp.o:
-	$(AM_V_CC)$(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS)
-
 .c.o:
 	$(AM_V_CC)$(CC) $(CFLAGS) -c $< -o $@
 endif
@@ -2107,10 +2089,6 @@ endif
 all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(ANALYZER_LIB_SOURCES) $(STRESS_LIB_SOURCES)
 DEPFILES = $(all_sources:.cc=.cc.d)
 
-ifeq ($(USE_FOLLY_DISTRIBUTED_MUTEX),1)
-  DEPFILES += $(FOLLY_SOURCES:.cpp=.cpp.d)
-endif
-
 # Add proper dependency support so changing a .h file forces a .cc file to
 # rebuild.
 
@@ -2120,10 +2098,6 @@ endif
 	@$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \
 	  -MM -MT'$@' -MT'$(<:.cc=.o)' "$<" -o '$@'
 
-%.cpp.d: %.cpp
-	@$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) \
-	  -MM -MT'$@' -MT'$(<:.cpp=.o)' "$<" -o '$@'
-
 ifeq ($(HAVE_POWER8),1)
 DEPFILES_C = $(LIB_SOURCES_C:.c=.c.d)
 DEPFILES_ASM = $(LIB_SOURCES_ASM:.S=.S.d)

TARGETS

@@ -26,7 +26,6 @@ ROCKSDB_EXTERNAL_DEPS = [
     ("lz4", None, "lz4"),
     ("zstd", None),
     ("tbb", None),
-    ("liburing", None, "uring"),
     ("googletest", None, "gtest"),
 ]
 
@@ -47,7 +46,6 @@ ROCKSDB_OS_PREPROCESSOR_FLAGS = [
             "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
             "-DROCKSDB_RANGESYNC_PRESENT",
             "-DROCKSDB_SCHED_GETCPU_PRESENT",
-            "-DROCKSDB_IOURING_PRESENT",
             "-DHAVE_SSE42",
             "-DNUMA",
         ],
@@ -72,7 +70,6 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
     "-DZSTD_STATIC_LINKING_ONLY",
     "-DGFLAGS=gflags",
     "-DTBB",
-    "-DLIBURING",
 
     # Added missing flags from output of build_detect_platform
     "-DROCKSDB_BACKTRACE",
@@ -402,17 +399,9 @@ cpp_library(
 cpp_library(
     name = "rocksdb_stress_lib",
     srcs = [
-        "db_stress_tool/batched_ops_stress.cc",
-        "db_stress_tool/cf_consistency_stress.cc",
-        "db_stress_tool/db_stress_common.cc",
-        "db_stress_tool/db_stress_driver.cc",
-        "db_stress_tool/db_stress_gflags.cc",
-        "db_stress_tool/db_stress_shared_state.cc",
-        "db_stress_tool/db_stress_test_base.cc",
-        "db_stress_tool/db_stress_tool.cc",
-        "db_stress_tool/no_batched_ops_stress.cc",
         "test_util/testutil.cc",
         "tools/block_cache_analyzer/block_cache_trace_analyzer.cc",
+        "tools/db_stress_tool.cc",
         "tools/trace_analyzer_tool.cc",
     ],
     auto_headers = AutoHeaders.RECURSIVE_GLOB,

appveyor.yml

@@ -29,7 +29,7 @@ install:
   - md %THIRDPARTY_HOME%
   - echo "Building Snappy dependency..."
   - cd %THIRDPARTY_HOME%
-  - curl -fsSL -o snappy-1.1.7.zip https://github.com/google/snappy/archive/1.1.7.zip
+  - curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
   - unzip snappy-1.1.7.zip
   - cd snappy-1.1.7
   - mkdir build
@@ -39,7 +39,7 @@ install:
   - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
   - echo "Building LZ4 dependency..."
   - cd %THIRDPARTY_HOME%
-  - curl -fsSL -o lz4-1.8.3.zip https://github.com/lz4/lz4/archive/v1.8.3.zip
+  - curl --fail --silent --show-error --output lz4-1.8.3.zip --location https://github.com/lz4/lz4/archive/v1.8.3.zip
   - unzip lz4-1.8.3.zip
   - cd lz4-1.8.3\visual\VS2010
   - ps: $CMD="$Env:DEV_ENV"; & $CMD lz4.sln /upgrade
@@ -47,7 +47,7 @@ install:
   - msbuild lz4.sln /p:Configuration=Release /p:Platform=x64
   - echo "Building ZStd dependency..."
   - cd %THIRDPARTY_HOME%
-  - curl -fsSL -o zstd-1.4.0.zip https://github.com/facebook/zstd/archive/v1.4.0.zip
+  - curl --fail --silent --show-error --output zstd-1.4.0.zip --location https://github.com/facebook/zstd/archive/v1.4.0.zip
   - unzip zstd-1.4.0.zip
   - cd zstd-1.4.0\build\VS2010
   - ps: $CMD="$Env:DEV_ENV"; & $CMD zstd.sln /upgrade

buckifier/targets_cfg.py

@@ -32,7 +32,6 @@ ROCKSDB_EXTERNAL_DEPS = [
     ("lz4", None, "lz4"),
     ("zstd", None),
     ("tbb", None),
-    ("liburing", None, "uring"),
     ("googletest", None, "gtest"),
 ]
 
@@ -53,7 +52,6 @@ ROCKSDB_OS_PREPROCESSOR_FLAGS = [
             "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
             "-DROCKSDB_RANGESYNC_PRESENT",
             "-DROCKSDB_SCHED_GETCPU_PRESENT",
-            "-DROCKSDB_IOURING_PRESENT",
             "-DHAVE_SSE42",
             "-DNUMA",
         ],
@@ -78,7 +76,6 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
     "-DZSTD_STATIC_LINKING_ONLY",
     "-DGFLAGS=gflags",
     "-DTBB",
-    "-DLIBURING",
 
     # Added missing flags from output of build_detect_platform
     "-DROCKSDB_BACKTRACE",

build_tools/build_detect_platform

@@ -150,19 +150,6 @@ case "$TARGET_OS" in
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
         fi
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
-        # check for liburing
-        $CXX $CFLAGS -x c++ - -luring -o /dev/null 2>/dev/null  <<EOF
-          #include <liburing.h>
-          int main() {
-            struct io_uring ring;
-            io_uring_queue_init(1, &ring, 0);
-            return 0;
-          }
-EOF
-        if [ "$?" = 0 ]; then
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -luring"
-          COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_IOURING_PRESENT"
-        fi
         if test -z "$USE_FOLLY_DISTRIBUTED_MUTEX"; then
           USE_FOLLY_DISTRIBUTED_MUTEX=1
         fi

build_tools/dependencies_platform007.sh

@@ -13,7 +13,6 @@ JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b86
 NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
 LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
 TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
-LIBURING_BASE=/mnt/gvfs/third-party2/liburing/79427253fd0d42677255aacfe6d13bfe63f752eb/20190828/platform007/ca4da3d
 KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
 BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
 VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d

build_tools/error_filter.py

@@ -133,16 +133,13 @@ _TEST_NAME_TO_PARSERS = {
     'lite_test': [CompilerErrorParser, GTestErrorParser],
     'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
     'stress_crash_with_atomic_flush': [CompilerErrorParser, DbCrashErrorParser],
-    'stress_crash_with_txn': [CompilerErrorParser, DbCrashErrorParser],
     'write_stress': [CompilerErrorParser, WriteStressErrorParser],
     'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
     'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
     'asan_crash_with_atomic_flush': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
-    'asan_crash_with_txn': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
     'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
     'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
     'ubsan_crash_with_atomic_flush': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
-    'ubsan_crash_with_txn': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
     'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
     'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
     'format_compatible': [CompilerErrorParser, CompatErrorParser],

build_tools/fbcode_config_platform007.sh

@@ -86,15 +86,6 @@ else
 fi
 CFLAGS+=" -DTBB"
 
-# location of LIBURING
-LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
-if test -z $PIC_BUILD; then
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
-else
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
-fi
-CFLAGS+=" -DLIBURING"
-
 test "$USE_SSE" || USE_SSE=1
 export USE_SSE
 test "$PORTABLE" || PORTABLE=1
@@ -103,7 +94,7 @@ export PORTABLE
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
 
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
 
 STDLIBS="-L $GCC_BASE/lib64"
 
@@ -144,10 +135,10 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"
 
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
 EXEC_LDFLAGS+=" -B$BINUTILS/gold"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
@@ -157,7 +148,7 @@ EXEC_LDFLAGS+=" -ldl"
 
 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
 
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
 
 VALGRIND_VER="$VALGRIND_BASE/bin/"
 

build_tools/regression_build_test.sh

@@ -378,7 +378,7 @@ function send_to_ods {
     echo >&2 "ERROR: Key $key doesn't have a value."
     return
   fi
-  curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \
+  curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \
     --connect-timeout 60
 }
 

build_tools/rocksdb-lego-determinator

@@ -450,36 +450,6 @@ STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
     }
 ]"
 
-#
-# RocksDB stress/crash test with txn
-#
-STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[
-    {
-        'name':'Rocksdb Stress and Crash Test with txn',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug stress tests',
-                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            {
-                'name':'Build and run RocksDB debug crash tests with txn',
-                'timeout': 86400,
-                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
 # RocksDB write stress test.
 # We run on disk device on purpose (i.e. no $SHM)
 # because we want to add some randomness to fsync commands
@@ -571,30 +541,6 @@ ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
     }
 ]"
 
-#
-# RocksDB crash testing with txn under address sanitizer
-#
-ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[
-    {
-        'name':'Rocksdb crash test with txn under ASAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug asan_crash_test_with_txn',
-                'timeout': 86400,
-                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
 #
 # RocksDB test under undefined behavior sanitizer
 #
@@ -663,30 +609,6 @@ UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
     }
 ]"
 
-#
-# RocksDB crash testing with txn under undefined behavior sanitizer
-#
-UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
-    {
-        'name':'Rocksdb crash test with txn under UBSAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug ubsan_crash_test_with_txn',
-                'timeout': 86400,
-                'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
 #
 # RocksDB unit test under valgrind
 #
@@ -780,30 +702,6 @@ TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
     }
 ]"
 
-#
-# RocksDB crash test with txn under TSAN
-#
-TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
-    {
-        'name':'Rocksdb Crash Test with txn under TSAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Compile and run',
-                'timeout': 86400,
-                'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
 #
 # RocksDB format compatible
 #
@@ -880,7 +778,7 @@ run_regression()
 
   # parameters: $1 -- key, $2 -- value
   function send_size_to_ods {
-    curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
+    curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
       --connect-timeout 60
   }
 
@@ -991,9 +889,6 @@ case $1 in
   stress_crash_with_atomic_flush)
     echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
     ;;
-  stress_crash_with_txn)
-    echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS
-    ;;
   write_stress)
     echo $WRITE_STRESS_COMMANDS
     ;;
@@ -1006,9 +901,6 @@ case $1 in
   asan_crash_with_atomic_flush)
     echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
     ;;
-  asan_crash_with_txn)
-    echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS
-    ;;
   ubsan)
     echo $UBSAN_TEST_COMMANDS
     ;;
@@ -1018,9 +910,6 @@ case $1 in
   ubsan_crash_with_atomic_flush)
     echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
     ;;
-  ubsan_crash_with_txn)
-    echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS
-    ;;
   valgrind)
     echo $VALGRIND_TEST_COMMANDS
     ;;
@@ -1033,9 +922,6 @@ case $1 in
   tsan_crash_with_atomic_flush)
     echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
     ;;
-  tsan_crash_with_txn)
-    echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS
-    ;;
   format_compatible)
     echo $FORMAT_COMPATIBLE_COMMANDS
     ;;

build_tools/update_dependencies.sh

@@ -92,7 +92,6 @@ get_lib_base jemalloc   LATEST  platform007
 get_lib_base numa       LATEST  platform007
 get_lib_base libunwind  LATEST  platform007
 get_lib_base tbb        LATEST  platform007
-get_lib_base liburing   LATEST  platform007
 
 get_lib_base kernel-headers fb platform007
 get_lib_base binutils   LATEST centos7-native

db/arena_wrapped_db_iter.cc

@@ -64,7 +64,7 @@ Status ArenaWrappedDBIter::Refresh() {
     arena_.~Arena();
     new (&arena_) Arena();
 
-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_->mutex());
+    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
     if (read_callback_) {
       read_callback_->Refresh(latest_seq);
     }

db/blob_index.h

@@ -111,7 +111,7 @@ class BlobIndex {
     return Status::OK();
   }
 
-  std::string DebugString(bool output_hex) const {
+  std::string DebugString(bool output_hex) {
     std::ostringstream oss;
 
     if (IsInlined()) {

db/c.cc

@@ -2032,17 +2032,6 @@ void rocksdb_block_based_options_set_index_type(
   options->rep.index_type = static_cast<BlockBasedTableOptions::IndexType>(v);
 }
 
-void rocksdb_block_based_options_set_data_block_index_type(
-    rocksdb_block_based_table_options_t* options, int v) {
-  options->rep.data_block_index_type =
-          static_cast<BlockBasedTableOptions::DataBlockIndexType>(v);
-}
-
-void rocksdb_block_based_options_set_data_block_hash_ratio(
-    rocksdb_block_based_table_options_t* options, double v) {
-  options->rep.data_block_hash_table_util_ratio = v;
-}
-
 void rocksdb_block_based_options_set_hash_index_allow_collision(
     rocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.hash_index_allow_collision = v;
@@ -3048,16 +3037,7 @@ rocksdb_filterpolicy_t* rocksdb_filterpolicy_create_bloom_format(int bits_per_ke
     bool KeyMayMatch(const Slice& key, const Slice& filter) const override {
       return rep_->KeyMayMatch(key, filter);
     }
-    // No need to override GetFilterBitsBuilder if this one is overridden
-    rocksdb::FilterBitsBuilder* GetBuilderWithContext(
-        const rocksdb::FilterBuildingContext& context) const override {
-      return rep_->GetBuilderWithContext(context);
-    }
-    rocksdb::FilterBitsReader* GetFilterBitsReader(
-        const Slice& contents) const override {
-      return rep_->GetFilterBitsReader(contents);
-    }
-    static void DoNothing(void*) {}
+    static void DoNothing(void*) { }
   };
   Wrapper* wrapper = new Wrapper;
   wrapper->rep_ = NewBloomFilterPolicy(bits_per_key, original_format);

db/c_test.c

@@ -487,11 +487,8 @@ int main(int argc, char** argv) {
   rocksdb_options_set_paranoid_checks(options, 1);
   rocksdb_options_set_max_open_files(options, 10);
   rocksdb_options_set_base_background_compactions(options, 1);
-
   table_options = rocksdb_block_based_options_create();
   rocksdb_block_based_options_set_block_cache(table_options, cache);
-  rocksdb_block_based_options_set_data_block_index_type(table_options, 1);
-  rocksdb_block_based_options_set_data_block_hash_ratio(table_options, 0.75);
   rocksdb_options_set_block_based_table_factory(options, table_options);
 
   rocksdb_options_set_compression(options, rocksdb_no_compression);
@@ -1051,20 +1048,17 @@ int main(int argc, char** argv) {
   }
 
   StartPhase("filter");
-  for (run = 0; run <= 2; run++) {
-    // First run uses custom filter
-    // Second run uses old block-based bloom filter
-    // Third run uses full bloom filter
+  for (run = 0; run < 2; run++) {
+    // First run uses custom filter, second run uses bloom filter
     CheckNoError(err);
     rocksdb_filterpolicy_t* policy;
     if (run == 0) {
-      policy = rocksdb_filterpolicy_create(NULL, FilterDestroy, FilterCreate,
-                                           FilterKeyMatch, NULL, FilterName);
-    } else if (run == 1) {
-      policy = rocksdb_filterpolicy_create_bloom(8);
+      policy = rocksdb_filterpolicy_create(
+          NULL, FilterDestroy, FilterCreate, FilterKeyMatch, NULL, FilterName);
     } else {
-      policy = rocksdb_filterpolicy_create_bloom_full(8);
+      policy = rocksdb_filterpolicy_create_bloom(10);
     }
+
     rocksdb_block_based_options_set_filter_policy(table_options, policy);
 
     // Create new database
@@ -1077,24 +1071,12 @@ int main(int argc, char** argv) {
     CheckNoError(err);
     rocksdb_put(db, woptions, "bar", 3, "barvalue", 8, &err);
     CheckNoError(err);
-
-    {
-      // Add enough keys to get just one reasonably populated Bloom filter
-      const int keys_to_add = 1500;
-      int i;
-      char keybuf[100];
-      for (i = 0; i < keys_to_add; i++) {
-        snprintf(keybuf, sizeof(keybuf), "yes%020d", i);
-        rocksdb_put(db, woptions, keybuf, strlen(keybuf), "val", 3, &err);
-        CheckNoError(err);
-      }
-    }
     rocksdb_compact_range(db, NULL, 0, NULL, 0);
 
     fake_filter_result = 1;
     CheckGet(db, roptions, "foo", "foovalue");
     CheckGet(db, roptions, "bar", "barvalue");
-    if (run == 0) {
+    if (phase == 0) {
       // Must not find value when custom filter returns false
       fake_filter_result = 0;
       CheckGet(db, roptions, "foo", NULL);
@@ -1104,43 +1086,6 @@ int main(int argc, char** argv) {
       CheckGet(db, roptions, "foo", "foovalue");
       CheckGet(db, roptions, "bar", "barvalue");
     }
-
-    {
-      // Query some keys not added to identify Bloom filter implementation
-      // from false positive queries, using perfcontext to detect Bloom
-      // filter behavior
-      rocksdb_perfcontext_t* perf = rocksdb_perfcontext_create();
-      rocksdb_perfcontext_reset(perf);
-
-      const int keys_to_query = 10000;
-      int i;
-      char keybuf[100];
-      for (i = 0; i < keys_to_query; i++) {
-        fake_filter_result = i % 2;
-        snprintf(keybuf, sizeof(keybuf), "no%020d", i);
-        CheckGet(db, roptions, keybuf, NULL);
-      }
-
-      const int hits =
-          (int)rocksdb_perfcontext_metric(perf, rocksdb_bloom_sst_hit_count);
-      if (run == 0) {
-        // Due to half true, half false with fake filter result
-        CheckCondition(hits == keys_to_query / 2);
-      } else if (run == 1) {
-        // Essentially a fingerprint of the block-based Bloom schema
-        CheckCondition(hits == 241);
-      } else {
-        // Essentially a fingerprint of the full Bloom schema(s),
-        // format_version < 5, which vary for three different CACHE_LINE_SIZEs
-        CheckCondition(hits == 224 || hits == 180 || hits == 125);
-      }
-      CheckCondition(
-          (keys_to_query - hits) ==
-          (int)rocksdb_perfcontext_metric(perf, rocksdb_bloom_sst_miss_count));
-
-      rocksdb_perfcontext_destroy(perf);
-    }
-
     // Reset the policy
     rocksdb_block_based_options_set_filter_policy(table_options, NULL);
     rocksdb_options_set_block_based_table_factory(options, table_options);

db/column_family.cc

@@ -1080,9 +1080,8 @@ Compaction* ColumnFamilyData::CompactRange(
   return result;
 }
 
-SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
-    InstrumentedMutex* db_mutex) {
-  SuperVersion* sv = GetThreadLocalSuperVersion(db_mutex);
+SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(DBImpl* db) {
+  SuperVersion* sv = GetThreadLocalSuperVersion(db);
   sv->Ref();
   if (!ReturnThreadLocalSuperVersion(sv)) {
     // This Unref() corresponds to the Ref() in GetThreadLocalSuperVersion()
@@ -1094,8 +1093,7 @@ SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
   return sv;
 }
 
-SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
-    InstrumentedMutex* db_mutex) {
+SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(DBImpl* db) {
   // The SuperVersion is cached in thread local storage to avoid acquiring
   // mutex when SuperVersion does not change since the last use. When a new
   // SuperVersion is installed, the compaction or flush thread cleans up
@@ -1122,16 +1120,21 @@ SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
 
     if (sv && sv->Unref()) {
       RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_CLEANUPS);
-      db_mutex->Lock();
+      db->mutex()->Lock();
       // NOTE: underlying resources held by superversion (sst files) might
       // not be released until the next background job.
       sv->Cleanup();
-      sv_to_delete = sv;
+      if (db->immutable_db_options().avoid_unnecessary_blocking_io) {
+        db->AddSuperVersionsToFreeQueue(sv);
+        db->SchedulePurge();
+      } else {
+        sv_to_delete = sv;
+      }
     } else {
-      db_mutex->Lock();
+      db->mutex()->Lock();
     }
     sv = super_version_->Ref();
-    db_mutex->Unlock();
+    db->mutex()->Unlock();
 
     delete sv_to_delete;
   }

db/column_family.h

@@ -430,11 +430,11 @@ class ColumnFamilyData {
   SuperVersion* GetSuperVersion() { return super_version_; }
   // thread-safe
   // Return a already referenced SuperVersion to be used safely.
-  SuperVersion* GetReferencedSuperVersion(InstrumentedMutex* db_mutex);
+  SuperVersion* GetReferencedSuperVersion(DBImpl* db);
   // thread-safe
   // Get SuperVersion stored in thread local storage. If it does not exist,
   // get a reference from a current SuperVersion.
-  SuperVersion* GetThreadLocalSuperVersion(InstrumentedMutex* db_mutex);
+  SuperVersion* GetThreadLocalSuperVersion(DBImpl* db);
   // Try to return SuperVersion back to thread local storage. Retrun true on
   // success and false on failure. It fails when the thread local storage
   // contains anything other than SuperVersion::kSVInUse flag.

db/column_family_test.cc

@@ -67,9 +67,9 @@ class ColumnFamilyTestBase : public testing::Test {
 #ifndef ROCKSDB_LITE
     const char* test_env_uri = getenv("TEST_ENV_URI");
     if (test_env_uri) {
-      Status s = ObjectRegistry::NewInstance()->NewSharedObject(test_env_uri,
-                                                                &env_guard_);
-      base_env = env_guard_.get();
+      Env* test_env = nullptr;
+      Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
+      base_env = test_env;
       EXPECT_OK(s);
       EXPECT_NE(Env::Default(), base_env);
     }

db/compaction/compaction.cc

@@ -547,11 +547,13 @@ bool Compaction::ShouldFormSubcompactions() const {
 
 uint64_t Compaction::MinInputFileOldestAncesterTime() const {
   uint64_t min_oldest_ancester_time = port::kMaxUint64;
-  for (const auto& file : inputs_[0].files) {
-    uint64_t oldest_ancester_time = file->TryGetOldestAncesterTime();
-    if (oldest_ancester_time != 0) {
-      min_oldest_ancester_time =
-          std::min(min_oldest_ancester_time, oldest_ancester_time);
+  for (const auto& level_files : inputs_) {
+    for (const auto& file : level_files.files) {
+      uint64_t oldest_ancester_time = file->TryGetOldestAncesterTime();
+      if (oldest_ancester_time != 0) {
+        min_oldest_ancester_time =
+            std::min(min_oldest_ancester_time, oldest_ancester_time);
+      }
     }
   }
   return min_oldest_ancester_time;

db/compaction/compaction_picker_universal.cc

@@ -247,8 +247,7 @@ bool UniversalCompactionBuilder::IsInputFilesNonOverlapping(Compaction* c) {
 
     next.f = nullptr;
 
-    if (c->level(curr.level) != 0 &&
-        curr.index < c->num_input_files(curr.level) - 1) {
+    if (curr.level != 0 && curr.index < c->num_input_files(curr.level) - 1) {
       next.f = c->input(curr.level, curr.index + 1);
       next.level = curr.level;
       next.index = curr.index + 1;

db/db_basic_test.cc

@@ -2045,81 +2045,48 @@ TEST_P(DBBasicTestWithTimestampWithParam, PutAndGet) {
       10 /*bits_per_key*/, false /*use_block_based_builder*/));
   bbto.whole_key_filtering = true;
   options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+  size_t num_cfs = handles_.size();
+  ASSERT_EQ(2, num_cfs);
+  std::vector<std::string> write_ts_strs(kNumTimestamps);
+  std::vector<std::string> read_ts_strs(kNumTimestamps);
+  std::vector<Slice> write_ts_list;
+  std::vector<Slice> read_ts_list;
 
-  std::vector<CompressionType> compression_types;
-  compression_types.push_back(kNoCompression);
-  if (Zlib_Supported()) {
-    compression_types.push_back(kZlibCompression);
-  }
-#if LZ4_VERSION_NUMBER >= 10400  // r124+
-  compression_types.push_back(kLZ4Compression);
-  compression_types.push_back(kLZ4HCCompression);
-#endif  // LZ4_VERSION_NUMBER >= 10400
-  if (ZSTD_Supported()) {
-    compression_types.push_back(kZSTD);
-  }
-
-  // Switch compression dictionary on/off to check key extraction
-  // correctness in kBuffered state
-  std::vector<uint32_t> max_dict_bytes_list = {0, 1 << 14};  // 0 or 16KB
-
-  for (auto compression_type : compression_types) {
-    for (uint32_t max_dict_bytes : max_dict_bytes_list) {
-      options.compression = compression_type;
-      options.compression_opts.max_dict_bytes = max_dict_bytes;
-      if (compression_type == kZSTD) {
-        options.compression_opts.zstd_max_train_bytes = max_dict_bytes;
+  for (size_t i = 0; i != kNumTimestamps; ++i) {
+    write_ts_list.emplace_back(EncodeTimestamp(i * 2, 0, &write_ts_strs[i]));
+    read_ts_list.emplace_back(EncodeTimestamp(1 + i * 2, 0, &read_ts_strs[i]));
+    const Slice& write_ts = write_ts_list.back();
+    WriteOptions wopts;
+    wopts.timestamp = &write_ts;
+    for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
+      for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps; ++j) {
+        ASSERT_OK(Put(cf, "key" + std::to_string(j),
+                      "value_" + std::to_string(j) + "_" + std::to_string(i),
+                      wopts));
       }
-      options.target_file_size_base = 1 << 26;  // 64MB
-
-      DestroyAndReopen(options);
-      CreateAndReopenWithCF({"pikachu"}, options);
-      size_t num_cfs = handles_.size();
-      ASSERT_EQ(2, num_cfs);
-      std::vector<std::string> write_ts_strs(kNumTimestamps);
-      std::vector<std::string> read_ts_strs(kNumTimestamps);
-      std::vector<Slice> write_ts_list;
-      std::vector<Slice> read_ts_list;
-
-      for (size_t i = 0; i != kNumTimestamps; ++i) {
-        write_ts_list.emplace_back(
-            EncodeTimestamp(i * 2, 0, &write_ts_strs[i]));
-        read_ts_list.emplace_back(
-            EncodeTimestamp(1 + i * 2, 0, &read_ts_strs[i]));
-        const Slice& write_ts = write_ts_list.back();
-        WriteOptions wopts;
-        wopts.timestamp = &write_ts;
-        for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
-          for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps; ++j) {
-            ASSERT_OK(Put(
-                cf, "key" + std::to_string(j),
-                "value_" + std::to_string(j) + "_" + std::to_string(i), wopts));
-          }
-          if (!memtable_only) {
-            ASSERT_OK(Flush(cf));
-          }
-        }
+      if (!memtable_only) {
+        ASSERT_OK(Flush(cf));
       }
-      const auto& verify_db_func = [&]() {
-        for (size_t i = 0; i != kNumTimestamps; ++i) {
-          ReadOptions ropts;
-          ropts.timestamp = &read_ts_list[i];
-          for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
-            ColumnFamilyHandle* cfh = handles_[cf];
-            for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps;
-                 ++j) {
-              std::string value;
-              ASSERT_OK(
-                  db_->Get(ropts, cfh, "key" + std::to_string(j), &value));
-              ASSERT_EQ("value_" + std::to_string(j) + "_" + std::to_string(i),
-                        value);
-            }
-          }
-        }
-      };
-      verify_db_func();
     }
   }
+  const auto& verify_db_func = [&]() {
+    for (size_t i = 0; i != kNumTimestamps; ++i) {
+      ReadOptions ropts;
+      ropts.timestamp = &read_ts_list[i];
+      for (int cf = 0; cf != static_cast<int>(num_cfs); ++cf) {
+        ColumnFamilyHandle* cfh = handles_[cf];
+        for (size_t j = 0; j != (kNumKeysPerFile - 1) / kNumTimestamps; ++j) {
+          std::string value;
+          ASSERT_OK(db_->Get(ropts, cfh, "key" + std::to_string(j), &value));
+          ASSERT_EQ("value_" + std::to_string(j) + "_" + std::to_string(i),
+                    value);
+        }
+      }
+    }
+  };
+  verify_db_func();
 }
 
 INSTANTIATE_TEST_CASE_P(Timestamp, DBBasicTestWithTimestampWithParam,

db/db_compaction_test.cc

@@ -3576,6 +3576,15 @@ TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) {
         ASSERT_OK(Put(Key(i), RandomString(&rnd, kValueSize)));
       }
       Flush();
+      // Get the first file's creation time. This will be the oldest file in the
+      // DB. Compactions inolving this file's descendents should keep getting
+      // this time.
+      std::vector<std::vector<FileMetaData>> level_to_files;
+      dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                      &level_to_files);
+      uint64_t oldest_time = level_to_files[0][0].oldest_ancester_time;
+      // Add 1 hour and do another flush.
+      env_->addon_time_.fetch_add(1 * 60 * 60);
       for (int i = 101; i <= 200; ++i) {
         ASSERT_OK(Put(Key(i), RandomString(&rnd, kValueSize)));
       }
@@ -3583,11 +3592,13 @@ TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) {
       MoveFilesToLevel(6);
       ASSERT_EQ("0,0,0,0,0,0,2", FilesPerLevel());
 
+      env_->addon_time_.fetch_add(1 * 60 * 60);
       // Add two L4 files with key ranges: [1 .. 50], [51 .. 150].
       for (int i = 1; i <= 50; ++i) {
         ASSERT_OK(Put(Key(i), RandomString(&rnd, kValueSize)));
       }
       Flush();
+      env_->addon_time_.fetch_add(1 * 60 * 60);
       for (int i = 51; i <= 150; ++i) {
         ASSERT_OK(Put(Key(i), RandomString(&rnd, kValueSize)));
       }
@@ -3595,6 +3606,7 @@ TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) {
       MoveFilesToLevel(4);
       ASSERT_EQ("0,0,0,0,2,0,2", FilesPerLevel());
 
+      env_->addon_time_.fetch_add(1 * 60 * 60);
       // Add one L1 file with key range: [26, 75].
       for (int i = 26; i <= 75; ++i) {
         ASSERT_OK(Put(Key(i), RandomString(&rnd, kValueSize)));
@@ -3636,6 +3648,10 @@ TEST_F(DBCompactionTest, LevelTtlCascadingCompactions) {
       ASSERT_EQ("1,0,0,0,0,0,1", FilesPerLevel());
       ASSERT_EQ(5, ttl_compactions);
 
+      dbfull()->TEST_GetFilesMetaData(dbfull()->DefaultColumnFamily(),
+                                      &level_to_files);
+      ASSERT_EQ(oldest_time, level_to_files[6][0].oldest_ancester_time);
+
       env_->addon_time_.fetch_add(25 * 60 * 60);
       ASSERT_OK(Put(Key(2), "1"));
       if (if_restart) {

db/db_flush_test.cc

@@ -212,30 +212,6 @@ TEST_F(DBFlushTest, ManualFlushWithMinWriteBufferNumberToMerge) {
   t.join();
 }
 
-TEST_F(DBFlushTest, ScheduleOnlyOneBgThread) {
-  Options options = CurrentOptions();
-  Reopen(options);
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-  int called = 0;
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::MaybeScheduleFlushOrCompaction:AfterSchedule:0", [&](void* arg) {
-        ASSERT_NE(nullptr, arg);
-        auto unscheduled_flushes = *reinterpret_cast<int*>(arg);
-        ASSERT_EQ(0, unscheduled_flushes);
-        ++called;
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(Put("a", "foo"));
-  FlushOptions flush_opts;
-  ASSERT_OK(dbfull()->Flush(flush_opts));
-  ASSERT_EQ(1, called);
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
 TEST_P(DBFlushDirectIOTest, DirectIO) {
   Options options;
   options.create_if_missing = true;

db/db_impl/db_impl.cc

@@ -141,6 +141,8 @@ void DumpSupportInfo(Logger* logger) {
   ROCKS_LOG_HEADER(logger, "Fast CRC32 supported: %s",
                    crc32c::IsFastCrc32Supported().c_str());
 }
+
+int64_t kDefaultLowPriThrottledRate = 2 * 1024 * 1024;
 }  // namespace
 
 DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
@@ -176,6 +178,11 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname,
       write_thread_(immutable_db_options_),
       nonmem_write_thread_(immutable_db_options_),
       write_controller_(mutable_db_options_.delayed_write_rate),
+      // Use delayed_write_rate as a base line to determine the initial
+      // low pri write rate limit. It may be adjusted later.
+      low_pri_write_rate_limiter_(NewGenericRateLimiter(std::min(
+          static_cast<int64_t>(mutable_db_options_.delayed_write_rate / 8),
+          kDefaultLowPriThrottledRate))),
       last_batch_group_size_(0),
       unscheduled_flushes_(0),
       unscheduled_compactions_(0),
@@ -872,7 +879,7 @@ Status DBImpl::TablesRangeTombstoneSummary(ColumnFamilyHandle* column_family,
           column_family);
   ColumnFamilyData* cfd = cfh->cfd();
 
-  SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+  SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
   Version* version = super_version->current;
 
   Status s =
@@ -888,7 +895,6 @@ void DBImpl::ScheduleBgLogWriterClose(JobContext* job_context) {
       AddToLogsToFreeQueue(l);
     }
     job_context->logs_to_free.clear();
-    SchedulePurge();
   }
 }
 
@@ -1315,19 +1321,34 @@ void DBImpl::BackgroundCallPurge() {
     delete log_writer;
     mutex_.Lock();
   }
-  for (const auto& file : purge_files_) {
-    const PurgeFileInfo& purge_file = file.second;
+  while (!superversions_to_free_queue_.empty()) {
+    assert(!superversions_to_free_queue_.empty());
+    SuperVersion* sv = superversions_to_free_queue_.front();
+    superversions_to_free_queue_.pop_front();
+    mutex_.Unlock();
+    delete sv;
+    mutex_.Lock();
+  }
+
+  // Can't use iterator to go over purge_files_ because inside the loop we're
+  // unlocking the mutex that protects purge_files_.
+  while (!purge_files_.empty()) {
+    auto it = purge_files_.begin();
+    // Need to make a copy of the PurgeFilesInfo before unlocking the mutex.
+    PurgeFileInfo purge_file = it->second;
+
     const std::string& fname = purge_file.fname;
     const std::string& dir_to_sync = purge_file.dir_to_sync;
     FileType type = purge_file.type;
     uint64_t number = purge_file.number;
     int job_id = purge_file.job_id;
 
+    purge_files_.erase(it);
+
     mutex_.Unlock();
     DeleteObsoleteFileImpl(job_id, fname, dir_to_sync, type, number);
     mutex_.Lock();
   }
-  purge_files_.clear();
 
   bg_purge_scheduled_--;
 
@@ -1367,10 +1388,14 @@ static void CleanupIteratorState(void* arg1, void* /*arg2*/) {
     state->db->FindObsoleteFiles(&job_context, false, true);
     if (state->background_purge) {
       state->db->ScheduleBgLogWriterClose(&job_context);
+      state->db->AddSuperVersionsToFreeQueue(state->super_version);
+      state->db->SchedulePurge();
     }
     state->mu->Unlock();
 
-    delete state->super_version;
+    if (!state->background_purge) {
+      delete state->super_version;
+    }
     if (job_context.HaveSomethingToDelete()) {
       if (state->background_purge) {
         // PurgeObsoleteFiles here does not delete files. Instead, it adds the
@@ -2445,7 +2470,7 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
     result = nullptr;
 
 #else
-    SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
     auto iter = new ForwardIterator(this, read_options, cfd, sv);
     result = NewDBIterator(
         env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
@@ -2471,7 +2496,7 @@ ArenaWrappedDBIter* DBImpl::NewIteratorImpl(const ReadOptions& read_options,
                                             ReadCallback* read_callback,
                                             bool allow_blob,
                                             bool allow_refresh) {
-  SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+  SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
 
   // Try to generate a DB iterator tree in continuous memory area to be
   // cache friendly. Here is an example of result:
@@ -2550,7 +2575,7 @@ Status DBImpl::NewIterators(
 #else
     for (auto cfh : column_families) {
       auto cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
-      SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+      SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
       auto iter = new ForwardIterator(this, read_options, cfd, sv);
       iterators->push_back(NewDBIterator(
           env_, read_options, *cfd->ioptions(), sv->mutable_cf_options,
@@ -2877,7 +2902,7 @@ bool DBImpl::GetAggregatedIntProperty(const Slice& property,
 
 SuperVersion* DBImpl::GetAndRefSuperVersion(ColumnFamilyData* cfd) {
   // TODO(ljin): consider using GetReferencedSuperVersion() directly
-  return cfd->GetThreadLocalSuperVersion(&mutex_);
+  return cfd->GetThreadLocalSuperVersion(this);
 }
 
 // REQUIRED: this function should only be called on the write thread or if the
@@ -2895,11 +2920,19 @@ SuperVersion* DBImpl::GetAndRefSuperVersion(uint32_t column_family_id) {
 void DBImpl::CleanupSuperVersion(SuperVersion* sv) {
   // Release SuperVersion
   if (sv->Unref()) {
+    bool defer_purge =
+            immutable_db_options().avoid_unnecessary_blocking_io;
     {
       InstrumentedMutexLock l(&mutex_);
       sv->Cleanup();
+      if (defer_purge) {
+        AddSuperVersionsToFreeQueue(sv);
+        SchedulePurge();
+      }
+    }
+    if (!defer_purge) {
+      delete sv;
     }
-    delete sv;
     RecordTick(stats_, NUMBER_SUPERVERSION_CLEANUPS);
   }
   RecordTick(stats_, NUMBER_SUPERVERSION_RELEASES);
@@ -3905,7 +3938,7 @@ Status DBImpl::IngestExternalFiles(
     start_file_number += args[i - 1].external_files.size();
     auto* cfd =
         static_cast<ColumnFamilyHandleImpl*>(args[i].column_family)->cfd();
-    SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
     exec_results[i].second = ingestion_jobs[i].Prepare(
         args[i].external_files, start_file_number, super_version);
     exec_results[i].first = true;
@@ -3916,7 +3949,7 @@ Status DBImpl::IngestExternalFiles(
   {
     auto* cfd =
         static_cast<ColumnFamilyHandleImpl*>(args[0].column_family)->cfd();
-    SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
     exec_results[0].second = ingestion_jobs[0].Prepare(
         args[0].external_files, next_file_number, super_version);
     exec_results[0].first = true;
@@ -4185,7 +4218,7 @@ Status DBImpl::CreateColumnFamilyWithImport(
   dummy_sv_ctx.Clean();
 
   if (status.ok()) {
-    SuperVersion* sv = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* sv = cfd->GetReferencedSuperVersion(this);
     status = import_job.Prepare(next_file_number, sv);
     CleanupSuperVersion(sv);
   }
@@ -4262,7 +4295,7 @@ Status DBImpl::VerifyChecksum(const ReadOptions& read_options) {
   }
   std::vector<SuperVersion*> sv_list;
   for (auto cfd : cfd_list) {
-    sv_list.push_back(cfd->GetReferencedSuperVersion(&mutex_));
+    sv_list.push_back(cfd->GetReferencedSuperVersion(this));
   }
   for (auto& sv : sv_list) {
     VersionStorageInfo* vstorage = sv->current->storage_info();
@@ -4287,14 +4320,23 @@ Status DBImpl::VerifyChecksum(const ReadOptions& read_options) {
       break;
     }
   }
+  bool defer_purge =
+          immutable_db_options().avoid_unnecessary_blocking_io;
   {
     InstrumentedMutexLock l(&mutex_);
     for (auto sv : sv_list) {
       if (sv && sv->Unref()) {
         sv->Cleanup();
-        delete sv;
+        if (defer_purge) {
+          AddSuperVersionsToFreeQueue(sv);
+        } else {
+          delete sv;
+        }
       }
     }
+    if (defer_purge) {
+      SchedulePurge();
+    }
     for (auto cfd : cfd_list) {
       cfd->Unref();
     }

db/db_impl/db_impl.h

@@ -798,6 +798,10 @@ class DBImpl : public DB {
     logs_to_free_queue_.push_back(log_writer);
   }
 
+  void AddSuperVersionsToFreeQueue(SuperVersion* sv) {
+    superversions_to_free_queue_.push_back(sv);
+  }
+
   void SetSnapshotChecker(SnapshotChecker* snapshot_checker);
 
   // Fill JobContext with snapshot information needed by flush and compaction.
@@ -1195,7 +1199,7 @@ class DBImpl : public DB {
   };
 
   // PurgeFileInfo is a structure to hold information of files to be deleted in
-  // purge_queue_
+  // purge_files_
   struct PurgeFileInfo {
     std::string fname;
     std::string dir_to_sync;
@@ -1831,6 +1835,8 @@ class DBImpl : public DB {
 
   WriteController write_controller_;
 
+  std::unique_ptr<RateLimiter> low_pri_write_rate_limiter_;
+
   // Size of the last batch group. In slowdown mode, next write needs to
   // sleep if it uses up the quota.
   // Note: This is to protect memtable and compaction. If the batch only writes
@@ -1888,6 +1894,7 @@ class DBImpl : public DB {
 
   // A queue to store log writers to close
   std::deque<log::Writer*> logs_to_free_queue_;
+  std::deque<SuperVersion*> superversions_to_free_queue_;
   int unscheduled_flushes_;
   int unscheduled_compactions_;
 

db/db_impl/db_impl_compaction_flush.cc

@@ -659,7 +659,7 @@ Status DBImpl::CompactRange(const CompactRangeOptions& options,
     // one/both sides of the interval are unbounded. But it requires more
     // changes to RangesOverlapWithMemtables.
     Range range(*begin, *end);
-    SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+    SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
     cfd->RangesOverlapWithMemtables({range}, super_version, &flush_needed);
     CleanupSuperVersion(super_version);
   }
@@ -1918,10 +1918,6 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
     fta->thread_pri_ = Env::Priority::HIGH;
     env_->Schedule(&DBImpl::BGWorkFlush, fta, Env::Priority::HIGH, this,
                    &DBImpl::UnscheduleFlushCallback);
-    --unscheduled_flushes_;
-    TEST_SYNC_POINT_CALLBACK(
-        "DBImpl::MaybeScheduleFlushOrCompaction:AfterSchedule:0",
-        &unscheduled_flushes_);
   }
 
   // special case -- if high-pri (flush) thread pool is empty, then schedule
@@ -1936,7 +1932,6 @@ void DBImpl::MaybeScheduleFlushOrCompaction() {
       fta->thread_pri_ = Env::Priority::LOW;
       env_->Schedule(&DBImpl::BGWorkFlush, fta, Env::Priority::LOW, this,
                      &DBImpl::UnscheduleFlushCallback);
-      --unscheduled_flushes_;
     }
   }
 
@@ -2020,6 +2015,8 @@ ColumnFamilyData* DBImpl::PopFirstFromCompactionQueue() {
 DBImpl::FlushRequest DBImpl::PopFirstFromFlushQueue() {
   assert(!flush_queue_.empty());
   FlushRequest flush_req = flush_queue_.front();
+  assert(unscheduled_flushes_ >= static_cast<int>(flush_req.size()));
+  unscheduled_flushes_ -= static_cast<int>(flush_req.size());
   flush_queue_.pop_front();
   // TODO: need to unset flush reason?
   return flush_req;
@@ -2061,7 +2058,7 @@ void DBImpl::SchedulePendingFlush(const FlushRequest& flush_req,
     cfd->Ref();
     cfd->SetFlushReason(flush_reason);
   }
-  ++unscheduled_flushes_;
+  unscheduled_flushes_ += static_cast<int>(flush_req.size());
   flush_queue_.push_back(flush_req);
 }
 

db/db_impl/db_impl_secondary.cc

@@ -406,7 +406,7 @@ ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
     const ReadOptions& read_options, ColumnFamilyData* cfd,
     SequenceNumber snapshot, ReadCallback* read_callback) {
   assert(nullptr != cfd);
-  SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
+  SuperVersion* super_version = cfd->GetReferencedSuperVersion(this);
   auto db_iter = NewArenaWrappedDbIterator(
       env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
       snapshot,

db/db_impl/db_impl_write.cc

@@ -1516,12 +1516,14 @@ Status DBImpl::TrimMemtableHistory(WriteContext* context) {
   for (auto& cfd : cfds) {
     autovector<MemTable*> to_delete;
     cfd->imm()->TrimHistory(&to_delete, cfd->mem()->ApproximateMemoryUsage());
-    for (auto m : to_delete) {
-      delete m;
+    if (!to_delete.empty()) {
+      for (auto m : to_delete) {
+        delete m;
+      }
+      context->superversion_context.NewSuperVersion();
+      assert(context->superversion_context.new_superversion.get() != nullptr);
+      cfd->InstallSuperVersion(&context->superversion_context, &mutex_);
     }
-    context->superversion_context.NewSuperVersion();
-    assert(context->superversion_context.new_superversion.get() != nullptr);
-    cfd->InstallSuperVersion(&context->superversion_context, &mutex_);
 
     if (cfd->Unref()) {
       delete cfd;

db/db_impl/db_secondary_test.cc

@@ -198,7 +198,7 @@ TEST_F(DBSecondaryTest, OpenAsSecondary) {
 namespace {
 class TraceFileEnv : public EnvWrapper {
  public:
-  explicit TraceFileEnv(Env* _target) : EnvWrapper(_target) {}
+  explicit TraceFileEnv(Env* target) : EnvWrapper(target) {}
   Status NewRandomAccessFile(const std::string& f,
                              std::unique_ptr<RandomAccessFile>* r,
                              const EnvOptions& env_options) override {

db/db_test2.cc

@@ -4210,6 +4210,42 @@ TEST_F(DBTest2, SeekFileRangeDeleteTail) {
   }
   db_->ReleaseSnapshot(s1);
 }
+
+TEST_F(DBTest2, BackgroundPurgeTest) {
+  Options options = CurrentOptions();
+  options.write_buffer_manager = std::make_shared<rocksdb::WriteBufferManager>(1 << 20);
+  options.avoid_unnecessary_blocking_io = true;
+  DestroyAndReopen(options);
+  size_t base_value = options.write_buffer_manager->memory_usage();
+
+  ASSERT_OK(Put("a", "a"));
+  Iterator* iter = db_->NewIterator(ReadOptions());
+  ASSERT_OK(Flush());
+  size_t value = options.write_buffer_manager->memory_usage();
+  ASSERT_GT(value, base_value);
+
+  db_->GetEnv()->SetBackgroundThreads(1, Env::Priority::HIGH);
+  test::SleepingBackgroundTask sleeping_task_after;
+  db_->GetEnv()->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                          &sleeping_task_after, Env::Priority::HIGH);
+  delete iter;
+
+  Env::Default()->SleepForMicroseconds(100000);
+  value = options.write_buffer_manager->memory_usage();
+  ASSERT_GT(value, base_value);
+
+  sleeping_task_after.WakeUp();
+  sleeping_task_after.WaitUntilDone();
+
+  test::SleepingBackgroundTask sleeping_task_after2;
+  db_->GetEnv()->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                          &sleeping_task_after2, Env::Priority::HIGH);
+  sleeping_task_after2.WakeUp();
+  sleeping_task_after2.WaitUntilDone();
+
+  value = options.write_buffer_manager->memory_usage();
+  ASSERT_EQ(base_value, value);
+}
 }  // namespace rocksdb
 
 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS

db/db_test_util.cc

@@ -53,9 +53,9 @@ DBTestBase::DBTestBase(const std::string path)
 #ifndef ROCKSDB_LITE
   const char* test_env_uri = getenv("TEST_ENV_URI");
   if (test_env_uri) {
-    Status s = ObjectRegistry::NewInstance()->NewSharedObject(test_env_uri,
-                                                              &env_guard_);
-    base_env = env_guard_.get();
+    Env* test_env = nullptr;
+    Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
+    base_env = test_env;
     EXPECT_OK(s);
     EXPECT_NE(Env::Default(), base_env);
   }

db/db_universal_compaction_test.cc

@@ -1753,14 +1753,13 @@ TEST_P(DBTestUniversalCompaction, RecalculateScoreAfterPicking) {
   // scheduling more; otherwise, other CFs/DBs may be delayed unnecessarily.
   const int kNumFilesTrigger = 8;
   Options options = CurrentOptions();
-  options.memtable_factory.reset(
-      new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
   options.compaction_options_universal.max_merge_width = kNumFilesTrigger / 2;
   options.compaction_options_universal.max_size_amplification_percent =
       static_cast<unsigned int>(-1);
   options.compaction_style = kCompactionStyleUniversal;
   options.level0_file_num_compaction_trigger = kNumFilesTrigger;
   options.num_levels = num_levels_;
+  options.write_buffer_size = 100 << 10;  // 100KB
   Reopen(options);
 
   std::atomic<int> num_compactions_attempted(0);

db/db_write_test.cc

@@ -39,6 +39,97 @@ TEST_P(DBWriteTest, SyncAndDisableWAL) {
   ASSERT_TRUE(dbfull()->Write(write_options, &batch).IsInvalidArgument());
 }
 
+TEST_P(DBWriteTest, WriteThreadHangOnWriteStall) {
+  Options options = GetOptions();
+  options.level0_stop_writes_trigger = options.level0_slowdown_writes_trigger = 4;
+  std::vector<port::Thread> threads;
+  std::atomic<int> thread_num(0);
+  port::Mutex mutex;
+  port::CondVar cv(&mutex);
+
+  Reopen(options);
+
+  std::function<void()> write_slowdown_func = [&]() {
+    int a = thread_num.fetch_add(1);
+    std::string key = "foo" + std::to_string(a);
+    WriteOptions wo;
+    wo.no_slowdown = false;
+    dbfull()->Put(wo, key, "bar");
+  };
+  std::function<void()> write_no_slowdown_func = [&]() {
+    int a = thread_num.fetch_add(1);
+    std::string key = "foo" + std::to_string(a);
+    WriteOptions wo;
+    wo.no_slowdown = true;
+    dbfull()->Put(wo, key, "bar");
+  };
+  std::function<void(void *)> unblock_main_thread_func = [&](void *) {
+    mutex.Lock();
+    cv.SignalAll();
+    mutex.Unlock();
+  };
+
+  // Create 3 L0 files and schedule 4th without waiting
+  Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar");
+  Flush();
+  Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar");
+  Flush();
+  Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar");
+  Flush();
+  Put("foo" + std::to_string(thread_num.fetch_add(1)), "bar");
+
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+    "WriteThread::JoinBatchGroup:Start", unblock_main_thread_func);
+  rocksdb::SyncPoint::GetInstance()->LoadDependency(
+    {{"DBWriteTest::WriteThreadHangOnWriteStall:1",
+      "DBImpl::BackgroundCallFlush:start"},
+     {"DBWriteTest::WriteThreadHangOnWriteStall:2",
+      "DBImpl::WriteImpl:BeforeLeaderEnters"},
+     // Make compaction start wait for the write stall to be detected and
+     // implemented by a write group leader
+     {"DBWriteTest::WriteThreadHangOnWriteStall:3",
+      "BackgroundCallCompaction:0"}});
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  // Schedule creation of 4th L0 file without waiting. This will seal the
+  // memtable and then wait for a sync point before writing the file. We need
+  // to do it this way because SwitchMemtable() needs to enter the
+  // write_thread
+  FlushOptions fopt;
+  fopt.wait = false;
+  dbfull()->Flush(fopt);
+
+  // Create a mix of slowdown/no_slowdown write threads
+  mutex.Lock();
+  // First leader
+  threads.emplace_back(write_slowdown_func);
+  cv.Wait();
+  // Second leader. Will stall writes
+  threads.emplace_back(write_slowdown_func);
+  cv.Wait();
+  threads.emplace_back(write_no_slowdown_func);
+  cv.Wait();
+  threads.emplace_back(write_slowdown_func);
+  cv.Wait();
+  threads.emplace_back(write_no_slowdown_func);
+  cv.Wait();
+  threads.emplace_back(write_slowdown_func);
+  cv.Wait();
+  mutex.Unlock();
+
+  TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:1");
+  dbfull()->TEST_WaitForFlushMemTable(nullptr);
+  // This would have triggered a write stall. Unblock the write group leader
+  TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:2");
+  // The leader is going to create missing newer links. When the leader finishes,
+  // the next leader is going to delay writes and fail writers with no_slowdown
+
+  TEST_SYNC_POINT("DBWriteTest::WriteThreadHangOnWriteStall:3");
+  for (auto& t : threads) {
+    t.join();
+  }
+}
+
 TEST_P(DBWriteTest, IOErrorOnWALWritePropagateToWriteThreadFollower) {
   constexpr int kNumThreads = 5;
   std::unique_ptr<FaultInjectionTestEnv> mock_env(

db/error_handler.cc

@@ -166,12 +166,6 @@ Status ErrorHandler::SetBGError(const Status& bg_err, BackgroundErrorReason reas
     return Status::OK();
   }
 
-  // Check if recovery is currently in progress. If it is, we will save this
-  // error so we can check it at the end to see if recovery succeeded or not
-  if (recovery_in_prog_ && recovery_error_.ok()) {
-    recovery_error_ = bg_err;
-  }
-
   bool paranoid = db_options_.paranoid_checks;
   Status::Severity sev = Status::Severity::kFatalError;
   Status new_bg_err;
@@ -204,10 +198,15 @@ Status ErrorHandler::SetBGError(const Status& bg_err, BackgroundErrorReason reas
 
   new_bg_err = Status(bg_err, sev);
 
+  // Check if recovery is currently in progress. If it is, we will save this
+  // error so we can check it at the end to see if recovery succeeded or not
+  if (recovery_in_prog_ && recovery_error_.ok()) {
+    recovery_error_ = new_bg_err;
+  }
+
   bool auto_recovery = auto_recovery_;
   if (new_bg_err.severity() >= Status::Severity::kFatalError && auto_recovery) {
     auto_recovery = false;
-    ;
   }
 
   // Allow some error specific overrides

db/error_handler_test.cc

@@ -22,6 +22,21 @@ namespace rocksdb {
 class DBErrorHandlingTest : public DBTestBase {
  public:
   DBErrorHandlingTest() : DBTestBase("/db_error_handling_test") {}
+
+  std::string GetManifestNameFromLiveFiles() {
+    std::vector<std::string> live_files;
+    uint64_t manifest_size;
+
+    dbfull()->GetLiveFiles(live_files, &manifest_size, false);
+    for (auto& file : live_files) {
+      uint64_t num = 0;
+      FileType type;
+      if (ParseFileName(file, &num, &type) && type == kDescriptorFile) {
+        return file;
+      }
+    }
+    return "";
+  }
 };
 
 class DBErrorHandlingEnv : public EnvWrapper {
@@ -161,6 +176,169 @@ TEST_F(DBErrorHandlingTest, FLushWriteError) {
   Destroy(options);
 }
 
+TEST_F(DBErrorHandlingTest, ManifestWriteError) {
+  std::unique_ptr<FaultInjectionTestEnv> fault_env(
+      new FaultInjectionTestEnv(Env::Default()));
+  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.env = fault_env.get();
+  options.listeners.emplace_back(listener);
+  Status s;
+  std::string old_manifest;
+  std::string new_manifest;
+
+  listener->EnableAutoRecovery(false);
+  DestroyAndReopen(options);
+  old_manifest = GetManifestNameFromLiveFiles();
+
+  Put(Key(0), "val");
+  Flush();
+  Put(Key(1), "val");
+  SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::LogAndApply:WriteManifest", [&](void *) {
+    fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+  s = Flush();
+  ASSERT_EQ(s.severity(), rocksdb::Status::Severity::kHardError);
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+  fault_env->SetFilesystemActive(true);
+  s = dbfull()->Resume();
+  ASSERT_EQ(s, Status::OK());
+
+  new_manifest = GetManifestNameFromLiveFiles();
+  ASSERT_NE(new_manifest, old_manifest);
+
+  Reopen(options);
+  ASSERT_EQ("val", Get(Key(0)));
+  ASSERT_EQ("val", Get(Key(1)));
+  Close();
+}
+
+TEST_F(DBErrorHandlingTest, DoubleManifestWriteError) {
+  std::unique_ptr<FaultInjectionTestEnv> fault_env(
+      new FaultInjectionTestEnv(Env::Default()));
+  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.env = fault_env.get();
+  options.listeners.emplace_back(listener);
+  Status s;
+  std::string old_manifest;
+  std::string new_manifest;
+
+  listener->EnableAutoRecovery(false);
+  DestroyAndReopen(options);
+  old_manifest = GetManifestNameFromLiveFiles();
+
+  Put(Key(0), "val");
+  Flush();
+  Put(Key(1), "val");
+  SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::LogAndApply:WriteManifest", [&](void *) {
+    fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+  });
+  SyncPoint::GetInstance()->EnableProcessing();
+  s = Flush();
+  ASSERT_EQ(s.severity(), rocksdb::Status::Severity::kHardError);
+  fault_env->SetFilesystemActive(true);
+
+  // This Resume() will attempt to create a new manifest file and fail again
+  s = dbfull()->Resume();
+  ASSERT_EQ(s.severity(), rocksdb::Status::Severity::kHardError);
+  fault_env->SetFilesystemActive(true);
+  SyncPoint::GetInstance()->ClearAllCallBacks();
+  SyncPoint::GetInstance()->DisableProcessing();
+
+  // A successful Resume() will create a new manifest file
+  s = dbfull()->Resume();
+  ASSERT_EQ(s, Status::OK());
+
+  new_manifest = GetManifestNameFromLiveFiles();
+  ASSERT_NE(new_manifest, old_manifest);
+
+  Reopen(options);
+  ASSERT_EQ("val", Get(Key(0)));
+  ASSERT_EQ("val", Get(Key(1)));
+  Close();
+}
+
+TEST_F(DBErrorHandlingTest, CompactionManifestWriteError) {
+  std::unique_ptr<FaultInjectionTestEnv> fault_env(
+      new FaultInjectionTestEnv(Env::Default()));
+  std::shared_ptr<ErrorHandlerListener> listener(new ErrorHandlerListener());
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.level0_file_num_compaction_trigger = 2;
+  options.listeners.emplace_back(listener);
+  options.env = fault_env.get();
+  Status s;
+  std::string old_manifest;
+  std::string new_manifest;
+  std::atomic<bool> fail_manifest(false);
+  DestroyAndReopen(options);
+  old_manifest = GetManifestNameFromLiveFiles();
+
+  Put(Key(0), "val");
+  Put(Key(2), "val");
+  s = Flush();
+  ASSERT_EQ(s, Status::OK());
+
+  rocksdb::SyncPoint::GetInstance()->LoadDependency(
+      // Wait for flush of 2nd L0 file before starting compaction
+      {{"FlushMemTableFinished",
+        "BackgroundCallCompaction:0"},
+      // Wait for compaction to detect manifest write error
+       {"BackgroundCallCompaction:1",
+        "CompactionManifestWriteError:0"},
+      // Make compaction thread wait for error to be cleared
+       {"CompactionManifestWriteError:1",
+        "DBImpl::BackgroundCallCompaction:FoundObsoleteFiles"},
+      // Wait for DB instance to clear bg_error before calling
+      // TEST_WaitForCompact
+       {"SstFileManagerImpl::ClearError",
+        "CompactionManifestWriteError:2"}});
+  // trigger manifest write failure in compaction thread
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "BackgroundCallCompaction:0", [&](void *) {
+      fail_manifest.store(true);
+      });
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "VersionSet::LogAndApply:WriteManifest", [&](void *) {
+      if (fail_manifest.load()) {
+        fault_env->SetFilesystemActive(false, Status::NoSpace("Out of space"));
+      }
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  Put(Key(1), "val");
+  // This Flush will trigger a compaction, which will fail when appending to
+  // the manifest
+  s = Flush();
+  ASSERT_EQ(s, Status::OK());
+
+  TEST_SYNC_POINT("CompactionManifestWriteError:0");
+  // Clear all errors so when the compaction is retried, it will succeed
+  fault_env->SetFilesystemActive(true);
+  rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
+  TEST_SYNC_POINT("CompactionManifestWriteError:1");
+  TEST_SYNC_POINT("CompactionManifestWriteError:2");
+
+  s = dbfull()->TEST_WaitForCompact();
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+  ASSERT_EQ(s, Status::OK());
+
+  new_manifest = GetManifestNameFromLiveFiles();
+  ASSERT_NE(new_manifest, old_manifest);
+  Reopen(options);
+  ASSERT_EQ("val", Get(Key(0)));
+  ASSERT_EQ("val", Get(Key(1)));
+  ASSERT_EQ("val", Get(Key(2)));
+  Close();
+}
+
 TEST_F(DBErrorHandlingTest, CompactionWriteError) {
   std::unique_ptr<FaultInjectionTestEnv> fault_env(
       new FaultInjectionTestEnv(Env::Default()));

db/forward_iterator.cc

@@ -239,9 +239,13 @@ void ForwardIterator::SVCleanup(DBImpl* db, SuperVersion* sv,
     db->FindObsoleteFiles(&job_context, false, true);
     if (background_purge_on_iterator_cleanup) {
       db->ScheduleBgLogWriterClose(&job_context);
+      db->AddSuperVersionsToFreeQueue(sv);
+      db->SchedulePurge();
     }
     db->mutex_.Unlock();
-    delete sv;
+    if (!background_purge_on_iterator_cleanup) {
+      delete sv;
+    }
     if (job_context.HaveSomethingToDelete()) {
       db->PurgeObsoleteFiles(job_context, background_purge_on_iterator_cleanup);
     }
@@ -614,7 +618,7 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
   Cleanup(refresh_sv);
   if (refresh_sv) {
     // New
-    sv_ = cfd_->GetReferencedSuperVersion(&(db_->mutex_));
+    sv_ = cfd_->GetReferencedSuperVersion(db_);
   }
   ReadRangeDelAggregator range_del_agg(&cfd_->internal_comparator(),
                                        kMaxSequenceNumber /* upper_bound */);
@@ -668,7 +672,7 @@ void ForwardIterator::RebuildIterators(bool refresh_sv) {
 void ForwardIterator::RenewIterators() {
   SuperVersion* svnew;
   assert(sv_);
-  svnew = cfd_->GetReferencedSuperVersion(&(db_->mutex_));
+  svnew = cfd_->GetReferencedSuperVersion(db_);
 
   if (mutable_iter_ != nullptr) {
     DeleteIterator(mutable_iter_, true /* is_arena */);

db/memtable.h

@@ -138,7 +138,7 @@ class MemTable {
 
   // As a cheap version of `ApproximateMemoryUsage()`, this function doens't
   // require external synchronization. The value may be less accurate though
-  size_t ApproximateMemoryUsageFast() {
+  size_t ApproximateMemoryUsageFast() const {
     return approximate_memory_usage_.load(std::memory_order_relaxed);
   }
 

db/memtable_list.cc

@@ -280,7 +280,7 @@ void MemTableListVersion::Remove(MemTable* m,
 }
 
 // return the total memory usage assuming the oldest flushed memtable is dropped
-size_t MemTableListVersion::ApproximateMemoryUsageExcludingLast() {
+size_t MemTableListVersion::ApproximateMemoryUsageExcludingLast() const {
   size_t total_memtable_size = 0;
   for (auto& memtable : memlist_) {
     total_memtable_size += memtable->ApproximateMemoryUsage();
@@ -503,7 +503,7 @@ Status MemTableList::TryInstallMemtableFlushResults(
                            cfd->GetName().c_str(), m->file_number_, mem_id);
           assert(m->file_number_ > 0);
           current_->Remove(m, to_delete);
-          UpdateMemoryUsageExcludingLast();
+          UpdateCachedValuesFromMemTableListVersion();
           ResetTrimHistoryNeeded();
           ++mem_id;
         }
@@ -544,14 +544,14 @@ void MemTableList::Add(MemTable* m, autovector<MemTable*>* to_delete) {
   if (num_flush_not_started_ == 1) {
     imm_flush_needed.store(true, std::memory_order_release);
   }
-  UpdateMemoryUsageExcludingLast();
+  UpdateCachedValuesFromMemTableListVersion();
   ResetTrimHistoryNeeded();
 }
 
 void MemTableList::TrimHistory(autovector<MemTable*>* to_delete, size_t usage) {
   InstallNewVersion();
   current_->TrimHistory(to_delete, usage);
-  UpdateMemoryUsageExcludingLast();
+  UpdateCachedValuesFromMemTableListVersion();
   ResetTrimHistoryNeeded();
 }
 
@@ -566,18 +566,25 @@ size_t MemTableList::ApproximateUnflushedMemTablesMemoryUsage() {
 
 size_t MemTableList::ApproximateMemoryUsage() { return current_memory_usage_; }
 
-size_t MemTableList::ApproximateMemoryUsageExcludingLast() {
-  size_t usage =
+size_t MemTableList::ApproximateMemoryUsageExcludingLast() const {
+  const size_t usage =
       current_memory_usage_excluding_last_.load(std::memory_order_relaxed);
   return usage;
 }
 
-// Update current_memory_usage_excluding_last_, need to call whenever state
-// changes for MemtableListVersion (whenever InstallNewVersion() is called)
-void MemTableList::UpdateMemoryUsageExcludingLast() {
-  size_t total_memtable_size = current_->ApproximateMemoryUsageExcludingLast();
+bool MemTableList::HasHistory() const {
+  const bool has_history = current_has_history_.load(std::memory_order_relaxed);
+  return has_history;
+}
+
+void MemTableList::UpdateCachedValuesFromMemTableListVersion() {
+  const size_t total_memtable_size =
+      current_->ApproximateMemoryUsageExcludingLast();
   current_memory_usage_excluding_last_.store(total_memtable_size,
                                              std::memory_order_relaxed);
+
+  const bool has_history = current_->HasHistory();
+  current_has_history_.store(has_history, std::memory_order_relaxed);
 }
 
 uint64_t MemTableList::ApproximateOldestKeyTime() const {
@@ -707,7 +714,7 @@ Status InstallMemtableAtomicFlushResults(
                          cfds[i]->GetName().c_str(), m->GetFileNumber(),
                          mem_id);
         imm->current_->Remove(m, to_delete);
-        imm->UpdateMemoryUsageExcludingLast();
+        imm->UpdateCachedValuesFromMemTableListVersion();
         imm->ResetTrimHistoryNeeded();
       }
     }
@@ -757,7 +764,7 @@ void MemTableList::RemoveOldMemTables(uint64_t log_number,
     }
   }
 
-  UpdateMemoryUsageExcludingLast();
+  UpdateCachedValuesFromMemTableListVersion();
   ResetTrimHistoryNeeded();
 }
 

db/memtable_list.h

@@ -162,7 +162,11 @@ class MemTableListVersion {
   // excluding the last MemTable in memlist_history_. The reason for excluding
   // the last MemTable is to see if dropping the last MemTable will keep total
   // memory usage above or equal to max_write_buffer_size_to_maintain_
-  size_t ApproximateMemoryUsageExcludingLast();
+  size_t ApproximateMemoryUsageExcludingLast() const;
+
+  // Whether this version contains flushed memtables that are only kept around
+  // for transaction conflict checking.
+  bool HasHistory() const { return !memlist_history_.empty(); }
 
   bool MemtableLimitExceeded(size_t usage);
 
@@ -211,7 +215,8 @@ class MemTableList {
         commit_in_progress_(false),
         flush_requested_(false),
         current_memory_usage_(0),
-        current_memory_usage_excluding_last_(0) {
+        current_memory_usage_excluding_last_(0),
+        current_has_history_(false) {
     current_->Ref();
   }
 
@@ -266,11 +271,16 @@ class MemTableList {
   // Returns an estimate of the number of bytes of data in use.
   size_t ApproximateMemoryUsage();
 
-  // Returns the cached current_memory_usage_excluding_last_ value
-  size_t ApproximateMemoryUsageExcludingLast();
+  // Returns the cached current_memory_usage_excluding_last_ value.
+  size_t ApproximateMemoryUsageExcludingLast() const;
 
-  // Update current_memory_usage_excluding_last_ from MemtableListVersion
-  void UpdateMemoryUsageExcludingLast();
+  // Returns the cached current_has_history_ value.
+  bool HasHistory() const;
+
+  // Updates current_memory_usage_excluding_last_ and current_has_history_
+  // from MemTableListVersion. Must be called whenever InstallNewVersion is
+  // called.
+  void UpdateCachedValuesFromMemTableListVersion();
 
   // `usage` is the current size of the mutable Memtable. When
   // max_write_buffer_size_to_maintain is used, total size of mutable and
@@ -388,7 +398,11 @@ class MemTableList {
   // The current memory usage.
   size_t current_memory_usage_;
 
+  // Cached value of current_->ApproximateMemoryUsageExcludingLast().
   std::atomic<size_t> current_memory_usage_excluding_last_;
+
+  // Cached value of current_->HasHistory().
+  std::atomic<bool> current_has_history_;
 };
 
 // Installs memtable atomic flush results.

db/version_set.cc

@@ -3942,12 +3942,15 @@ Status VersionSet::ProcessManifestWrites(
     for (auto v : versions) {
       delete v;
     }
+    // If manifest append failed for whatever reason, the file could be
+    // corrupted. So we need to force the next version update to start a
+    // new manifest file.
+    descriptor_log_.reset();
     if (new_descriptor_log) {
       ROCKS_LOG_INFO(db_options_->info_log,
                      "Deleting manifest %" PRIu64 " current manifest %" PRIu64
                      "\n",
                      manifest_file_number_, pending_manifest_file_number_);
-      descriptor_log_.reset();
       env_->DeleteFile(
           DescriptorFileName(dbname_, pending_manifest_file_number_));
     }

db/write_batch.cc

@@ -1784,14 +1784,28 @@ class MemTableInserter : public WriteBatch::Handler {
     // check if memtable_list size exceeds max_write_buffer_size_to_maintain
     if (trim_history_scheduler_ != nullptr) {
       auto* cfd = cf_mems_->current();
-      assert(cfd != nullptr);
-      if (cfd->ioptions()->max_write_buffer_size_to_maintain > 0 &&
-          cfd->mem()->ApproximateMemoryUsageFast() +
-                  cfd->imm()->ApproximateMemoryUsageExcludingLast() >=
-              static_cast<size_t>(
-                  cfd->ioptions()->max_write_buffer_size_to_maintain) &&
-          cfd->imm()->MarkTrimHistoryNeeded()) {
-        trim_history_scheduler_->ScheduleWork(cfd);
+
+      assert(cfd);
+      assert(cfd->ioptions());
+
+      const size_t size_to_maintain = static_cast<size_t>(
+          cfd->ioptions()->max_write_buffer_size_to_maintain);
+
+      if (size_to_maintain > 0) {
+        MemTableList* const imm = cfd->imm();
+        assert(imm);
+
+        if (imm->HasHistory()) {
+          const MemTable* const mem = cfd->mem();
+          assert(mem);
+
+          if (mem->ApproximateMemoryUsageFast() +
+                      imm->ApproximateMemoryUsageExcludingLast() >=
+                  size_to_maintain &&
+              imm->MarkTrimHistoryNeeded()) {
+            trim_history_scheduler_->ScheduleWork(cfd);
+          }
+        }
       }
     }
   }

db/write_thread.cc

@@ -344,6 +344,9 @@ void WriteThread::BeginWriteStall() {
       prev->link_older = w->link_older;
       w->status = Status::Incomplete("Write stall");
       SetState(w, STATE_COMPLETED);
+      if (prev->link_older) {
+        prev->link_older->link_newer = prev;
+      }
       w = prev->link_older;
     } else {
       prev = w;
@@ -355,7 +358,11 @@ void WriteThread::BeginWriteStall() {
 void WriteThread::EndWriteStall() {
   MutexLock lock(&stall_mu_);
 
+  // Unlink write_stall_dummy_ from the write queue. This will unblock
+  // pending write threads to enqueue themselves
   assert(newest_writer_.load(std::memory_order_relaxed) == &write_stall_dummy_);
+  assert(write_stall_dummy_.link_older != nullptr);
+  write_stall_dummy_.link_older->link_newer = write_stall_dummy_.link_newer;
   newest_writer_.exchange(write_stall_dummy_.link_older);
 
   // Wake up writers

db_stress_tool/CMakeLists.txt

@@ -1,14 +0,0 @@
-add_executable(db_stress${ARTIFACT_SUFFIX}
-  db_stress.cc
-  db_stress_tool.cc
-  batched_ops_stress.cc
-  cf_consistency_stress.cc
-  db_stress_common.cc
-  db_stress_driver.cc
-  db_stress_test_base.cc
-  db_stress_shared_state.cc
-  db_stress_gflags.cc
-  db_stress_tool.cc
-  no_batched_ops_stress.cc)
-target_link_libraries(db_stress${ARTIFACT_SUFFIX} ${ROCKSDB_LIB})
-list(APPEND tool_deps db_stress)

db_stress_tool/batched_ops_stress.cc

@@ -1,340 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifdef GFLAGS
-#include "db_stress_tool/db_stress_common.h"
-
-namespace rocksdb {
-class BatchedOpsStressTest : public StressTest {
- public:
-  BatchedOpsStressTest() {}
-  virtual ~BatchedOpsStressTest() {}
-
-  // Given a key K and value V, this puts ("0"+K, "0"+V), ("1"+K, "1"+V), ...
-  // ("9"+K, "9"+V) in DB atomically i.e in a single batch.
-  // Also refer BatchedOpsStressTest::TestGet
-  Status TestPut(ThreadState* thread, WriteOptions& write_opts,
-                 const ReadOptions& /* read_opts */,
-                 const std::vector<int>& rand_column_families,
-                 const std::vector<int64_t>& rand_keys, char (&value)[100],
-                 std::unique_ptr<MutexLock>& /* lock */) override {
-    uint32_t value_base =
-        thread->rand.Next() % thread->shared->UNKNOWN_SENTINEL;
-    size_t sz = GenerateValue(value_base, value, sizeof(value));
-    Slice v(value, sz);
-    std::string keys[10] = {"9", "8", "7", "6", "5", "4", "3", "2", "1", "0"};
-    std::string values[10] = {"9", "8", "7", "6", "5", "4", "3", "2", "1", "0"};
-    Slice value_slices[10];
-    WriteBatch batch;
-    Status s;
-    auto cfh = column_families_[rand_column_families[0]];
-    std::string key_str = Key(rand_keys[0]);
-    for (int i = 0; i < 10; i++) {
-      keys[i] += key_str;
-      values[i] += v.ToString();
-      value_slices[i] = values[i];
-      if (FLAGS_use_merge) {
-        batch.Merge(cfh, keys[i], value_slices[i]);
-      } else {
-        batch.Put(cfh, keys[i], value_slices[i]);
-      }
-    }
-
-    s = db_->Write(write_opts, &batch);
-    if (!s.ok()) {
-      fprintf(stderr, "multiput error: %s\n", s.ToString().c_str());
-      thread->stats.AddErrors(1);
-    } else {
-      // we did 10 writes each of size sz + 1
-      thread->stats.AddBytesForWrites(10, (sz + 1) * 10);
-    }
-
-    return s;
-  }
-
-  // Given a key K, this deletes ("0"+K), ("1"+K),... ("9"+K)
-  // in DB atomically i.e in a single batch. Also refer MultiGet.
-  Status TestDelete(ThreadState* thread, WriteOptions& writeoptions,
-                    const std::vector<int>& rand_column_families,
-                    const std::vector<int64_t>& rand_keys,
-                    std::unique_ptr<MutexLock>& /* lock */) override {
-    std::string keys[10] = {"9", "7", "5", "3", "1", "8", "6", "4", "2", "0"};
-
-    WriteBatch batch;
-    Status s;
-    auto cfh = column_families_[rand_column_families[0]];
-    std::string key_str = Key(rand_keys[0]);
-    for (int i = 0; i < 10; i++) {
-      keys[i] += key_str;
-      batch.Delete(cfh, keys[i]);
-    }
-
-    s = db_->Write(writeoptions, &batch);
-    if (!s.ok()) {
-      fprintf(stderr, "multidelete error: %s\n", s.ToString().c_str());
-      thread->stats.AddErrors(1);
-    } else {
-      thread->stats.AddDeletes(10);
-    }
-
-    return s;
-  }
-
-  Status TestDeleteRange(ThreadState* /* thread */,
-                         WriteOptions& /* write_opts */,
-                         const std::vector<int>& /* rand_column_families */,
-                         const std::vector<int64_t>& /* rand_keys */,
-                         std::unique_ptr<MutexLock>& /* lock */) override {
-    assert(false);
-    return Status::NotSupported(
-        "BatchedOpsStressTest does not support "
-        "TestDeleteRange");
-  }
-
-  void TestIngestExternalFile(
-      ThreadState* /* thread */,
-      const std::vector<int>& /* rand_column_families */,
-      const std::vector<int64_t>& /* rand_keys */,
-      std::unique_ptr<MutexLock>& /* lock */) override {
-    assert(false);
-    fprintf(stderr,
-            "BatchedOpsStressTest does not support "
-            "TestIngestExternalFile\n");
-    std::terminate();
-  }
-
-  // Given a key K, this gets values for "0"+K, "1"+K,..."9"+K
-  // in the same snapshot, and verifies that all the values are of the form
-  // "0"+V, "1"+V,..."9"+V.
-  // ASSUMES that BatchedOpsStressTest::TestPut was used to put (K, V) into
-  // the DB.
-  Status TestGet(ThreadState* thread, const ReadOptions& readoptions,
-                 const std::vector<int>& rand_column_families,
-                 const std::vector<int64_t>& rand_keys) override {
-    std::string keys[10] = {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"};
-    Slice key_slices[10];
-    std::string values[10];
-    ReadOptions readoptionscopy = readoptions;
-    readoptionscopy.snapshot = db_->GetSnapshot();
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    auto cfh = column_families_[rand_column_families[0]];
-    std::string from_db;
-    Status s;
-    for (int i = 0; i < 10; i++) {
-      keys[i] += key.ToString();
-      key_slices[i] = keys[i];
-      s = db_->Get(readoptionscopy, cfh, key_slices[i], &from_db);
-      if (!s.ok() && !s.IsNotFound()) {
-        fprintf(stderr, "get error: %s\n", s.ToString().c_str());
-        values[i] = "";
-        thread->stats.AddErrors(1);
-        // we continue after error rather than exiting so that we can
-        // find more errors if any
-      } else if (s.IsNotFound()) {
-        values[i] = "";
-        thread->stats.AddGets(1, 0);
-      } else {
-        values[i] = from_db;
-
-        char expected_prefix = (keys[i])[0];
-        char actual_prefix = (values[i])[0];
-        if (actual_prefix != expected_prefix) {
-          fprintf(stderr, "error expected prefix = %c actual = %c\n",
-                  expected_prefix, actual_prefix);
-        }
-        (values[i])[0] = ' ';  // blank out the differing character
-        thread->stats.AddGets(1, 1);
-      }
-    }
-    db_->ReleaseSnapshot(readoptionscopy.snapshot);
-
-    // Now that we retrieved all values, check that they all match
-    for (int i = 1; i < 10; i++) {
-      if (values[i] != values[0]) {
-        fprintf(stderr, "error : inconsistent values for key %s: %s, %s\n",
-                key.ToString(true).c_str(), StringToHex(values[0]).c_str(),
-                StringToHex(values[i]).c_str());
-        // we continue after error rather than exiting so that we can
-        // find more errors if any
-      }
-    }
-
-    return s;
-  }
-
-  std::vector<Status> TestMultiGet(
-      ThreadState* thread, const ReadOptions& readoptions,
-      const std::vector<int>& rand_column_families,
-      const std::vector<int64_t>& rand_keys) override {
-    size_t num_keys = rand_keys.size();
-    std::vector<Status> ret_status(num_keys);
-    std::array<std::string, 10> keys = {"0", "1", "2", "3", "4",
-                                        "5", "6", "7", "8", "9"};
-    size_t num_prefixes = keys.size();
-    for (size_t rand_key = 0; rand_key < num_keys; ++rand_key) {
-      std::vector<Slice> key_slices;
-      std::vector<PinnableSlice> values(num_prefixes);
-      std::vector<Status> statuses(num_prefixes);
-      ReadOptions readoptionscopy = readoptions;
-      readoptionscopy.snapshot = db_->GetSnapshot();
-      std::vector<std::string> key_str;
-      key_str.reserve(num_prefixes);
-      key_slices.reserve(num_prefixes);
-      std::string from_db;
-      ColumnFamilyHandle* cfh = column_families_[rand_column_families[0]];
-
-      for (size_t key = 0; key < num_prefixes; ++key) {
-        key_str.emplace_back(keys[key] + Key(rand_keys[rand_key]));
-        key_slices.emplace_back(key_str.back());
-      }
-      db_->MultiGet(readoptionscopy, cfh, num_prefixes, key_slices.data(),
-                    values.data(), statuses.data());
-      for (size_t i = 0; i < num_prefixes; i++) {
-        Status s = statuses[i];
-        if (!s.ok() && !s.IsNotFound()) {
-          fprintf(stderr, "get error: %s\n", s.ToString().c_str());
-          thread->stats.AddErrors(1);
-          ret_status[rand_key] = s;
-          // we continue after error rather than exiting so that we can
-          // find more errors if any
-        } else if (s.IsNotFound()) {
-          thread->stats.AddGets(1, 0);
-          ret_status[rand_key] = s;
-        } else {
-          char expected_prefix = (keys[i])[0];
-          char actual_prefix = (values[i])[0];
-          if (actual_prefix != expected_prefix) {
-            fprintf(stderr, "error expected prefix = %c actual = %c\n",
-                    expected_prefix, actual_prefix);
-          }
-          std::string str;
-          str.assign(values[i].data(), values[i].size());
-          values[i].Reset();
-          str[0] = ' ';  // blank out the differing character
-          values[i].PinSelf(str);
-          thread->stats.AddGets(1, 1);
-        }
-      }
-      db_->ReleaseSnapshot(readoptionscopy.snapshot);
-
-      // Now that we retrieved all values, check that they all match
-      for (size_t i = 1; i < num_prefixes; i++) {
-        if (values[i] != values[0]) {
-          fprintf(stderr, "error : inconsistent values for key %s: %s, %s\n",
-                  key_str[i].c_str(), StringToHex(values[0].ToString()).c_str(),
-                  StringToHex(values[i].ToString()).c_str());
-          // we continue after error rather than exiting so that we can
-          // find more errors if any
-        }
-      }
-    }
-
-    return ret_status;
-  }
-
-  // Given a key, this does prefix scans for "0"+P, "1"+P,..."9"+P
-  // in the same snapshot where P is the first FLAGS_prefix_size - 1 bytes
-  // of the key. Each of these 10 scans returns a series of values;
-  // each series should be the same length, and it is verified for each
-  // index i that all the i'th values are of the form "0"+V, "1"+V,..."9"+V.
-  // ASSUMES that MultiPut was used to put (K, V)
-  Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions,
-                        const std::vector<int>& rand_column_families,
-                        const std::vector<int64_t>& rand_keys) override {
-    size_t prefix_to_use =
-        (FLAGS_prefix_size < 0) ? 7 : static_cast<size_t>(FLAGS_prefix_size);
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    auto cfh = column_families_[rand_column_families[0]];
-    std::string prefixes[10] = {"0", "1", "2", "3", "4",
-                                "5", "6", "7", "8", "9"};
-    Slice prefix_slices[10];
-    ReadOptions readoptionscopy[10];
-    const Snapshot* snapshot = db_->GetSnapshot();
-    Iterator* iters[10];
-    std::string upper_bounds[10];
-    Slice ub_slices[10];
-    Status s = Status::OK();
-    for (int i = 0; i < 10; i++) {
-      prefixes[i] += key.ToString();
-      prefixes[i].resize(prefix_to_use);
-      prefix_slices[i] = Slice(prefixes[i]);
-      readoptionscopy[i] = readoptions;
-      readoptionscopy[i].snapshot = snapshot;
-      if (thread->rand.OneIn(2) &&
-          GetNextPrefix(prefix_slices[i], &(upper_bounds[i]))) {
-        // For half of the time, set the upper bound to the next prefix
-        ub_slices[i] = Slice(upper_bounds[i]);
-        readoptionscopy[i].iterate_upper_bound = &(ub_slices[i]);
-      }
-      iters[i] = db_->NewIterator(readoptionscopy[i], cfh);
-      iters[i]->Seek(prefix_slices[i]);
-    }
-
-    long count = 0;
-    while (iters[0]->Valid() && iters[0]->key().starts_with(prefix_slices[0])) {
-      count++;
-      std::string values[10];
-      // get list of all values for this iteration
-      for (int i = 0; i < 10; i++) {
-        // no iterator should finish before the first one
-        assert(iters[i]->Valid() &&
-               iters[i]->key().starts_with(prefix_slices[i]));
-        values[i] = iters[i]->value().ToString();
-
-        char expected_first = (prefixes[i])[0];
-        char actual_first = (values[i])[0];
-
-        if (actual_first != expected_first) {
-          fprintf(stderr, "error expected first = %c actual = %c\n",
-                  expected_first, actual_first);
-        }
-        (values[i])[0] = ' ';  // blank out the differing character
-      }
-      // make sure all values are equivalent
-      for (int i = 0; i < 10; i++) {
-        if (values[i] != values[0]) {
-          fprintf(stderr,
-                  "error : %d, inconsistent values for prefix %s: %s, %s\n", i,
-                  prefixes[i].c_str(), StringToHex(values[0]).c_str(),
-                  StringToHex(values[i]).c_str());
-          // we continue after error rather than exiting so that we can
-          // find more errors if any
-        }
-        iters[i]->Next();
-      }
-    }
-
-    // cleanup iterators and snapshot
-    for (int i = 0; i < 10; i++) {
-      // if the first iterator finished, they should have all finished
-      assert(!iters[i]->Valid() ||
-             !iters[i]->key().starts_with(prefix_slices[i]));
-      assert(iters[i]->status().ok());
-      delete iters[i];
-    }
-    db_->ReleaseSnapshot(snapshot);
-
-    if (s.ok()) {
-      thread->stats.AddPrefixes(1, count);
-    } else {
-      thread->stats.AddErrors(1);
-    }
-
-    return s;
-  }
-
-  void VerifyDb(ThreadState* /* thread */) const override {}
-};
-
-StressTest* CreateBatchedOpsStressTest() { return new BatchedOpsStressTest(); }
-
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/cf_consistency_stress.cc

@@ -1,508 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifdef GFLAGS
-#include "db_stress_tool/db_stress_common.h"
-
-namespace rocksdb {
-class CfConsistencyStressTest : public StressTest {
- public:
-  CfConsistencyStressTest() : batch_id_(0) {}
-
-  virtual ~CfConsistencyStressTest() {}
-
-  Status TestPut(ThreadState* thread, WriteOptions& write_opts,
-                 const ReadOptions& /* read_opts */,
-                 const std::vector<int>& rand_column_families,
-                 const std::vector<int64_t>& rand_keys, char (&value)[100],
-                 std::unique_ptr<MutexLock>& /* lock */) override {
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    uint64_t value_base = batch_id_.fetch_add(1);
-    size_t sz =
-        GenerateValue(static_cast<uint32_t>(value_base), value, sizeof(value));
-    Slice v(value, sz);
-    WriteBatch batch;
-    for (auto cf : rand_column_families) {
-      ColumnFamilyHandle* cfh = column_families_[cf];
-      if (FLAGS_use_merge) {
-        batch.Merge(cfh, key, v);
-      } else { /* !FLAGS_use_merge */
-        batch.Put(cfh, key, v);
-      }
-    }
-    Status s = db_->Write(write_opts, &batch);
-    if (!s.ok()) {
-      fprintf(stderr, "multi put or merge error: %s\n", s.ToString().c_str());
-      thread->stats.AddErrors(1);
-    } else {
-      auto num = static_cast<long>(rand_column_families.size());
-      thread->stats.AddBytesForWrites(num, (sz + 1) * num);
-    }
-
-    return s;
-  }
-
-  Status TestDelete(ThreadState* thread, WriteOptions& write_opts,
-                    const std::vector<int>& rand_column_families,
-                    const std::vector<int64_t>& rand_keys,
-                    std::unique_ptr<MutexLock>& /* lock */) override {
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    WriteBatch batch;
-    for (auto cf : rand_column_families) {
-      ColumnFamilyHandle* cfh = column_families_[cf];
-      batch.Delete(cfh, key);
-    }
-    Status s = db_->Write(write_opts, &batch);
-    if (!s.ok()) {
-      fprintf(stderr, "multidel error: %s\n", s.ToString().c_str());
-      thread->stats.AddErrors(1);
-    } else {
-      thread->stats.AddDeletes(static_cast<long>(rand_column_families.size()));
-    }
-    return s;
-  }
-
-  Status TestDeleteRange(ThreadState* thread, WriteOptions& write_opts,
-                         const std::vector<int>& rand_column_families,
-                         const std::vector<int64_t>& rand_keys,
-                         std::unique_ptr<MutexLock>& /* lock */) override {
-    int64_t rand_key = rand_keys[0];
-    auto shared = thread->shared;
-    int64_t max_key = shared->GetMaxKey();
-    if (rand_key > max_key - FLAGS_range_deletion_width) {
-      rand_key =
-          thread->rand.Next() % (max_key - FLAGS_range_deletion_width + 1);
-    }
-    std::string key_str = Key(rand_key);
-    Slice key = key_str;
-    std::string end_key_str = Key(rand_key + FLAGS_range_deletion_width);
-    Slice end_key = end_key_str;
-    WriteBatch batch;
-    for (auto cf : rand_column_families) {
-      ColumnFamilyHandle* cfh = column_families_[rand_column_families[cf]];
-      batch.DeleteRange(cfh, key, end_key);
-    }
-    Status s = db_->Write(write_opts, &batch);
-    if (!s.ok()) {
-      fprintf(stderr, "multi del range error: %s\n", s.ToString().c_str());
-      thread->stats.AddErrors(1);
-    } else {
-      thread->stats.AddRangeDeletions(
-          static_cast<long>(rand_column_families.size()));
-    }
-    return s;
-  }
-
-  void TestIngestExternalFile(
-      ThreadState* /* thread */,
-      const std::vector<int>& /* rand_column_families */,
-      const std::vector<int64_t>& /* rand_keys */,
-      std::unique_ptr<MutexLock>& /* lock */) override {
-    assert(false);
-    fprintf(stderr,
-            "CfConsistencyStressTest does not support TestIngestExternalFile "
-            "because it's not possible to verify the result\n");
-    std::terminate();
-  }
-
-  Status TestGet(ThreadState* thread, const ReadOptions& readoptions,
-                 const std::vector<int>& rand_column_families,
-                 const std::vector<int64_t>& rand_keys) override {
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    Status s;
-    bool is_consistent = true;
-
-    if (thread->rand.OneIn(2)) {
-      // 1/2 chance, does a random read from random CF
-      auto cfh =
-          column_families_[rand_column_families[thread->rand.Next() %
-                                                rand_column_families.size()]];
-      std::string from_db;
-      s = db_->Get(readoptions, cfh, key, &from_db);
-    } else {
-      // 1/2 chance, comparing one key is the same across all CFs
-      const Snapshot* snapshot = db_->GetSnapshot();
-      ReadOptions readoptionscopy = readoptions;
-      readoptionscopy.snapshot = snapshot;
-
-      std::string value0;
-      s = db_->Get(readoptionscopy, column_families_[rand_column_families[0]],
-                   key, &value0);
-      if (s.ok() || s.IsNotFound()) {
-        bool found = s.ok();
-        for (size_t i = 1; i < rand_column_families.size(); i++) {
-          std::string value1;
-          s = db_->Get(readoptionscopy,
-                       column_families_[rand_column_families[i]], key, &value1);
-          if (!s.ok() && !s.IsNotFound()) {
-            break;
-          }
-          if (!found && s.ok()) {
-            fprintf(stderr, "Get() return different results with key %s\n",
-                    Slice(key_str).ToString(true).c_str());
-            fprintf(stderr, "CF %s is not found\n",
-                    column_family_names_[0].c_str());
-            fprintf(stderr, "CF %s returns value %s\n",
-                    column_family_names_[i].c_str(),
-                    Slice(value1).ToString(true).c_str());
-            is_consistent = false;
-          } else if (found && s.IsNotFound()) {
-            fprintf(stderr, "Get() return different results with key %s\n",
-                    Slice(key_str).ToString(true).c_str());
-            fprintf(stderr, "CF %s returns value %s\n",
-                    column_family_names_[0].c_str(),
-                    Slice(value0).ToString(true).c_str());
-            fprintf(stderr, "CF %s is not found\n",
-                    column_family_names_[i].c_str());
-            is_consistent = false;
-          } else if (s.ok() && value0 != value1) {
-            fprintf(stderr, "Get() return different results with key %s\n",
-                    Slice(key_str).ToString(true).c_str());
-            fprintf(stderr, "CF %s returns value %s\n",
-                    column_family_names_[0].c_str(),
-                    Slice(value0).ToString(true).c_str());
-            fprintf(stderr, "CF %s returns value %s\n",
-                    column_family_names_[i].c_str(),
-                    Slice(value1).ToString(true).c_str());
-            is_consistent = false;
-          }
-          if (!is_consistent) {
-            break;
-          }
-        }
-      }
-
-      db_->ReleaseSnapshot(snapshot);
-    }
-    if (!is_consistent) {
-      thread->stats.AddErrors(1);
-      // Fail fast to preserve the DB state.
-      thread->shared->SetVerificationFailure();
-    } else if (s.ok()) {
-      thread->stats.AddGets(1, 1);
-    } else if (s.IsNotFound()) {
-      thread->stats.AddGets(1, 0);
-    } else {
-      thread->stats.AddErrors(1);
-    }
-    return s;
-  }
-
-  std::vector<Status> TestMultiGet(
-      ThreadState* thread, const ReadOptions& read_opts,
-      const std::vector<int>& rand_column_families,
-      const std::vector<int64_t>& rand_keys) override {
-    size_t num_keys = rand_keys.size();
-    std::vector<std::string> key_str;
-    std::vector<Slice> keys;
-    keys.reserve(num_keys);
-    key_str.reserve(num_keys);
-    std::vector<PinnableSlice> values(num_keys);
-    std::vector<Status> statuses(num_keys);
-    ColumnFamilyHandle* cfh = column_families_[rand_column_families[0]];
-
-    for (size_t i = 0; i < num_keys; ++i) {
-      key_str.emplace_back(Key(rand_keys[i]));
-      keys.emplace_back(key_str.back());
-    }
-    db_->MultiGet(read_opts, cfh, num_keys, keys.data(), values.data(),
-                  statuses.data());
-    for (auto s : statuses) {
-      if (s.ok()) {
-        // found case
-        thread->stats.AddGets(1, 1);
-      } else if (s.IsNotFound()) {
-        // not found case
-        thread->stats.AddGets(1, 0);
-      } else {
-        // errors case
-        thread->stats.AddErrors(1);
-      }
-    }
-    return statuses;
-  }
-
-  Status TestPrefixScan(ThreadState* thread, const ReadOptions& readoptions,
-                        const std::vector<int>& rand_column_families,
-                        const std::vector<int64_t>& rand_keys) override {
-    size_t prefix_to_use =
-        (FLAGS_prefix_size < 0) ? 7 : static_cast<size_t>(FLAGS_prefix_size);
-
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    Slice prefix = Slice(key.data(), prefix_to_use);
-
-    std::string upper_bound;
-    Slice ub_slice;
-    ReadOptions ro_copy = readoptions;
-    if (thread->rand.OneIn(2) && GetNextPrefix(prefix, &upper_bound)) {
-      ub_slice = Slice(upper_bound);
-      ro_copy.iterate_upper_bound = &ub_slice;
-    }
-    auto cfh =
-        column_families_[rand_column_families[thread->rand.Next() %
-                                              rand_column_families.size()]];
-    Iterator* iter = db_->NewIterator(ro_copy, cfh);
-    long count = 0;
-    for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix);
-         iter->Next()) {
-      ++count;
-    }
-    assert(prefix_to_use == 0 ||
-           count <= (static_cast<long>(1) << ((8 - prefix_to_use) * 8)));
-    Status s = iter->status();
-    if (s.ok()) {
-      thread->stats.AddPrefixes(1, count);
-    } else {
-      thread->stats.AddErrors(1);
-    }
-    delete iter;
-    return s;
-  }
-
-  ColumnFamilyHandle* GetControlCfh(ThreadState* thread,
-                                    int /*column_family_id*/
-                                    ) override {
-    // All column families should contain the same data. Randomly pick one.
-    return column_families_[thread->rand.Next() % column_families_.size()];
-  }
-
-#ifdef ROCKSDB_LITE
-  Status TestCheckpoint(ThreadState* /* thread */,
-                        const std::vector<int>& /* rand_column_families */,
-                        const std::vector<int64_t>& /* rand_keys */) override {
-    assert(false);
-    fprintf(stderr,
-            "RocksDB lite does not support "
-            "TestCheckpoint\n");
-    std::terminate();
-  }
-#else
-  Status TestCheckpoint(ThreadState* thread,
-                        const std::vector<int>& /* rand_column_families */,
-                        const std::vector<int64_t>& /* rand_keys */) override {
-    std::string checkpoint_dir =
-        FLAGS_db + "/.checkpoint" + ToString(thread->tid);
-    DestroyDB(checkpoint_dir, options_);
-    Checkpoint* checkpoint = nullptr;
-    Status s = Checkpoint::Create(db_, &checkpoint);
-    if (s.ok()) {
-      s = checkpoint->CreateCheckpoint(checkpoint_dir);
-    }
-    std::vector<ColumnFamilyHandle*> cf_handles;
-    DB* checkpoint_db = nullptr;
-    if (s.ok()) {
-      delete checkpoint;
-      checkpoint = nullptr;
-      Options options(options_);
-      options.listeners.clear();
-      std::vector<ColumnFamilyDescriptor> cf_descs;
-      // TODO(ajkr): `column_family_names_` is not safe to access here when
-      // `clear_column_family_one_in != 0`. But we can't easily switch to
-      // `ListColumnFamilies` to get names because it won't necessarily give
-      // the same order as `column_family_names_`.
-      if (FLAGS_clear_column_family_one_in == 0) {
-        for (const auto& name : column_family_names_) {
-          cf_descs.emplace_back(name, ColumnFamilyOptions(options));
-        }
-        s = DB::OpenForReadOnly(DBOptions(options), checkpoint_dir, cf_descs,
-                                &cf_handles, &checkpoint_db);
-      }
-    }
-    if (checkpoint_db != nullptr) {
-      for (auto cfh : cf_handles) {
-        delete cfh;
-      }
-      cf_handles.clear();
-      delete checkpoint_db;
-      checkpoint_db = nullptr;
-    }
-    DestroyDB(checkpoint_dir, options_);
-    if (!s.ok()) {
-      fprintf(stderr, "A checkpoint operation failed with: %s\n",
-              s.ToString().c_str());
-    }
-    return s;
-  }
-#endif  // !ROCKSDB_LITE
-
-  void VerifyDb(ThreadState* thread) const override {
-    ReadOptions options(FLAGS_verify_checksum, true);
-    // We must set total_order_seek to true because we are doing a SeekToFirst
-    // on a column family whose memtables may support (by default) prefix-based
-    // iterator. In this case, NewIterator with options.total_order_seek being
-    // false returns a prefix-based iterator. Calling SeekToFirst using this
-    // iterator causes the iterator to become invalid. That means we cannot
-    // iterate the memtable using this iterator any more, although the memtable
-    // contains the most up-to-date key-values.
-    options.total_order_seek = true;
-    assert(thread != nullptr);
-    auto shared = thread->shared;
-    std::vector<std::unique_ptr<Iterator>> iters(column_families_.size());
-    for (size_t i = 0; i != column_families_.size(); ++i) {
-      iters[i].reset(db_->NewIterator(options, column_families_[i]));
-    }
-    for (auto& iter : iters) {
-      iter->SeekToFirst();
-    }
-    size_t num = column_families_.size();
-    assert(num == iters.size());
-    std::vector<Status> statuses(num, Status::OK());
-    do {
-      if (shared->HasVerificationFailedYet()) {
-        break;
-      }
-      size_t valid_cnt = 0;
-      size_t idx = 0;
-      for (auto& iter : iters) {
-        if (iter->Valid()) {
-          ++valid_cnt;
-        } else {
-          statuses[idx] = iter->status();
-        }
-        ++idx;
-      }
-      if (valid_cnt == 0) {
-        Status status;
-        for (size_t i = 0; i != num; ++i) {
-          const auto& s = statuses[i];
-          if (!s.ok()) {
-            status = s;
-            fprintf(stderr, "Iterator on cf %s has error: %s\n",
-                    column_families_[i]->GetName().c_str(),
-                    s.ToString().c_str());
-            shared->SetVerificationFailure();
-          }
-        }
-        if (status.ok()) {
-          fprintf(stdout, "Finished scanning all column families.\n");
-        }
-        break;
-      } else if (valid_cnt != iters.size()) {
-        shared->SetVerificationFailure();
-        for (size_t i = 0; i != num; ++i) {
-          if (!iters[i]->Valid()) {
-            if (statuses[i].ok()) {
-              fprintf(stderr, "Finished scanning cf %s\n",
-                      column_families_[i]->GetName().c_str());
-            } else {
-              fprintf(stderr, "Iterator on cf %s has error: %s\n",
-                      column_families_[i]->GetName().c_str(),
-                      statuses[i].ToString().c_str());
-            }
-          } else {
-            fprintf(stderr, "cf %s has remaining data to scan\n",
-                    column_families_[i]->GetName().c_str());
-          }
-        }
-        break;
-      }
-      if (shared->HasVerificationFailedYet()) {
-        break;
-      }
-      // If the program reaches here, then all column families' iterators are
-      // still valid.
-      if (shared->PrintingVerificationResults()) {
-        continue;
-      }
-      Slice key;
-      Slice value;
-      int num_mismatched_cfs = 0;
-      for (size_t i = 0; i != num; ++i) {
-        if (i == 0) {
-          key = iters[i]->key();
-          value = iters[i]->value();
-        } else {
-          int cmp = key.compare(iters[i]->key());
-          if (cmp != 0) {
-            ++num_mismatched_cfs;
-            if (1 == num_mismatched_cfs) {
-              fprintf(stderr, "Verification failed\n");
-              fprintf(stderr, "Latest Sequence Number: %" PRIu64 "\n",
-                      db_->GetLatestSequenceNumber());
-              fprintf(stderr, "[%s] %s => %s\n",
-                      column_families_[0]->GetName().c_str(),
-                      key.ToString(true /* hex */).c_str(),
-                      value.ToString(true /* hex */).c_str());
-            }
-            fprintf(stderr, "[%s] %s => %s\n",
-                    column_families_[i]->GetName().c_str(),
-                    iters[i]->key().ToString(true /* hex */).c_str(),
-                    iters[i]->value().ToString(true /* hex */).c_str());
-#ifndef ROCKSDB_LITE
-            Slice begin_key;
-            Slice end_key;
-            if (cmp < 0) {
-              begin_key = key;
-              end_key = iters[i]->key();
-            } else {
-              begin_key = iters[i]->key();
-              end_key = key;
-            }
-            std::vector<KeyVersion> versions;
-            const size_t kMaxNumIKeys = 8;
-            const auto print_key_versions = [&](ColumnFamilyHandle* cfh) {
-              Status s = GetAllKeyVersions(db_, cfh, begin_key, end_key,
-                                           kMaxNumIKeys, &versions);
-              if (!s.ok()) {
-                fprintf(stderr, "%s\n", s.ToString().c_str());
-                return;
-              }
-              assert(nullptr != cfh);
-              fprintf(stderr,
-                      "Internal keys in CF '%s', [%s, %s] (max %" ROCKSDB_PRIszt
-                      ")\n",
-                      cfh->GetName().c_str(),
-                      begin_key.ToString(true /* hex */).c_str(),
-                      end_key.ToString(true /* hex */).c_str(), kMaxNumIKeys);
-              for (const KeyVersion& kv : versions) {
-                fprintf(stderr, "  key %s seq %" PRIu64 " type %d\n",
-                        Slice(kv.user_key).ToString(true).c_str(), kv.sequence,
-                        kv.type);
-              }
-            };
-            if (1 == num_mismatched_cfs) {
-              print_key_versions(column_families_[0]);
-            }
-            print_key_versions(column_families_[i]);
-#endif  // ROCKSDB_LITE
-            shared->SetVerificationFailure();
-          }
-        }
-      }
-      shared->FinishPrintingVerificationResults();
-      for (auto& iter : iters) {
-        iter->Next();
-      }
-    } while (true);
-  }
-
-  std::vector<int> GenerateColumnFamilies(
-      const int /* num_column_families */,
-      int /* rand_column_family */) const override {
-    std::vector<int> ret;
-    int num = static_cast<int>(column_families_.size());
-    int k = 0;
-    std::generate_n(back_inserter(ret), num, [&k]() -> int { return k++; });
-    return ret;
-  }
-
- private:
-  std::atomic<int64_t> batch_id_;
-};
-
-StressTest* CreateCfConsistencyStressTest() {
-  return new CfConsistencyStressTest();
-}
-
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_common.cc

@@ -1,108 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-
-#ifdef GFLAGS
-#include "db_stress_tool/db_stress_common.h"
-
-rocksdb::Env* FLAGS_env = rocksdb::Env::Default();
-enum rocksdb::CompressionType FLAGS_compression_type_e =
-    rocksdb::kSnappyCompression;
-enum rocksdb::ChecksumType FLAGS_checksum_type_e = rocksdb::kCRC32c;
-enum RepFactory FLAGS_rep_factory = kSkipList;
-
-namespace rocksdb {
-void PoolSizeChangeThread(void* v) {
-  assert(FLAGS_compaction_thread_pool_adjust_interval > 0);
-  ThreadState* thread = reinterpret_cast<ThreadState*>(v);
-  SharedState* shared = thread->shared;
-
-  while (true) {
-    {
-      MutexLock l(shared->GetMutex());
-      if (shared->ShoudStopBgThread()) {
-        shared->SetBgThreadFinish();
-        shared->GetCondVar()->SignalAll();
-        return;
-      }
-    }
-
-    auto thread_pool_size_base = FLAGS_max_background_compactions;
-    auto thread_pool_size_var = FLAGS_compaction_thread_pool_variations;
-    int new_thread_pool_size =
-        thread_pool_size_base - thread_pool_size_var +
-        thread->rand.Next() % (thread_pool_size_var * 2 + 1);
-    if (new_thread_pool_size < 1) {
-      new_thread_pool_size = 1;
-    }
-    FLAGS_env->SetBackgroundThreads(new_thread_pool_size);
-    // Sleep up to 3 seconds
-    FLAGS_env->SleepForMicroseconds(
-        thread->rand.Next() % FLAGS_compaction_thread_pool_adjust_interval *
-            1000 +
-        1);
-  }
-}
-
-void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz) {
-  if (!FLAGS_verbose) {
-    return;
-  }
-  std::string tmp;
-  tmp.reserve(sz * 2 + 16);
-  char buf[4];
-  for (size_t i = 0; i < sz; i++) {
-    snprintf(buf, 4, "%X", value[i]);
-    tmp.append(buf);
-  }
-  fprintf(stdout, "[CF %d] %" PRIi64 " == > (%" ROCKSDB_PRIszt ") %s\n", cf,
-          key, sz, tmp.c_str());
-}
-
-int64_t GenerateOneKey(ThreadState* thread, uint64_t iteration) {
-  const double completed_ratio =
-      static_cast<double>(iteration) / FLAGS_ops_per_thread;
-  const int64_t base_key = static_cast<int64_t>(
-      completed_ratio * (FLAGS_max_key - FLAGS_active_width));
-  return base_key + thread->rand.Next() % FLAGS_active_width;
-}
-
-std::vector<int64_t> GenerateNKeys(ThreadState* thread, int num_keys,
-                                   uint64_t iteration) {
-  const double completed_ratio =
-      static_cast<double>(iteration) / FLAGS_ops_per_thread;
-  const int64_t base_key = static_cast<int64_t>(
-      completed_ratio * (FLAGS_max_key - FLAGS_active_width));
-  std::vector<int64_t> keys;
-  keys.reserve(num_keys);
-  int64_t next_key = base_key + thread->rand.Next() % FLAGS_active_width;
-  keys.push_back(next_key);
-  for (int i = 1; i < num_keys; ++i) {
-    // This may result in some duplicate keys
-    next_key = next_key + thread->rand.Next() %
-                              (FLAGS_active_width - (next_key - base_key));
-    keys.push_back(next_key);
-  }
-  return keys;
-}
-
-size_t GenerateValue(uint32_t rand, char* v, size_t max_sz) {
-  size_t value_sz =
-      ((rand % kRandomValueMaxFactor) + 1) * FLAGS_value_size_mult;
-  assert(value_sz <= max_sz && value_sz >= sizeof(uint32_t));
-  (void)max_sz;
-  *((uint32_t*)v) = rand;
-  for (size_t i = sizeof(uint32_t); i < value_sz; i++) {
-    v[i] = (char)(rand ^ i);
-  }
-  v[value_sz] = '\0';
-  return value_sz;  // the size of the value set.
-}
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_common.h

@@ -1,373 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// The test uses an array to compare against values written to the database.
-// Keys written to the array are in 1:1 correspondence to the actual values in
-// the database according to the formula in the function GenerateValue.
-
-// Space is reserved in the array from 0 to FLAGS_max_key and values are
-// randomly written/deleted/read from those positions. During verification we
-// compare all the positions in the array. To shorten/elongate the running
-// time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread,
-// (sometimes also FLAGS_threads).
-//
-// NOTE that if FLAGS_test_batches_snapshots is set, the test will have
-// different behavior. See comment of the flag for details.
-
-#ifdef GFLAGS
-#pragma once
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <algorithm>
-#include <array>
-#include <chrono>
-#include <cinttypes>
-#include <exception>
-#include <queue>
-#include <thread>
-
-#include "db/db_impl/db_impl.h"
-#include "db/version_set.h"
-#include "db_stress_tool/db_stress_listener.h"
-#include "db_stress_tool/db_stress_shared_state.h"
-#include "db_stress_tool/db_stress_test_base.h"
-#include "hdfs/env_hdfs.h"
-#include "logging/logging.h"
-#include "monitoring/histogram.h"
-#include "options/options_helper.h"
-#include "port/port.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/env.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/slice_transform.h"
-#include "rocksdb/statistics.h"
-#include "rocksdb/utilities/backupable_db.h"
-#include "rocksdb/utilities/checkpoint.h"
-#include "rocksdb/utilities/db_ttl.h"
-#include "rocksdb/utilities/debug.h"
-#include "rocksdb/utilities/options_util.h"
-#include "rocksdb/utilities/transaction.h"
-#include "rocksdb/utilities/transaction_db.h"
-#include "rocksdb/write_batch.h"
-#include "util/coding.h"
-#include "util/compression.h"
-#include "util/crc32c.h"
-#include "util/gflags_compat.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
-#include "util/string_util.h"
-// SyncPoint is not supported in Released Windows Mode.
-#if !(defined NDEBUG) || !defined(OS_WIN)
-#include "test_util/sync_point.h"
-#endif  // !(defined NDEBUG) || !defined(OS_WIN)
-#include "test_util/testutil.h"
-
-#include "utilities/merge_operators.h"
-
-using GFLAGS_NAMESPACE::ParseCommandLineFlags;
-using GFLAGS_NAMESPACE::RegisterFlagValidator;
-using GFLAGS_NAMESPACE::SetUsageMessage;
-
-DECLARE_uint64(seed);
-DECLARE_bool(read_only);
-DECLARE_int64(max_key);
-DECLARE_int32(column_families);
-DECLARE_string(options_file);
-DECLARE_int64(active_width);
-DECLARE_bool(test_batches_snapshots);
-DECLARE_bool(atomic_flush);
-DECLARE_bool(test_cf_consistency);
-DECLARE_int32(threads);
-DECLARE_int32(ttl);
-DECLARE_int32(value_size_mult);
-DECLARE_int32(compaction_readahead_size);
-DECLARE_bool(enable_pipelined_write);
-DECLARE_bool(verify_before_write);
-DECLARE_bool(histogram);
-DECLARE_bool(destroy_db_initially);
-DECLARE_bool(verbose);
-DECLARE_bool(progress_reports);
-DECLARE_uint64(db_write_buffer_size);
-DECLARE_int32(write_buffer_size);
-DECLARE_int32(max_write_buffer_number);
-DECLARE_int32(min_write_buffer_number_to_merge);
-DECLARE_int32(max_write_buffer_number_to_maintain);
-DECLARE_int64(max_write_buffer_size_to_maintain);
-DECLARE_double(memtable_prefix_bloom_size_ratio);
-DECLARE_bool(memtable_whole_key_filtering);
-DECLARE_int32(open_files);
-DECLARE_int64(compressed_cache_size);
-DECLARE_int32(compaction_style);
-DECLARE_int32(level0_file_num_compaction_trigger);
-DECLARE_int32(level0_slowdown_writes_trigger);
-DECLARE_int32(level0_stop_writes_trigger);
-DECLARE_int32(block_size);
-DECLARE_int32(format_version);
-DECLARE_int32(index_block_restart_interval);
-DECLARE_int32(max_background_compactions);
-DECLARE_int32(num_bottom_pri_threads);
-DECLARE_int32(compaction_thread_pool_adjust_interval);
-DECLARE_int32(compaction_thread_pool_variations);
-DECLARE_int32(max_background_flushes);
-DECLARE_int32(universal_size_ratio);
-DECLARE_int32(universal_min_merge_width);
-DECLARE_int32(universal_max_merge_width);
-DECLARE_int32(universal_max_size_amplification_percent);
-DECLARE_int32(clear_column_family_one_in);
-DECLARE_int32(set_options_one_in);
-DECLARE_int32(set_in_place_one_in);
-DECLARE_int64(cache_size);
-DECLARE_bool(cache_index_and_filter_blocks);
-DECLARE_bool(use_clock_cache);
-DECLARE_uint64(subcompactions);
-DECLARE_uint64(periodic_compaction_seconds);
-DECLARE_uint64(compaction_ttl);
-DECLARE_bool(allow_concurrent_memtable_write);
-DECLARE_bool(enable_write_thread_adaptive_yield);
-DECLARE_int32(reopen);
-DECLARE_double(bloom_bits);
-DECLARE_bool(use_block_based_filter);
-DECLARE_bool(partition_filters);
-DECLARE_int32(index_type);
-DECLARE_string(db);
-DECLARE_string(secondaries_base);
-DECLARE_bool(enable_secondary);
-DECLARE_string(expected_values_path);
-DECLARE_bool(verify_checksum);
-DECLARE_bool(mmap_read);
-DECLARE_bool(mmap_write);
-DECLARE_bool(use_direct_reads);
-DECLARE_bool(use_direct_io_for_flush_and_compaction);
-DECLARE_bool(statistics);
-DECLARE_bool(sync);
-DECLARE_bool(use_fsync);
-DECLARE_int32(kill_random_test);
-DECLARE_string(kill_prefix_blacklist);
-DECLARE_bool(disable_wal);
-DECLARE_uint64(recycle_log_file_num);
-DECLARE_int64(target_file_size_base);
-DECLARE_int32(target_file_size_multiplier);
-DECLARE_uint64(max_bytes_for_level_base);
-DECLARE_double(max_bytes_for_level_multiplier);
-DECLARE_int32(range_deletion_width);
-DECLARE_uint64(rate_limiter_bytes_per_sec);
-DECLARE_bool(rate_limit_bg_reads);
-DECLARE_bool(use_txn);
-DECLARE_int32(backup_one_in);
-DECLARE_int32(checkpoint_one_in);
-DECLARE_int32(ingest_external_file_one_in);
-DECLARE_int32(ingest_external_file_width);
-DECLARE_int32(compact_files_one_in);
-DECLARE_int32(compact_range_one_in);
-DECLARE_int32(flush_one_in);
-DECLARE_int32(pause_background_one_in);
-DECLARE_int32(compact_range_width);
-DECLARE_int32(acquire_snapshot_one_in);
-DECLARE_bool(compare_full_db_state_snapshot);
-DECLARE_uint64(snapshot_hold_ops);
-DECLARE_bool(use_multiget);
-DECLARE_int32(readpercent);
-DECLARE_int32(prefixpercent);
-DECLARE_int32(writepercent);
-DECLARE_int32(delpercent);
-DECLARE_int32(delrangepercent);
-DECLARE_int32(nooverwritepercent);
-DECLARE_int32(iterpercent);
-DECLARE_uint64(num_iterations);
-DECLARE_string(compression_type);
-DECLARE_int32(compression_max_dict_bytes);
-DECLARE_int32(compression_zstd_max_train_bytes);
-DECLARE_string(checksum_type);
-DECLARE_string(hdfs);
-DECLARE_string(env_uri);
-DECLARE_uint64(ops_per_thread);
-DECLARE_uint64(log2_keys_per_lock);
-DECLARE_uint64(max_manifest_file_size);
-DECLARE_bool(in_place_update);
-DECLARE_int32(secondary_catch_up_one_in);
-DECLARE_string(memtablerep);
-DECLARE_int32(prefix_size);
-DECLARE_bool(use_merge);
-DECLARE_bool(use_full_merge_v1);
-DECLARE_int32(sync_wal_one_in);
-
-const long KB = 1024;
-const int kRandomValueMaxFactor = 3;
-const int kValueMaxLen = 100;
-
-// posix or hdfs environment
-extern rocksdb::Env* FLAGS_env;
-
-extern enum rocksdb::CompressionType FLAGS_compression_type_e;
-extern enum rocksdb::ChecksumType FLAGS_checksum_type_e;
-
-enum RepFactory { kSkipList, kHashSkipList, kVectorRep };
-
-inline enum RepFactory StringToRepFactory(const char* ctype) {
-  assert(ctype);
-
-  if (!strcasecmp(ctype, "skip_list"))
-    return kSkipList;
-  else if (!strcasecmp(ctype, "prefix_hash"))
-    return kHashSkipList;
-  else if (!strcasecmp(ctype, "vector"))
-    return kVectorRep;
-
-  fprintf(stdout, "Cannot parse memreptable %s\n", ctype);
-  return kSkipList;
-}
-
-extern enum RepFactory FLAGS_rep_factory;
-
-namespace rocksdb {
-inline enum rocksdb::CompressionType StringToCompressionType(
-    const char* ctype) {
-  assert(ctype);
-
-  if (!strcasecmp(ctype, "none"))
-    return rocksdb::kNoCompression;
-  else if (!strcasecmp(ctype, "snappy"))
-    return rocksdb::kSnappyCompression;
-  else if (!strcasecmp(ctype, "zlib"))
-    return rocksdb::kZlibCompression;
-  else if (!strcasecmp(ctype, "bzip2"))
-    return rocksdb::kBZip2Compression;
-  else if (!strcasecmp(ctype, "lz4"))
-    return rocksdb::kLZ4Compression;
-  else if (!strcasecmp(ctype, "lz4hc"))
-    return rocksdb::kLZ4HCCompression;
-  else if (!strcasecmp(ctype, "xpress"))
-    return rocksdb::kXpressCompression;
-  else if (!strcasecmp(ctype, "zstd"))
-    return rocksdb::kZSTD;
-
-  fprintf(stderr, "Cannot parse compression type '%s'\n", ctype);
-  return rocksdb::kSnappyCompression;  // default value
-}
-
-inline enum rocksdb::ChecksumType StringToChecksumType(const char* ctype) {
-  assert(ctype);
-  auto iter = rocksdb::checksum_type_string_map.find(ctype);
-  if (iter != rocksdb::checksum_type_string_map.end()) {
-    return iter->second;
-  }
-  fprintf(stderr, "Cannot parse checksum type '%s'\n", ctype);
-  return rocksdb::kCRC32c;
-}
-
-inline std::string ChecksumTypeToString(rocksdb::ChecksumType ctype) {
-  auto iter = std::find_if(
-      rocksdb::checksum_type_string_map.begin(),
-      rocksdb::checksum_type_string_map.end(),
-      [&](const std::pair<std::string, rocksdb::ChecksumType>&
-              name_and_enum_val) { return name_and_enum_val.second == ctype; });
-  assert(iter != rocksdb::checksum_type_string_map.end());
-  return iter->first;
-}
-
-inline std::vector<std::string> SplitString(std::string src) {
-  std::vector<std::string> ret;
-  if (src.empty()) {
-    return ret;
-  }
-  size_t pos = 0;
-  size_t pos_comma;
-  while ((pos_comma = src.find(',', pos)) != std::string::npos) {
-    ret.push_back(src.substr(pos, pos_comma - pos));
-    pos = pos_comma + 1;
-  }
-  ret.push_back(src.substr(pos, src.length()));
-  return ret;
-}
-
-#ifdef _MSC_VER
-#pragma warning(push)
-// truncation of constant value on static_cast
-#pragma warning(disable : 4309)
-#endif
-inline bool GetNextPrefix(const rocksdb::Slice& src, std::string* v) {
-  std::string ret = src.ToString();
-  for (int i = static_cast<int>(ret.size()) - 1; i >= 0; i--) {
-    if (ret[i] != static_cast<char>(255)) {
-      ret[i] = ret[i] + 1;
-      break;
-    } else if (i != 0) {
-      ret[i] = 0;
-    } else {
-      // all FF. No next prefix
-      return false;
-    }
-  }
-  *v = ret;
-  return true;
-}
-#ifdef _MSC_VER
-#pragma warning(pop)
-#endif
-
-// convert long to a big-endian slice key
-extern inline std::string Key(int64_t val) {
-  std::string little_endian_key;
-  std::string big_endian_key;
-  PutFixed64(&little_endian_key, val);
-  assert(little_endian_key.size() == sizeof(val));
-  big_endian_key.resize(sizeof(val));
-  for (size_t i = 0; i < sizeof(val); ++i) {
-    big_endian_key[i] = little_endian_key[sizeof(val) - 1 - i];
-  }
-  return big_endian_key;
-}
-
-extern inline bool GetIntVal(std::string big_endian_key, uint64_t* key_p) {
-  unsigned int size_key = sizeof(*key_p);
-  assert(big_endian_key.size() == size_key);
-  std::string little_endian_key;
-  little_endian_key.resize(size_key);
-  for (size_t i = 0; i < size_key; ++i) {
-    little_endian_key[i] = big_endian_key[size_key - 1 - i];
-  }
-  Slice little_endian_slice = Slice(little_endian_key);
-  return GetFixed64(&little_endian_slice, key_p);
-}
-
-extern inline std::string StringToHex(const std::string& str) {
-  std::string result = "0x";
-  result.append(Slice(str).ToString(true));
-  return result;
-}
-
-// Unified output format for double parameters
-extern inline std::string FormatDoubleParam(double param) {
-  return std::to_string(param);
-}
-
-// Make sure that double parameter is a value we can reproduce by
-// re-inputting the value printed.
-extern inline void SanitizeDoubleParam(double* param) {
-  *param = std::atof(FormatDoubleParam(*param).c_str());
-}
-
-extern void PoolSizeChangeThread(void* v);
-
-extern void PrintKeyValue(int cf, uint64_t key, const char* value, size_t sz);
-
-extern int64_t GenerateOneKey(ThreadState* thread, uint64_t iteration);
-
-extern std::vector<int64_t> GenerateNKeys(ThreadState* thread, int num_keys,
-                                          uint64_t iteration);
-
-extern size_t GenerateValue(uint32_t rand, char* v, size_t max_sz);
-
-extern StressTest* CreateCfConsistencyStressTest();
-extern StressTest* CreateBatchedOpsStressTest();
-extern StressTest* CreateNonBatchedOpsStressTest();
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_driver.cc

@@ -1,157 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-
-#ifdef GFLAGS
-#include "db_stress_tool/db_stress_common.h"
-
-namespace rocksdb {
-void ThreadBody(void* v) {
-  ThreadState* thread = reinterpret_cast<ThreadState*>(v);
-  SharedState* shared = thread->shared;
-
-  if (shared->ShouldVerifyAtBeginning()) {
-    thread->shared->GetStressTest()->VerifyDb(thread);
-  }
-  {
-    MutexLock l(shared->GetMutex());
-    shared->IncInitialized();
-    if (shared->AllInitialized()) {
-      shared->GetCondVar()->SignalAll();
-    }
-    while (!shared->Started()) {
-      shared->GetCondVar()->Wait();
-    }
-  }
-  thread->shared->GetStressTest()->OperateDb(thread);
-
-  {
-    MutexLock l(shared->GetMutex());
-    shared->IncOperated();
-    if (shared->AllOperated()) {
-      shared->GetCondVar()->SignalAll();
-    }
-    while (!shared->VerifyStarted()) {
-      shared->GetCondVar()->Wait();
-    }
-  }
-
-  thread->shared->GetStressTest()->VerifyDb(thread);
-
-  {
-    MutexLock l(shared->GetMutex());
-    shared->IncDone();
-    if (shared->AllDone()) {
-      shared->GetCondVar()->SignalAll();
-    }
-  }
-}
-
-bool RunStressTest(StressTest* stress) {
-  stress->InitDb();
-
-  SharedState shared(FLAGS_env, stress);
-  if (FLAGS_read_only) {
-    stress->InitReadonlyDb(&shared);
-  }
-
-  uint32_t n = shared.GetNumThreads();
-
-  uint64_t now = FLAGS_env->NowMicros();
-  fprintf(stdout, "%s Initializing worker threads\n",
-          FLAGS_env->TimeToString(now / 1000000).c_str());
-  std::vector<ThreadState*> threads(n);
-  for (uint32_t i = 0; i < n; i++) {
-    threads[i] = new ThreadState(i, &shared);
-    FLAGS_env->StartThread(ThreadBody, threads[i]);
-  }
-  ThreadState bg_thread(0, &shared);
-  if (FLAGS_compaction_thread_pool_adjust_interval > 0) {
-    FLAGS_env->StartThread(PoolSizeChangeThread, &bg_thread);
-  }
-
-  // Each thread goes through the following states:
-  // initializing -> wait for others to init -> read/populate/depopulate
-  // wait for others to operate -> verify -> done
-
-  {
-    MutexLock l(shared.GetMutex());
-    while (!shared.AllInitialized()) {
-      shared.GetCondVar()->Wait();
-    }
-    if (shared.ShouldVerifyAtBeginning()) {
-      if (shared.HasVerificationFailedYet()) {
-        printf("Crash-recovery verification failed :(\n");
-      } else {
-        printf("Crash-recovery verification passed :)\n");
-      }
-    }
-
-    now = FLAGS_env->NowMicros();
-    fprintf(stdout, "%s Starting database operations\n",
-            FLAGS_env->TimeToString(now / 1000000).c_str());
-
-    shared.SetStart();
-    shared.GetCondVar()->SignalAll();
-    while (!shared.AllOperated()) {
-      shared.GetCondVar()->Wait();
-    }
-
-    now = FLAGS_env->NowMicros();
-    if (FLAGS_test_batches_snapshots) {
-      fprintf(stdout, "%s Limited verification already done during gets\n",
-              FLAGS_env->TimeToString((uint64_t)now / 1000000).c_str());
-    } else {
-      fprintf(stdout, "%s Starting verification\n",
-              FLAGS_env->TimeToString((uint64_t)now / 1000000).c_str());
-    }
-
-    shared.SetStartVerify();
-    shared.GetCondVar()->SignalAll();
-    while (!shared.AllDone()) {
-      shared.GetCondVar()->Wait();
-    }
-  }
-
-  for (unsigned int i = 1; i < n; i++) {
-    threads[0]->stats.Merge(threads[i]->stats);
-  }
-  threads[0]->stats.Report("Stress Test");
-
-  for (unsigned int i = 0; i < n; i++) {
-    delete threads[i];
-    threads[i] = nullptr;
-  }
-  now = FLAGS_env->NowMicros();
-  if (!FLAGS_test_batches_snapshots && !shared.HasVerificationFailedYet()) {
-    fprintf(stdout, "%s Verification successful\n",
-            FLAGS_env->TimeToString(now / 1000000).c_str());
-  }
-  stress->PrintStatistics();
-
-  if (FLAGS_compaction_thread_pool_adjust_interval > 0) {
-    MutexLock l(shared.GetMutex());
-    shared.SetShouldStopBgThread();
-    while (!shared.BgThreadFinished()) {
-      shared.GetCondVar()->Wait();
-    }
-  }
-
-  if (!stress->VerifySecondaries()) {
-    return false;
-  }
-
-  if (shared.HasVerificationFailedYet()) {
-    printf("Verification failed :(\n");
-    return false;
-  }
-  return true;
-}
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_driver.h

@@ -1,17 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifdef GFLAGS
-#pragma once
-#include "db_stress_tool/db_stress_test_base.h"
-namespace rocksdb {
-extern void ThreadBody(void* /*thread_state*/);
-extern bool RunStressTest(StressTest*);
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_gflags.cc

@@ -1,540 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifdef GFLAGS
-#include "db_stress_tool/db_stress_common.h"
-
-static bool ValidateUint32Range(const char* flagname, uint64_t value) {
-  if (value > std::numeric_limits<uint32_t>::max()) {
-    fprintf(stderr, "Invalid value for --%s: %lu, overflow\n", flagname,
-            (unsigned long)value);
-    return false;
-  }
-  return true;
-}
-
-DEFINE_uint64(seed, 2341234, "Seed for PRNG");
-static const bool FLAGS_seed_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_seed, &ValidateUint32Range);
-
-DEFINE_bool(read_only, false, "True if open DB in read-only mode during tests");
-
-DEFINE_int64(max_key, 1 * KB * KB,
-             "Max number of key/values to place in database");
-
-DEFINE_int32(column_families, 10, "Number of column families");
-
-DEFINE_string(
-    options_file, "",
-    "The path to a RocksDB options file.  If specified, then db_stress will "
-    "run with the RocksDB options in the default column family of the "
-    "specified options file. Note that, when an options file is provided, "
-    "db_stress will ignore the flag values for all options that may be passed "
-    "via options file.");
-
-DEFINE_int64(
-    active_width, 0,
-    "Number of keys in active span of the key-range at any given time. The "
-    "span begins with its left endpoint at key 0, gradually moves rightwards, "
-    "and ends with its right endpoint at max_key. If set to 0, active_width "
-    "will be sanitized to be equal to max_key.");
-
-// TODO(noetzli) Add support for single deletes
-DEFINE_bool(test_batches_snapshots, false,
-            "If set, the test uses MultiGet(), MultiPut() and MultiDelete()"
-            " which read/write/delete multiple keys in a batch. In this mode,"
-            " we do not verify db content by comparing the content with the "
-            "pre-allocated array. Instead, we do partial verification inside"
-            " MultiGet() by checking various values in a batch. Benefit of"
-            " this mode:\n"
-            "\t(a) No need to acquire mutexes during writes (less cache "
-            "flushes in multi-core leading to speed up)\n"
-            "\t(b) No long validation at the end (more speed up)\n"
-            "\t(c) Test snapshot and atomicity of batch writes");
-
-DEFINE_bool(atomic_flush, false,
-            "If set, enables atomic flush in the options.\n");
-
-DEFINE_bool(test_cf_consistency, false,
-            "If set, runs the stress test dedicated to verifying writes to "
-            "multiple column families are consistent. Setting this implies "
-            "`atomic_flush=true` is set true if `disable_wal=false`.\n");
-
-DEFINE_int32(threads, 32, "Number of concurrent threads to run.");
-
-DEFINE_int32(ttl, -1,
-             "Opens the db with this ttl value if this is not -1. "
-             "Carefully specify a large value such that verifications on "
-             "deleted values don't fail");
-
-DEFINE_int32(value_size_mult, 8,
-             "Size of value will be this number times rand_int(1,3) bytes");
-
-DEFINE_int32(compaction_readahead_size, 0, "Compaction readahead size");
-
-DEFINE_bool(enable_pipelined_write, false, "Pipeline WAL/memtable writes");
-
-DEFINE_bool(verify_before_write, false, "Verify before write");
-
-DEFINE_bool(histogram, false, "Print histogram of operation timings");
-
-DEFINE_bool(destroy_db_initially, true,
-            "Destroys the database dir before start if this is true");
-
-DEFINE_bool(verbose, false, "Verbose");
-
-DEFINE_bool(progress_reports, true,
-            "If true, db_stress will report number of finished operations");
-
-DEFINE_uint64(db_write_buffer_size, rocksdb::Options().db_write_buffer_size,
-              "Number of bytes to buffer in all memtables before compacting");
-
-DEFINE_int32(write_buffer_size,
-             static_cast<int32_t>(rocksdb::Options().write_buffer_size),
-             "Number of bytes to buffer in memtable before compacting");
-
-DEFINE_int32(max_write_buffer_number,
-             rocksdb::Options().max_write_buffer_number,
-             "The number of in-memory memtables. "
-             "Each memtable is of size FLAGS_write_buffer_size.");
-
-DEFINE_int32(min_write_buffer_number_to_merge,
-             rocksdb::Options().min_write_buffer_number_to_merge,
-             "The minimum number of write buffers that will be merged together "
-             "before writing to storage. This is cheap because it is an "
-             "in-memory merge. If this feature is not enabled, then all these "
-             "write buffers are flushed to L0 as separate files and this "
-             "increases read amplification because a get request has to check "
-             "in all of these files. Also, an in-memory merge may result in "
-             "writing less data to storage if there are duplicate records in"
-             " each of these individual write buffers.");
-
-DEFINE_int32(max_write_buffer_number_to_maintain,
-             rocksdb::Options().max_write_buffer_number_to_maintain,
-             "The total maximum number of write buffers to maintain in memory "
-             "including copies of buffers that have already been flushed. "
-             "Unlike max_write_buffer_number, this parameter does not affect "
-             "flushing. This controls the minimum amount of write history "
-             "that will be available in memory for conflict checking when "
-             "Transactions are used. If this value is too low, some "
-             "transactions may fail at commit time due to not being able to "
-             "determine whether there were any write conflicts. Setting this "
-             "value to 0 will cause write buffers to be freed immediately "
-             "after they are flushed.  If this value is set to -1, "
-             "'max_write_buffer_number' will be used.");
-
-DEFINE_int64(max_write_buffer_size_to_maintain,
-             rocksdb::Options().max_write_buffer_size_to_maintain,
-             "The total maximum size of write buffers to maintain in memory "
-             "including copies of buffers that have already been flushed. "
-             "Unlike max_write_buffer_number, this parameter does not affect "
-             "flushing. This controls the minimum amount of write history "
-             "that will be available in memory for conflict checking when "
-             "Transactions are used. If this value is too low, some "
-             "transactions may fail at commit time due to not being able to "
-             "determine whether there were any write conflicts. Setting this "
-             "value to 0 will cause write buffers to be freed immediately "
-             "after they are flushed.  If this value is set to -1, "
-             "'max_write_buffer_number' will be used.");
-
-DEFINE_double(memtable_prefix_bloom_size_ratio,
-              rocksdb::Options().memtable_prefix_bloom_size_ratio,
-              "creates prefix blooms for memtables, each with size "
-              "`write_buffer_size * memtable_prefix_bloom_size_ratio`.");
-
-DEFINE_bool(memtable_whole_key_filtering,
-            rocksdb::Options().memtable_whole_key_filtering,
-            "Enable whole key filtering in memtables.");
-
-DEFINE_int32(open_files, rocksdb::Options().max_open_files,
-             "Maximum number of files to keep open at the same time "
-             "(use default if == 0)");
-
-DEFINE_int64(compressed_cache_size, -1,
-             "Number of bytes to use as a cache of compressed data."
-             " Negative means use default settings.");
-
-DEFINE_int32(compaction_style, rocksdb::Options().compaction_style, "");
-
-DEFINE_int32(level0_file_num_compaction_trigger,
-             rocksdb::Options().level0_file_num_compaction_trigger,
-             "Level0 compaction start trigger");
-
-DEFINE_int32(level0_slowdown_writes_trigger,
-             rocksdb::Options().level0_slowdown_writes_trigger,
-             "Number of files in level-0 that will slow down writes");
-
-DEFINE_int32(level0_stop_writes_trigger,
-             rocksdb::Options().level0_stop_writes_trigger,
-             "Number of files in level-0 that will trigger put stop.");
-
-DEFINE_int32(block_size,
-             static_cast<int32_t>(rocksdb::BlockBasedTableOptions().block_size),
-             "Number of bytes in a block.");
-
-DEFINE_int32(
-    format_version,
-    static_cast<int32_t>(rocksdb::BlockBasedTableOptions().format_version),
-    "Format version of SST files.");
-
-DEFINE_int32(index_block_restart_interval,
-             rocksdb::BlockBasedTableOptions().index_block_restart_interval,
-             "Number of keys between restart points "
-             "for delta encoding of keys in index block.");
-
-DEFINE_int32(max_background_compactions,
-             rocksdb::Options().max_background_compactions,
-             "The maximum number of concurrent background compactions "
-             "that can occur in parallel.");
-
-DEFINE_int32(num_bottom_pri_threads, 0,
-             "The number of threads in the bottom-priority thread pool (used "
-             "by universal compaction only).");
-
-DEFINE_int32(compaction_thread_pool_adjust_interval, 0,
-             "The interval (in milliseconds) to adjust compaction thread pool "
-             "size. Don't change it periodically if the value is 0.");
-
-DEFINE_int32(compaction_thread_pool_variations, 2,
-             "Range of background thread pool size variations when adjusted "
-             "periodically.");
-
-DEFINE_int32(max_background_flushes, rocksdb::Options().max_background_flushes,
-             "The maximum number of concurrent background flushes "
-             "that can occur in parallel.");
-
-DEFINE_int32(universal_size_ratio, 0,
-             "The ratio of file sizes that trigger"
-             " compaction in universal style");
-
-DEFINE_int32(universal_min_merge_width, 0,
-             "The minimum number of files to "
-             "compact in universal style compaction");
-
-DEFINE_int32(universal_max_merge_width, 0,
-             "The max number of files to compact"
-             " in universal style compaction");
-
-DEFINE_int32(universal_max_size_amplification_percent, 0,
-             "The max size amplification for universal style compaction");
-
-DEFINE_int32(clear_column_family_one_in, 1000000,
-             "With a chance of 1/N, delete a column family and then recreate "
-             "it again. If N == 0, never drop/create column families. "
-             "When test_batches_snapshots is true, this flag has no effect");
-
-DEFINE_int32(set_options_one_in, 0,
-             "With a chance of 1/N, change some random options");
-
-DEFINE_int32(set_in_place_one_in, 0,
-             "With a chance of 1/N, toggle in place support option");
-
-DEFINE_int64(cache_size, 2LL * KB * KB * KB,
-             "Number of bytes to use as a cache of uncompressed data.");
-
-DEFINE_bool(cache_index_and_filter_blocks, false,
-            "True if indexes/filters should be cached in block cache.");
-
-DEFINE_bool(use_clock_cache, false,
-            "Replace default LRU block cache with clock cache.");
-
-DEFINE_uint64(subcompactions, 1,
-              "Maximum number of subcompactions to divide L0-L1 compactions "
-              "into.");
-
-DEFINE_uint64(periodic_compaction_seconds, 1000,
-              "Files older than this value will be picked up for compaction.");
-
-DEFINE_uint64(compaction_ttl, 1000,
-              "Files older than TTL will be compacted to the next level.");
-
-DEFINE_bool(allow_concurrent_memtable_write, false,
-            "Allow multi-writers to update mem tables in parallel.");
-
-DEFINE_bool(enable_write_thread_adaptive_yield, true,
-            "Use a yielding spin loop for brief writer thread waits.");
-
-static const bool FLAGS_subcompactions_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_subcompactions, &ValidateUint32Range);
-
-static bool ValidateInt32Positive(const char* flagname, int32_t value) {
-  if (value < 0) {
-    fprintf(stderr, "Invalid value for --%s: %d, must be >=0\n", flagname,
-            value);
-    return false;
-  }
-  return true;
-}
-DEFINE_int32(reopen, 10, "Number of times database reopens");
-static const bool FLAGS_reopen_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_reopen, &ValidateInt32Positive);
-
-DEFINE_double(bloom_bits, 10,
-              "Bloom filter bits per key. "
-              "Negative means use default settings.");
-
-DEFINE_bool(use_block_based_filter, false,
-            "use block based filter"
-            "instead of full filter for block based table");
-
-DEFINE_bool(partition_filters, false,
-            "use partitioned filters "
-            "for block-based table");
-
-DEFINE_int32(
-    index_type,
-    static_cast<int32_t>(rocksdb::BlockBasedTableOptions::kBinarySearch),
-    "Type of block-based table index (see `enum IndexType` in table.h)");
-
-DEFINE_string(db, "", "Use the db with the following name.");
-
-DEFINE_string(secondaries_base, "",
-              "Use this path as the base path for secondary instances.");
-
-DEFINE_bool(enable_secondary, false, "Enable secondary instance.");
-
-DEFINE_string(
-    expected_values_path, "",
-    "File where the array of expected uint32_t values will be stored. If "
-    "provided and non-empty, the DB state will be verified against these "
-    "values after recovery. --max_key and --column_family must be kept the "
-    "same across invocations of this program that use the same "
-    "--expected_values_path.");
-
-DEFINE_bool(verify_checksum, false,
-            "Verify checksum for every block read from storage");
-
-DEFINE_bool(mmap_read, rocksdb::Options().allow_mmap_reads,
-            "Allow reads to occur via mmap-ing files");
-
-DEFINE_bool(mmap_write, rocksdb::Options().allow_mmap_writes,
-            "Allow writes to occur via mmap-ing files");
-
-DEFINE_bool(use_direct_reads, rocksdb::Options().use_direct_reads,
-            "Use O_DIRECT for reading data");
-
-DEFINE_bool(use_direct_io_for_flush_and_compaction,
-            rocksdb::Options().use_direct_io_for_flush_and_compaction,
-            "Use O_DIRECT for writing data");
-
-DEFINE_bool(statistics, false, "Create database statistics");
-
-DEFINE_bool(sync, false, "Sync all writes to disk");
-
-DEFINE_bool(use_fsync, false, "If true, issue fsync instead of fdatasync");
-
-DEFINE_int32(kill_random_test, 0,
-             "If non-zero, kill at various points in source code with "
-             "probability 1/this");
-static const bool FLAGS_kill_random_test_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_kill_random_test, &ValidateInt32Positive);
-extern int rocksdb_kill_odds;
-
-DEFINE_string(kill_prefix_blacklist, "",
-              "If non-empty, kill points with prefix in the list given will be"
-              " skipped. Items are comma-separated.");
-extern std::vector<std::string> rocksdb_kill_prefix_blacklist;
-
-DEFINE_bool(disable_wal, false, "If true, do not write WAL for write.");
-
-DEFINE_uint64(recycle_log_file_num, rocksdb::Options().recycle_log_file_num,
-              "Number of old WAL files to keep around for later recycling");
-
-DEFINE_int64(target_file_size_base, rocksdb::Options().target_file_size_base,
-             "Target level-1 file size for compaction");
-
-DEFINE_int32(target_file_size_multiplier, 1,
-             "A multiplier to compute target level-N file size (N >= 2)");
-
-DEFINE_uint64(max_bytes_for_level_base,
-              rocksdb::Options().max_bytes_for_level_base,
-              "Max bytes for level-1");
-
-DEFINE_double(max_bytes_for_level_multiplier, 2,
-              "A multiplier to compute max bytes for level-N (N >= 2)");
-
-DEFINE_int32(range_deletion_width, 10,
-             "The width of the range deletion intervals.");
-
-DEFINE_uint64(rate_limiter_bytes_per_sec, 0, "Set options.rate_limiter value.");
-
-DEFINE_bool(rate_limit_bg_reads, false,
-            "Use options.rate_limiter on compaction reads");
-
-DEFINE_bool(use_txn, false,
-            "Use TransactionDB. Currently the default write policy is "
-            "TxnDBWritePolicy::WRITE_PREPARED");
-
-DEFINE_int32(backup_one_in, 0,
-             "If non-zero, then CreateNewBackup() will be called once for "
-             "every N operations on average.  0 indicates CreateNewBackup() "
-             "is disabled.");
-
-DEFINE_int32(checkpoint_one_in, 0,
-             "If non-zero, then CreateCheckpoint() will be called once for "
-             "every N operations on average.  0 indicates CreateCheckpoint() "
-             "is disabled.");
-
-DEFINE_int32(ingest_external_file_one_in, 0,
-             "If non-zero, then IngestExternalFile() will be called once for "
-             "every N operations on average.  0 indicates IngestExternalFile() "
-             "is disabled.");
-
-DEFINE_int32(ingest_external_file_width, 1000,
-             "The width of the ingested external files.");
-
-DEFINE_int32(compact_files_one_in, 0,
-             "If non-zero, then CompactFiles() will be called once for every N "
-             "operations on average.  0 indicates CompactFiles() is disabled.");
-
-DEFINE_int32(compact_range_one_in, 0,
-             "If non-zero, then CompactRange() will be called once for every N "
-             "operations on average.  0 indicates CompactRange() is disabled.");
-
-DEFINE_int32(flush_one_in, 0,
-             "If non-zero, then Flush() will be called once for every N ops "
-             "on average.  0 indicates calls to Flush() are disabled.");
-
-DEFINE_int32(pause_background_one_in, 0,
-             "If non-zero, then PauseBackgroundWork()+Continue will be called "
-             "once for every N ops on average.  0 disables.");
-
-DEFINE_int32(compact_range_width, 10000,
-             "The width of the ranges passed to CompactRange().");
-
-DEFINE_int32(acquire_snapshot_one_in, 0,
-             "If non-zero, then acquires a snapshot once every N operations on "
-             "average.");
-
-DEFINE_bool(compare_full_db_state_snapshot, false,
-            "If set we compare state of entire db (in one of the threads) with"
-            "each snapshot.");
-
-DEFINE_uint64(snapshot_hold_ops, 0,
-              "If non-zero, then releases snapshots N operations after they're "
-              "acquired.");
-
-DEFINE_bool(use_multiget, false,
-            "If set, use the batched MultiGet API for reads");
-
-static bool ValidateInt32Percent(const char* flagname, int32_t value) {
-  if (value < 0 || value > 100) {
-    fprintf(stderr, "Invalid value for --%s: %d, 0<= pct <=100 \n", flagname,
-            value);
-    return false;
-  }
-  return true;
-}
-
-DEFINE_int32(readpercent, 10,
-             "Ratio of reads to total workload (expressed as a percentage)");
-static const bool FLAGS_readpercent_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_readpercent, &ValidateInt32Percent);
-
-DEFINE_int32(prefixpercent, 20,
-             "Ratio of prefix iterators to total workload (expressed as a"
-             " percentage)");
-static const bool FLAGS_prefixpercent_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_prefixpercent, &ValidateInt32Percent);
-
-DEFINE_int32(writepercent, 45,
-             "Ratio of writes to total workload (expressed as a percentage)");
-static const bool FLAGS_writepercent_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_writepercent, &ValidateInt32Percent);
-
-DEFINE_int32(delpercent, 15,
-             "Ratio of deletes to total workload (expressed as a percentage)");
-static const bool FLAGS_delpercent_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_delpercent, &ValidateInt32Percent);
-
-DEFINE_int32(delrangepercent, 0,
-             "Ratio of range deletions to total workload (expressed as a "
-             "percentage). Cannot be used with test_batches_snapshots");
-static const bool FLAGS_delrangepercent_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_delrangepercent, &ValidateInt32Percent);
-
-DEFINE_int32(nooverwritepercent, 60,
-             "Ratio of keys without overwrite to total workload (expressed as "
-             " a percentage)");
-static const bool FLAGS_nooverwritepercent_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_nooverwritepercent, &ValidateInt32Percent);
-
-DEFINE_int32(iterpercent, 10,
-             "Ratio of iterations to total workload"
-             " (expressed as a percentage)");
-static const bool FLAGS_iterpercent_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_iterpercent, &ValidateInt32Percent);
-
-DEFINE_uint64(num_iterations, 10, "Number of iterations per MultiIterate run");
-static const bool FLAGS_num_iterations_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_num_iterations, &ValidateUint32Range);
-
-DEFINE_string(compression_type, "snappy",
-              "Algorithm to use to compress the database");
-
-DEFINE_int32(compression_max_dict_bytes, 0,
-             "Maximum size of dictionary used to prime the compression "
-             "library.");
-
-DEFINE_int32(compression_zstd_max_train_bytes, 0,
-             "Maximum size of training data passed to zstd's dictionary "
-             "trainer.");
-
-DEFINE_string(checksum_type, "kCRC32c", "Algorithm to use to checksum blocks");
-
-DEFINE_string(hdfs, "", "Name of hdfs environment");
-
-DEFINE_string(env_uri, "",
-              "URI for env lookup. Mutually exclusive with --hdfs");
-
-DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread.");
-static const bool FLAGS_ops_per_thread_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_ops_per_thread, &ValidateUint32Range);
-
-DEFINE_uint64(log2_keys_per_lock, 2, "Log2 of number of keys per lock");
-static const bool FLAGS_log2_keys_per_lock_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_log2_keys_per_lock, &ValidateUint32Range);
-
-DEFINE_uint64(max_manifest_file_size, 16384, "Maximum size of a MANIFEST file");
-
-DEFINE_bool(in_place_update, false, "On true, does inplace update in memtable");
-
-DEFINE_int32(secondary_catch_up_one_in, 0,
-             "If non-zero, the secondaries attemp to catch up with the primary "
-             "once for every N operations on average. 0 indicates the "
-             "secondaries do not try to catch up after open.");
-
-DEFINE_string(memtablerep, "skip_list", "");
-
-inline static bool ValidatePrefixSize(const char* flagname, int32_t value) {
-  if (value < -1 || value > 8) {
-    fprintf(stderr, "Invalid value for --%s: %d. -1 <= PrefixSize <= 8\n",
-            flagname, value);
-    return false;
-  }
-  return true;
-}
-DEFINE_int32(prefix_size, 7,
-             "Control the prefix size for HashSkipListRep. "
-             "-1 is disabled.");
-static const bool FLAGS_prefix_size_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize);
-
-DEFINE_bool(use_merge, false,
-            "On true, replaces all writes with a Merge "
-            "that behaves like a Put");
-
-DEFINE_bool(use_full_merge_v1, false,
-            "On true, use a merge operator that implement the deprecated "
-            "version of FullMerge");
-
-DEFINE_int32(sync_wal_one_in, 0,
-             "If non-zero, then SyncWAL() will be called once for every N ops "
-             "on average. 0 indicates that calls to SyncWAL() are disabled.");
-#endif  // GFLAGS

db_stress_tool/db_stress_listener.h

@@ -1,142 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifdef GFLAGS
-#pragma once
-
-#include "rocksdb/listener.h"
-#include "util/gflags_compat.h"
-
-DECLARE_int32(compact_files_one_in);
-
-namespace rocksdb {
-class DbStressListener : public EventListener {
- public:
-  DbStressListener(const std::string& db_name,
-                   const std::vector<DbPath>& db_paths,
-                   const std::vector<ColumnFamilyDescriptor>& column_families)
-      : db_name_(db_name),
-        db_paths_(db_paths),
-        column_families_(column_families),
-        num_pending_file_creations_(0) {}
-  virtual ~DbStressListener() { assert(num_pending_file_creations_ == 0); }
-#ifndef ROCKSDB_LITE
-  virtual void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
-    assert(IsValidColumnFamilyName(info.cf_name));
-    VerifyFilePath(info.file_path);
-    // pretending doing some work here
-    std::this_thread::sleep_for(
-        std::chrono::microseconds(Random::GetTLSInstance()->Uniform(5000)));
-  }
-
-  virtual void OnCompactionCompleted(DB* /*db*/,
-                                     const CompactionJobInfo& ci) override {
-    assert(IsValidColumnFamilyName(ci.cf_name));
-    assert(ci.input_files.size() + ci.output_files.size() > 0U);
-    for (const auto& file_path : ci.input_files) {
-      VerifyFilePath(file_path);
-    }
-    for (const auto& file_path : ci.output_files) {
-      VerifyFilePath(file_path);
-    }
-    // pretending doing some work here
-    std::this_thread::sleep_for(
-        std::chrono::microseconds(Random::GetTLSInstance()->Uniform(5000)));
-  }
-
-  virtual void OnTableFileCreationStarted(
-      const TableFileCreationBriefInfo& /*info*/) override {
-    ++num_pending_file_creations_;
-  }
-  virtual void OnTableFileCreated(const TableFileCreationInfo& info) override {
-    assert(info.db_name == db_name_);
-    assert(IsValidColumnFamilyName(info.cf_name));
-    if (info.file_size) {
-      VerifyFilePath(info.file_path);
-    }
-    assert(info.job_id > 0 || FLAGS_compact_files_one_in > 0);
-    if (info.status.ok() && info.file_size > 0) {
-      assert(info.table_properties.data_size > 0 ||
-             info.table_properties.num_range_deletions > 0);
-      assert(info.table_properties.raw_key_size > 0);
-      assert(info.table_properties.num_entries > 0);
-    }
-    --num_pending_file_creations_;
-  }
-
- protected:
-  bool IsValidColumnFamilyName(const std::string& cf_name) const {
-    if (cf_name == kDefaultColumnFamilyName) {
-      return true;
-    }
-    // The column family names in the stress tests are numbers.
-    for (size_t i = 0; i < cf_name.size(); ++i) {
-      if (cf_name[i] < '0' || cf_name[i] > '9') {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  void VerifyFileDir(const std::string& file_dir) {
-#ifndef NDEBUG
-    if (db_name_ == file_dir) {
-      return;
-    }
-    for (const auto& db_path : db_paths_) {
-      if (db_path.path == file_dir) {
-        return;
-      }
-    }
-    for (auto& cf : column_families_) {
-      for (const auto& cf_path : cf.options.cf_paths) {
-        if (cf_path.path == file_dir) {
-          return;
-        }
-      }
-    }
-    assert(false);
-#else
-    (void)file_dir;
-#endif  // !NDEBUG
-  }
-
-  void VerifyFileName(const std::string& file_name) {
-#ifndef NDEBUG
-    uint64_t file_number;
-    FileType file_type;
-    bool result = ParseFileName(file_name, &file_number, &file_type);
-    assert(result);
-    assert(file_type == kTableFile);
-#else
-    (void)file_name;
-#endif  // !NDEBUG
-  }
-
-  void VerifyFilePath(const std::string& file_path) {
-#ifndef NDEBUG
-    size_t pos = file_path.find_last_of("/");
-    if (pos == std::string::npos) {
-      VerifyFileName(file_path);
-    } else {
-      if (pos > 0) {
-        VerifyFileDir(file_path.substr(0, pos));
-      }
-      VerifyFileName(file_path.substr(pos));
-    }
-#else
-    (void)file_path;
-#endif  // !NDEBUG
-  }
-#endif  // !ROCKSDB_LITE
-
- private:
-  std::string db_name_;
-  std::vector<DbPath> db_paths_;
-  std::vector<ColumnFamilyDescriptor> column_families_;
-  std::atomic<int> num_pending_file_creations_;
-};
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_shared_state.cc

@@ -1,19 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-
-#ifdef GFLAGS
-#include "db_stress_tool/db_stress_shared_state.h"
-
-namespace rocksdb {
-const uint32_t SharedState::UNKNOWN_SENTINEL = 0xfffffffe;
-// indicates a key should definitely be deleted
-const uint32_t SharedState::DELETION_SENTINEL = 0xffffffff;
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_shared_state.h

@@ -1,370 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors
-
-#ifdef GFLAGS
-#pragma once
-
-#include "db_stress_tool/db_stress_stat.h"
-#include "util/gflags_compat.h"
-
-DECLARE_uint64(seed);
-DECLARE_int64(max_key);
-DECLARE_uint64(log2_keys_per_lock);
-DECLARE_int32(threads);
-DECLARE_int32(column_families);
-DECLARE_int32(nooverwritepercent);
-DECLARE_string(expected_values_path);
-DECLARE_int32(clear_column_family_one_in);
-DECLARE_bool(test_batches_snapshots);
-
-namespace rocksdb {
-class StressTest;
-
-// State shared by all concurrent executions of the same benchmark.
-class SharedState {
- public:
-  // indicates a key may have any value (or not be present) as an operation on
-  // it is incomplete.
-  static const uint32_t UNKNOWN_SENTINEL;
-  // indicates a key should definitely be deleted
-  static const uint32_t DELETION_SENTINEL;
-
-  SharedState(Env* env, StressTest* stress_test)
-      : cv_(&mu_),
-        seed_(static_cast<uint32_t>(FLAGS_seed)),
-        max_key_(FLAGS_max_key),
-        log2_keys_per_lock_(static_cast<uint32_t>(FLAGS_log2_keys_per_lock)),
-        num_threads_(FLAGS_threads),
-        num_initialized_(0),
-        num_populated_(0),
-        vote_reopen_(0),
-        num_done_(0),
-        start_(false),
-        start_verify_(false),
-        should_stop_bg_thread_(false),
-        bg_thread_finished_(false),
-        stress_test_(stress_test),
-        verification_failure_(false),
-        no_overwrite_ids_(FLAGS_column_families),
-        values_(nullptr),
-        printing_verification_results_(false) {
-    // Pick random keys in each column family that will not experience
-    // overwrite
-
-    printf("Choosing random keys with no overwrite\n");
-    Random64 rnd(seed_);
-    // Start with the identity permutation. Subsequent iterations of
-    // for loop below will start with perm of previous for loop
-    int64_t* permutation = new int64_t[max_key_];
-    for (int64_t i = 0; i < max_key_; i++) {
-      permutation[i] = i;
-    }
-    // Now do the Knuth shuffle
-    int64_t num_no_overwrite_keys = (max_key_ * FLAGS_nooverwritepercent) / 100;
-    // Only need to figure out first num_no_overwrite_keys of permutation
-    no_overwrite_ids_.reserve(num_no_overwrite_keys);
-    for (int64_t i = 0; i < num_no_overwrite_keys; i++) {
-      int64_t rand_index = i + rnd.Next() % (max_key_ - i);
-      // Swap i and rand_index;
-      int64_t temp = permutation[i];
-      permutation[i] = permutation[rand_index];
-      permutation[rand_index] = temp;
-      // Fill no_overwrite_ids_ with the first num_no_overwrite_keys of
-      // permutation
-      no_overwrite_ids_.insert(permutation[i]);
-    }
-    delete[] permutation;
-
-    size_t expected_values_size =
-        sizeof(std::atomic<uint32_t>) * FLAGS_column_families * max_key_;
-    bool values_init_needed = false;
-    Status status;
-    if (!FLAGS_expected_values_path.empty()) {
-      if (!std::atomic<uint32_t>{}.is_lock_free()) {
-        status = Status::InvalidArgument(
-            "Cannot use --expected_values_path on platforms without lock-free "
-            "std::atomic<uint32_t>");
-      }
-      if (status.ok() && FLAGS_clear_column_family_one_in > 0) {
-        status = Status::InvalidArgument(
-            "Cannot use --expected_values_path on when "
-            "--clear_column_family_one_in is greater than zero.");
-      }
-      uint64_t size = 0;
-      if (status.ok()) {
-        status = env->GetFileSize(FLAGS_expected_values_path, &size);
-      }
-      std::unique_ptr<WritableFile> wfile;
-      if (status.ok() && size == 0) {
-        const EnvOptions soptions;
-        status =
-            env->NewWritableFile(FLAGS_expected_values_path, &wfile, soptions);
-      }
-      if (status.ok() && size == 0) {
-        std::string buf(expected_values_size, '\0');
-        status = wfile->Append(buf);
-        values_init_needed = true;
-      }
-      if (status.ok()) {
-        status = env->NewMemoryMappedFileBuffer(FLAGS_expected_values_path,
-                                                &expected_mmap_buffer_);
-      }
-      if (status.ok()) {
-        assert(expected_mmap_buffer_->GetLen() == expected_values_size);
-        values_ = static_cast<std::atomic<uint32_t>*>(
-            expected_mmap_buffer_->GetBase());
-        assert(values_ != nullptr);
-      } else {
-        fprintf(stderr, "Failed opening shared file '%s' with error: %s\n",
-                FLAGS_expected_values_path.c_str(), status.ToString().c_str());
-        assert(values_ == nullptr);
-      }
-    }
-    if (values_ == nullptr) {
-      values_allocation_.reset(
-          new std::atomic<uint32_t>[FLAGS_column_families * max_key_]);
-      values_ = &values_allocation_[0];
-      values_init_needed = true;
-    }
-    assert(values_ != nullptr);
-    if (values_init_needed) {
-      for (int i = 0; i < FLAGS_column_families; ++i) {
-        for (int j = 0; j < max_key_; ++j) {
-          Delete(i, j, false /* pending */);
-        }
-      }
-    }
-
-    if (FLAGS_test_batches_snapshots) {
-      fprintf(stdout, "No lock creation because test_batches_snapshots set\n");
-      return;
-    }
-
-    long num_locks = static_cast<long>(max_key_ >> log2_keys_per_lock_);
-    if (max_key_ & ((1 << log2_keys_per_lock_) - 1)) {
-      num_locks++;
-    }
-    fprintf(stdout, "Creating %ld locks\n", num_locks * FLAGS_column_families);
-    key_locks_.resize(FLAGS_column_families);
-
-    for (int i = 0; i < FLAGS_column_families; ++i) {
-      key_locks_[i].resize(num_locks);
-      for (auto& ptr : key_locks_[i]) {
-        ptr.reset(new port::Mutex);
-      }
-    }
-  }
-
-  ~SharedState() {}
-
-  port::Mutex* GetMutex() { return &mu_; }
-
-  port::CondVar* GetCondVar() { return &cv_; }
-
-  StressTest* GetStressTest() const { return stress_test_; }
-
-  int64_t GetMaxKey() const { return max_key_; }
-
-  uint32_t GetNumThreads() const { return num_threads_; }
-
-  void IncInitialized() { num_initialized_++; }
-
-  void IncOperated() { num_populated_++; }
-
-  void IncDone() { num_done_++; }
-
-  void IncVotedReopen() { vote_reopen_ = (vote_reopen_ + 1) % num_threads_; }
-
-  bool AllInitialized() const { return num_initialized_ >= num_threads_; }
-
-  bool AllOperated() const { return num_populated_ >= num_threads_; }
-
-  bool AllDone() const { return num_done_ >= num_threads_; }
-
-  bool AllVotedReopen() { return (vote_reopen_ == 0); }
-
-  void SetStart() { start_ = true; }
-
-  void SetStartVerify() { start_verify_ = true; }
-
-  bool Started() const { return start_; }
-
-  bool VerifyStarted() const { return start_verify_; }
-
-  void SetVerificationFailure() { verification_failure_.store(true); }
-
-  bool HasVerificationFailedYet() { return verification_failure_.load(); }
-
-  port::Mutex* GetMutexForKey(int cf, int64_t key) {
-    return key_locks_[cf][key >> log2_keys_per_lock_].get();
-  }
-
-  void LockColumnFamily(int cf) {
-    for (auto& mutex : key_locks_[cf]) {
-      mutex->Lock();
-    }
-  }
-
-  void UnlockColumnFamily(int cf) {
-    for (auto& mutex : key_locks_[cf]) {
-      mutex->Unlock();
-    }
-  }
-
-  std::atomic<uint32_t>& Value(int cf, int64_t key) const {
-    return values_[cf * max_key_ + key];
-  }
-
-  void ClearColumnFamily(int cf) {
-    std::fill(&Value(cf, 0 /* key */), &Value(cf + 1, 0 /* key */),
-              DELETION_SENTINEL);
-  }
-
-  // @param pending True if the update may have started but is not yet
-  //    guaranteed finished. This is useful for crash-recovery testing when the
-  //    process may crash before updating the expected values array.
-  void Put(int cf, int64_t key, uint32_t value_base, bool pending) {
-    if (!pending) {
-      // prevent expected-value update from reordering before Write
-      std::atomic_thread_fence(std::memory_order_release);
-    }
-    Value(cf, key).store(pending ? UNKNOWN_SENTINEL : value_base,
-                         std::memory_order_relaxed);
-    if (pending) {
-      // prevent Write from reordering before expected-value update
-      std::atomic_thread_fence(std::memory_order_release);
-    }
-  }
-
-  uint32_t Get(int cf, int64_t key) const { return Value(cf, key); }
-
-  // @param pending See comment above Put()
-  // Returns true if the key was not yet deleted.
-  bool Delete(int cf, int64_t key, bool pending) {
-    if (Value(cf, key) == DELETION_SENTINEL) {
-      return false;
-    }
-    Put(cf, key, DELETION_SENTINEL, pending);
-    return true;
-  }
-
-  // @param pending See comment above Put()
-  // Returns true if the key was not yet deleted.
-  bool SingleDelete(int cf, int64_t key, bool pending) {
-    return Delete(cf, key, pending);
-  }
-
-  // @param pending See comment above Put()
-  // Returns number of keys deleted by the call.
-  int DeleteRange(int cf, int64_t begin_key, int64_t end_key, bool pending) {
-    int covered = 0;
-    for (int64_t key = begin_key; key < end_key; ++key) {
-      if (Delete(cf, key, pending)) {
-        ++covered;
-      }
-    }
-    return covered;
-  }
-
-  bool AllowsOverwrite(int64_t key) {
-    return no_overwrite_ids_.find(key) == no_overwrite_ids_.end();
-  }
-
-  bool Exists(int cf, int64_t key) {
-    // UNKNOWN_SENTINEL counts as exists. That assures a key for which overwrite
-    // is disallowed can't be accidentally added a second time, in which case
-    // SingleDelete wouldn't be able to properly delete the key. It does allow
-    // the case where a SingleDelete might be added which covers nothing, but
-    // that's not a correctness issue.
-    uint32_t expected_value = Value(cf, key).load();
-    return expected_value != DELETION_SENTINEL;
-  }
-
-  uint32_t GetSeed() const { return seed_; }
-
-  void SetShouldStopBgThread() { should_stop_bg_thread_ = true; }
-
-  bool ShoudStopBgThread() { return should_stop_bg_thread_; }
-
-  void SetBgThreadFinish() { bg_thread_finished_ = true; }
-
-  bool BgThreadFinished() const { return bg_thread_finished_; }
-
-  bool ShouldVerifyAtBeginning() const {
-    return expected_mmap_buffer_.get() != nullptr;
-  }
-
-  bool PrintingVerificationResults() {
-    bool tmp = false;
-    return !printing_verification_results_.compare_exchange_strong(
-        tmp, true, std::memory_order_relaxed);
-  }
-
-  void FinishPrintingVerificationResults() {
-    printing_verification_results_.store(false, std::memory_order_relaxed);
-  }
-
- private:
-  port::Mutex mu_;
-  port::CondVar cv_;
-  const uint32_t seed_;
-  const int64_t max_key_;
-  const uint32_t log2_keys_per_lock_;
-  const int num_threads_;
-  long num_initialized_;
-  long num_populated_;
-  long vote_reopen_;
-  long num_done_;
-  bool start_;
-  bool start_verify_;
-  bool should_stop_bg_thread_;
-  bool bg_thread_finished_;
-  StressTest* stress_test_;
-  std::atomic<bool> verification_failure_;
-
-  // Keys that should not be overwritten
-  std::unordered_set<size_t> no_overwrite_ids_;
-
-  std::atomic<uint32_t>* values_;
-  std::unique_ptr<std::atomic<uint32_t>[]> values_allocation_;
-  // Has to make it owned by a smart ptr as port::Mutex is not copyable
-  // and storing it in the container may require copying depending on the impl.
-  std::vector<std::vector<std::unique_ptr<port::Mutex>>> key_locks_;
-  std::unique_ptr<MemoryMappedFileBuffer> expected_mmap_buffer_;
-  std::atomic<bool> printing_verification_results_;
-};
-
-// Per-thread state for concurrent executions of the same benchmark.
-struct ThreadState {
-  uint32_t tid;  // 0..n-1
-  Random rand;   // Has different seeds for different threads
-  SharedState* shared;
-  Stats stats;
-  struct SnapshotState {
-    const Snapshot* snapshot;
-    // The cf from which we did a Get at this snapshot
-    int cf_at;
-    // The name of the cf at the time that we did a read
-    std::string cf_at_name;
-    // The key with which we did a Get at this snapshot
-    std::string key;
-    // The status of the Get
-    Status status;
-    // The value of the Get
-    std::string value;
-    // optional state of all keys in the db
-    std::vector<bool>* key_vec;
-  };
-  std::queue<std::pair<uint64_t, SnapshotState>> snapshot_queue;
-
-  ThreadState(uint32_t index, SharedState* _shared)
-      : tid(index), rand(1000 + index + _shared->GetSeed()), shared(_shared) {}
-};
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_stat.h

@@ -1,213 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-#include <cinttypes>
-#include <memory>
-#include <queue>
-#include <unordered_set>
-
-#include "monitoring/histogram.h"
-#include "port/port.h"
-#include "rocksdb/env.h"
-#include "rocksdb/snapshot.h"
-#include "rocksdb/statistics.h"
-#include "util/gflags_compat.h"
-#include "util/random.h"
-
-DECLARE_bool(histogram);
-DECLARE_bool(progress_reports);
-
-namespace rocksdb {
-// Database statistics
-static std::shared_ptr<rocksdb::Statistics> dbstats;
-static std::shared_ptr<rocksdb::Statistics> dbstats_secondaries;
-
-class Stats {
- private:
-  uint64_t start_;
-  uint64_t finish_;
-  double seconds_;
-  long done_;
-  long gets_;
-  long prefixes_;
-  long writes_;
-  long deletes_;
-  size_t single_deletes_;
-  long iterator_size_sums_;
-  long founds_;
-  long iterations_;
-  long range_deletions_;
-  long covered_by_range_deletions_;
-  long errors_;
-  long num_compact_files_succeed_;
-  long num_compact_files_failed_;
-  int next_report_;
-  size_t bytes_;
-  uint64_t last_op_finish_;
-  HistogramImpl hist_;
-
- public:
-  Stats() {}
-
-  void Start() {
-    next_report_ = 100;
-    hist_.Clear();
-    done_ = 0;
-    gets_ = 0;
-    prefixes_ = 0;
-    writes_ = 0;
-    deletes_ = 0;
-    single_deletes_ = 0;
-    iterator_size_sums_ = 0;
-    founds_ = 0;
-    iterations_ = 0;
-    range_deletions_ = 0;
-    covered_by_range_deletions_ = 0;
-    errors_ = 0;
-    bytes_ = 0;
-    seconds_ = 0;
-    num_compact_files_succeed_ = 0;
-    num_compact_files_failed_ = 0;
-    start_ = Env::Default()->NowMicros();
-    last_op_finish_ = start_;
-    finish_ = start_;
-  }
-
-  void Merge(const Stats& other) {
-    hist_.Merge(other.hist_);
-    done_ += other.done_;
-    gets_ += other.gets_;
-    prefixes_ += other.prefixes_;
-    writes_ += other.writes_;
-    deletes_ += other.deletes_;
-    single_deletes_ += other.single_deletes_;
-    iterator_size_sums_ += other.iterator_size_sums_;
-    founds_ += other.founds_;
-    iterations_ += other.iterations_;
-    range_deletions_ += other.range_deletions_;
-    covered_by_range_deletions_ = other.covered_by_range_deletions_;
-    errors_ += other.errors_;
-    bytes_ += other.bytes_;
-    seconds_ += other.seconds_;
-    num_compact_files_succeed_ += other.num_compact_files_succeed_;
-    num_compact_files_failed_ += other.num_compact_files_failed_;
-    if (other.start_ < start_) start_ = other.start_;
-    if (other.finish_ > finish_) finish_ = other.finish_;
-  }
-
-  void Stop() {
-    finish_ = Env::Default()->NowMicros();
-    seconds_ = (finish_ - start_) * 1e-6;
-  }
-
-  void FinishedSingleOp() {
-    if (FLAGS_histogram) {
-      auto now = Env::Default()->NowMicros();
-      auto micros = now - last_op_finish_;
-      hist_.Add(micros);
-      if (micros > 20000) {
-        fprintf(stdout, "long op: %" PRIu64 " micros%30s\r", micros, "");
-      }
-      last_op_finish_ = now;
-    }
-
-    done_++;
-    if (FLAGS_progress_reports) {
-      if (done_ >= next_report_) {
-        if (next_report_ < 1000)
-          next_report_ += 100;
-        else if (next_report_ < 5000)
-          next_report_ += 500;
-        else if (next_report_ < 10000)
-          next_report_ += 1000;
-        else if (next_report_ < 50000)
-          next_report_ += 5000;
-        else if (next_report_ < 100000)
-          next_report_ += 10000;
-        else if (next_report_ < 500000)
-          next_report_ += 50000;
-        else
-          next_report_ += 100000;
-        fprintf(stdout, "... finished %ld ops%30s\r", done_, "");
-      }
-    }
-  }
-
-  void AddBytesForWrites(long nwrites, size_t nbytes) {
-    writes_ += nwrites;
-    bytes_ += nbytes;
-  }
-
-  void AddGets(long ngets, long nfounds) {
-    founds_ += nfounds;
-    gets_ += ngets;
-  }
-
-  void AddPrefixes(long nprefixes, long count) {
-    prefixes_ += nprefixes;
-    iterator_size_sums_ += count;
-  }
-
-  void AddIterations(long n) { iterations_ += n; }
-
-  void AddDeletes(long n) { deletes_ += n; }
-
-  void AddSingleDeletes(size_t n) { single_deletes_ += n; }
-
-  void AddRangeDeletions(long n) { range_deletions_ += n; }
-
-  void AddCoveredByRangeDeletions(long n) { covered_by_range_deletions_ += n; }
-
-  void AddErrors(long n) { errors_ += n; }
-
-  void AddNumCompactFilesSucceed(long n) { num_compact_files_succeed_ += n; }
-
-  void AddNumCompactFilesFailed(long n) { num_compact_files_failed_ += n; }
-
-  void Report(const char* name) {
-    std::string extra;
-    if (bytes_ < 1 || done_ < 1) {
-      fprintf(stderr, "No writes or ops?\n");
-      return;
-    }
-
-    double elapsed = (finish_ - start_) * 1e-6;
-    double bytes_mb = bytes_ / 1048576.0;
-    double rate = bytes_mb / elapsed;
-    double throughput = (double)done_ / elapsed;
-
-    fprintf(stdout, "%-12s: ", name);
-    fprintf(stdout, "%.3f micros/op %ld ops/sec\n", seconds_ * 1e6 / done_,
-            (long)throughput);
-    fprintf(stdout, "%-12s: Wrote %.2f MB (%.2f MB/sec) (%ld%% of %ld ops)\n",
-            "", bytes_mb, rate, (100 * writes_) / done_, done_);
-    fprintf(stdout, "%-12s: Wrote %ld times\n", "", writes_);
-    fprintf(stdout, "%-12s: Deleted %ld times\n", "", deletes_);
-    fprintf(stdout, "%-12s: Single deleted %" ROCKSDB_PRIszt " times\n", "",
-            single_deletes_);
-    fprintf(stdout, "%-12s: %ld read and %ld found the key\n", "", gets_,
-            founds_);
-    fprintf(stdout, "%-12s: Prefix scanned %ld times\n", "", prefixes_);
-    fprintf(stdout, "%-12s: Iterator size sum is %ld\n", "",
-            iterator_size_sums_);
-    fprintf(stdout, "%-12s: Iterated %ld times\n", "", iterations_);
-    fprintf(stdout, "%-12s: Deleted %ld key-ranges\n", "", range_deletions_);
-    fprintf(stdout, "%-12s: Range deletions covered %ld keys\n", "",
-            covered_by_range_deletions_);
-
-    fprintf(stdout, "%-12s: Got errors %ld times\n", "", errors_);
-    fprintf(stdout, "%-12s: %ld CompactFiles() succeed\n", "",
-            num_compact_files_succeed_);
-    fprintf(stdout, "%-12s: %ld CompactFiles() did not succeed\n", "",
-            num_compact_files_failed_);
-
-    if (FLAGS_histogram) {
-      fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());
-    }
-    fflush(stdout);
-  }
-};
-}  // namespace rocksdb

db_stress_tool/db_stress_test_base.h

@@ -1,198 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifdef GFLAGS
-#pragma once
-#include "db_stress_tool/db_stress_common.h"
-#include "db_stress_tool/db_stress_shared_state.h"
-
-namespace rocksdb {
-class Transaction;
-class TransactionDB;
-
-class StressTest {
- public:
-  StressTest();
-
-  virtual ~StressTest();
-
-  std::shared_ptr<Cache> NewCache(size_t capacity);
-
-  bool BuildOptionsTable();
-
-  void InitDb();
-  void InitReadonlyDb(SharedState*);
-
-  // Return false if verification fails.
-  bool VerifySecondaries();
-
-  void OperateDb(ThreadState* thread);
-  virtual void VerifyDb(ThreadState* thread) const = 0;
-
-  void PrintStatistics();
-
- protected:
-  Status AssertSame(DB* db, ColumnFamilyHandle* cf,
-                    ThreadState::SnapshotState& snap_state);
-
-  // Currently PreloadDb has to be single-threaded.
-  void PreloadDbAndReopenAsReadOnly(int64_t number_of_keys,
-                                    SharedState* shared);
-
-  Status SetOptions(ThreadState* thread);
-
-#ifndef ROCKSDB_LITE
-  Status NewTxn(WriteOptions& write_opts, Transaction** txn);
-
-  Status CommitTxn(Transaction* txn);
-#endif
-
-  virtual void MaybeClearOneColumnFamily(ThreadState* /* thread */) {}
-
-  virtual bool ShouldAcquireMutexOnKey() const { return false; }
-
-  virtual std::vector<int> GenerateColumnFamilies(
-      const int /* num_column_families */, int rand_column_family) const {
-    return {rand_column_family};
-  }
-
-  virtual std::vector<int64_t> GenerateKeys(int64_t rand_key) const {
-    return {rand_key};
-  }
-
-  virtual Status TestGet(ThreadState* thread, const ReadOptions& read_opts,
-                         const std::vector<int>& rand_column_families,
-                         const std::vector<int64_t>& rand_keys) = 0;
-
-  virtual std::vector<Status> TestMultiGet(
-      ThreadState* thread, const ReadOptions& read_opts,
-      const std::vector<int>& rand_column_families,
-      const std::vector<int64_t>& rand_keys) = 0;
-
-  virtual Status TestPrefixScan(ThreadState* thread,
-                                const ReadOptions& read_opts,
-                                const std::vector<int>& rand_column_families,
-                                const std::vector<int64_t>& rand_keys) = 0;
-
-  virtual Status TestPut(ThreadState* thread, WriteOptions& write_opts,
-                         const ReadOptions& read_opts,
-                         const std::vector<int>& cf_ids,
-                         const std::vector<int64_t>& keys, char (&value)[100],
-                         std::unique_ptr<MutexLock>& lock) = 0;
-
-  virtual Status TestDelete(ThreadState* thread, WriteOptions& write_opts,
-                            const std::vector<int>& rand_column_families,
-                            const std::vector<int64_t>& rand_keys,
-                            std::unique_ptr<MutexLock>& lock) = 0;
-
-  virtual Status TestDeleteRange(ThreadState* thread, WriteOptions& write_opts,
-                                 const std::vector<int>& rand_column_families,
-                                 const std::vector<int64_t>& rand_keys,
-                                 std::unique_ptr<MutexLock>& lock) = 0;
-
-  virtual void TestIngestExternalFile(
-      ThreadState* thread, const std::vector<int>& rand_column_families,
-      const std::vector<int64_t>& rand_keys,
-      std::unique_ptr<MutexLock>& lock) = 0;
-
-  // Issue compact range, starting with start_key, whose integer value
-  // is rand_key.
-  virtual void TestCompactRange(ThreadState* thread, int64_t rand_key,
-                                const Slice& start_key,
-                                ColumnFamilyHandle* column_family);
-
-  // Calculate a hash value for all keys in range [start_key, end_key]
-  // at a certain snapshot.
-  uint32_t GetRangeHash(ThreadState* thread, const Snapshot* snapshot,
-                        ColumnFamilyHandle* column_family,
-                        const Slice& start_key, const Slice& end_key);
-
-  // Return a column family handle that mirrors what is pointed by
-  // `column_family_id`, which will be used to validate data to be correct.
-  // By default, the column family itself will be returned.
-  virtual ColumnFamilyHandle* GetControlCfh(ThreadState* /* thread*/,
-                                            int column_family_id) {
-    return column_families_[column_family_id];
-  }
-
-#ifndef ROCKSDB_LITE
-  // Generated a list of keys that close to boundaries of SST keys.
-  // If there isn't any SST file in the DB, return empty list.
-  std::vector<std::string> GetWhiteBoxKeys(ThreadState* thread, DB* db,
-                                           ColumnFamilyHandle* cfh,
-                                           size_t num_keys);
-#else   // !ROCKSDB_LITE
-  std::vector<std::string> GetWhiteBoxKeys(ThreadState*, DB*,
-                                           ColumnFamilyHandle*, size_t) {
-    // Not supported in LITE mode.
-    return {};
-  }
-#endif  // !ROCKSDB_LITE
-
-  // Given a key K, this creates an iterator which scans to K and then
-  // does a random sequence of Next/Prev operations.
-  virtual Status TestIterate(ThreadState* thread, const ReadOptions& read_opts,
-                             const std::vector<int>& rand_column_families,
-                             const std::vector<int64_t>& rand_keys);
-
-  // Enum used by VerifyIterator() to identify the mode to validate.
-  enum LastIterateOp { kLastOpSeek, kLastOpSeekForPrev, kLastOpNextOrPrev };
-
-  // Compare the two iterator, iter and cmp_iter are in the same position,
-  // unless iter might be made invalidate or undefined because of
-  // upper or lower bounds, or prefix extractor.
-  // Will flag failure if the verification fails.
-  // diverged = true if the two iterator is already diverged.
-  // True if verification passed, false if not.
-  void VerifyIterator(ThreadState* thread, ColumnFamilyHandle* cmp_cfh,
-                      const ReadOptions& ro, Iterator* iter, Iterator* cmp_iter,
-                      LastIterateOp op, const Slice& seek_key, bool* diverged);
-
-  virtual Status TestBackupRestore(ThreadState* thread,
-                                   const std::vector<int>& rand_column_families,
-                                   const std::vector<int64_t>& rand_keys);
-
-  virtual Status TestCheckpoint(ThreadState* thread,
-                                const std::vector<int>& rand_column_families,
-                                const std::vector<int64_t>& rand_keys);
-
-  void VerificationAbort(SharedState* shared, std::string msg, Status s) const;
-
-  void VerificationAbort(SharedState* shared, std::string msg, int cf,
-                         int64_t key) const;
-
-  void PrintEnv() const;
-
-  void Open();
-
-  void Reopen(ThreadState* thread);
-
-  std::shared_ptr<Cache> cache_;
-  std::shared_ptr<Cache> compressed_cache_;
-  std::shared_ptr<const FilterPolicy> filter_policy_;
-  DB* db_;
-#ifndef ROCKSDB_LITE
-  TransactionDB* txn_db_;
-#endif
-  Options options_;
-  std::vector<ColumnFamilyHandle*> column_families_;
-  std::vector<std::string> column_family_names_;
-  std::atomic<int> new_column_family_name_;
-  int num_times_reopened_;
-  std::unordered_map<std::string, std::vector<std::string>> options_table_;
-  std::vector<std::string> options_index_;
-  std::atomic<bool> db_preload_finished_;
-
-  // Fields used for stress-testing secondary instance in the same process
-  std::vector<DB*> secondaries_;
-  std::vector<std::vector<ColumnFamilyHandle*>> secondary_cfh_lists_;
-};
-
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/db_stress_tool.cc

@@ -1,201 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// The test uses an array to compare against values written to the database.
-// Keys written to the array are in 1:1 correspondence to the actual values in
-// the database according to the formula in the function GenerateValue.
-
-// Space is reserved in the array from 0 to FLAGS_max_key and values are
-// randomly written/deleted/read from those positions. During verification we
-// compare all the positions in the array. To shorten/elongate the running
-// time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread,
-// (sometimes also FLAGS_threads).
-//
-// NOTE that if FLAGS_test_batches_snapshots is set, the test will have
-// different behavior. See comment of the flag for details.
-
-#ifdef GFLAGS
-#include "db_stress_tool/db_stress_common.h"
-#include "db_stress_tool/db_stress_driver.h"
-
-namespace rocksdb {
-namespace {
-static std::shared_ptr<rocksdb::Env> env_guard;
-}  // namespace
-
-int db_stress_tool(int argc, char** argv) {
-  SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
-                  " [OPTIONS]...");
-  ParseCommandLineFlags(&argc, &argv, true);
-
-  SanitizeDoubleParam(&FLAGS_bloom_bits);
-  SanitizeDoubleParam(&FLAGS_memtable_prefix_bloom_size_ratio);
-  SanitizeDoubleParam(&FLAGS_max_bytes_for_level_multiplier);
-
-  if (FLAGS_statistics) {
-    dbstats = rocksdb::CreateDBStatistics();
-    if (FLAGS_enable_secondary) {
-      dbstats_secondaries = rocksdb::CreateDBStatistics();
-    }
-  }
-  FLAGS_compression_type_e =
-      StringToCompressionType(FLAGS_compression_type.c_str());
-  FLAGS_checksum_type_e = StringToChecksumType(FLAGS_checksum_type.c_str());
-  if (!FLAGS_hdfs.empty()) {
-    if (!FLAGS_env_uri.empty()) {
-      fprintf(stderr, "Cannot specify both --hdfs and --env_uri.\n");
-      exit(1);
-    }
-    FLAGS_env = new rocksdb::HdfsEnv(FLAGS_hdfs);
-  } else if (!FLAGS_env_uri.empty()) {
-    Status s = Env::LoadEnv(FLAGS_env_uri, &FLAGS_env, &env_guard);
-    if (FLAGS_env == nullptr) {
-      fprintf(stderr, "No Env registered for URI: %s\n", FLAGS_env_uri.c_str());
-      exit(1);
-    }
-  }
-  FLAGS_rep_factory = StringToRepFactory(FLAGS_memtablerep.c_str());
-
-  // The number of background threads should be at least as much the
-  // max number of concurrent compactions.
-  FLAGS_env->SetBackgroundThreads(FLAGS_max_background_compactions);
-  FLAGS_env->SetBackgroundThreads(FLAGS_num_bottom_pri_threads,
-                                  rocksdb::Env::Priority::BOTTOM);
-  if (FLAGS_prefixpercent > 0 && FLAGS_prefix_size < 0) {
-    fprintf(stderr,
-            "Error: prefixpercent is non-zero while prefix_size is "
-            "not positive!\n");
-    exit(1);
-  }
-  if (FLAGS_test_batches_snapshots && FLAGS_prefix_size <= 0) {
-    fprintf(stderr,
-            "Error: please specify prefix_size for "
-            "test_batches_snapshots test!\n");
-    exit(1);
-  }
-  if (FLAGS_memtable_prefix_bloom_size_ratio > 0.0 && FLAGS_prefix_size < 0) {
-    fprintf(stderr,
-            "Error: please specify positive prefix_size in order to use "
-            "memtable_prefix_bloom_size_ratio\n");
-    exit(1);
-  }
-  if ((FLAGS_readpercent + FLAGS_prefixpercent + FLAGS_writepercent +
-       FLAGS_delpercent + FLAGS_delrangepercent + FLAGS_iterpercent) != 100) {
-    fprintf(stderr,
-            "Error: Read+Prefix+Write+Delete+DeleteRange+Iterate percents != "
-            "100!\n");
-    exit(1);
-  }
-  if (FLAGS_disable_wal == 1 && FLAGS_reopen > 0) {
-    fprintf(stderr, "Error: Db cannot reopen safely with disable_wal set!\n");
-    exit(1);
-  }
-  if ((unsigned)FLAGS_reopen >= FLAGS_ops_per_thread) {
-    fprintf(stderr,
-            "Error: #DB-reopens should be < ops_per_thread\n"
-            "Provided reopens = %d and ops_per_thread = %lu\n",
-            FLAGS_reopen, (unsigned long)FLAGS_ops_per_thread);
-    exit(1);
-  }
-  if (FLAGS_test_batches_snapshots && FLAGS_delrangepercent > 0) {
-    fprintf(stderr,
-            "Error: nonzero delrangepercent unsupported in "
-            "test_batches_snapshots mode\n");
-    exit(1);
-  }
-  if (FLAGS_active_width > FLAGS_max_key) {
-    fprintf(stderr, "Error: active_width can be at most max_key\n");
-    exit(1);
-  } else if (FLAGS_active_width == 0) {
-    FLAGS_active_width = FLAGS_max_key;
-  }
-  if (FLAGS_value_size_mult * kRandomValueMaxFactor > kValueMaxLen) {
-    fprintf(stderr, "Error: value_size_mult can be at most %d\n",
-            kValueMaxLen / kRandomValueMaxFactor);
-    exit(1);
-  }
-  if (FLAGS_use_merge && FLAGS_nooverwritepercent == 100) {
-    fprintf(
-        stderr,
-        "Error: nooverwritepercent must not be 100 when using merge operands");
-    exit(1);
-  }
-  if (FLAGS_ingest_external_file_one_in > 0 && FLAGS_nooverwritepercent > 0) {
-    fprintf(stderr,
-            "Error: nooverwritepercent must be 0 when using file ingestion\n");
-    exit(1);
-  }
-  if (FLAGS_clear_column_family_one_in > 0 && FLAGS_backup_one_in > 0) {
-    fprintf(stderr,
-            "Error: clear_column_family_one_in must be 0 when using backup\n");
-    exit(1);
-  }
-  if (FLAGS_test_cf_consistency && FLAGS_disable_wal) {
-    FLAGS_atomic_flush = true;
-  }
-
-  if (FLAGS_read_only) {
-    if (FLAGS_writepercent != 0 || FLAGS_delpercent != 0 ||
-        FLAGS_delrangepercent != 0) {
-      fprintf(stderr, "Error: updates are not supported in read only mode\n");
-      exit(1);
-    } else if (FLAGS_checkpoint_one_in > 0 &&
-               FLAGS_clear_column_family_one_in > 0) {
-      fprintf(stdout,
-              "Warn: checkpoint won't be validated since column families may "
-              "be dropped.\n");
-    }
-  }
-
-  // Choose a location for the test database if none given with --db=<path>
-  if (FLAGS_db.empty()) {
-    std::string default_db_path;
-    FLAGS_env->GetTestDirectory(&default_db_path);
-    default_db_path += "/dbstress";
-    FLAGS_db = default_db_path;
-  }
-
-  if (FLAGS_enable_secondary && FLAGS_secondaries_base.empty()) {
-    std::string default_secondaries_path;
-    FLAGS_env->GetTestDirectory(&default_secondaries_path);
-    default_secondaries_path += "/dbstress_secondaries";
-    rocksdb::Status s = FLAGS_env->CreateDirIfMissing(default_secondaries_path);
-    if (!s.ok()) {
-      fprintf(stderr, "Failed to create directory %s: %s\n",
-              default_secondaries_path.c_str(), s.ToString().c_str());
-      exit(1);
-    }
-    FLAGS_secondaries_base = default_secondaries_path;
-  }
-
-  if (!FLAGS_enable_secondary && FLAGS_secondary_catch_up_one_in > 0) {
-    fprintf(stderr, "Secondary instance is disabled.\n");
-    exit(1);
-  }
-
-  rocksdb_kill_odds = FLAGS_kill_random_test;
-  rocksdb_kill_prefix_blacklist = SplitString(FLAGS_kill_prefix_blacklist);
-
-  std::unique_ptr<rocksdb::StressTest> stress;
-  if (FLAGS_test_cf_consistency) {
-    stress.reset(CreateCfConsistencyStressTest());
-  } else if (FLAGS_test_batches_snapshots) {
-    stress.reset(CreateBatchedOpsStressTest());
-  } else {
-    stress.reset(CreateNonBatchedOpsStressTest());
-  }
-  if (RunStressTest(stress.get())) {
-    return 0;
-  } else {
-    return 1;
-  }
-}
-
-}  // namespace rocksdb
-#endif  // GFLAGS

db_stress_tool/no_batched_ops_stress.cc

@@ -1,553 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifdef GFLAGS
-#include "db_stress_tool/db_stress_common.h"
-
-namespace rocksdb {
-class NonBatchedOpsStressTest : public StressTest {
- public:
-  NonBatchedOpsStressTest() {}
-
-  virtual ~NonBatchedOpsStressTest() {}
-
-  void VerifyDb(ThreadState* thread) const override {
-    ReadOptions options(FLAGS_verify_checksum, true);
-    auto shared = thread->shared;
-    const int64_t max_key = shared->GetMaxKey();
-    const int64_t keys_per_thread = max_key / shared->GetNumThreads();
-    int64_t start = keys_per_thread * thread->tid;
-    int64_t end = start + keys_per_thread;
-    uint64_t prefix_to_use =
-        (FLAGS_prefix_size < 0) ? 1 : static_cast<size_t>(FLAGS_prefix_size);
-    if (thread->tid == shared->GetNumThreads() - 1) {
-      end = max_key;
-    }
-    for (size_t cf = 0; cf < column_families_.size(); ++cf) {
-      if (thread->shared->HasVerificationFailedYet()) {
-        break;
-      }
-      if (!thread->rand.OneIn(2)) {
-        // Use iterator to verify this range
-        std::unique_ptr<Iterator> iter(
-            db_->NewIterator(options, column_families_[cf]));
-        iter->Seek(Key(start));
-        for (auto i = start; i < end; i++) {
-          if (thread->shared->HasVerificationFailedYet()) {
-            break;
-          }
-          // TODO(ljin): update "long" to uint64_t
-          // Reseek when the prefix changes
-          if (prefix_to_use > 0 &&
-              i % (static_cast<int64_t>(1) << 8 * (8 - prefix_to_use)) == 0) {
-            iter->Seek(Key(i));
-          }
-          std::string from_db;
-          std::string keystr = Key(i);
-          Slice k = keystr;
-          Status s = iter->status();
-          if (iter->Valid()) {
-            if (iter->key().compare(k) > 0) {
-              s = Status::NotFound(Slice());
-            } else if (iter->key().compare(k) == 0) {
-              from_db = iter->value().ToString();
-              iter->Next();
-            } else if (iter->key().compare(k) < 0) {
-              VerificationAbort(shared, "An out of range key was found",
-                                static_cast<int>(cf), i);
-            }
-          } else {
-            // The iterator found no value for the key in question, so do not
-            // move to the next item in the iterator
-            s = Status::NotFound(Slice());
-          }
-          VerifyValue(static_cast<int>(cf), i, options, shared, from_db, s,
-                      true);
-          if (from_db.length()) {
-            PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i),
-                          from_db.data(), from_db.length());
-          }
-        }
-      } else {
-        // Use Get to verify this range
-        for (auto i = start; i < end; i++) {
-          if (thread->shared->HasVerificationFailedYet()) {
-            break;
-          }
-          std::string from_db;
-          std::string keystr = Key(i);
-          Slice k = keystr;
-          Status s = db_->Get(options, column_families_[cf], k, &from_db);
-          VerifyValue(static_cast<int>(cf), i, options, shared, from_db, s,
-                      true);
-          if (from_db.length()) {
-            PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i),
-                          from_db.data(), from_db.length());
-          }
-        }
-      }
-    }
-  }
-
-  void MaybeClearOneColumnFamily(ThreadState* thread) override {
-    if (FLAGS_clear_column_family_one_in != 0 && FLAGS_column_families > 1) {
-      if (thread->rand.OneIn(FLAGS_clear_column_family_one_in)) {
-        // drop column family and then create it again (can't drop default)
-        int cf = thread->rand.Next() % (FLAGS_column_families - 1) + 1;
-        std::string new_name = ToString(new_column_family_name_.fetch_add(1));
-        {
-          MutexLock l(thread->shared->GetMutex());
-          fprintf(
-              stdout,
-              "[CF %d] Dropping and recreating column family. new name: %s\n",
-              cf, new_name.c_str());
-        }
-        thread->shared->LockColumnFamily(cf);
-        Status s = db_->DropColumnFamily(column_families_[cf]);
-        delete column_families_[cf];
-        if (!s.ok()) {
-          fprintf(stderr, "dropping column family error: %s\n",
-                  s.ToString().c_str());
-          std::terminate();
-        }
-        s = db_->CreateColumnFamily(ColumnFamilyOptions(options_), new_name,
-                                    &column_families_[cf]);
-        column_family_names_[cf] = new_name;
-        thread->shared->ClearColumnFamily(cf);
-        if (!s.ok()) {
-          fprintf(stderr, "creating column family error: %s\n",
-                  s.ToString().c_str());
-          std::terminate();
-        }
-        thread->shared->UnlockColumnFamily(cf);
-      }
-    }
-  }
-
-  bool ShouldAcquireMutexOnKey() const override { return true; }
-
-  Status TestGet(ThreadState* thread, const ReadOptions& read_opts,
-                 const std::vector<int>& rand_column_families,
-                 const std::vector<int64_t>& rand_keys) override {
-    auto cfh = column_families_[rand_column_families[0]];
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    std::string from_db;
-    Status s = db_->Get(read_opts, cfh, key, &from_db);
-    if (s.ok()) {
-      // found case
-      thread->stats.AddGets(1, 1);
-    } else if (s.IsNotFound()) {
-      // not found case
-      thread->stats.AddGets(1, 0);
-    } else {
-      // errors case
-      thread->stats.AddErrors(1);
-    }
-    return s;
-  }
-
-  std::vector<Status> TestMultiGet(
-      ThreadState* thread, const ReadOptions& read_opts,
-      const std::vector<int>& rand_column_families,
-      const std::vector<int64_t>& rand_keys) override {
-    size_t num_keys = rand_keys.size();
-    std::vector<std::string> key_str;
-    std::vector<Slice> keys;
-    key_str.reserve(num_keys);
-    keys.reserve(num_keys);
-    std::vector<PinnableSlice> values(num_keys);
-    std::vector<Status> statuses(num_keys);
-    ColumnFamilyHandle* cfh = column_families_[rand_column_families[0]];
-
-    for (size_t i = 0; i < num_keys; ++i) {
-      key_str.emplace_back(Key(rand_keys[i]));
-      keys.emplace_back(key_str.back());
-    }
-    db_->MultiGet(read_opts, cfh, num_keys, keys.data(), values.data(),
-                  statuses.data());
-    for (const auto& s : statuses) {
-      if (s.ok()) {
-        // found case
-        thread->stats.AddGets(1, 1);
-      } else if (s.IsNotFound()) {
-        // not found case
-        thread->stats.AddGets(1, 0);
-      } else {
-        // errors case
-        thread->stats.AddErrors(1);
-      }
-    }
-    return statuses;
-  }
-
-  Status TestPrefixScan(ThreadState* thread, const ReadOptions& read_opts,
-                        const std::vector<int>& rand_column_families,
-                        const std::vector<int64_t>& rand_keys) override {
-    auto cfh = column_families_[rand_column_families[0]];
-    std::string key_str = Key(rand_keys[0]);
-    Slice key = key_str;
-    Slice prefix = Slice(key.data(), FLAGS_prefix_size);
-
-    std::string upper_bound;
-    Slice ub_slice;
-    ReadOptions ro_copy = read_opts;
-    if (thread->rand.OneIn(2) && GetNextPrefix(prefix, &upper_bound)) {
-      // For half of the time, set the upper bound to the next prefix
-      ub_slice = Slice(upper_bound);
-      ro_copy.iterate_upper_bound = &ub_slice;
-    }
-
-    Iterator* iter = db_->NewIterator(ro_copy, cfh);
-    long count = 0;
-    for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix);
-         iter->Next()) {
-      ++count;
-    }
-    assert(count <= (static_cast<long>(1) << ((8 - FLAGS_prefix_size) * 8)));
-    Status s = iter->status();
-    if (iter->status().ok()) {
-      thread->stats.AddPrefixes(1, count);
-    } else {
-      thread->stats.AddErrors(1);
-    }
-    delete iter;
-    return s;
-  }
-
-  Status TestPut(ThreadState* thread, WriteOptions& write_opts,
-                 const ReadOptions& read_opts,
-                 const std::vector<int>& rand_column_families,
-                 const std::vector<int64_t>& rand_keys, char (&value)[100],
-                 std::unique_ptr<MutexLock>& lock) override {
-    auto shared = thread->shared;
-    int64_t max_key = shared->GetMaxKey();
-    int64_t rand_key = rand_keys[0];
-    int rand_column_family = rand_column_families[0];
-    while (!shared->AllowsOverwrite(rand_key) &&
-           (FLAGS_use_merge || shared->Exists(rand_column_family, rand_key))) {
-      lock.reset();
-      rand_key = thread->rand.Next() % max_key;
-      rand_column_family = thread->rand.Next() % FLAGS_column_families;
-      lock.reset(
-          new MutexLock(shared->GetMutexForKey(rand_column_family, rand_key)));
-    }
-
-    std::string key_str = Key(rand_key);
-    Slice key = key_str;
-    ColumnFamilyHandle* cfh = column_families_[rand_column_family];
-
-    if (FLAGS_verify_before_write) {
-      std::string key_str2 = Key(rand_key);
-      Slice k = key_str2;
-      std::string from_db;
-      Status s = db_->Get(read_opts, cfh, k, &from_db);
-      if (!VerifyValue(rand_column_family, rand_key, read_opts, shared, from_db,
-                       s, true)) {
-        return s;
-      }
-    }
-    uint32_t value_base = thread->rand.Next() % shared->UNKNOWN_SENTINEL;
-    size_t sz = GenerateValue(value_base, value, sizeof(value));
-    Slice v(value, sz);
-    shared->Put(rand_column_family, rand_key, value_base, true /* pending */);
-    Status s;
-    if (FLAGS_use_merge) {
-      if (!FLAGS_use_txn) {
-        s = db_->Merge(write_opts, cfh, key, v);
-      } else {
-#ifndef ROCKSDB_LITE
-        Transaction* txn;
-        s = NewTxn(write_opts, &txn);
-        if (s.ok()) {
-          s = txn->Merge(cfh, key, v);
-          if (s.ok()) {
-            s = CommitTxn(txn);
-          }
-        }
-#endif
-      }
-    } else {
-      if (!FLAGS_use_txn) {
-        s = db_->Put(write_opts, cfh, key, v);
-      } else {
-#ifndef ROCKSDB_LITE
-        Transaction* txn;
-        s = NewTxn(write_opts, &txn);
-        if (s.ok()) {
-          s = txn->Put(cfh, key, v);
-          if (s.ok()) {
-            s = CommitTxn(txn);
-          }
-        }
-#endif
-      }
-    }
-    shared->Put(rand_column_family, rand_key, value_base, false /* pending */);
-    if (!s.ok()) {
-      fprintf(stderr, "put or merge error: %s\n", s.ToString().c_str());
-      std::terminate();
-    }
-    thread->stats.AddBytesForWrites(1, sz);
-    PrintKeyValue(rand_column_family, static_cast<uint32_t>(rand_key), value,
-                  sz);
-    return s;
-  }
-
-  Status TestDelete(ThreadState* thread, WriteOptions& write_opts,
-                    const std::vector<int>& rand_column_families,
-                    const std::vector<int64_t>& rand_keys,
-                    std::unique_ptr<MutexLock>& lock) override {
-    int64_t rand_key = rand_keys[0];
-    int rand_column_family = rand_column_families[0];
-    auto shared = thread->shared;
-    int64_t max_key = shared->GetMaxKey();
-
-    // OPERATION delete
-    // If the chosen key does not allow overwrite and it does not exist,
-    // choose another key.
-    while (!shared->AllowsOverwrite(rand_key) &&
-           !shared->Exists(rand_column_family, rand_key)) {
-      lock.reset();
-      rand_key = thread->rand.Next() % max_key;
-      rand_column_family = thread->rand.Next() % FLAGS_column_families;
-      lock.reset(
-          new MutexLock(shared->GetMutexForKey(rand_column_family, rand_key)));
-    }
-
-    std::string key_str = Key(rand_key);
-    Slice key = key_str;
-    auto cfh = column_families_[rand_column_family];
-
-    // Use delete if the key may be overwritten and a single deletion
-    // otherwise.
-    Status s;
-    if (shared->AllowsOverwrite(rand_key)) {
-      shared->Delete(rand_column_family, rand_key, true /* pending */);
-      if (!FLAGS_use_txn) {
-        s = db_->Delete(write_opts, cfh, key);
-      } else {
-#ifndef ROCKSDB_LITE
-        Transaction* txn;
-        s = NewTxn(write_opts, &txn);
-        if (s.ok()) {
-          s = txn->Delete(cfh, key);
-          if (s.ok()) {
-            s = CommitTxn(txn);
-          }
-        }
-#endif
-      }
-      shared->Delete(rand_column_family, rand_key, false /* pending */);
-      thread->stats.AddDeletes(1);
-      if (!s.ok()) {
-        fprintf(stderr, "delete error: %s\n", s.ToString().c_str());
-        std::terminate();
-      }
-    } else {
-      shared->SingleDelete(rand_column_family, rand_key, true /* pending */);
-      if (!FLAGS_use_txn) {
-        s = db_->SingleDelete(write_opts, cfh, key);
-      } else {
-#ifndef ROCKSDB_LITE
-        Transaction* txn;
-        s = NewTxn(write_opts, &txn);
-        if (s.ok()) {
-          s = txn->SingleDelete(cfh, key);
-          if (s.ok()) {
-            s = CommitTxn(txn);
-          }
-        }
-#endif
-      }
-      shared->SingleDelete(rand_column_family, rand_key, false /* pending */);
-      thread->stats.AddSingleDeletes(1);
-      if (!s.ok()) {
-        fprintf(stderr, "single delete error: %s\n", s.ToString().c_str());
-        std::terminate();
-      }
-    }
-    return s;
-  }
-
-  Status TestDeleteRange(ThreadState* thread, WriteOptions& write_opts,
-                         const std::vector<int>& rand_column_families,
-                         const std::vector<int64_t>& rand_keys,
-                         std::unique_ptr<MutexLock>& lock) override {
-    // OPERATION delete range
-    std::vector<std::unique_ptr<MutexLock>> range_locks;
-    // delete range does not respect disallowed overwrites. the keys for
-    // which overwrites are disallowed are randomly distributed so it
-    // could be expensive to find a range where each key allows
-    // overwrites.
-    int64_t rand_key = rand_keys[0];
-    int rand_column_family = rand_column_families[0];
-    auto shared = thread->shared;
-    int64_t max_key = shared->GetMaxKey();
-    if (rand_key > max_key - FLAGS_range_deletion_width) {
-      lock.reset();
-      rand_key =
-          thread->rand.Next() % (max_key - FLAGS_range_deletion_width + 1);
-      range_locks.emplace_back(
-          new MutexLock(shared->GetMutexForKey(rand_column_family, rand_key)));
-    } else {
-      range_locks.emplace_back(std::move(lock));
-    }
-    for (int j = 1; j < FLAGS_range_deletion_width; ++j) {
-      if (((rand_key + j) & ((1 << FLAGS_log2_keys_per_lock) - 1)) == 0) {
-        range_locks.emplace_back(new MutexLock(
-            shared->GetMutexForKey(rand_column_family, rand_key + j)));
-      }
-    }
-    shared->DeleteRange(rand_column_family, rand_key,
-                        rand_key + FLAGS_range_deletion_width,
-                        true /* pending */);
-
-    std::string keystr = Key(rand_key);
-    Slice key = keystr;
-    auto cfh = column_families_[rand_column_family];
-    std::string end_keystr = Key(rand_key + FLAGS_range_deletion_width);
-    Slice end_key = end_keystr;
-    Status s = db_->DeleteRange(write_opts, cfh, key, end_key);
-    if (!s.ok()) {
-      fprintf(stderr, "delete range error: %s\n", s.ToString().c_str());
-      std::terminate();
-    }
-    int covered = shared->DeleteRange(rand_column_family, rand_key,
-                                      rand_key + FLAGS_range_deletion_width,
-                                      false /* pending */);
-    thread->stats.AddRangeDeletions(1);
-    thread->stats.AddCoveredByRangeDeletions(covered);
-    return s;
-  }
-
-#ifdef ROCKSDB_LITE
-  void TestIngestExternalFile(
-      ThreadState* /* thread */,
-      const std::vector<int>& /* rand_column_families */,
-      const std::vector<int64_t>& /* rand_keys */,
-      std::unique_ptr<MutexLock>& /* lock */) override {
-    assert(false);
-    fprintf(stderr,
-            "RocksDB lite does not support "
-            "TestIngestExternalFile\n");
-    std::terminate();
-  }
-#else
-  void TestIngestExternalFile(ThreadState* thread,
-                              const std::vector<int>& rand_column_families,
-                              const std::vector<int64_t>& rand_keys,
-                              std::unique_ptr<MutexLock>& lock) override {
-    const std::string sst_filename =
-        FLAGS_db + "/." + ToString(thread->tid) + ".sst";
-    Status s;
-    if (FLAGS_env->FileExists(sst_filename).ok()) {
-      // Maybe we terminated abnormally before, so cleanup to give this file
-      // ingestion a clean slate
-      s = FLAGS_env->DeleteFile(sst_filename);
-    }
-
-    SstFileWriter sst_file_writer(EnvOptions(options_), options_);
-    if (s.ok()) {
-      s = sst_file_writer.Open(sst_filename);
-    }
-    int64_t key_base = rand_keys[0];
-    int column_family = rand_column_families[0];
-    std::vector<std::unique_ptr<MutexLock>> range_locks;
-    std::vector<uint32_t> values;
-    SharedState* shared = thread->shared;
-
-    // Grab locks, set pending state on expected values, and add keys
-    for (int64_t key = key_base;
-         s.ok() && key < std::min(key_base + FLAGS_ingest_external_file_width,
-                                  shared->GetMaxKey());
-         ++key) {
-      if (key == key_base) {
-        range_locks.emplace_back(std::move(lock));
-      } else if ((key & ((1 << FLAGS_log2_keys_per_lock) - 1)) == 0) {
-        range_locks.emplace_back(
-            new MutexLock(shared->GetMutexForKey(column_family, key)));
-      }
-
-      uint32_t value_base = thread->rand.Next() % shared->UNKNOWN_SENTINEL;
-      values.push_back(value_base);
-      shared->Put(column_family, key, value_base, true /* pending */);
-
-      char value[100];
-      size_t value_len = GenerateValue(value_base, value, sizeof(value));
-      auto key_str = Key(key);
-      s = sst_file_writer.Put(Slice(key_str), Slice(value, value_len));
-    }
-
-    if (s.ok()) {
-      s = sst_file_writer.Finish();
-    }
-    if (s.ok()) {
-      s = db_->IngestExternalFile(column_families_[column_family],
-                                  {sst_filename}, IngestExternalFileOptions());
-    }
-    if (!s.ok()) {
-      fprintf(stderr, "file ingestion error: %s\n", s.ToString().c_str());
-      std::terminate();
-    }
-    int64_t key = key_base;
-    for (int32_t value : values) {
-      shared->Put(column_family, key, value, false /* pending */);
-      ++key;
-    }
-  }
-#endif  // ROCKSDB_LITE
-
-  bool VerifyValue(int cf, int64_t key, const ReadOptions& /*opts*/,
-                   SharedState* shared, const std::string& value_from_db,
-                   Status s, bool strict = false) const {
-    if (shared->HasVerificationFailedYet()) {
-      return false;
-    }
-    // compare value_from_db with the value in the shared state
-    char value[kValueMaxLen];
-    uint32_t value_base = shared->Get(cf, key);
-    if (value_base == SharedState::UNKNOWN_SENTINEL) {
-      return true;
-    }
-    if (value_base == SharedState::DELETION_SENTINEL && !strict) {
-      return true;
-    }
-
-    if (s.ok()) {
-      if (value_base == SharedState::DELETION_SENTINEL) {
-        VerificationAbort(shared, "Unexpected value found", cf, key);
-        return false;
-      }
-      size_t sz = GenerateValue(value_base, value, sizeof(value));
-      if (value_from_db.length() != sz) {
-        VerificationAbort(shared, "Length of value read is not equal", cf, key);
-        return false;
-      }
-      if (memcmp(value_from_db.data(), value, sz) != 0) {
-        VerificationAbort(shared, "Contents of value read don't match", cf,
-                          key);
-        return false;
-      }
-    } else {
-      if (value_base != SharedState::DELETION_SENTINEL) {
-        VerificationAbort(shared, "Value not found: " + s.ToString(), cf, key);
-        return false;
-      }
-    }
-    return true;
-  }
-};
-
-StressTest* CreateNonBatchedOpsStressTest() {
-  return new NonBatchedOpsStressTest();
-}
-
-}  // namespace rocksdb
-#endif  // GFLAGS

env/env_posix.cc

@@ -16,9 +16,6 @@
 #if defined(OS_LINUX)
 #include <linux/fs.h>
 #endif
-#if defined(ROCKSDB_IOURING_PRESENT)
-#include <liburing.h>
-#endif
 #include <pthread.h>
 #include <signal.h>
 #include <stdio.h>
@@ -35,9 +32,6 @@
 #include <sys/statvfs.h>
 #include <sys/time.h>
 #include <sys/types.h>
-#if defined(ROCKSDB_IOURING_PRESENT)
-#include <sys/uio.h>
-#endif
 #include <time.h>
 #include <algorithm>
 // Get nano time includes
@@ -292,12 +286,7 @@ class PosixEnv : public Env {
         }
 #endif
       }
-      result->reset(new PosixRandomAccessFile(fname, fd, options
-#if defined(ROCKSDB_IOURING_PRESENT)
-                                              ,
-                                              thread_local_io_urings_.get()
-#endif
-                                                  ));
+      result->reset(new PosixRandomAccessFile(fname, fd, options));
     }
     return s;
   }
@@ -1116,11 +1105,6 @@ class PosixEnv : public Env {
 #endif
   }
 
-#if defined(ROCKSDB_IOURING_PRESENT)
-  // io_uring instance
-  std::unique_ptr<ThreadLocalPtr> thread_local_io_urings_;
-#endif
-
   size_t page_size_;
 
   std::vector<ThreadPoolImpl> thread_pools_;
@@ -1145,17 +1129,6 @@ PosixEnv::PosixEnv()
     thread_pools_[pool_id].SetHostEnv(this);
   }
   thread_status_updater_ = CreateThreadStatusUpdater();
-
-#if defined(ROCKSDB_IOURING_PRESENT)
-  // Test whether IOUring is supported, and if it does, create a managing
-  // object for thread local point so that in the future thread-local
-  // io_uring can be created.
-  struct io_uring* new_io_uring = CreateIOUring();
-  if (new_io_uring != nullptr) {
-    thread_local_io_urings_.reset(new ThreadLocalPtr(DeleteIOUring));
-    delete new_io_uring;
-  }
-#endif
 }
 
 void PosixEnv::Schedule(void (*function)(void* arg1), void* arg, Priority pri,

env/io_posix.cc

@@ -34,7 +34,6 @@
 #include "port/port.h"
 #include "rocksdb/slice.h"
 #include "test_util/sync_point.h"
-#include "util/autovector.h"
 #include "util/coding.h"
 #include "util/string_util.h"
 
@@ -410,22 +409,12 @@ size_t PosixHelper::GetUniqueIdFromFile(int fd, char* id, size_t max_size) {
  *
  * pread() based random-access
  */
-PosixRandomAccessFile::PosixRandomAccessFile(
-    const std::string& fname, int fd, const EnvOptions& options
-#if defined(ROCKSDB_IOURING_PRESENT)
-    ,
-    ThreadLocalPtr* thread_local_io_urings
-#endif
-    )
+PosixRandomAccessFile::PosixRandomAccessFile(const std::string& fname, int fd,
+                                             const EnvOptions& options)
     : filename_(fname),
       fd_(fd),
       use_direct_io_(options.use_direct_reads),
-      logical_sector_size_(GetLogicalBufferSize(fd_))
-#if defined(ROCKSDB_IOURING_PRESENT)
-      ,
-      thread_local_io_urings_(thread_local_io_urings)
-#endif
-{
+      logical_sector_size_(GetLogicalBufferSize(fd_)) {
   assert(!options.use_direct_reads || !options.use_mmap_reads);
   assert(!options.use_mmap_reads || sizeof(void*) < 8);
 }
@@ -471,95 +460,6 @@ Status PosixRandomAccessFile::Read(uint64_t offset, size_t n, Slice* result,
   return s;
 }
 
-Status PosixRandomAccessFile::MultiRead(ReadRequest* reqs, size_t num_reqs) {
-#if defined(ROCKSDB_IOURING_PRESENT)
-  size_t reqs_off;
-  ssize_t ret __attribute__((__unused__));
-
-  struct io_uring* iu = nullptr;
-  if (thread_local_io_urings_) {
-    iu = static_cast<struct io_uring*>(thread_local_io_urings_->Get());
-    if (iu == nullptr) {
-      iu = CreateIOUring();
-      if (iu != nullptr) {
-        thread_local_io_urings_->Reset(iu);
-      }
-    }
-  }
-
-  // Init failed, platform doesn't support io_uring. Fall back to
-  // serialized reads
-  if (iu == nullptr) {
-    return RandomAccessFile::MultiRead(reqs, num_reqs);
-  }
-
-  struct WrappedReadRequest {
-    ReadRequest* req;
-    struct iovec iov;
-    explicit WrappedReadRequest(ReadRequest* r) : req(r) {}
-  };
-
-  autovector<WrappedReadRequest, 32> req_wraps;
-
-  for (size_t i = 0; i < num_reqs; i++) {
-    req_wraps.emplace_back(&reqs[i]);
-  }
-
-  reqs_off = 0;
-  while (num_reqs) {
-    size_t this_reqs = num_reqs;
-
-    // If requests exceed depth, split it into batches
-    if (this_reqs > kIoUringDepth) this_reqs = kIoUringDepth;
-
-    for (size_t i = 0; i < this_reqs; i++) {
-      size_t index = i + reqs_off;
-      struct io_uring_sqe* sqe;
-
-      sqe = io_uring_get_sqe(iu);
-      req_wraps[index].iov.iov_base = reqs[index].scratch;
-      req_wraps[index].iov.iov_len = reqs[index].len;
-      io_uring_prep_readv(sqe, fd_, &req_wraps[index].iov, 1,
-                          reqs[index].offset);
-      io_uring_sqe_set_data(sqe, &req_wraps[index]);
-    }
-
-    ret = io_uring_submit_and_wait(iu, static_cast<unsigned int>(this_reqs));
-    if (static_cast<size_t>(ret) != this_reqs) {
-      fprintf(stderr, "ret = %ld this_reqs: %ld\n", (long)ret, (long)this_reqs);
-    }
-    assert(static_cast<size_t>(ret) == this_reqs);
-
-    for (size_t i = 0; i < this_reqs; i++) {
-      struct io_uring_cqe* cqe;
-      WrappedReadRequest* req_wrap;
-
-      // We could use the peek variant here, but this seems safer in terms
-      // of our initial wait not reaping all completions
-      ret = io_uring_wait_cqe(iu, &cqe);
-      assert(!ret);
-      req_wrap = static_cast<WrappedReadRequest*>(io_uring_cqe_get_data(cqe));
-      ReadRequest* req = req_wrap->req;
-      if (static_cast<size_t>(cqe->res) == req_wrap->iov.iov_len) {
-        req->result = Slice(req->scratch, cqe->res);
-        req->status = Status::OK();
-      } else if (cqe->res >= 0) {
-        req->result = Slice(req->scratch, req_wrap->iov.iov_len - cqe->res);
-      } else {
-        req->result = Slice(req->scratch, 0);
-        req->status = IOError("Req failed", filename_, cqe->res);
-      }
-      io_uring_cqe_seen(iu, cqe);
-    }
-    num_reqs -= this_reqs;
-    reqs_off += this_reqs;
-  }
-  return Status::OK();
-#else
-  return RandomAccessFile::MultiRead(reqs, num_reqs);
-#endif
-}
-
 Status PosixRandomAccessFile::Prefetch(uint64_t offset, size_t n) {
   Status s;
   if (!use_direct_io()) {

env/io_posix.h

@@ -8,15 +8,10 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #pragma once
 #include <errno.h>
-#if defined(ROCKSDB_IOURING_PRESENT)
-#include <liburing.h>
-#include <sys/uio.h>
-#endif
 #include <unistd.h>
 #include <atomic>
 #include <string>
 #include "rocksdb/env.h"
-#include "util/thread_local.h"
 
 // For non linux platform, the following macros are used only as place
 // holder.
@@ -84,51 +79,21 @@ class PosixSequentialFile : public SequentialFile {
   }
 };
 
-#if defined(ROCKSDB_IOURING_PRESENT)
-// io_uring instance queue depth
-const unsigned int kIoUringDepth = 256;
-
-inline void DeleteIOUring(void* p) {
-  struct io_uring* iu = static_cast<struct io_uring*>(p);
-  delete iu;
-}
-
-inline struct io_uring* CreateIOUring() {
-  struct io_uring* new_io_uring = new struct io_uring;
-  int ret = io_uring_queue_init(kIoUringDepth, new_io_uring, 0);
-  if (ret) {
-    delete new_io_uring;
-    new_io_uring = nullptr;
-  }
-  return new_io_uring;
-}
-#endif  // defined(ROCKSDB_IOURING_PRESENT)
-
 class PosixRandomAccessFile : public RandomAccessFile {
  protected:
   std::string filename_;
   int fd_;
   bool use_direct_io_;
   size_t logical_sector_size_;
-#if defined(ROCKSDB_IOURING_PRESENT)
-  ThreadLocalPtr* thread_local_io_urings_;
-#endif
 
  public:
   PosixRandomAccessFile(const std::string& fname, int fd,
-                        const EnvOptions& options
-#if defined(ROCKSDB_IOURING_PRESENT)
-                        ,
-                        ThreadLocalPtr* thread_local_io_urings
-#endif
-  );
+                        const EnvOptions& options);
   virtual ~PosixRandomAccessFile();
 
   virtual Status Read(uint64_t offset, size_t n, Slice* result,
                       char* scratch) const override;
 
-  virtual Status MultiRead(ReadRequest* reqs, size_t num_reqs) override;
-
   virtual Status Prefetch(uint64_t offset, size_t n) override;
 
 #if defined(OS_LINUX) || defined(OS_MACOSX) || defined(OS_AIX)

examples/transaction_example.cc

@@ -94,25 +94,14 @@ int main() {
   s = txn_db->Put(write_options, "abc", "xyz");
   assert(s.ok());
 
-  // Read the latest committed value.
-  s = txn->Get(read_options, "abc", &value);
-  assert(s.ok());
-  assert(value == "xyz");
-
-  // Read the snapshotted value.
-  read_options.snapshot = snapshot;
-  s = txn->Get(read_options, "abc", &value);
-  assert(s.ok());
-  assert(value == "def");
-
   // Attempt to read a key using the snapshot.  This will fail since
   // the previous write outside this txn conflicts with this read.
+  read_options.snapshot = snapshot;
   s = txn->GetForUpdate(read_options, "abc", &value);
   assert(s.IsBusy());
 
   txn->Rollback();
 
-  // Snapshot will be released upon deleting the transaction.
   delete txn;
   // Clear snapshot from read options since it is no longer valid
   read_options.snapshot = nullptr;
@@ -136,13 +125,10 @@ int main() {
   // Do some reads and writes to key "x"
   read_options.snapshot = txn_db->GetSnapshot();
   s = txn->Get(read_options, "x", &value);
-  assert(s.IsNotFound());
-  s = txn->Put("x", "x");
-  assert(s.ok());
+  txn->Put("x", "x");
 
   // Do a write outside of the transaction to key "y"
-  s = txn_db->Put(write_options, "y", "y1");
-  assert(s.ok());
+  s = txn_db->Put(write_options, "y", "y");
 
   // Set a new snapshot in the transaction
   txn->SetSnapshot();
@@ -153,10 +139,7 @@ int main() {
   // Since the snapshot was advanced, the write done outside of the
   // transaction does not conflict.
   s = txn->GetForUpdate(read_options, "y", &value);
-  assert(s.ok());
-  assert(value == "y1");
-  s = txn->Put("y", "y2");
-  assert(s.ok());
+  txn->Put("y", "y");
 
   // Decide we want to revert the last write from this transaction.
   txn->RollbackToSavePoint();
@@ -168,15 +151,6 @@ int main() {
   // Clear snapshot from read options since it is no longer valid
   read_options.snapshot = nullptr;
 
-  // db state is at the save point.
-  s = txn_db->Get(read_options, "x", &value);
-  assert(s.ok());
-  assert(value == "x");
-
-  s = txn_db->Get(read_options, "y", &value);
-  assert(s.ok());
-  assert(value == "y1");
-
   // Cleanup
   delete txn_db;
   DestroyDB(kDBPath, options);

file/delete_scheduler.cc

@@ -73,7 +73,8 @@ Status DeleteScheduler::DeleteFile(const std::string& file_path,
   s = MarkAsTrash(file_path, &trash_file);
 
   if (!s.ok()) {
-    ROCKS_LOG_ERROR(info_log_, "Failed to mark %s as trash", file_path.c_str());
+    ROCKS_LOG_ERROR(info_log_, "Failed to mark %s as trash -- %s",
+                    file_path.c_str(), s.ToString().c_str());
     s = env_->DeleteFile(file_path);
     if (s.ok()) {
       sst_file_manager_->OnDeleteFile(file_path);

file/sst_file_manager_impl.cc

@@ -306,6 +306,7 @@ void SstFileManagerImpl::ClearError() {
       // since the ErrorHandler::recovery_in_prog_ flag would be true
       cur_instance_ = error_handler;
       mu_.Unlock();
+      TEST_SYNC_POINT("SstFileManagerImpl::ClearError");
       s = error_handler->RecoverFromBGError();
       mu_.Lock();
       // The DB instance might have been deleted while we were

include/rocksdb/c.h

@@ -706,14 +706,6 @@ enum {
 };
 extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_index_type(
     rocksdb_block_based_table_options_t*, int);  // uses one of the above enums
-enum {
-  rocksdb_block_based_table_data_block_index_type_binary_search = 0,
-  rocksdb_block_based_table_data_block_index_type_binary_search_and_hash = 1,
-};
-extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_data_block_index_type(
-    rocksdb_block_based_table_options_t*, int);  // uses one of the above enums
-extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_data_block_hash_ratio(
-    rocksdb_block_based_table_options_t* options, double v);
 extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_hash_index_allow_collision(
     rocksdb_block_based_table_options_t*, unsigned char);

include/rocksdb/cache.h

@@ -255,7 +255,7 @@ class Cache {
   // Always delete the DB object before calling this method!
   virtual void DisownData(){
       // default implementation is noop
-  };
+  }
 
   // Apply callback to all entries in the cache
   // If thread_safe is true, it will also lock the accesses. Otherwise, it will

include/rocksdb/env.h

@@ -672,7 +672,7 @@ class RandomAccessFile {
   virtual size_t GetUniqueId(char* /*id*/, size_t /*max_size*/) const {
     return 0;  // Default implementation to prevent issues with backwards
                // compatibility.
-  };
+  }
 
   enum AccessPattern { NORMAL, RANDOM, SEQUENTIAL, WILLNEED, DONTNEED };
 
@@ -1414,7 +1414,7 @@ class RandomAccessFileWrapper : public RandomAccessFile {
   }
   size_t GetUniqueId(char* id, size_t max_size) const override {
     return target_->GetUniqueId(id, max_size);
-  };
+  }
   void Hint(AccessPattern pattern) override { target_->Hint(pattern); }
   bool use_direct_io() const override { return target_->use_direct_io(); }
   size_t GetRequiredBufferAlignment() const override {

include/rocksdb/options.h

@@ -1079,10 +1079,10 @@ struct DBOptions {
   // independently if the process crashes later and tries to recover.
   bool atomic_flush = false;
 
-  // If true, ColumnFamilyHandle's and Iterator's destructors won't delete
-  // obsolete files directly and will instead schedule a background job
-  // to do it. Use it if you're destroying iterators or ColumnFamilyHandle-s
-  // from latency-sensitive threads.
+  // If true, working thread may avoid doing unnecessary and long-latency
+  // operation (such as deleting obsolete files directly or deleting memtable)
+  // and will instead schedule a background job to do it.
+  // Use it if you're latency-sensitive.
   // If set to true, takes precedence over
   // ReadOptions::background_purge_on_iterator_cleanup.
   bool avoid_unnecessary_blocking_io = false;
@@ -1181,7 +1181,7 @@ struct ReadOptions {
   // "iterate_upper_bound" defines the extent upto which the forward iterator
   // can returns entries. Once the bound is reached, Valid() will be false.
   // "iterate_upper_bound" is exclusive ie the bound value is
-  // not a valid entry. If prefix_extractor is not null, the Seek target
+  // not a valid entry.  If iterator_extractor is not null, the Seek target
   // and iterate_upper_bound need to have the same prefix.
   // This is because ordering is not guaranteed outside of prefix domain.
   //

include/rocksdb/slice.h

@@ -195,8 +195,15 @@ class PinnableSlice : public Slice, public Cleanable {
     }
   }
 
-  void remove_prefix(size_t /*n*/) {
-    assert(0);  // Not implemented
+  void remove_prefix(size_t n) {
+    assert(n <= size());
+    if (pinned_) {
+      data_ += n;
+      size_ -= n;
+    } else {
+      buf_->erase(0, n);
+      PinSelf();
+    }
   }
 
   void Reset() {

include/rocksdb/statistics.h

@@ -523,7 +523,7 @@ class Statistics {
   virtual bool getTickerMap(std::map<std::string, uint64_t>*) const {
     // Do nothing by default
     return false;
-  };
+  }
 
   // Override this function to disable particular histogram collection
   virtual bool HistEnabledForType(uint32_t type) const {

include/rocksdb/version.h

@@ -6,7 +6,7 @@
 
 #define ROCKSDB_MAJOR 6
 #define ROCKSDB_MINOR 6
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 4
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

java/CMakeLists.txt

@@ -312,17 +312,19 @@ if(NOT EXISTS ${JAVA_TEST_LIBDIR})
   file(MAKE_DIRECTORY mkdir ${JAVA_TEST_LIBDIR})
 endif()
 
-if (DEFINED CUSTOM_REPO_URL)
-  set(SEARCH_REPO_URL ${CUSTOM_REPO_URL}/)
-  set(CENTRAL_REPO_URL ${CUSTOM_REPO_URL}/)
+if (DEFINED CUSTOM_DEPS_URL)
+  set(DEPS_URL ${CUSTOM_DEPS_URL}/)
 else ()
-  set(SEARCH_REPO_URL "http://search.maven.org/remotecontent?filepath=")
-  set(CENTRAL_REPO_URL "http://central.maven.org/maven2/")
+  # This is a URL for artifacts from a "fake" release on pdillinger's fork,
+  # so as not to put binaries in git (ew). We should move to hosting these
+  # under the facebook account on github, or something else more reliable
+  # than maven.org, which has been failing frequently from Travis.
+  set(DEPS_URL "https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps")
 endif()
 
 if(NOT EXISTS ${JAVA_JUNIT_JAR})
   message("Downloading ${JAVA_JUNIT_JAR}")
-  file(DOWNLOAD ${SEARCH_REPO_URL}junit/junit/4.12/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${DEPS_URL}/junit-4.12.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_JUNIT_JAR}")
@@ -331,7 +333,7 @@ if(NOT EXISTS ${JAVA_JUNIT_JAR})
 endif()
 if(NOT EXISTS ${JAVA_HAMCR_JAR})
   message("Downloading ${JAVA_HAMCR_JAR}")
-  file(DOWNLOAD ${SEARCH_REPO_URL}org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${DEPS_URL}/hamcrest-core-1.3.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_HAMCR_JAR}")
@@ -340,7 +342,7 @@ if(NOT EXISTS ${JAVA_HAMCR_JAR})
 endif()
 if(NOT EXISTS ${JAVA_MOCKITO_JAR})
   message("Downloading ${JAVA_MOCKITO_JAR}")
-  file(DOWNLOAD ${SEARCH_REPO_URL}org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${DEPS_URL}/mockito-all-1.10.19.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_MOCKITO_JAR}")
@@ -349,7 +351,7 @@ if(NOT EXISTS ${JAVA_MOCKITO_JAR})
 endif()
 if(NOT EXISTS ${JAVA_CGLIB_JAR})
   message("Downloading ${JAVA_CGLIB_JAR}")
-  file(DOWNLOAD ${SEARCH_REPO_URL}cglib/cglib/2.2.2/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${DEPS_URL}/cglib-2.2.2.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_CGLIB_JAR}")
@@ -358,7 +360,7 @@ if(NOT EXISTS ${JAVA_CGLIB_JAR})
 endif()
 if(NOT EXISTS ${JAVA_ASSERTJ_JAR})
   message("Downloading ${JAVA_ASSERTJ_JAR}")
-  file(DOWNLOAD ${CENTRAL_REPO_URL}org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
+  file(DOWNLOAD ${DEPS_URL}/assertj-core-1.7.1.jar ${JAVA_TMP_JAR} STATUS downloadStatus)
   list(GET downloadStatus 0 error_code)
   if(NOT error_code EQUAL 0)
     message(FATAL_ERROR "Failed downloading ${JAVA_ASSERTJ_JAR}")
@@ -484,13 +486,13 @@ endif()
 set(ROCKSDBJNI_STATIC_LIB rocksdbjni${ARTIFACT_SUFFIX})
 add_library(${ROCKSDBJNI_STATIC_LIB} ${JNI_NATIVE_SOURCES})
 add_dependencies(${ROCKSDBJNI_STATIC_LIB} rocksdbjni_headers)
-target_link_libraries(${ROCKSDBJNI_STATIC_LIB} ${ROCKSDB_STATIC_LIB} ${ROCKSDB_LIB})
+target_link_libraries(${ROCKSDBJNI_STATIC_LIB} ${ROCKSDB_STATIC_LIB} ${LIBS})
 
 if(NOT MINGW)
   set(ROCKSDBJNI_SHARED_LIB rocksdbjni-shared${ARTIFACT_SUFFIX})
   add_library(${ROCKSDBJNI_SHARED_LIB} SHARED ${JNI_NATIVE_SOURCES})
   add_dependencies(${ROCKSDBJNI_SHARED_LIB} rocksdbjni_headers)
-  target_link_libraries(${ROCKSDBJNI_SHARED_LIB} ${ROCKSDB_STATIC_LIB} ${ROCKSDB_LIB})
+  target_link_libraries(${ROCKSDBJNI_SHARED_LIB} ${ROCKSDB_STATIC_LIB} ${LIBS})
 
   set_target_properties(
     ${ROCKSDBJNI_SHARED_LIB}

java/Makefile

@@ -213,16 +213,23 @@ ifneq ($(DEBUG_LEVEL),0)
 	JAVAC_ARGS = -Xlint:deprecation -Xlint:unchecked
 endif
 
-SEARCH_REPO_URL?=http://search.maven.org/remotecontent?filepath=
-CENTRAL_REPO_URL?=http://central.maven.org/maven2/
+# This is a URL for artifacts from a "fake" release on pdillinger's fork,
+# so as not to put binaries in git (ew). We should move to hosting these
+# under the facebook account on github, or something else more reliable
+# than maven.org, which has been failing frequently from Travis.
+DEPS_URL?=https://github.com/pdillinger/rocksdb/releases/download/v6.6.x-java-deps
 
-clean:
-	$(AM_V_at)rm -rf include/*
-	$(AM_V_at)rm -rf test-libs/
+clean: clean-not-downloaded clean-downloaded
+
+clean-not-downloaded:
+	$(AM_V_at)rm -rf $(NATIVE_INCLUDE)
 	$(AM_V_at)rm -rf $(OUTPUT)
 	$(AM_V_at)rm -rf $(BENCHMARK_OUTPUT)
 	$(AM_V_at)rm -rf $(SAMPLES_OUTPUT)
 
+clean-downloaded:
+	$(AM_V_at)rm -rf $(JAVA_TEST_LIBDIR)
+
 
 javadocs: java
 	$(AM_V_GEN)mkdir -p $(JAVADOC)
@@ -279,11 +286,11 @@ optimistic_transaction_sample: java
 
 resolve_test_deps:
 	test -d "$(JAVA_TEST_LIBDIR)" || mkdir -p "$(JAVA_TEST_LIBDIR)"
-	test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_JUNIT_JAR) $(SEARCH_REPO_URL)junit/junit/4.12/junit-4.12.jar
-	test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o $(JAVA_HAMCR_JAR) $(SEARCH_REPO_URL)org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar
-	test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_MOCKITO_JAR)" $(SEARCH_REPO_URL)org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar
-	test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_CGLIB_JAR)" $(SEARCH_REPO_URL)cglib/cglib/2.2.2/cglib-2.2.2.jar
-	test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" $(CENTRAL_REPO_URL)org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar
+	test -s "$(JAVA_JUNIT_JAR)" || cp $(MVN_LOCAL)/junit/junit/4.12/junit-4.12.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output $(JAVA_JUNIT_JAR) --location $(DEPS_URL)/junit-4.12.jar
+	test -s "$(JAVA_HAMCR_JAR)" || cp $(MVN_LOCAL)/org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output $(JAVA_HAMCR_JAR) --location $(DEPS_URL)/hamcrest-core-1.3.jar
+	test -s "$(JAVA_MOCKITO_JAR)" || cp $(MVN_LOCAL)/org/mockito/mockito-all/1.10.19/mockito-all-1.10.19.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_MOCKITO_JAR)" --location $(DEPS_URL)/mockito-all-1.10.19.jar
+	test -s "$(JAVA_CGLIB_JAR)" || cp $(MVN_LOCAL)/cglib/cglib/2.2.2/cglib-2.2.2.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_CGLIB_JAR)" --location $(DEPS_URL)/cglib-2.2.2.jar
+	test -s "$(JAVA_ASSERTJ_JAR)" || cp $(MVN_LOCAL)/org/assertj/assertj-core/1.7.1/assertj-core-1.7.1.jar $(JAVA_TEST_LIBDIR) || curl --fail --insecure --output "$(JAVA_ASSERTJ_JAR)" --location $(DEPS_URL)/assertj-core-1.7.1.jar
 
 java_test: java resolve_test_deps
 	$(AM_V_GEN)mkdir -p $(TEST_CLASSES)

java/rocksjni/options.cc

@@ -1247,9 +1247,51 @@ jint Java_org_rocksdb_Options_statsDumpPeriodSec(
  */
 void Java_org_rocksdb_Options_setStatsDumpPeriodSec(
     JNIEnv*, jobject, jlong jhandle,
-    jint stats_dump_period_sec) {
+    jint jstats_dump_period_sec) {
   reinterpret_cast<rocksdb::Options*>(jhandle)->stats_dump_period_sec =
-      static_cast<int>(stats_dump_period_sec);
+      static_cast<unsigned int>(jstats_dump_period_sec);
+}
+
+/*
+ * Class:     org_rocksdb_Options
+ * Method:    statsPersistPeriodSec
+ * Signature: (J)I
+ */
+jint Java_org_rocksdb_Options_statsPersistPeriodSec(
+    JNIEnv*, jobject, jlong jhandle) {
+  return reinterpret_cast<rocksdb::Options*>(jhandle)->stats_persist_period_sec;
+}
+
+/*
+ * Class:     org_rocksdb_Options
+ * Method:    setStatsPersistPeriodSec
+ * Signature: (JI)V
+ */
+void Java_org_rocksdb_Options_setStatsPersistPeriodSec(
+    JNIEnv*, jobject, jlong jhandle, jint jstats_persist_period_sec) {
+  reinterpret_cast<rocksdb::Options*>(jhandle)->stats_persist_period_sec =
+      static_cast<unsigned int>(jstats_persist_period_sec);
+}
+
+/*
+ * Class:     org_rocksdb_Options
+ * Method:    statsHistoryBufferSize
+ * Signature: (J)J
+ */
+jlong Java_org_rocksdb_Options_statsHistoryBufferSize(
+    JNIEnv*, jobject, jlong jhandle) {
+  return reinterpret_cast<rocksdb::Options*>(jhandle)->stats_history_buffer_size;
+}
+
+/*
+ * Class:     org_rocksdb_Options
+ * Method:    setStatsHistoryBufferSize
+ * Signature: (JJ)V
+ */
+void Java_org_rocksdb_Options_setStatsHistoryBufferSize(
+    JNIEnv*, jobject, jlong jhandle, jlong jstats_history_buffer_size) {
+  reinterpret_cast<rocksdb::Options*>(jhandle)->stats_history_buffer_size =
+      static_cast<size_t>(jstats_history_buffer_size);
 }
 
 /*
@@ -1481,6 +1523,28 @@ jlong Java_org_rocksdb_Options_walBytesPerSync(
   return static_cast<jlong>(opt->wal_bytes_per_sync);
 }
 
+/*
+ * Class:     org_rocksdb_Options
+ * Method:    setStrictBytesPerSync
+ * Signature: (JZ)V
+ */
+void Java_org_rocksdb_Options_setStrictBytesPerSync(
+    JNIEnv*, jobject, jlong jhandle, jboolean jstrict_bytes_per_sync) {
+  reinterpret_cast<rocksdb::Options*>(jhandle)->strict_bytes_per_sync =
+      jstrict_bytes_per_sync == JNI_TRUE;
+}
+
+/*
+ * Class:     org_rocksdb_Options
+ * Method:    strictBytesPerSync
+ * Signature: (J)Z
+ */
+jboolean Java_org_rocksdb_Options_strictBytesPerSync(
+    JNIEnv*, jobject, jlong jhandle) {
+  auto* opt = reinterpret_cast<rocksdb::Options*>(jhandle);
+  return static_cast<jboolean>(opt->strict_bytes_per_sync);
+}
+
 /*
  * Class:     org_rocksdb_Options
  * Method:    setEnableThreadTracking
@@ -5441,9 +5505,9 @@ jboolean Java_org_rocksdb_DBOptions_isFdCloseOnExec(
  * Signature: (JI)V
  */
 void Java_org_rocksdb_DBOptions_setStatsDumpPeriodSec(
-    JNIEnv*, jobject, jlong jhandle, jint stats_dump_period_sec) {
+    JNIEnv*, jobject, jlong jhandle, jint jstats_dump_period_sec) {
   reinterpret_cast<rocksdb::DBOptions*>(jhandle)->stats_dump_period_sec =
-      static_cast<int>(stats_dump_period_sec);
+      static_cast<unsigned int>(jstats_dump_period_sec);
 }
 
 /*
@@ -5456,6 +5520,48 @@ jint Java_org_rocksdb_DBOptions_statsDumpPeriodSec(
   return reinterpret_cast<rocksdb::DBOptions*>(jhandle)->stats_dump_period_sec;
 }
 
+/*
+ * Class:     org_rocksdb_DBOptions
+ * Method:    setStatsPersistPeriodSec
+ * Signature: (JI)V
+ */
+void Java_org_rocksdb_DBOptions_setStatsPersistPeriodSec(
+    JNIEnv*, jobject, jlong jhandle, jint jstats_persist_period_sec) {
+  reinterpret_cast<rocksdb::DBOptions*>(jhandle)->stats_persist_period_sec =
+      static_cast<unsigned int>(jstats_persist_period_sec);
+}
+
+/*
+ * Class:     org_rocksdb_DBOptions
+ * Method:    statsPersistPeriodSec
+ * Signature: (J)I
+ */
+jint Java_org_rocksdb_DBOptions_statsPersistPeriodSec(
+    JNIEnv*, jobject, jlong jhandle) {
+  return reinterpret_cast<rocksdb::DBOptions*>(jhandle)->stats_persist_period_sec;
+}
+
+/*
+ * Class:     org_rocksdb_DBOptions
+ * Method:    setStatsHistoryBufferSize
+ * Signature: (JJ)V
+ */
+void Java_org_rocksdb_DBOptions_setStatsHistoryBufferSize(
+    JNIEnv*, jobject, jlong jhandle, jlong jstats_history_buffer_size) {
+  reinterpret_cast<rocksdb::DBOptions*>(jhandle)->stats_history_buffer_size =
+      static_cast<size_t>(jstats_history_buffer_size);
+}
+
+/*
+ * Class:     org_rocksdb_DBOptions
+ * Method:    statsHistoryBufferSize
+ * Signature: (J)J
+ */
+jlong Java_org_rocksdb_DBOptions_statsHistoryBufferSize(
+    JNIEnv*, jobject, jlong jhandle) {
+  return reinterpret_cast<rocksdb::DBOptions*>(jhandle)->stats_history_buffer_size;
+}
+
 /*
  * Class:     org_rocksdb_DBOptions
  * Method:    setAdviseRandomOnOpen
@@ -5694,6 +5800,28 @@ jlong Java_org_rocksdb_DBOptions_walBytesPerSync(
   return static_cast<jlong>(opt->wal_bytes_per_sync);
 }
 
+/*
+ * Class:     org_rocksdb_DBOptions
+ * Method:    setStrictBytesPerSync
+ * Signature: (JZ)V
+ */
+void Java_org_rocksdb_DBOptions_setStrictBytesPerSync(
+    JNIEnv*, jobject, jlong jhandle, jboolean jstrict_bytes_per_sync) {
+  reinterpret_cast<rocksdb::DBOptions*>(jhandle)->strict_bytes_per_sync =
+      jstrict_bytes_per_sync == JNI_TRUE;
+}
+
+/*
+ * Class:     org_rocksdb_DBOptions
+ * Method:    strictBytesPerSync
+ * Signature: (J)Z
+ */
+jboolean Java_org_rocksdb_DBOptions_strictBytesPerSync(
+    JNIEnv*, jobject, jlong jhandle) {
+  return static_cast<jboolean>(
+      reinterpret_cast<rocksdb::DBOptions*>(jhandle)->strict_bytes_per_sync);
+}
+
 /*
  * Class:     org_rocksdb_DBOptions
  * Method:    setDelayedWriteRate

java/src/main/java/org/rocksdb/DBOptions.java

@@ -660,6 +660,34 @@ public class DBOptions extends RocksObject
     return statsDumpPeriodSec(nativeHandle_);
   }
 
+  @Override
+  public DBOptions setStatsPersistPeriodSec(
+      final int statsPersistPeriodSec) {
+    assert(isOwningHandle());
+    setStatsPersistPeriodSec(nativeHandle_, statsPersistPeriodSec);
+    return this;
+  }
+
+  @Override
+  public int statsPersistPeriodSec() {
+    assert(isOwningHandle());
+    return statsPersistPeriodSec(nativeHandle_);
+  }
+
+  @Override
+  public DBOptions setStatsHistoryBufferSize(
+      final long statsHistoryBufferSize) {
+    assert(isOwningHandle());
+    setStatsHistoryBufferSize(nativeHandle_, statsHistoryBufferSize);
+    return this;
+  }
+
+  @Override
+  public long statsHistoryBufferSize() {
+    assert(isOwningHandle());
+    return statsHistoryBufferSize(nativeHandle_);
+  }
+
   @Override
   public DBOptions setAdviseRandomOnOpen(
       final boolean adviseRandomOnOpen) {
@@ -807,6 +835,19 @@ public class DBOptions extends RocksObject
     return walBytesPerSync(nativeHandle_);
   }
 
+  @Override
+  public DBOptions setStrictBytesPerSync(final boolean strictBytesPerSync) {
+    assert(isOwningHandle());
+    setStrictBytesPerSync(nativeHandle_, strictBytesPerSync);
+    return this;
+  }
+
+  @Override
+  public boolean strictBytesPerSync() {
+    assert(isOwningHandle());
+    return strictBytesPerSync(nativeHandle_);
+  }
+
   //TODO(AR) NOW
 //  @Override
 //  public DBOptions setListeners(final List<EventListener> listeners) {
@@ -1239,6 +1280,14 @@ public class DBOptions extends RocksObject
   private native void setStatsDumpPeriodSec(
       long handle, int statsDumpPeriodSec);
   private native int statsDumpPeriodSec(long handle);
+  private native void setStatsPersistPeriodSec(
+      final long handle, final int statsPersistPeriodSec);
+  private native int statsPersistPeriodSec(
+      final long handle);
+  private native void setStatsHistoryBufferSize(
+      final long handle, final long statsHistoryBufferSize);
+  private native long statsHistoryBufferSize(
+      final long handle);
   private native void setAdviseRandomOnOpen(
       long handle, boolean adviseRandomOnOpen);
   private native boolean adviseRandomOnOpen(long handle);
@@ -1270,6 +1319,10 @@ public class DBOptions extends RocksObject
   private native long bytesPerSync(long handle);
   private native void setWalBytesPerSync(long handle, long walBytesPerSync);
   private native long walBytesPerSync(long handle);
+  private native void setStrictBytesPerSync(
+      final long handle, final boolean strictBytesPerSync);
+  private native boolean strictBytesPerSync(
+      final long handle);
   private native void setEnableThreadTracking(long handle,
       boolean enableThreadTracking);
   private native boolean enableThreadTracking(long handle);

java/src/main/java/org/rocksdb/Env.java

@@ -13,6 +13,10 @@ import java.util.List;
  */
 public abstract class Env extends RocksObject {
 
+  static {
+    RocksDB.loadLibrary();
+  }
+
   private static final Env DEFAULT_ENV = new RocksEnv(getDefaultEnvInternal());
   static {
     /**

java/src/main/java/org/rocksdb/MutableDBOptions.java

@@ -89,9 +89,12 @@ public class MutableDBOptions extends AbstractMutableOptions {
     max_total_wal_size(ValueType.LONG),
     delete_obsolete_files_period_micros(ValueType.LONG),
     stats_dump_period_sec(ValueType.INT),
+    stats_persist_period_sec(ValueType.INT),
+    stats_history_buffer_size(ValueType.LONG),
     max_open_files(ValueType.INT),
     bytes_per_sync(ValueType.LONG),
     wal_bytes_per_sync(ValueType.LONG),
+    strict_bytes_per_sync(ValueType.BOOLEAN),
     compaction_readahead_size(ValueType.LONG);
 
     private final ValueType valueType;
@@ -240,6 +243,28 @@ public class MutableDBOptions extends AbstractMutableOptions {
       return getInt(DBOption.stats_dump_period_sec);
     }
 
+    @Override
+    public MutableDBOptionsBuilder setStatsPersistPeriodSec(
+        final int statsPersistPeriodSec) {
+      return setInt(DBOption.stats_persist_period_sec, statsPersistPeriodSec);
+    }
+
+    @Override
+    public int statsPersistPeriodSec() {
+      return getInt(DBOption.stats_persist_period_sec);
+    }
+
+    @Override
+    public MutableDBOptionsBuilder setStatsHistoryBufferSize(
+        final long statsHistoryBufferSize) {
+      return setLong(DBOption.stats_history_buffer_size, statsHistoryBufferSize);
+    }
+
+    @Override
+    public long statsHistoryBufferSize() {
+      return getLong(DBOption.stats_history_buffer_size);
+    }
+
     @Override
     public MutableDBOptionsBuilder setMaxOpenFiles(final int maxOpenFiles) {
       return setInt(DBOption.max_open_files, maxOpenFiles);
@@ -271,6 +296,17 @@ public class MutableDBOptions extends AbstractMutableOptions {
       return getLong(DBOption.wal_bytes_per_sync);
     }
 
+    @Override
+    public MutableDBOptionsBuilder setStrictBytesPerSync(
+        final boolean strictBytesPerSync) {
+      return setBoolean(DBOption.strict_bytes_per_sync, strictBytesPerSync);
+    }
+
+    @Override
+    public boolean strictBytesPerSync() {
+      return getBoolean(DBOption.strict_bytes_per_sync);
+    }
+
     @Override
     public MutableDBOptionsBuilder setCompactionReadaheadSize(
         final long compactionReadaheadSize) {

java/src/main/java/org/rocksdb/MutableDBOptionsInterface.java

@@ -237,6 +237,44 @@ public interface MutableDBOptionsInterface<T extends MutableDBOptionsInterface<T
    */
   int statsDumpPeriodSec();
 
+  /**
+   * If not zero, dump rocksdb.stats to RocksDB every
+   * {@code statsPersistPeriodSec}
+   *
+   * Default: 600
+   *
+   * @param statsPersistPeriodSec time interval in seconds.
+   * @return the instance of the current object.
+   */
+  T setStatsPersistPeriodSec(int statsPersistPeriodSec);
+
+  /**
+   * If not zero, dump rocksdb.stats to RocksDB every
+   * {@code statsPersistPeriodSec}
+   *
+   * @return time interval in seconds.
+   */
+  int statsPersistPeriodSec();
+
+  /**
+   * If not zero, periodically take stats snapshots and store in memory, the
+   * memory size for stats snapshots is capped at {@code statsHistoryBufferSize}
+   *
+   * Default: 1MB
+   *
+   * @param statsHistoryBufferSize the size of the buffer.
+   * @return the instance of the current object.
+   */
+  T setStatsHistoryBufferSize(long statsHistoryBufferSize);
+
+  /**
+   * If not zero, periodically take stats snapshots and store in memory, the
+   * memory size for stats snapshots is capped at {@code statsHistoryBufferSize}
+   *
+   * @return the size of the buffer.
+   */
+  long statsHistoryBufferSize();
+
   /**
    * Number of open files that can be used by the DB.  You may need to
    * increase this if your database has a large working set. Value -1 means
@@ -303,6 +341,42 @@ public interface MutableDBOptionsInterface<T extends MutableDBOptionsInterface<T
    */
   long walBytesPerSync();
 
+  /**
+   * When true, guarantees WAL files have at most {@link #walBytesPerSync()}
+   * bytes submitted for writeback at any given time, and SST files have at most
+   * {@link #bytesPerSync()} bytes pending writeback at any given time. This
+   * can be used to handle cases where processing speed exceeds I/O speed
+   * during file generation, which can lead to a huge sync when the file is
+   * finished, even with {@link #bytesPerSync()} / {@link #walBytesPerSync()}
+   * properly configured.
+   *
+   * - If `sync_file_range` is supported it achieves this by waiting for any
+   *   prior `sync_file_range`s to finish before proceeding. In this way,
+   *   processing (compression, etc.) can proceed uninhibited in the gap
+   *   between `sync_file_range`s, and we block only when I/O falls
+   *   behind.
+   * - Otherwise the `WritableFile::Sync` method is used. Note this mechanism
+   *   always blocks, thus preventing the interleaving of I/O and processing.
+   *
+   * Note: Enabling this option does not provide any additional persistence
+   * guarantees, as it may use `sync_file_range`, which does not write out
+   * metadata.
+   *
+   * Default: false
+   *
+   * @param strictBytesPerSync the bytes per sync
+   * @return the instance of the current object.
+   */
+  T setStrictBytesPerSync(boolean strictBytesPerSync);
+
+  /**
+   * Return the strict byte limit per sync.
+   *
+   * See {@link #setStrictBytesPerSync(boolean)}
+   *
+   * @return the limit in bytes.
+   */
+  boolean strictBytesPerSync();
 
   /**
    * If non-zero, we perform bigger reads when doing compaction. If you're

java/src/main/java/org/rocksdb/Options.java

@@ -739,6 +739,34 @@ public class Options extends RocksObject
     return this;
   }
 
+  @Override
+  public Options setStatsPersistPeriodSec(
+      final int statsPersistPeriodSec) {
+    assert(isOwningHandle());
+    setStatsPersistPeriodSec(nativeHandle_, statsPersistPeriodSec);
+    return this;
+  }
+
+  @Override
+  public int statsPersistPeriodSec() {
+    assert(isOwningHandle());
+    return statsPersistPeriodSec(nativeHandle_);
+  }
+
+  @Override
+  public Options setStatsHistoryBufferSize(
+      final long statsHistoryBufferSize) {
+    assert(isOwningHandle());
+    setStatsHistoryBufferSize(nativeHandle_, statsHistoryBufferSize);
+    return this;
+  }
+
+  @Override
+  public long statsHistoryBufferSize() {
+    assert(isOwningHandle());
+    return statsHistoryBufferSize(nativeHandle_);
+  }
+
   @Override
   public boolean adviseRandomOnOpen() {
     return adviseRandomOnOpen(nativeHandle_);
@@ -883,6 +911,19 @@ public class Options extends RocksObject
     return walBytesPerSync(nativeHandle_);
   }
 
+  @Override
+  public Options setStrictBytesPerSync(final boolean strictBytesPerSync) {
+    assert(isOwningHandle());
+    setStrictBytesPerSync(nativeHandle_, strictBytesPerSync);
+    return this;
+  }
+
+  @Override
+  public boolean strictBytesPerSync() {
+    assert(isOwningHandle());
+    return strictBytesPerSync(nativeHandle_);
+  }
+
   @Override
   public Options setEnableThreadTracking(final boolean enableThreadTracking) {
     assert(isOwningHandle());
@@ -1858,6 +1899,14 @@ public class Options extends RocksObject
   private native void setStatsDumpPeriodSec(
       long handle, int statsDumpPeriodSec);
   private native int statsDumpPeriodSec(long handle);
+  private native void setStatsPersistPeriodSec(
+      final long handle, final int statsPersistPeriodSec);
+  private native int statsPersistPeriodSec(
+      final long handle);
+  private native void setStatsHistoryBufferSize(
+      final long handle, final long statsHistoryBufferSize);
+  private native long statsHistoryBufferSize(
+      final long handle);
   private native void setAdviseRandomOnOpen(
       long handle, boolean adviseRandomOnOpen);
   private native boolean adviseRandomOnOpen(long handle);
@@ -1889,6 +1938,10 @@ public class Options extends RocksObject
   private native long bytesPerSync(long handle);
   private native void setWalBytesPerSync(long handle, long walBytesPerSync);
   private native long walBytesPerSync(long handle);
+  private native void setStrictBytesPerSync(
+      final long handle, final boolean strictBytesPerSync);
+  private native boolean strictBytesPerSync(
+      final long handle);
   private native void setEnableThreadTracking(long handle,
       boolean enableThreadTracking);
   private native boolean enableThreadTracking(long handle);

java/src/main/java/org/rocksdb/ReadOptions.java

@@ -476,7 +476,7 @@ public class ReadOptions extends RocksObject {
    *
    * The upper bound is exclusive i.e. the bound value is not a valid entry.
    *
-   * If prefix_extractor is not null, the Seek target and iterate_upper_bound
+   * If iterator_extractor is not null, the Seek target and iterate_upper_bound
    * need to have the same prefix. This is because ordering is not guaranteed
    * outside of prefix domain.
    *

java/src/test/java/org/rocksdb/DBOptionsTest.java

@@ -406,6 +406,24 @@ public class DBOptionsTest {
     }
   }
 
+  @Test
+  public void statsPersistPeriodSec() {
+    try (final DBOptions opt = new DBOptions()) {
+      final int intValue = rand.nextInt();
+      opt.setStatsPersistPeriodSec(intValue);
+      assertThat(opt.statsPersistPeriodSec()).isEqualTo(intValue);
+    }
+  }
+
+  @Test
+  public void statsHistoryBufferSize() {
+    try (final DBOptions opt = new DBOptions()) {
+      final long longValue = rand.nextLong();
+      opt.setStatsHistoryBufferSize(longValue);
+      assertThat(opt.statsHistoryBufferSize()).isEqualTo(longValue);
+    }
+  }
+
   @Test
   public void adviseRandomOnOpen() {
     try(final DBOptions opt = new DBOptions()) {
@@ -516,6 +534,15 @@ public class DBOptionsTest {
     }
   }
 
+  @Test
+  public void strictBytesPerSync() {
+    try (final DBOptions opt = new DBOptions()) {
+      assertThat(opt.strictBytesPerSync()).isFalse();
+      opt.setStrictBytesPerSync(true);
+      assertThat(opt.strictBytesPerSync()).isTrue();
+    }
+  }
+
   @Test
   public void enableThreadTracking() {
     try (final DBOptions opt = new DBOptions()) {

java/src/test/java/org/rocksdb/MutableDBOptionsTest.java

@@ -56,21 +56,22 @@ public class MutableDBOptionsTest {
   }
 
   @Test
-  public void mutableColumnFamilyOptions_toString() {
+  public void mutableDBOptions_toString() {
     final String str = MutableDBOptions
         .builder()
         .setMaxOpenFiles(99)
         .setDelayedWriteRate(789)
         .setAvoidFlushDuringShutdown(true)
+        .setStrictBytesPerSync(true)
         .build()
         .toString();
 
     assertThat(str).isEqualTo("max_open_files=99;delayed_write_rate=789;"
-        + "avoid_flush_during_shutdown=true");
+        + "avoid_flush_during_shutdown=true;strict_bytes_per_sync=true");
   }
 
   @Test
-  public void mutableColumnFamilyOptions_parse() {
+  public void mutableDBOptions_parse() {
     final String str = "max_open_files=99;delayed_write_rate=789;"
         + "avoid_flush_during_shutdown=true";
 

java/src/test/java/org/rocksdb/OptionsTest.java

@@ -625,6 +625,24 @@ public class OptionsTest {
     }
   }
 
+  @Test
+  public void statsPersistPeriodSec() {
+    try (final Options opt = new Options()) {
+      final int intValue = rand.nextInt();
+      opt.setStatsPersistPeriodSec(intValue);
+      assertThat(opt.statsPersistPeriodSec()).isEqualTo(intValue);
+    }
+  }
+
+  @Test
+  public void statsHistoryBufferSize() {
+    try (final Options opt = new Options()) {
+      final long longValue = rand.nextLong();
+      opt.setStatsHistoryBufferSize(longValue);
+      assertThat(opt.statsHistoryBufferSize()).isEqualTo(longValue);
+    }
+  }
+
   @Test
   public void adviseRandomOnOpen() {
     try (final Options opt = new Options()) {
@@ -735,6 +753,15 @@ public class OptionsTest {
     }
   }
 
+  @Test
+  public void strictBytesPerSync() {
+    try (final Options opt = new Options()) {
+      assertThat(opt.strictBytesPerSync()).isFalse();
+      opt.setStrictBytesPerSync(true);
+      assertThat(opt.strictBytesPerSync()).isTrue();
+    }
+  }
+
   @Test
   public void enableThreadTracking() {
     try (final Options opt = new Options()) {

src.mk

@@ -265,15 +265,7 @@ BENCH_LIB_SOURCES =                                             \
   tools/db_bench_tool.cc                                        \
 
 STRESS_LIB_SOURCES =                                            \
-  db_stress_tool/batched_ops_stress.cc                         \
-  db_stress_tool/cf_consistency_stress.cc                      \
-  db_stress_tool/db_stress_common.cc                           \
-  db_stress_tool/db_stress_driver.cc                           \
-  db_stress_tool/db_stress_test_base.cc                        \
-  db_stress_tool/db_stress_gflags.cc                           \
-  db_stress_tool/db_stress_shared_state.cc                     \
-  db_stress_tool/db_stress_tool.cc                             \
-  db_stress_tool/no_batched_ops_stress.cc                      \
+  tools/db_stress_tool.cc                                       \
 
 TEST_LIB_SOURCES =                                              \
   db/db_test_util.cc                                            \
@@ -291,7 +283,6 @@ FOLLY_SOURCES = \
 MAIN_SOURCES =                                                          \
   cache/cache_bench.cc                                                  \
   cache/cache_test.cc                                                   \
-  db_stress_tool/db_stress.cc                                           \
   db/column_family_test.cc                                              \
   db/compact_files_test.cc                                              \
   db/compaction/compaction_iterator_test.cc                             \
@@ -407,6 +398,7 @@ MAIN_SOURCES =                                                          \
   tools/db_bench.cc                                                     \
   tools/db_bench_tool_test.cc                                           \
   tools/db_sanity_test.cc                                               \
+  tools/db_stress.cc                                                    \
   tools/ldb_cmd_test.cc                                                 \
   tools/reduce_levels_test.cc                                           \
   tools/sst_dump_test.cc                                                \

table/block_based/block_based_table_builder.cc

@@ -1108,9 +1108,7 @@ void BlockBasedTableBuilder::EnterUnbuffered() {
 
     for (const auto& key : keys) {
       if (r->filter_builder != nullptr) {
-        size_t ts_sz =
-            r->internal_comparator.user_comparator()->timestamp_size();
-        r->filter_builder->Add(ExtractUserKeyAndStripTimestamp(key, ts_sz));
+        r->filter_builder->Add(ExtractUserKey(key));
       }
       r->index_builder->OnKeyAdded(key);
     }

table/block_based/filter_policy.cc

@@ -443,7 +443,8 @@ void BloomFilterPolicy::CreateFilter(const Slice* keys, int n,
                                      std::string* dst) const {
   // We should ideally only be using this deprecated interface for
   // appropriately constructed BloomFilterPolicy
-  assert(mode_ == kDeprecatedBlock);
+  // FIXME disabled because of bug in C interface; see issue #6129
+  //assert(mode_ == kDeprecatedBlock);
 
   // Compute bloom filter size (in both bits and bytes)
   uint32_t bits = static_cast<uint32_t>(n * whole_bits_per_key_);

third-party/folly/folly/synchronization/test/DistributedMutexTest.cpp

@@ -13,7 +13,7 @@
 #include <gtest/gtest.h>
 #endif
 
-#if !defined(ROCKSDB_LITE) && !defined(__ARM_ARCH)
+#ifndef ROCKSDB_LITE
 
 #include <chrono>
 #include <cmath>
@@ -1127,16 +1127,10 @@ TEST(DistributedMutex, StressBigValueReturnSixtyFourThreads) {
   concurrentBigValueReturnStress(64, std::chrono::seconds{kStressTestSeconds});
 }
 
-}  // namespace folly
+} // namespace folly
+#endif  // ROCKSDB_LITE
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   return RUN_ALL_TESTS();
 }
-
-#else
-int main(int /*argc*/, char** /*argv*/) {
-  printf("DistributedMutex is not supported in ROCKSDB_LITE or on ARM\n");
-  return 0;
-}
-#endif  // !ROCKSDB_LITE && !__ARM_ARCH

tools/CMakeLists.txt

@@ -1,6 +1,7 @@
 set(TOOLS
   sst_dump.cc
   db_sanity_test.cc
+  db_stress.cc
   write_stress.cc
   ldb.cc
   db_repl_stress.cc
@@ -10,12 +11,11 @@ foreach(src ${TOOLS})
   get_filename_component(exename ${src} NAME_WE)
   add_executable(${exename}${ARTIFACT_SUFFIX}
     ${src})
-  target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${ROCKSDB_LIB})
+  target_link_libraries(${exename}${ARTIFACT_SUFFIX} ${LIBS})
   list(APPEND tool_deps ${exename})
 endforeach()
-
-list(APPEND tool_deps)
-
+add_custom_target(tools
+  DEPENDS ${tool_deps})
 add_custom_target(ldb_tests
   COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/ldb_tests.py
   DEPENDS ldb)

tools/block_cache_analyzer/block_cache_trace_analyzer.cc

@@ -577,11 +577,10 @@ void BlockCacheTraceAnalyzer::WriteSkewness(
     pairs.push_back(itr);
   }
   // Sort in descending order.
-  sort(pairs.begin(), pairs.end(),
-       [=](const std::pair<std::string, uint64_t>& a,
-           const std::pair<std::string, uint64_t>& b) {
-         return b.second < a.second;
-       });
+  sort(
+      pairs.begin(), pairs.end(),
+      [=](std::pair<std::string, uint64_t>& a,
+          std::pair<std::string, uint64_t>& b) { return b.second < a.second; });
 
   size_t prev_start_index = 0;
   for (auto const& percent : percent_buckets) {

tools/db_crashtest.py

@@ -19,16 +19,12 @@ import argparse
 #   for cf_consistency:
 #       default_params < {blackbox,whitebox}_default_params <
 #       cf_consistency_params < args
-#   for txn:
-#       default_params < {blackbox,whitebox}_default_params < txn_params < args
 
 expected_values_file = tempfile.NamedTemporaryFile()
 
 default_params = {
     "acquire_snapshot_one_in": 10000,
     "block_size": 16384,
-    "bloom_bits": lambda: random.choice([random.randint(0,19),
-                                         random.lognormvariate(2.3, 1.3)]),
     "cache_index_and_filter_blocks": lambda: random.randint(0, 1),
     "cache_size": 1048576,
     "checkpoint_one_in": 1000000,
@@ -54,7 +50,6 @@ default_params = {
     "nooverwritepercent": 1,
     "open_files": lambda : random.choice([-1, 500000]),
     "partition_filters": lambda: random.randint(0, 1),
-    "pause_background_one_in": 1000000,
     "prefixpercent": 5,
     "progress_reports": 0,
     "readpercent": 45,
@@ -71,7 +66,7 @@ default_params = {
     "verify_checksum": 1,
     "write_buffer_size": 4 * 1024 * 1024,
     "writepercent": 35,
-    "format_version": lambda: random.choice([2, 3, 4, 5, 5]),
+    "format_version": lambda: random.randint(2, 4),
     "index_block_restart_interval": lambda: random.choice(range(1, 16)),
     "use_multiget" : lambda: random.randint(0, 1),
     "periodic_compaction_seconds" :
@@ -157,14 +152,6 @@ cf_consistency_params = {
     "enable_pipelined_write": lambda: random.randint(0, 1),
 }
 
-txn_params = {
-    "use_txn" : 1,
-    "disable_wal": 0,
-    # OpenReadOnly after checkpoint is not currnetly compatible with WritePrepared txns
-    "checkpoint_one_in": 0,
-    # pipeline write is not currnetly compatible with WritePrepared txns
-    "enable_pipelined_write": 0,
-}
 
 def finalize_and_sanitize(src_params):
     dest_params = dict([(k,  v() if callable(v) else v)
@@ -178,9 +165,7 @@ def finalize_and_sanitize(src_params):
             dest_params["db"]):
         dest_params["use_direct_io_for_flush_and_compaction"] = 0
         dest_params["use_direct_reads"] = 0
-    # DeleteRange is not currnetly compatible with Txns
-    if dest_params.get("test_batches_snapshots") == 1 or \
-            dest_params.get("use_txn") == 1:
+    if dest_params.get("test_batches_snapshots") == 1:
         dest_params["delpercent"] += dest_params["delrangepercent"]
         dest_params["delrangepercent"] = 0
     if dest_params.get("disable_wal", 0) == 1:
@@ -221,8 +206,6 @@ def gen_cmd_params(args):
             params.update(whitebox_simple_default_params)
     if args.cf_consistency:
         params.update(cf_consistency_params)
-    if args.txn:
-        params.update(txn_params)
 
     for k, v in vars(args).items():
         if v is not None:
@@ -235,7 +218,7 @@ def gen_cmd(params, unknown_params):
         '--{0}={1}'.format(k, v)
         for k, v in finalize_and_sanitize(params).items()
         if k not in set(['test_type', 'simple', 'duration', 'interval',
-                         'random_kill_odd', 'cf_consistency', 'txn'])
+                         'random_kill_odd', 'cf_consistency'])
         and v is not None] + unknown_params
     return cmd
 
@@ -433,7 +416,6 @@ def main():
     parser.add_argument("test_type", choices=["blackbox", "whitebox"])
     parser.add_argument("--simple", action="store_true")
     parser.add_argument("--cf_consistency", action='store_true')
-    parser.add_argument("--txn", action='store_true')
 
     all_params = dict(default_params.items()
                       + blackbox_default_params.items()

tools/ldb_cmd.cc

@@ -1016,8 +1016,8 @@ void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
                       /*block_cache_tracer=*/nullptr);
   Status s = versions.DumpManifest(options, file, verbose, hex, json);
   if (!s.ok()) {
-    fprintf(stderr, "Error in processing file %s %s\n", file.c_str(),
-            s.ToString().c_str());
+    printf("Error in processing file %s %s\n", file.c_str(),
+           s.ToString().c_str());
   }
 }
 
@@ -1066,72 +1066,45 @@ void ManifestDumpCommand::DoCommand() {
   if (!path_.empty()) {
     manifestfile = path_;
   } else {
+    bool found = false;
     // We need to find the manifest file by searching the directory
     // containing the db for files of the form MANIFEST_[0-9]+
 
-    std::vector<std::string> files;
-    Status s = options_.env->GetChildren(db_path_, &files);
-    if (!s.ok()) {
-      std::string err_msg = s.ToString();
-      err_msg.append(": Failed to list the content of ");
-      err_msg.append(db_path_);
-      exec_state_ = LDBCommandExecuteResult::Failed(err_msg);
+    auto CloseDir = [](DIR* p) { closedir(p); };
+    std::unique_ptr<DIR, decltype(CloseDir)> d(opendir(db_path_.c_str()),
+                                               CloseDir);
+
+    if (d == nullptr) {
+      exec_state_ =
+          LDBCommandExecuteResult::Failed(db_path_ + " is not a directory");
       return;
     }
-    const std::string kManifestNamePrefix = "MANIFEST-";
-    std::string matched_file;
-#ifdef OS_WIN
-    const char kPathDelim = '\\';
-#else
-    const char kPathDelim = '/';
-#endif
-    for (const auto& file_path : files) {
-      // Some Env::GetChildren() return absolute paths. Some directories' path
-      // end with path delim, e.g. '/' or '\\'.
-      size_t pos = file_path.find_last_of(kPathDelim);
-      if (pos == file_path.size() - 1) {
-        continue;
-      }
-      std::string fname;
-      if (pos != std::string::npos) {
-        // Absolute path.
-        fname.assign(file_path, pos + 1, file_path.size() - pos - 1);
-      } else {
-        fname = file_path;
-      }
-      uint64_t file_num = 0;
-      FileType file_type = kLogFile;  // Just for initialization
-      if (ParseFileName(fname, &file_num, &file_type) &&
-          file_type == kDescriptorFile) {
-        if (!matched_file.empty()) {
+    struct dirent* entry;
+    while ((entry = readdir(d.get())) != nullptr) {
+      unsigned int match;
+      uint64_t num;
+      if (sscanf(entry->d_name, "MANIFEST-%" PRIu64 "%n", &num, &match) &&
+          match == strlen(entry->d_name)) {
+        if (!found) {
+          manifestfile = db_path_ + "/" + std::string(entry->d_name);
+          found = true;
+        } else {
           exec_state_ = LDBCommandExecuteResult::Failed(
               "Multiple MANIFEST files found; use --path to select one");
           return;
-        } else {
-          matched_file.swap(fname);
         }
       }
     }
-    if (matched_file.empty()) {
-      std::string err_msg("No MANIFEST found in ");
-      err_msg.append(db_path_);
-      exec_state_ = LDBCommandExecuteResult::Failed(err_msg);
-      return;
-    }
-    if (db_path_[db_path_.length() - 1] != '/') {
-      db_path_.append("/");
-    }
-    manifestfile = db_path_ + matched_file;
   }
 
   if (verbose_) {
-    fprintf(stdout, "Processing Manifest file %s\n", manifestfile.c_str());
+    printf("Processing Manifest file %s\n", manifestfile.c_str());
   }
 
   DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_);
 
   if (verbose_) {
-    fprintf(stdout, "Processing Manifest file %s done\n", manifestfile.c_str());
+    printf("Processing Manifest file %s done\n", manifestfile.c_str());
   }
 }
 
@@ -1153,19 +1126,19 @@ void ListColumnFamiliesCommand::DoCommand() {
   std::vector<std::string> column_families;
   Status s = DB::ListColumnFamilies(options_, db_path_, &column_families);
   if (!s.ok()) {
-    fprintf(stderr, "Error in processing db %s %s\n", db_path_.c_str(),
-            s.ToString().c_str());
+    printf("Error in processing db %s %s\n", db_path_.c_str(),
+           s.ToString().c_str());
   } else {
-    fprintf(stdout, "Column families in %s: \n{", db_path_.c_str());
+    printf("Column families in %s: \n{", db_path_.c_str());
     bool first = true;
     for (auto cf : column_families) {
       if (!first) {
-        fprintf(stdout, ", ");
+        printf(", ");
       }
       first = false;
-      fprintf(stdout, "%s", cf.c_str());
+      printf("%s", cf.c_str());
     }
-    fprintf(stdout, "}\n");
+    printf("}\n");
   }
 }
 
@@ -2788,7 +2761,7 @@ void CheckPointCommand::DoCommand() {
   Status status = Checkpoint::Create(db_, &checkpoint);
   status = checkpoint->CreateCheckpoint(checkpoint_dir_);
   if (status.ok()) {
-    fprintf(stdout, "OK\n");
+    printf("OK\n");
   } else {
     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
   }
@@ -2812,7 +2785,7 @@ void RepairCommand::DoCommand() {
   options.info_log.reset(new StderrLogger(InfoLogLevel::WARN_LEVEL));
   Status status = RepairDB(db_path_, options);
   if (status.ok()) {
-    fprintf(stdout, "OK\n");
+    printf("OK\n");
   } else {
     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
   }
@@ -2892,7 +2865,7 @@ void BackupCommand::DoCommand() {
     assert(GetExecuteState().IsFailed());
     return;
   }
-  fprintf(stdout, "open db OK\n");
+  printf("open db OK\n");
   Env* custom_env = nullptr;
   Env::LoadEnv(backup_env_uri_, &custom_env, &backup_env_guard_);
   assert(custom_env != nullptr);
@@ -2903,14 +2876,14 @@ void BackupCommand::DoCommand() {
   backup_options.max_background_operations = num_threads_;
   status = BackupEngine::Open(custom_env, backup_options, &backup_engine);
   if (status.ok()) {
-    fprintf(stdout, "open backup engine OK\n");
+    printf("open backup engine OK\n");
   } else {
     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
     return;
   }
   status = backup_engine->CreateNewBackup(db_);
   if (status.ok()) {
-    fprintf(stdout, "create new backup OK\n");
+    printf("create new backup OK\n");
   } else {
     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
     return;
@@ -2948,11 +2921,11 @@ void RestoreCommand::DoCommand() {
     }
   }
   if (status.ok()) {
-    fprintf(stdout, "open restore engine OK\n");
+    printf("open restore engine OK\n");
     status = restore_engine->RestoreDBFromLatestBackup(db_path_, db_path_);
   }
   if (status.ok()) {
-    fprintf(stdout, "restore from backup OK\n");
+    printf("restore from backup OK\n");
   } else {
     exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
   }

util/channel.h

@@ -60,7 +60,7 @@ class channel {
 
  private:
   std::condition_variable cv_;
-  std::mutex lock_;
+  mutable std::mutex lock_;
   std::queue<T> buffer_;
   bool eof_;
 };

util/thread_local_test.cc

@@ -65,45 +65,45 @@ TEST_F(ThreadLocalTest, UniqueIdTest) {
   port::Mutex mu;
   port::CondVar cv(&mu);
 
-  uint32_t base_id = IDChecker::PeekId();
+  ASSERT_EQ(IDChecker::PeekId(), 0u);
   // New ThreadLocal instance bumps id by 1
   {
     // Id used 0
     Params p1(&mu, &cv, nullptr, 1u);
-    ASSERT_EQ(IDChecker::PeekId(), base_id + 1u);
+    ASSERT_EQ(IDChecker::PeekId(), 1u);
     // Id used 1
     Params p2(&mu, &cv, nullptr, 1u);
-    ASSERT_EQ(IDChecker::PeekId(), base_id + 2u);
+    ASSERT_EQ(IDChecker::PeekId(), 2u);
     // Id used 2
     Params p3(&mu, &cv, nullptr, 1u);
-    ASSERT_EQ(IDChecker::PeekId(), base_id + 3u);
+    ASSERT_EQ(IDChecker::PeekId(), 3u);
     // Id used 3
     Params p4(&mu, &cv, nullptr, 1u);
-    ASSERT_EQ(IDChecker::PeekId(), base_id + 4u);
+    ASSERT_EQ(IDChecker::PeekId(), 4u);
   }
   // id 3, 2, 1, 0 are in the free queue in order
-  ASSERT_EQ(IDChecker::PeekId(), base_id + 0u);
+  ASSERT_EQ(IDChecker::PeekId(), 0u);
 
   // pick up 0
   Params p1(&mu, &cv, nullptr, 1u);
-  ASSERT_EQ(IDChecker::PeekId(), base_id + 1u);
+  ASSERT_EQ(IDChecker::PeekId(), 1u);
   // pick up 1
   Params* p2 = new Params(&mu, &cv, nullptr, 1u);
-  ASSERT_EQ(IDChecker::PeekId(), base_id + 2u);
+  ASSERT_EQ(IDChecker::PeekId(), 2u);
   // pick up 2
   Params p3(&mu, &cv, nullptr, 1u);
-  ASSERT_EQ(IDChecker::PeekId(), base_id + 3u);
+  ASSERT_EQ(IDChecker::PeekId(), 3u);
   // return up 1
   delete p2;
-  ASSERT_EQ(IDChecker::PeekId(), base_id + 1u);
+  ASSERT_EQ(IDChecker::PeekId(), 1u);
   // Now we have 3, 1 in queue
   // pick up 1
   Params p4(&mu, &cv, nullptr, 1u);
-  ASSERT_EQ(IDChecker::PeekId(), base_id + 3u);
+  ASSERT_EQ(IDChecker::PeekId(), 3u);
   // pick up 3
   Params p5(&mu, &cv, nullptr, 1u);
   // next new id
-  ASSERT_EQ(IDChecker::PeekId(), base_id + 4u);
+  ASSERT_EQ(IDChecker::PeekId(), 4u);
   // After exit, id sequence in queue:
   // 3, 1, 2, 0
 }
@@ -111,7 +111,7 @@ TEST_F(ThreadLocalTest, UniqueIdTest) {
 
 TEST_F(ThreadLocalTest, SequentialReadWriteTest) {
   // global id list carries over 3, 1, 2, 0
-  uint32_t base_id = IDChecker::PeekId();
+  ASSERT_EQ(IDChecker::PeekId(), 0u);
 
   port::Mutex mu;
   port::CondVar cv(&mu);
@@ -141,7 +141,7 @@ TEST_F(ThreadLocalTest, SequentialReadWriteTest) {
   };
 
   for (int iter = 0; iter < 1024; ++iter) {
-    ASSERT_EQ(IDChecker::PeekId(), base_id + 1u);
+    ASSERT_EQ(IDChecker::PeekId(), 1u);
     // Another new thread, read/write should not see value from previous thread
     env_->StartThread(func, static_cast<void*>(&p));
     mu.Lock();
@@ -149,13 +149,13 @@ TEST_F(ThreadLocalTest, SequentialReadWriteTest) {
       cv.Wait();
     }
     mu.Unlock();
-    ASSERT_EQ(IDChecker::PeekId(), base_id + 1u);
+    ASSERT_EQ(IDChecker::PeekId(), 1u);
   }
 }
 
 TEST_F(ThreadLocalTest, ConcurrentReadWriteTest) {
   // global id list carries over 3, 1, 2, 0
-  uint32_t base_id = IDChecker::PeekId();
+  ASSERT_EQ(IDChecker::PeekId(), 0u);
 
   ThreadLocalPtr tls2;
   port::Mutex mu1;
@@ -236,10 +236,12 @@ TEST_F(ThreadLocalTest, ConcurrentReadWriteTest) {
   }
   mu2.Unlock();
 
-  ASSERT_EQ(IDChecker::PeekId(), base_id + 3u);
+  ASSERT_EQ(IDChecker::PeekId(), 3u);
 }
 
 TEST_F(ThreadLocalTest, Unref) {
+  ASSERT_EQ(IDChecker::PeekId(), 0u);
+
   auto unref = [](void* ptr) {
     auto& p = *static_cast<Params*>(ptr);
     p.mu->Lock();

utilities/blob_db/blob_db.h

@@ -87,7 +87,7 @@ class BlobDB : public StackableDB {
   virtual Status Put(const WriteOptions& options,
                      ColumnFamilyHandle* column_family, const Slice& key,
                      const Slice& value) override {
-    if (column_family != DefaultColumnFamily()) {
+    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
       return Status::NotSupported(
           "Blob DB doesn't support non-default column family.");
     }
@@ -98,7 +98,7 @@ class BlobDB : public StackableDB {
   virtual Status Delete(const WriteOptions& options,
                         ColumnFamilyHandle* column_family,
                         const Slice& key) override {
-    if (column_family != DefaultColumnFamily()) {
+    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
       return Status::NotSupported(
           "Blob DB doesn't support non-default column family.");
     }
@@ -111,7 +111,7 @@ class BlobDB : public StackableDB {
   virtual Status PutWithTTL(const WriteOptions& options,
                             ColumnFamilyHandle* column_family, const Slice& key,
                             const Slice& value, uint64_t ttl) {
-    if (column_family != DefaultColumnFamily()) {
+    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
       return Status::NotSupported(
           "Blob DB doesn't support non-default column family.");
     }
@@ -125,7 +125,7 @@ class BlobDB : public StackableDB {
   virtual Status PutUntil(const WriteOptions& options,
                           ColumnFamilyHandle* column_family, const Slice& key,
                           const Slice& value, uint64_t expiration) {
-    if (column_family != DefaultColumnFamily()) {
+    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
       return Status::NotSupported(
           "Blob DB doesn't support non-default column family.");
     }
@@ -157,7 +157,7 @@ class BlobDB : public StackableDB {
       const std::vector<Slice>& keys,
       std::vector<std::string>* values) override {
     for (auto column_family : column_families) {
-      if (column_family != DefaultColumnFamily()) {
+      if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
         return std::vector<Status>(
             column_families.size(),
             Status::NotSupported(
@@ -197,7 +197,7 @@ class BlobDB : public StackableDB {
   virtual Iterator* NewIterator(const ReadOptions& options) override = 0;
   virtual Iterator* NewIterator(const ReadOptions& options,
                                 ColumnFamilyHandle* column_family) override {
-    if (column_family != DefaultColumnFamily()) {
+    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
       // Blob DB doesn't support non-default column family.
       return nullptr;
     }
@@ -217,7 +217,7 @@ class BlobDB : public StackableDB {
       const int output_path_id = -1,
       std::vector<std::string>* const output_file_names = nullptr,
       CompactionJobInfo* compaction_job_info = nullptr) override {
-    if (column_family != DefaultColumnFamily()) {
+    if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
       return Status::NotSupported(
           "Blob DB doesn't support non-default column family.");
     }

utilities/blob_db/blob_db_impl.cc

@@ -459,6 +459,10 @@ void BlobDBImpl::ProcessFlushJobInfo(const FlushJobInfo& info) {
 void BlobDBImpl::ProcessCompactionJobInfo(const CompactionJobInfo& info) {
   assert(bdb_options_.enable_garbage_collection);
 
+  if (!info.status.ok()) {
+    return;
+  }
+
   // Note: the same SST file may appear in both the input and the output
   // file list in case of a trivial move. We process the inputs first
   // to ensure the blob file still has a link after processing all updates.
@@ -1462,7 +1466,7 @@ Status BlobDBImpl::Get(const ReadOptions& read_options,
 Status BlobDBImpl::GetImpl(const ReadOptions& read_options,
                            ColumnFamilyHandle* column_family, const Slice& key,
                            PinnableSlice* value, uint64_t* expiration) {
-  if (column_family != DefaultColumnFamily()) {
+  if (column_family->GetID() != DefaultColumnFamily()->GetID()) {
     return Status::NotSupported(
         "Blob DB doesn't support non-default column family.");
   }