Update HISTORY.md

Summary:
As discovered during v5.9.2 release, and forward-ported.
Closes https://github.com/facebook/rocksdb/pull/3323

Differential Revision: D6657209

Pulled By: siying

fbshipit-source-id: b560d9f8ddb89e0ffaff7c895ec80f68ddf7dab4

HISTORY.md

@@ -1,5 +1,20 @@
 # Rocksdb Change Log
-## Unreleased
+## 5.10.4 (02/22/2018)
+### New Features
+* Follow rsync-style naming convention for BackupEngine tempfiles. This enables some optimizations when run on GlusterFS.
+* Fix regression of Java build break on Windows.
+
+## 5.10.3 (02/21/2018)
+### Bug fixes
+* Fix build break regression using gcc-7
+* Direct I/O writable file should do fsync in Close()
+
+### New Features
+* Add rocksdb.iterator.internal-key property
+
+## 5.10.1 (01/18/2018)
+### Bug Fixes
+* Fix DB::Flush() keep waiting after flush finish under certain condition.
 
 ## 5.10.0 (12/11/2017)
 ### Public API Change

INSTALL.md

@@ -107,6 +107,41 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
             * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
     * run `brew install rocksdb`
 
+* **FreeBSD** (11.01):
+
+    * You can either install RocksDB from the Ports system using `cd /usr/ports/databases/rocksdb && make install`, or you can follow the details below to install dependencies and compile from source code:
+
+    * Install the dependencies for RocksDB:
+
+        export BATCH=YES
+        cd /usr/ports/devel/gmake && make install
+        cd /usr/ports/devel/gflags && make install
+
+        cd /usr/ports/archivers/snappy && make install
+        cd /usr/ports/archivers/bzip2 && make install
+        cd /usr/ports/archivers/liblz4 && make install
+        cd /usr/ports/archivesrs/zstd && make install
+
+        cd /usr/ports/devel/git && make install
+
+
+    * Install the dependencies for RocksJava (optional):
+
+        export BATCH=yes
+        cd /usr/ports/java/openjdk7 && make install
+
+    * Build RocksDB from source:
+        cd ~
+        git clone https://github.com/facebook/rocksdb.git
+        cd rocksdb
+        gmake static_lib
+
+    * Build RocksJava from source (optional):
+        cd rocksdb
+        export JAVA_HOME=/usr/local/openjdk7
+        gmake rocksdbjava
+
+
 * **iOS**:
   * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
 

Makefile

@@ -344,6 +344,8 @@ ifeq ($(HAVE_POWER8),1)
 LIB_CC_OBJECTS = $(LIB_SOURCES:.cc=.o)
 LIBOBJECTS += $(LIB_SOURCES_C:.c=.o)
 LIBOBJECTS += $(LIB_SOURCES_ASM:.S=.o)
+else
+LIB_CC_OBJECTS = $(LIB_SOURCES:.cc=.o)
 endif
 
 LIBOBJECTS += $(TOOL_LIB_SOURCES:.cc=.o)
@@ -1571,7 +1573,7 @@ else
 endif
 endif
 ifeq ($(PLATFORM), OS_FREEBSD)
-	JAVA_INCLUDE += -I$(JAVA_HOME)/include/freebsd
+	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/freebsd
 	ROCKSDBJNILIB = librocksdbjni-freebsd$(ARCH).so
 	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-freebsd$(ARCH).jar
 endif
@@ -1597,7 +1599,7 @@ libz.a:
 		exit 1; \
 	fi
 	tar xvzf zlib-$(ZLIB_VER).tar.gz
-	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && make
+	cd zlib-$(ZLIB_VER) && CFLAGS='-fPIC ${EXTRA_CFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --static && $(MAKE)
 	cp zlib-$(ZLIB_VER)/libz.a .
 
 libbz2.a:
@@ -1609,7 +1611,7 @@ libbz2.a:
 		exit 1; \
 	fi
 	tar xvzf bzip2-$(BZIP2_VER).tar.gz
-	cd bzip2-$(BZIP2_VER) && make CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
+	cd bzip2-$(BZIP2_VER) && $(MAKE) CFLAGS='-fPIC -O2 -g -D_FILE_OFFSET_BITS=64 ${EXTRA_CFLAGS}' AR='ar ${EXTRA_ARFLAGS}'
 	cp bzip2-$(BZIP2_VER)/libbz2.a .
 
 libsnappy.a:
@@ -1622,7 +1624,7 @@ libsnappy.a:
 	fi
 	tar xvzf snappy-$(SNAPPY_VER).tar.gz
 	cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
-	cd snappy-$(SNAPPY_VER) && make ${SNAPPY_MAKE_TARGET}
+	cd snappy-$(SNAPPY_VER) && $(MAKE) ${SNAPPY_MAKE_TARGET}
 	cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
 
 liblz4.a:
@@ -1635,7 +1637,7 @@ liblz4.a:
 		exit 1; \
 	fi
 	tar xvzf lz4-$(LZ4_VER).tar.gz
-	cd lz4-$(LZ4_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
+	cd lz4-$(LZ4_VER)/lib && $(MAKE) CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
 	cp lz4-$(LZ4_VER)/lib/liblz4.a .
 
 libzstd.a:
@@ -1648,29 +1650,45 @@ libzstd.a:
 		exit 1; \
 	fi
 	tar xvzf zstd-$(ZSTD_VER).tar.gz
-	cd zstd-$(ZSTD_VER)/lib && make CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' all
+	cd zstd-$(ZSTD_VER)/lib && DESTDIR=. PREFIX= $(MAKE) CFLAGS='-fPIC -O2 ${EXTRA_CFLAGS}' install
 	cp zstd-$(ZSTD_VER)/lib/libzstd.a .
 
 # A version of each $(LIBOBJECTS) compiled with -fPIC and a fixed set of static compression libraries
-java_static_libobjects = $(patsubst %,jls/%,$(LIBOBJECTS))
+java_static_libobjects = $(patsubst %,jls/%,$(LIB_CC_OBJECTS))
 CLEAN_FILES += jls
+java_static_all_libobjects = $(java_static_libobjects)
 
 ifneq ($(ROCKSDB_JAVA_NO_COMPRESSION), 1)
 JAVA_COMPRESSIONS = libz.a libbz2.a libsnappy.a liblz4.a libzstd.a
 endif
 
 JAVA_STATIC_FLAGS = -DZLIB -DBZIP2 -DSNAPPY -DLZ4 -DZSTD
-JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib
+JAVA_STATIC_INCLUDES = -I./zlib-$(ZLIB_VER) -I./bzip2-$(BZIP2_VER) -I./snappy-$(SNAPPY_VER) -I./lz4-$(LZ4_VER)/lib -I./zstd-$(ZSTD_VER)/lib/include
+
+ifeq ($(HAVE_POWER8),1)
+JAVA_STATIC_C_LIBOBJECTS = $(patsubst %.c.o,jls/%.c.o,$(LIB_SOURCES_C:.c=.o))
+JAVA_STATIC_ASM_LIBOBJECTS = $(patsubst %.S.o,jls/%.S.o,$(LIB_SOURCES_ASM:.S=.o))
+
+java_static_ppc_libobjects = $(JAVA_STATIC_C_LIBOBJECTS) $(JAVA_STATIC_ASM_LIBOBJECTS)
+
+jls/util/crc32c_ppc.o: util/crc32c_ppc.c
+	$(AM_V_CC)$(CC) $(CFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -c $< -o $@
+
+jls/util/crc32c_ppc_asm.o: util/crc32c_ppc_asm.S
+	$(AM_V_CC)$(CC) $(CFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -c $< -o $@
+
+java_static_all_libobjects += $(java_static_ppc_libobjects)
+endif
 
 $(java_static_libobjects): jls/%.o: %.cc $(JAVA_COMPRESSIONS)
 	$(AM_V_CC)mkdir -p $(@D) && $(CXX) $(CXXFLAGS) $(JAVA_STATIC_FLAGS) $(JAVA_STATIC_INCLUDES) -fPIC -c $< -o $@ $(COVERAGEFLAGS)
 
-rocksdbjavastatic: $(java_static_libobjects)
+rocksdbjavastatic: $(java_static_all_libobjects)
 	cd java;$(MAKE) javalib;
 	rm -f ./java/target/$(ROCKSDBJNILIB)
 	$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC \
 	  -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
-	  $(java_static_libobjects) $(COVERAGEFLAGS) \
+	  $(java_static_all_libobjects) $(COVERAGEFLAGS) \
 	  $(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
 	cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
@@ -1731,7 +1749,7 @@ JAVA_C_LIBOBJECTS = $(patsubst %.c.o,jl/%.c.o,$(JAVA_C_OBJECTS))
 JAVA_ASM_LIBOBJECTS = $(patsubst %.S.o,jl/%.S.o,$(JAVA_ASM_OBJECTS))
 endif
 
-java_libobjects = $(patsubst %,jl/%,$(LIBOBJECTS))
+java_libobjects = $(patsubst %,jl/%,$(LIB_CC_OBJECTS))
 CLEAN_FILES += jl
 java_all_libobjects = $(java_libobjects)
 

buckifier/buckify_rocksdb.py

@@ -36,7 +36,7 @@ def parse_src_mk(repo_path):
 # get all .cc / .c files
 def get_cc_files(repo_path):
     cc_files = []
-    for root, dirnames, filenames in os.walk(repo_path):
+    for root, dirnames, filenames in os.walk(repo_path):  # noqa: B007 T25377293 Grandfathered in
         root = root[(len(repo_path) + 1):]
         if "java" in root:
             # Skip java

buckifier/rocks_test_runner.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
+# shellcheck disable=SC2068
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"

build_tools/build_detect_platform

@@ -141,6 +141,7 @@ case "$TARGET_OS" in
         ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
+        CXX=clang++
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
         # PORT_FILES=port/freebsd/freebsd_specific.cc
@@ -481,6 +482,9 @@ if test -z "$PORTABLE"; then
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
   elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
     COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
+  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^arm`"; then
+    # TODO: Handle this with approprite options.
+    COMMON_FLAGS="$COMMON_FLAGS"
   elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
   elif test "$USE_SSE"; then

build_tools/cont_integration.sh

@@ -13,10 +13,12 @@ error=0
 
 function log {
   DATE=`date +%Y-%m-%d:%H:%M:%S`
+  # shellcheck disable=SC2068
   echo $DATE $@
 }
 
 function log_err {
+  # shellcheck disable=SC2145
   log "ERROR: $@ Error code: $error."
 }
 

build_tools/dependencies.sh

@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 GCC_BASE=/mnt/gvfs/third-party2/gcc/8219ec1bcedf8ad9da05e121e193364de2cc4f61/5.x/centos6-native/c447969
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/64d8d58e3d84f8bde7a029763d4f5baf39d0d5b9/stable/centos6-native/6aaf4de
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ba9be983c81de7299b59fe71950c664a84dcb5f8/5.x/gcc-5-glibc-2.23/339d858

build_tools/dependencies_4.8.1.sh

@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc

build_tools/make_package.sh

@@ -1,3 +1,4 @@
+# shellcheck disable=SC1113
 #/usr/bin/env bash
 
 set -e
@@ -28,12 +29,14 @@ function package() {
     if dpkg --get-selections | grep --quiet $1; then
       log "$1 is already installed. skipping."
     else
+      # shellcheck disable=SC2068
       apt-get install $@ -y
     fi
   elif [[ $OS = "centos" ]]; then
     if rpm -qa | grep --quiet $1; then
       log "$1 is already installed. skipping."
     else
+      # shellcheck disable=SC2068
       yum install $@ -y
     fi
   fi
@@ -52,6 +55,7 @@ function gem_install() {
   if gem list | grep --quiet $1; then
     log "$1 is already installed. skipping."
   else
+    # shellcheck disable=SC2068
     gem install $@
   fi
 }
@@ -125,4 +129,5 @@ function main() {
     include $LIB_DIR
 }
 
+# shellcheck disable=SC2068
 main $@

coverage/parse_gcov_output.py

@@ -72,7 +72,7 @@ def display_file_coverage(per_file_coverage, total_coverage):
     header_template = \
         "%" + str(max_file_name_length) + "s\t%s\t%s"
     separator = "-" * (max_file_name_length + 10 + 20)
-    print header_template % ("Filename", "Coverage", "Lines")
+    print header_template % ("Filename", "Coverage", "Lines")  # noqa: E999 T25377293 Grandfathered in
     print separator
 
     # -- Print body

db/column_family.cc

@@ -344,12 +344,13 @@ void SuperVersionUnrefHandle(void* ptr) {
   // When latter happens, we are in ~ColumnFamilyData(), no get should happen as
   // well.
   SuperVersion* sv = static_cast<SuperVersion*>(ptr);
-  if (sv->Unref()) {
-    sv->db_mutex->Lock();
-    sv->Cleanup();
-    sv->db_mutex->Unlock();
-    delete sv;
-  }
+  bool was_last_ref __attribute__((__unused__));
+  was_last_ref = sv->Unref();
+  // Thread-local SuperVersions can't outlive ColumnFamilyData::super_version_.
+  // This is important because we can't do SuperVersion cleanup here.
+  // That would require locking DB mutex, which would deadlock because
+  // SuperVersionUnrefHandle is called with locked ThreadLocalPtr mutex.
+  assert(!was_last_ref);
 }
 }  // anonymous namespace
 
@@ -385,7 +386,8 @@ ColumnFamilyData::ColumnFamilyData(
       pending_flush_(false),
       pending_compaction_(false),
       prev_compaction_needed_bytes_(0),
-      allow_2pc_(db_options.allow_2pc) {
+      allow_2pc_(db_options.allow_2pc),
+      last_memtable_id_(0) {
   Ref();
 
   // Convert user defined table properties collector factories to internal ones.
@@ -965,6 +967,12 @@ void ColumnFamilyData::InstallSuperVersion(
       RecalculateWriteStallConditions(mutable_cf_options);
 
   if (old_superversion != nullptr) {
+    // Reset SuperVersions cached in thread local storage.
+    // This should be done before old_superversion->Unref(). That's to ensure
+    // that local_sv_ never holds the last reference to SuperVersion, since
+    // it has no means to safely do SuperVersion cleanup.
+    ResetThreadLocalSuperVersions();
+
     if (old_superversion->mutable_cf_options.write_buffer_size !=
         mutable_cf_options.write_buffer_size) {
       mem_->UpdateWriteBufferSize(mutable_cf_options.write_buffer_size);
@@ -980,9 +988,6 @@ void ColumnFamilyData::InstallSuperVersion(
       sv_context->superversions_to_free.push_back(old_superversion);
     }
   }
-
-  // Reset SuperVersions cached in thread local storage
-  ResetThreadLocalSuperVersions();
 }
 
 void ColumnFamilyData::ResetThreadLocalSuperVersions() {
@@ -994,10 +999,12 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
       continue;
     }
     auto sv = static_cast<SuperVersion*>(ptr);
-    if (sv->Unref()) {
-      sv->Cleanup();
-      delete sv;
-    }
+    bool was_last_ref __attribute__((__unused__));
+    was_last_ref = sv->Unref();
+    // sv couldn't have been the last reference because
+    // ResetThreadLocalSuperVersions() is called before
+    // unref'ing super_version_.
+    assert(!was_last_ref);
   }
 }
 

db/column_family.h

@@ -239,7 +239,11 @@ class ColumnFamilyData {
   void SetCurrent(Version* _current);
   uint64_t GetNumLiveVersions() const;  // REQUIRE: DB mutex held
   uint64_t GetTotalSstFilesSize() const;  // REQUIRE: DB mutex held
-  void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
+  void SetMemtable(MemTable* new_mem) {
+    uint64_t memtable_id = last_memtable_id_.fetch_add(1) + 1;
+    new_mem->SetID(memtable_id);
+    mem_ = new_mem;
+  }
 
   // calculate the oldest log needed for the durability of this column family
   uint64_t OldestLogToKeep();
@@ -419,6 +423,9 @@ class ColumnFamilyData {
 
   // if the database was opened with 2pc enabled
   bool allow_2pc_;
+
+  // Memtable id to track flush.
+  std::atomic<uint64_t> last_memtable_id_;
 };
 
 // ColumnFamilySet has interesting thread-safety requirements

db/compaction_job.cc

@@ -625,8 +625,8 @@ Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
       "[%s] compacted to: %s, MB/sec: %.1f rd, %.1f wr, level %d, "
       "files in(%d, %d) out(%d) "
       "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
-      "write-amplify(%.1f) %s, records in: %d, records dropped: %d "
-      "output_compression: %s\n",
+      "write-amplify(%.1f) %s, records in: %" PRIu64
+      ", records dropped: %" PRIu64 " output_compression: %s\n",
       cfd->GetName().c_str(), vstorage->LevelSummary(&tmp), bytes_read_per_sec,
       bytes_written_per_sec, compact_->compaction->output_level(),
       stats.num_input_files_in_non_output_levels,

db/db_flush_test.cc

@@ -126,6 +126,41 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
   ASSERT_EQ(1, num_compactions);
 }
 
+TEST_F(DBFlushTest, ManualFlushWithMinWriteBufferNumberToMerge) {
+  Options options = CurrentOptions();
+  options.write_buffer_size = 100;
+  options.max_write_buffer_number = 4;
+  options.min_write_buffer_number_to_merge = 3;
+  Reopen(options);
+
+  SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::BGWorkFlush",
+        "DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1"},
+       {"DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2",
+        "DBImpl::FlushMemTableToOutputFile:BeforeInstallSV"}});
+  SyncPoint::GetInstance()->EnableProcessing();
+
+  ASSERT_OK(Put("key1", "value1"));
+
+  port::Thread t([&]() {
+    // The call wait for flush to finish, i.e. with flush_options.wait = true.
+    ASSERT_OK(Flush());
+  });
+
+  // Wait for flush start.
+  TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1");
+  // Insert a second memtable before the manual flush finish.
+  // At the end of the manual flush job, it will check if further flush
+  // is needed, but it will not trigger flush of the second memtable because
+  // min_write_buffer_number_to_merge is not reached.
+  ASSERT_OK(Put("key2", "value2"));
+  ASSERT_OK(dbfull()->TEST_SwitchMemtable());
+  TEST_SYNC_POINT("DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2");
+
+  // Manual flush should return, without waiting for flush indefinitely.
+  t.join();
+}
+
 TEST_P(DBFlushDirectIOTest, DirectIO) {
   Options options;
   options.create_if_missing = true;

db/db_impl.h

@@ -810,8 +810,12 @@ class DBImpl : public DB {
   Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options,
                        bool writes_stopped = false);
 
-  // Wait for memtable flushed
-  Status WaitForFlushMemTable(ColumnFamilyData* cfd);
+  // Wait for memtable flushed.
+  // If flush_memtable_id is non-null, wait until the memtable with the ID
+  // gets flush. Otherwise, wait until the column family don't have any
+  // memtable pending flush.
+  Status WaitForFlushMemTable(ColumnFamilyData* cfd,
+                              const uint64_t* flush_memtable_id = nullptr);
 
   // REQUIRES: mutex locked
   Status SwitchWAL(WriteContext* write_context);

db/db_impl_compaction_flush.cc

@@ -134,6 +134,7 @@ Status DBImpl::FlushMemTableToOutputFile(
   }
 
   if (s.ok()) {
+    TEST_SYNC_POINT("DBImpl::FlushMemTableToOutputFile:BeforeInstallSV");
     InstallSuperVersionAndScheduleWork(cfd, &job_context->superversion_context,
                                        mutable_cf_options);
     if (made_progress) {
@@ -809,7 +810,13 @@ int DBImpl::Level0StopWriteTrigger(ColumnFamilyHandle* column_family) {
 Status DBImpl::Flush(const FlushOptions& flush_options,
                      ColumnFamilyHandle* column_family) {
   auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
-  return FlushMemTable(cfh->cfd(), flush_options);
+  ROCKS_LOG_INFO(immutable_db_options_.info_log, "[%s] Manual flush start.",
+                 cfh->GetName().c_str());
+  Status s = FlushMemTable(cfh->cfd(), flush_options);
+  ROCKS_LOG_INFO(immutable_db_options_.info_log,
+                 "[%s] Manual flush finished, status: %s\n",
+                 cfh->GetName().c_str(), s.ToString().c_str());
+  return s;
 }
 
 Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level,
@@ -944,6 +951,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
                              const FlushOptions& flush_options,
                              bool writes_stopped) {
   Status s;
+  uint64_t flush_memtable_id = 0;
   {
     WriteContext context;
     InstrumentedMutexLock guard_lock(&mutex_);
@@ -961,6 +969,7 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
 
     // SwitchMemtable() will release and reacquire mutex during execution
     s = SwitchMemtable(cfd, &context);
+    flush_memtable_id = cfd->imm()->GetLatestMemTableID();
 
     if (!writes_stopped) {
       write_thread_.ExitUnbatched(&w);
@@ -975,16 +984,19 @@ Status DBImpl::FlushMemTable(ColumnFamilyData* cfd,
 
   if (s.ok() && flush_options.wait) {
     // Wait until the compaction completes
-    s = WaitForFlushMemTable(cfd);
+    s = WaitForFlushMemTable(cfd, &flush_memtable_id);
   }
   return s;
 }
 
-Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd) {
+Status DBImpl::WaitForFlushMemTable(ColumnFamilyData* cfd,
+                                    const uint64_t* flush_memtable_id) {
   Status s;
   // Wait until the compaction completes
   InstrumentedMutexLock l(&mutex_);
-  while (cfd->imm()->NumNotFlushed() > 0 && bg_error_.ok()) {
+  while (cfd->imm()->NumNotFlushed() > 0 && bg_error_.ok() &&
+         (flush_memtable_id == nullptr ||
+          cfd->imm()->GetEarliestMemTableID() <= *flush_memtable_id)) {
     if (shutting_down_.load(std::memory_order_acquire)) {
       return Status::ShutdownInProgress();
     }

db/db_iter.cc

@@ -217,6 +217,9 @@ class DBIter final: public Iterator {
         *prop = "Iterator is not valid.";
       }
       return Status::OK();
+    } else if (prop_name == "rocksdb.iterator.internal-key") {
+      *prop = saved_key_.GetUserKey().ToString();
+      return Status::OK();
     }
     return Status::InvalidArgument("Undentified property.");
   }

db/db_iterator_test.cc

@@ -55,6 +55,7 @@ TEST_F(DBIteratorTest, IteratorProperty) {
   Options options = CurrentOptions();
   CreateAndReopenWithCF({"pikachu"}, options);
   Put(1, "1", "2");
+  Delete(1, "2");
   ReadOptions ropt;
   ropt.pin_data = false;
   {
@@ -64,9 +65,15 @@ TEST_F(DBIteratorTest, IteratorProperty) {
     ASSERT_NOK(iter->GetProperty("non_existing.value", &prop_value));
     ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
     ASSERT_EQ("0", prop_value);
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
+    ASSERT_EQ("1", prop_value);
     iter->Next();
     ASSERT_OK(iter->GetProperty("rocksdb.iterator.is-key-pinned", &prop_value));
     ASSERT_EQ("Iterator is not valid.", prop_value);
+
+    // Get internal key at which the iteration stopped (tombstone in this case).
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
+    ASSERT_EQ("2", prop_value);
   }
   Close();
 }
@@ -2157,6 +2164,48 @@ TEST_F(DBIteratorTest, SkipStatistics) {
   ASSERT_EQ(skip_count, TestGetTickerCount(options, NUMBER_ITER_SKIP));
 }
 
+TEST_F(DBIteratorTest, SeekAfterHittingManyInternalKeys) {
+  Options options = CurrentOptions();
+  DestroyAndReopen(options);
+  ReadOptions ropts;
+  ropts.max_skippable_internal_keys = 2;
+
+  Put("1", "val_1");
+  // Add more tombstones than max_skippable_internal_keys so that Next() fails.
+  Delete("2");
+  Delete("3");
+  Delete("4");
+  Delete("5");
+  Put("6", "val_6");
+
+  unique_ptr<Iterator> iter(db_->NewIterator(ropts));
+  iter->SeekToFirst();
+
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ(iter->key().ToString(), "1");
+  ASSERT_EQ(iter->value().ToString(), "val_1");
+
+  // This should fail as incomplete due to too many non-visible internal keys on
+  // the way to the next valid user key.
+  iter->Next();
+  ASSERT_TRUE(!iter->Valid());
+  ASSERT_TRUE(iter->status().IsIncomplete());
+
+  // Get the internal key at which Next() failed.
+  std::string prop_value;
+  ASSERT_OK(iter->GetProperty("rocksdb.iterator.internal-key", &prop_value));
+  ASSERT_EQ("4", prop_value);
+
+  // Create a new iterator to seek to the internal key.
+  unique_ptr<Iterator> iter2(db_->NewIterator(ropts));
+  iter2->Seek(prop_value);
+  ASSERT_TRUE(iter2->Valid());
+  ASSERT_OK(iter2->status());
+
+  ASSERT_EQ(iter2->key().ToString(), "6");
+  ASSERT_EQ(iter2->value().ToString(), "val_6");
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/db_test.cc

@@ -5651,6 +5651,50 @@ TEST_F(DBTest, PauseBackgroundWorkTest) {
   // now it's done
   ASSERT_TRUE(done.load());
 }
+
+// Keep spawning short-living threads that create an iterator and quit.
+// Meanwhile in another thread keep flushing memtables.
+// This used to cause a deadlock.
+TEST_F(DBTest, ThreadLocalPtrDeadlock) {
+  std::atomic<int> flushes_done{0};
+  std::atomic<int> threads_destroyed{0};
+  auto done = [&] {
+    return flushes_done.load() > 10;
+  };
+
+  std::thread flushing_thread([&] {
+    for (int i = 0; !done(); ++i) {
+      ASSERT_OK(db_->Put(WriteOptions(), Slice("hi"),
+                         Slice(std::to_string(i).c_str())));
+      ASSERT_OK(db_->Flush(FlushOptions()));
+      int cnt = ++flushes_done;
+      fprintf(stderr, "Flushed %d times\n", cnt);
+    }
+  });
+
+  std::vector<std::thread> thread_spawning_threads(10);
+  for (auto& t: thread_spawning_threads) {
+    t = std::thread([&] {
+      while (!done()) {
+        {
+          std::thread tmp_thread([&] {
+            auto it = db_->NewIterator(ReadOptions());
+            delete it;
+          });
+          tmp_thread.join();
+        }
+        ++threads_destroyed;
+      }
+    });
+  }
+
+  for (auto& t: thread_spawning_threads) {
+    t.join();
+  }
+  flushing_thread.join();
+  fprintf(stderr, "Done. Flushed %d times, destroyed %d threads\n",
+          flushes_done.load(), threads_destroyed.load());
+}
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/log_reader.h

@@ -54,6 +54,7 @@ class Reader {
   // The Reader will start reading at the first record located at physical
   // position >= initial_offset within the file.
   Reader(std::shared_ptr<Logger> info_log,
+  // @lint-ignore TXT2 T25377293 Grandfathered in
 	 unique_ptr<SequentialFileReader>&& file,
          Reporter* reporter, bool checksum, uint64_t initial_offset,
          uint64_t log_num);

db/memtable.h

@@ -368,6 +368,11 @@ class MemTable {
     return oldest_key_time_.load(std::memory_order_relaxed);
   }
 
+  // REQUIRES: db_mutex held.
+  void SetID(uint64_t id) { id_ = id; }
+
+  uint64_t GetID() const { return id_; }
+
  private:
   enum FlushStateEnum { FLUSH_NOT_REQUESTED, FLUSH_REQUESTED, FLUSH_SCHEDULED };
 
@@ -437,6 +442,9 @@ class MemTable {
   // Timestamp of oldest key
   std::atomic<uint64_t> oldest_key_time_;
 
+  // Memtable id to track flush.
+  uint64_t id_ = 0;
+
   // Returns a heuristic flush decision
   bool ShouldFlushNow() const;
 

db/memtable_list.h

@@ -5,11 +5,12 @@
 //
 #pragma once
 
-#include <string>
-#include <list>
-#include <vector>
-#include <set>
 #include <deque>
+#include <limits>
+#include <list>
+#include <set>
+#include <string>
+#include <vector>
 
 #include "db/dbformat.h"
 #include "db/memtable.h"
@@ -244,6 +245,22 @@ class MemTableList {
 
   uint64_t GetMinLogContainingPrepSection();
 
+  uint64_t GetEarliestMemTableID() const {
+    auto& memlist = current_->memlist_;
+    if (memlist.empty()) {
+      return std::numeric_limits<uint64_t>::max();
+    }
+    return memlist.back()->GetID();
+  }
+
+  uint64_t GetLatestMemTableID() const {
+    auto& memlist = current_->memlist_;
+    if (memlist.empty()) {
+      return 0;
+    }
+    return memlist.front()->GetID();
+  }
+
  private:
   // DB mutex held
   void InstallNewVersion();

hdfs/setup.sh

@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 export USE_HDFS=1
 export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:/usr/lib/hadoop/lib/native
 

include/rocksdb/iterator.h

@@ -97,6 +97,9 @@ class Iterator : public Cleanable {
   // Property "rocksdb.iterator.super-version-number":
   //   LSM version used by the iterator. The same format as DB Property
   //   kCurrentSuperVersionNumber. See its comment for more information.
+  // Property "rocksdb.iterator.internal-key":
+  //   Get the user-key portion of the internal key at which the iteration
+  //   stopped.
   virtual Status GetProperty(std::string prop_name, std::string* prop);
 
  private:

include/rocksdb/utilities/stackable_db.h

@@ -4,6 +4,7 @@
 
 #pragma once
 #include <map>
+#include <memory>
 #include <string>
 #include "rocksdb/db.h"
 
@@ -18,11 +19,20 @@ namespace rocksdb {
 // This class contains APIs to stack rocksdb wrappers.Eg. Stack TTL over base d
 class StackableDB : public DB {
  public:
-  // StackableDB is the owner of db now!
+  // StackableDB take sole ownership of the underlying db.
   explicit StackableDB(DB* db) : db_(db) {}
 
+  // StackableDB take shared ownership of the underlying db.
+  explicit StackableDB(std::shared_ptr<DB> db)
+      : db_(db.get()), shared_db_ptr_(db) {}
+
   ~StackableDB() {
-    delete db_;
+    if (shared_db_ptr_ == nullptr) {
+      delete db_;
+    } else {
+      assert(shared_db_ptr_.get() == db_);
+    }
+    db_ = nullptr;
   }
 
   virtual DB* GetBaseDB() {
@@ -373,6 +383,7 @@ class StackableDB : public DB {
 
  protected:
   DB* db_;
+  std::shared_ptr<DB> shared_db_ptr_;
 };
 
 } //  namespace rocksdb

include/rocksdb/version.h

@@ -6,7 +6,7 @@
 
 #define ROCKSDB_MAJOR 5
 #define ROCKSDB_MINOR 10
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 4
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

java/CMakeLists.txt

@@ -7,6 +7,8 @@ set(JNI_NATIVE_SOURCES
         rocksjni/clock_cache.cc
         rocksjni/columnfamilyhandle.cc
         rocksjni/compaction_filter.cc
+        rocksjni/compaction_filter_factory.cc
+        rocksjni/compaction_filter_factory_jnicallback.cc
         rocksjni/compaction_options_fifo.cc
         rocksjni/compaction_options_universal.cc
         rocksjni/comparator.cc
@@ -17,6 +19,7 @@ set(JNI_NATIVE_SOURCES
         rocksjni/filter.cc
         rocksjni/ingest_external_file_options.cc
         rocksjni/iterator.cc
+        rocksjni/jnicallback.cc
         rocksjni/loggerjnicallback.cc
         rocksjni/lru_cache.cc
         rocksjni/memtablejni.cc
@@ -25,6 +28,7 @@ set(JNI_NATIVE_SOURCES
         rocksjni/options_util.cc
         rocksjni/ratelimiterjni.cc
         rocksjni/remove_emptyvalue_compactionfilterjni.cc
+        rocksjni/rocks_callback_object.cc
         rocksjni/cassandra_compactionfilterjni.cc
         rocksjni/cassandra_value_operator.cc
         rocksjni/restorejni.cc

java/jdb_bench.sh

@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 PLATFORM=64
 if [ `getconf LONG_BIT` != "64" ]
 then
@@ -7,4 +8,5 @@ fi
 ROCKS_JAR=`find target -name rocksdbjni*.jar`
 
 echo "Running benchmark in $PLATFORM-Bit mode."
+# shellcheck disable=SC2068
 java -server -d$PLATFORM -XX:NewSize=4m -XX:+AggressiveOpts -Djava.library.path=target -cp "${ROCKS_JAR}:benchmark/target/classes" org.rocksdb.benchmark.DbBenchmark $@

java/rocksjni/compaction_filter_factory.cc

@@ -34,4 +34,5 @@ void Java_org_rocksdb_AbstractCompactionFilterFactory_disposeInternal(
   auto* ptr_sptr_cff =
       reinterpret_cast<std::shared_ptr<rocksdb::CompactionFilterFactoryJniCallback> *>(jhandle);
   delete ptr_sptr_cff;
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }

java/rocksjni/ingest_external_file_options.cc

@@ -146,4 +146,5 @@ void Java_org_rocksdb_IngestExternalFileOptions_disposeInternal(
   auto* options =
       reinterpret_cast<rocksdb::IngestExternalFileOptions*>(jhandle);
   delete options;
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }

java/rocksjni/jnicallback.cc

@@ -49,4 +49,5 @@ JniCallback::~JniCallback() {
 
   releaseJniEnv(attached_thread);
 }
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }  // namespace rocksdb

java/rocksjni/jnicallback.h

@@ -25,4 +25,5 @@ namespace rocksdb {
   };
 }
 
+// @lint-ignore TXT4 T25377293 Grandfathered in
 #endif  // JAVA_ROCKSJNI_JNICALLBACK_H_

java/rocksjni/rocks_callback_object.cc

@@ -24,4 +24,5 @@ void Java_org_rocksdb_RocksCallbackObject_disposeInternal(
       //   2) Comparator -> BaseComparatorJniCallback + JniCallback -> ComparatorJniCallback
       // I think this is okay, as Comparator and JniCallback both have virtual destructors...
       delete reinterpret_cast<rocksdb::JniCallback*>(handle);
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }

java/rocksjni/statisticsjni.cc

@@ -30,4 +30,5 @@ namespace rocksdb {
 
     return true;
   }
+// @lint-ignore TXT4 T25377293 Grandfathered in
 };

java/rocksjni/statisticsjni.h

@@ -30,4 +30,5 @@ namespace rocksdb {
 
 }  // namespace rocksdb
 
+// @lint-ignore TXT4 T25377293 Grandfathered in
 #endif  // JAVA_ROCKSJNI_STATISTICSJNI_H_

java/src/main/java/org/rocksdb/util/Environment.java

@@ -12,6 +12,10 @@ public class Environment {
     return (OS.contains("win"));
   }
 
+  public static boolean isFreeBSD() {
+    return (OS.contains("freebsd"));
+  }
+
   public static boolean isMac() {
     return (OS.contains("mac"));
   }
@@ -54,6 +58,8 @@ public class Environment {
       }
     } else if (isMac()) {
       return String.format("%sjni-osx", name);
+    } else if (isFreeBSD()) {
+      return String.format("%sjni-freebsd%s", name, is64Bit() ? "64" : "32");
     } else if (isAix() && is64Bit()) {
       return String.format("%sjni-aix64", name);
     } else if (isSolaris()) {
@@ -71,7 +77,7 @@ public class Environment {
   }
 
   private static String appendLibOsSuffix(final String libraryFileName, final boolean shared) {
-    if (isUnix() || isAix() || isSolaris()) {
+    if (isUnix() || isAix() || isSolaris() || isFreeBSD()) {
       return libraryFileName + ".so";
     } else if (isMac()) {
       return libraryFileName + (shared ? ".dylib" : ".jnilib");

port/stack_trace.cc

@@ -32,7 +32,7 @@ namespace port {
 
 namespace {
 
-#ifdef OS_LINUX
+#if defined(OS_LINUX) || defined(OS_FREEBSD)
 const char* GetExecutableName() {
   static char name[1024];
 

table/cuckoo_table_builder_test.cc

@@ -49,6 +49,7 @@ class CuckooBuilderTest : public testing::Test {
     uint64_t read_file_size;
     ASSERT_OK(env_->GetFileSize(fname, &read_file_size));
 
+   // @lint-ignore TXT2 T25377293 Grandfathered in
 	  Options options;
 	  options.allow_mmap_reads = true;
 	  ImmutableCFOptions ioptions(options);

tools/auto_sanity_test.sh

@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 TMP_DIR="${TMPDIR:-/tmp}/rocksdb-sanity-test"
 
 if [ "$#" -lt 2 ]; then

tools/benchmark.sh

@@ -449,6 +449,7 @@ echo "===== Benchmark ====="
 
 # Run!!!
 IFS=',' read -a jobs <<< $1
+# shellcheck disable=SC2068
 for job in ${jobs[@]}; do
 
   if [ $job != debug ]; then

tools/benchmark_leveldb.sh

@@ -151,6 +151,7 @@ echo "===== Benchmark ====="
 
 # Run!!!
 IFS=',' read -a jobs <<< $1
+# shellcheck disable=SC2068
 for job in ${jobs[@]}; do
 
   if [ $job != debug ]; then

tools/db_bench_tool.cc

@@ -3661,9 +3661,7 @@ void VerifyDBFromDB(std::string& truth_db_name) {
         }
       }
       if (!use_blob_db_) {
-#ifndef ROCKSDB_LITE
         s = db_with_cfh->db->Write(write_options_, &batch);
-#endif  //  ROCKSDB_LITE
       }
       thread->stats.FinishedOps(db_with_cfh, db_with_cfh->db,
                                 entries_per_batch_, kWrite);

tools/db_crashtest.py

@@ -317,7 +317,7 @@ def whitebox_crash_main(args):
         cmd = gen_cmd(dict(cmd_params.items() + additional_opts.items()
                            + {'db': dbname}.items()))
 
-        print "Running:" + ' '.join(cmd) + "\n"
+        print "Running:" + ' '.join(cmd) + "\n"  # noqa: E999 T25377293 Grandfathered in
 
         popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                  stderr=subprocess.STDOUT)

tools/ldb_test.py

@@ -229,7 +229,7 @@ class LDBTestCase(unittest.TestCase):
         self.assertRunFAIL("get --ttl a3")
         self.assertRunOK("checkconsistency", "OK")
 
-    def testInvalidCmdLines(self):
+    def testInvalidCmdLines(self):  # noqa: F811 T25377293 Grandfathered in
         print "Running testInvalidCmdLines..."
         # db not specified
         self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
@@ -516,7 +516,7 @@ class LDBTestCase(unittest.TestCase):
 
     def testColumnFamilies(self):
         print "Running testColumnFamilies..."
-        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)  # noqa: F841 T25377293 Grandfathered in
         self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
         self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
         self.assertRunOK("put cf1_3 3 --try_load_options", "OK")

tools/regression_test.sh

@@ -465,4 +465,5 @@ function cleanup_test_directory {
 
 ############################################################################
 
+# shellcheck disable=SC2068
 main $@

tools/rocksdb_dump_test.sh

@@ -1,3 +1,4 @@
+# shellcheck disable=SC2148
 TESTDIR=`mktemp -d ${TMPDIR:-/tmp}/rocksdb-dump-test.XXXXX`
 DUMPFILE="tools/sample-dump.dmp"
 

tools/write_stress_runner.py

@@ -21,7 +21,7 @@ def generate_runtimes(total_runtime):
 
 def main(args):
     runtimes = generate_runtimes(int(args.runtime_sec))
-    print "Going to execute write stress for " + str(runtimes)
+    print "Going to execute write stress for " + str(runtimes)  # noqa: E999 T25377293 Grandfathered in
     first_time = True
 
     for runtime in runtimes:

util/crc32c.cc

@@ -688,259 +688,386 @@ uint32_t crc32c_3way(uint32_t crc, const char* buf, size_t len) {
           do {
             // jumps here for a full block of len 128
             CRCtriplet(crc, next, -128);
+            // FALLTHRU
             case 127:
               // jumps here or below for the first block smaller
               CRCtriplet(crc, next, -127);
+              // FALLTHRU
             case 126:
               CRCtriplet(crc, next, -126); // than 128
+              // FALLTHRU
             case 125:
               CRCtriplet(crc, next, -125);
+              // FALLTHRU
             case 124:
               CRCtriplet(crc, next, -124);
+              // FALLTHRU
             case 123:
               CRCtriplet(crc, next, -123);
+              // FALLTHRU
             case 122:
               CRCtriplet(crc, next, -122);
+              // FALLTHRU
             case 121:
               CRCtriplet(crc, next, -121);
+              // FALLTHRU
             case 120:
               CRCtriplet(crc, next, -120);
+              // FALLTHRU
             case 119:
               CRCtriplet(crc, next, -119);
+              // FALLTHRU
             case 118:
               CRCtriplet(crc, next, -118);
+              // FALLTHRU
             case 117:
               CRCtriplet(crc, next, -117);
+              // FALLTHRU
             case 116:
               CRCtriplet(crc, next, -116);
+              // FALLTHRU
             case 115:
               CRCtriplet(crc, next, -115);
+              // FALLTHRU
             case 114:
               CRCtriplet(crc, next, -114);
+              // FALLTHRU
             case 113:
               CRCtriplet(crc, next, -113);
+              // FALLTHRU
             case 112:
               CRCtriplet(crc, next, -112);
+              // FALLTHRU
             case 111:
               CRCtriplet(crc, next, -111);
+              // FALLTHRU
             case 110:
               CRCtriplet(crc, next, -110);
+              // FALLTHRU
             case 109:
               CRCtriplet(crc, next, -109);
+              // FALLTHRU
             case 108:
               CRCtriplet(crc, next, -108);
+              // FALLTHRU
             case 107:
               CRCtriplet(crc, next, -107);
+              // FALLTHRU
             case 106:
               CRCtriplet(crc, next, -106);
+              // FALLTHRU
             case 105:
               CRCtriplet(crc, next, -105);
+              // FALLTHRU
             case 104:
               CRCtriplet(crc, next, -104);
+              // FALLTHRU
             case 103:
               CRCtriplet(crc, next, -103);
+              // FALLTHRU
             case 102:
               CRCtriplet(crc, next, -102);
+              // FALLTHRU
             case 101:
               CRCtriplet(crc, next, -101);
+              // FALLTHRU
             case 100:
               CRCtriplet(crc, next, -100);
+              // FALLTHRU
             case 99:
               CRCtriplet(crc, next, -99);
+              // FALLTHRU
             case 98:
               CRCtriplet(crc, next, -98);
+              // FALLTHRU
             case 97:
               CRCtriplet(crc, next, -97);
+              // FALLTHRU
             case 96:
               CRCtriplet(crc, next, -96);
+              // FALLTHRU
             case 95:
               CRCtriplet(crc, next, -95);
+              // FALLTHRU
             case 94:
               CRCtriplet(crc, next, -94);
+              // FALLTHRU
             case 93:
               CRCtriplet(crc, next, -93);
+              // FALLTHRU
             case 92:
               CRCtriplet(crc, next, -92);
+              // FALLTHRU
             case 91:
               CRCtriplet(crc, next, -91);
+              // FALLTHRU
             case 90:
               CRCtriplet(crc, next, -90);
+              // FALLTHRU
             case 89:
               CRCtriplet(crc, next, -89);
+              // FALLTHRU
             case 88:
               CRCtriplet(crc, next, -88);
+              // FALLTHRU
             case 87:
               CRCtriplet(crc, next, -87);
+              // FALLTHRU
             case 86:
               CRCtriplet(crc, next, -86);
+              // FALLTHRU
             case 85:
               CRCtriplet(crc, next, -85);
+              // FALLTHRU
             case 84:
               CRCtriplet(crc, next, -84);
+              // FALLTHRU
             case 83:
               CRCtriplet(crc, next, -83);
+              // FALLTHRU
             case 82:
               CRCtriplet(crc, next, -82);
+              // FALLTHRU
             case 81:
               CRCtriplet(crc, next, -81);
+              // FALLTHRU
             case 80:
               CRCtriplet(crc, next, -80);
+              // FALLTHRU
             case 79:
               CRCtriplet(crc, next, -79);
+              // FALLTHRU
             case 78:
               CRCtriplet(crc, next, -78);
+              // FALLTHRU
             case 77:
               CRCtriplet(crc, next, -77);
+              // FALLTHRU
             case 76:
               CRCtriplet(crc, next, -76);
+              // FALLTHRU
             case 75:
               CRCtriplet(crc, next, -75);
+              // FALLTHRU
             case 74:
               CRCtriplet(crc, next, -74);
+              // FALLTHRU
             case 73:
               CRCtriplet(crc, next, -73);
+              // FALLTHRU
             case 72:
               CRCtriplet(crc, next, -72);
+              // FALLTHRU
             case 71:
               CRCtriplet(crc, next, -71);
+              // FALLTHRU
             case 70:
               CRCtriplet(crc, next, -70);
+              // FALLTHRU
             case 69:
               CRCtriplet(crc, next, -69);
+              // FALLTHRU
             case 68:
               CRCtriplet(crc, next, -68);
+              // FALLTHRU
             case 67:
               CRCtriplet(crc, next, -67);
+              // FALLTHRU
             case 66:
               CRCtriplet(crc, next, -66);
+              // FALLTHRU
             case 65:
               CRCtriplet(crc, next, -65);
+              // FALLTHRU
             case 64:
               CRCtriplet(crc, next, -64);
+              // FALLTHRU
             case 63:
               CRCtriplet(crc, next, -63);
+              // FALLTHRU
             case 62:
               CRCtriplet(crc, next, -62);
+              // FALLTHRU
             case 61:
               CRCtriplet(crc, next, -61);
+              // FALLTHRU
             case 60:
               CRCtriplet(crc, next, -60);
+              // FALLTHRU
             case 59:
               CRCtriplet(crc, next, -59);
+              // FALLTHRU
             case 58:
               CRCtriplet(crc, next, -58);
+              // FALLTHRU
             case 57:
               CRCtriplet(crc, next, -57);
+              // FALLTHRU
             case 56:
               CRCtriplet(crc, next, -56);
+              // FALLTHRU
             case 55:
               CRCtriplet(crc, next, -55);
+              // FALLTHRU
             case 54:
               CRCtriplet(crc, next, -54);
+              // FALLTHRU
             case 53:
               CRCtriplet(crc, next, -53);
+              // FALLTHRU
             case 52:
               CRCtriplet(crc, next, -52);
+              // FALLTHRU
             case 51:
               CRCtriplet(crc, next, -51);
+              // FALLTHRU
             case 50:
               CRCtriplet(crc, next, -50);
+              // FALLTHRU
             case 49:
               CRCtriplet(crc, next, -49);
+              // FALLTHRU
             case 48:
               CRCtriplet(crc, next, -48);
+              // FALLTHRU
             case 47:
               CRCtriplet(crc, next, -47);
+              // FALLTHRU
             case 46:
               CRCtriplet(crc, next, -46);
+              // FALLTHRU
             case 45:
               CRCtriplet(crc, next, -45);
+              // FALLTHRU
             case 44:
               CRCtriplet(crc, next, -44);
+              // FALLTHRU
             case 43:
               CRCtriplet(crc, next, -43);
+              // FALLTHRU
             case 42:
               CRCtriplet(crc, next, -42);
+              // FALLTHRU
             case 41:
               CRCtriplet(crc, next, -41);
+              // FALLTHRU
             case 40:
               CRCtriplet(crc, next, -40);
+              // FALLTHRU
             case 39:
               CRCtriplet(crc, next, -39);
+              // FALLTHRU
             case 38:
               CRCtriplet(crc, next, -38);
+              // FALLTHRU
             case 37:
               CRCtriplet(crc, next, -37);
+              // FALLTHRU
             case 36:
               CRCtriplet(crc, next, -36);
+              // FALLTHRU
             case 35:
               CRCtriplet(crc, next, -35);
+              // FALLTHRU
             case 34:
               CRCtriplet(crc, next, -34);
+              // FALLTHRU
             case 33:
               CRCtriplet(crc, next, -33);
+              // FALLTHRU
             case 32:
               CRCtriplet(crc, next, -32);
+              // FALLTHRU
             case 31:
               CRCtriplet(crc, next, -31);
+              // FALLTHRU
             case 30:
               CRCtriplet(crc, next, -30);
+              // FALLTHRU
             case 29:
               CRCtriplet(crc, next, -29);
+              // FALLTHRU
             case 28:
               CRCtriplet(crc, next, -28);
+              // FALLTHRU
             case 27:
               CRCtriplet(crc, next, -27);
+              // FALLTHRU
             case 26:
               CRCtriplet(crc, next, -26);
+              // FALLTHRU
             case 25:
               CRCtriplet(crc, next, -25);
+              // FALLTHRU
             case 24:
               CRCtriplet(crc, next, -24);
+              // FALLTHRU
             case 23:
               CRCtriplet(crc, next, -23);
+              // FALLTHRU
             case 22:
               CRCtriplet(crc, next, -22);
+              // FALLTHRU
             case 21:
               CRCtriplet(crc, next, -21);
+              // FALLTHRU
             case 20:
               CRCtriplet(crc, next, -20);
+              // FALLTHRU
             case 19:
               CRCtriplet(crc, next, -19);
+              // FALLTHRU
             case 18:
               CRCtriplet(crc, next, -18);
+              // FALLTHRU
             case 17:
               CRCtriplet(crc, next, -17);
+              // FALLTHRU
             case 16:
               CRCtriplet(crc, next, -16);
+              // FALLTHRU
             case 15:
               CRCtriplet(crc, next, -15);
+              // FALLTHRU
             case 14:
               CRCtriplet(crc, next, -14);
+              // FALLTHRU
             case 13:
               CRCtriplet(crc, next, -13);
+              // FALLTHRU
             case 12:
               CRCtriplet(crc, next, -12);
+              // FALLTHRU
             case 11:
               CRCtriplet(crc, next, -11);
+              // FALLTHRU
             case 10:
               CRCtriplet(crc, next, -10);
+              // FALLTHRU
             case 9:
               CRCtriplet(crc, next, -9);
+              // FALLTHRU
             case 8:
               CRCtriplet(crc, next, -8);
+              // FALLTHRU
             case 7:
               CRCtriplet(crc, next, -7);
+              // FALLTHRU
             case 6:
               CRCtriplet(crc, next, -6);
+              // FALLTHRU
             case 5:
               CRCtriplet(crc, next, -5);
+              // FALLTHRU
             case 4:
               CRCtriplet(crc, next, -4);
+              // FALLTHRU
             case 3:
               CRCtriplet(crc, next, -3);
+              // FALLTHRU
             case 2:
               CRCtriplet(crc, next, -2);
+              // FALLTHRU
             case 1:
               CRCduplet(crc, next, -1); // the final triplet is actually only 2
               //{ CombineCRC(); }
@@ -953,6 +1080,7 @@ uint32_t crc32c_3way(uint32_t crc, const char* buf, size_t len) {
                 next1 = next0 + 128; // from here on all blocks are 128 long
                 next2 = next1 + 128;
               }
+              // FALLTHRU
             case 0:;
           } while (n > 0);
       }
@@ -964,58 +1092,85 @@ uint32_t crc32c_3way(uint32_t crc, const char* buf, size_t len) {
     switch (count2) {
       case 27:
         CRCsinglet(crc0, next, -27 * 8);
+        // FALLTHRU
       case 26:
         CRCsinglet(crc0, next, -26 * 8);
+        // FALLTHRU
       case 25:
         CRCsinglet(crc0, next, -25 * 8);
+        // FALLTHRU
       case 24:
         CRCsinglet(crc0, next, -24 * 8);
+        // FALLTHRU
       case 23:
         CRCsinglet(crc0, next, -23 * 8);
+        // FALLTHRU
       case 22:
         CRCsinglet(crc0, next, -22 * 8);
+        // FALLTHRU
       case 21:
         CRCsinglet(crc0, next, -21 * 8);
+        // FALLTHRU
       case 20:
         CRCsinglet(crc0, next, -20 * 8);
+        // FALLTHRU
       case 19:
         CRCsinglet(crc0, next, -19 * 8);
+        // FALLTHRU
       case 18:
         CRCsinglet(crc0, next, -18 * 8);
+        // FALLTHRU
       case 17:
         CRCsinglet(crc0, next, -17 * 8);
+        // FALLTHRU
       case 16:
         CRCsinglet(crc0, next, -16 * 8);
+        // FALLTHRU
       case 15:
         CRCsinglet(crc0, next, -15 * 8);
+        // FALLTHRU
       case 14:
         CRCsinglet(crc0, next, -14 * 8);
+        // FALLTHRU
       case 13:
         CRCsinglet(crc0, next, -13 * 8);
+        // FALLTHRU
       case 12:
         CRCsinglet(crc0, next, -12 * 8);
+        // FALLTHRU
       case 11:
         CRCsinglet(crc0, next, -11 * 8);
+        // FALLTHRU
       case 10:
         CRCsinglet(crc0, next, -10 * 8);
+        // FALLTHRU
       case 9:
         CRCsinglet(crc0, next, -9 * 8);
+        // FALLTHRU
       case 8:
         CRCsinglet(crc0, next, -8 * 8);
+        // FALLTHRU
       case 7:
         CRCsinglet(crc0, next, -7 * 8);
+        // FALLTHRU
       case 6:
         CRCsinglet(crc0, next, -6 * 8);
+        // FALLTHRU
       case 5:
         CRCsinglet(crc0, next, -5 * 8);
+        // FALLTHRU
       case 4:
         CRCsinglet(crc0, next, -4 * 8);
+        // FALLTHRU
       case 3:
         CRCsinglet(crc0, next, -3 * 8);
+        // FALLTHRU
       case 2:
         CRCsinglet(crc0, next, -2 * 8);
+        // FALLTHRU
       case 1:
         CRCsinglet(crc0, next, -1 * 8);
+        // FALLTHRU
       case 0:;
     }
   }

util/file_reader_writer.cc

@@ -239,6 +239,9 @@ Status WritableFileWriter::Close() {
   // we need to let the file know where data ends.
   if (use_direct_io()) {
     interim = writable_file_->Truncate(filesize_);
+    if (interim.ok()) {
+      interim = writable_file_->Fsync();
+    }
     if (!interim.ok() && s.ok()) {
       s = interim;
     }

util/thread_local.h

@@ -24,6 +24,13 @@ namespace rocksdb {
 // pointer (if not NULL) when one of the following happens:
 // (1) a thread terminates
 // (2) a ThreadLocalPtr is destroyed
+//
+// Warning: this function is called while holding a global mutex. The same mutex
+// is used (at least in some cases) by most methods of ThreadLocalPtr, and it's
+// shared across all instances of ThreadLocalPtr. Thereforere extra care
+// is needed to avoid deadlocks. In particular, the handler shouldn't lock any
+// mutexes and shouldn't call any methods of any ThreadLocalPtr instances,
+// unless you know what you're doing.
 typedef void (*UnrefHandler)(void* ptr);
 
 // ThreadLocalPtr stores only values of pointer type.  Different from

util/transaction_test_util.cc

@@ -77,8 +77,9 @@ Status RandomTransactionInserter::DBGet(
     uint64_t ikey, bool get_for_update, uint64_t* int_value,
     std::string* full_key, bool* unexpected_error) {
   Status s;
-  // four digits and zero end char
-  char prefix_buf[5];
+  // Five digits (since the largest uint16_t is 65535) plus the NUL
+  // end char.
+  char prefix_buf[6];
   // Pad prefix appropriately so we can iterate over each set
   snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1);
   // key format:  [SET#][random#]
@@ -261,8 +262,10 @@ Status RandomTransactionInserter::Verify(DB* db, uint16_t num_sets,
   }
   // For each set of keys with the same prefix, sum all the values
   for (uint16_t set_i : set_vec) {
-    // four digits and zero end char
-    char prefix_buf[5];
+    // Five digits (since the largest uint16_t is 65535) plus the NUL
+    // end char.
+    char prefix_buf[6];
+    assert(set_i + 1 <= 9999);
     snprintf(prefix_buf, sizeof(prefix_buf), "%.4u", set_i + 1);
     uint64_t total = 0;
 

utilities/backupable/backupable_db.cc

@@ -146,11 +146,17 @@ class BackupEngineImpl : public BackupEngine {
 
   class BackupMeta {
    public:
-    BackupMeta(const std::string& meta_filename,
+    BackupMeta(
+        const std::string& meta_filename, const std::string& meta_tmp_filename,
         std::unordered_map<std::string, std::shared_ptr<FileInfo>>* file_infos,
         Env* env)
-      : timestamp_(0), sequence_number_(0), size_(0),
-        meta_filename_(meta_filename), file_infos_(file_infos), env_(env) {}
+        : timestamp_(0),
+          sequence_number_(0),
+          size_(0),
+          meta_filename_(meta_filename),
+          meta_tmp_filename_(meta_tmp_filename),
+          file_infos_(file_infos),
+          env_(env) {}
 
     BackupMeta(const BackupMeta&) = delete;
     BackupMeta& operator=(const BackupMeta&) = delete;
@@ -228,6 +234,7 @@ class BackupEngineImpl : public BackupEngine {
     uint64_t size_;
     std::string app_metadata_;
     std::string const meta_filename_;
+    std::string const meta_tmp_filename_;
     // files with relative paths (without "/" prefix!!)
     std::vector<std::shared_ptr<FileInfo>> files_;
     std::unordered_map<std::string, std::shared_ptr<FileInfo>>* file_infos_;
@@ -257,12 +264,14 @@ class BackupEngineImpl : public BackupEngine {
   inline std::string GetSharedFileRel(const std::string& file = "",
                                       bool tmp = false) const {
     assert(file.size() == 0 || file[0] != '/');
-    return "shared/" + file + (tmp ? ".tmp" : "");
+    return std::string("shared/") + (tmp ? "." : "") + file +
+           (tmp ? ".tmp" : "");
   }
   inline std::string GetSharedFileWithChecksumRel(const std::string& file = "",
                                                   bool tmp = false) const {
     assert(file.size() == 0 || file[0] != '/');
-    return GetSharedChecksumDirRel() + "/" + file + (tmp ? ".tmp" : "");
+    return GetSharedChecksumDirRel() + "/" + (tmp ? "." : "") + file +
+           (tmp ? ".tmp" : "");
   }
   inline std::string GetSharedFileWithChecksum(const std::string& file,
                                                const uint32_t checksum_value,
@@ -283,8 +292,9 @@ class BackupEngineImpl : public BackupEngine {
   inline std::string GetBackupMetaDir() const {
     return GetAbsolutePath("meta");
   }
-  inline std::string GetBackupMetaFile(BackupID backup_id) const {
-    return GetBackupMetaDir() + "/" + rocksdb::ToString(backup_id);
+  inline std::string GetBackupMetaFile(BackupID backup_id, bool tmp) const {
+    return GetBackupMetaDir() + "/" + (tmp ? "." : "") +
+           rocksdb::ToString(backup_id) + (tmp ? ".tmp" : "");
   }
 
   // If size_limit == 0, there is no size limit, copy everything.
@@ -605,10 +615,11 @@ Status BackupEngineImpl::Initialize() {
       continue;
     }
     assert(backups_.find(backup_id) == backups_.end());
-    backups_.insert(
-        std::make_pair(backup_id, unique_ptr<BackupMeta>(new BackupMeta(
-                                      GetBackupMetaFile(backup_id),
-                                      &backuped_file_infos_, backup_env_))));
+    backups_.insert(std::make_pair(
+        backup_id, unique_ptr<BackupMeta>(new BackupMeta(
+                       GetBackupMetaFile(backup_id, false /* tmp */),
+                       GetBackupMetaFile(backup_id, true /* tmp */),
+                       &backuped_file_infos_, backup_env_))));
   }
 
   latest_backup_id_ = 0;
@@ -736,10 +747,11 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
   BackupID new_backup_id = latest_backup_id_ + 1;
 
   assert(backups_.find(new_backup_id) == backups_.end());
-  auto ret = backups_.insert(
-      std::make_pair(new_backup_id, unique_ptr<BackupMeta>(new BackupMeta(
-                                        GetBackupMetaFile(new_backup_id),
-                                        &backuped_file_infos_, backup_env_))));
+  auto ret = backups_.insert(std::make_pair(
+      new_backup_id, unique_ptr<BackupMeta>(new BackupMeta(
+                         GetBackupMetaFile(new_backup_id, false /* tmp */),
+                         GetBackupMetaFile(new_backup_id, true /* tmp */),
+                         &backuped_file_infos_, backup_env_))));
   assert(ret.second == true);
   auto& new_backup = ret.first->second;
   new_backup->RecordTimestamp();
@@ -1708,8 +1720,7 @@ Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) {
   EnvOptions env_options;
   env_options.use_mmap_writes = false;
   env_options.use_direct_writes = false;
-  s = env_->NewWritableFile(meta_filename_ + ".tmp", &backup_meta_file,
-                            env_options);
+  s = env_->NewWritableFile(meta_tmp_filename_, &backup_meta_file, env_options);
   if (!s.ok()) {
     return s;
   }
@@ -1749,7 +1760,7 @@ Status BackupEngineImpl::BackupMeta::StoreToFile(bool sync) {
     s = backup_meta_file->Close();
   }
   if (s.ok()) {
-    s = env_->RenameFile(meta_filename_ + ".tmp", meta_filename_);
+    s = env_->RenameFile(meta_tmp_filename_, meta_filename_);
   }
   return s;
 }

utilities/backupable/backupable_db_test.cc

@@ -810,9 +810,9 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
   test_db_env_->SetFilenamesForMockedAttrs(dummy_db_->live_files_);
   ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
   std::vector<std::string> should_have_written = {
-      "/shared/00010.sst.tmp",    "/shared/00011.sst.tmp",
+      "/shared/.00010.sst.tmp",    "/shared/.00011.sst.tmp",
       "/private/1.tmp/CURRENT",   "/private/1.tmp/MANIFEST-01",
-      "/private/1.tmp/00011.log", "/meta/1.tmp"};
+      "/private/1.tmp/00011.log", "/meta/.1.tmp"};
   AppendPath(backupdir_, should_have_written);
   test_backup_env_->AssertWrittenFiles(should_have_written);
 
@@ -828,9 +828,9 @@ TEST_F(BackupableDBTest, NoDoubleCopy) {
   ASSERT_OK(backup_engine_->CreateNewBackup(db_.get(), false));
   // should not open 00010.sst - it's already there
 
-  should_have_written = {"/shared/00015.sst.tmp", "/private/2.tmp/CURRENT",
+  should_have_written = {"/shared/.00015.sst.tmp", "/private/2.tmp/CURRENT",
                          "/private/2.tmp/MANIFEST-01",
-                         "/private/2.tmp/00011.log", "/meta/2.tmp"};
+                         "/private/2.tmp/00011.log", "/meta/.2.tmp"};
   AppendPath(backupdir_, should_have_written);
   test_backup_env_->AssertWrittenFiles(should_have_written);
 
@@ -1169,7 +1169,7 @@ TEST_F(BackupableDBTest, DeleteTmpFiles) {
     } else {
       shared_tmp += "/shared";
     }
-    shared_tmp += "/00006.sst.tmp";
+    shared_tmp += "/.00006.sst.tmp";
     std::string private_tmp_dir = backupdir_ + "/private/10.tmp";
     std::string private_tmp_file = private_tmp_dir + "/00003.sst";
     file_manager_->WriteToFile(shared_tmp, "tmp");

utilities/env_librados.cc

@@ -1485,4 +1485,5 @@ EnvLibrados* EnvLibrados::Default() {
                                  default_pool_name);
   return &default_env;
 }
+// @lint-ignore TXT4 T25377293 Grandfathered in
 }

utilities/simulator_cache/sim_cache.cc

@@ -89,6 +89,7 @@ class CacheActivityLogger {
     log_line += key.ToString(true);
     log_line += " - ";
     AppendNumberTo(&log_line, size);
+  // @lint-ignore TXT2 T25377293 Grandfathered in
 		log_line += "\n";
 
     // line format: "ADD - <KEY> - <KEY-SIZE>"

utilities/simulator_cache/sim_cache_test.cc

@@ -193,6 +193,7 @@ TEST_F(SimCacheTest, SimCacheLogging) {
   ASSERT_EQ(add_num, num_block_entries);
 
   // Log things again but stop logging automatically after reaching 512 bytes
+ // @lint-ignore TXT2 T25377293 Grandfathered in
 	int max_size = 512;
   ASSERT_OK(sim_cache->StartActivityLogging(log_file, env_, max_size));
   for (int it = 0; it < 10; it++) {