Disable error as warning

Add one more #include<functional>
Add some include<functional>
2026-11-06 11:02:36 -05:00 · 2019-11-05 11:03:59 -08:00 · 2019-11-05 11:03:58 -08:00 · 2019-10-31 14:29:25 -07:00 · 2019-10-31 14:29:24 -07:00 · 2017-07-17 22:22:06 -07:00
260 changed files with 11314 additions and 9395 deletions
@@ -207,6 +207,7 @@ set(SOURCES
        util/filter_policy.cc
        util/hash.cc
        util/histogram.cc
+        util/histogram_windowing.cc
        util/instrumented_mutex.cc
        util/iostats_context.cc
        tools/ldb_cmd.cc
@@ -238,6 +239,7 @@ set(SOURCES
        util/thread_status_updater.cc
        util/thread_status_util.cc
        util/thread_status_util_debug.cc
+        util/transaction_test_util.cc
        util/xfunc.cc
        util/xxhash.cc
        utilities/backupable/backupable_db.cc
@@ -338,6 +340,7 @@ set(TESTS
        db/db_tailing_iter_test.cc
        db/db_test.cc
        db/db_test2.cc
+        db/db_block_cache_test.cc
        db/db_universal_compaction_test.cc
        db/db_wal_test.cc
        db/dbformat_test.cc
@@ -357,6 +360,7 @@ set(TESTS
        db/perf_context_test.cc
        db/plain_table_db_test.cc
        db/prefix_test.cc
+        db/repair_test.cc
        db/skiplist_test.cc
        db/table_properties_collector_test.cc
        db/version_builder_test.cc
@@ -395,6 +399,7 @@ set(TESTS
        util/iostats_context_test.cc
        util/memenv_test.cc
        util/mock_env_test.cc
+        util/options_settable_test.cc
        util/options_test.cc
        util/rate_limiter_test.cc
        util/slice_transform_test.cc
@@ -0,0 +1,9 @@
+# RocksDB default options change log
+## 4.7.0 (4/8/2016)
+* options.write_buffer_size changes from 4MB to 64MB
+* options.target_file_size_base changes from 2MB to 64MB
+* options.max_bytes_for_level_base changes from 10MB to 256MB
+* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB
+* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB
+* table_cache_numshardbits changes from 4 to 6
+* max_file_opening_threads changes from 1 to 16 
@@ -1,7 +1,15 @@
 # Rocksdb Change Log
-## Unreleased
+## 4.7.0 (4/8/2016)
+### Public API Change
+* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
+* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
+
+## 4.6.0 (3/10/2016)
 ### Public API Changes
 * Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
+* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signiture of Cache::Insert() is updated accordingly.
+* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.  
+* Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree.
 ### New Features
 * Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
 * Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
@@ -0,0 +1,11 @@
+This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
+
+* Java - https://github.com/facebook/rocksdb/tree/master/java
+* Python - http://pyrocksdb.readthedocs.org/en/latest/
+* Perl - https://metacpan.org/pod/RocksDB
+* Node.js - https://npmjs.org/package/rocksdb
+* Go - https://github.com/tecbot/gorocksdb
+* Ruby - http://rubygems.org/gems/rocksdb-ruby
+* Haskell - https://hackage.haskell.org/package/rocksdb-haskell
+* PHP - https://github.com/Photonios/rocksdb-php
+* C# - https://github.com/warrenfalk/rocksdb-sharp
@@ -174,6 +174,15 @@ else
 	pg = -pg
 endif

+# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
+ifdef COMPILE_WITH_UBSAN
+	DISABLE_JEMALLOC=1
+	EXEC_LDFLAGS += -fsanitize=undefined
+	PLATFORM_CCFLAGS += -fsanitize=undefined
+	PLATFORM_CXXFLAGS += -fsanitize=undefined
+endif
+
+
 ifndef DISABLE_JEMALLOC
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
@@ -191,10 +200,6 @@ default: all
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
  -Wno-unused-parameter

-ifndef DISABLE_WARNING_AS_ERROR
-	WARNING_FLAGS += -Werror
-endif
-
 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
 CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers

@@ -246,6 +251,7 @@ BENCHTOOLOBJECTS = $(BENCH_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
 TESTS = \
 	db_test \
 	db_test2 \
+	db_block_cache_test \
 	db_iter_test \
 	db_log_iter_test \
 	db_compaction_filter_test \
@@ -320,6 +326,7 @@ TESTS = \
 	rate_limiter_test \
 	delete_scheduler_test \
 	options_test \
+	options_settable_test \
 	options_util_test \
 	event_logger_test \
 	cuckoo_table_builder_test \
@@ -599,7 +606,7 @@ ldb_tests: ldb
 crash_test: whitebox_crash_test blackbox_crash_test

 blackbox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple blackbox 
+	python -u tools/db_crashtest.py --simple blackbox
 	python -u tools/db_crashtest.py blackbox

 whitebox_crash_test: db_stress
@@ -616,8 +623,18 @@ asan_crash_test:
 	COMPILE_WITH_ASAN=1 $(MAKE) crash_test
 	$(MAKE) clean

+ubsan_check:
+	$(MAKE) clean
+	COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
+	$(MAKE) clean
+
+ubsan_crash_test:
+	$(MAKE) clean
+	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test
+	$(MAKE) clean
+
 valgrind_check: $(TESTS)
-	for t in $(filter-out %skiplist_test,$(TESTS)); do \
+	for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
 		$(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \
 		ret_code=$$?; \
 		if [ $$ret_code -ne 0 ]; then \
@@ -794,6 +811,9 @@ db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 db_test2: db/db_test2.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+db_block_cache_test: db/db_block_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 db_log_iter_test: db/db_log_iter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1001,6 +1021,9 @@ compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
 options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1043,6 +1066,9 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)

+repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 ldb_cmd_test: tools/ldb_cmd_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -1214,7 +1240,7 @@ jdb_bench:

 commit_prereq: build_tools/rocksdb-lego-determinator \
               build_tools/precommit_checker.py
-	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan lite
+	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan ubsan lite
 	$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;

 xfunc:
@@ -52,3 +52,7 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub

 ## VWO, Wingify
 [VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
+
+## quasardb
+[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
+quasardb uses a heavily tuned RocksDB as its persistence layer.
@@ -52,12 +52,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
    FBCODE_BUILD="true"
    # If we're compiling with TSAN we need pic build
    PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      source "$PWD/build_tools/fbcode_config.sh"
-    else
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    fi
+    source "$PWD/build_tools/fbcode_config.sh"
 fi

 # Delete existing output, if it exists
@@ -1,16 +1,19 @@
-GCC_BASE=/mnt/vol/engshare/fbcode/third-party2/gcc/4.9.x/centos6-native/1317bc4/
-CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ea2fd1278810d3af2ea52218d2767e09d786dbd0/4.9.x/gcc-4.9-glibc-2.20/024dbc3
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.20/gcc-4.9-glibc-2.20/500e281
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.9-glibc-2.20/e9936bf
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.9-glibc-2.20/e9936bf
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.9-glibc-2.20/e9936bf
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/cb6c4880fcb4fee471574ba6af63a3882155a16a/0.5.1/gcc-4.9-glibc-2.20/e9936bf
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/40791a3fef9206a77f2c4bc51f8169e5bf10d68e/master/gcc-4.9-glibc-2.20/a6c5e1e
-NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.9-glibc-2.20/e9936bf
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/303048f72efc92ae079e62dfc84823401aecfd94/trunk/gcc-4.9-glibc-2.20/12266b1
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.9-glibc-2.20/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/a5b8152b2a15ce8a98808cf954fbccec825a97bc/2.25/centos6-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.10.0/gcc-4.9-glibc-2.20/e9936bf
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
+NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
@@ -1,6 +1,6 @@
-GCC_BASE=/mnt/vol/engshare/fbcode/third-party2/gcc/4.8.1/centos6-native/cc6c9dc/
+GCC_BASE=/mnt/gvfs/third-party2/gcc/ebc96bc2fb751b5a0300b8d91a95bdf24ac1d88b/4.8.1/centos6-native/cc6c9dc
 CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ea2fd1278810d3af2ea52218d2767e09d786dbd0/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/f97108c12512b3b0789ac4515d836bdb1eae1142/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
 GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.17/gcc-4.8.1-glibc-2.17/99df8fc
 SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
 ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
@@ -8,9 +8,9 @@ BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a
 LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.8.1-glibc-2.17/c3f970a
 ZSTD_BASE=/mnt/gvfs/third-party2/zstd/cb6c4880fcb4fee471574ba6af63a3882155a16a/0.5.1/gcc-4.8.1-glibc-2.17/c3f970a
 GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/40791a3fef9206a77f2c4bc51f8169e5bf10d68e/master/gcc-4.8.1-glibc-2.17/8d31e51
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9acac8a9a2c9378eb696e22ffa8bd0e97d9206b/master/gcc-4.8.1-glibc-2.17/8d31e51
 NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/303048f72efc92ae079e62dfc84823401aecfd94/trunk/gcc-4.8.1-glibc-2.17/675d945
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/41bfa3759ce52c071f5fd547ec9ecd2522929f0a/trunk/gcc-4.8.1-glibc-2.17/675d945
 KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/a5b8152b2a15ce8a98808cf954fbccec825a97bc/2.25/centos6-native/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/71454c53fffcb716a0beb9a90047aff7fb5c984a/2.26/centos6-native/da39a3e
 VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
@@ -13,12 +13,12 @@ source "$BASEDIR/dependencies.sh"
 CFLAGS=""

 # libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"

 # glibc
 GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIB_BASE/lib"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"

 # snappy
 SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
@@ -43,12 +43,16 @@ if test -z $PIC_BUILD; then
  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
  CFLAGS+=" -DLZ4"
-
-  ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-  CFLAGS+=" -DZSTD"
 fi

+ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+if test -z $PIC_BUILD; then
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+else
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+fi
+CFLAGS+=" -DZSTD"
+
 # location of gflags headers and libraries
 GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
 if test -z $PIC_BUILD; then
@@ -56,7 +60,7 @@ if test -z $PIC_BUILD; then
 else
  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
 fi
-CFLAGS+=" -DGFLAGS=google"
+CFLAGS+=" -DGFLAGS=gflags"

 # location of jemalloc
 JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
@@ -72,13 +76,22 @@ if test -z $PIC_BUILD; then
  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
 fi

+# location of TBB
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+if test -z $PIC_BUILD; then
+  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+else
+  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
+fi
+CFLAGS+=" -DTBB"
+
 # use Intel SSE support for checksum calculations
 export USE_SSE=1

 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"

-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"

 STDLIBS="-L $GCC_BASE/lib64"

@@ -87,7 +100,7 @@ CLANG_LIB="$CLANG_BASE/lib"
 CLANG_SRC="$CLANG_BASE/../../src"

 CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/clang/tools/scan-build/scan-build"
+CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"

 if [ -z "$USE_CLANG" ]; then
  # gcc
@@ -95,40 +108,45 @@ if [ -z "$USE_CLANG" ]; then
  CXX="$GCC_BASE/bin/g++"

  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  JEMALLOC=1
 else
  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
+  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
  CC="$CLANG_BIN/clang"
  CXX="$CLANG_BIN/clang++"

  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"

  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
  CFLAGS+=" -isystem $CLANG_INCLUDE"
  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
+  CFLAGS+=" -Wno-expansion-to-defined "
  CXXFLAGS="-nostdinc++"
 fi

 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"

-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.9-glibc-2.20/lib"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"

 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"

-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"

 VALGRIND_VER="$VALGRIND_BASE/bin/"

-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD 
@@ -184,7 +184,7 @@ parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')

 # <test ....>
 parser.add_argument('test', nargs='+',
-                    help='CI test(s) to run. e.g: unit punit asan tsan')
+                    help='CI test(s) to run. e.g: unit punit asan tsan ubsan')

 print("Please follow log %s" % Log.LOG_FILE)

@@ -67,6 +67,7 @@ ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
 LITE="OPT=-DROCKSDB_LITE"
 TSAN="COMPILE_WITH_TSAN=1"
+UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 PARSER="'parser':'egrep \'Failure|^#|Abort|Expected|Actual|GoogleTestFailure|^==\''"

@@ -376,6 +377,48 @@ ASAN_CRASH_TEST_COMMANDS="[
    }
 ]"

+#
+# RocksDB test under undefined behavior sanitizer
+#
+UBSAN_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under UBSAN',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Test RocksDB debug under UBSAN',
+                'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check',
+                'user':'root',
+                $PARSER
+            }
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB crash testing under udnefined behavior sanitizer
+#
+UBSAN_CRASH_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb crash test under UBSAN',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and run RocksDB debug ubsan_crash_test',
+                'timeout': 86400,
+                'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
 #
 # RocksDB unit test under valgrind
 #
@@ -462,9 +505,9 @@ run_format_compatible()
  if [ -e "build_detect_platform" ]
  then
    sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
-    rm -rf build_detect_platform 
+    rm -rf build_detect_platform
    cp $TEST_TMPDIR/temp_build_file build_detect_platform
-    chmod +x build_detect_platform 
+    chmod +x build_detect_platform
  fi

  make ldb -j32
@@ -632,6 +675,12 @@ case $1 in
  asan_crash)
    echo $ASAN_CRASH_TEST_COMMANDS
    ;;
+  ubsan)
+    echo $UBSAN_TEST_COMMANDS
+    ;;
+  ubsan_crash)
+    echo $UBSAN_CRASH_TEST_COMMANDS
+    ;;
  valgrind)
    echo $VALGRIND_TEST_COMMANDS
    ;;
@@ -9,7 +9,7 @@ Param(
  [string]$WorkFolder = "",  # Direct tests to use that folder
  [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
  [string]$Exclude = "", # Expect a comma separated list, no spaces
-  [string]$Run = "db_test",  # Run db_test|tests|testname1,testname2...
+  [string]$Run = "db_test",  # Run db_test|db_test2|tests|testname1,testname2...
   # Number of async tasks that would run concurrently. Recommend a number below 64.
   # However, CPU utlization really depends on the storage media. Recommend ram based disk.
   # a value of 1 will run everything serially
@@ -46,15 +46,25 @@ if($WorkFolder -eq "") {
  $Env:TEST_TMPDIR = $WorkFolder
 }

+Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
+
 # Use JEMALLOC executables
-if($EnableJE) {
-    $db_test = -Join ($BinariesFolder, "db_test_je.exe")
-} else {
-    $db_test = -Join ($BinariesFolder, "db_test.exe")
+if($Run -ceq "db_test" -or
+   $Run -ceq "db_test2" ) {
+
+   $file_name = $Run
+
+   if($EnableJE) {
+     $file_name += "_je"
+   }
+
+   $file_name += ".exe"
+
+   $db_test = -Join ($BinariesFolder, $file_name)
+
+   Write-Output "Binaries: $BinariesFolder db_test: $db_test"
 }

-Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-Write-Output "Binaries: $BinariesFolder db_test: $db_test"

 #Exclusions that we do not want to run
 $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
@@ -153,7 +163,8 @@ function Discover-TestBinaries([string]$Pattern, $HashTable) {

 $TestsToRun = [ordered]@{}

-if($Run -ceq "db_test") {
+if($Run -ceq "db_test" -or
+   $Run -ceq "db_test2") {
    Normalize-DbTests -HashTable $TestsToRun
 } elseif($Run -ceq "tests") {
    if($EnableJE) {
@@ -225,7 +236,8 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
            Write-Host "Starting $k"
            $log_path = ($TestToLog.$k)

-            if($Run -ceq "db_test") {
+            if($Run -ceq "db_test" -or
+               $Run -ceq "db_test2") {
              $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
            } else {
              [string]$Exe =  -Join ($BinariesFolder, $k)
@@ -64,7 +64,7 @@ touch "$OUTPUT"
 echo "Writing dependencies to $OUTPUT"

 # Compilers locations
-GCC_BASE=`ls -d1 $TP2_LATEST/gcc/4.9.x/centos6-native/*/ | head -n1`
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
 CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`

 log_variable GCC_BASE
@@ -101,7 +101,7 @@ touch "$OUTPUT"
 echo "Writing 4.8.1 dependencies to $OUTPUT"

 # Compilers locations
-GCC_BASE=`ls -d1 $TP2_LATEST/gcc/4.8.1/centos6-native/*/ | head -n1`
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
 CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`

 log_variable GCC_BASE
@@ -41,13 +41,17 @@ TableBuilder* NewTableBuilder(
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, WritableFileWriter* file,
-    const CompressionType compression_type,
+    uint32_t column_family_id, const std::string& column_family_name,
+    WritableFileWriter* file, const CompressionType compression_type,
    const CompressionOptions& compression_opts, const bool skip_filters) {
+  assert((column_family_id ==
+          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
+         column_family_name.empty());
  return ioptions.table_factory->NewTableBuilder(
      TableBuilderOptions(ioptions, internal_comparator,
                          int_tbl_prop_collector_factories, compression_type,
-                          compression_opts, skip_filters),
+                          compression_opts, skip_filters,
+                          column_family_name),
      column_family_id, file);
 }

@@ -58,12 +62,16 @@ Status BuildTable(
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, const std::string& column_family_name,
+    std::vector<SequenceNumber> snapshots,
    SequenceNumber earliest_write_conflict_snapshot,
    const CompressionType compression,
    const CompressionOptions& compression_opts, bool paranoid_file_checks,
    InternalStats* internal_stats, const Env::IOPriority io_priority,
-    TableProperties* table_properties) {
+    TableProperties* table_properties, int level) {
+  assert((column_family_id ==
+          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
+         column_family_name.empty());
  // Reports the IOStats for flush for every following bytes.
  const size_t kReportFlushIOStatsEvery = 1048576;
  Status s;
@@ -87,7 +95,8 @@ Status BuildTable(

      builder = NewTableBuilder(
          ioptions, internal_comparator, int_tbl_prop_collector_factories,
-          column_family_id, file_writer.get(), compression, compression_opts);
+          column_family_id, column_family_name, file_writer.get(), compression,
+          compression_opts);
    }

    MergeHelper merge(env, internal_comparator.user_comparator(),
@@ -149,7 +158,8 @@ Status BuildTable(
          ReadOptions(), env_options, internal_comparator, meta->fd, nullptr,
          (internal_stats == nullptr) ? nullptr
                                      : internal_stats->GetFileReadHist(0),
-          false));
+          false /* for_compaction */, nullptr /* arena */,
+          false /* skip_filter */, level));
      s = it->status();
      if (s.ok() && paranoid_file_checks) {
        for (it->SeekToFirst(); it->Valid(); it->Next()) {
@@ -33,13 +33,16 @@ class WritableFileWriter;
 class InternalStats;
 class InternalIterator;

+// @param column_family_name Name of the column family that is also identified
+//    by column_family_id, or empty string if unknown. It must outlive the
+//    TableBuilder returned by this function.
 TableBuilder* NewTableBuilder(
    const ImmutableCFOptions& options,
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, WritableFileWriter* file,
-    const CompressionType compression_type,
+    uint32_t column_family_id, const std::string& column_family_name,
+    WritableFileWriter* file, const CompressionType compression_type,
    const CompressionOptions& compression_opts,
    const bool skip_filters = false);

@@ -48,6 +51,9 @@ TableBuilder* NewTableBuilder(
 // *meta will be filled with metadata about the generated table.
 // If no data is present in *iter, meta->file_size will be set to
 // zero, and no Table file will be produced.
+//
+// @param column_family_name Name of the column family that is also identified
+//    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
    const std::string& dbname, Env* env, const ImmutableCFOptions& options,
    const EnvOptions& env_options, TableCache* table_cache,
@@ -55,12 +61,13 @@ extern Status BuildTable(
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, const std::string& column_family_name,
+    std::vector<SequenceNumber> snapshots,
    SequenceNumber earliest_write_conflict_snapshot,
    const CompressionType compression,
    const CompressionOptions& compression_opts, bool paranoid_file_checks,
    InternalStats* internal_stats,
    const Env::IOPriority io_priority = Env::IO_HIGH,
-    TableProperties* table_properties = nullptr);
+    TableProperties* table_properties = nullptr, int level = -1);

 }  // namespace rocksdb
@@ -1288,6 +1288,11 @@ void rocksdb_block_based_options_set_cache_index_and_filter_blocks(
  options->rep.cache_index_and_filter_blocks = v;
 }

+void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
+    rocksdb_block_based_table_options_t* options, unsigned char v) {
+  options->rep.pin_l0_filter_and_index_blocks_in_cache = v;
+}
+
 void rocksdb_block_based_options_set_skip_table_builder_flush(
    rocksdb_block_based_table_options_t* options, unsigned char v) {
  options->rep.skip_table_builder_flush = v;
@@ -147,13 +147,10 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
                                    const ColumnFamilyOptions& src) {
  ColumnFamilyOptions result = src;
  result.comparator = icmp;
-#ifdef OS_MACOSX
-  // TODO(icanadi) make write_buffer_size uint64_t instead of size_t
-  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, ((size_t)1) << 30);
-#else
-  ClipToRange(&result.write_buffer_size,
-              ((size_t)64) << 10, ((size_t)64) << 30);
-#endif
+  size_t clamp_max = std::conditional<
+      sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
+      std::integral_constant<size_t, 64ull << 30>>::type::value;
+  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, clamp_max);
  // if user sets arena_block_size, we trust user to use this value. Otherwise,
  // calculate a proper value from writer_buffer_size;
  if (result.arena_block_size <= 0) {
@@ -465,6 +465,8 @@ class ColumnFamilySet {
  // Don't call while iterating over ColumnFamilySet
  void FreeDeadColumnFamilies();

+  Cache* get_table_cache() { return table_cache_; }
+
 private:
  friend class ColumnFamilyData;
  // helper function that gets called from cfd destructor
@@ -162,9 +162,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
      deletion_compaction_(_deletion_compaction),
      inputs_(std::move(_inputs)),
      grandparents_(std::move(_grandparents)),
-      grandparent_index_(0),
-      seen_key_(false),
-      overlapped_bytes_(0),
      score_(_score),
      bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
      is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
@@ -290,32 +287,6 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
  return true;
 }

-bool Compaction::ShouldStopBefore(const Slice& internal_key) {
-  // Scan to find earliest grandparent file that contains key.
-  const InternalKeyComparator* icmp = &cfd_->internal_comparator();
-  while (grandparent_index_ < grandparents_.size() &&
-      icmp->Compare(internal_key,
-                    grandparents_[grandparent_index_]->largest.Encode()) > 0) {
-    if (seen_key_) {
-      overlapped_bytes_ += grandparents_[grandparent_index_]->fd.GetFileSize();
-    }
-    assert(grandparent_index_ + 1 >= grandparents_.size() ||
-           icmp->Compare(grandparents_[grandparent_index_]->largest.Encode(),
-                         grandparents_[grandparent_index_+1]->smallest.Encode())
-                         < 0);
-    grandparent_index_++;
-  }
-  seen_key_ = true;
-
-  if (overlapped_bytes_ > max_grandparent_overlap_bytes_) {
-    // Too much overlap for current output; start new output
-    overlapped_bytes_ = 0;
-    return true;
-  } else {
-    return false;
-  }
-}
-
 // Mark (or clear) each file that is being compacted
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
  for (size_t i = 0; i < num_input_levels(); i++) {
@@ -421,7 +392,7 @@ void Compaction::Summary(char* output, int len) {
  snprintf(output + write, len - write, "]");
 }

-uint64_t Compaction::OutputFilePreallocationSize() {
+uint64_t Compaction::OutputFilePreallocationSize() const {
  uint64_t preallocation_size = 0;

  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
@@ -104,7 +104,7 @@ class Compaction {
  }

  // Returns the LevelFilesBrief of the specified compaction input level.
-  LevelFilesBrief* input_levels(size_t compaction_input_level) {
+  const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
    return &input_levels_[compaction_input_level];
  }

@@ -132,10 +132,6 @@ class Compaction {
  bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
                                     std::vector<size_t>* level_ptrs) const;

-  // Returns true iff we should stop building the current output
-  // before processing "internal_key".
-  bool ShouldStopBefore(const Slice& internal_key);
-
  // Clear all files to indicate that they are not being compacted
  // Delete this compaction from the list of running compactions.
  //
@@ -151,13 +147,13 @@ class Compaction {
  double score() const { return score_; }

  // Is this compaction creating a file in the bottom most level?
-  bool bottommost_level() { return bottommost_level_; }
+  bool bottommost_level() const { return bottommost_level_; }

  // Does this compaction include all sst files?
-  bool is_full_compaction() { return is_full_compaction_; }
+  bool is_full_compaction() const { return is_full_compaction_; }

  // Was this compaction triggered manually by the client?
-  bool is_manual_compaction() { return is_manual_compaction_; }
+  bool is_manual_compaction() const { return is_manual_compaction_; }

  // Used when allow_trivial_move option is set in
  // Universal compaction. If all the input files are
@@ -170,19 +166,21 @@ class Compaction {
  // Used when allow_trivial_move option is set in
  // Universal compaction. Returns true, if the input files
  // are non-overlapping and can be trivially moved.
-  bool is_trivial_move() { return is_trivial_move_; }
+  bool is_trivial_move() const { return is_trivial_move_; }

  // How many total levels are there?
  int number_levels() const { return number_levels_; }

  // Return the MutableCFOptions that should be used throughout the compaction
  // procedure
-  const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
+  const MutableCFOptions* mutable_cf_options() const {
+    return &mutable_cf_options_;
+  }

  // Returns the size in bytes that the output file should be preallocated to.
  // In level compaction, that is max_file_size_. In universal compaction, that
  // is the sum of all input file sizes.
-  uint64_t OutputFilePreallocationSize();
+  uint64_t OutputFilePreallocationSize() const;

  void SetInputVersion(Version* input_version);

@@ -225,6 +223,14 @@ class Compaction {

  CompactionReason compaction_reason() { return compaction_reason_; }

+  const std::vector<FileMetaData*>& grandparents() const {
+    return grandparents_;
+  }
+
+  uint64_t max_grandparent_overlap_bytes() const {
+    return max_grandparent_overlap_bytes_;
+  }
+
 private:
  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool mark_as_compacted);
@@ -268,10 +274,6 @@ class Compaction {
  // State used to check for number of of overlapping grandparent files
  // (grandparent == "output_level_ + 1")
  std::vector<FileMetaData*> grandparents_;
-  size_t grandparent_index_;   // Index in grandparent_starts_
-  bool seen_key_;              // Some output key has been seen
-  uint64_t overlapped_bytes_;  // Bytes of overlap between current output
-                               // and grandparent files
  const double score_;         // score that was used to pick this compaction.

  // Is this compaction creating a file in the bottom most level?
@@ -14,7 +14,7 @@ CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
    SequenceNumber earliest_write_conflict_snapshot, Env* env,
-    bool expect_valid_internal_key, Compaction* compaction,
+    bool expect_valid_internal_key, const Compaction* compaction,
    const CompactionFilter* compaction_filter, LogBuffer* log_buffer)
    : input_(input),
      cmp_(cmp),
@@ -42,7 +42,7 @@ class CompactionIterator {
                     std::vector<SequenceNumber>* snapshots,
                     SequenceNumber earliest_write_conflict_snapshot, Env* env,
                     bool expect_valid_internal_key,
-                     Compaction* compaction = nullptr,
+                     const Compaction* compaction = nullptr,
                     const CompactionFilter* compaction_filter = nullptr,
                     LogBuffer* log_buffer = nullptr);

@@ -92,7 +92,7 @@ class CompactionIterator {
  const SequenceNumber earliest_write_conflict_snapshot_;
  Env* env_;
  bool expect_valid_internal_key_;
-  Compaction* compaction_;
+  const Compaction* compaction_;
  const CompactionFilter* compaction_filter_;
  LogBuffer* log_buffer_;
  bool bottommost_level_;
@@ -63,7 +63,7 @@ namespace rocksdb {

 // Maintains state for each sub-compaction
 struct CompactionJob::SubcompactionState {
-  Compaction* compaction;
+  const Compaction* compaction;
  std::unique_ptr<CompactionIterator> c_iter;

  // The boundaries of the key-range this compaction is interested in. No two
@@ -104,6 +104,14 @@ struct CompactionJob::SubcompactionState {
  uint64_t num_output_records;
  CompactionJobStats compaction_job_stats;
  uint64_t approx_size;
+  // An index that used to speed up Compaction::ShouldStopBefore().
+  size_t grandparent_index = 0;
+  // The number of bytes overlapping between the current output and
+  // grandparent files used in Compaction::ShouldStopBefore().
+  uint64_t overlapped_bytes = 0;
+  // A flag determine whether the key has been seen in
+  // Compaction::ShouldStopBefore()
+  bool seen_key = false;

  SubcompactionState(Compaction* c, Slice* _start, Slice* _end,
                     uint64_t size = 0)
@@ -115,7 +123,10 @@ struct CompactionJob::SubcompactionState {
        total_bytes(0),
        num_input_records(0),
        num_output_records(0),
-        approx_size(size) {
+        approx_size(size),
+        grandparent_index(0),
+        overlapped_bytes(0),
+        seen_key(false) {
    assert(compaction != nullptr);
  }

@@ -134,6 +145,9 @@ struct CompactionJob::SubcompactionState {
    num_output_records = std::move(o.num_output_records);
    compaction_job_stats = std::move(o.compaction_job_stats);
    approx_size = std::move(o.approx_size);
+    grandparent_index = std::move(o.grandparent_index);
+    overlapped_bytes = std::move(o.overlapped_bytes);
+    seen_key = std::move(o.seen_key);
    return *this;
  }

@@ -141,6 +155,38 @@ struct CompactionJob::SubcompactionState {
  SubcompactionState(const SubcompactionState&) = delete;

  SubcompactionState& operator=(const SubcompactionState&) = delete;
+
+  // Returns true iff we should stop building the current output
+  // before processing "internal_key".
+  bool ShouldStopBefore(const Slice& internal_key) {
+    const InternalKeyComparator* icmp =
+        &compaction->column_family_data()->internal_comparator();
+    const std::vector<FileMetaData*>& grandparents = compaction->grandparents();
+
+    // Scan to find earliest grandparent file that contains key.
+    while (grandparent_index < grandparents.size() &&
+           icmp->Compare(internal_key,
+                         grandparents[grandparent_index]->largest.Encode()) >
+               0) {
+      if (seen_key) {
+        overlapped_bytes += grandparents[grandparent_index]->fd.GetFileSize();
+      }
+      assert(grandparent_index + 1 >= grandparents.size() ||
+             icmp->Compare(
+                 grandparents[grandparent_index]->largest.Encode(),
+                 grandparents[grandparent_index + 1]->smallest.Encode()) < 0);
+      grandparent_index++;
+    }
+    seen_key = true;
+
+    if (overlapped_bytes > compaction->max_grandparent_overlap_bytes()) {
+      // Too much overlap for current output; start new output
+      overlapped_bytes = 0;
+      return true;
+    }
+
+    return false;
+  }
 };

 // Maintains state for the entire compaction
@@ -669,7 +715,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    if (end != nullptr &&
        cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
      break;
-    } else if (sub_compact->compaction->ShouldStopBefore(key) &&
+    } else if (sub_compact->ShouldStopBefore(key) &&
               sub_compact->builder != nullptr) {
      status = FinishCompactionOutputFile(input->status(), sub_compact);
      if (!status.ok()) {
@@ -973,7 +1019,7 @@ Status CompactionJob::OpenCompactionOutputFile(
      cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
  sub_compact->builder.reset(NewTableBuilder(
      *cfd->ioptions(), cfd->internal_comparator(),
-      cfd->int_tbl_prop_collector_factories(), cfd->GetID(),
+      cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
      sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
      cfd->ioptions()->compression_opts, skip_filters));
  LogFlush(db_options_.info_log);
@@ -660,7 +660,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
  options.max_subcompactions = max_subcompactions_;
  options.bytes_per_sync = 512 * 1024;

-  options.compaction_measure_io_stats = true;
+  options.report_bg_io_stats = true;
  for (int test = 0; test < 2; ++test) {
    DestroyAndReopen(options);
    CreateAndReopenWithCF({"pikachu"}, options);
@@ -195,6 +195,7 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
  NewVersionStorage(6, kCompactionStyleLevel);
  mutable_cf_options_.target_file_size_base = 10000000;
  mutable_cf_options_.target_file_size_multiplier = 10;
+  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
  Add(0, 1U, "150", "200", 1000000000U);
  // Level 1 score 1.2
  Add(1, 66U, "150", "200", 6000000U);
@@ -491,6 +492,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
  NewVersionStorage(6, kCompactionStyleLevel);
  mutable_cf_options_.target_file_size_base = 10000000;
  mutable_cf_options_.target_file_size_multiplier = 10;
+  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;

  Add(2, 6U, "150", "179", 50000000U);
@@ -517,6 +519,7 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
  NewVersionStorage(6, kCompactionStyleLevel);
  mutable_cf_options_.target_file_size_base = 10000000;
  mutable_cf_options_.target_file_size_multiplier = 10;
+  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;

  Add(2, 6U, "150", "175",
@@ -544,20 +547,21 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {

 TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
  NewVersionStorage(6, kCompactionStyleLevel);
-  mutable_cf_options_.target_file_size_base = 10000000;
-  mutable_cf_options_.target_file_size_multiplier = 10;
+  mutable_cf_options_.max_bytes_for_level_base = 10000000;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;

  // file 7 and 8 over lap with the same file, but file 8 is smaller so
  // it will be picked.
-  Add(2, 6U, "150", "175", 60000000U);  // Overlaps with file 26, 27
-  Add(2, 7U, "176", "200", 60000000U);  // Overlaps with file 27
-  Add(2, 8U, "201", "300", 61000000U);  // Overlaps with file 27
+  Add(2, 6U, "150", "167", 60000000U);  // Overlaps with file 26, 27
+  Add(2, 7U, "168", "169", 60000000U);  // Overlaps with file 27
+  Add(2, 8U, "201", "300", 61000000U);  // Overlaps with file 28, but the file
+                                        // itself is larger. Should be picked.

  Add(3, 26U, "160", "165", 260000000U);
-  Add(3, 26U, "166", "170", 260000000U);
-  Add(3, 27U, "180", "400", 260000000U);
-  Add(3, 28U, "401", "500", 260000000U);
+  Add(3, 27U, "166", "170", 260000000U);
+  Add(3, 28U, "180", "400", 260000000U);
+  Add(3, 29U, "401", "500", 260000000U);
  UpdateVersionStorageInfo();

  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
@@ -0,0 +1,240 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include <cstdlib>
+#include "db/db_test_util.h"
+#include "port/stack_trace.h"
+
+namespace rocksdb {
+
+static uint64_t TestGetTickerCount(const Options& options,
+                                   Tickers ticker_type) {
+  return options.statistics->getTickerCount(ticker_type);
+}
+
+class DBBlockCacheTest : public DBTestBase {
+ private:
+  size_t miss_count_ = 0;
+  size_t hit_count_ = 0;
+  size_t insert_count_ = 0;
+  size_t failure_count_ = 0;
+  size_t compressed_miss_count_ = 0;
+  size_t compressed_hit_count_ = 0;
+  size_t compressed_insert_count_ = 0;
+  size_t compressed_failure_count_ = 0;
+
+ public:
+  const size_t kNumBlocks = 10;
+  const size_t kValueSize = 100;
+
+  DBBlockCacheTest() : DBTestBase("/db_block_cache_test") {}
+
+  BlockBasedTableOptions GetTableOptions() {
+    BlockBasedTableOptions table_options;
+    // Set a small enough block size so that each key-value get its own block.
+    table_options.block_size = 1;
+    return table_options;
+  }
+
+  Options GetOptions(const BlockBasedTableOptions& table_options) {
+    Options options = CurrentOptions();
+    options.create_if_missing = true;
+    // options.compression = kNoCompression;
+    options.statistics = rocksdb::CreateDBStatistics();
+    options.table_factory.reset(new BlockBasedTableFactory(table_options));
+    return options;
+  }
+
+  void InitTable(const Options& options) {
+    std::string value(kValueSize, 'a');
+    for (size_t i = 0; i < kNumBlocks; i++) {
+      ASSERT_OK(Put(ToString(i), value.c_str()));
+    }
+  }
+
+  void RecordCacheCounters(const Options& options) {
+    miss_count_ = TestGetTickerCount(options, BLOCK_CACHE_MISS);
+    hit_count_ = TestGetTickerCount(options, BLOCK_CACHE_HIT);
+    insert_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD);
+    failure_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
+    compressed_miss_count_ =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS);
+    compressed_hit_count_ =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT);
+    compressed_insert_count_ =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD);
+    compressed_failure_count_ =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES);
+  }
+
+  void CheckCacheCounters(const Options& options, size_t expected_misses,
+                          size_t expected_hits, size_t expected_inserts,
+                          size_t expected_failures) {
+    size_t new_miss_count = TestGetTickerCount(options, BLOCK_CACHE_MISS);
+    size_t new_hit_count = TestGetTickerCount(options, BLOCK_CACHE_HIT);
+    size_t new_insert_count = TestGetTickerCount(options, BLOCK_CACHE_ADD);
+    size_t new_failure_count =
+        TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
+    ASSERT_EQ(miss_count_ + expected_misses, new_miss_count);
+    ASSERT_EQ(hit_count_ + expected_hits, new_hit_count);
+    ASSERT_EQ(insert_count_ + expected_inserts, new_insert_count);
+    ASSERT_EQ(failure_count_ + expected_failures, new_failure_count);
+    miss_count_ = new_miss_count;
+    hit_count_ = new_hit_count;
+    insert_count_ = new_insert_count;
+    failure_count_ = new_failure_count;
+  }
+
+  void CheckCompressedCacheCounters(const Options& options,
+                                    size_t expected_misses,
+                                    size_t expected_hits,
+                                    size_t expected_inserts,
+                                    size_t expected_failures) {
+    size_t new_miss_count =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS);
+    size_t new_hit_count =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT);
+    size_t new_insert_count =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD);
+    size_t new_failure_count =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES);
+    ASSERT_EQ(compressed_miss_count_ + expected_misses, new_miss_count);
+    ASSERT_EQ(compressed_hit_count_ + expected_hits, new_hit_count);
+    ASSERT_EQ(compressed_insert_count_ + expected_inserts, new_insert_count);
+    ASSERT_EQ(compressed_failure_count_ + expected_failures, new_failure_count);
+    compressed_miss_count_ = new_miss_count;
+    compressed_hit_count_ = new_hit_count;
+    compressed_insert_count_ = new_insert_count;
+    compressed_failure_count_ = new_failure_count;
+  }
+};
+
+TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
+  ReadOptions read_options;
+  auto table_options = GetTableOptions();
+  auto options = GetOptions(table_options);
+  InitTable(options);
+
+  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
+  table_options.block_cache = cache;
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  Reopen(options);
+  RecordCacheCounters(options);
+
+  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
+  Iterator* iter = nullptr;
+
+  // Load blocks into cache.
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    iter = db_->NewIterator(read_options);
+    iter->Seek(ToString(i));
+    ASSERT_OK(iter->status());
+    CheckCacheCounters(options, 1, 0, 1, 0);
+    iterators[i].reset(iter);
+  }
+  size_t usage = cache->GetUsage();
+  ASSERT_LT(0, usage);
+  cache->SetCapacity(usage);
+  ASSERT_EQ(usage, cache->GetPinnedUsage());
+
+  // Test with strict capacity limit.
+  cache->SetStrictCapacityLimit(true);
+  iter = db_->NewIterator(read_options);
+  iter->Seek(ToString(kNumBlocks - 1));
+  ASSERT_TRUE(iter->status().IsIncomplete());
+  CheckCacheCounters(options, 1, 0, 0, 1);
+  delete iter;
+  iter = nullptr;
+
+  // Release interators and access cache again.
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    iterators[i].reset();
+    CheckCacheCounters(options, 0, 0, 0, 0);
+  }
+  ASSERT_EQ(0, cache->GetPinnedUsage());
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    iter = db_->NewIterator(read_options);
+    iter->Seek(ToString(i));
+    ASSERT_OK(iter->status());
+    CheckCacheCounters(options, 0, 1, 0, 0);
+    iterators[i].reset(iter);
+  }
+}
+
+#ifdef SNAPPY
+TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
+  ReadOptions read_options;
+  auto table_options = GetTableOptions();
+  auto options = GetOptions(table_options);
+  options.compression = CompressionType::kSnappyCompression;
+  InitTable(options);
+
+  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
+  std::shared_ptr<Cache> compressed_cache = NewLRUCache(0, 0, false);
+  table_options.block_cache = cache;
+  table_options.block_cache_compressed = compressed_cache;
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  Reopen(options);
+  RecordCacheCounters(options);
+
+  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
+  Iterator* iter = nullptr;
+
+  // Load blocks into cache.
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    iter = db_->NewIterator(read_options);
+    iter->Seek(ToString(i));
+    ASSERT_OK(iter->status());
+    CheckCacheCounters(options, 1, 0, 1, 0);
+    CheckCompressedCacheCounters(options, 1, 0, 1, 0);
+    iterators[i].reset(iter);
+  }
+  size_t usage = cache->GetUsage();
+  ASSERT_LT(0, usage);
+  ASSERT_EQ(usage, cache->GetPinnedUsage());
+  size_t compressed_usage = compressed_cache->GetUsage();
+  ASSERT_LT(0, compressed_usage);
+  // Compressed block cache cannot be pinned.
+  ASSERT_EQ(0, compressed_cache->GetPinnedUsage());
+
+  // Set strict capacity limit flag. Now block will only load into compressed
+  // block cache.
+  cache->SetCapacity(usage);
+  cache->SetStrictCapacityLimit(true);
+  ASSERT_EQ(usage, cache->GetPinnedUsage());
+  // compressed_cache->SetCapacity(compressed_usage);
+  compressed_cache->SetCapacity(0);
+  // compressed_cache->SetStrictCapacityLimit(true);
+  iter = db_->NewIterator(read_options);
+  iter->Seek(ToString(kNumBlocks - 1));
+  ASSERT_TRUE(iter->status().IsIncomplete());
+  CheckCacheCounters(options, 1, 0, 0, 1);
+  CheckCompressedCacheCounters(options, 1, 0, 1, 0);
+  delete iter;
+  iter = nullptr;
+
+  // Clear strict capacity limit flag. This time we shall hit compressed block
+  // cache.
+  cache->SetStrictCapacityLimit(false);
+  iter = db_->NewIterator(read_options);
+  iter->Seek(ToString(kNumBlocks - 1));
+  ASSERT_OK(iter->status());
+  CheckCacheCounters(options, 1, 0, 1, 0);
+  CheckCompressedCacheCounters(options, 0, 1, 0, 0);
+  delete iter;
+  iter = nullptr;
+}
+#endif
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  rocksdb::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -368,11 +368,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
 // entries are deleted. The compaction should create bunch of 'DeleteFile'
 // entries in VersionEdit, but none of the 'AddFile's.
 TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // put some data
@@ -400,11 +399,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {

 TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.num_levels = 3;
    options.compaction_filter_factory =
      std::make_shared<ChangeFilterFactory>();
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Write 100K+1 keys, these are written to a few files
@@ -467,8 +465,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
  PutFixed64(&three, 3);
  PutFixed64(&four, 4);

-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();
  options.create_if_missing = true;
  options.merge_operator = MergeOperators::CreateUInt64AddOperator();
  options.num_levels = 3;
@@ -621,11 +618,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) {
 // Compaction filters should only be applied to records that are newer than the
 // latest snapshot. This test inserts records and applies a delete filter.
 TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // Put some data.
@@ -659,11 +655,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
 // records newer than the snapshot will also be processed
 TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
  std::string five = ToString(5);
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>();
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // Put some data.
@@ -288,8 +288,7 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
 }

 TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();
  options.env = env_;
  options.new_table_reader_for_compaction_inputs = true;
  options.max_open_files = 100;
@@ -493,14 +492,13 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
 TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
  const int kNumKeysPerFile = 100;

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 110 << 10;  // 110KB
  options.arena_block_size = 4 << 10;
  options.num_levels = 3;
  options.level0_file_num_compaction_trigger = 3;
  options.max_subcompactions = max_subcompactions_;
  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  Random rnd(301);
@@ -538,7 +536,7 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
  // and see number of compactions scheduled to be less than allowed.
  const int kNumKeysPerFile = 100;

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 110 << 10;  // 110KB
  options.arena_block_size = 4 << 10;
  options.num_levels = 3;
@@ -549,7 +547,6 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
  options.base_background_compactions = 1;
  options.max_background_compactions = 3;
  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-  options = CurrentOptions(options);

  // Block all threads in thread pool.
  const size_t kTotalTasks = 4;
@@ -632,10 +629,9 @@ TEST_F(DBCompactionTest, BGCompactionsAllowed) {
 }

 TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;        // Large write buffer
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  Random rnd(301);
@@ -662,9 +658,8 @@ TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {

 TEST_F(DBCompactionTest, MinorCompactionsHappen) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.write_buffer_size = 10000;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    const int N = 500;
@@ -689,7 +684,7 @@ TEST_F(DBCompactionTest, MinorCompactionsHappen) {
 }

 TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleLevel;
  options.level0_file_num_compaction_trigger = 3;

@@ -703,7 +698,6 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
  const size_t key_len =
    static_cast<size_t>(compact_opt.output_file_size_limit) / 5;

-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  std::vector<const Snapshot*> snaps;
@@ -747,10 +741,8 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
 // if the database is shutdown during the memtable compaction.
 TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.env = env_;
-    options.write_buffer_size = 1000000;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Trigger a long memtable compaction and reopen the database during it
@@ -774,10 +766,9 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
      [&](void* arg) { trivial_move++; });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  int32_t num_keys = 80;
@@ -1369,10 +1360,9 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
      [&](void* arg) { non_trivial_move++; });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  int32_t value_size = 10 * 1024;  // 10 KB
@@ -1661,7 +1651,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
  int max_key_universal_insert = 600;

  // Stage 1: generate a db with level compaction
-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 110 << 10;  // 110KB
  options.arena_block_size = 4 << 10;
  options.num_levels = 4;
@@ -1671,7 +1661,6 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
  options.target_file_size_base = 200 << 10;  // 200KB
  options.target_file_size_multiplier = 1;
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  for (int i = 0; i <= max_key_level_insert; i++) {
@@ -2386,10 +2375,9 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
      [&](void* arg) { non_trivial_move++; });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  int32_t value_size = 10 * 1024;  // 10 KB
@@ -2479,14 +2467,14 @@ class CompactionPriTest : public DBTestBase,
 };

 TEST_P(CompactionPriTest, Test) {
-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 16 * 1024;
  options.compaction_pri = static_cast<CompactionPri>(compaction_pri_);
  options.hard_pending_compaction_bytes_limit = 256 * 1024;
  options.max_bytes_for_level_base = 64 * 1024;
  options.max_bytes_for_level_multiplier = 4;
  options.compression = kNoCompression;
-  options = CurrentOptions(options);
+
  DestroyAndReopen(options);

  Random rnd(301);
@@ -405,6 +405,21 @@ DBImpl::~DBImpl() {
  }
  logs_.clear();

+  // Table cache may have table handles holding blocks from the block cache.
+  // We need to release them before the block cache is destroyed. The block
+  // cache may be destroyed inside versions_.reset(), when column family data
+  // list is destroyed, so leaving handles in table cache after
+  // versions_.reset() may cause issues.
+  // Here we clean all unreferenced handles in table cache.
+  // Now we assume all user queries have finished, so only version set itself
+  // can possibly hold the blocks from block cache. After releasing unreferenced
+  // handles here, only handles held by version set left and inside
+  // versions_.reset(), we will release them. There, we need to make sure every
+  // time a handle is released, we erase it from the cache too. By doing that,
+  // we can guarantee that after versions_.reset(), table cache is empty
+  // so the cache can be safely destroyed.
+  table_cache_->EraseUnRefEntries();
+
  // versions need to be destroyed before table_cache since it can hold
  // references to table_cache.
  versions_.reset();
@@ -572,6 +587,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
  // Get obsolete files.  This function will also update the list of
  // pending files in VersionSet().
  versions_->GetObsoleteFiles(&job_context->sst_delete_files,
+                              &job_context->manifest_delete_files,
                              job_context->min_pending_output);

  // store the current filenum, lognum, etc
@@ -689,9 +705,9 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
  }

  auto candidate_files = state.full_scan_candidate_files;
-  candidate_files.reserve(candidate_files.size() +
-                          state.sst_delete_files.size() +
-                          state.log_delete_files.size());
+  candidate_files.reserve(
+      candidate_files.size() + state.sst_delete_files.size() +
+      state.log_delete_files.size() + state.manifest_delete_files.size());
  // We may ignore the dbname when generating the file names.
  const char* kDumbDbName = "";
  for (auto file : state.sst_delete_files) {
@@ -707,6 +723,9 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {
                                   0);
    }
  }
+  for (const auto& filename : state.manifest_delete_files) {
+    candidate_files.emplace_back(filename, 0);
+  }

  // dedup state.candidate_files so we don't try to delete the same
  // file twice
@@ -825,7 +844,8 @@ void DBImpl::PurgeObsoleteFiles(const JobContext& state) {

  // Delete old info log files.
  size_t old_info_log_file_count = old_info_log_files.size();
-  if (old_info_log_file_count >= db_options_.keep_log_file_num) {
+  if (old_info_log_file_count != 0 &&
+      old_info_log_file_count >= db_options_.keep_log_file_num) {
    std::sort(old_info_log_files.begin(), old_info_log_files.end());
    size_t end = old_info_log_file_count - db_options_.keep_log_file_num;
    for (unsigned int i = 0; i <= end; i++) {
@@ -1098,6 +1118,23 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
    stream.EndArray();
  }

+#ifndef ROCKSDB_LITE
+  if (db_options_.wal_filter != nullptr) {
+    std::map<std::string, uint32_t> cf_name_id_map;
+    std::map<uint32_t, uint64_t> cf_lognumber_map;
+    for (auto cfd : *versions_->GetColumnFamilySet()) {
+      cf_name_id_map.insert(
+        std::make_pair(cfd->GetName(), cfd->GetID()));
+      cf_lognumber_map.insert(
+        std::make_pair(cfd->GetID(), cfd->GetLogNumber()));
+    }
+
+    db_options_.wal_filter->ColumnFamilyLogNumberMap(
+      cf_lognumber_map,
+      cf_name_id_map);
+  }
+#endif
+
  bool continue_replay_log = true;
  for (auto log_number : log_numbers) {
    // The previous incarnation may not have written any MANIFEST
@@ -1164,7 +1201,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
        continue_replay_log &&
        reader.ReadRecord(&record, &scratch, db_options_.wal_recovery_mode) &&
        status.ok()) {
-      if (record.size() < 12) {
+      if (record.size() < WriteBatchInternal::kHeader) {
        reporter.Corruption(record.size(),
                            Status::Corruption("log record too small"));
        continue;
@@ -1177,8 +1214,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
        bool batch_changed = false;

        WalFilter::WalProcessingOption wal_processing_option =
-            db_options_.wal_filter->LogRecord(batch, &new_batch,
-                                              &batch_changed);
+            db_options_.wal_filter->LogRecordFound(log_number, fname, batch,
+                                              &new_batch, &batch_changed);

        switch (wal_processing_option) {
          case WalFilter::WalProcessingOption::kContinueProcessing:
@@ -1415,8 +1452,8 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
      s = BuildTable(
          dbname_, env_, *cfd->ioptions(), env_options_, cfd->table_cache(),
          iter.get(), &meta, cfd->internal_comparator(),
-          cfd->int_tbl_prop_collector_factories(), cfd->GetID(), snapshot_seqs,
-          earliest_write_conflict_snapshot,
+          cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
+          snapshot_seqs, earliest_write_conflict_snapshot,
          GetCompressionFlush(*cfd->ioptions()),
          cfd->ioptions()->compression_opts, paranoid_file_checks,
          cfd->internal_stats(), Env::IO_HIGH, &info.table_properties);
@@ -1476,12 +1513,13 @@ Status DBImpl::FlushMemTableToOutputFile(
  std::vector<SequenceNumber> snapshot_seqs =
      snapshots_.GetAll(&earliest_write_conflict_snapshot);

-  FlushJob flush_job(
-      dbname_, cfd, db_options_, mutable_cf_options, env_options_,
-      versions_.get(), &mutex_, &shutting_down_, snapshot_seqs,
-      earliest_write_conflict_snapshot, job_context, log_buffer,
-      directories_.GetDbDir(), directories_.GetDataDir(0U),
-      GetCompressionFlush(*cfd->ioptions()), stats_, &event_logger_);
+  FlushJob flush_job(dbname_, cfd, db_options_, mutable_cf_options,
+                     env_options_, versions_.get(), &mutex_, &shutting_down_,
+                     snapshot_seqs, earliest_write_conflict_snapshot,
+                     job_context, log_buffer, directories_.GetDbDir(),
+                     directories_.GetDataDir(0U),
+                     GetCompressionFlush(*cfd->ioptions()), stats_,
+                     &event_logger_, mutable_cf_options.report_bg_io_stats);

  FileMetaData file_meta;

@@ -1830,7 +1868,7 @@ Status DBImpl::CompactFilesImpl(
      directories_.GetDataDir(c->output_path_id()), stats_, &mutex_, &bg_error_,
      snapshot_seqs, earliest_write_conflict_snapshot, table_cache_,
      &event_logger_, c->mutable_cf_options()->paranoid_file_checks,
-      c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
+      c->mutable_cf_options()->report_bg_io_stats, dbname_,
      nullptr);  // Here we pass a nullptr for CompactionJobStats because
                 // CompactFiles does not trigger OnCompactionCompleted(),
                 // which is the only place where CompactionJobStats is
@@ -1844,6 +1882,17 @@ Status DBImpl::CompactFilesImpl(
                 // support for CompactFiles, we should have CompactFiles API
                 // pass a pointer of CompactionJobStats as the out-value
                 // instead of using EventListener.
+
+  // Creating a compaction influences the compaction score because the score
+  // takes running compactions into account (by skipping files that are already
+  // being compacted). Since we just changed compaction score, we recalculate it
+  // here.
+  {
+    CompactionOptionsFIFO dummy_compaction_options_fifo;
+    version->storage_info()->ComputeCompactionScore(
+        *c->mutable_cf_options(), dummy_compaction_options_fifo);
+  }
+
  compaction_job.Prepare();

  mutex_.Unlock();
@@ -3025,7 +3074,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
        &bg_error_, snapshot_seqs, earliest_write_conflict_snapshot,
        table_cache_, &event_logger_,
        c->mutable_cf_options()->paranoid_file_checks,
-        c->mutable_cf_options()->compaction_measure_io_stats, dbname_,
+        c->mutable_cf_options()->report_bg_io_stats, dbname_,
        &compaction_job_stats);
    compaction_job.Prepare();

@@ -523,7 +523,7 @@ class DBImpl : public DB {
                         SequenceNumber* max_sequence, bool read_only);

  // The following two methods are used to flush a memtable to
-  // storage. The first one is used atdatabase RecoveryTime (when the
+  // storage. The first one is used at database RecoveryTime (when the
  // database is opened) and is heavyweight because it holds the mutex
  // for the entire period. The second method WriteLevel0Table supports
  // concurrent flush memtables to storage.
@@ -18,12 +18,11 @@ class DBTestInPlaceUpdate : public DBTestBase {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;
    options.env = env_;
    options.write_buffer_size = 100000;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of smaller size
@@ -41,12 +40,11 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;
    options.env = env_;
    options.write_buffer_size = 100000;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of larger size
@@ -64,7 +62,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;

@@ -72,7 +70,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
    options.write_buffer_size = 100000;
    options.inplace_callback =
      rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerSize;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of smaller size
@@ -92,7 +89,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;

@@ -100,7 +97,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
    options.write_buffer_size = 100000;
    options.inplace_callback =
      rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of smaller varint size
@@ -120,7 +116,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;

@@ -128,7 +124,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
    options.write_buffer_size = 100000;
    options.inplace_callback =
      rocksdb::DBTestInPlaceUpdate::updateInPlaceLargerSize;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of larger size
@@ -146,7 +141,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;

@@ -154,7 +149,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
    options.write_buffer_size = 100000;
    options.inplace_callback =
      rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Callback function requests no actions from db
@@ -15,6 +15,7 @@

 #include "db/dbformat.h"
 #include "db/filename.h"
+#include "db/merge_context.h"
 #include "port/port.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
@@ -60,6 +61,44 @@ class DBIter: public Iterator {
    kReverse
  };

+  // LocalStatistics contain Statistics counters that will be aggregated per
+  // each iterator instance and then will be sent to the global statistics when
+  // the iterator is destroyed.
+  //
+  // The purpose of this approach is to avoid perf regression happening
+  // when multiple threads bump the atomic counters from a DBIter::Next().
+  struct LocalStatistics {
+    explicit LocalStatistics() { ResetCounters(); }
+
+    void ResetCounters() {
+      next_count_ = 0;
+      next_found_count_ = 0;
+      prev_count_ = 0;
+      prev_found_count_ = 0;
+      bytes_read_ = 0;
+    }
+
+    void BumpGlobalStatistics(Statistics* global_statistics) {
+      RecordTick(global_statistics, NUMBER_DB_NEXT, next_count_);
+      RecordTick(global_statistics, NUMBER_DB_NEXT_FOUND, next_found_count_);
+      RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_);
+      RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_);
+      RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_);
+      ResetCounters();
+    }
+
+    // Map to Tickers::NUMBER_DB_NEXT
+    uint64_t next_count_;
+    // Map to Tickers::NUMBER_DB_NEXT_FOUND
+    uint64_t next_found_count_;
+    // Map to Tickers::NUMBER_DB_PREV
+    uint64_t prev_count_;
+    // Map to Tickers::NUMBER_DB_PREV_FOUND
+    uint64_t prev_found_count_;
+    // Map to Tickers::ITER_BYTES_READ
+    uint64_t bytes_read_;
+  };
+
  DBIter(Env* env, const ImmutableCFOptions& ioptions, const Comparator* cmp,
         InternalIterator* iter, SequenceNumber s, bool arena_mode,
         uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
@@ -86,6 +125,7 @@ class DBIter: public Iterator {
  }
  virtual ~DBIter() {
    RecordTick(statistics_, NO_ITERATORS, -1);
+    local_stats_.BumpGlobalStatistics(statistics_);
    if (!arena_mode_) {
      delete iter_;
    } else {
@@ -212,7 +252,8 @@ class DBIter: public Iterator {
  bool prefix_same_as_start_;
  bool iter_pinned_;
  // List of operands for merge operator.
-  std::deque<std::string> merge_operands_;
+  MergeContext merge_context_;
+  LocalStatistics local_stats_;

  // No copying allowed
  DBIter(const DBIter&);
@@ -250,6 +291,9 @@ void DBIter::Next() {
    PERF_COUNTER_ADD(internal_key_skipped_count, 1);
  }

+  if (statistics_ != nullptr) {
+    local_stats_.next_count_++;
+  }
  // Now we point to the next internal position, for both of merge and
  // not merge cases.
  if (!iter_->Valid()) {
@@ -257,18 +301,15 @@ void DBIter::Next() {
    return;
  }
  FindNextUserEntry(true /* skipping the current user key */);
-  if (statistics_ != nullptr) {
-    RecordTick(statistics_, NUMBER_DB_NEXT);
-    if (valid_) {
-      RecordTick(statistics_, NUMBER_DB_NEXT_FOUND);
-      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-    }
-  }
  if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
      prefix_extractor_->Transform(saved_key_.GetKey())
              .compare(prefix_start_.GetKey()) != 0) {
    valid_ = false;
  }
+  if (statistics_ != nullptr && valid_) {
+    local_stats_.next_found_count_++;
+    local_stats_.bytes_read_ += (key().size() + value().size());
+  }
 }

 // PRE: saved_key_ has the current user key if skipping
@@ -371,9 +412,9 @@ void DBIter::MergeValuesNewToOld() {
    return;
  }

+  merge_context_.Clear();
  // Start the merge process by pushing the first operand
-  std::deque<std::string> operands;
-  operands.push_front(iter_->value().ToString());
+  merge_context_.PushOperand(iter_->value());

  ParsedInternalKey ikey;
  for (iter_->Next(); iter_->Valid(); iter_->Next()) {
@@ -398,7 +439,8 @@ void DBIter::MergeValuesNewToOld() {
      {
        StopWatchNano timer(env_, statistics_ != nullptr);
        PERF_TIMER_GUARD(merge_operator_time_nanos);
-        user_merge_operator_->FullMerge(ikey.user_key, &val, operands,
+        user_merge_operator_->FullMerge(ikey.user_key, &val,
+                                        merge_context_.GetOperands(),
                                        &saved_value_, logger_);
        RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                   timer.ElapsedNanos());
@@ -410,7 +452,7 @@ void DBIter::MergeValuesNewToOld() {
      // hit a merge, add the value as an operand and run associative merge.
      // when complete, add result to operands and continue.
      const Slice& val = iter_->value();
-      operands.push_front(val.ToString());
+      merge_context_.PushOperand(val);
    } else {
      assert(false);
    }
@@ -423,8 +465,9 @@ void DBIter::MergeValuesNewToOld() {
    // a deletion marker.
    // feed null as the existing value to the merge operator, such that
    // client can differentiate this scenario and do things accordingly.
-    user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
-                                    &saved_value_, logger_);
+    user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
+                                    merge_context_.GetOperands(), &saved_value_,
+                                    logger_);
    RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
  }
 }
@@ -436,10 +479,10 @@ void DBIter::Prev() {
  }
  PrevInternal();
  if (statistics_ != nullptr) {
-    RecordTick(statistics_, NUMBER_DB_PREV);
+    local_stats_.prev_count_++;
    if (valid_) {
-      RecordTick(statistics_, NUMBER_DB_PREV_FOUND);
-      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
+      local_stats_.prev_found_count_++;
+      local_stats_.bytes_read_ += (key().size() + value().size());
    }
  }
  if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
@@ -516,7 +559,7 @@ void DBIter::PrevInternal() {
 // saved_value_
 bool DBIter::FindValueForCurrentKey() {
  assert(iter_->Valid());
-  merge_operands_.clear();
+  merge_context_.Clear();
  // last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or
  // kTypeValue)
  ValueType last_not_merge_type = kTypeDeletion;
@@ -536,19 +579,19 @@ bool DBIter::FindValueForCurrentKey() {
    last_key_entry_type = ikey.type;
    switch (last_key_entry_type) {
      case kTypeValue:
-        merge_operands_.clear();
+        merge_context_.Clear();
        saved_value_ = iter_->value().ToString();
        last_not_merge_type = kTypeValue;
        break;
      case kTypeDeletion:
      case kTypeSingleDeletion:
-        merge_operands_.clear();
+        merge_context_.Clear();
        last_not_merge_type = last_key_entry_type;
        PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
        break;
      case kTypeMerge:
        assert(user_merge_operator_ != nullptr);
-        merge_operands_.push_back(iter_->value().ToString());
+        merge_context_.PushOperandBack(iter_->value());
        break;
      default:
        assert(false);
@@ -571,8 +614,8 @@ bool DBIter::FindValueForCurrentKey() {
        StopWatchNano timer(env_, statistics_ != nullptr);
        PERF_TIMER_GUARD(merge_operator_time_nanos);
        user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
-                                        merge_operands_, &saved_value_,
-                                        logger_);
+                                        merge_context_.GetOperands(),
+                                        &saved_value_, logger_);
        RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                   timer.ElapsedNanos());
      } else {
@@ -583,8 +626,8 @@ bool DBIter::FindValueForCurrentKey() {
          StopWatchNano timer(env_, statistics_ != nullptr);
          PERF_TIMER_GUARD(merge_operator_time_nanos);
          user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice,
-                                          merge_operands_, &saved_value_,
-                                          logger_);
+                                          merge_context_.GetOperands(),
+                                          &saved_value_, logger_);
          RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                     timer.ElapsedNanos());
        }
@@ -627,11 +670,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {

  // kTypeMerge. We need to collect all kTypeMerge values and save them
  // in operands
-  std::deque<std::string> operands;
+  merge_context_.Clear();
  while (iter_->Valid() &&
         user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) &&
         ikey.type == kTypeMerge) {
-    operands.push_front(iter_->value().ToString());
+    merge_context_.PushOperand(iter_->value());
    iter_->Next();
    FindParseableKey(&ikey, kForward);
  }
@@ -642,7 +685,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    {
      StopWatchNano timer(env_, statistics_ != nullptr);
      PERF_TIMER_GUARD(merge_operator_time_nanos);
-      user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
+      user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
+                                      merge_context_.GetOperands(),
                                      &saved_value_, logger_);
      RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
    }
@@ -660,8 +704,9 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
  {
    StopWatchNano timer(env_, statistics_ != nullptr);
    PERF_TIMER_GUARD(merge_operator_time_nanos);
-    user_merge_operator_->FullMerge(saved_key_.GetKey(), &val, operands,
-                                    &saved_value_, logger_);
+    user_merge_operator_->FullMerge(saved_key_.GetKey(), &val,
+                                    merge_context_.GetOperands(), &saved_value_,
+                                    logger_);
    RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
  }
  valid_ = true;
@@ -253,6 +253,35 @@ TEST_F(DBPropertiesTest, ValidatePropertyInfo) {
  }
 }

+TEST_F(DBPropertiesTest, ValidateSampleNumber) {
+  // When "max_open_files" is -1, we read all the files for
+  // "rocksdb.estimate-num-keys" computation, which is the ground truth.
+  // Otherwise, we sample 20 newest files to make an estimation.
+  // Formula: lastest_20_files_active_key_ratio * total_files
+  Options options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  options.level0_stop_writes_trigger = 1000;
+  DestroyAndReopen(options);
+  int key = 0;
+  for (int files = 20; files >= 10; files -= 10) {
+    for (int i = 0; i < files; i++) {
+      int rows = files / 10;
+      for (int j = 0; j < rows; j++) {
+        db_->Put(WriteOptions(), std::to_string(++key), "foo");
+      }
+      db_->Flush(FlushOptions());
+    }
+  }
+  std::string num;
+  Reopen(options);
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
+  ASSERT_EQ("45", num);
+  options.max_open_files = -1;
+  Reopen(options);
+  ASSERT_TRUE(dbfull()->GetProperty("rocksdb.estimate-num-keys", &num));
+  ASSERT_EQ("50", num);
+}
+
 TEST_F(DBPropertiesTest, AggregatedTableProperties) {
  for (int kTableCount = 40; kTableCount <= 100; kTableCount += 30) {
    const int kKeysPerTable = 100;
@@ -215,6 +215,34 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) {
    TestGetPropertiesOfTablesInRange(std::move(ranges));
  }
 }
+
+TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) {
+  std::string kExtraCfName = "pikachu";
+  CreateAndReopenWithCF({kExtraCfName}, CurrentOptions());
+
+  // Create one table per CF, then verify it was created with the column family
+  // name property.
+  for (int cf = 0; cf < 2; ++cf) {
+    Put(cf, "key", "val");
+    Flush(cf);
+
+    TablePropertiesCollection fname_to_props;
+    ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props));
+    ASSERT_EQ(1U, fname_to_props.size());
+
+    std::string expected_cf_name;
+    if (cf > 0) {
+      expected_cf_name = kExtraCfName;
+    } else {
+      expected_cf_name = kDefaultColumnFamilyName;
+    }
+    ASSERT_EQ(expected_cf_name,
+              fname_to_props.begin()->second->column_family_name);
+    ASSERT_EQ(cf, static_cast<uint32_t>(
+                      fname_to_props.begin()->second->column_family_id));
+  }
+}
+
 }  // namespace rocksdb

 #endif  // ROCKSDB_LITE
@@ -228,10 +228,9 @@ TEST_F(DBTest, MemEnvTest) {
 #endif  // ROCKSDB_LITE

 TEST_F(DBTest, WriteEmptyBatch) {
-  Options options;
+  Options options = CurrentOptions();
  options.env = env_;
  options.write_buffer_size = 100000;
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  ASSERT_OK(Put(1, "foo", "bar"));
@@ -281,13 +280,12 @@ TEST_F(DBTest, ReadOnlyDB) {

 TEST_F(DBTest, CompactedDB) {
  const uint64_t kFileSize = 1 << 20;
-  Options options;
+  Options options = CurrentOptions();
  options.disable_auto_compactions = true;
  options.write_buffer_size = kFileSize;
  options.target_file_size_base = kFileSize;
  options.max_bytes_for_level_base = 1 << 30;
  options.compression = kNoCompression;
-  options = CurrentOptions(options);
  Reopen(options);
  // 1 L0 file, use CompactedDB if max_open_files = -1
  ASSERT_OK(Put("aaa", DummyString(kFileSize / 2, '1')));
@@ -764,11 +762,10 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) {
  std::string key1(kKeySize, 'c');
  std::string key2(kKeySize, 'd');

-  Options options;
+  Options options = CurrentOptions();
  options.env = env_;
  options.write_buffer_size = 100000;  // Small write buffer
  options.paranoid_checks = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  ASSERT_OK(Put("boo", "v1"));
@@ -811,10 +808,8 @@ TEST_F(DBTest, DISABLED_VeryLargeValue) {

 TEST_F(DBTest, GetFromImmutableLayer) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.env = env_;
-    options.write_buffer_size = 100000;  // Small write buffer
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    ASSERT_OK(Put(1, "foo", "v1"));
@@ -1916,11 +1911,9 @@ TEST_F(DBTest, Recover) {

 TEST_F(DBTest, RecoverWithTableHandle) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
-    options.write_buffer_size = 100;
    options.disable_auto_compactions = true;
-    options = CurrentOptions(options);
    DestroyAndReopen(options);
    CreateAndReopenWithCF({"pikachu"}, options);

@@ -2368,10 +2361,9 @@ TEST_F(DBTest, CompressedCache) {
  // Iteration 4: both block cache and compressed cache, but DB is not
  // compressed
  for (int iter = 0; iter < 4; iter++) {
-    Options options;
+    Options options = CurrentOptions();
    options.write_buffer_size = 64*1024;        // small write buffer
    options.statistics = rocksdb::CreateDBStatistics();
-    options = CurrentOptions(options);

    BlockBasedTableOptions table_options;
    switch (iter) {
@@ -2636,10 +2628,9 @@ TEST_F(DBTest, MinLevelToCompress2) {

 TEST_F(DBTest, RepeatedWritesToSameKey) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.env = env_;
    options.write_buffer_size = 100000;  // Small write buffer
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // We must have at most one file per level except for level-0,
@@ -2716,11 +2707,10 @@ static bool Between(uint64_t val, uint64_t low, uint64_t high) {
 }

 TEST_F(DBTest, ApproximateSizesMemTable) {
-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;  // Large write buffer
  options.compression = kNoCompression;
  options.create_if_missing = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  const int N = 128;
@@ -2823,11 +2813,10 @@ TEST_F(DBTest, ApproximateSizesMemTable) {

 TEST_F(DBTest, ApproximateSizes) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.write_buffer_size = 100000000;        // Large write buffer
    options.compression = kNoCompression;
    options.create_if_missing = true;
-    options = CurrentOptions(options);
    DestroyAndReopen(options);
    CreateAndReopenWithCF({"pikachu"}, options);

@@ -7843,8 +7832,7 @@ TEST_F(DBTest, MergeTestTime) {
  this->env_->addon_time_.store(0);
  this->env_->time_elapse_only_sleep_ = true;
  this->env_->no_sleep_ = true;
-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();
  options.statistics = rocksdb::CreateDBStatistics();
  options.merge_operator.reset(new DelayedMergeOperator(this));
  DestroyAndReopen(options);
@@ -7884,8 +7872,7 @@ TEST_F(DBTest, MergeTestTime) {
 #ifndef ROCKSDB_LITE
 TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
  SetPerfLevel(kEnableTime);
-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();
  options.compaction_filter_factory = std::make_shared<KeepFilterFactory>();
  options.statistics = rocksdb::CreateDBStatistics();
  options.merge_operator.reset(new DelayedMergeOperator(this));
@@ -7904,14 +7891,13 @@ TEST_P(DBTestWithParam, MergeCompactionTimeTest) {
 }

 TEST_P(DBTestWithParam, FilterCompactionTimeTest) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_filter_factory =
      std::make_shared<DelayFilterFactory>(this);
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
  options.statistics = rocksdb::CreateDBStatistics();
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // put some data
@@ -7953,9 +7939,8 @@ TEST_F(DBTest, TestLogCleanup) {

 #ifndef ROCKSDB_LITE
 TEST_F(DBTest, EmptyCompactedDB) {
-  Options options;
+  Options options = CurrentOptions();
  options.max_open_files = -1;
-  options = CurrentOptions(options);
  Close();
  ASSERT_OK(ReadOnlyReopen(options));
  Status s = Put("new", "value");
@@ -8191,9 +8176,8 @@ TEST_F(DBTest, AutomaticConflictsWithManualCompaction) {
 // Github issue #595
 // Large write batch with column families
 TEST_F(DBTest, LargeBatchWithColumnFamilies) {
-  Options options;
+  Options options = CurrentOptions();
  options.env = env_;
-  options = CurrentOptions(options);
  options.write_buffer_size = 100000;  // Small write buffer
  CreateAndReopenWithCF({"pikachu"}, options);
  int64_t j = 0;
@@ -8287,11 +8271,10 @@ TEST_F(DBTest, DelayedWriteRate) {
  const int kEntriesPerMemTable = 100;
  const int kTotalFlushes = 20;

-  Options options;
+  Options options = CurrentOptions();
  env_->SetBackgroundThreads(1, Env::LOW);
  options.env = env_;
  env_->no_sleep_ = true;
-  options = CurrentOptions(options);
  options.write_buffer_size = 100000000;
  options.max_write_buffer_number = 256;
  options.max_background_compactions = 1;
@@ -8352,10 +8335,9 @@ TEST_F(DBTest, DelayedWriteRate) {
 }

 TEST_F(DBTest, HardLimit) {
-  Options options;
+  Options options = CurrentOptions();
  options.env = env_;
  env_->SetBackgroundThreads(1, Env::LOW);
-  options = CurrentOptions(options);
  options.max_write_buffer_number = 256;
  options.write_buffer_size = 110 << 10;  // 110KB
  options.arena_block_size = 4 * 1024;
@@ -8403,9 +8385,8 @@ TEST_F(DBTest, HardLimit) {

 #ifndef ROCKSDB_LITE
 TEST_F(DBTest, SoftLimit) {
-  Options options;
+  Options options = CurrentOptions();
  options.env = env_;
-  options = CurrentOptions(options);
  options.write_buffer_size = 100000;  // Small write buffer
  options.max_write_buffer_number = 256;
  options.level0_file_num_compaction_trigger = 1;
@@ -8521,9 +8502,8 @@ TEST_F(DBTest, SoftLimit) {
 }

 TEST_F(DBTest, LastWriteBufferDelay) {
-  Options options;
+  Options options = CurrentOptions();
  options.env = env_;
-  options = CurrentOptions(options);
  options.write_buffer_size = 100000;
  options.max_write_buffer_number = 4;
  options.delayed_write_rate = 20000;
@@ -10108,9 +10088,8 @@ INSTANTIATE_TEST_CASE_P(DBTestWithParam, DBTestWithParam,
                                           ::testing::Bool()));

 TEST_F(DBTest, PauseBackgroundWorkTest) {
-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000;  // Small write buffer
-  options = CurrentOptions(options);
  Reopen(options);

  std::vector<std::thread> threads;
@@ -10134,403 +10113,6 @@ TEST_F(DBTest, PauseBackgroundWorkTest) {
  ASSERT_EQ(true, done.load());
 }

-#ifndef ROCKSDB_LITE
-namespace {
-void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
-                          const std::vector<Slice>& keys_must_not_exist) {
-  // Ensure that expected keys exist
-  std::vector<std::string> values;
-  if (keys_must_exist.size() > 0) {
-    std::vector<Status> status_list =
-        db->MultiGet(ReadOptions(), keys_must_exist, &values);
-    for (size_t i = 0; i < keys_must_exist.size(); i++) {
-      ASSERT_OK(status_list[i]);
-    }
-  }
-
-  // Ensure that given keys don't exist
-  if (keys_must_not_exist.size() > 0) {
-    std::vector<Status> status_list =
-        db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
-    for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
-      ASSERT_TRUE(status_list[i].IsNotFound());
-    }
-  }
-}
-
-}  // namespace
-
-TEST_F(DBTest, WalFilterTest) {
-  class TestWalFilter : public WalFilter {
-   private:
-    // Processing option that is requested to be applied at the given index
-    WalFilter::WalProcessingOption wal_processing_option_;
-    // Index at which to apply wal_processing_option_
-    // At other indexes default wal_processing_option::kContinueProcessing is
-    // returned.
-    size_t apply_option_at_record_index_;
-    // Current record index, incremented with each record encountered.
-    size_t current_record_index_;
-
-   public:
-    TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
-                  size_t apply_option_for_record_index)
-        : wal_processing_option_(wal_processing_option),
-          apply_option_at_record_index_(apply_option_for_record_index),
-          current_record_index_(0) {}
-
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-                                          WriteBatch* new_batch,
-                                          bool* batch_changed) const override {
-      WalFilter::WalProcessingOption option_to_return;
-
-      if (current_record_index_ == apply_option_at_record_index_) {
-        option_to_return = wal_processing_option_;
-      } else {
-        option_to_return = WalProcessingOption::kContinueProcessing;
-      }
-
-      // Filter is passed as a const object for RocksDB to not modify the
-      // object, however we modify it for our own purpose here and hence
-      // cast the constness away.
-      (const_cast<TestWalFilter*>(this)->current_record_index_)++;
-
-      return option_to_return;
-    }
-
-    virtual const char* Name() const override { return "TestWalFilter"; }
-  };
-
-  // Create 3 batches with two keys each
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  // Test with all WAL processing options
-  for (int option = 0;
-       option < static_cast<int>(
-                    WalFilter::WalProcessingOption::kWalProcessingOptionMax);
-       option++) {
-    Options options = OptionsForLogIterTest();
-    DestroyAndReopen(options);
-    CreateAndReopenWithCF({"pikachu"}, options);
-
-    // Write given keys in given batches
-    for (size_t i = 0; i < batch_keys.size(); i++) {
-      WriteBatch batch;
-      for (size_t j = 0; j < batch_keys[i].size(); j++) {
-        batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-      }
-      dbfull()->Write(WriteOptions(), &batch);
-    }
-
-    WalFilter::WalProcessingOption wal_processing_option =
-        static_cast<WalFilter::WalProcessingOption>(option);
-
-    // Create a test filter that would apply wal_processing_option at the first
-    // record
-    size_t apply_option_for_record_index = 1;
-    TestWalFilter test_wal_filter(wal_processing_option,
-                                  apply_option_for_record_index);
-
-    // Reopen database with option to use WAL filter
-    options = OptionsForLogIterTest();
-    options.wal_filter = &test_wal_filter;
-    Status status =
-        TryReopenWithColumnFamilies({"default", "pikachu"}, options);
-    if (wal_processing_option ==
-        WalFilter::WalProcessingOption::kCorruptedRecord) {
-      assert(!status.ok());
-      // In case of corruption we can turn off paranoid_checks to reopen
-      // databse
-      options.paranoid_checks = false;
-      ReopenWithColumnFamilies({"default", "pikachu"}, options);
-    } else {
-      assert(status.ok());
-    }
-
-    // Compute which keys we expect to be found
-    // and which we expect not to be found after recovery.
-    std::vector<Slice> keys_must_exist;
-    std::vector<Slice> keys_must_not_exist;
-    switch (wal_processing_option) {
-      case WalFilter::WalProcessingOption::kCorruptedRecord:
-      case WalFilter::WalProcessingOption::kContinueProcessing: {
-        fprintf(stderr, "Testing with complete WAL processing\n");
-        // we expect all records to be processed
-        for (size_t i = 0; i < batch_keys.size(); i++) {
-          for (size_t j = 0; j < batch_keys[i].size(); j++) {
-            keys_must_exist.push_back(Slice(batch_keys[i][j]));
-          }
-        }
-        break;
-      }
-      case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
-        fprintf(stderr,
-                "Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
-                apply_option_for_record_index);
-        // We expect the record with apply_option_for_record_index to be not
-        // found.
-        for (size_t i = 0; i < batch_keys.size(); i++) {
-          for (size_t j = 0; j < batch_keys[i].size(); j++) {
-            if (i == apply_option_for_record_index) {
-              keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-            } else {
-              keys_must_exist.push_back(Slice(batch_keys[i][j]));
-            }
-          }
-        }
-        break;
-      }
-      case WalFilter::WalProcessingOption::kStopReplay: {
-        fprintf(stderr,
-                "Testing with stopping replay from record %" ROCKSDB_PRIszt
-                "\n",
-                apply_option_for_record_index);
-        // We expect records beyond apply_option_for_record_index to be not
-        // found.
-        for (size_t i = 0; i < batch_keys.size(); i++) {
-          for (size_t j = 0; j < batch_keys[i].size(); j++) {
-            if (i >= apply_option_for_record_index) {
-              keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-            } else {
-              keys_must_exist.push_back(Slice(batch_keys[i][j]));
-            }
-          }
-        }
-        break;
-      }
-      default:
-        assert(false);  // unhandled case
-    }
-
-    bool checked_after_reopen = false;
-
-    while (true) {
-      // Ensure that expected keys exists
-      // and not expected keys don't exist after recovery
-      ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-
-      if (checked_after_reopen) {
-        break;
-      }
-
-      // reopen database again to make sure previous log(s) are not used
-      //(even if they were skipped)
-      // reopn database with option to use WAL filter
-      options = OptionsForLogIterTest();
-      ReopenWithColumnFamilies({"default", "pikachu"}, options);
-
-      checked_after_reopen = true;
-    }
-  }
-}
-
-TEST_F(DBTest, WalFilterTestWithChangeBatch) {
-  class ChangeBatchHandler : public WriteBatch::Handler {
-   private:
-    // Batch to insert keys in
-    WriteBatch* new_write_batch_;
-    // Number of keys to add in the new batch
-    size_t num_keys_to_add_in_new_batch_;
-    // Number of keys added to new batch
-    size_t num_keys_added_;
-
-   public:
-    ChangeBatchHandler(WriteBatch* new_write_batch,
-                       size_t num_keys_to_add_in_new_batch)
-        : new_write_batch_(new_write_batch),
-          num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
-          num_keys_added_(0) {}
-    virtual void Put(const Slice& key, const Slice& value) override {
-      if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
-        new_write_batch_->Put(key, value);
-        ++num_keys_added_;
-      }
-    }
-  };
-
-  class TestWalFilterWithChangeBatch : public WalFilter {
-   private:
-    // Index at which to start changing records
-    size_t change_records_from_index_;
-    // Number of keys to add in the new batch
-    size_t num_keys_to_add_in_new_batch_;
-    // Current record index, incremented with each record encountered.
-    size_t current_record_index_;
-
-   public:
-    TestWalFilterWithChangeBatch(size_t change_records_from_index,
-                                 size_t num_keys_to_add_in_new_batch)
-        : change_records_from_index_(change_records_from_index),
-          num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
-          current_record_index_(0) {}
-
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-                                          WriteBatch* new_batch,
-                                          bool* batch_changed) const override {
-      if (current_record_index_ >= change_records_from_index_) {
-        ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
-        batch.Iterate(&handler);
-        *batch_changed = true;
-      }
-
-      // Filter is passed as a const object for RocksDB to not modify the
-      // object, however we modify it for our own purpose here and hence
-      // cast the constness away.
-      (const_cast<TestWalFilterWithChangeBatch*>(this)
-           ->current_record_index_)++;
-
-      return WalProcessingOption::kContinueProcessing;
-    }
-
-    virtual const char* Name() const override {
-      return "TestWalFilterWithChangeBatch";
-    }
-  };
-
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  Options options = OptionsForLogIterTest();
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  // Create a test filter that would apply wal_processing_option at the first
-  // record
-  size_t change_records_from_index = 1;
-  size_t num_keys_to_add_in_new_batch = 1;
-  TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
-      change_records_from_index, num_keys_to_add_in_new_batch);
-
-  // Reopen database with option to use WAL filter
-  options = OptionsForLogIterTest();
-  options.wal_filter = &test_wal_filter_with_change_batch;
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-
-  // Ensure that all keys exist before change_records_from_index_
-  // And after that index only single key exists
-  // as our filter adds only single key for each batch
-  std::vector<Slice> keys_must_exist;
-  std::vector<Slice> keys_must_not_exist;
-
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      if (i >= change_records_from_index && j >= num_keys_to_add_in_new_batch) {
-        keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
-      } else {
-        keys_must_exist.push_back(Slice(batch_keys[i][j]));
-      }
-    }
-  }
-
-  bool checked_after_reopen = false;
-
-  while (true) {
-    // Ensure that expected keys exists
-    // and not expected keys don't exist after recovery
-    ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-
-    if (checked_after_reopen) {
-      break;
-    }
-
-    // reopen database again to make sure previous log(s) are not used
-    //(even if they were skipped)
-    // reopn database with option to use WAL filter
-    options = OptionsForLogIterTest();
-    ReopenWithColumnFamilies({"default", "pikachu"}, options);
-
-    checked_after_reopen = true;
-  }
-}
-
-TEST_F(DBTest, WalFilterTestWithChangeBatchExtraKeys) {
-  class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
-   public:
-    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
-                                          WriteBatch* new_batch,
-                                          bool* batch_changed) const override {
-      *new_batch = batch;
-      new_batch->Put("key_extra", "value_extra");
-      *batch_changed = true;
-      return WalProcessingOption::kContinueProcessing;
-    }
-
-    virtual const char* Name() const override {
-      return "WalFilterTestWithChangeBatchExtraKeys";
-    }
-  };
-
-  std::vector<std::vector<std::string>> batch_keys(3);
-
-  batch_keys[0].push_back("key1");
-  batch_keys[0].push_back("key2");
-  batch_keys[1].push_back("key3");
-  batch_keys[1].push_back("key4");
-  batch_keys[2].push_back("key5");
-  batch_keys[2].push_back("key6");
-
-  Options options = OptionsForLogIterTest();
-  DestroyAndReopen(options);
-  CreateAndReopenWithCF({"pikachu"}, options);
-
-  // Write given keys in given batches
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    WriteBatch batch;
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
-    }
-    dbfull()->Write(WriteOptions(), &batch);
-  }
-
-  // Create a test filter that would add extra keys
-  TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;
-
-  // Reopen database with option to use WAL filter
-  options = OptionsForLogIterTest();
-  options.wal_filter = &test_wal_filter_extra_keys;
-  Status status = TryReopenWithColumnFamilies({"default", "pikachu"}, options);
-  ASSERT_TRUE(status.IsNotSupported());
-
-  // Reopen without filter, now reopen should succeed - previous
-  // attempt to open must not have altered the db.
-  options = OptionsForLogIterTest();
-  ReopenWithColumnFamilies({"default", "pikachu"}, options);
-
-  std::vector<Slice> keys_must_exist;
-  std::vector<Slice> keys_must_not_exist;  // empty vector
-
-  for (size_t i = 0; i < batch_keys.size(); i++) {
-    for (size_t j = 0; j < batch_keys[i].size(); j++) {
-      keys_must_exist.push_back(Slice(batch_keys[i][j]));
-    }
-  }
-
-  ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
-}
-#endif  // ROCKSDB_LITE
-
 class SliceTransformLimitedDomain : public SliceTransform {
  const char* Name() const override { return "SliceTransformLimitedDomain"; }

@@ -10611,6 +10193,88 @@ TEST_F(DBTest, PrefixExtractorBlockFilter) {
  delete iter;
 }

+TEST_F(DBTest, IteratorWithLocalStatistics) {
+  Options options = CurrentOptions();
+  options.statistics = rocksdb::CreateDBStatistics();
+  DestroyAndReopen(options);
+
+  Random rnd(301);
+  for (int i = 0; i < 1000; i++) {
+    // Key 10 bytes / Value 10 bytes
+    ASSERT_OK(Put(RandomString(&rnd, 10), RandomString(&rnd, 10)));
+  }
+
+  std::atomic<uint64_t> total_next(0);
+  std::atomic<uint64_t> total_next_found(0);
+  std::atomic<uint64_t> total_prev(0);
+  std::atomic<uint64_t> total_prev_found(0);
+  std::atomic<uint64_t> total_bytes(0);
+
+  std::vector<std::thread> threads;
+  std::function<void()> reader_func_next = [&]() {
+    Iterator* iter = db_->NewIterator(ReadOptions());
+
+    iter->SeekToFirst();
+    // Seek will bump ITER_BYTES_READ
+    total_bytes += iter->key().size();
+    total_bytes += iter->value().size();
+    while (true) {
+      iter->Next();
+      total_next++;
+
+      if (!iter->Valid()) {
+        break;
+      }
+      total_next_found++;
+      total_bytes += iter->key().size();
+      total_bytes += iter->value().size();
+    }
+
+    delete iter;
+  };
+
+  std::function<void()> reader_func_prev = [&]() {
+    Iterator* iter = db_->NewIterator(ReadOptions());
+
+    iter->SeekToLast();
+    // Seek will bump ITER_BYTES_READ
+    total_bytes += iter->key().size();
+    total_bytes += iter->value().size();
+    while (true) {
+      iter->Prev();
+      total_prev++;
+
+      if (!iter->Valid()) {
+        break;
+      }
+      total_prev_found++;
+      total_bytes += iter->key().size();
+      total_bytes += iter->value().size();
+    }
+
+    delete iter;
+  };
+
+  for (int i = 0; i < 10; i++) {
+    threads.emplace_back(reader_func_next);
+  }
+  for (int i = 0; i < 15; i++) {
+    threads.emplace_back(reader_func_prev);
+  }
+
+  for (auto& t : threads) {
+    t.join();
+  }
+
+  ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_NEXT), total_next);
+  ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_NEXT_FOUND),
+            total_next_found);
+  ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV), total_prev);
+  ASSERT_EQ(TestGetTickerCount(options, NUMBER_DB_PREV_FOUND),
+            total_prev_found);
+  ASSERT_EQ(TestGetTickerCount(options, ITER_BYTES_READ), total_bytes);
+}
+
 #ifndef ROCKSDB_LITE
 class BloomStatsTestWithParam
    : public DBTest,
@@ -9,14 +9,82 @@
 #include <cstdlib>
 #include "db/db_test_util.h"
 #include "port/stack_trace.h"
+#include "rocksdb/wal_filter.h"

 namespace rocksdb {

+static uint64_t TestGetTickerCount(const Options& options,
+                                   Tickers ticker_type) {
+  return options.statistics->getTickerCount(ticker_type);
+}
+
 class DBTest2 : public DBTestBase {
 public:
  DBTest2() : DBTestBase("/db_test2") {}
 };

+class PrefixFullBloomWithReverseComparator
+    : public DBTestBase,
+      public ::testing::WithParamInterface<bool> {
+ public:
+  PrefixFullBloomWithReverseComparator()
+      : DBTestBase("/prefix_bloom_reverse") {}
+  virtual void SetUp() override { if_cache_filter_ = GetParam(); }
+  bool if_cache_filter_;
+};
+
+TEST_P(PrefixFullBloomWithReverseComparator,
+       PrefixFullBloomWithReverseComparator) {
+  Options options = last_options_;
+  options.comparator = ReverseBytewiseComparator();
+  options.prefix_extractor.reset(NewCappedPrefixTransform(3));
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions bbto;
+  if (if_cache_filter_) {
+    bbto.no_block_cache = false;
+    bbto.cache_index_and_filter_blocks = true;
+    bbto.block_cache = NewLRUCache(1);
+  }
+  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(options);
+
+  ASSERT_OK(dbfull()->Put(WriteOptions(), "bar123", "foo"));
+  ASSERT_OK(dbfull()->Put(WriteOptions(), "bar234", "foo2"));
+  ASSERT_OK(dbfull()->Put(WriteOptions(), "foo123", "foo3"));
+
+  dbfull()->Flush(FlushOptions());
+
+  if (bbto.block_cache) {
+    bbto.block_cache->EraseUnRefEntries();
+  }
+
+  unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
+  iter->Seek("bar345");
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ("bar234", iter->key().ToString());
+  ASSERT_EQ("foo2", iter->value().ToString());
+  iter->Next();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ("bar123", iter->key().ToString());
+  ASSERT_EQ("foo", iter->value().ToString());
+
+  iter->Seek("foo234");
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ("foo123", iter->key().ToString());
+  ASSERT_EQ("foo3", iter->value().ToString());
+
+  iter->Seek("bar");
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(!iter->Valid());
+}
+
+INSTANTIATE_TEST_CASE_P(PrefixFullBloomWithReverseComparator,
+                        PrefixFullBloomWithReverseComparator, testing::Bool());
+
 TEST_F(DBTest2, IteratorPropertyVersionNumber) {
  Put("", "");
  Iterator* iter1 = db_->NewIterator(ReadOptions());
@@ -58,6 +126,735 @@ TEST_F(DBTest2, IteratorPropertyVersionNumber) {
  delete iter2;
  delete iter3;
 }
+
+TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  Put(1, "a", "begin");
+  Put(1, "z", "end");
+  ASSERT_OK(Flush(1));
+  TryReopenWithColumnFamilies({"default", "pikachu"}, options);
+
+  std::string value;
+  value = Get(1, "a");
+}
+
+#ifndef ROCKSDB_LITE
+namespace {
+  void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
+    const std::vector<Slice>& keys_must_not_exist) {
+    // Ensure that expected keys exist
+    std::vector<std::string> values;
+    if (keys_must_exist.size() > 0) {
+      std::vector<Status> status_list =
+        db->MultiGet(ReadOptions(), keys_must_exist, &values);
+      for (size_t i = 0; i < keys_must_exist.size(); i++) {
+        ASSERT_OK(status_list[i]);
+      }
+    }
+
+    // Ensure that given keys don't exist
+    if (keys_must_not_exist.size() > 0) {
+      std::vector<Status> status_list =
+        db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
+      for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
+        ASSERT_TRUE(status_list[i].IsNotFound());
+      }
+    }
+  }
+
+}  // namespace
+
+TEST_F(DBTest2, WalFilterTest) {
+  class TestWalFilter : public WalFilter {
+  private:
+    // Processing option that is requested to be applied at the given index
+    WalFilter::WalProcessingOption wal_processing_option_;
+    // Index at which to apply wal_processing_option_
+    // At other indexes default wal_processing_option::kContinueProcessing is
+    // returned.
+    size_t apply_option_at_record_index_;
+    // Current record index, incremented with each record encountered.
+    size_t current_record_index_;
+
+  public:
+    TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
+      size_t apply_option_for_record_index)
+      : wal_processing_option_(wal_processing_option),
+      apply_option_at_record_index_(apply_option_for_record_index),
+      current_record_index_(0) {}
+
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+      WriteBatch* new_batch,
+      bool* batch_changed) const override {
+      WalFilter::WalProcessingOption option_to_return;
+
+      if (current_record_index_ == apply_option_at_record_index_) {
+        option_to_return = wal_processing_option_;
+      }
+      else {
+        option_to_return = WalProcessingOption::kContinueProcessing;
+      }
+
+      // Filter is passed as a const object for RocksDB to not modify the
+      // object, however we modify it for our own purpose here and hence
+      // cast the constness away.
+      (const_cast<TestWalFilter*>(this)->current_record_index_)++;
+
+      return option_to_return;
+    }
+
+    virtual const char* Name() const override { return "TestWalFilter"; }
+  };
+
+  // Create 3 batches with two keys each
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  // Test with all WAL processing options
+  for (int option = 0;
+    option < static_cast<int>(
+    WalFilter::WalProcessingOption::kWalProcessingOptionMax);
+  option++) {
+    Options options = OptionsForLogIterTest();
+    DestroyAndReopen(options);
+    CreateAndReopenWithCF({ "pikachu" }, options);
+
+    // Write given keys in given batches
+    for (size_t i = 0; i < batch_keys.size(); i++) {
+      WriteBatch batch;
+      for (size_t j = 0; j < batch_keys[i].size(); j++) {
+        batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+      }
+      dbfull()->Write(WriteOptions(), &batch);
+    }
+
+    WalFilter::WalProcessingOption wal_processing_option =
+      static_cast<WalFilter::WalProcessingOption>(option);
+
+    // Create a test filter that would apply wal_processing_option at the first
+    // record
+    size_t apply_option_for_record_index = 1;
+    TestWalFilter test_wal_filter(wal_processing_option,
+      apply_option_for_record_index);
+
+    // Reopen database with option to use WAL filter
+    options = OptionsForLogIterTest();
+    options.wal_filter = &test_wal_filter;
+    Status status =
+      TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
+    if (wal_processing_option ==
+      WalFilter::WalProcessingOption::kCorruptedRecord) {
+      assert(!status.ok());
+      // In case of corruption we can turn off paranoid_checks to reopen
+      // databse
+      options.paranoid_checks = false;
+      ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+    }
+    else {
+      assert(status.ok());
+    }
+
+    // Compute which keys we expect to be found
+    // and which we expect not to be found after recovery.
+    std::vector<Slice> keys_must_exist;
+    std::vector<Slice> keys_must_not_exist;
+    switch (wal_processing_option) {
+    case WalFilter::WalProcessingOption::kCorruptedRecord:
+    case WalFilter::WalProcessingOption::kContinueProcessing: {
+      fprintf(stderr, "Testing with complete WAL processing\n");
+      // we expect all records to be processed
+      for (size_t i = 0; i < batch_keys.size(); i++) {
+        for (size_t j = 0; j < batch_keys[i].size(); j++) {
+          keys_must_exist.push_back(Slice(batch_keys[i][j]));
+        }
+      }
+      break;
+    }
+    case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
+      fprintf(stderr,
+        "Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
+        apply_option_for_record_index);
+      // We expect the record with apply_option_for_record_index to be not
+      // found.
+      for (size_t i = 0; i < batch_keys.size(); i++) {
+        for (size_t j = 0; j < batch_keys[i].size(); j++) {
+          if (i == apply_option_for_record_index) {
+            keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+          }
+          else {
+            keys_must_exist.push_back(Slice(batch_keys[i][j]));
+          }
+        }
+      }
+      break;
+    }
+    case WalFilter::WalProcessingOption::kStopReplay: {
+      fprintf(stderr,
+        "Testing with stopping replay from record %" ROCKSDB_PRIszt
+        "\n",
+        apply_option_for_record_index);
+      // We expect records beyond apply_option_for_record_index to be not
+      // found.
+      for (size_t i = 0; i < batch_keys.size(); i++) {
+        for (size_t j = 0; j < batch_keys[i].size(); j++) {
+          if (i >= apply_option_for_record_index) {
+            keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+          }
+          else {
+            keys_must_exist.push_back(Slice(batch_keys[i][j]));
+          }
+        }
+      }
+      break;
+    }
+    default:
+      assert(false);  // unhandled case
+    }
+
+    bool checked_after_reopen = false;
+
+    while (true) {
+      // Ensure that expected keys exists
+      // and not expected keys don't exist after recovery
+      ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+
+      if (checked_after_reopen) {
+        break;
+      }
+
+      // reopen database again to make sure previous log(s) are not used
+      //(even if they were skipped)
+      // reopn database with option to use WAL filter
+      options = OptionsForLogIterTest();
+      ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+
+      checked_after_reopen = true;
+    }
+  }
+}
+
+TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
+  class ChangeBatchHandler : public WriteBatch::Handler {
+  private:
+    // Batch to insert keys in
+    WriteBatch* new_write_batch_;
+    // Number of keys to add in the new batch
+    size_t num_keys_to_add_in_new_batch_;
+    // Number of keys added to new batch
+    size_t num_keys_added_;
+
+  public:
+    ChangeBatchHandler(WriteBatch* new_write_batch,
+      size_t num_keys_to_add_in_new_batch)
+      : new_write_batch_(new_write_batch),
+      num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
+      num_keys_added_(0) {}
+    virtual void Put(const Slice& key, const Slice& value) override {
+      if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
+        new_write_batch_->Put(key, value);
+        ++num_keys_added_;
+      }
+    }
+  };
+
+  class TestWalFilterWithChangeBatch : public WalFilter {
+  private:
+    // Index at which to start changing records
+    size_t change_records_from_index_;
+    // Number of keys to add in the new batch
+    size_t num_keys_to_add_in_new_batch_;
+    // Current record index, incremented with each record encountered.
+    size_t current_record_index_;
+
+  public:
+    TestWalFilterWithChangeBatch(size_t change_records_from_index,
+      size_t num_keys_to_add_in_new_batch)
+      : change_records_from_index_(change_records_from_index),
+      num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
+      current_record_index_(0) {}
+
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+      WriteBatch* new_batch,
+      bool* batch_changed) const override {
+      if (current_record_index_ >= change_records_from_index_) {
+        ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
+        batch.Iterate(&handler);
+        *batch_changed = true;
+      }
+
+      // Filter is passed as a const object for RocksDB to not modify the
+      // object, however we modify it for our own purpose here and hence
+      // cast the constness away.
+      (const_cast<TestWalFilterWithChangeBatch*>(this)
+        ->current_record_index_)++;
+
+      return WalProcessingOption::kContinueProcessing;
+    }
+
+    virtual const char* Name() const override {
+      return "TestWalFilterWithChangeBatch";
+    }
+  };
+
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  Options options = OptionsForLogIterTest();
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({ "pikachu" }, options);
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  // Create a test filter that would apply wal_processing_option at the first
+  // record
+  size_t change_records_from_index = 1;
+  size_t num_keys_to_add_in_new_batch = 1;
+  TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
+    change_records_from_index, num_keys_to_add_in_new_batch);
+
+  // Reopen database with option to use WAL filter
+  options = OptionsForLogIterTest();
+  options.wal_filter = &test_wal_filter_with_change_batch;
+  ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+
+  // Ensure that all keys exist before change_records_from_index_
+  // And after that index only single key exists
+  // as our filter adds only single key for each batch
+  std::vector<Slice> keys_must_exist;
+  std::vector<Slice> keys_must_not_exist;
+
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      if (i >= change_records_from_index && j >= 
+        num_keys_to_add_in_new_batch) {
+        keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+      }
+      else {
+        keys_must_exist.push_back(Slice(batch_keys[i][j]));
+      }
+    }
+  }
+
+  bool checked_after_reopen = false;
+
+  while (true) {
+    // Ensure that expected keys exists
+    // and not expected keys don't exist after recovery
+    ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+
+    if (checked_after_reopen) {
+      break;
+    }
+
+    // reopen database again to make sure previous log(s) are not used
+    //(even if they were skipped)
+    // reopn database with option to use WAL filter
+    options = OptionsForLogIterTest();
+    ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+
+    checked_after_reopen = true;
+  }
+}
+
+TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
+  class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
+  public:
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+      WriteBatch* new_batch,
+      bool* batch_changed) const override {
+      *new_batch = batch;
+      new_batch->Put("key_extra", "value_extra");
+      *batch_changed = true;
+      return WalProcessingOption::kContinueProcessing;
+    }
+
+    virtual const char* Name() const override {
+      return "WalFilterTestWithChangeBatchExtraKeys";
+    }
+  };
+
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  Options options = OptionsForLogIterTest();
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({ "pikachu" }, options);
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  // Create a test filter that would add extra keys
+  TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;
+
+  // Reopen database with option to use WAL filter
+  options = OptionsForLogIterTest();
+  options.wal_filter = &test_wal_filter_extra_keys;
+  Status status = TryReopenWithColumnFamilies({ "default", "pikachu" }, 
+    options);
+  ASSERT_TRUE(status.IsNotSupported());
+
+  // Reopen without filter, now reopen should succeed - previous
+  // attempt to open must not have altered the db.
+  options = OptionsForLogIterTest();
+  ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+
+  std::vector<Slice> keys_must_exist;
+  std::vector<Slice> keys_must_not_exist;  // empty vector
+
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      keys_must_exist.push_back(Slice(batch_keys[i][j]));
+    }
+  }
+
+  ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+}
+
+TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
+  class TestWalFilterWithColumnFamilies : public WalFilter {
+  private:
+    // column_family_id -> log_number map (provided to WALFilter)
+    std::map<uint32_t, uint64_t> cf_log_number_map_;
+    // column_family_name -> column_family_id map (provided to WALFilter)
+    std::map<std::string, uint32_t> cf_name_id_map_;
+    // column_family_name -> keys_found_in_wal map
+    // We store keys that are applicable to the column_family
+    // during recovery (i.e. aren't already flushed to SST file(s))
+    // for verification against the keys we expect.
+    std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
+  public:
+    virtual void ColumnFamilyLogNumberMap(
+      const std::map<uint32_t, uint64_t>& cf_lognumber_map,
+      const std::map<std::string, uint32_t>& cf_name_id_map) override {
+      cf_log_number_map_ = cf_lognumber_map;
+      cf_name_id_map_ = cf_name_id_map;
+    }
+
+    virtual WalProcessingOption LogRecordFound(unsigned long long log_number,
+      const std::string& log_file_name,
+      const WriteBatch& batch,
+      WriteBatch* new_batch,
+      bool* batch_changed) override {
+      class LogRecordBatchHandler : public WriteBatch::Handler {
+      private:
+        const std::map<uint32_t, uint64_t> & cf_log_number_map_;
+        std::map<uint32_t, std::vector<std::string>> & cf_wal_keys_;
+        unsigned long long log_number_;
+      public:
+        LogRecordBatchHandler(unsigned long long current_log_number,
+          const std::map<uint32_t, uint64_t> & cf_log_number_map,
+          std::map<uint32_t, std::vector<std::string>> & cf_wal_keys) :
+          cf_log_number_map_(cf_log_number_map),
+          cf_wal_keys_(cf_wal_keys),
+          log_number_(current_log_number){}
+
+        virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+          const Slice& /*value*/) override {
+          auto it = cf_log_number_map_.find(column_family_id);
+          assert(it != cf_log_number_map_.end());
+          unsigned long long log_number_for_cf = it->second;
+          // If the current record is applicable for column_family_id
+          // (i.e. isn't flushed to SST file(s) for column_family_id)
+          // add it to the cf_wal_keys_ map for verification.
+          if (log_number_ >= log_number_for_cf) {
+            cf_wal_keys_[column_family_id].push_back(std::string(key.data(),
+              key.size()));
+          }
+          return Status::OK();
+        }
+      } handler(log_number, cf_log_number_map_, cf_wal_keys_);
+
+      batch.Iterate(&handler);
+
+      return WalProcessingOption::kContinueProcessing;
+    }
+
+    virtual const char* Name() const override {
+      return "WalFilterTestWithColumnFamilies";
+    }
+
+    const std::map<uint32_t, std::vector<std::string>> & 
+    GetColumnFamilyKeys() {
+      return cf_wal_keys_;
+    }
+
+    const std::map<std::string, uint32_t> & GetColumnFamilyNameIdMap() {
+      return cf_name_id_map_;
+    }
+  };
+
+  std::vector<std::vector<std::string>> batch_keys_pre_flush(3);
+
+  batch_keys_pre_flush[0].push_back("key1");
+  batch_keys_pre_flush[0].push_back("key2");
+  batch_keys_pre_flush[1].push_back("key3");
+  batch_keys_pre_flush[1].push_back("key4");
+  batch_keys_pre_flush[2].push_back("key5");
+  batch_keys_pre_flush[2].push_back("key6");
+
+  Options options = OptionsForLogIterTest();
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({ "pikachu" }, options);
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys_pre_flush[i][j], DummyString(1024));
+      batch.Put(handles_[1], batch_keys_pre_flush[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  //Flush default column-family
+  db_->Flush(FlushOptions(), handles_[0]);
+
+  // Do some more writes
+  std::vector<std::vector<std::string>> batch_keys_post_flush(3);
+
+  batch_keys_post_flush[0].push_back("key7");
+  batch_keys_post_flush[0].push_back("key8");
+  batch_keys_post_flush[1].push_back("key9");
+  batch_keys_post_flush[1].push_back("key10");
+  batch_keys_post_flush[2].push_back("key11");
+  batch_keys_post_flush[2].push_back("key12");
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys_post_flush[i][j], DummyString(1024));
+      batch.Put(handles_[1], batch_keys_post_flush[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  // On Recovery we should only find the second batch applicable to default CF
+  // But both batches applicable to pikachu CF
+
+  // Create a test filter that would add extra keys
+  TestWalFilterWithColumnFamilies test_wal_filter_column_families;
+
+  // Reopen database with option to use WAL filter
+  options = OptionsForLogIterTest();
+  options.wal_filter = &test_wal_filter_column_families;
+  Status status =
+    TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
+  ASSERT_TRUE(status.ok());
+
+  // verify that handles_[0] only has post_flush keys
+  // while handles_[1] has pre and post flush keys
+  auto cf_wal_keys = test_wal_filter_column_families.GetColumnFamilyKeys();
+  auto name_id_map = 
+    test_wal_filter_column_families.GetColumnFamilyNameIdMap();
+  size_t index = 0;
+  auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]];
+  //default column-family, only post_flush keys are expected
+  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
+    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
+      Slice key_from_the_log(keys_cf[index++]);
+      Slice batch_key(batch_keys_post_flush[i][j]);
+      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
+    }
+  }
+  ASSERT_TRUE(index == keys_cf.size());
+
+  index = 0;
+  keys_cf = cf_wal_keys[name_id_map["pikachu"]];
+  //pikachu column-family, all keys are expected
+  for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
+    for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
+      Slice key_from_the_log(keys_cf[index++]);
+      Slice batch_key(batch_keys_pre_flush[i][j]);
+      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
+    }
+  }
+
+  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
+    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
+      Slice key_from_the_log(keys_cf[index++]);
+      Slice batch_key(batch_keys_post_flush[i][j]);
+      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
+    }
+  }
+  ASSERT_TRUE(index == keys_cf.size());
+}
+#endif  // ROCKSDB_LITE
+
+TEST_F(DBTest2, DISABLED_FirstSnapshotTest) {
+  Options options;
+  options.write_buffer_size = 100000;  // Small write buffer
+  options = CurrentOptions(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  // This snapshot will have sequence number 0.  When compaction encounters
+  // this snapshot, CompactionIterator::findEarliestVisibleSnapshot() will
+  // assert as it expects non-zero snapshots.
+  //
+  // One fix would be to simply remove this assert.  However, a better fix
+  // might
+  // be to always have db sequence numbers start from 1 so that no code is
+  // ever
+  // confused by 0.
+  const Snapshot* s1 = db_->GetSnapshot();
+
+  Put(1, "k1", std::string(100000, 'x'));  // Fill memtable
+  Put(1, "k2", std::string(100000, 'y'));  // Trigger flush
+
+  db_->ReleaseSnapshot(s1);
+}
+
+class PinL0IndexAndFilterBlocksTest : public DBTestBase,
+                                      public testing::WithParamInterface<bool> {
+ public:
+  PinL0IndexAndFilterBlocksTest() : DBTestBase("/db_pin_l0_index_bloom_test") {}
+  virtual void SetUp() override { infinite_max_files_ = GetParam(); }
+
+  bool infinite_max_files_;
+};
+
+TEST_P(PinL0IndexAndFilterBlocksTest,
+       IndexAndFilterBlocksOfNewTableAddedToCacheWithPinning) {
+  Options options = CurrentOptions();
+  if (infinite_max_files_) {
+    options.max_open_files = -1;
+  }
+  options.create_if_missing = true;
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  ASSERT_OK(Put(1, "key", "val"));
+  // Create a new table.
+  ASSERT_OK(Flush(1));
+
+  // index/filter blocks added to block cache right after table creation.
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+
+  // only index/filter were added
+  ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
+
+  std::string value;
+  // Miss and hit count should remain the same, they're all pinned.
+  db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+
+  // Miss and hit count should remain the same, they're all pinned.
+  value = Get(1, "key");
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+}
+
+TEST_P(PinL0IndexAndFilterBlocksTest,
+       MultiLevelIndexAndFilterBlocksCachedWithPinning) {
+  Options options = CurrentOptions();
+  if (infinite_max_files_) {
+    options.max_open_files = -1;
+  }
+  options.create_if_missing = true;
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  Put(1, "a", "begin");
+  Put(1, "z", "end");
+  ASSERT_OK(Flush(1));
+  // move this table to L1
+  dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
+
+  // reset block cache
+  table_options.block_cache = NewLRUCache(64 * 1024);
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  TryReopenWithColumnFamilies({"default", "pikachu"}, options);
+  // create new table at L0
+  Put(1, "a2", "begin2");
+  Put(1, "z2", "end2");
+  ASSERT_OK(Flush(1));
+
+  table_options.block_cache->EraseUnRefEntries();
+
+  // get base cache values
+  uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS);
+  uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
+  uint64_t im = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS);
+  uint64_t ih = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT);
+
+  std::string value;
+  // this should be read from L0
+  // so cache values don't change
+  value = Get(1, "a2");
+  ASSERT_EQ(fm, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
+  ASSERT_EQ(im, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+
+  // this should be read from L1
+  // the file is opened, prefetching results in a cache filter miss
+  // the block is loaded and added to the cache,
+  // then the get results in a cache hit for L1
+  value = Get(1, "a");
+  ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+}
+
+INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest,
+                        PinL0IndexAndFilterBlocksTest, ::testing::Bool());
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -210,6 +210,8 @@ Options DBTestBase::CurrentOptions(
    const anon::OptionsOverride& options_override) {
  Options options;
  options.write_buffer_size = 4090 * 4096;
+  options.target_file_size_base = 2 * 1024 * 1024;
+  options.max_bytes_for_level_base = 10 * 1024 * 1024;
  return CurrentOptions(options, options_override);
 }

@@ -301,6 +303,7 @@ Options DBTestBase::CurrentOptions(
    case kPerfOptions:
      options.soft_rate_limit = 2.0;
      options.delayed_write_rate = 8 * 1024 * 1024;
+      options.report_bg_io_stats = true;
      // TODO(3.13) -- test more options
      break;
    case kDeletesFilterFirst:
@@ -124,8 +124,7 @@ class DelayFilterFactory : public CompactionFilterFactory {
 // Make sure we don't trigger a problem if the trigger conditon is given
 // to be 0, which is invalid.
 TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();

  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = num_levels_;
@@ -160,8 +159,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
 }

 TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.compaction_options_universal.size_ratio = 5;
  options.num_levels = num_levels_;
@@ -337,13 +335,12 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
 }

 TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = num_levels_;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.target_file_size_base = 32 << 10;  // 32KB
  options.level0_file_num_compaction_trigger = 3;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -388,12 +385,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
  ChangeCompactOptions();
  Options options;
  options.create_if_missing = true;
-  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
  options.compaction_style = kCompactionStyleLevel;
  options.num_levels = 1;
  options.target_file_size_base = options.write_buffer_size;
  options.compression = kNoCompression;
  options = CurrentOptions(options);
+  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
  CreateAndReopenWithCF({"pikachu"}, options);
  ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
  Random rnd(301);
@@ -451,12 +448,11 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
 }

 TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.num_levels = 7;
  options.disable_auto_compactions = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // Generate 3 overlapping files
@@ -496,14 +492,13 @@ class DBTestUniversalCompactionMultiLevels
 };

 TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = num_levels_;
  options.write_buffer_size = 100 << 10;  // 100KB
  options.level0_file_num_compaction_trigger = 8;
  options.max_background_compactions = 3;
  options.target_file_size_base = 32 * 1024;
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  // Trigger compaction if size amplification exceeds 110%
@@ -539,7 +534,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
      });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.compaction_options_universal.allow_trivial_move = true;
  options.num_levels = 3;
@@ -547,7 +542,6 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
  options.level0_file_num_compaction_trigger = 3;
  options.max_background_compactions = 2;
  options.target_file_size_base = 32 * 1024;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -586,7 +580,7 @@ class DBTestUniversalCompactionParallel :
 };

 TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = num_levels_;
  options.write_buffer_size = 1 << 10;  // 1KB
@@ -595,7 +589,6 @@ TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
  options.max_background_flushes = 3;
  options.target_file_size_base = 1 * 1024;
  options.compaction_options_universal.max_size_amplification_percent = 110;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -652,7 +645,7 @@ INSTANTIATE_TEST_CASE_P(DBTestUniversalCompactionParallel,
                                           ::testing::Bool()));

 TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.write_buffer_size = 105 << 10;    // 105KB
  options.arena_block_size = 4 << 10;       // 4KB
@@ -660,7 +653,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
  options.level0_file_num_compaction_trigger = 4;
  options.num_levels = num_levels_;
  options.compaction_options_universal.compression_size_percent = -1;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -773,14 +765,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
    return;
  }

-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.target_file_size_base = 32 << 10;  // 32KB
  options.level0_file_num_compaction_trigger = 2;
  options.num_levels = num_levels_;
  options.compaction_options_universal.compression_size_percent = 70;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  Random rnd(301);
@@ -841,14 +832,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
  if (!Snappy_Supported()) {
    return;
  }
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.target_file_size_base = 32 << 10;  // 32KB
  options.level0_file_num_compaction_trigger = 2;
  options.num_levels = num_levels_;
  options.compaction_options_universal.compression_size_percent = 95;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  Random rnd(301);
@@ -884,7 +874,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
      });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.compaction_options_universal.allow_trivial_move = true;
  options.num_levels = 2;
@@ -892,7 +882,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
  options.level0_file_num_compaction_trigger = 3;
  options.max_background_compactions = 1;
  options.target_file_size_base = 32 * 1024;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -929,7 +918,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {

  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.compaction_options_universal.allow_trivial_move = true;
  options.num_levels = 15;
@@ -937,7 +926,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
  options.level0_file_num_compaction_trigger = 8;
  options.max_background_compactions = 4;
  options.target_file_size_base = 64 * 1024;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -963,7 +951,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
 }

 TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
-  Options options;
+  Options options = CurrentOptions();
  options.db_paths.emplace_back(dbname_, 300 * 1024);
  options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024);
  options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
@@ -976,7 +964,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
  options.arena_block_size = 4 << 10;
  options.level0_file_num_compaction_trigger = 2;
  options.num_levels = 1;
-  options = CurrentOptions(options);

  std::vector<std::string> filenames;
  env_->GetChildren(options.db_paths[1].path, &filenames);
@@ -1163,7 +1150,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
  if (!Snappy_Supported()) {
    return;
  }
-  Options options;
+  Options options = CurrentOptions();
  options.db_paths.emplace_back(dbname_, 500 * 1024);
  options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024);
  options.compaction_style = kCompactionStyleUniversal;
@@ -1175,7 +1162,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
  options.num_levels = 1;
  options.memtable_factory.reset(
      new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
-  options = CurrentOptions(options);

  std::vector<std::string> filenames;
  env_->GetChildren(options.db_paths[1].path, &filenames);
@@ -465,6 +465,12 @@ class InternalKeySliceTransform : public SliceTransform {
  const SliceTransform* const transform_;
 };

+// Read the key of a record from a write batch.
+// if this record represent the default column family then cf_record
+// must be passed as false, otherwise it must be passed as true.
+extern bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key,
+                                       bool cf_record);
+
 // Read record from a write batch piece from input.
 // tag, column_family, key, value and blob are return values. Callers own the
 // Slice they point to.
@@ -66,7 +66,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                   JobContext* job_context, LogBuffer* log_buffer,
                   Directory* db_directory, Directory* output_file_directory,
                   CompressionType output_compression, Statistics* stats,
-                   EventLogger* event_logger)
+                   EventLogger* event_logger, bool measure_io_stats)
    : dbname_(dbname),
      cfd_(cfd),
      db_options_(db_options),
@@ -83,7 +83,8 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
      output_file_directory_(output_file_directory),
      output_compression_(output_compression),
      stats_(stats),
-      event_logger_(event_logger) {
+      event_logger_(event_logger),
+      measure_io_stats_(measure_io_stats) {
  // Update the thread status to indicate flush.
  ReportStartedFlush();
  TEST_SYNC_POINT("FlushJob::FlushJob()");
@@ -121,6 +122,21 @@ void FlushJob::RecordFlushIOStats() {
 Status FlushJob::Run(FileMetaData* file_meta) {
  AutoThreadOperationStageUpdater stage_run(
      ThreadStatus::STAGE_FLUSH_RUN);
+  // I/O measurement variables
+  PerfLevel prev_perf_level = PerfLevel::kEnableTime;
+  uint64_t prev_write_nanos = 0;
+  uint64_t prev_fsync_nanos = 0;
+  uint64_t prev_range_sync_nanos = 0;
+  uint64_t prev_prepare_write_nanos = 0;
+  if (measure_io_stats_) {
+    prev_perf_level = GetPerfLevel();
+    SetPerfLevel(PerfLevel::kEnableTime);
+    prev_write_nanos = IOSTATS(write_nanos);
+    prev_fsync_nanos = IOSTATS(fsync_nanos);
+    prev_range_sync_nanos = IOSTATS(range_sync_nanos);
+    prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
+  }
+
  // Save the contents of the earliest memtable as a new Table
  FileMetaData meta;
  autovector<MemTable*> mems;
@@ -180,6 +196,18 @@ Status FlushJob::Run(FileMetaData* file_meta) {
  }
  stream.EndArray();

+  if (measure_io_stats_) {
+    if (prev_perf_level != PerfLevel::kEnableTime) {
+      SetPerfLevel(prev_perf_level);
+    }
+    stream << "file_write_nanos" << (IOSTATS(write_nanos) - prev_write_nanos);
+    stream << "file_range_sync_nanos"
+           << (IOSTATS(range_sync_nanos) - prev_range_sync_nanos);
+    stream << "file_fsync_nanos" << (IOSTATS(fsync_nanos) - prev_fsync_nanos);
+    stream << "file_prepare_write_nanos"
+           << (IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos);
+  }
+
  return s;
 }

@@ -234,14 +262,15 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,

      TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:output_compression",
                               &output_compression_);
-      s = BuildTable(dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
-                     cfd_->table_cache(), iter.get(), meta,
-                     cfd_->internal_comparator(),
-                     cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
-                     existing_snapshots_, earliest_write_conflict_snapshot_,
-                     output_compression_, cfd_->ioptions()->compression_opts,
-                     mutable_cf_options_.paranoid_file_checks,
-                     cfd_->internal_stats(), Env::IO_HIGH, &table_properties_);
+      s = BuildTable(
+          dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
+          cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(),
+          cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
+          cfd_->GetName(), existing_snapshots_,
+          earliest_write_conflict_snapshot_, output_compression_,
+          cfd_->ioptions()->compression_opts,
+          mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
+          Env::IO_HIGH, &table_properties_, 0 /* level */);
      info.table_properties = table_properties_;
      LogFlush(db_options_.info_log);
    }
@@ -62,7 +62,7 @@ class FlushJob {
           JobContext* job_context, LogBuffer* log_buffer,
           Directory* db_directory, Directory* output_file_directory,
           CompressionType output_compression, Statistics* stats,
-           EventLogger* event_logger);
+           EventLogger* event_logger, bool measure_io_stats);

  ~FlushJob();

@@ -93,6 +93,7 @@ class FlushJob {
  Statistics* stats_;
  EventLogger* event_logger_;
  TableProperties table_properties_;
+  bool measure_io_stats_;
 };

 }  // namespace rocksdb
@@ -93,7 +93,7 @@ TEST_F(FlushJobTest, Empty) {
                     db_options_, *cfd->GetLatestMutableCFOptions(),
                     env_options_, versions_.get(), &mutex_, &shutting_down_,
                     {}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
-                     nullptr, kNoCompression, nullptr, &event_logger);
+                     nullptr, kNoCompression, nullptr, &event_logger, false);
  ASSERT_OK(flush_job.Run());
  job_context.Clean();
 }
@@ -132,7 +132,7 @@ TEST_F(FlushJobTest, NonEmpty) {
                     db_options_, *cfd->GetLatestMutableCFOptions(),
                     env_options_, versions_.get(), &mutex_, &shutting_down_,
                     {}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
-                     nullptr, kNoCompression, nullptr, &event_logger);
+                     nullptr, kNoCompression, nullptr, &event_logger, true);
  FileMetaData fd;
  mutex_.Lock();
  ASSERT_OK(flush_job.Run(&fd));
@@ -192,11 +192,11 @@ TEST_F(FlushJobTest, Snapshots) {
  }

  EventLogger event_logger(db_options_.info_log.get());
-  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
-                     db_options_, *cfd->GetLatestMutableCFOptions(),
-                     env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     snapshots, kMaxSequenceNumber, &job_context, nullptr,
-                     nullptr, nullptr, kNoCompression, nullptr, &event_logger);
+  FlushJob flush_job(
+      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
+      *cfd->GetLatestMutableCFOptions(), env_options_, versions_.get(), &mutex_,
+      &shutting_down_, snapshots, kMaxSequenceNumber, &job_context, nullptr,
+      nullptr, nullptr, kNoCompression, nullptr, &event_logger, true);
  mutex_.Lock();
  ASSERT_OK(flush_job.Run());
  mutex_.Unlock();
@@ -22,9 +22,9 @@ class MemTable;
 struct JobContext {
  inline bool HaveSomethingToDelete() const {
    return full_scan_candidate_files.size() || sst_delete_files.size() ||
-           log_delete_files.size() || new_superversion != nullptr ||
-           superversions_to_free.size() > 0 || memtables_to_free.size() > 0 ||
-           logs_to_free.size() > 0;
+           log_delete_files.size() || manifest_delete_files.size() ||
+           new_superversion != nullptr || superversions_to_free.size() > 0 ||
+           memtables_to_free.size() > 0 || logs_to_free.size() > 0;
  }

  // Structure to store information for candidate files to delete.
@@ -56,6 +56,9 @@ struct JobContext {
  // a list of log files that we need to delete
  std::vector<uint64_t> log_delete_files;

+  // a list of manifest files that we need to delete
+  std::vector<std::string> manifest_delete_files;
+
  // a list of memtables to be free
  autovector<MemTable*> memtables_to_free;

@@ -79,7 +79,7 @@ ManagedIterator::ManagedIterator(DBImpl* db, const ReadOptions& read_options,
      release_supported_(true) {
  read_options_.managed = false;
  if ((!read_options_.tailing) && (read_options_.snapshot == nullptr)) {
-    assert(read_options_.snapshot = db_->GetSnapshot());
+    assert(nullptr != (read_options_.snapshot = db_->GetSnapshot()));
    snapshot_created_ = true;
  }
  cfh_.SetCFD(cfd);
@@ -210,7 +210,8 @@ void ManagedIterator::RebuildIterator() {
 void ManagedIterator::UpdateCurrent() {
  assert(mutable_iter_ != nullptr);

-  if (!(valid_ = mutable_iter_->Valid())) {
+  valid_ = mutable_iter_->Valid();
+  if (!valid_) {
    status_ = mutable_iter_->status();
    return;
  }
@@ -345,8 +345,8 @@ Status MemTableList::InstallMemtableFlushResults(
        imm_flush_needed.store(true, std::memory_order_release);
      }
      ++mem_id;
-    } while (!current_->memlist_.empty() && (m = current_->memlist_.back()) &&
-             m->file_number_ == file_number);
+    } while (!current_->memlist_.empty() && (nullptr != (m = current_->memlist_.back())) &&
+             (m->file_number_ == file_number));
  }
  commit_in_progress_ = false;
  return s;
@@ -37,6 +37,11 @@ public:
    Initialize();
    operand_list->push_front(operand_slice.ToString());
  }
+  // Push back a merge operand
+  void PushOperandBack(const Slice& operand_slice) {
+    Initialize();
+    operand_list->push_back(operand_slice.ToString());
+  }
  // return total number of operands in the list
  size_t GetNumOperands() const {
    if (!operand_list) {
@@ -9,16 +9,16 @@
 #include <vector>

 #include "rocksdb/db.h"
+#include "rocksdb/memtablerep.h"
 #include "rocksdb/perf_context.h"
 #include "rocksdb/slice_transform.h"
-#include "rocksdb/memtablerep.h"
 #include "util/histogram.h"
 #include "util/instrumented_mutex.h"
 #include "util/stop_watch.h"
+#include "util/string_util.h"
 #include "util/testharness.h"
 #include "util/thread_status_util.h"
-#include "util/string_util.h"
-
+#include "utilities/merge_operators.h"

 bool FLAGS_random_key = false;
 bool FLAGS_use_set_based_memetable = false;
@@ -602,6 +602,41 @@ TEST_F(PerfContextTest, ToString) {
  ASSERT_EQ(std::string::npos, zero_excluded.find("= 0"));
  ASSERT_NE(std::string::npos, zero_excluded.find("= 12345"));
 }
+
+TEST_F(PerfContextTest, MergeOperatorTime) {
+  DestroyDB(kDbName, Options());
+  DB* db;
+  Options options;
+  options.create_if_missing = true;
+  options.merge_operator = MergeOperators::CreateStringAppendOperator();
+  Status s = DB::Open(options, kDbName, &db);
+  EXPECT_OK(s);
+
+  std::string val;
+  ASSERT_OK(db->Merge(WriteOptions(), "k1", "val1"));
+  ASSERT_OK(db->Merge(WriteOptions(), "k1", "val2"));
+  ASSERT_OK(db->Merge(WriteOptions(), "k1", "val3"));
+  ASSERT_OK(db->Merge(WriteOptions(), "k1", "val4"));
+
+  SetPerfLevel(kEnableTime);
+  perf_context.Reset();
+  ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+  EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
+
+  ASSERT_OK(db->Flush(FlushOptions()));
+
+  perf_context.Reset();
+  ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+  EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
+
+  ASSERT_OK(db->CompactRange(CompactRangeOptions(), nullptr, nullptr));
+
+  perf_context.Reset();
+  ASSERT_OK(db->Get(ReadOptions(), "k1", &val));
+  EXPECT_GT(perf_context.merge_operator_time_nanos, 0);
+
+  delete db;
+}
 }

 int main(int argc, char** argv) {
@@ -263,7 +263,9 @@ class TestPlainTableReader : public PlainTableReader {
                       const TableProperties* table_properties,
                       unique_ptr<RandomAccessFileReader>&& file,
                       const ImmutableCFOptions& ioptions,
-                       bool* expect_bloom_not_match, bool store_index_in_file)
+                       bool* expect_bloom_not_match, bool store_index_in_file,
+                       uint32_t column_family_id,
+                       const std::string& column_family_name)
      : PlainTableReader(ioptions, std::move(file), env_options, icomparator,
                         encoding_type, file_size, table_properties),
        expect_bloom_not_match_(expect_bloom_not_match) {
@@ -276,6 +278,8 @@ class TestPlainTableReader : public PlainTableReader {
    EXPECT_TRUE(s.ok());

    TableProperties* props = const_cast<TableProperties*>(table_properties);
+    EXPECT_EQ(column_family_id, static_cast<uint32_t>(props->column_family_id));
+    EXPECT_EQ(column_family_name, props->column_family_name);
    if (store_index_in_file) {
      auto bloom_version_ptr = props->user_collected_properties.find(
          PlainTablePropertyNames::kBloomVersion);
@@ -308,13 +312,17 @@ extern const uint64_t kPlainTableMagicNumber;
 class TestPlainTableFactory : public PlainTableFactory {
 public:
  explicit TestPlainTableFactory(bool* expect_bloom_not_match,
-                                 const PlainTableOptions& options)
+                                 const PlainTableOptions& options,
+                                 uint32_t column_family_id,
+                                 std::string column_family_name)
      : PlainTableFactory(options),
        bloom_bits_per_key_(options.bloom_bits_per_key),
        hash_table_ratio_(options.hash_table_ratio),
        index_sparseness_(options.index_sparseness),
        store_index_in_file_(options.store_index_in_file),
-        expect_bloom_not_match_(expect_bloom_not_match) {}
+        expect_bloom_not_match_(expect_bloom_not_match),
+        column_family_id_(column_family_id),
+        column_family_name_(std::move(column_family_name)) {}

  Status NewTableReader(const TableReaderOptions& table_reader_options,
                        unique_ptr<RandomAccessFileReader>&& file,
@@ -354,7 +362,7 @@ class TestPlainTableFactory : public PlainTableFactory {
        table_reader_options.internal_comparator, encoding_type, file_size,
        bloom_bits_per_key_, hash_table_ratio_, index_sparseness_, props,
        std::move(file), table_reader_options.ioptions, expect_bloom_not_match_,
-        store_index_in_file_));
+        store_index_in_file_, column_family_id_, column_family_name_));

    *table = std::move(new_reader);
    return s;
@@ -366,6 +374,8 @@ class TestPlainTableFactory : public PlainTableFactory {
  size_t index_sparseness_;
  bool store_index_in_file_;
  bool* expect_bloom_not_match_;
+  const uint32_t column_family_id_;
+  const std::string column_family_name_;
 };

 TEST_P(PlainTableDBTest, Flush) {
@@ -492,7 +502,8 @@ TEST_P(PlainTableDBTest, Flush2) {
        plain_table_options.encoding_type = encoding_type;
        plain_table_options.store_index_in_file = store_index_in_file;
        options.table_factory.reset(new TestPlainTableFactory(
-            &expect_bloom_not_match, plain_table_options));
+            &expect_bloom_not_match, plain_table_options,
+            0 /* column_family_id */, kDefaultColumnFamilyName));

        DestroyAndReopen(&options);
        ASSERT_OK(Put("0000000000000bar", "b"));
@@ -561,7 +572,8 @@ TEST_P(PlainTableDBTest, Iterator) {
          plain_table_options.encoding_type = encoding_type;

          options.table_factory.reset(new TestPlainTableFactory(
-              &expect_bloom_not_match, plain_table_options));
+              &expect_bloom_not_match, plain_table_options,
+              0 /* column_family_id */, kDefaultColumnFamilyName));
        } else {
          PlainTableOptions plain_table_options;
          plain_table_options.user_key_len = 16;
@@ -572,7 +584,8 @@ TEST_P(PlainTableDBTest, Iterator) {
          plain_table_options.encoding_type = encoding_type;

          options.table_factory.reset(new TestPlainTableFactory(
-              &expect_bloom_not_match, plain_table_options));
+              &expect_bloom_not_match, plain_table_options,
+              0 /* column_family_id */, kDefaultColumnFamilyName));
        }
        DestroyAndReopen(&options);

@@ -265,7 +265,7 @@ class Repairer {
    mem->Ref();
    int counter = 0;
    while (reader.ReadRecord(&record, &scratch)) {
-      if (record.size() < 12) {
+      if (record.size() < WriteBatchInternal::kHeader) {
        reporter.Corruption(
            record.size(), Status::Corruption("log record too small"));
        continue;
@@ -294,9 +294,10 @@ class Repairer {
      status = BuildTable(
          dbname_, env_, ioptions_, env_options_, table_cache_, iter.get(),
          &meta, icmp_, &int_tbl_prop_collector_factories_,
-          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, {},
-          kMaxSequenceNumber, kNoCompression, CompressionOptions(), false,
-          nullptr);
+          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily,
+          std::string() /* column_family_name */, {}, kMaxSequenceNumber,
+          kNoCompression, CompressionOptions(), false,
+          nullptr /* internal_stats */);
    }
    delete mem->Unref();
    delete cf_mems_default;
@@ -0,0 +1,177 @@
+//  Copyright (c) 2016-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include "db/db_impl.h"
+#include "db/db_test_util.h"
+#include "rocksdb/db.h"
+#include "rocksdb/transaction_log.h"
+#include "util/file_util.h"
+
+namespace rocksdb {
+
+class RepairTest : public DBTestBase {
+ public:
+  RepairTest() : DBTestBase("/repair_test") {}
+
+  std::string GetFirstSstPath() {
+    uint64_t manifest_size;
+    std::vector<std::string> files;
+    db_->GetLiveFiles(files, &manifest_size);
+    auto sst_iter =
+        std::find_if(files.begin(), files.end(), [](const std::string& file) {
+          uint64_t number;
+          FileType type;
+          bool ok = ParseFileName(file, &number, &type);
+          return ok && type == kTableFile;
+        });
+    return sst_iter == files.end() ? "" : dbname_ + *sst_iter;
+  }
+};
+
+TEST_F(RepairTest, LostManifest) {
+  // Add a couple SST files, delete the manifest, and verify RepairDB() saves
+  // the day.
+  Put("key", "val");
+  Flush();
+  Put("key2", "val2");
+  Flush();
+  // Need to get path before Close() deletes db_, but delete it after Close() to
+  // ensure Close() didn't change the manifest.
+  std::string manifest_path =
+      DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
+
+  Close();
+  ASSERT_OK(env_->FileExists(manifest_path));
+  ASSERT_OK(env_->DeleteFile(manifest_path));
+  RepairDB(dbname_, CurrentOptions());
+  Reopen(CurrentOptions());
+
+  ASSERT_EQ(Get("key"), "val");
+  ASSERT_EQ(Get("key2"), "val2");
+}
+
+TEST_F(RepairTest, CorruptManifest) {
+  // Manifest is in an invalid format. Expect a full recovery.
+  Put("key", "val");
+  Flush();
+  Put("key2", "val2");
+  Flush();
+  // Need to get path before Close() deletes db_, but overwrite it after Close()
+  // to ensure Close() didn't change the manifest.
+  std::string manifest_path =
+      DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
+
+  Close();
+  ASSERT_OK(env_->FileExists(manifest_path));
+  CreateFile(env_, manifest_path, "blah");
+  RepairDB(dbname_, CurrentOptions());
+  Reopen(CurrentOptions());
+
+  ASSERT_EQ(Get("key"), "val");
+  ASSERT_EQ(Get("key2"), "val2");
+}
+
+TEST_F(RepairTest, IncompleteManifest) {
+  // In this case, the manifest is valid but does not reference all of the SST
+  // files. Expect a full recovery.
+  Put("key", "val");
+  Flush();
+  std::string orig_manifest_path =
+      DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
+  CopyFile(orig_manifest_path, orig_manifest_path + ".tmp");
+  Put("key2", "val2");
+  Flush();
+  // Need to get path before Close() deletes db_, but overwrite it after Close()
+  // to ensure Close() didn't change the manifest.
+  std::string new_manifest_path =
+      DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
+
+  Close();
+  ASSERT_OK(env_->FileExists(new_manifest_path));
+  // Replace the manifest with one that is only aware of the first SST file.
+  CopyFile(orig_manifest_path + ".tmp", new_manifest_path);
+  RepairDB(dbname_, CurrentOptions());
+  Reopen(CurrentOptions());
+
+  ASSERT_EQ(Get("key"), "val");
+  ASSERT_EQ(Get("key2"), "val2");
+}
+
+TEST_F(RepairTest, LostSst) {
+  // Delete one of the SST files but preserve the manifest that refers to it,
+  // then verify the DB is still usable for the intact SST.
+  Put("key", "val");
+  Flush();
+  Put("key2", "val2");
+  Flush();
+  auto sst_path = GetFirstSstPath();
+  ASSERT_FALSE(sst_path.empty());
+  ASSERT_OK(env_->DeleteFile(sst_path));
+
+  Close();
+  RepairDB(dbname_, CurrentOptions());
+  Reopen(CurrentOptions());
+
+  // Exactly one of the key-value pairs should be in the DB now.
+  ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
+}
+
+TEST_F(RepairTest, CorruptSst) {
+  // Corrupt one of the SST files but preserve the manifest that refers to it,
+  // then verify the DB is still usable for the intact SST.
+  Put("key", "val");
+  Flush();
+  Put("key2", "val2");
+  Flush();
+  auto sst_path = GetFirstSstPath();
+  ASSERT_FALSE(sst_path.empty());
+  CreateFile(env_, sst_path, "blah");
+
+  Close();
+  RepairDB(dbname_, CurrentOptions());
+  Reopen(CurrentOptions());
+
+  // Exactly one of the key-value pairs should be in the DB now.
+  ASSERT_TRUE((Get("key") == "val") != (Get("key2") == "val2"));
+}
+
+TEST_F(RepairTest, UnflushedSst) {
+  // This test case invokes repair while some data is unflushed, then verifies
+  // that data is in the db.
+  Put("key", "val");
+  VectorLogPtr wal_files;
+  ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
+  ASSERT_EQ(wal_files.size(), 1);
+  uint64_t total_ssts_size;
+  GetAllSSTFiles(&total_ssts_size);
+  ASSERT_EQ(total_ssts_size, 0);
+  // Need to get path before Close() deletes db_, but delete it after Close() to
+  // ensure Close() didn't change the manifest.
+  std::string manifest_path =
+      DescriptorFileName(dbname_, dbfull()->TEST_Current_Manifest_FileNo());
+
+  Close();
+  ASSERT_OK(env_->FileExists(manifest_path));
+  ASSERT_OK(env_->DeleteFile(manifest_path));
+  RepairDB(dbname_, CurrentOptions());
+  Reopen(CurrentOptions());
+
+  ASSERT_OK(dbfull()->GetSortedWalFiles(wal_files));
+  ASSERT_EQ(wal_files.size(), 0);
+  GetAllSSTFiles(&total_ssts_size);
+  ASSERT_GT(total_ssts_size, 0);
+  ASSERT_EQ(Get("key"), "val");
+}
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -88,7 +88,7 @@ Status TableCache::GetTableReader(
    const EnvOptions& env_options,
    const InternalKeyComparator& internal_comparator, const FileDescriptor& fd,
    bool sequential_mode, bool record_read_stats, HistogramImpl* file_read_hist,
-    unique_ptr<TableReader>* table_reader, bool skip_filters) {
+    unique_ptr<TableReader>* table_reader, bool skip_filters, int level) {
  std::string fname =
      TableFileName(ioptions_.db_paths, fd.GetNumber(), fd.GetPathId());
  unique_ptr<RandomAccessFile> file;
@@ -109,18 +109,26 @@ Status TableCache::GetTableReader(
                                   file_read_hist));
    s = ioptions_.table_factory->NewTableReader(
        TableReaderOptions(ioptions_, env_options, internal_comparator,
-                           skip_filters),
+                           skip_filters, level),
        std::move(file_reader), fd.GetFileSize(), table_reader);
    TEST_SYNC_POINT("TableCache::GetTableReader:0");
  }
  return s;
 }

+void TableCache::EraseHandle(const FileDescriptor& fd, Cache::Handle* handle) {
+  ReleaseHandle(handle);
+  uint64_t number = fd.GetNumber();
+  Slice key = GetSliceForFileNumber(&number);
+  cache_->Erase(key);
+}
+
 Status TableCache::FindTable(const EnvOptions& env_options,
                             const InternalKeyComparator& internal_comparator,
                             const FileDescriptor& fd, Cache::Handle** handle,
                             const bool no_io, bool record_read_stats,
-                             HistogramImpl* file_read_hist, bool skip_filters) {
+                             HistogramImpl* file_read_hist, bool skip_filters,
+                             int level) {
  PERF_TIMER_GUARD(find_table_nanos);
  Status s;
  uint64_t number = fd.GetNumber();
@@ -136,15 +144,19 @@ Status TableCache::FindTable(const EnvOptions& env_options,
    unique_ptr<TableReader> table_reader;
    s = GetTableReader(env_options, internal_comparator, fd,
                       false /* sequential mode */, record_read_stats,
-                       file_read_hist, &table_reader, skip_filters);
+                       file_read_hist, &table_reader, skip_filters, level);
    if (!s.ok()) {
      assert(table_reader == nullptr);
      RecordTick(ioptions_.statistics, NO_FILE_ERRORS);
      // We do not cache error results so that if the error is transient,
      // or somebody repairs the file, we recover automatically.
    } else {
-      *handle = cache_->Insert(key, table_reader.release(), 1,
-                               &DeleteEntry<TableReader>);
+      s = cache_->Insert(key, table_reader.get(), 1, &DeleteEntry<TableReader>,
+                         handle);
+      if (s.ok()) {
+        // Release ownership of table reader.
+        table_reader.release();
+      }
    }
  }
  return s;
@@ -154,7 +166,7 @@ InternalIterator* TableCache::NewIterator(
    const ReadOptions& options, const EnvOptions& env_options,
    const InternalKeyComparator& icomparator, const FileDescriptor& fd,
    TableReader** table_reader_ptr, HistogramImpl* file_read_hist,
-    bool for_compaction, Arena* arena, bool skip_filters) {
+    bool for_compaction, Arena* arena, bool skip_filters, int level) {
  PERF_TIMER_GUARD(new_table_iterator_nanos);

  if (table_reader_ptr != nullptr) {
@@ -169,7 +181,8 @@ InternalIterator* TableCache::NewIterator(
    unique_ptr<TableReader> table_reader_unique_ptr;
    Status s = GetTableReader(
        env_options, icomparator, fd, /* sequential mode */ true,
-        /* record stats */ false, nullptr, &table_reader_unique_ptr);
+        /* record stats */ false, nullptr, &table_reader_unique_ptr,
+        false /* skip_filters */, level);
    if (!s.ok()) {
      return NewErrorInternalIterator(s, arena);
    }
@@ -180,7 +193,7 @@ InternalIterator* TableCache::NewIterator(
      Status s = FindTable(env_options, icomparator, fd, &handle,
                           options.read_tier == kBlockCacheTier /* no_io */,
                           !for_compaction /* record read_stats */,
-                           file_read_hist, skip_filters);
+                           file_read_hist, skip_filters, level);
      if (!s.ok()) {
        return NewErrorInternalIterator(s, arena);
      }
@@ -212,7 +225,7 @@ Status TableCache::Get(const ReadOptions& options,
                       const InternalKeyComparator& internal_comparator,
                       const FileDescriptor& fd, const Slice& k,
                       GetContext* get_context, HistogramImpl* file_read_hist,
-                       bool skip_filters) {
+                       bool skip_filters, int level) {
  TableReader* t = fd.table_reader;
  Status s;
  Cache::Handle* handle = nullptr;
@@ -261,7 +274,8 @@ Status TableCache::Get(const ReadOptions& options,
  if (!t) {
    s = FindTable(env_options_, internal_comparator, fd, &handle,
                  options.read_tier == kBlockCacheTier /* no_io */,
-                  true /* record_read_stats */, file_read_hist, skip_filters);
+                  true /* record_read_stats */, file_read_hist, skip_filters,
+                  level);
    if (s.ok()) {
      t = GetTableReaderFromHandle(handle);
    }
@@ -285,9 +299,8 @@ Status TableCache::Get(const ReadOptions& options,
    size_t charge =
        row_cache_key.Size() + row_cache_entry->size() + sizeof(std::string);
    void* row_ptr = new std::string(std::move(*row_cache_entry));
-    auto row_handle = ioptions_.row_cache->Insert(
-        row_cache_key.GetKey(), row_ptr, charge, &DeleteEntry<std::string>);
-    ioptions_.row_cache->Release(row_handle);
+    ioptions_.row_cache->Insert(row_cache_key.GetKey(), row_ptr, charge,
+                                &DeleteEntry<std::string>);
  }
 #endif  // ROCKSDB_LITE

@@ -45,34 +45,41 @@ class TableCache {
  // the cache and should not be deleted, and is valid for as long as the
  // returned iterator is live.
  // @param skip_filters Disables loading/accessing the filter block
+  // @param level The level this table is at, -1 for "not set / don't know"
  InternalIterator* NewIterator(
      const ReadOptions& options, const EnvOptions& toptions,
      const InternalKeyComparator& internal_comparator,
      const FileDescriptor& file_fd, TableReader** table_reader_ptr = nullptr,
      HistogramImpl* file_read_hist = nullptr, bool for_compaction = false,
-      Arena* arena = nullptr, bool skip_filters = false);
+      Arena* arena = nullptr, bool skip_filters = false, int level = -1);

  // If a seek to internal key "k" in specified file finds an entry,
  // call (*handle_result)(arg, found_key, found_value) repeatedly until
  // it returns false.
  // @param skip_filters Disables loading/accessing the filter block
+  // @param level The level this table is at, -1 for "not set / don't know"
  Status Get(const ReadOptions& options,
             const InternalKeyComparator& internal_comparator,
             const FileDescriptor& file_fd, const Slice& k,
             GetContext* get_context, HistogramImpl* file_read_hist = nullptr,
-             bool skip_filters = false);
+             bool skip_filters = false, int level = -1);

  // Evict any entry for the specified file number
  static void Evict(Cache* cache, uint64_t file_number);

+  // Clean table handle and erase it from the table cache
+  // Used in DB close, or the file is not live anymore.
+  void EraseHandle(const FileDescriptor& fd, Cache::Handle* handle);
+
  // Find table reader
  // @param skip_filters Disables loading/accessing the filter block
+  // @param level == -1 means not specified
  Status FindTable(const EnvOptions& toptions,
                   const InternalKeyComparator& internal_comparator,
                   const FileDescriptor& file_fd, Cache::Handle**,
                   const bool no_io = false, bool record_read_stats = true,
                   HistogramImpl* file_read_hist = nullptr,
-                   bool skip_filters = false);
+                   bool skip_filters = false, int level = -1);

  // Get TableReader from a cache handle.
  TableReader* GetTableReaderFromHandle(Cache::Handle* handle);
@@ -106,7 +113,7 @@ class TableCache {
                        const FileDescriptor& fd, bool sequential_mode,
                        bool record_read_stats, HistogramImpl* file_read_hist,
                        unique_ptr<TableReader>* table_reader,
-                        bool skip_filters = false);
+                        bool skip_filters = false, int level = -1);

  const ImmutableCFOptions& ioptions_;
  const EnvOptions& env_options_;
@@ -36,6 +36,7 @@ class TablePropertiesTest : public testing::Test,
 // Utilities test functions
 namespace {
 static const uint32_t kTestColumnFamilyId = 66;
+static const std::string kTestColumnFamilyName = "test_column_fam";

 void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,
                 const InternalKeyComparator& internal_comparator,
@@ -48,8 +49,8 @@ void MakeBuilder(const Options& options, const ImmutableCFOptions& ioptions,

  builder->reset(NewTableBuilder(
      ioptions, internal_comparator, int_tbl_prop_collector_factories,
-      kTestColumnFamilyId /* column_family_id */, writable->get(),
-      options.compression, options.compression_opts));
+      kTestColumnFamilyId, kTestColumnFamilyName,
+      writable->get(), options.compression, options.compression_opts));
 }
 }  // namespace

@@ -107,7 +107,7 @@ void TransactionLogIteratorImpl::SeekToStartSequence(
    return;
  }
  while (RestrictedRead(&record, &scratch)) {
-    if (record.size() < 12) {
+    if (record.size() < WriteBatchInternal::kHeader) {
      reporter_.Corruption(
        record.size(), Status::Corruption("very small log record"));
      continue;
@@ -167,7 +167,7 @@ void TransactionLogIteratorImpl::NextImpl(bool internal) {
      currentLogReader_->UnmarkEOF();
    }
    while (RestrictedRead(&record, &scratch)) {
-      if (record.size() < 12) {
+      if (record.size() < WriteBatchInternal::kHeader) {
        reporter_.Corruption(
          record.size(), Status::Corruption("very small log record"));
        continue;
@@ -309,11 +309,11 @@ class VersionBuilder::Rep {

        auto* file_meta = files_meta[file_idx].first;
        int level = files_meta[file_idx].second;
-        table_cache_->FindTable(env_options_,
-                                *(base_vstorage_->InternalComparator()),
-                                file_meta->fd, &file_meta->table_reader_handle,
-                                false /*no_io */, true /* record_read_stats */,
-                                internal_stats->GetFileReadHist(level));
+        table_cache_->FindTable(
+            env_options_, *(base_vstorage_->InternalComparator()),
+            file_meta->fd, &file_meta->table_reader_handle, false /*no_io */,
+            true /* record_read_stats */,
+            internal_stats->GetFileReadHist(level), false, level);
        if (file_meta->table_reader_handle != nullptr) {
          // Load table_reader
          file_meta->fd.table_reader = table_cache_->GetTableReaderFromHandle(
@@ -23,6 +23,7 @@
 #include <vector>
 #include <string>

+#include "db/compaction.h"
 #include "db/filename.h"
 #include "db/internal_stats.h"
 #include "db/log_reader.h"
@@ -30,22 +31,22 @@
 #include "db/memtable.h"
 #include "db/merge_context.h"
 #include "db/table_cache.h"
-#include "db/compaction.h"
 #include "db/version_builder.h"
 #include "db/writebuffer.h"
 #include "rocksdb/env.h"
 #include "rocksdb/merge_operator.h"
-#include "table/internal_iterator.h"
-#include "table/table_reader.h"
-#include "table/merger.h"
-#include "table/two_level_iterator.h"
 #include "table/format.h"
-#include "table/plain_table_factory.h"
-#include "table/meta_blocks.h"
 #include "table/get_context.h"
+#include "table/internal_iterator.h"
+#include "table/merger.h"
+#include "table/meta_blocks.h"
+#include "table/plain_table_factory.h"
+#include "table/table_reader.h"
+#include "table/two_level_iterator.h"
 #include "util/coding.h"
 #include "util/file_reader_writer.h"
 #include "util/logging.h"
+#include "util/perf_context_imp.h"
 #include "util/stop_watch.h"
 #include "util/sync_point.h"

@@ -90,8 +91,9 @@ class FilePicker {
             const Comparator* user_comparator,
             const InternalKeyComparator* internal_comparator)
      : num_levels_(num_levels),
-        curr_level_(-1),
-        hit_file_level_(-1),
+        curr_level_(static_cast<unsigned int>(-1)),
+        returned_file_level_(static_cast<unsigned int>(-1)),
+        hit_file_level_(static_cast<unsigned int>(-1)),
        search_left_bound_(0),
        search_right_bound_(FileIndexer::kLevelMaxIndex),
 #ifndef NDEBUG
@@ -117,6 +119,8 @@ class FilePicker {
    }
  }

+  int GetCurrentLevel() { return returned_file_level_; }
+
  FdWithKeyRange* GetNextFile() {
    while (!search_ended_) {  // Loops over different levels.
      while (curr_index_in_curr_level_ < curr_file_level_->num_files) {
@@ -189,6 +193,7 @@ class FilePicker {
        }
        prev_file_ = f;
 #endif
+        returned_file_level_ = curr_level_;
        if (curr_level_ > 0 && cmp_largest < 0) {
          // No more files to search in this level.
          search_ended_ = !PrepareNextLevel();
@@ -215,6 +220,7 @@ class FilePicker {
 private:
  unsigned int num_levels_;
  unsigned int curr_level_;
+  unsigned int returned_file_level_;
  unsigned int hit_file_level_;
  int32_t search_left_bound_;
  int32_t search_right_bound_;
@@ -321,7 +327,7 @@ Version::~Version() {
      f->refs--;
      if (f->refs <= 0) {
        if (f->table_reader_handle) {
-          cfd_->table_cache()->ReleaseHandle(f->table_reader_handle);
+          cfd_->table_cache()->EraseHandle(f->fd, f->table_reader_handle);
          f->table_reader_handle = nullptr;
        }
        vset_->obsolete_files_.push_back(f);
@@ -485,7 +491,7 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
                         const EnvOptions& env_options,
                         const InternalKeyComparator& icomparator,
                         HistogramImpl* file_read_hist, bool for_compaction,
-                         bool prefix_enabled, bool skip_filters)
+                         bool prefix_enabled, bool skip_filters, int level)
      : TwoLevelIteratorState(prefix_enabled),
        table_cache_(table_cache),
        read_options_(read_options),
@@ -493,7 +499,8 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
        icomparator_(icomparator),
        file_read_hist_(file_read_hist),
        for_compaction_(for_compaction),
-        skip_filters_(skip_filters) {}
+        skip_filters_(skip_filters),
+        level_(level) {}

  InternalIterator* NewSecondaryIterator(const Slice& meta_handle) override {
    if (meta_handle.size() != sizeof(FileDescriptor)) {
@@ -505,7 +512,7 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
      return table_cache_->NewIterator(
          read_options_, env_options_, icomparator_, *fd,
          nullptr /* don't need reference to table*/, file_read_hist_,
-          for_compaction_, nullptr /* arena */, skip_filters_);
+          for_compaction_, nullptr /* arena */, skip_filters_, level_);
    }
  }

@@ -521,6 +528,7 @@ class LevelFileIteratorState : public TwoLevelIteratorState {
  HistogramImpl* file_read_hist_;
  bool for_compaction_;
  bool skip_filters_;
+  int level_;
 };

 // A wrapper of version builder which references the current version in
@@ -788,7 +796,8 @@ void Version::AddIterators(const ReadOptions& read_options,
    const auto& file = storage_info_.LevelFilesBrief(0).files[i];
    merge_iter_builder->AddIterator(cfd_->table_cache()->NewIterator(
        read_options, soptions, cfd_->internal_comparator(), file.fd, nullptr,
-        cfd_->internal_stats()->GetFileReadHist(0), false, arena));
+        cfd_->internal_stats()->GetFileReadHist(0), false, arena,
+        false /* skip_filters */, 0 /* level */));
  }

  // For levels > 0, we can use a concatenating iterator that sequentially
@@ -803,7 +812,7 @@ void Version::AddIterators(const ReadOptions& read_options,
                                 cfd_->internal_stats()->GetFileReadHist(level),
                                 false /* for_compaction */,
                                 cfd_->ioptions()->prefix_extractor != nullptr,
-                                 IsFilterSkipped(level));
+                                 IsFilterSkipped(level), level);
      mem = arena->AllocateAligned(sizeof(LevelFileNumIterator));
      auto* first_level_iter = new (mem) LevelFileNumIterator(
          cfd_->internal_comparator(), &storage_info_.LevelFilesBrief(level));
@@ -908,7 +917,8 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
        read_options, *internal_comparator(), f->fd, ikey, &get_context,
        cfd_->internal_stats()->GetFileReadHist(fp.GetHitFileLevel()),
        IsFilterSkipped(static_cast<int>(fp.GetHitFileLevel()),
-                        fp.IsHitFileLastInLevel()));
+                        fp.IsHitFileLastInLevel()),
+        fp.GetCurrentLevel());
    // TODO: examine the behavior for corrupted key
    if (!status->ok()) {
      return;
@@ -948,9 +958,16 @@ void Version::Get(const ReadOptions& read_options, const LookupKey& k,
    }
    // merge_operands are in saver and we hit the beginning of the key history
    // do a final merge of nullptr and operands;
-    if (merge_operator_->FullMerge(user_key, nullptr,
-                                   merge_context->GetOperands(), value,
-                                   info_log_)) {
+    bool merge_success = false;
+    {
+      StopWatchNano timer(env_, db_statistics_ != nullptr);
+      PERF_TIMER_GUARD(merge_operator_time_nanos);
+      merge_success = merge_operator_->FullMerge(
+          user_key, nullptr, merge_context->GetOperands(), value, info_log_);
+      RecordTick(db_statistics_, MERGE_OPERATION_TOTAL_TIME,
+                 timer.ElapsedNanos());
+    }
+    if (merge_success) {
      *status = Status::OK();
    } else {
      RecordTick(db_statistics_, NUMBER_MERGE_FAILURES);
@@ -1062,6 +1079,12 @@ void Version::UpdateAccumulatedStats(bool update_stats) {
        if (MaybeInitializeFileMetaData(file_meta)) {
          // each FileMeta will be initialized only once.
          storage_info_.UpdateAccumulatedStats(file_meta);
+          // when option "max_open_files" is -1, all the file metadata has
+          // already been read, so MaybeInitializeFileMetaData() won't incur
+          // any I/O cost.
+          if (vset_->db_options_->max_open_files == -1) {
+            continue;
+          }
          if (++init_count >= kMaxInitCount) {
            break;
          }
@@ -2054,9 +2077,16 @@ VersionSet::VersionSet(const std::string& dbname, const DBOptions* db_options,
      env_options_(storage_options),
      env_options_compactions_(env_options_) {}

+void CloseTables(void* ptr, size_t) {
+  TableReader* table_reader = reinterpret_cast<TableReader*>(ptr);
+  table_reader->Close();
+}
+
 VersionSet::~VersionSet() {
  // we need to delete column_family_set_ because its destructor depends on
  // VersionSet
+  column_family_set_->get_table_cache()->ApplyToAllCacheEntries(&CloseTables,
+                                                                false);
  column_family_set_.reset();
  for (auto file : obsolete_files_) {
    delete file;
@@ -2254,6 +2284,8 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
                         db_options_->disableDataSync ? nullptr : db_directory);
      // Leave the old file behind since PurgeObsoleteFiles will take care of it
      // later. It's unsafe to delete now since file deletion may be disabled.
+      obsolete_manifests_.emplace_back(
+          DescriptorFileName("", manifest_file_number_));
    }

    if (s.ok()) {
@@ -2304,11 +2336,14 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
    manifest_file_size_ = new_manifest_file_size;
    prev_log_number_ = edit->prev_log_number_;
  } else {
+    std::string version_edits;
+    for (auto& e : batch_edits) {
+      version_edits = version_edits + "\n" + e->DebugString(true);
+    }
    Log(InfoLogLevel::ERROR_LEVEL, db_options_->info_log,
-        "Error in committing version %lu to [%s]",
-        (unsigned long)v->GetVersionNumber(),
-        column_family_data ? column_family_data->GetName().c_str()
-                           : "<null>");
+        "[%s] Error in committing version edit to MANIFEST: %s",
+        column_family_data ? column_family_data->GetName().c_str() : "<null>",
+        version_edits.c_str());
    delete v;
    if (new_descriptor_log) {
      Log(InfoLogLevel::INFO_LEVEL, db_options_->info_log,
@@ -3238,7 +3273,7 @@ void VersionSet::AddLiveFiles(std::vector<FileDescriptor>* live_list) {
  }
 }

-InternalIterator* VersionSet::MakeInputIterator(Compaction* c) {
+InternalIterator* VersionSet::MakeInputIterator(const Compaction* c) {
  auto cfd = c->column_family_data();
  ReadOptions read_options;
  read_options.verify_checksums =
@@ -3265,7 +3300,8 @@ InternalIterator* VersionSet::MakeInputIterator(Compaction* c) {
              read_options, env_options_compactions_,
              cfd->internal_comparator(), flevel->files[i].fd, nullptr,
              nullptr, /* no per level latency histogram*/
-              true /* for compaction */);
+              true /* for_compaction */, nullptr /* arena */,
+              false /* skip_filters */, (int)which /* level */);
        }
      } else {
        // Create concatenating iterator for the files from this level
@@ -3275,7 +3311,7 @@ InternalIterator* VersionSet::MakeInputIterator(Compaction* c) {
                cfd->internal_comparator(),
                nullptr /* no per level latency histogram */,
                true /* for_compaction */, false /* prefix enabled */,
-                false /* skip_filters */),
+                false /* skip_filters */, (int)which /* level */),
            new LevelFileNumIterator(cfd->internal_comparator(),
                                     c->input_levels(which)));
      }
@@ -3388,7 +3424,10 @@ void VersionSet::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
 }

 void VersionSet::GetObsoleteFiles(std::vector<FileMetaData*>* files,
+                                  std::vector<std::string>* manifest_filenames,
                                  uint64_t min_pending_output) {
+  assert(manifest_filenames->empty());
+  obsolete_manifests_.swap(*manifest_filenames);
  std::vector<FileMetaData*> pending_files;
  for (auto f : obsolete_files_) {
    if (f->fd.GetNumber() < min_pending_output) {
@@ -24,6 +24,7 @@
 #include <map>
 #include <memory>
 #include <set>
+#include <string>
 #include <utility>
 #include <vector>

@@ -670,7 +671,7 @@ class VersionSet {

  // Create an iterator that reads over the compaction inputs for "*c".
  // The caller should delete the iterator when no longer needed.
-  InternalIterator* MakeInputIterator(Compaction* c);
+  InternalIterator* MakeInputIterator(const Compaction* c);

  // Add all files listed in any live version to *live.
  void AddLiveFiles(std::vector<FileDescriptor>* live_list);
@@ -697,6 +698,7 @@ class VersionSet {
  void GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata);

  void GetObsoleteFiles(std::vector<FileMetaData*>* files,
+                        std::vector<std::string>* manifest_filenames,
                        uint64_t min_pending_output);

  ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); }
@@ -758,6 +760,7 @@ class VersionSet {
  uint64_t manifest_file_size_;

  std::vector<FileMetaData*> obsolete_files_;
+  std::vector<std::string> obsolete_manifests_;

  // env options for all reads and writes except compactions
  const EnvOptions& env_options_;
@@ -455,7 +455,7 @@ Status WalManager::ReadFirstLine(const std::string& fname,

  if (reader.ReadRecord(&record, &scratch) &&
      (status.ok() || !db_options_.paranoid_checks)) {
-    if (record.size() < 12) {
+    if (record.size() < WriteBatchInternal::kHeader) {
      reporter.Corruption(record.size(),
                          Status::Corruption("log record too small"));
      // TODO read record's till the first no corrupt entry?
@@ -81,8 +81,6 @@ struct BatchContentClassifier : public WriteBatch::Handler {

 }  // anon namespace

-// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
-static const size_t kHeader = 12;

 struct SavePoint {
  size_t size;  // size of rep_
@@ -96,8 +94,9 @@ struct SavePoints {

 WriteBatch::WriteBatch(size_t reserved_bytes)
    : save_points_(nullptr), content_flags_(0), rep_() {
-  rep_.reserve((reserved_bytes > kHeader) ? reserved_bytes : kHeader);
-  rep_.resize(kHeader);
+  rep_.reserve((reserved_bytes > WriteBatchInternal::kHeader) ?
+    reserved_bytes : WriteBatchInternal::kHeader);
+  rep_.resize(WriteBatchInternal::kHeader);
 }

 WriteBatch::WriteBatch(const std::string& rep)
@@ -146,7 +145,7 @@ bool WriteBatch::Handler::Continue() {

 void WriteBatch::Clear() {
  rep_.clear();
-  rep_.resize(kHeader);
+  rep_.resize(WriteBatchInternal::kHeader);

  content_flags_.store(0, std::memory_order_relaxed);

@@ -193,6 +192,23 @@ bool WriteBatch::HasMerge() const {
  return (ComputeContentFlags() & ContentFlags::HAS_MERGE) != 0;
 }

+bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key, bool cf_record) {
+  assert(input != nullptr && key != nullptr);
+  // Skip tag byte
+  input->remove_prefix(1);
+
+  if (cf_record) {
+    // Skip column_family bytes
+    uint32_t cf;
+    if (!GetVarint32(input, &cf)) {
+      return false;
+    }
+  }
+
+  // Extract key
+  return GetLengthPrefixedSlice(input, key);
+}
+
 Status ReadRecordFromWriteBatch(Slice* input, char* tag,
                                uint32_t* column_family, Slice* key,
                                Slice* value, Slice* blob) {
@@ -249,11 +265,11 @@ Status ReadRecordFromWriteBatch(Slice* input, char* tag,

 Status WriteBatch::Iterate(Handler* handler) const {
  Slice input(rep_);
-  if (input.size() < kHeader) {
+  if (input.size() < WriteBatchInternal::kHeader) {
    return Status::Corruption("malformed WriteBatch (too small)");
  }

-  input.remove_prefix(kHeader);
+  input.remove_prefix(WriteBatchInternal::kHeader);
  Slice key, value, blob;
  int found = 0;
  Status s;
@@ -329,7 +345,9 @@ void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) {
  EncodeFixed64(&b->rep_[0], seq);
 }

-size_t WriteBatchInternal::GetFirstOffset(WriteBatch* b) { return kHeader; }
+size_t WriteBatchInternal::GetFirstOffset(WriteBatch* b) {
+  return WriteBatchInternal::kHeader;
+}

 void WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id,
                             const Slice& key, const Slice& value) {
@@ -832,15 +850,16 @@ Status WriteBatchInternal::InsertInto(const WriteBatch* batch,
 }

 void WriteBatchInternal::SetContents(WriteBatch* b, const Slice& contents) {
-  assert(contents.size() >= kHeader);
+  assert(contents.size() >= WriteBatchInternal::kHeader);
  b->rep_.assign(contents.data(), contents.size());
  b->content_flags_.store(ContentFlags::DEFERRED, std::memory_order_relaxed);
 }

 void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) {
  SetCount(dst, Count(dst) + Count(src));
-  assert(src->rep_.size() >= kHeader);
-  dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader);
+  assert(src->rep_.size() >= WriteBatchInternal::kHeader);
+  dst->rep_.append(src->rep_.data() + WriteBatchInternal::kHeader,
+    src->rep_.size() - WriteBatchInternal::kHeader);
  dst->content_flags_.store(
      dst->content_flags_.load(std::memory_order_relaxed) |
          src->content_flags_.load(std::memory_order_relaxed),
@@ -852,7 +871,7 @@ size_t WriteBatchInternal::AppendedByteSize(size_t leftByteSize,
  if (leftByteSize == 0 || rightByteSize == 0) {
    return leftByteSize + rightByteSize;
  } else {
-    return leftByteSize + rightByteSize - kHeader;
+    return leftByteSize + rightByteSize - WriteBatchInternal::kHeader;
  }
 }

@@ -63,6 +63,10 @@ class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
 // WriteBatch that we don't want in the public WriteBatch interface.
 class WriteBatchInternal {
 public:
+
+  // WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
+  static const size_t kHeader = 12;
+
  // WriteBatch methods with column_family_id instead of ColumnFamilyHandle*
  static void Put(WriteBatch* batch, uint32_t column_family_id,
                  const Slice& key, const Slice& value);
@@ -138,6 +138,7 @@
  block_size=8192
  block_restart_interval=16
  cache_index_and_filter_blocks=false
+  pin_l0_filter_and_index_blocks_in_cache=false
  index_type=kBinarySearch
  hash_index_allow_collision=true
  flush_block_policy_factory=FlushBlockBySizePolicyFactory
@@ -12,7 +12,7 @@ underlying filesystem.
 If you want to compile rocksdb with hdfs support, please set the following
 environment variables appropriately (also defined in setup.sh for convenience)
   USE_HDFS=1
-   JAVA_HOME=/usr/local/jdk-6u22-64
+   JAVA_HOME=/usr/local/jdk-7u79-64
   LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/jdk-6u22-64/jre/lib/amd64/server:/usr/local/jdk-6u22-64/jre/lib/amd64/:./snappy/libs
   make clean all db_bench

@@ -451,6 +451,9 @@ extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_cache_index_and_filter_blocks(
    rocksdb_block_based_table_options_t*, unsigned char);
 extern ROCKSDB_LIBRARY_API void
+rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
+    rocksdb_block_based_table_options_t*, unsigned char);
+extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_skip_table_builder_flush(
    rocksdb_block_based_table_options_t* options, unsigned char);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_block_based_table_factory(
@@ -25,6 +25,7 @@
 #include <memory>
 #include <stdint.h>
 #include "rocksdb/slice.h"
+#include "rocksdb/status.h"

 namespace rocksdb {

@@ -33,12 +34,15 @@ using std::shared_ptr;
 class Cache;

 // Create a new cache with a fixed size capacity. The cache is sharded
-// to 2^numShardBits shards, by hash of the key. The total capacity
+// to 2^num_shard_bits shards, by hash of the key. The total capacity
 // is divided and evenly assigned to each shard.
 //
-// The functions without parameter numShardBits uses default value, which is 4
+// The parameter num_shard_bits defaults to 4, and strict_capacity_limit
+// defaults to false.
 extern shared_ptr<Cache> NewLRUCache(size_t capacity);
-extern shared_ptr<Cache> NewLRUCache(size_t capacity, int numShardBits);
+extern shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits);
+extern shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
+                                     bool strict_capacity_limit);

 class Cache {
 public:
@@ -55,15 +59,22 @@ class Cache {

  // Insert a mapping from key->value into the cache and assign it
  // the specified charge against the total cache capacity.
+  // If strict_capacity_limit is true and cache reaches its full capacity,
+  // return Status::Incomplete.
  //
-  // Returns a handle that corresponds to the mapping.  The caller
-  // must call this->Release(handle) when the returned mapping is no
-  // longer needed.
+  // If handle is not nullptr, returns a handle that corresponds to the
+  // mapping. The caller must call this->Release(handle) when the returned
+  // mapping is no longer needed. In case of error caller is responsible to
+  // cleanup the value (i.e. calling "deleter").
+  //
+  // If handle is nullptr, it is as if Release is called immediately after
+  // insert. In case of error value will be cleanup.
  //
  // When the inserted entry is no longer needed, the key and
  // value will be passed to "deleter".
-  virtual Handle* Insert(const Slice& key, void* value, size_t charge,
-                         void (*deleter)(const Slice& key, void* value)) = 0;
+  virtual Status Insert(const Slice& key, void* value, size_t charge,
+                        void (*deleter)(const Slice& key, void* value),
+                        Handle** handle = nullptr) = 0;

  // If the cache has no mapping for "key", returns nullptr.
  //
@@ -100,6 +111,14 @@ class Cache {
  // purge the released entries from the cache in order to lower the usage
  virtual void SetCapacity(size_t capacity) = 0;

+  // Set whether to return error on insertion when cache reaches its full
+  // capacity.
+  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
+
+  // Set whether to return error on insertion when cache reaches its full
+  // capacity.
+  virtual bool HasStrictCapacityLimit() const = 0;
+
  // returns the maximum configured capacity of the cache
  virtual size_t GetCapacity() const = 0;

@@ -127,6 +146,10 @@ class Cache {
  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
                                      bool thread_safe) = 0;

+  // Remove all entries.
+  // Prerequisit: no entry is referenced.
+  virtual void EraseUnRefEntries() = 0;
+
 private:
  void LRU_Remove(Handle* e);
  void LRU_Append(Handle* e);
@@ -61,7 +61,7 @@ struct CompactionJobStats {
  uint64_t num_corrupt_keys;

  // Following counters are only populated if
-  // options.compaction_measure_io_stats = true;
+  // options.report_bg_io_stats = true;

  // Time spent on file's Append() call.
  uint64_t file_write_nanos;
@@ -561,7 +561,7 @@ class WritableFile {
   * underlying storage of a file (generally via fallocate) if the Env
   * instance supports it.
   */
-  void SetPreallocationBlockSize(size_t size) {
+  virtual void SetPreallocationBlockSize(size_t size) {
    preallocation_block_size_ = size;
  }

@@ -597,7 +597,7 @@ class WritableFile {
  // of space on devices where it can result in less file
  // fragmentation and/or less waste from over-zealous filesystem
  // pre-allocation.
-  void PrepareWrite(size_t offset, size_t len) {
+  virtual void PrepareWrite(size_t offset, size_t len) {
    if (preallocation_block_size_ == 0) {
      return;
    }
@@ -950,6 +950,13 @@ class WritableFileWrapper : public WritableFile {
    return target_->InvalidateCache(offset, length);
  }

+  virtual void SetPreallocationBlockSize(size_t size) override {
+    target_->SetPreallocationBlockSize(size);
+  }
+  virtual void PrepareWrite(size_t offset, size_t len) override {
+    target_->PrepareWrite(offset, len);
+  }
+
 protected:
  Status Allocate(uint64_t offset, uint64_t len) override {
    return target_->Allocate(offset, len);
@@ -158,7 +158,15 @@ struct DbPath {
 struct Options;

 struct ColumnFamilyOptions {
+  // The function recovers options to a previous version. Only 4.6 or later
+  // versions are supported.
+  ColumnFamilyOptions* OldDefaults(int rocksdb_major_version = 4,
+                                   int rocksdb_minor_version = 6);
+
  // Some functions that make it easier to optimize RocksDB
+  // Use if if your DB is very small (like under 1GB) and you don't want to
+  // spend lots of memory for memtables.
+  ColumnFamilyOptions* OptimizeForSmallDb();

  // Use this if you don't need to keep the data sorted, i.e. you'll never use
  // an iterator, only Put() and Get() API calls
@@ -254,7 +262,7 @@ struct ColumnFamilyOptions {
  // Note that write_buffer_size is enforced per column family.
  // See db_write_buffer_size for sharing memory across column families.
  //
-  // Default: 4MB
+  // Default: 64MB
  //
  // Dynamically changeable through SetOptions() API
  size_t write_buffer_size;
@@ -400,7 +408,7 @@ struct ColumnFamilyOptions {
  // be 2MB, and each file on level 2 will be 20MB,
  // and each file on level-3 will be 200MB.
  //
-  // Default: 2MB.
+  // Default: 64MB.
  //
  // Dynamically changeable through SetOptions() API
  uint64_t target_file_size_base;
@@ -415,12 +423,12 @@ struct ColumnFamilyOptions {
  // max_bytes_for_level_base is the max total for level-1.
  // Maximum number of bytes for level L can be calculated as
  // (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1))
-  // For example, if max_bytes_for_level_base is 20MB, and if
+  // For example, if max_bytes_for_level_base is 200MB, and if
  // max_bytes_for_level_multiplier is 10, total data size for level-1
-  // will be 20MB, total file size for level-2 will be 200MB,
-  // and total file size for level-3 will be 2GB.
+  // will be 2GB, total file size for level-2 will be 20GB,
+  // and total file size for level-3 will be 200GB.
  //
-  // Default: 10MB.
+  // Default: 256MB.
  //
  // Dynamically changeable through SetOptions() API
  uint64_t max_bytes_for_level_base;
@@ -538,13 +546,13 @@ struct ColumnFamilyOptions {
  // All writes will be slowed down to at least delayed_write_rate if estimated
  // bytes needed to be compaction exceed this threshold.
  //
-  // Default: 0 (disabled)
+  // Default: 64GB
  uint64_t soft_pending_compaction_bytes_limit;

  // All writes are stopped if estimated bytes needed to be compaction exceed
  // this threshold.
  //
-  // Default: 0 (disabled)
+  // Default: 256GB
  uint64_t hard_pending_compaction_bytes_limit;

  // DEPRECATED -- this options is no longer used
@@ -782,9 +790,9 @@ struct ColumnFamilyOptions {
  // Default: false
  bool paranoid_file_checks;

-  // Measure IO stats in compactions, if true.
+  // Measure IO stats in compactions and flushes, if true.
  // Default: false
-  bool compaction_measure_io_stats;
+  bool report_bg_io_stats;

  // Create ColumnFamilyOptions with default values for all fields
  ColumnFamilyOptions();
@@ -795,6 +803,10 @@ struct ColumnFamilyOptions {
 };

 struct DBOptions {
+  // The function recovers options to the option as in version 4.6.
+  DBOptions* OldDefaults(int rocksdb_major_version = 4,
+                         int rocksdb_minor_version = 6);
+
  // Some functions that make it easier to optimize RocksDB

 #ifndef ROCKSDB_LITE
@@ -837,10 +849,19 @@ struct DBOptions {
  // Default: nullptr
  std::shared_ptr<RateLimiter> rate_limiter;

-  // Use to track SST files and control their file deletion rate, can be used
-  // among multiple RocksDB instances, sst_file_manager only track and throttle
-  // deletes of SST files in first db_path (db_name if db_paths is empty), other
-  // files and other db_paths wont be tracked or affected by sst_file_manager.
+  // Use to track SST files and control their file deletion rate.
+  //
+  // Features:
+  //  - Throttle the deletion rate of the SST files.
+  //  - Keep track the total size of all SST files.
+  //  - Set a maximum allowed space limit for SST files that when reached
+  //    the DB wont do any further flushes or compactions and will set the
+  //    background error.
+  //  - Can be shared between multiple dbs.
+  // Limitations:
+  //  - Only track and throttle deletes of SST files in
+  //    first db_path (db_name if db_paths is empty).
+  //
  // Default: nullptr
  std::shared_ptr<SstFileManager> sst_file_manager;

@@ -1041,7 +1062,23 @@ struct DBOptions {
  // large amounts of data (such as xfs's allocsize option).
  size_t manifest_preallocation_size;

-  // Data being read from file storage may be buffered in the OS
+  // Hint the OS that it should not buffer disk I/O. Enabling this
+  // parameter may improve performance but increases pressure on the
+  // system cache.
+  //
+  // The exact behavior of this parameter is platform dependent.
+  //
+  // On POSIX systems, after RocksDB reads data from disk it will
+  // mark the pages as "unneeded". The operating system may - or may not
+  // - evict these pages from memory, reducing pressure on the system
+  // cache. If the disk block is requested again this can result in
+  // additional disk I/O.
+  //
+  // On WINDOWS system, files will be opened in "unbuffered I/O" mode
+  // which means that data read from the disk will not be cached or
+  // bufferized. The hardware buffer of the devices may however still
+  // be used. Memory mapped files are not impacted by this parameter.
+  //
  // Default: true
  bool allow_os_buffer;

@@ -1258,7 +1295,7 @@ struct DBOptions {
  // records, ignoring a particular record or skipping replay.
  // The filter is invoked at startup and is invoked from a single-thread
  // currently.
-  const WalFilter* wal_filter;
+  WalFilter* wal_filter;
 #endif  // ROCKSDB_LITE

  // If true, then DB::Open / CreateColumnFamily / DropColumnFamily
@@ -1278,6 +1315,10 @@ struct Options : public DBOptions, public ColumnFamilyOptions {
          const ColumnFamilyOptions& column_family_options)
      : DBOptions(db_options), ColumnFamilyOptions(column_family_options) {}

+  // The function recovers options to the option as in version 4.6.
+  Options* OldDefaults(int rocksdb_major_version = 4,
+                       int rocksdb_minor_version = 6);
+
  void Dump(Logger* log) const;

  void DumpCFOptions(Logger* log) const;
@@ -30,10 +30,32 @@ struct PerfContext {
  uint64_t block_read_time;           // total nanos spent on block reads
  uint64_t block_checksum_time;       // total nanos spent on block checksum
  uint64_t block_decompress_time;  // total nanos spent on block decompression
-  // total number of internal keys skipped over during iteration (overwritten or
-  // deleted, to be more specific, hidden by a put or delete of the same key)
+  // total number of internal keys skipped over during iteration.
+  // There are several reasons for it:
+  // 1. when calling Next(), the iterator is in the position of the previous
+  //    key, so that we'll need to skip it. It means this counter will always
+  //    be incremented in Next().
+  // 2. when calling Next(), we need to skip internal entries for the previous
+  //    keys that are overwritten.
+  // 3. when calling Next(), Seek() or SeekToFirst(), after previous key
+  //    before calling Next(), the seek key in Seek() or the beginning for
+  //    SeekToFirst(), there may be one or more deleted keys before the next
+  //    valid key that the operation should place the iterator to. We need
+  //    to skip both of the tombstone and updates hidden by the tombstones. The
+  //    tombstones are not included in this counter, while previous updates
+  //    hidden by the tombstones will be included here.
+  // 4. symmetric cases for Prev() and SeekToLast()
+  // We sometimes also skip entries of more recent updates than the snapshot
+  // we read from, but they are not included in this counter.
+  //
  uint64_t internal_key_skipped_count;
-  // total number of deletes and single deletes skipped over during iteration
+  // Total number of deletes and single deletes skipped over during iteration
+  // When calling Next(), Seek() or SeekToFirst(), after previous position
+  // before calling Next(), the seek key in Seek() or the beginning for
+  // SeekToFirst(), there may be one or more deleted keys before the next valid
+  // key. Every deleted key is counted once. We don't recount here if there are
+  // still older updates invalidated by the tombstones.
+  //
  uint64_t internal_delete_skipped_count;

  uint64_t get_snapshot_time;       // total nanos spent on getting snapshot
@@ -79,8 +79,16 @@ class Slice {
  }

  // Return a string that contains the copy of the referenced data.
+  // when hex is true, returns a string of twice the length hex encoded (0-9A-F)
  std::string ToString(bool hex = false) const;

+  // Decodes the current slice interpreted as an hexadecimal string into result,
+  // if successful returns true, if this isn't a valid hex string
+  // (e.g not coming from Slice::ToString(true)) DecodeHex returns false.
+  // This slice is expected to have an even number of 0-9A-F characters
+  // also accepts lowercase (a-f)
+  bool DecodeHex(std::string* result) const;
+
  // Three-way comparison.  Returns value:
  //   <  0 iff "*this" <  "b",
  //   == 0 iff "*this" == "b",
@@ -33,6 +33,8 @@ enum Tickers : uint32_t {
  BLOCK_CACHE_HIT,
  // # of blocks added to block cache.
  BLOCK_CACHE_ADD,
+  // # of failures when adding blocks to block cache.
+  BLOCK_CACHE_ADD_FAILURES,
  // # of times cache miss when accessing index block from block cache.
  BLOCK_CACHE_INDEX_MISS,
  // # of times cache hit when accessing index block from block cache.
@@ -140,8 +142,12 @@ enum Tickers : uint32_t {
  GET_UPDATES_SINCE_CALLS,
  BLOCK_CACHE_COMPRESSED_MISS,  // miss in the compressed block cache
  BLOCK_CACHE_COMPRESSED_HIT,   // hit in the compressed block cache
-  WAL_FILE_SYNCED,              // Number of times WAL sync is done
-  WAL_FILE_BYTES,               // Number of bytes written to WAL
+  // Number of blocks added to comopressed block cache
+  BLOCK_CACHE_COMPRESSED_ADD,
+  // Number of failures when adding blocks to compressed block cache
+  BLOCK_CACHE_COMPRESSED_ADD_FAILURES,
+  WAL_FILE_SYNCED,  // Number of times WAL sync is done
+  WAL_FILE_BYTES,   // Number of bytes written to WAL

  // Writes can be processed by requesting thread or by the thread at the
  // head of the writers queue.
@@ -176,6 +182,7 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
    {BLOCK_CACHE_MISS, "rocksdb.block.cache.miss"},
    {BLOCK_CACHE_HIT, "rocksdb.block.cache.hit"},
    {BLOCK_CACHE_ADD, "rocksdb.block.cache.add"},
+    {BLOCK_CACHE_ADD_FAILURES, "rocksdb.block.cache.add.failures"},
    {BLOCK_CACHE_INDEX_MISS, "rocksdb.block.cache.index.miss"},
    {BLOCK_CACHE_INDEX_HIT, "rocksdb.block.cache.index.hit"},
    {BLOCK_CACHE_FILTER_MISS, "rocksdb.block.cache.filter.miss"},
@@ -227,6 +234,9 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
    {GET_UPDATES_SINCE_CALLS, "rocksdb.getupdatessince.calls"},
    {BLOCK_CACHE_COMPRESSED_MISS, "rocksdb.block.cachecompressed.miss"},
    {BLOCK_CACHE_COMPRESSED_HIT, "rocksdb.block.cachecompressed.hit"},
+    {BLOCK_CACHE_COMPRESSED_ADD, "rocksdb.block.cachecompressed.add"},
+    {BLOCK_CACHE_COMPRESSED_ADD_FAILURES,
+     "rocksdb.block.cachecompressed.add.failures"},
    {WAL_FILE_SYNCED, "rocksdb.wal.synced"},
    {WAL_FILE_BYTES, "rocksdb.wal.bytes"},
    {WRITE_DONE_BY_SELF, "rocksdb.write.self"},
@@ -64,6 +64,12 @@ struct BlockBasedTableOptions {
  // block during table initialization.
  bool cache_index_and_filter_blocks = false;

+  // if cache_index_and_filter_blocks is true and the below is true, then
+  // filter and index blocks are stored in the cache, but a reference is
+  // held in the "table reader" object so the blocks are pinned and only
+  // evicted from cache when the table reader is freed.
+  bool pin_l0_filter_and_index_blocks_in_cache = false;
+
  // The index type that will be used for this table.
  enum IndexType : char {
    // A space efficient index block that is optimized for
@@ -27,47 +27,6 @@ namespace rocksdb {
 // }
 typedef std::map<std::string, std::string> UserCollectedProperties;

-// TableProperties contains a bunch of read-only properties of its associated
-// table.
-struct TableProperties {
- public:
-  // the total size of all data blocks.
-  uint64_t data_size = 0;
-  // the size of index block.
-  uint64_t index_size = 0;
-  // the size of filter block.
-  uint64_t filter_size = 0;
-  // total raw key size
-  uint64_t raw_key_size = 0;
-  // total raw value size
-  uint64_t raw_value_size = 0;
-  // the number of blocks in this table
-  uint64_t num_data_blocks = 0;
-  // the number of entries in this table
-  uint64_t num_entries = 0;
-  // format version, reserved for backward compatibility
-  uint64_t format_version = 0;
-  // If 0, key is variable length. Otherwise number of bytes for each key.
-  uint64_t fixed_key_len = 0;
-
-  // The name of the filter policy used in this table.
-  // If no filter policy is used, `filter_policy_name` will be an empty string.
-  std::string filter_policy_name;
-
-  // user collected properties
-  UserCollectedProperties user_collected_properties;
-  UserCollectedProperties readable_properties;
-
-  // convert this object to a human readable form
-  //   @prop_delim: delimiter for each property.
-  std::string ToString(const std::string& prop_delim = "; ",
-                       const std::string& kv_delim = "=") const;
-
-  // Aggregate the numerical member variables of the specified
-  // TableProperties.
-  void Add(const TableProperties& tp);
-};
-
 // table properties' human-readable names in the property block.
 struct TablePropertiesNames {
  static const std::string kDataSize;
@@ -80,6 +39,8 @@ struct TablePropertiesNames {
  static const std::string kFormatVersion;
  static const std::string kFixedKeyLen;
  static const std::string kFilterPolicy;
+  static const std::string kColumnFamilyName;
+  static const std::string kColumnFamilyId;
 };

 extern const std::string kPropertiesBlock;
@@ -158,6 +119,55 @@ class TablePropertiesCollectorFactory {
  virtual const char* Name() const = 0;
 };

+// TableProperties contains a bunch of read-only properties of its associated
+// table.
+struct TableProperties {
+ public:
+  // the total size of all data blocks.
+  uint64_t data_size = 0;
+  // the size of index block.
+  uint64_t index_size = 0;
+  // the size of filter block.
+  uint64_t filter_size = 0;
+  // total raw key size
+  uint64_t raw_key_size = 0;
+  // total raw value size
+  uint64_t raw_value_size = 0;
+  // the number of blocks in this table
+  uint64_t num_data_blocks = 0;
+  // the number of entries in this table
+  uint64_t num_entries = 0;
+  // format version, reserved for backward compatibility
+  uint64_t format_version = 0;
+  // If 0, key is variable length. Otherwise number of bytes for each key.
+  uint64_t fixed_key_len = 0;
+  // ID of column family for this SST file, corresponding to the CF identified
+  // by column_family_name.
+  uint64_t column_family_id =
+      rocksdb::TablePropertiesCollectorFactory::Context::kUnknownColumnFamily;
+
+  // Name of the column family with which this SST file is associated.
+  // If column family is unknown, `column_family_name` will be an empty string.
+  std::string column_family_name;
+
+  // The name of the filter policy used in this table.
+  // If no filter policy is used, `filter_policy_name` will be an empty string.
+  std::string filter_policy_name;
+
+  // user collected properties
+  UserCollectedProperties user_collected_properties;
+  UserCollectedProperties readable_properties;
+
+  // convert this object to a human readable form
+  //   @prop_delim: delimiter for each property.
+  std::string ToString(const std::string& prop_delim = "; ",
+                       const std::string& kv_delim = "=") const;
+
+  // Aggregate the numerical member variables of the specified
+  // TableProperties.
+  void Add(const TableProperties& tp);
+};
+
 // Extra properties
 // Below is a list of non-basic properties that are collected by database
 // itself. Especially some properties regarding to the internal keys (which
@@ -143,13 +143,17 @@ struct BackupInfo {
  uint64_t size;

  uint32_t number_files;
+  std::string app_metadata;

  BackupInfo() {}

  BackupInfo(BackupID _backup_id, int64_t _timestamp, uint64_t _size,
-             uint32_t _number_files)
-      : backup_id(_backup_id), timestamp(_timestamp), size(_size),
-        number_files(_number_files) {}
+             uint32_t _number_files, const std::string& _app_metadata)
+      : backup_id(_backup_id),
+        timestamp(_timestamp),
+        size(_size),
+        number_files(_number_files),
+        app_metadata(_app_metadata) {}
 };

 class BackupStatistics {
@@ -218,9 +222,17 @@ class BackupEngine {
                     const BackupableDBOptions& options,
                     BackupEngine** backup_engine_ptr);

-  virtual Status CreateNewBackup(
-      DB* db, bool flush_before_backup = false,
+  /// same as CreateNewBackup, but stores extra application metadata
+  virtual Status CreateNewBackupWithMetadata(
+      DB* db, const std::string& app_metadata, bool flush_before_backup = false,
      std::function<void()> progress_callback = []() {}) = 0;
+
+  virtual Status CreateNewBackup(DB* db, bool flush_before_backup = false,
+                                 std::function<void()> progress_callback =
+                                     []() {}) {
+    return CreateNewBackupWithMetadata(db, "", flush_before_backup,
+                                       progress_callback);
+  }
  virtual Status PurgeOldBackups(uint32_t num_backups_to_keep) = 0;
  virtual Status DeleteBackup(BackupID backup_id) = 0;
  virtual void StopBackup() = 0;
@@ -5,8 +5,8 @@
 #pragma once

 #define ROCKSDB_MAJOR 4
-#define ROCKSDB_MINOR 6
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_MINOR 7
+#define ROCKSDB_PATCH 1

 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these
@@ -4,6 +4,8 @@
 // of patent rights can be found in the PATENTS file in the same directory.

 #pragma once
+#include <string>
+#include <map>

 namespace rocksdb {

@@ -30,6 +32,21 @@ class WalFilter {

  virtual ~WalFilter() {}

+  // Provide ColumnFamily->LogNumber map to filter
+  // so that filter can determine whether a log number applies to a given 
+  // column family (i.e. that log hasn't been flushed to SST already for the
+  // column family).
+  // We also pass in name->id map as only name is known during
+  // recovery (as handles are opened post-recovery).
+  // while write batch callbacks happen in terms of column family id.
+  //
+  // @params cf_lognumber_map column_family_id to lognumber map
+  // @params cf_name_id_map   column_family_name to column_family_id map
+
+  virtual void ColumnFamilyLogNumberMap(
+    const std::map<uint32_t, uint64_t>& cf_lognumber_map,
+    const std::map<std::string, uint32_t>& cf_name_id_map) {}
+
  // LogRecord is invoked for each log record encountered for all the logs
  // during replay on logs on recovery. This method can be used to:
  //  * inspect the record (using the batch parameter)
@@ -41,6 +58,10 @@ class WalFilter {
  //    (by returning kStop replay) - please note that this implies
  //    discarding the logs from current record onwards.
  //
+  // @params log_number     log_number of the current log.
+  //                        Filter might use this to determine if the log
+  //                        record is applicable to a certain column family.
+  // @params log_file_name  log file name - only for informational purposes
  // @params batch          batch encountered in the log during recovery
  // @params new_batch      new_batch to populate if filter wants to change
  //                        the batch (for example to filter some records out,
@@ -54,9 +75,23 @@ class WalFilter {
  // @returns               Processing option for the current record.
  //                        Please see WalProcessingOption enum above for
  //                        details.
+  virtual WalProcessingOption LogRecordFound(unsigned long long log_number,
+                                        const std::string& log_file_name,
+                                        const WriteBatch& batch,
+                                        WriteBatch* new_batch,
+                                        bool* batch_changed) {
+    // Default implementation falls back to older function for compatibility
+    return LogRecord(batch, new_batch, batch_changed);
+  }
+
+  // Please see the comments for LogRecord above. This function is for 
+  // compatibility only and contains a subset of parameters. 
+  // New code should use the function above.
  virtual WalProcessingOption LogRecord(const WriteBatch& batch,
                                        WriteBatch* new_batch,
-                                        bool* batch_changed) const = 0;
+                                        bool* batch_changed) const {
+    return WalProcessingOption::kContinueProcessing;
+  }

  // Returns a name that identifies this WAL filter.
  // The name will be printed to LOG file on start up for diagnosis.
@@ -64,6 +64,7 @@ JAVA_TESTS = org.rocksdb.BackupableDBOptionsTest\
        org.rocksdb.BackupEngineTest\
 	org.rocksdb.BackupableDBTest\
 	org.rocksdb.BlockBasedTableConfigTest\
+        org.rocksdb.util.BytewiseComparatorTest\
 	org.rocksdb.CheckPointTest\
 	org.rocksdb.ColumnFamilyOptionsTest\
 	org.rocksdb.ColumnFamilyTest\
@@ -21,10 +21,14 @@
 */
 package org.rocksdb.benchmark;

+import java.io.IOException;
 import java.lang.Runnable;
 import java.lang.Math;
 import java.io.File;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.nio.ByteBuffer;
+import java.nio.file.Files;
 import java.util.Collection;
 import java.util.Date;
 import java.util.EnumMap;
@@ -601,6 +605,11 @@ public class DbBenchmark {
        (Integer)flags_.get(Flag.max_successive_merges));
    options.setWalTtlSeconds((Long)flags_.get(Flag.wal_ttl_seconds));
    options.setWalSizeLimitMB((Long)flags_.get(Flag.wal_size_limit_MB));
+    if(flags_.get(Flag.java_comparator) != null) {
+      options.setComparator(
+          (AbstractComparator)flags_.get(Flag.java_comparator));
+    }
+
    /* TODO(yhchiang): enable the following parameters
    options.setCompressionType((String)flags_.get(Flag.compression_type));
    options.setCompressionLevel((Integer)flags_.get(Flag.compression_level));
@@ -774,6 +783,7 @@ public class DbBenchmark {
  }

  private void open(Options options) throws RocksDBException {
+    System.out.println("Using database directory: " + databaseDir_);
    db_ = RocksDB.open(options, databaseDir_);
  }

@@ -1475,7 +1485,7 @@ public class DbBenchmark {
        return Integer.parseInt(value);
      }
    },
-    db("/tmp/rocksdbjni-bench",
+    db(getTempDir("rocksdb-jni"),
       "Use the db with the following name.") {
      @Override public Object parseValue(String value) {
        return value;
@@ -1486,6 +1496,31 @@ public class DbBenchmark {
      @Override public Object parseValue(String value) {
        return parseBoolean(value);
      }
+    },
+    java_comparator(null, "Class name of a Java Comparator to use instead\n" +
+        "\tof the default C++ ByteWiseComparatorImpl. Must be available on\n" +
+        "\tthe classpath") {
+      @Override
+      protected Object parseValue(final String value) {
+        try {
+          final ComparatorOptions copt = new ComparatorOptions();
+          final Class<AbstractComparator> clsComparator =
+              (Class<AbstractComparator>)Class.forName(value);
+          final Constructor cstr =
+              clsComparator.getConstructor(ComparatorOptions.class);
+          return cstr.newInstance(copt);
+        } catch(final ClassNotFoundException cnfe) {
+          throw new IllegalArgumentException("Java Comparator '" + value + "'" +
+              " not found on the classpath", cnfe);
+        } catch(final NoSuchMethodException nsme) {
+          throw new IllegalArgumentException("Java Comparator '" + value + "'" +
+              " does not have a public ComparatorOptions constructor", nsme);
+        } catch(final IllegalAccessException | InstantiationException
+            | InvocationTargetException ie) {
+          throw new IllegalArgumentException("Unable to construct Java" +
+              " Comparator '" + value + "'", ie);
+        }
+      }
    };

    private Flag(Object defaultValue, String desc) {
@@ -1516,6 +1551,18 @@ public class DbBenchmark {
    private final String desc_;
  }

+  private final static String DEFAULT_TEMP_DIR = "/tmp";
+
+  private static String getTempDir(final String dirName) {
+    try {
+      return Files.createTempDirectory(dirName).toAbsolutePath().toString();
+    } catch(final IOException ioe) {
+      System.err.println("Unable to create temp directory, defaulting to: " +
+          DEFAULT_TEMP_DIR);
+      return DEFAULT_TEMP_DIR + File.pathSeparator + dirName;
+    }
+  }
+
  private static class RandomGenerator {
    private final byte[] data_;
    private int dataLength_;
@@ -21,17 +21,17 @@
 /*
 * Class:     org_rocksdb_BackupableDB
 * Method:    open
- * Signature: (JJ)V
+ * Signature: (JJ)J
 */
-void Java_org_rocksdb_BackupableDB_open(
-    JNIEnv* env, jobject jbdb, jlong jdb_handle, jlong jopt_handle) {
-  auto db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
-  auto opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
+jlong Java_org_rocksdb_BackupableDB_open(
+    JNIEnv* env, jclass jcls, jlong jdb_handle, jlong jopt_handle) {
+  auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
+  auto* opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
  auto bdb = new rocksdb::BackupableDB(db, *opt);

  // as BackupableDB extends RocksDB on the java side, we can reuse
  // the RocksDB portal here.
-  rocksdb::RocksDBJni::setHandle(env, jbdb, bdb);
+  return reinterpret_cast<jlong>(bdb);
 }

 /*
@@ -135,14 +135,14 @@ void Java_org_rocksdb_BackupableDB_garbageCollect(JNIEnv* env,
 /*
 * Class:     org_rocksdb_BackupableDBOptions
 * Method:    newBackupableDBOptions
- * Signature: (Ljava/lang/String;)V
+ * Signature: (Ljava/lang/String;)J
 */
-void Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions(
-    JNIEnv* env, jobject jobj, jstring jpath) {
-  const char* cpath = env->GetStringUTFChars(jpath, 0);
+jlong Java_org_rocksdb_BackupableDBOptions_newBackupableDBOptions(
+    JNIEnv* env, jclass jcls, jstring jpath) {
+  const char* cpath = env->GetStringUTFChars(jpath, NULL);
  auto bopt = new rocksdb::BackupableDBOptions(cpath);
  env->ReleaseStringUTFChars(jpath, cpath);
-  rocksdb::BackupableDBOptionsJni::setHandle(env, jobj, bopt);
+  return reinterpret_cast<jlong>(bopt);
 }

 /*
@@ -320,5 +320,4 @@ void Java_org_rocksdb_BackupableDBOptions_disposeInternal(
  auto bopt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jhandle);
  assert(bopt);
  delete bopt;
-  rocksdb::BackupableDBOptionsJni::setHandle(env, jopt, nullptr);
 }
@@ -16,10 +16,10 @@
 /*
 * Class:     org_rocksdb_BackupEngine
 * Method:    open
- * Signature: (JJ)V
+ * Signature: (JJ)J
 */
-void Java_org_rocksdb_BackupEngine_open(
-    JNIEnv* env, jobject jbe, jlong env_handle,
+jlong Java_org_rocksdb_BackupEngine_open(
+    JNIEnv* env, jclass jcls, jlong env_handle,
    jlong backupable_db_options_handle) {
  auto* rocks_env = reinterpret_cast<rocksdb::Env*>(env_handle);
  auto* backupable_db_options =
@@ -30,11 +30,11 @@ void Java_org_rocksdb_BackupEngine_open(
      *backupable_db_options, &backup_engine);

  if (status.ok()) {
-    rocksdb::BackupEngineJni::setHandle(env, jbe, backup_engine);
-    return;
+    return reinterpret_cast<jlong>(backup_engine);
+  } else {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, status);
+    return 0;
  }
-
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, status);
 }

 /*
@@ -36,15 +36,15 @@ void Java_org_rocksdb_AbstractComparator_disposeInternal(
 /*
 * Class:     org_rocksdb_Comparator
 * Method:    createNewComparator0
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_Comparator_createNewComparator0(
+jlong Java_org_rocksdb_Comparator_createNewComparator0(
    JNIEnv* env, jobject jobj, jlong copt_handle) {
  const rocksdb::ComparatorJniCallbackOptions* copt =
    reinterpret_cast<rocksdb::ComparatorJniCallbackOptions*>(copt_handle);
  const rocksdb::ComparatorJniCallback* c =
    new rocksdb::ComparatorJniCallback(env, jobj, copt);
-  rocksdb::AbstractComparatorJni::setHandle(env, jobj, c);
+  return reinterpret_cast<jlong>(c);
 }
 // </editor-fold>

@@ -53,14 +53,14 @@ void Java_org_rocksdb_Comparator_createNewComparator0(
 /*
 * Class:     org_rocksdb_DirectComparator
 * Method:    createNewDirectComparator0
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_DirectComparator_createNewDirectComparator0(
+jlong Java_org_rocksdb_DirectComparator_createNewDirectComparator0(
    JNIEnv* env, jobject jobj, jlong copt_handle) {
  const rocksdb::ComparatorJniCallbackOptions* copt =
    reinterpret_cast<rocksdb::ComparatorJniCallbackOptions*>(copt_handle);
  const rocksdb::DirectComparatorJniCallback* c =
    new rocksdb::DirectComparatorJniCallback(env, jobj, copt);
-  rocksdb::AbstractComparatorJni::setHandle(env, jobj, c);
+  return reinterpret_cast<jlong>(c);
 }
 // </editor-fold>
@@ -60,8 +60,8 @@ int BaseComparatorJniCallback::Compare(const Slice& a, const Slice& b) const {
  // performance.
  mtx_compare->Lock();

-  AbstractSliceJni::setHandle(m_env, m_jSliceA, &a);
-  AbstractSliceJni::setHandle(m_env, m_jSliceB, &b);
+  AbstractSliceJni::setHandle(m_env, m_jSliceA, &a, JNI_FALSE);
+  AbstractSliceJni::setHandle(m_env, m_jSliceB, &b, JNI_FALSE);
  jint result =
    m_env->CallIntMethod(m_jComparator, m_jCompareMethodId, m_jSliceA,
      m_jSliceB);
@@ -89,7 +89,7 @@ void BaseComparatorJniCallback::FindShortestSeparator(
  // performance.
  mtx_findShortestSeparator->Lock();

-  AbstractSliceJni::setHandle(m_env, m_jSliceLimit, &limit);
+  AbstractSliceJni::setHandle(m_env, m_jSliceLimit, &limit, JNI_FALSE);
  jstring jsResultStart =
    (jstring)m_env->CallObjectMethod(m_jComparator,
      m_jFindShortestSeparatorMethodId, jsStart, m_jSliceLimit);
@@ -19,17 +19,17 @@
 /*
 * Class:     org_rocksdb_BloomFilter
 * Method:    createBloomFilter
- * Signature: (IZ)V
+ * Signature: (IZ)J
 */
-void Java_org_rocksdb_BloomFilter_createNewBloomFilter(
-    JNIEnv* env, jobject jobj, jint bits_per_key,
+jlong Java_org_rocksdb_BloomFilter_createNewBloomFilter(
+    JNIEnv* env, jclass jcls, jint bits_per_key,
    jboolean use_block_base_builder) {
-  rocksdb::FilterPolicy* fp = const_cast<rocksdb::FilterPolicy *>(
+  auto* fp = const_cast<rocksdb::FilterPolicy *>(
      rocksdb::NewBloomFilterPolicy(bits_per_key, use_block_base_builder));
-  std::shared_ptr<rocksdb::FilterPolicy> *pFilterPolicy =
+  auto* pFilterPolicy =
      new std::shared_ptr<rocksdb::FilterPolicy>;
  *pFilterPolicy = std::shared_ptr<rocksdb::FilterPolicy>(fp);
-  rocksdb::FilterJni::setHandle(env, jobj, pFilterPolicy);
+  return reinterpret_cast<jlong>(pFilterPolicy);
 }

 /*
@@ -125,9 +125,9 @@ LoggerJniCallback::~LoggerJniCallback() {
 /*
 * Class:     org_rocksdb_Logger
 * Method:    createNewLoggerOptions
- * Signature: (J)V
+ * Signature: (J)J
 */
-void Java_org_rocksdb_Logger_createNewLoggerOptions(
+jlong Java_org_rocksdb_Logger_createNewLoggerOptions(
    JNIEnv* env, jobject jobj, jlong joptions) {
  rocksdb::LoggerJniCallback* c =
      new rocksdb::LoggerJniCallback(env, jobj);
@@ -137,15 +137,15 @@ void Java_org_rocksdb_Logger_createNewLoggerOptions(
  std::shared_ptr<rocksdb::LoggerJniCallback> *pLoggerJniCallback =
      new std::shared_ptr<rocksdb::LoggerJniCallback>;
  *pLoggerJniCallback = std::shared_ptr<rocksdb::LoggerJniCallback>(c);
-  rocksdb::LoggerJni::setHandle(env, jobj, pLoggerJniCallback);
+  return reinterpret_cast<jlong>(pLoggerJniCallback);
 }

 /*
 * Class:     org_rocksdb_Logger
 * Method:    createNewLoggerDbOptions
- * Signature: (J)V
+ * Signature: (J)J
 */
-void Java_org_rocksdb_Logger_createNewLoggerDbOptions(
+jlong Java_org_rocksdb_Logger_createNewLoggerDbOptions(
    JNIEnv* env, jobject jobj, jlong jdb_options) {
  rocksdb::LoggerJniCallback* c =
      new rocksdb::LoggerJniCallback(env, jobj);
@@ -155,7 +155,7 @@ void Java_org_rocksdb_Logger_createNewLoggerDbOptions(
  std::shared_ptr<rocksdb::LoggerJniCallback> *pLoggerJniCallback =
      new std::shared_ptr<rocksdb::LoggerJniCallback>;
  *pLoggerJniCallback = std::shared_ptr<rocksdb::LoggerJniCallback>(c);
-  rocksdb::LoggerJni::setHandle(env, jobj, pLoggerJniCallback);
+  return reinterpret_cast<jlong>(pLoggerJniCallback);
 }

 /*
@@ -36,25 +36,25 @@
 /*
 * Class:     org_rocksdb_Options
 * Method:    newOptions
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_Options_newOptions__(JNIEnv* env, jobject jobj) {
+jlong Java_org_rocksdb_Options_newOptions__(JNIEnv* env, jclass jcls) {
  rocksdb::Options* op = new rocksdb::Options();
-  rocksdb::OptionsJni::setHandle(env, jobj, op);
+  return reinterpret_cast<jlong>(op);
 }

 /*
 * Class:     org_rocksdb_Options
 * Method:    newOptions
- * Signature: (JJ)V
+ * Signature: (JJ)J
 */
-void Java_org_rocksdb_Options_newOptions__JJ(JNIEnv* env, jobject jobj,
+jlong Java_org_rocksdb_Options_newOptions__JJ(JNIEnv* env, jclass jcls,
    jlong jdboptions, jlong jcfoptions) {
-  auto dbOpt = reinterpret_cast<const rocksdb::DBOptions*>(jdboptions);
-  auto cfOpt = reinterpret_cast<const rocksdb::ColumnFamilyOptions*>(
+  auto* dbOpt = reinterpret_cast<const rocksdb::DBOptions*>(jdboptions);
+  auto* cfOpt = reinterpret_cast<const rocksdb::ColumnFamilyOptions*>(
      jcfoptions);
  rocksdb::Options* op = new rocksdb::Options(*dbOpt, *cfOpt);
-  rocksdb::OptionsJni::setHandle(env, jobj, op);
+  return reinterpret_cast<jlong>(op);
 }

 /*
@@ -1081,21 +1081,20 @@ jbyte Java_org_rocksdb_Options_compressionType(
 * vector.
 */
 std::vector<rocksdb::CompressionType> rocksdb_compression_vector_helper(
-    JNIEnv* env, jobject jcompressionLevels) {
+    JNIEnv* env, jbyteArray jcompressionLevels) {
  std::vector<rocksdb::CompressionType> compressionLevels;
-  // iterate over compressionLevels
-  jobject iteratorObj = env->CallObjectMethod(
-        jcompressionLevels, rocksdb::ListJni::getIteratorMethod(env));
-  while (env->CallBooleanMethod(
-    iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
-    // get compression
-    jobject jcompression_obj = env->CallObjectMethod(iteratorObj,
-        rocksdb::ListJni::getNextMethod(env));
-    jbyte jcompression = env->CallByteMethod(jcompression_obj,
-        rocksdb::ByteJni::getByteValueMethod(env));
-    compressionLevels.push_back(static_cast<rocksdb::CompressionType>(
-        jcompression));
+
+  jsize len = env->GetArrayLength(jcompressionLevels);
+  jbyte* jcompressionLevel = env->GetByteArrayElements(jcompressionLevels,
+    NULL);
+  for(int i = 0; i < len; i++) {
+    jbyte jcl;
+    jcl = jcompressionLevel[i];
+    compressionLevels.push_back(static_cast<rocksdb::CompressionType>(jcl));
  }
+  env->ReleaseByteArrayElements(jcompressionLevels, jcompressionLevel,
+      JNI_ABORT);
+
  return compressionLevels;
 }

@@ -1103,34 +1102,29 @@ std::vector<rocksdb::CompressionType> rocksdb_compression_vector_helper(
 * Helper method to convert a CompressionType vector to a Java
 * List.
 */
-jobject rocksdb_compression_list_helper(JNIEnv* env,
+jbyteArray rocksdb_compression_list_helper(JNIEnv* env,
    std::vector<rocksdb::CompressionType> compressionLevels) {
-  jclass jListClazz = env->FindClass("java/util/ArrayList");
-  jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
-      env, jListClazz);
-  jobject jcompressionLevels = env->NewObject(jListClazz,
-    midList, compressionLevels.size());
-  // insert in java list
+  jbyte jbuf[compressionLevels.size()];
  for (std::vector<rocksdb::CompressionType>::size_type i = 0;
        i != compressionLevels.size(); i++) {
-    jclass jByteClazz = env->FindClass("java/lang/Byte");
-    jmethodID midByte = env->GetMethodID(jByteClazz, "<init>", "(B)V");
-    jobject obj = env->NewObject(jByteClazz, midByte,
-        compressionLevels[i]);
-    env->CallBooleanMethod(jcompressionLevels,
-        rocksdb::ListJni::getListAddMethodId(env), obj);
+      jbuf[i] = compressionLevels[i];
  }
+  // insert in java array
+  jbyteArray jcompressionLevels = env->NewByteArray(
+      static_cast<jsize>(compressionLevels.size()));
+  env->SetByteArrayRegion(jcompressionLevels, 0,
+      static_cast<jsize>(compressionLevels.size()), jbuf);
  return jcompressionLevels;
 }

 /*
 * Class:     org_rocksdb_Options
 * Method:    setCompressionPerLevel
- * Signature: (JLjava/util/List;)V
+ * Signature: (J[B)V
 */
 void Java_org_rocksdb_Options_setCompressionPerLevel(
    JNIEnv* env, jobject jobj, jlong jhandle,
-    jobject jcompressionLevels) {
+    jbyteArray jcompressionLevels) {
  auto* options = reinterpret_cast<rocksdb::Options*>(jhandle);
  std::vector<rocksdb::CompressionType> compressionLevels =
      rocksdb_compression_vector_helper(env, jcompressionLevels);
@@ -1140,9 +1134,9 @@ void Java_org_rocksdb_Options_setCompressionPerLevel(
 /*
 * Class:     org_rocksdb_Options
 * Method:    compressionPerLevel
- * Signature: (J)Ljava/util/List;
+ * Signature: (J)[B
 */
-jobject Java_org_rocksdb_Options_compressionPerLevel(
+jbyteArray Java_org_rocksdb_Options_compressionPerLevel(
    JNIEnv* env, jobject jobj, jlong jhandle) {
  auto* options = reinterpret_cast<rocksdb::Options*>(jhandle);
  return rocksdb_compression_list_helper(env,
@@ -1932,12 +1926,12 @@ void Java_org_rocksdb_Options_prepareForBulkLoad(
 /*
 * Class:     org_rocksdb_ColumnFamilyOptions
 * Method:    newColumnFamilyOptions
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptions(
-    JNIEnv* env, jobject jobj) {
+jlong Java_org_rocksdb_ColumnFamilyOptions_newColumnFamilyOptions(
+    JNIEnv* env, jclass jcls) {
  rocksdb::ColumnFamilyOptions* op = new rocksdb::ColumnFamilyOptions();
-  rocksdb::ColumnFamilyOptionsJni::setHandle(env, jobj, op);
+  return reinterpret_cast<jlong>(op);
 }

 /*
@@ -2285,11 +2279,11 @@ jbyte Java_org_rocksdb_ColumnFamilyOptions_compressionType(
 /*
 * Class:     org_rocksdb_ColumnFamilyOptions
 * Method:    setCompressionPerLevel
- * Signature: (JLjava/util/List;)V
+ * Signature: (J[B)V
 */
 void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel(
    JNIEnv* env, jobject jobj, jlong jhandle,
-    jobject jcompressionLevels) {
+    jbyteArray jcompressionLevels) {
  auto* options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
  std::vector<rocksdb::CompressionType> compressionLevels =
      rocksdb_compression_vector_helper(env, jcompressionLevels);
@@ -2299,9 +2293,9 @@ void Java_org_rocksdb_ColumnFamilyOptions_setCompressionPerLevel(
 /*
 * Class:     org_rocksdb_ColumnFamilyOptions
 * Method:    compressionPerLevel
- * Signature: (J)Ljava/util/List;
+ * Signature: (J)[B
 */
-jobject Java_org_rocksdb_ColumnFamilyOptions_compressionPerLevel(
+jbyteArray Java_org_rocksdb_ColumnFamilyOptions_compressionPerLevel(
    JNIEnv* env, jobject jobj, jlong jhandle) {
  auto* options = reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle);
  return rocksdb_compression_list_helper(env,
@@ -3072,12 +3066,12 @@ void Java_org_rocksdb_ColumnFamilyOptions_setOptimizeFiltersForHits(
 /*
 * Class:     org_rocksdb_DBOptions
 * Method:    newDBOptions
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_DBOptions_newDBOptions(JNIEnv* env,
-    jobject jobj) {
+jlong Java_org_rocksdb_DBOptions_newDBOptions(JNIEnv* env,
+    jclass jcls) {
  rocksdb::DBOptions* dbop = new rocksdb::DBOptions();
-  rocksdb::DBOptionsJni::setHandle(env, jobj, dbop);
+  return reinterpret_cast<jlong>(dbop);
 }

 /*
@@ -3872,12 +3866,12 @@ jlong Java_org_rocksdb_DBOptions_bytesPerSync(
 /*
 * Class:     org_rocksdb_WriteOptions
 * Method:    newWriteOptions
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_WriteOptions_newWriteOptions(
-    JNIEnv* env, jobject jwrite_options) {
+jlong Java_org_rocksdb_WriteOptions_newWriteOptions(
+    JNIEnv* env, jclass jcls) {
  rocksdb::WriteOptions* op = new rocksdb::WriteOptions();
-  rocksdb::WriteOptionsJni::setHandle(env, jwrite_options, op);
+  return reinterpret_cast<jlong>(op);
 }

 /*
@@ -3889,8 +3883,6 @@ void Java_org_rocksdb_WriteOptions_disposeInternal(
    JNIEnv* env, jobject jwrite_options, jlong jhandle) {
  auto write_options = reinterpret_cast<rocksdb::WriteOptions*>(jhandle);
  delete write_options;
-
-  rocksdb::WriteOptionsJni::setHandle(env, jwrite_options, nullptr);
 }

 /*
@@ -3939,12 +3931,12 @@ jboolean Java_org_rocksdb_WriteOptions_disableWAL(
 /*
 * Class:     org_rocksdb_ReadOptions
 * Method:    newReadOptions
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_ReadOptions_newReadOptions(
-    JNIEnv* env, jobject jobj) {
+jlong Java_org_rocksdb_ReadOptions_newReadOptions(
+    JNIEnv* env, jclass jcls) {
  auto read_opt = new rocksdb::ReadOptions();
-  rocksdb::ReadOptionsJni::setHandle(env, jobj, read_opt);
+  return reinterpret_cast<jlong>(read_opt);
 }

 /*
@@ -3955,7 +3947,6 @@ void Java_org_rocksdb_ReadOptions_newReadOptions(
 void Java_org_rocksdb_ReadOptions_disposeInternal(
    JNIEnv* env, jobject jobj, jlong jhandle) {
  delete reinterpret_cast<rocksdb::ReadOptions*>(jhandle);
-  rocksdb::ReadOptionsJni::setHandle(env, jobj, nullptr);
 }

 /*
@@ -4052,12 +4043,12 @@ jlong Java_org_rocksdb_ReadOptions_snapshot(
 /*
 * Class:     org_rocksdb_ComparatorOptions
 * Method:    newComparatorOptions
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_ComparatorOptions_newComparatorOptions(
-    JNIEnv* env, jobject jobj) {
+jlong Java_org_rocksdb_ComparatorOptions_newComparatorOptions(
+    JNIEnv* env, jclass jcls) {
  auto comparator_opt = new rocksdb::ComparatorJniCallbackOptions();
-  rocksdb::ComparatorOptionsJni::setHandle(env, jobj, comparator_opt);
+  return reinterpret_cast<jlong>(comparator_opt);
 }

 /*
@@ -4090,7 +4081,6 @@ void Java_org_rocksdb_ComparatorOptions_setUseAdaptiveMutex(
 void Java_org_rocksdb_ComparatorOptions_disposeInternal(
    JNIEnv * env, jobject jobj, jlong jhandle) {
  delete reinterpret_cast<rocksdb::ComparatorJniCallbackOptions*>(jhandle);
-  rocksdb::ComparatorOptionsJni::setHandle(env, jobj, nullptr);
 }

 /////////////////////////////////////////////////////////////////////
@@ -4099,12 +4089,12 @@ void Java_org_rocksdb_ComparatorOptions_disposeInternal(
 /*
 * Class:     org_rocksdb_FlushOptions
 * Method:    newFlushOptions
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_FlushOptions_newFlushOptions(
-    JNIEnv* env, jobject jobj) {
+jlong Java_org_rocksdb_FlushOptions_newFlushOptions(
+    JNIEnv* env, jclass jcls) {
  auto flush_opt = new rocksdb::FlushOptions();
-  rocksdb::FlushOptionsJni::setHandle(env, jobj, flush_opt);
+  return reinterpret_cast<jlong>(flush_opt);
 }

 /*
@@ -4137,5 +4127,4 @@ jboolean Java_org_rocksdb_FlushOptions_waitForFlush(
 void Java_org_rocksdb_FlushOptions_disposeInternal(
    JNIEnv * env, jobject jobj, jlong jhandle) {
  delete reinterpret_cast<rocksdb::FlushOptions*>(jhandle);
-  rocksdb::FlushOptionsJni::setHandle(env, jobj, nullptr);
 }
@@ -49,27 +49,25 @@ template<class PTR, class DERIVED> class RocksDBNativeClass {
    assert(jclazz != nullptr);
    return jclazz;
  }
+};

-  // Get the field id of the member variable to store
-  // the ptr
-  static jfieldID getHandleFieldID(JNIEnv* env) {
-    static jfieldID fid = env->GetFieldID(
-        DERIVED::getJClass(env), "nativeHandle_", "J");
-    assert(fid != nullptr);
-    return fid;
-  }
+// Native class template for sub-classes of RocksMutableObject
+template<class PTR, class DERIVED> class NativeRocksMutableObject
+    : public RocksDBNativeClass<PTR, DERIVED> {
+ public:

-  // Get the pointer from Java
-  static PTR getHandle(JNIEnv* env, jobject jobj) {
-    return reinterpret_cast<PTR>(
-        env->GetLongField(jobj, getHandleFieldID(env)));
+  static jmethodID getSetNativeHandleMethod(JNIEnv* env) {
+    static jmethodID mid = env->GetMethodID(
+        DERIVED::getJClass(env), "setNativeHandle", "(JZ)V");
+    assert(mid != nullptr);
+    return mid;
  }

  // Pass the pointer to the java side.
-  static void setHandle(JNIEnv* env, jobject jdb, PTR ptr) {
-    env->SetLongField(
-        jdb, getHandleFieldID(env),
-        reinterpret_cast<jlong>(ptr));
+  static void setHandle(JNIEnv* env, jobject jobj, PTR ptr,
+      jboolean java_owns_handle) {
+    env->CallVoidMethod(jobj, getSetNativeHandleMethod(env),
+      reinterpret_cast<jlong>(ptr), java_owns_handle);
  }
 };

@@ -407,7 +405,7 @@ class AbstractComparatorJni : public RocksDBNativeClass<
 };

 // The portal class for org.rocksdb.AbstractSlice
-class AbstractSliceJni : public RocksDBNativeClass<
+class AbstractSliceJni : public NativeRocksMutableObject<
    const rocksdb::Slice*, AbstractSliceJni> {
 public:
  static jclass getJClass(JNIEnv* env) {
@@ -649,67 +647,6 @@ class WriteEntryJni {
      assert(jclazz != nullptr);
      return jclazz;
    }
-
-    static void setWriteType(JNIEnv* env, jobject jwrite_entry,
-        WriteType write_type) {
-      jobject jwrite_type;
-      switch (write_type) {
-        case kPutRecord:
-          jwrite_type = WriteTypeJni::PUT(env);
-          break;
-
-        case kMergeRecord:
-          jwrite_type = WriteTypeJni::MERGE(env);
-          break;
-
-        case kDeleteRecord:
-          jwrite_type = WriteTypeJni::DELETE(env);
-          break;
-
-        case kLogDataRecord:
-          jwrite_type = WriteTypeJni::LOG(env);
-          break;
-
-        default:
-          jwrite_type = nullptr;
-      }
-      assert(jwrite_type != nullptr);
-      env->SetObjectField(jwrite_entry, getWriteTypeField(env), jwrite_type);
-    }
-
-    static void setKey(JNIEnv* env, jobject jwrite_entry,
-        const rocksdb::Slice* slice) {
-      jobject jkey = env->GetObjectField(jwrite_entry, getKeyField(env));
-      AbstractSliceJni::setHandle(env, jkey, slice);
-    }
-
-    static void setValue(JNIEnv* env, jobject jwrite_entry,
-        const rocksdb::Slice* slice) {
-      jobject jvalue = env->GetObjectField(jwrite_entry, getValueField(env));
-      AbstractSliceJni::setHandle(env, jvalue, slice);
-    }
-
- private:
-    static jfieldID getWriteTypeField(JNIEnv* env) {
-      static jfieldID fid = env->GetFieldID(
-          getJClass(env), "type", "Lorg/rocksdb/WBWIRocksIterator$WriteType;");
-        assert(fid != nullptr);
-        return fid;
-    }
-
-    static jfieldID getKeyField(JNIEnv* env) {
-      static jfieldID fid = env->GetFieldID(
-          getJClass(env), "key", "Lorg/rocksdb/DirectSlice;");
-      assert(fid != nullptr);
-      return fid;
-    }
-
-    static jfieldID getValueField(JNIEnv* env) {
-      static jfieldID fid = env->GetFieldID(
-          getJClass(env), "value", "Lorg/rocksdb/DirectSlice;");
-      assert(fid != nullptr);
-      return fid;
-    }
 };

 class InfoLogLevelJni {
@@ -12,16 +12,13 @@
 /*
 * Class:     org_rocksdb_RemoveEmptyValueCompactionFilter
 * Method:    createNewRemoveEmptyValueCompactionFilter0
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_RemoveEmptyValueCompactionFilter_createNewRemoveEmptyValueCompactionFilter0(
-    JNIEnv* env, jobject jobj) {
-  const rocksdb::RemoveEmptyValueCompactionFilter* compaction_filter =
+jlong Java_org_rocksdb_RemoveEmptyValueCompactionFilter_createNewRemoveEmptyValueCompactionFilter0(
+    JNIEnv* env, jclass jcls) {
+  auto* compaction_filter =
      new rocksdb::RemoveEmptyValueCompactionFilter();

  // set the native handle to our native compaction filter
-  static jclass jclazz =
-      env->FindClass("org/rocksdb/RemoveEmptyValueCompactionFilter");
-  static jfieldID fid = env->GetFieldID(jclazz, "nativeHandle_", "J");
-  env->SetLongField(jobj, fid, reinterpret_cast<jlong>(compaction_filter));
+  return reinterpret_cast<jlong>(compaction_filter);
 }
@@ -22,17 +22,17 @@
 * Signature: (Z)J
 */
 jlong Java_org_rocksdb_RestoreOptions_newRestoreOptions(JNIEnv* env,
-    jobject jobj, jboolean keep_log_files) {
+    jclass jcls, jboolean keep_log_files) {
  auto ropt = new rocksdb::RestoreOptions(keep_log_files);
  return reinterpret_cast<jlong>(ropt);
 }

 /*
 * Class:     org_rocksdb_RestoreOptions
- * Method:    dispose
+ * Method:    disposeInternal
 * Signature: (J)V
 */
-void Java_org_rocksdb_RestoreOptions_dispose(JNIEnv* env, jobject jobj,
+void Java_org_rocksdb_RestoreOptions_disposeInternal(JNIEnv* env, jobject jobj,
    jlong jhandle) {
  auto ropt = reinterpret_cast<rocksdb::RestoreOptions*>(jhandle);
  assert(ropt);
@@ -45,8 +45,8 @@ void Java_org_rocksdb_RestoreOptions_dispose(JNIEnv* env, jobject jobj,
 * Signature: (J)J
 */
 jlong Java_org_rocksdb_RestoreBackupableDB_newRestoreBackupableDB(JNIEnv* env,
-    jobject jobj, jlong jopt_handle) {
-  auto opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
+    jclass jcls, jlong jopt_handle) {
+  auto* opt = reinterpret_cast<rocksdb::BackupableDBOptions*>(jopt_handle);
  auto rdb = new rocksdb::RestoreBackupableDB(rocksdb::Env::Default(), *opt);
  return reinterpret_cast<jlong>(rdb);
 }
@@ -185,11 +185,11 @@ void Java_org_rocksdb_RestoreBackupableDB_garbageCollect(

 /*
 * Class:     org_rocksdb_RestoreBackupableDB
- * Method:    dispose
+ * Method:    disposeInternal
 * Signature: (J)V
 */
-void Java_org_rocksdb_RestoreBackupableDB_dispose(JNIEnv* env, jobject jobj,
-    jlong jhandle) {
+void Java_org_rocksdb_RestoreBackupableDB_disposeInternal(JNIEnv* env,
+    jobject jobj, jlong jhandle) {
  auto ropt = reinterpret_cast<rocksdb::RestoreBackupableDB*>(jhandle);
  assert(ropt);
  delete ropt;
@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <memory>
 #include <string>
+#include <tuple>
 #include <vector>
 #include <algorithm>

@@ -26,217 +27,142 @@

 //////////////////////////////////////////////////////////////////////////////
 // rocksdb::DB::Open
-
-/*
- * Class:     org_rocksdb_RocksDB
- * Method:    open
- * Signature: (JLjava/lang/String;)V
- */
-void Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2(
-    JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path) {
-  auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
+jlong rocksdb_open_helper(JNIEnv* env, jlong jopt_handle, jstring jdb_path,
+    std::function<rocksdb::Status(
+      const rocksdb::Options&, const std::string&, rocksdb::DB**)> open_fn
+    ) {
+  auto* opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
  rocksdb::DB* db = nullptr;
-  const char* db_path = env->GetStringUTFChars(jdb_path, 0);
-  rocksdb::Status s = rocksdb::DB::Open(*opt, db_path, &db);
+  const char* db_path = env->GetStringUTFChars(jdb_path, NULL);
+  rocksdb::Status s = open_fn(*opt, db_path, &db);
  env->ReleaseStringUTFChars(jdb_path, db_path);

  if (s.ok()) {
-    rocksdb::RocksDBJni::setHandle(env, jdb, db);
-    return;
+    return reinterpret_cast<jlong>(db);
+  } else {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+    return 0;
  }
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
-}
-
-/*
- * Class:     org_rocksdb_RocksDB
- * Method:    openROnly
- * Signature: (JLjava/lang/String;)V
- */
-void Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2(
-    JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path) {
-  auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
-  rocksdb::DB* db = nullptr;
-  const char* db_path = env->GetStringUTFChars(jdb_path, 0);
-  rocksdb::Status s = rocksdb::DB::OpenForReadOnly(*opt,
-      db_path, &db);
-  env->ReleaseStringUTFChars(jdb_path, db_path);
-
-  if (s.ok()) {
-    rocksdb::RocksDBJni::setHandle(env, jdb, db);
-    return;
-  }
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
-}
-
-/*
- * Class:     org_rocksdb_RocksDB
- * Method:    openROnly
- * Signature: (JLjava/lang/String;Ljava/util/List;I)Ljava/util/List;
- */
-jobject
-    Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2Ljava_util_List_2I(
-    JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path,
-    jobject jcfdesc_list, jint jcfdesc_count) {
-  auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
-  rocksdb::DB* db = nullptr;
-  const char* db_path = env->GetStringUTFChars(jdb_path, 0);
-
-  std::vector<jbyte*> cfnames_to_free;
-  std::vector<jbyteArray> jcfnames_for_free;
-
-  std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
-  std::vector<rocksdb::ColumnFamilyHandle* > handles;
-  // get iterator for ColumnFamilyDescriptors
-  jobject iteratorObj = env->CallObjectMethod(
-      jcfdesc_list, rocksdb::ListJni::getIteratorMethod(env));
-
-  // iterate over ColumnFamilyDescriptors
-  while (env->CallBooleanMethod(
-      iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
-      // get ColumnFamilyDescriptor
-      jobject jcf_descriptor = env->CallObjectMethod(iteratorObj,
-          rocksdb::ListJni::getNextMethod(env));
-      // get ColumnFamilyName
-      jbyteArray cf_name_in_byte_array = static_cast<jbyteArray>(
-          env->CallObjectMethod(jcf_descriptor,
-          rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
-              env)));
-      // get CF Options
-      jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
-          rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
-              env));
-      rocksdb::ColumnFamilyOptions* cfOptions =
-          rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
-
-      jbyte* cfname = env->GetByteArrayElements(cf_name_in_byte_array, 0);
-      const int len = env->GetArrayLength(cf_name_in_byte_array);
-
-      // free allocated cfnames after call to open
-      cfnames_to_free.push_back(cfname);
-      jcfnames_for_free.push_back(cf_name_in_byte_array);
-      column_families.push_back(rocksdb::ColumnFamilyDescriptor(
-          std::string(reinterpret_cast<char *>(cfname), len), *cfOptions));
-  }
-
-  rocksdb::Status s = rocksdb::DB::OpenForReadOnly(*opt,
-      db_path, column_families, &handles, &db);
-  env->ReleaseStringUTFChars(jdb_path, db_path);
-  // free jbyte allocations
-  for (std::vector<jbyte*>::size_type i = 0;
-      i != cfnames_to_free.size(); i++) {
-    // free  cfnames
-    env->ReleaseByteArrayElements(jcfnames_for_free[i], cfnames_to_free[i], 0);
-  }
-
-  // check if open operation was successful
-  if (s.ok()) {
-    rocksdb::RocksDBJni::setHandle(env, jdb, db);
-    jclass jListClazz = env->FindClass("java/util/ArrayList");
-    jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
-        env, jListClazz);
-    jobject jcfhandle_list = env->NewObject(jListClazz,
-        midList, handles.size());
-    // insert in java list
-    for (std::vector<rocksdb::ColumnFamilyHandle*>::size_type i = 0;
-        i != handles.size(); i++) {
-      // jlong must be converted to Long due to collections restrictions
-      jclass jLongClazz = env->FindClass("java/lang/Long");
-      jmethodID midLong = env->GetMethodID(jLongClazz, "<init>", "(J)V");
-      jobject obj = env->NewObject(jLongClazz, midLong,
-          reinterpret_cast<jlong>(handles[i]));
-      env->CallBooleanMethod(jcfhandle_list,
-          rocksdb::ListJni::getListAddMethodId(env), obj);
-    }
-
-    return jcfhandle_list;
-  }
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
-  return nullptr;
 }

 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    open
- * Signature: (JLjava/lang/String;Ljava/util/List;I)Ljava/util/List;
+ * Signature: (JLjava/lang/String;)J
 */
-jobject Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2Ljava_util_List_2I(
-    JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path,
-    jobject jcfdesc_list, jint jcfdesc_count) {
-  auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
-  rocksdb::DB* db = nullptr;
-  const char* db_path = env->GetStringUTFChars(jdb_path, 0);
+jlong Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2(
+    JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path) {
+  return rocksdb_open_helper(env, jopt_handle, jdb_path,
+    (rocksdb::Status(*)
+      (const rocksdb::Options&, const std::string&, rocksdb::DB**)
+    )&rocksdb::DB::Open
+  );
+}

-  std::vector<jbyte*> cfnames_to_free;
-  std::vector<jbyteArray> jcfnames_for_free;
+/*
+ * Class:     org_rocksdb_RocksDB
+ * Method:    openROnly
+ * Signature: (JLjava/lang/String;)J
+ */
+jlong Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2(
+    JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path) {
+  return rocksdb_open_helper(env, jopt_handle, jdb_path, [](
+      const rocksdb::Options& options,
+      const std::string& db_path, rocksdb::DB** db) {
+    return rocksdb::DB::OpenForReadOnly(options, db_path, db);
+  });
+}
+
+jlongArray rocksdb_open_helper(JNIEnv* env, jlong jopt_handle,
+    jstring jdb_path, jobjectArray jcolumn_names, jlongArray jcolumn_options,
+    std::function<rocksdb::Status(
+      const rocksdb::DBOptions&, const std::string&,
+      const std::vector<rocksdb::ColumnFamilyDescriptor>&,
+      std::vector<rocksdb::ColumnFamilyHandle*>*,
+      rocksdb::DB**)> open_fn
+    ) {
+  auto* opt = reinterpret_cast<rocksdb::DBOptions*>(jopt_handle);
+  const char* db_path = env->GetStringUTFChars(jdb_path, NULL);

  std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
-  std::vector<rocksdb::ColumnFamilyHandle* > handles;
-  // get iterator for ColumnFamilyDescriptors
-  jobject iteratorObj = env->CallObjectMethod(
-      jcfdesc_list, rocksdb::ListJni::getIteratorMethod(env));

-  // iterate over ColumnFamilyDescriptors
-  while (env->CallBooleanMethod(
-      iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
-      // get ColumnFamilyDescriptor
-      jobject jcf_descriptor = env->CallObjectMethod(iteratorObj,
-          rocksdb::ListJni::getNextMethod(env));
-      // get ColumnFamilyName
-      jbyteArray cf_name_in_byte_array = static_cast<jbyteArray>(
-          env->CallObjectMethod(jcf_descriptor,
-          rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
-              env)));
-      // get CF Options
-      jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
-          rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
-              env));
-      rocksdb::ColumnFamilyOptions* cfOptions =
-          rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
+  jsize len_cols = env->GetArrayLength(jcolumn_names);
+  jlong* jco = env->GetLongArrayElements(jcolumn_options, NULL);
+  for(int i = 0; i < len_cols; i++) {
+    jobject jcn = env->GetObjectArrayElement(jcolumn_names, i);
+    jbyteArray jcn_ba = reinterpret_cast<jbyteArray>(jcn);
+    jbyte* jcf_name = env->GetByteArrayElements(jcn_ba, NULL);
+    const int jcf_name_len = env->GetArrayLength(jcn_ba);

-      jbyte* cfname = env->GetByteArrayElements(cf_name_in_byte_array, 0);
-      const int len = env->GetArrayLength(cf_name_in_byte_array);
+    //TODO(AR) do I need to make a copy of jco[i] ?

-      // free allocated cfnames after call to open
-      cfnames_to_free.push_back(cfname);
-      jcfnames_for_free.push_back(cf_name_in_byte_array);
-      column_families.push_back(rocksdb::ColumnFamilyDescriptor(
-          std::string(reinterpret_cast<char *>(cfname), len), *cfOptions));
+    std::string cf_name (reinterpret_cast<char *>(jcf_name), jcf_name_len);
+    rocksdb::ColumnFamilyOptions* cf_options =
+      reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jco[i]);
+    column_families.push_back(
+      rocksdb::ColumnFamilyDescriptor(cf_name, *cf_options));
+
+    env->ReleaseByteArrayElements(jcn_ba, jcf_name, JNI_ABORT);
+    env->DeleteLocalRef(jcn);
  }
+  env->ReleaseLongArrayElements(jcolumn_options, jco, JNI_ABORT);

-  rocksdb::Status s = rocksdb::DB::Open(*opt, db_path, column_families,
+  std::vector<rocksdb::ColumnFamilyHandle*> handles;
+  rocksdb::DB* db = nullptr;
+  rocksdb::Status s = open_fn(*opt, db_path, column_families,
      &handles, &db);
-  env->ReleaseStringUTFChars(jdb_path, db_path);
-  // free jbyte allocations
-  for (std::vector<jbyte*>::size_type i = 0;
-      i != cfnames_to_free.size(); i++) {
-    // free  cfnames
-    env->ReleaseByteArrayElements(jcfnames_for_free[i], cfnames_to_free[i], 0);
-  }

  // check if open operation was successful
  if (s.ok()) {
-    rocksdb::RocksDBJni::setHandle(env, jdb, db);
-    jclass jListClazz = env->FindClass("java/util/ArrayList");
-    jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
-        env, jListClazz);
-    jobject jcfhandle_list = env->NewObject(jListClazz,
-        midList, handles.size());
-    // insert in java list
-    for (std::vector<rocksdb::ColumnFamilyHandle*>::size_type i = 0;
-        i != handles.size(); i++) {
-      // jlong must be converted to Long due to collections restrictions
-      jclass jLongClazz = env->FindClass("java/lang/Long");
-      jmethodID midLong = env->GetMethodID(jLongClazz, "<init>", "(J)V");
-      jobject obj = env->NewObject(jLongClazz, midLong,
-          reinterpret_cast<jlong>(handles[i]));
-      env->CallBooleanMethod(jcfhandle_list,
-          rocksdb::ListJni::getListAddMethodId(env), obj);
+    jsize resultsLen = 1 + len_cols; //db handle + column family handles
+    jlong results[resultsLen];
+    results[0] = reinterpret_cast<jlong>(db);
+    for(int i = 1; i <= len_cols; i++) {
+      results[i] = reinterpret_cast<jlong>(handles[i - 1]);
    }

-    return jcfhandle_list;
+    jlongArray jresults = env->NewLongArray(resultsLen);
+    env->SetLongArrayRegion(jresults, 0, resultsLen, results);
+    return jresults;
+  } else {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+    return NULL;
  }
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
-  return nullptr;
+}
+
+/*
+ * Class:     org_rocksdb_RocksDB
+ * Method:    openROnly
+ * Signature: (JLjava/lang/String;[[B[J)[J
+ */
+jlongArray Java_org_rocksdb_RocksDB_openROnly__JLjava_lang_String_2_3_3B_3J(
+    JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path,
+    jobjectArray jcolumn_names, jlongArray jcolumn_options) {
+  return rocksdb_open_helper(env, jopt_handle, jdb_path, jcolumn_names,
+    jcolumn_options, [](
+        const rocksdb::DBOptions& options, const std::string& db_path,
+        const std::vector<rocksdb::ColumnFamilyDescriptor>& column_families,
+        std::vector<rocksdb::ColumnFamilyHandle*>* handles, rocksdb::DB** db) {
+      return rocksdb::DB::OpenForReadOnly(options, db_path, column_families,
+        handles, db);
+  });
+}
+
+/*
+ * Class:     org_rocksdb_RocksDB
+ * Method:    open
+ * Signature: (JLjava/lang/String;[[B[J)[J
+ */
+jlongArray Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2_3_3B_3J(
+    JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path,
+    jobjectArray jcolumn_names, jlongArray jcolumn_options) {
+  return rocksdb_open_helper(env, jopt_handle, jdb_path, jcolumn_names,
+    jcolumn_options, (rocksdb::Status(*)
+      (const rocksdb::DBOptions&, const std::string&,
+        const std::vector<rocksdb::ColumnFamilyDescriptor>&,
+        std::vector<rocksdb::ColumnFamilyHandle*>*, rocksdb::DB**)
+      )&rocksdb::DB::Open
+    );
 }

 //////////////////////////////////////////////////////////////////////////////
@@ -245,25 +171,21 @@ jobject Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2Ljava_util_List_2I(
 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    listColumnFamilies
- * Signature: (JLjava/lang/String;)Ljava/util/List;
+ * Signature: (JLjava/lang/String;)[[B
 */
-jobject Java_org_rocksdb_RocksDB_listColumnFamilies(
+jobjectArray Java_org_rocksdb_RocksDB_listColumnFamilies(
    JNIEnv* env, jclass jclazz, jlong jopt_handle, jstring jdb_path) {
  std::vector<std::string> column_family_names;
-  auto opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
+  auto* opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
  const char* db_path = env->GetStringUTFChars(jdb_path, 0);
-  jobject jvalue_list = nullptr;
-
  rocksdb::Status s = rocksdb::DB::ListColumnFamilies(*opt, db_path,
      &column_family_names);
  env->ReleaseStringUTFChars(jdb_path, db_path);
-  if (s.ok()) {
-    // Don't reuse class pointer
-    jclass jListClazz = env->FindClass("java/util/ArrayList");
-    jmethodID mid = rocksdb::ListJni::getArrayListConstructorMethodId(env,
-        jListClazz);
-    jvalue_list = env->NewObject(jListClazz, mid, column_family_names.size());

+  jclass jcls_ba = env->FindClass("[B");
+  jobjectArray jresults = env->NewObjectArray(
+      static_cast<jsize>(column_family_names.size()), jcls_ba, NULL);
+  if (s.ok()) {
    for (std::vector<std::string>::size_type i = 0;
        i < column_family_names.size(); i++) {
      jbyteArray jcf_value =
@@ -271,11 +193,11 @@ jobject Java_org_rocksdb_RocksDB_listColumnFamilies(
      env->SetByteArrayRegion(
          jcf_value, 0, static_cast<jsize>(column_family_names[i].size()),
          reinterpret_cast<const jbyte*>(column_family_names[i].data()));
-      env->CallBooleanMethod(jvalue_list,
-          rocksdb::ListJni::getListAddMethodId(env), jcf_value);
+      env->SetObjectArrayElement(jresults, static_cast<jsize>(i), jcf_value);
+      env->DeleteLocalRef(jcf_value);
    }
  }
-  return jvalue_list;
+  return jresults;
 }

 //////////////////////////////////////////////////////////////////////////////
@@ -398,12 +320,12 @@ void Java_org_rocksdb_RocksDB_put__JJ_3BI_3BIJ(
 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    write0
- * Signature: (JJ)V
+ * Signature: (JJJ)V
 */
 void Java_org_rocksdb_RocksDB_write0(
-    JNIEnv* env, jobject jdb,
+    JNIEnv* env, jobject jdb, jlong jdb_handle,
    jlong jwrite_options_handle, jlong jwb_handle) {
-  rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
+  auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
  auto* write_options = reinterpret_cast<rocksdb::WriteOptions*>(
      jwrite_options_handle);
  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
@@ -418,12 +340,12 @@ void Java_org_rocksdb_RocksDB_write0(
 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    write1
- * Signature: (JJ)V
+ * Signature: (JJJ)V
 */
 void Java_org_rocksdb_RocksDB_write1(
-    JNIEnv* env, jobject jdb,
+    JNIEnv* env, jobject jdb, jlong jdb_handle,
    jlong jwrite_options_handle, jlong jwbwi_handle) {
-  rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
+  auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
  auto* write_options = reinterpret_cast<rocksdb::WriteOptions*>(
      jwrite_options_handle);
  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
@@ -467,52 +389,16 @@ jboolean key_may_exist_helper(JNIEnv* env, rocksdb::DB* db,
  return static_cast<jboolean>(keyMayExist);
 }

-/*
- * Class:     org_rocksdb_RocksDB
- * Method:    keyMayExist
- * Signature: ([BILjava/lang/StringBuffer;)Z
- */
-jboolean Java_org_rocksdb_RocksDB_keyMayExist___3BILjava_lang_StringBuffer_2(
-    JNIEnv* env, jobject jdb, jbyteArray jkey, jint jkey_len,
-    jobject jstring_buffer) {
-  rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
-  return key_may_exist_helper(env, db, rocksdb::ReadOptions(),
-      nullptr, jkey, jkey_len, jstring_buffer);
-}
-
-/*
- * Class:     org_rocksdb_RocksDB
- * Method:    keyMayExist
- * Signature: ([BIJLjava/lang/StringBuffer;)Z
- */
-jboolean Java_org_rocksdb_RocksDB_keyMayExist___3BIJLjava_lang_StringBuffer_2(
-    JNIEnv* env, jobject jdb, jbyteArray jkey, jint jkey_len,
-    jlong jcf_handle, jobject jstring_buffer) {
-  rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
-  auto cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(
-      jcf_handle);
-  if (cf_handle != nullptr) {
-    return key_may_exist_helper(env, db, rocksdb::ReadOptions(),
-        cf_handle, jkey, jkey_len, jstring_buffer);
-  } else {
-    rocksdb::RocksDBExceptionJni::ThrowNew(env,
-        rocksdb::Status::InvalidArgument("Invalid ColumnFamilyHandle."));
-  }
-  return true;
-}
-
 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    keyMayExist
 * Signature: (J[BILjava/lang/StringBuffer;)Z
 */
 jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BILjava_lang_StringBuffer_2(
-    JNIEnv* env, jobject jdb, jlong jread_options_handle,
-    jbyteArray jkey, jint jkey_len, jobject jstring_buffer) {
-  rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
-  auto& read_options = *reinterpret_cast<rocksdb::ReadOptions*>(
-      jread_options_handle);
-  return key_may_exist_helper(env, db, read_options,
+    JNIEnv* env, jobject jdb, jlong jdb_handle, jbyteArray jkey, jint jkey_len,
+    jobject jstring_buffer) {
+  auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
+  return key_may_exist_helper(env, db, rocksdb::ReadOptions(),
      nullptr, jkey, jkey_len, jstring_buffer);
 }

@@ -522,12 +408,48 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BILjava_lang_StringBuffer_2(
 * Signature: (J[BIJLjava/lang/StringBuffer;)Z
 */
 jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BIJLjava_lang_StringBuffer_2(
-    JNIEnv* env, jobject jdb, jlong jread_options_handle,
-    jbyteArray jkey, jint jkey_len, jlong jcf_handle, jobject jstring_buffer) {
-  rocksdb::DB* db = rocksdb::RocksDBJni::getHandle(env, jdb);
+    JNIEnv* env, jobject jdb, jlong jdb_handle, jbyteArray jkey, jint jkey_len,
+    jlong jcf_handle, jobject jstring_buffer) {
+  auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
+  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(
+      jcf_handle);
+  if (cf_handle != nullptr) {
+    return key_may_exist_helper(env, db, rocksdb::ReadOptions(),
+        cf_handle, jkey, jkey_len, jstring_buffer);
+  } else {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env,
+        rocksdb::Status::InvalidArgument("Invalid ColumnFamilyHandle."));
+    return true;
+  }
+}
+
+/*
+ * Class:     org_rocksdb_RocksDB
+ * Method:    keyMayExist
+ * Signature: (JJ[BILjava/lang/StringBuffer;)Z
+ */
+jboolean Java_org_rocksdb_RocksDB_keyMayExist__JJ_3BILjava_lang_StringBuffer_2(
+    JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jread_options_handle,
+    jbyteArray jkey, jint jkey_len, jobject jstring_buffer) {
+  auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
  auto& read_options = *reinterpret_cast<rocksdb::ReadOptions*>(
      jread_options_handle);
-  auto cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(
+  return key_may_exist_helper(env, db, read_options,
+      nullptr, jkey, jkey_len, jstring_buffer);
+}
+
+/*
+ * Class:     org_rocksdb_RocksDB
+ * Method:    keyMayExist
+ * Signature: (JJ[BIJLjava/lang/StringBuffer;)Z
+ */
+jboolean Java_org_rocksdb_RocksDB_keyMayExist__JJ_3BIJLjava_lang_StringBuffer_2(
+    JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jread_options_handle,
+    jbyteArray jkey, jint jkey_len, jlong jcf_handle, jobject jstring_buffer) {
+  auto* db = reinterpret_cast<rocksdb::DB*>(jdb_handle);
+  auto& read_options = *reinterpret_cast<rocksdb::ReadOptions*>(
+      jread_options_handle);
+  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(
      jcf_handle);
  if (cf_handle != nullptr) {
    return key_may_exist_helper(env, db, read_options, cf_handle,
@@ -535,8 +457,8 @@ jboolean Java_org_rocksdb_RocksDB_keyMayExist__J_3BIJLjava_lang_StringBuffer_2(
  } else {
    rocksdb::RocksDBExceptionJni::ThrowNew(env,
        rocksdb::Status::InvalidArgument("Invalid ColumnFamilyHandle."));
+    return true;
  }
-  return true;
 }

 //////////////////////////////////////////////////////////////////////////////
@@ -703,49 +625,38 @@ jint rocksdb_get_helper(
 }

 // cf multi get
-jobject multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db,
-    const rocksdb::ReadOptions& rOpt, jobject jkey_list, jint jkeys_count,
-    jobject jcfhandle_list) {
-  std::vector<rocksdb::Slice> keys;
-  std::vector<jbyte*> keys_to_free;
+jobjectArray multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db,
+    const rocksdb::ReadOptions& rOpt, jobjectArray jkeys,
+    jlongArray jcolumn_family_handles) {
  std::vector<rocksdb::ColumnFamilyHandle*> cf_handles;
-
-  if (jcfhandle_list != nullptr) {
-    // get cf iterator
-    jobject cfIteratorObj = env->CallObjectMethod(
-        jcfhandle_list, rocksdb::ListJni::getIteratorMethod(env));
-
-    // iterate over keys and convert java byte array to slice
-    while (env->CallBooleanMethod(
-        cfIteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
-      jobject jobj = (jbyteArray) env->CallObjectMethod(
-          cfIteratorObj, rocksdb::ListJni::getNextMethod(env));
-      rocksdb::ColumnFamilyHandle* cfHandle =
-          rocksdb::ColumnFamilyHandleJni::getHandle(env, jobj);
-      cf_handles.push_back(cfHandle);
+  if (jcolumn_family_handles != nullptr) {
+    jsize len_cols = env->GetArrayLength(jcolumn_family_handles);
+    jlong* jcfh = env->GetLongArrayElements(jcolumn_family_handles, NULL);
+    for (int i = 0; i < len_cols; i++) {
+      auto* cf_handle =
+          reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcfh[i]);
+      cf_handles.push_back(cf_handle);
    }
+    env->ReleaseLongArrayElements(jcolumn_family_handles, jcfh, JNI_ABORT);
  }

-  // Process key list
-  // get iterator
-  jobject iteratorObj = env->CallObjectMethod(
-      jkey_list, rocksdb::ListJni::getIteratorMethod(env));
+  std::vector<rocksdb::Slice> keys;
+  std::vector<std::tuple<jbyteArray, jbyte*, jobject>> keys_to_free;
+  jsize len_keys = env->GetArrayLength(jkeys);
+  if(env->EnsureLocalCapacity(len_keys) != 0) {
+    // out of memory
+    return NULL;
+  }
+  for (int i = 0; i < len_keys; i++) {
+    jobject jk = env->GetObjectArrayElement(jkeys, i);
+    jbyteArray jk_ba = reinterpret_cast<jbyteArray>(jk);
+    jsize len_key = env->GetArrayLength(jk_ba);
+    jbyte* jk_val = env->GetByteArrayElements(jk_ba, NULL);

-  // iterate over keys and convert java byte array to slice
-  while (env->CallBooleanMethod(
-      iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
-    jbyteArray jkey = (jbyteArray) env->CallObjectMethod(
-       iteratorObj, rocksdb::ListJni::getNextMethod(env));
-    jint key_length = env->GetArrayLength(jkey);
-
-    jbyte* key = new jbyte[key_length];
-    env->GetByteArrayRegion(jkey, 0, key_length, key);
-    // store allocated jbyte to free it after multiGet call
-    keys_to_free.push_back(key);
-
-    rocksdb::Slice key_slice(
-      reinterpret_cast<char*>(key), key_length);
+    rocksdb::Slice key_slice(reinterpret_cast<char*>(jk_val), len_key);
    keys.push_back(key_slice);
+
+    keys_to_free.push_back(std::make_tuple(jk_ba, jk_val, jk));
  }

  std::vector<std::string> values;
@@ -756,13 +667,23 @@ jobject multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db,
    s = db->MultiGet(rOpt, cf_handles, keys, &values);
  }

-  // Don't reuse class pointer
-  jclass jclazz = env->FindClass("java/util/ArrayList");
-  jmethodID mid = rocksdb::ListJni::getArrayListConstructorMethodId(
-      env, jclazz);
-  jobject jvalue_list = env->NewObject(jclazz, mid, jkeys_count);
+  // free up allocated byte arrays
+  for (std::vector<std::tuple<jbyteArray, jbyte*, jobject>>::size_type i = 0;
+      i < keys_to_free.size(); i++) {
+    jobject jk;
+    jbyteArray jk_ba;
+    jbyte* jk_val;
+    std::tie(jk_ba, jk_val, jk) = keys_to_free[i];
+    env->ReleaseByteArrayElements(jk_ba, jk_val, JNI_ABORT);
+    env->DeleteLocalRef(jk);
+  }

-  // insert in java list
+  // prepare the results
+  jclass jcls_ba = env->FindClass("[B");
+  jobjectArray jresults =
+      env->NewObjectArray(static_cast<jsize>(s.size()), jcls_ba, NULL);
+
+  // add to the jresults
  for (std::vector<rocksdb::Status>::size_type i = 0; i != s.size(); i++) {
    if (s[i].ok()) {
      jbyteArray jentry_value =
@@ -770,73 +691,60 @@ jobject multi_get_helper(JNIEnv* env, jobject jdb, rocksdb::DB* db,
      env->SetByteArrayRegion(
          jentry_value, 0, static_cast<jsize>(values[i].size()),
          reinterpret_cast<const jbyte*>(values[i].c_str()));
-      env->CallBooleanMethod(
-          jvalue_list, rocksdb::ListJni::getListAddMethodId(env),
-              jentry_value);
-    } else {
-      env->CallBooleanMethod(
-          jvalue_list, rocksdb::ListJni::getListAddMethodId(env), nullptr);
+      env->SetObjectArrayElement(jresults, static_cast<jsize>(i), jentry_value);
+      env->DeleteLocalRef(jentry_value);
    }
  }
-  // free up allocated byte arrays
-  for (std::vector<jbyte*>::size_type i = 0; i != keys_to_free.size(); i++) {
-    delete[] keys_to_free[i];
-  }
-  keys_to_free.clear();
-  return jvalue_list;
+
+  return jresults;
 }

 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    multiGet
- * Signature: (JLjava/util/List;I)Ljava/util/List;
+ * Signature: (J[[B)[[B
 */
-jobject Java_org_rocksdb_RocksDB_multiGet__JLjava_util_List_2I(
-    JNIEnv* env, jobject jdb, jlong jdb_handle,
-    jobject jkey_list, jint jkeys_count) {
+jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B(
+    JNIEnv* env, jobject jdb, jlong jdb_handle, jobjectArray jkeys) {
  return multi_get_helper(env, jdb, reinterpret_cast<rocksdb::DB*>(jdb_handle),
-      rocksdb::ReadOptions(), jkey_list, jkeys_count, nullptr);
+      rocksdb::ReadOptions(), jkeys, nullptr);
 }

 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    multiGet
- * Signature: (JLjava/util/List;ILjava/util/List;)Ljava/util/List;
+ * Signature: (J[[B[J)[[B
 */
-jobject
-    Java_org_rocksdb_RocksDB_multiGet__JLjava_util_List_2ILjava_util_List_2(
-    JNIEnv* env, jobject jdb, jlong jdb_handle,
-    jobject jkey_list, jint jkeys_count, jobject jcfhandle_list) {
+jobjectArray Java_org_rocksdb_RocksDB_multiGet__J_3_3B_3J(
+    JNIEnv* env, jobject jdb, jlong jdb_handle, jobjectArray jkeys,
+    jlongArray jcolumn_family_handles) {
  return multi_get_helper(env, jdb, reinterpret_cast<rocksdb::DB*>(jdb_handle),
-      rocksdb::ReadOptions(), jkey_list, jkeys_count, jcfhandle_list);
+      rocksdb::ReadOptions(), jkeys, jcolumn_family_handles);
 }

 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    multiGet
- * Signature: (JJLjava/util/List;I)Ljava/util/List;
+ * Signature: (JJ[[B)[[B
 */
-jobject Java_org_rocksdb_RocksDB_multiGet__JJLjava_util_List_2I(
-    JNIEnv* env, jobject jdb, jlong jdb_handle,
-    jlong jropt_handle, jobject jkey_list, jint jkeys_count) {
+jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B(
+    JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle,
+    jobjectArray jkeys) {
  return multi_get_helper(env, jdb, reinterpret_cast<rocksdb::DB*>(jdb_handle),
-      *reinterpret_cast<rocksdb::ReadOptions*>(jropt_handle), jkey_list,
-      jkeys_count, nullptr);
+      *reinterpret_cast<rocksdb::ReadOptions*>(jropt_handle), jkeys, nullptr);
 }

 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    multiGet
- * Signature: (JJLjava/util/List;ILjava/util/List;)Ljava/util/List;
+ * Signature: (JJ[[B[J)[[B
 */
-jobject
-    Java_org_rocksdb_RocksDB_multiGet__JJLjava_util_List_2ILjava_util_List_2(
-    JNIEnv* env, jobject jdb, jlong jdb_handle,
-    jlong jropt_handle, jobject jkey_list, jint jkeys_count,
-    jobject jcfhandle_list) {
+jobjectArray Java_org_rocksdb_RocksDB_multiGet__JJ_3_3B_3J(
+    JNIEnv* env, jobject jdb, jlong jdb_handle, jlong jropt_handle,
+    jobjectArray jkeys, jlongArray jcolumn_family_handles) {
  return multi_get_helper(env, jdb, reinterpret_cast<rocksdb::DB*>(jdb_handle),
-      *reinterpret_cast<rocksdb::ReadOptions*>(jropt_handle), jkey_list,
-      jkeys_count, jcfhandle_list);
+      *reinterpret_cast<rocksdb::ReadOptions*>(jropt_handle), jkeys,
+      jcolumn_family_handles);
 }

 /*
@@ -1204,47 +1112,42 @@ jlong Java_org_rocksdb_RocksDB_iteratorCF__JJJ(
 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    iterators
- * Signature: (JLjava/util/List;J)[J
+ * Signature: (J[JJ)[J
 */
 jlongArray Java_org_rocksdb_RocksDB_iterators(
-    JNIEnv* env, jobject jdb, jlong db_handle, jobject jcfhandle_list,
-    jlong jread_options_handle) {
-  auto db = reinterpret_cast<rocksdb::DB*>(db_handle);
+    JNIEnv* env, jobject jdb, jlong db_handle,
+    jlongArray jcolumn_family_handles, jlong jread_options_handle) {
+  auto* db = reinterpret_cast<rocksdb::DB*>(db_handle);
  auto& read_options = *reinterpret_cast<rocksdb::ReadOptions*>(
        jread_options_handle);
  std::vector<rocksdb::ColumnFamilyHandle*> cf_handles;
-  std::vector<rocksdb::Iterator*> iterators;
-
-  if (jcfhandle_list != nullptr) {
-    // get cf iterator
-    jobject cfIteratorObj = env->CallObjectMethod(
-        jcfhandle_list, rocksdb::ListJni::getIteratorMethod(env));
-
-    // iterate over keys and convert java byte array to slice
-    while (env->CallBooleanMethod(
-        cfIteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
-      jobject jobj = (jbyteArray) env->CallObjectMethod(
-          cfIteratorObj, rocksdb::ListJni::getNextMethod(env));
-      rocksdb::ColumnFamilyHandle* cfHandle =
-          rocksdb::ColumnFamilyHandleJni::getHandle(env, jobj);
-      cf_handles.push_back(cfHandle);
+  if (jcolumn_family_handles != nullptr) {
+    jsize len_cols = env->GetArrayLength(jcolumn_family_handles);
+    jlong* jcfh = env->GetLongArrayElements(jcolumn_family_handles, NULL);
+    for (int i = 0; i < len_cols; i++) {
+      auto* cf_handle =
+          reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcfh[i]);
+      cf_handles.push_back(cf_handle);
    }
+    env->ReleaseLongArrayElements(jcolumn_family_handles, jcfh, JNI_ABORT);
  }

+  std::vector<rocksdb::Iterator*> iterators;
  rocksdb::Status s = db->NewIterators(read_options,
      cf_handles, &iterators);
  if (s.ok()) {
    jlongArray jLongArray =
        env->NewLongArray(static_cast<jsize>(iterators.size()));
-    for (std::vector<rocksdb::Iterator*>::size_type i = 0; i < iterators.size();
-         i++) {
+    for (std::vector<rocksdb::Iterator*>::size_type i = 0;
+        i < iterators.size(); i++) {
      env->SetLongArrayRegion(jLongArray, static_cast<jsize>(i), 1,
                              reinterpret_cast<const jlong*>(&iterators[i]));
    }
    return jLongArray;
+  } else {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+    return NULL;
  }
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
-  return env->NewLongArray(0);
 }

 /*
@@ -1262,32 +1165,23 @@ jlong Java_org_rocksdb_RocksDB_getDefaultColumnFamily(
 /*
 * Class:     org_rocksdb_RocksDB
 * Method:    createColumnFamily
- * Signature: (JLorg/rocksdb/ColumnFamilyDescriptor;)J;
+ * Signature: (J[BJ)J
 */
 jlong Java_org_rocksdb_RocksDB_createColumnFamily(
    JNIEnv* env, jobject jdb, jlong jdb_handle,
-    jobject jcf_descriptor) {
+    jbyteArray jcolumn_name, jlong jcolumn_options) {
  rocksdb::ColumnFamilyHandle* handle;
  auto db_handle = reinterpret_cast<rocksdb::DB*>(jdb_handle);

-  // get ColumnFamilyName
-  jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
-      jcf_descriptor,
-      rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
-          env)));
-  // get CF Options
-  jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
-      rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
-          env));
-  rocksdb::ColumnFamilyOptions* cfOptions =
-      rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
+  jbyte* cfname = env->GetByteArrayElements(jcolumn_name, 0);
+  const int len = env->GetArrayLength(jcolumn_name);

-  jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
-  const int len = env->GetArrayLength(byteArray);
+  auto* cfOptions =
+      reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jcolumn_options);

  rocksdb::Status s = db_handle->CreateColumnFamily(
      *cfOptions, std::string(reinterpret_cast<char *>(cfname), len), &handle);
-  env->ReleaseByteArrayElements(byteArray, cfname, 0);
+  env->ReleaseByteArrayElements(jcolumn_name, cfname, 0);

  if (s.ok()) {
    return reinterpret_cast<jlong>(handle);
@@ -22,12 +22,11 @@
 /*
 * Class:     org_rocksdb_AbstractSlice
 * Method:    createNewSliceFromString
- * Signature: (Ljava/lang/String;)V
+ * Signature: (Ljava/lang/String;)J
 */
-void Java_org_rocksdb_AbstractSlice_createNewSliceFromString(
-    JNIEnv* env, jobject jobj, jstring jstr) {
-
-  const auto* str = env->GetStringUTFChars(jstr, 0);
+jlong Java_org_rocksdb_AbstractSlice_createNewSliceFromString(
+    JNIEnv * env, jclass jcls, jstring jstr) {
+  const auto* str = env->GetStringUTFChars(jstr, NULL);
  const size_t len = strlen(str);
  char* buf = new char[len + 1];
  memcpy(buf, str, len);
@@ -35,7 +34,7 @@ void Java_org_rocksdb_AbstractSlice_createNewSliceFromString(
  env->ReleaseStringUTFChars(jstr, str);

  const auto* slice = new rocksdb::Slice(buf);
-  rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
+  return reinterpret_cast<jlong>(slice);
 }

 /*
@@ -115,10 +114,10 @@ void Java_org_rocksdb_AbstractSlice_disposeInternal(
 /*
 * Class:     org_rocksdb_Slice
 * Method:    createNewSlice0
- * Signature: ([BI)V
+ * Signature: ([BI)J
 */
-void Java_org_rocksdb_Slice_createNewSlice0(
-    JNIEnv * env, jobject jobj, jbyteArray data, jint offset) {
+jlong Java_org_rocksdb_Slice_createNewSlice0(
+    JNIEnv * env, jclass jcls, jbyteArray data, jint offset) {

  const jsize dataSize = env->GetArrayLength(data);
  const int len = dataSize - offset;
@@ -126,32 +125,33 @@ void Java_org_rocksdb_Slice_createNewSlice0(
  env->GetByteArrayRegion(data, offset, len, ptrData);

  const auto* slice = new rocksdb::Slice((const char*)ptrData, len);
-  rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
+  return reinterpret_cast<jlong>(slice);
 }

 /*
 * Class:     org_rocksdb_Slice
 * Method:    createNewSlice1
- * Signature: ([B)V
+ * Signature: ([B)J
 */
-void Java_org_rocksdb_Slice_createNewSlice1(
-    JNIEnv * env, jobject jobj, jbyteArray data) {
+jlong Java_org_rocksdb_Slice_createNewSlice1(
+    JNIEnv * env, jclass jcls, jbyteArray data) {

  const int len = env->GetArrayLength(data) + 1;

  jboolean isCopy;
  jbyte* ptrData = env->GetByteArrayElements(data, &isCopy);
-  char* buf = new char[len];

+  // NOTE: buf will be deleted in the org.rocksdb.Slice#dispose method
+  char* buf = new char[len];
  memcpy(buf, ptrData, len - 1);
  buf[len-1]='\0';

  const auto* slice =
      new rocksdb::Slice(buf, len - 1);

-  rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
  env->ReleaseByteArrayElements(data, ptrData, JNI_ABORT);
-  // NOTE: buf will be deleted in the org.rocksdb.Slice#dispose method
+
+  return reinterpret_cast<jlong>(slice);
 }

 /*
@@ -187,27 +187,27 @@ void Java_org_rocksdb_Slice_disposeInternalBuf(
 /*
 * Class:     org_rocksdb_DirectSlice
 * Method:    createNewDirectSlice0
- * Signature: (Ljava/nio/ByteBuffer;I)V
+ * Signature: (Ljava/nio/ByteBuffer;I)J
 */
-void Java_org_rocksdb_DirectSlice_createNewDirectSlice0(
-    JNIEnv* env, jobject jobj, jobject data, jint length) {
+jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice0(
+    JNIEnv* env, jclass jcls, jobject data, jint length) {
  const auto* ptrData =
     reinterpret_cast<char*>(env->GetDirectBufferAddress(data));
  const auto* slice = new rocksdb::Slice(ptrData, length);
-  rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
+  return reinterpret_cast<jlong>(slice);
 }

 /*
 * Class:     org_rocksdb_DirectSlice
 * Method:    createNewDirectSlice1
- * Signature: (Ljava/nio/ByteBuffer;)V
+ * Signature: (Ljava/nio/ByteBuffer;)J
 */
-void Java_org_rocksdb_DirectSlice_createNewDirectSlice1(
-    JNIEnv* env, jobject jobj, jobject data) {
+jlong Java_org_rocksdb_DirectSlice_createNewDirectSlice1(
+    JNIEnv* env, jclass jcls, jobject data) {
  const auto* ptrData =
    reinterpret_cast<char*>(env->GetDirectBufferAddress(data));
  const auto* slice = new rocksdb::Slice(ptrData);
-  rocksdb::AbstractSliceJni::setHandle(env, jobj, slice);
+  return reinterpret_cast<jlong>(slice);
 }

 /*
@@ -38,13 +38,14 @@ jlong Java_org_rocksdb_PlainTableConfig_newTableFactoryHandle(
 /*
 * Class:     org_rocksdb_BlockBasedTableConfig
 * Method:    newTableFactoryHandle
- * Signature: (ZJIJIIZIZZJIBBI)J
+ * Signature: (ZJIJIIZIZZZJIBBI)J
 */
 jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
    JNIEnv* env, jobject jobj, jboolean no_block_cache, jlong block_cache_size,
    jint block_cache_num_shardbits, jlong block_size, jint block_size_deviation,
    jint block_restart_interval, jboolean whole_key_filtering,
    jlong jfilterPolicy, jboolean cache_index_and_filter_blocks,
+    jboolean pin_l0_filter_and_index_blocks_in_cache,
    jboolean hash_index_allow_collision, jlong block_cache_compressed_size,
    jint block_cache_compressd_num_shard_bits, jbyte jchecksum_type,
    jbyte jindex_type, jint jformat_version) {
@@ -70,6 +71,8 @@ jlong Java_org_rocksdb_BlockBasedTableConfig_newTableFactoryHandle(
    options.filter_policy = *pFilterPolicy;
  }
  options.cache_index_and_filter_blocks = cache_index_and_filter_blocks;
+  options.pin_l0_filter_and_index_blocks_in_cache =
+      pin_l0_filter_and_index_blocks_in_cache;
  options.hash_index_allow_collision = hash_index_allow_collision;
  if (block_cache_compressed_size > 0) {
    if (block_cache_compressd_num_shard_bits > 0) {
@@ -20,10 +20,10 @@
 /*
 * Class:     org_rocksdb_TtlDB
 * Method:    open
- * Signature: (JLjava/lang/String;IZ)V
+ * Signature: (JLjava/lang/String;IZ)J
 */
-void Java_org_rocksdb_TtlDB_open(JNIEnv* env,
-    jobject jttldb, jlong joptions_handle, jstring jdb_path,
+jlong Java_org_rocksdb_TtlDB_open(JNIEnv* env,
+    jclass jcls, jlong joptions_handle, jstring jdb_path,
    jint jttl, jboolean jread_only) {
  auto* opt = reinterpret_cast<rocksdb::Options*>(joptions_handle);
  rocksdb::DBWithTTL* db = nullptr;
@@ -35,145 +35,102 @@ void Java_org_rocksdb_TtlDB_open(JNIEnv* env,
  // as TTLDB extends RocksDB on the java side, we can reuse
  // the RocksDB portal here.
  if (s.ok()) {
-      rocksdb::RocksDBJni::setHandle(env, jttldb, db);
-      return;
+    return reinterpret_cast<jlong>(db);
+  } else {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+    return 0;
  }
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
 }

 /*
 * Class:     org_rocksdb_TtlDB
 * Method:    openCF
- * Signature: (JLjava/lang/String;Ljava/util/List;
- *    ILjava/util/List;Z)Ljava/util/List;
+ * Signature: (JLjava/lang/String;[[B[J[IZ)[J
 */
-jobject
+jlongArray
    Java_org_rocksdb_TtlDB_openCF(
-    JNIEnv* env, jobject jdb, jlong jopt_handle, jstring jdb_path,
-    jobject jcfdesc_list, jint jcfdesc_count, jobject jttl_list,
-    jboolean jread_only) {
-  auto* opt = reinterpret_cast<rocksdb::Options*>(jopt_handle);
-  rocksdb::DBWithTTL* db = nullptr;
-  const char* db_path = env->GetStringUTFChars(jdb_path, 0);
-
-  std::vector<jbyte*> cfnames_to_free;
-  std::vector<jbyteArray> jcfnames_for_free;
+    JNIEnv* env, jclass jcls, jlong jopt_handle, jstring jdb_path,
+    jobjectArray jcolumn_names, jlongArray jcolumn_options,
+    jintArray jttls, jboolean jread_only) {
+  auto* opt = reinterpret_cast<rocksdb::DBOptions*>(jopt_handle);
+  const char* db_path = env->GetStringUTFChars(jdb_path, NULL);

  std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
+
+  jsize len_cols = env->GetArrayLength(jcolumn_names);
+  jlong* jco = env->GetLongArrayElements(jcolumn_options, NULL);
+  for(int i = 0; i < len_cols; i++) {
+    jobject jcn = env->GetObjectArrayElement(jcolumn_names, i);
+    jbyteArray jcn_ba = reinterpret_cast<jbyteArray>(jcn);
+    jbyte* jcf_name = env->GetByteArrayElements(jcn_ba, NULL);
+    const int jcf_name_len = env->GetArrayLength(jcn_ba);
+
+    //TODO(AR) do I need to make a copy of jco[i] ?
+
+    std::string cf_name (reinterpret_cast<char *>(jcf_name), jcf_name_len);
+    rocksdb::ColumnFamilyOptions* cf_options =
+      reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jco[i]);
+    column_families.push_back(
+      rocksdb::ColumnFamilyDescriptor(cf_name, *cf_options));
+
+    env->ReleaseByteArrayElements(jcn_ba, jcf_name, JNI_ABORT);
+    env->DeleteLocalRef(jcn);
+  }
+  env->ReleaseLongArrayElements(jcolumn_options, jco, JNI_ABORT);
+
+  std::vector<rocksdb::ColumnFamilyHandle*> handles;
+  rocksdb::DBWithTTL* db = nullptr;
+
  std::vector<int32_t> ttl_values;
-  std::vector<rocksdb::ColumnFamilyHandle* > handles;
-  // get iterator for ColumnFamilyDescriptors
-  jobject iteratorObj = env->CallObjectMethod(
-      jcfdesc_list, rocksdb::ListJni::getIteratorMethod(env));
-
-  // iterate over ColumnFamilyDescriptors
-  while (env->CallBooleanMethod(
-      iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
-      // get ColumnFamilyDescriptor
-      jobject jcf_descriptor = env->CallObjectMethod(iteratorObj,
-          rocksdb::ListJni::getNextMethod(env));
-      // get ColumnFamilyName
-      jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
-          jcf_descriptor,
-          rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
-              env)));
-      // get CF Options
-      jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
-          rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
-              env));
-      rocksdb::ColumnFamilyOptions* cfOptions =
-          rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
-
-      jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
-      const int len = env->GetArrayLength(byteArray);
-
-      // free allocated cfnames after call to open
-      cfnames_to_free.push_back(cfname);
-      jcfnames_for_free.push_back(byteArray);
-      column_families.push_back(rocksdb::ColumnFamilyDescriptor(
-          std::string(reinterpret_cast<char *>(cfname), len), *cfOptions));
-  }
-  // get iterator for TTL values
-  iteratorObj = env->CallObjectMethod(
-        jttl_list, rocksdb::ListJni::getIteratorMethod(env));
-  // iterate over TTL values
-  while (env->CallBooleanMethod(
-      iteratorObj, rocksdb::ListJni::getHasNextMethod(env)) == JNI_TRUE) {
-     // get TTL object
-     jobject jttl_object = env->CallObjectMethod(iteratorObj,
-       rocksdb::ListJni::getNextMethod(env));
-     // get Integer value
-     jclass jIntClazz = env->FindClass("java/lang/Integer");
-     jmethodID getVal = env->GetMethodID(jIntClazz, "intValue", "()I");
-     ttl_values.push_back(env->CallIntMethod(jttl_object, getVal));
+  jint* jttlv = env->GetIntArrayElements(jttls, NULL);
+  jsize len_ttls = env->GetArrayLength(jttls);
+  for(int i = 0; i < len_ttls; i++) {
+    ttl_values.push_back(jttlv[i]);
  }
+  env->ReleaseIntArrayElements(jttls, jttlv, JNI_ABORT);
+
  rocksdb::Status s = rocksdb::DBWithTTL::Open(*opt, db_path, column_families,
      &handles, &db, ttl_values, jread_only);

-  env->ReleaseStringUTFChars(jdb_path, db_path);
-  // free jbyte allocations
-  for (std::vector<jbyte*>::size_type i = 0;
-      i != cfnames_to_free.size(); i++) {
-    // free  cfnames
-    env->ReleaseByteArrayElements(jcfnames_for_free[i], cfnames_to_free[i], 0);
-  }
-
  // check if open operation was successful
  if (s.ok()) {
-    rocksdb::RocksDBJni::setHandle(env, jdb, db);
-    jclass jListClazz = env->FindClass("java/util/ArrayList");
-    jmethodID midList = rocksdb::ListJni::getArrayListConstructorMethodId(
-        env, jListClazz);
-    jobject jcfhandle_list = env->NewObject(jListClazz,
-        midList, handles.size());
-    // insert in java list
-    for (std::vector<rocksdb::ColumnFamilyHandle*>::size_type i = 0;
-        i != handles.size(); i++) {
-      // jlong must be converted to Long due to collections restrictions
-      jclass jLongClazz = env->FindClass("java/lang/Long");
-      jmethodID midLong = env->GetMethodID(jLongClazz, "<init>", "(J)V");
-      jobject obj = env->NewObject(jLongClazz, midLong,
-          reinterpret_cast<jlong>(handles[i]));
-      env->CallBooleanMethod(jcfhandle_list,
-          rocksdb::ListJni::getListAddMethodId(env), obj);
+    jsize resultsLen = 1 + len_cols; //db handle + column family handles
+    jlong results[resultsLen];
+    results[0] = reinterpret_cast<jlong>(db);
+    for(int i = 1; i <= len_cols; i++) {
+      results[i] = reinterpret_cast<jlong>(handles[i - 1]);
    }

-    return jcfhandle_list;
+    jlongArray jresults = env->NewLongArray(resultsLen);
+    env->SetLongArrayRegion(jresults, 0, resultsLen, results);
+    return jresults;
+  } else {
+    rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
+    return NULL;
  }
-  rocksdb::RocksDBExceptionJni::ThrowNew(env, s);
-  return nullptr;
 }

 /*
 * Class:     org_rocksdb_TtlDB
 * Method:    createColumnFamilyWithTtl
- * Signature: (JLorg/rocksdb/ColumnFamilyDescriptor;I)J;
+ * Signature: (JLorg/rocksdb/ColumnFamilyDescriptor;[BJI)J;
 */
 jlong Java_org_rocksdb_TtlDB_createColumnFamilyWithTtl(
    JNIEnv* env, jobject jobj, jlong jdb_handle,
-    jobject jcf_descriptor, jint jttl) {
+    jbyteArray jcolumn_name, jlong jcolumn_options, jint jttl) {
  rocksdb::ColumnFamilyHandle* handle;
  auto* db_handle = reinterpret_cast<rocksdb::DBWithTTL*>(jdb_handle);

-  // get ColumnFamilyName
-  jbyteArray byteArray = static_cast<jbyteArray>(env->CallObjectMethod(
-      jcf_descriptor,
-      rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyNameMethod(
-          env)));
-  // get CF Options
-  jobject jcf_opt_obj = env->CallObjectMethod(jcf_descriptor,
-      rocksdb::ColumnFamilyDescriptorJni::getColumnFamilyOptionsMethod(
-      env));
-  rocksdb::ColumnFamilyOptions* cfOptions =
-      rocksdb::ColumnFamilyOptionsJni::getHandle(env, jcf_opt_obj);
+  jbyte* cfname = env->GetByteArrayElements(jcolumn_name, 0);
+  const int len = env->GetArrayLength(jcolumn_name);

-  jbyte* cfname = env->GetByteArrayElements(byteArray, 0);
-  const int len = env->GetArrayLength(byteArray);
+  auto* cfOptions =
+      reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jcolumn_options);

  rocksdb::Status s = db_handle->CreateColumnFamilyWithTtl(
      *cfOptions, std::string(reinterpret_cast<char *>(cfname),
          len), &handle, jttl);
-  env->ReleaseByteArrayElements(byteArray, cfname, 0);
+  env->ReleaseByteArrayElements(jcolumn_name, cfname, 0);

  if (s.ok()) {
    return reinterpret_cast<jlong>(handle);
@@ -27,23 +27,23 @@
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    newWriteBatch
- * Signature: (I)V
+ * Signature: (I)J
 */
-void Java_org_rocksdb_WriteBatch_newWriteBatch(
-    JNIEnv* env, jobject jobj, jint jreserved_bytes) {
+jlong Java_org_rocksdb_WriteBatch_newWriteBatch(
+    JNIEnv* env, jclass jcls, jint jreserved_bytes) {
  rocksdb::WriteBatch* wb = new rocksdb::WriteBatch(
      static_cast<size_t>(jreserved_bytes));
-
-  rocksdb::WriteBatchJni::setHandle(env, jobj, wb);
+  return reinterpret_cast<jlong>(wb);
 }

 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    count0
- * Signature: ()I
+ * Signature: (J)I
 */
-jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* env, jobject jobj) {
-  rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* env, jobject jobj,
+    jlong jwb_handle) {
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);

  return static_cast<jint>(wb->Count());
@@ -52,10 +52,11 @@ jint Java_org_rocksdb_WriteBatch_count0(JNIEnv* env, jobject jobj) {
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    clear0
- * Signature: ()V
+ * Signature: (J)V
 */
-void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* env, jobject jobj) {
-  rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* env, jobject jobj,
+    jlong jwb_handle) {
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);

  wb->Clear();
@@ -64,13 +65,13 @@ void Java_org_rocksdb_WriteBatch_clear0(JNIEnv* env, jobject jobj) {
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    put
- * Signature: ([BI[BI)V
+ * Signature: (J[BI[BI)V
 */
-void Java_org_rocksdb_WriteBatch_put___3BI_3BI(
-    JNIEnv* env, jobject jobj,
+void Java_org_rocksdb_WriteBatch_put__J_3BI_3BI(
+    JNIEnv* env, jobject jobj, jlong jwb_handle,
    jbyteArray jkey, jint jkey_len,
    jbyteArray jentry_value, jint jentry_value_len) {
-  auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);
  auto put = [&wb] (rocksdb::Slice key, rocksdb::Slice value) {
    wb->Put(key, value);
@@ -82,13 +83,13 @@ void Java_org_rocksdb_WriteBatch_put___3BI_3BI(
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    put
- * Signature: ([BI[BIJ)V
+ * Signature: (J[BI[BIJ)V
 */
-void Java_org_rocksdb_WriteBatch_put___3BI_3BIJ(
-    JNIEnv* env, jobject jobj,
+void Java_org_rocksdb_WriteBatch_put__J_3BI_3BIJ(
+    JNIEnv* env, jobject jobj, jlong jwb_handle,
    jbyteArray jkey, jint jkey_len,
    jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) {
-  auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);
  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
  assert(cf_handle != nullptr);
@@ -102,13 +103,13 @@ void Java_org_rocksdb_WriteBatch_put___3BI_3BIJ(
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    merge
- * Signature: ([BI[BI)V
+ * Signature: (J[BI[BI)V
 */
-void Java_org_rocksdb_WriteBatch_merge___3BI_3BI(
-    JNIEnv* env, jobject jobj,
+void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BI(
+    JNIEnv* env, jobject jobj, jlong jwb_handle,
    jbyteArray jkey, jint jkey_len,
    jbyteArray jentry_value, jint jentry_value_len) {
-  auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);
  auto merge = [&wb] (rocksdb::Slice key, rocksdb::Slice value) {
    wb->Merge(key, value);
@@ -120,13 +121,13 @@ void Java_org_rocksdb_WriteBatch_merge___3BI_3BI(
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    merge
- * Signature: ([BI[BIJ)V
+ * Signature: (J[BI[BIJ)V
 */
-void Java_org_rocksdb_WriteBatch_merge___3BI_3BIJ(
-    JNIEnv* env, jobject jobj,
+void Java_org_rocksdb_WriteBatch_merge__J_3BI_3BIJ(
+    JNIEnv* env, jobject jobj, jlong jwb_handle,
    jbyteArray jkey, jint jkey_len,
    jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) {
-  auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);
  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
  assert(cf_handle != nullptr);
@@ -140,12 +141,12 @@ void Java_org_rocksdb_WriteBatch_merge___3BI_3BIJ(
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    remove
- * Signature: ([BI)V
+ * Signature: (J[BI)V
 */
-void Java_org_rocksdb_WriteBatch_remove___3BI(
-    JNIEnv* env, jobject jobj,
+void Java_org_rocksdb_WriteBatch_remove__J_3BI(
+    JNIEnv* env, jobject jobj, jlong jwb_handle,
    jbyteArray jkey, jint jkey_len) {
-  auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);
  auto remove = [&wb] (rocksdb::Slice key) {
    wb->Delete(key);
@@ -156,12 +157,12 @@ void Java_org_rocksdb_WriteBatch_remove___3BI(
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    remove
- * Signature: ([BIJ)V
+ * Signature: (J[BIJ)V
 */
-void Java_org_rocksdb_WriteBatch_remove___3BIJ(
-    JNIEnv* env, jobject jobj,
+void Java_org_rocksdb_WriteBatch_remove__J_3BIJ(
+    JNIEnv* env, jobject jobj, jlong jwb_handle,
    jbyteArray jkey, jint jkey_len, jlong jcf_handle) {
-  auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);
  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
  assert(cf_handle != nullptr);
@@ -174,11 +175,12 @@ void Java_org_rocksdb_WriteBatch_remove___3BIJ(
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    putLogData
- * Signature: ([BI)V
+ * Signature: (J[BI)V
 */
 void Java_org_rocksdb_WriteBatch_putLogData(
-    JNIEnv* env, jobject jobj, jbyteArray jblob, jint jblob_len) {
-  auto* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+    JNIEnv* env, jobject jobj, jlong jwb_handle, jbyteArray jblob,
+    jint jblob_len) {
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);
  auto putLogData = [&wb] (rocksdb::Slice blob) {
    wb->PutLogData(blob);
@@ -189,11 +191,11 @@ void Java_org_rocksdb_WriteBatch_putLogData(
 /*
 * Class:     org_rocksdb_WriteBatch
 * Method:    iterate
- * Signature: (J)V
+ * Signature: (JJ)V
 */
 void Java_org_rocksdb_WriteBatch_iterate(
-    JNIEnv* env , jobject jobj, jlong handlerHandle) {
-  rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+    JNIEnv* env , jobject jobj, jlong jwb_handle, jlong handlerHandle) {
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);

  rocksdb::Status s = wb->Iterate(
@@ -218,13 +220,13 @@ void Java_org_rocksdb_WriteBatch_disposeInternal(
 /*
 * Class:     org_rocksdb_WriteBatch_Handler
 * Method:    createNewHandler0
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_WriteBatch_00024Handler_createNewHandler0(
+jlong Java_org_rocksdb_WriteBatch_00024Handler_createNewHandler0(
    JNIEnv* env, jobject jobj) {
  const rocksdb::WriteBatchHandlerJniCallback* h =
    new rocksdb::WriteBatchHandlerJniCallback(env, jobj);
-  rocksdb::WriteBatchHandlerJni::setHandle(env, jobj, h);
+  return reinterpret_cast<jlong>(h);
 }

 /*
@@ -28,11 +28,11 @@
 /*
 * Class:     org_rocksdb_WriteBatchTest
 * Method:    getContents
- * Signature: (Lorg/rocksdb/WriteBatch;)[B
+ * Signature: (J)[B
 */
 jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(
-    JNIEnv* env, jclass jclazz, jobject jobj) {
-  rocksdb::WriteBatch* b = rocksdb::WriteBatchJni::getHandle(env, jobj);
+    JNIEnv* env, jclass jclazz, jlong jwb_handle) {
+  auto* b = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(b != nullptr);

  // todo: Currently the following code is directly copied from
@@ -109,11 +109,11 @@ jbyteArray Java_org_rocksdb_WriteBatchTest_getContents(
 /*
 * Class:     org_rocksdb_WriteBatchTestInternalHelper
 * Method:    setSequence
- * Signature: (Lorg/rocksdb/WriteBatch;J)V
+ * Signature: (JJ)V
 */
 void Java_org_rocksdb_WriteBatchTestInternalHelper_setSequence(
-    JNIEnv* env, jclass jclazz, jobject jobj, jlong jsn) {
-  rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+    JNIEnv* env, jclass jclazz, jlong jwb_handle, jlong jsn) {
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);

  rocksdb::WriteBatchInternal::SetSequence(
@@ -123,11 +123,11 @@ void Java_org_rocksdb_WriteBatchTestInternalHelper_setSequence(
 /*
 * Class:     org_rocksdb_WriteBatchTestInternalHelper
 * Method:    sequence
- * Signature: (Lorg/rocksdb/WriteBatch;)J
+ * Signature: (J)J
 */
 jlong Java_org_rocksdb_WriteBatchTestInternalHelper_sequence(
-    JNIEnv* env, jclass jclazz, jobject jobj) {
-  rocksdb::WriteBatch* wb = rocksdb::WriteBatchJni::getHandle(env, jobj);
+    JNIEnv* env, jclass jclazz, jlong jwb_handle) {
+  auto* wb = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle);
  assert(wb != nullptr);

  return static_cast<jlong>(rocksdb::WriteBatchInternal::Sequence(wb));
@@ -136,13 +136,13 @@ jlong Java_org_rocksdb_WriteBatchTestInternalHelper_sequence(
 /*
 * Class:     org_rocksdb_WriteBatchTestInternalHelper
 * Method:    append
- * Signature: (Lorg/rocksdb/WriteBatch;Lorg/rocksdb/WriteBatch;)V
+ * Signature: (JJ)V
 */
 void Java_org_rocksdb_WriteBatchTestInternalHelper_append(
-    JNIEnv* env, jclass jclazz, jobject jwb1, jobject jwb2) {
-  rocksdb::WriteBatch* wb1 = rocksdb::WriteBatchJni::getHandle(env, jwb1);
+    JNIEnv* env, jclass jclazz, jlong jwb_handle_1, jlong jwb_handle_2) {
+  auto* wb1 = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle_1);
  assert(wb1 != nullptr);
-  rocksdb::WriteBatch* wb2 = rocksdb::WriteBatchJni::getHandle(env, jwb2);
+  auto* wb2 = reinterpret_cast<rocksdb::WriteBatch*>(jwb_handle_2);
  assert(wb2 != nullptr);

  rocksdb::WriteBatchInternal::Append(wb1, wb2);
@@ -15,51 +15,50 @@
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    newWriteBatchWithIndex
- * Signature: ()V
+ * Signature: ()J
 */
-void Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__(
-    JNIEnv* env, jobject jobj) {
+jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__(
+    JNIEnv* env, jclass jcls) {
  rocksdb::WriteBatchWithIndex* wbwi = new rocksdb::WriteBatchWithIndex();
-  rocksdb::WriteBatchWithIndexJni::setHandle(env, jobj, wbwi);
+  return reinterpret_cast<jlong>(wbwi);
 }

 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    newWriteBatchWithIndex
- * Signature: (Z)V
+ * Signature: (Z)J
 */
-void Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z(
-    JNIEnv* env, jobject jobj, jboolean joverwrite_key) {
+jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__Z(
+    JNIEnv* env, jclass jcls, jboolean joverwrite_key) {
  rocksdb::WriteBatchWithIndex* wbwi =
      new rocksdb::WriteBatchWithIndex(rocksdb::BytewiseComparator(), 0,
      static_cast<bool>(joverwrite_key));
-  rocksdb::WriteBatchWithIndexJni::setHandle(env, jobj, wbwi);
+  return reinterpret_cast<jlong>(wbwi);
 }

 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    newWriteBatchWithIndex
- * Signature: (JIZ)V
+ * Signature: (JIZ)J
 */
-void Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__JIZ(
-    JNIEnv* env, jobject jobj, jlong jfallback_index_comparator_handle,
+jlong Java_org_rocksdb_WriteBatchWithIndex_newWriteBatchWithIndex__JIZ(
+    JNIEnv* env, jclass jcls, jlong jfallback_index_comparator_handle,
    jint jreserved_bytes, jboolean joverwrite_key) {
  rocksdb::WriteBatchWithIndex* wbwi =
      new rocksdb::WriteBatchWithIndex(
      reinterpret_cast<rocksdb::Comparator*>(jfallback_index_comparator_handle),
      static_cast<size_t>(jreserved_bytes), static_cast<bool>(joverwrite_key));
-  rocksdb::WriteBatchWithIndexJni::setHandle(env, jobj, wbwi);
+  return reinterpret_cast<jlong>(wbwi);
 }

 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
- * Method:    count
- * Signature: ()I
+ * Method:    count0
+ * Signature: (J)I
 */
 jint Java_org_rocksdb_WriteBatchWithIndex_count0(
-    JNIEnv* env, jobject jobj) {
-  rocksdb::WriteBatchWithIndex* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);

  return static_cast<jint>(wbwi->GetWriteBatch()->Count());
@@ -68,13 +67,12 @@ jint Java_org_rocksdb_WriteBatchWithIndex_count0(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    put
- * Signature: ([BI[BI)V
+ * Signature: (J[BI[BI)V
 */
-void Java_org_rocksdb_WriteBatchWithIndex_put___3BI_3BI(
-    JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len,
-    jbyteArray jentry_value, jint jentry_value_len) {
-  auto* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BI(
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
+    jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);
  auto put = [&wbwi] (rocksdb::Slice key, rocksdb::Slice value) {
    wbwi->Put(key, value);
@@ -86,13 +84,13 @@ void Java_org_rocksdb_WriteBatchWithIndex_put___3BI_3BI(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    put
- * Signature: ([BI[BIJ)V
+ * Signature: (J[BI[BIJ)V
 */
-void Java_org_rocksdb_WriteBatchWithIndex_put___3BI_3BIJ(
-    JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len,
-    jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) {
-  auto* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+void Java_org_rocksdb_WriteBatchWithIndex_put__J_3BI_3BIJ(
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
+    jint jkey_len, jbyteArray jentry_value, jint jentry_value_len,
+    jlong jcf_handle) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);
  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
  assert(cf_handle != nullptr);
@@ -106,13 +104,12 @@ void Java_org_rocksdb_WriteBatchWithIndex_put___3BI_3BIJ(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    merge
- * Signature: ([BI[BI)V
+ * Signature: (J[BI[BI)V
 */
-void Java_org_rocksdb_WriteBatchWithIndex_merge___3BI_3BI(
-    JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len,
-    jbyteArray jentry_value, jint jentry_value_len) {
-  auto* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BI(
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
+    jint jkey_len, jbyteArray jentry_value, jint jentry_value_len) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);
  auto merge = [&wbwi] (rocksdb::Slice key, rocksdb::Slice value) {
    wbwi->Merge(key, value);
@@ -124,13 +121,13 @@ void Java_org_rocksdb_WriteBatchWithIndex_merge___3BI_3BI(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    merge
- * Signature: ([BI[BIJ)V
+ * Signature: (J[BI[BIJ)V
 */
-void Java_org_rocksdb_WriteBatchWithIndex_merge___3BI_3BIJ(
-    JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len,
-    jbyteArray jentry_value, jint jentry_value_len, jlong jcf_handle) {
-  auto* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+void Java_org_rocksdb_WriteBatchWithIndex_merge__J_3BI_3BIJ(
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
+    jint jkey_len, jbyteArray jentry_value, jint jentry_value_len,
+    jlong jcf_handle) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);
  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
  assert(cf_handle != nullptr);
@@ -144,12 +141,12 @@ void Java_org_rocksdb_WriteBatchWithIndex_merge___3BI_3BIJ(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    remove
- * Signature: ([BI)V
+ * Signature: (J[BI)V
 */
-void Java_org_rocksdb_WriteBatchWithIndex_remove___3BI(
-    JNIEnv* env, jobject jobj, jbyteArray jkey, jint jkey_len) {
-  auto* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+void Java_org_rocksdb_WriteBatchWithIndex_remove__J_3BI(
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
+    jint jkey_len) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);
  auto remove = [&wbwi] (rocksdb::Slice key) {
    wbwi->Delete(key);
@@ -160,13 +157,12 @@ void Java_org_rocksdb_WriteBatchWithIndex_remove___3BI(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    remove
- * Signature: ([BIJ)V
+ * Signature: (J[BIJ)V
 */
-void Java_org_rocksdb_WriteBatchWithIndex_remove___3BIJ(
-    JNIEnv* env, jobject jobj,
-    jbyteArray jkey, jint jkey_len, jlong jcf_handle) {
-  auto* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+void Java_org_rocksdb_WriteBatchWithIndex_remove__J_3BIJ(
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jkey,
+    jint jkey_len, jlong jcf_handle) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);
  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
  assert(cf_handle != nullptr);
@@ -179,12 +175,12 @@ void Java_org_rocksdb_WriteBatchWithIndex_remove___3BIJ(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    putLogData
- * Signature: ([BI)V
+ * Signature: (J[BI)V
 */
 void Java_org_rocksdb_WriteBatchWithIndex_putLogData(
-    JNIEnv* env, jobject jobj, jbyteArray jblob, jint jblob_len) {
-  auto* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jbyteArray jblob,
+    jint jblob_len) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);
  auto putLogData = [&wbwi] (rocksdb::Slice blob) {
    wbwi->PutLogData(blob);
@@ -195,12 +191,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_putLogData(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    clear
- * Signature: ()V
+ * Signature: (J)V
 */
 void Java_org_rocksdb_WriteBatchWithIndex_clear0(
-    JNIEnv* env, jobject jobj) {
-  rocksdb::WriteBatchWithIndex* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  assert(wbwi != nullptr);

  wbwi->GetWriteBatch()->Clear();
@@ -209,12 +204,11 @@ void Java_org_rocksdb_WriteBatchWithIndex_clear0(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    iterator0
- * Signature: ()J
+ * Signature: (J)J
 */
 jlong Java_org_rocksdb_WriteBatchWithIndex_iterator0(
-    JNIEnv* env, jobject jobj) {
-  rocksdb::WriteBatchWithIndex* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  rocksdb::WBWIIterator* wbwi_iterator = wbwi->NewIterator();
  return reinterpret_cast<jlong>(wbwi_iterator);
 }
@@ -222,12 +216,11 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_iterator0(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    iterator1
- * Signature: (J)J
+ * Signature: (JJ)J
 */
 jlong Java_org_rocksdb_WriteBatchWithIndex_iterator1(
-    JNIEnv* env, jobject jobj, jlong jcf_handle) {
-  rocksdb::WriteBatchWithIndex* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jlong jcf_handle) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
  rocksdb::WBWIIterator* wbwi_iterator = wbwi->NewIterator(cf_handle);
  return reinterpret_cast<jlong>(wbwi_iterator);
@@ -236,12 +229,12 @@ jlong Java_org_rocksdb_WriteBatchWithIndex_iterator1(
 /*
 * Class:     org_rocksdb_WriteBatchWithIndex
 * Method:    iteratorWithBase
- * Signature: (JJ)J
+ * Signature: (JJJ)J
 */
 jlong Java_org_rocksdb_WriteBatchWithIndex_iteratorWithBase(
-    JNIEnv* env, jobject jobj, jlong jcf_handle, jlong jbi_handle) {
-  rocksdb::WriteBatchWithIndex* wbwi =
-      rocksdb::WriteBatchWithIndexJni::getHandle(env, jobj);
+    JNIEnv* env, jobject jobj, jlong jwbwi_handle, jlong jcf_handle,
+    jlong jbi_handle) {
+  auto* wbwi = reinterpret_cast<rocksdb::WriteBatchWithIndex*>(jwbwi_handle);
  auto* cf_handle = reinterpret_cast<rocksdb::ColumnFamilyHandle*>(jcf_handle);
  auto* base_iterator = reinterpret_cast<rocksdb::Iterator*>(jbi_handle);
  auto* iterator = wbwi->NewIteratorWithBase(cf_handle, base_iterator);
@@ -360,27 +353,57 @@ void Java_org_rocksdb_WBWIRocksIterator_status0(
 /*
 * Class:     org_rocksdb_WBWIRocksIterator
 * Method:    entry1
- * Signature: (JLorg/rocksdb/WBWIRocksIterator/WriteEntry;)V
+ * Signature: (J)[J
 */
-void Java_org_rocksdb_WBWIRocksIterator_entry1(
-    JNIEnv* env, jobject jobj, jlong handle, jobject jwrite_entry) {
+jlongArray Java_org_rocksdb_WBWIRocksIterator_entry1(
+    JNIEnv* env, jobject jobj, jlong handle) {
  auto* it = reinterpret_cast<rocksdb::WBWIIterator*>(handle);
  const rocksdb::WriteEntry& we = it->Entry();
-  jobject jwe = rocksdb::WBWIRocksIteratorJni::getWriteEntry(env, jobj);
-  rocksdb::WriteEntryJni::setWriteType(env, jwe, we.type);

+  jlong results[3];
+
+  //set the type of the write entry
+  switch (we.type) {
+    case rocksdb::kPutRecord:
+      results[0] = 0x1;
+      break;
+
+    case rocksdb::kMergeRecord:
+      results[0] = 0x2;
+      break;
+
+    case rocksdb::kDeleteRecord:
+      results[0] = 0x4;
+      break;
+
+    case rocksdb::kLogDataRecord:
+      results[0] = 0x8;
+      break;
+
+    default:
+      results[0] = 0x0;
+  }
+
+  //TODO(AR) do we leak buf and value_buf?
+
+  //set the pointer to the key slice
  char* buf = new char[we.key.size()];
  memcpy(buf, we.key.data(), we.key.size());
  auto* key_slice = new rocksdb::Slice(buf, we.key.size());
-  rocksdb::WriteEntryJni::setKey(env, jwe, key_slice);
+  results[1] = reinterpret_cast<jlong>(key_slice);

+  //set the pointer to the value slice
  if (we.type == rocksdb::kDeleteRecord || we.type == rocksdb::kLogDataRecord) {
    // set native handle of value slice to null if no value available
-    rocksdb::WriteEntryJni::setValue(env, jwe, nullptr);
+    results[2] = 0;
  } else {
    char* value_buf = new char[we.value.size()];
    memcpy(value_buf, we.value.data(), we.value.size());
    auto* value_slice = new rocksdb::Slice(value_buf, we.value.size());
-    rocksdb::WriteEntryJni::setValue(env, jwe, value_slice);
+    results[2] = reinterpret_cast<jlong>(value_slice);
  }
+
+  jlongArray jresults = env->NewLongArray(3);
+  env->SetLongArrayRegion(jresults, 0, 3, results);
+  return jresults;
 }
@@ -22,73 +22,57 @@ public class RocksDBColumnFamilySample {
    String db_path = args[0];

    System.out.println("RocksDBColumnFamilySample");
-    RocksDB db = null;
-    Options options = null;
-    ColumnFamilyHandle columnFamilyHandle = null;
-    WriteBatch wb = null;
-    try {
-      options = new Options().setCreateIfMissing(true);
-      db = RocksDB.open(options, db_path);
+    try(final Options options = new Options().setCreateIfMissing(true);
+        final RocksDB db = RocksDB.open(options, db_path)) {
+
      assert(db != null);

      // create column family
-      columnFamilyHandle = db.createColumnFamily(
+      try(final ColumnFamilyHandle columnFamilyHandle = db.createColumnFamily(
          new ColumnFamilyDescriptor("new_cf".getBytes(),
-          new ColumnFamilyOptions()));
-      assert(columnFamilyHandle != null);
-
-    } finally {
-      if (columnFamilyHandle != null) {
-        columnFamilyHandle.dispose();
-      }
-      if (db != null) {
-        db.close();
-        db = null;
-      }
-      if (options != null) {
-        options.dispose();
+          new ColumnFamilyOptions()))) {
+        assert (columnFamilyHandle != null);
      }
    }

    // open DB with two column families
-    List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>();
+    final List<ColumnFamilyDescriptor> columnFamilyDescriptors =
+        new ArrayList<>();
    // have to open default column family
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
        RocksDB.DEFAULT_COLUMN_FAMILY, new ColumnFamilyOptions()));
    // open the new one, too
    columnFamilyDescriptors.add(new ColumnFamilyDescriptor(
        "new_cf".getBytes(), new ColumnFamilyOptions()));
-    List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
-    try {
-      db = RocksDB.open(new DBOptions(), db_path,
-          columnFamilyDescriptors, columnFamilyHandles);
+    final List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>();
+    try(final DBOptions options = new DBOptions();
+        final RocksDB db = RocksDB.open(options, db_path,
+            columnFamilyDescriptors, columnFamilyHandles)) {
      assert(db != null);

-      // put and get from non-default column family
-      db.put(columnFamilyHandles.get(0), new WriteOptions(),
-          "key".getBytes(), "value".getBytes());
-      String value = new String(db.get(columnFamilyHandles.get(0),
-          "key".getBytes()));
+      try {
+        // put and get from non-default column family
+        db.put(columnFamilyHandles.get(0), new WriteOptions(),
+            "key".getBytes(), "value".getBytes());
+        String value = new String(db.get(columnFamilyHandles.get(0),
+            "key".getBytes()));

-      // atomic write
-      wb = new WriteBatch();
-      wb.put(columnFamilyHandles.get(0), "key2".getBytes(), "value2".getBytes());
-      wb.put(columnFamilyHandles.get(1), "key3".getBytes(), "value3".getBytes());
-      wb.remove(columnFamilyHandles.get(0), "key".getBytes());
-      db.write(new WriteOptions(), wb);
+        // atomic write
+        try (final WriteBatch wb = new WriteBatch()) {
+          wb.put(columnFamilyHandles.get(0), "key2".getBytes(),
+              "value2".getBytes());
+          wb.put(columnFamilyHandles.get(1), "key3".getBytes(),
+              "value3".getBytes());
+          wb.remove(columnFamilyHandles.get(0), "key".getBytes());
+          db.write(new WriteOptions(), wb);
+        }

-      // drop column family
-      db.dropColumnFamily(columnFamilyHandles.get(1));
-
-    } finally {
-      for (ColumnFamilyHandle handle : columnFamilyHandles){
-        handle.dispose();
-      }
-      if (db != null) {
-        db.close();
-      }
-      if (wb != null) {
-        wb.dispose();
+        // drop column family
+        db.dropColumnFamily(columnFamilyHandles.get(1));
+      } finally {
+        for (final ColumnFamilyHandle handle : columnFamilyHandles) {
+          handle.close();
+        }
      }
    }
  }
@@ -8,8 +8,10 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.ArrayList;
+
 import org.rocksdb.*;
 import org.rocksdb.util.SizeUnit;
+
 import java.io.IOException;

 public class RocksDBSample {
@@ -26,287 +28,273 @@ public class RocksDBSample {
    String db_path_not_found = db_path + "_not_found";

    System.out.println("RocksDBSample");
-    RocksDB db = null;
-    Options options = new Options();
-    try {
-      db = RocksDB.open(options, db_path_not_found);
-      assert(false);
-    } catch (RocksDBException e) {
-      System.out.format("caught the expceted exception -- %s\n", e);
-      assert(db == null);
-    }
+    try (final Options options = new Options();
+         final Filter bloomFilter = new BloomFilter(10);
+         final ReadOptions readOptions = new ReadOptions()
+             .setFillCache(false)) {

-    try {
-      options.setCreateIfMissing(true)
-          .createStatistics()
-          .setWriteBufferSize(8 * SizeUnit.KB)
-          .setMaxWriteBufferNumber(3)
-          .setMaxBackgroundCompactions(10)
-          .setCompressionType(CompressionType.SNAPPY_COMPRESSION)
-          .setCompactionStyle(CompactionStyle.UNIVERSAL);
-    } catch (IllegalArgumentException e) {
-      assert(false);
-    }
-
-    Statistics stats = options.statisticsPtr();
-
-    assert(options.createIfMissing() == true);
-    assert(options.writeBufferSize() == 8 * SizeUnit.KB);
-    assert(options.maxWriteBufferNumber() == 3);
-    assert(options.maxBackgroundCompactions() == 10);
-    assert(options.compressionType() == CompressionType.SNAPPY_COMPRESSION);
-    assert(options.compactionStyle() == CompactionStyle.UNIVERSAL);
-
-    assert(options.memTableFactoryName().equals("SkipListFactory"));
-    options.setMemTableConfig(
-        new HashSkipListMemTableConfig()
-            .setHeight(4)
-            .setBranchingFactor(4)
-            .setBucketCount(2000000));
-    assert(options.memTableFactoryName().equals("HashSkipListRepFactory"));
-
-    options.setMemTableConfig(
-        new HashLinkedListMemTableConfig()
-            .setBucketCount(100000));
-    assert(options.memTableFactoryName().equals("HashLinkedListRepFactory"));
-
-    options.setMemTableConfig(
-        new VectorMemTableConfig().setReservedSize(10000));
-    assert(options.memTableFactoryName().equals("VectorRepFactory"));
-
-    options.setMemTableConfig(new SkipListMemTableConfig());
-    assert(options.memTableFactoryName().equals("SkipListFactory"));
-
-    options.setTableFormatConfig(new PlainTableConfig());
-    // Plain-Table requires mmap read
-    options.setAllowMmapReads(true);
-    assert(options.tableFactoryName().equals("PlainTable"));
-
-    options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000,
-        10000, 10));
-    options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000));
-
-
-    Filter bloomFilter = new BloomFilter(10);
-    BlockBasedTableConfig table_options = new BlockBasedTableConfig();
-    table_options.setBlockCacheSize(64 * SizeUnit.KB)
-                 .setFilter(bloomFilter)
-                 .setCacheNumShardBits(6)
-                 .setBlockSizeDeviation(5)
-                 .setBlockRestartInterval(10)
-                 .setCacheIndexAndFilterBlocks(true)
-                 .setHashIndexAllowCollision(false)
-                 .setBlockCacheCompressedSize(64 * SizeUnit.KB)
-                 .setBlockCacheCompressedNumShardBits(10);
-
-    assert(table_options.blockCacheSize() == 64 * SizeUnit.KB);
-    assert(table_options.cacheNumShardBits() == 6);
-    assert(table_options.blockSizeDeviation() == 5);
-    assert(table_options.blockRestartInterval() == 10);
-    assert(table_options.cacheIndexAndFilterBlocks() == true);
-    assert(table_options.hashIndexAllowCollision() == false);
-    assert(table_options.blockCacheCompressedSize() == 64 * SizeUnit.KB);
-    assert(table_options.blockCacheCompressedNumShardBits() == 10);
-
-    options.setTableFormatConfig(table_options);
-    assert(options.tableFactoryName().equals("BlockBasedTable"));
-
-    try {
-      db = RocksDB.open(options, db_path);
-      db.put("hello".getBytes(), "world".getBytes());
-      byte[] value = db.get("hello".getBytes());
-      assert("world".equals(new String(value)));
-      String str = db.getProperty("rocksdb.stats");
-      assert(str != null && !str.equals(""));
-    } catch (RocksDBException e) {
-      System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
-      assert(db == null);
-      assert(false);
-    }
-    // be sure to release the c++ pointer
-    db.close();
-
-    ReadOptions readOptions = new ReadOptions();
-    readOptions.setFillCache(false);
-
-    try {
-      db = RocksDB.open(options, db_path);
-      db.put("hello".getBytes(), "world".getBytes());
-      byte[] value = db.get("hello".getBytes());
-      System.out.format("Get('hello') = %s\n",
-          new String(value));
-
-      for (int i = 1; i <= 9; ++i) {
-        for (int j = 1; j <= 9; ++j) {
-          db.put(String.format("%dx%d", i, j).getBytes(),
-                 String.format("%d", i * j).getBytes());
-        }
+      try (final RocksDB db = RocksDB.open(options, db_path_not_found)) {
+        assert (false);
+      } catch (RocksDBException e) {
+        System.out.format("caught the expected exception -- %s\n", e);
      }

-      for (int i = 1; i <= 9; ++i) {
-        for (int j = 1; j <= 9; ++j) {
-          System.out.format("%s ", new String(db.get(
-              String.format("%dx%d", i, j).getBytes())));
-        }
-        System.out.println("");
+      try {
+        options.setCreateIfMissing(true)
+            .createStatistics()
+            .setWriteBufferSize(8 * SizeUnit.KB)
+            .setMaxWriteBufferNumber(3)
+            .setMaxBackgroundCompactions(10)
+            .setCompressionType(CompressionType.SNAPPY_COMPRESSION)
+            .setCompactionStyle(CompactionStyle.UNIVERSAL);
+      } catch (IllegalArgumentException e) {
+        assert (false);
      }

-      // write batch test
-      WriteOptions writeOpt = new WriteOptions();
-      for (int i = 10; i <= 19; ++i) {
-        WriteBatch batch = new WriteBatch();
-        for (int j = 10; j <= 19; ++j) {
-          batch.put(String.format("%dx%d", i, j).getBytes(),
+      Statistics stats = options.statisticsPtr();
+
+      assert (options.createIfMissing() == true);
+      assert (options.writeBufferSize() == 8 * SizeUnit.KB);
+      assert (options.maxWriteBufferNumber() == 3);
+      assert (options.maxBackgroundCompactions() == 10);
+      assert (options.compressionType() == CompressionType.SNAPPY_COMPRESSION);
+      assert (options.compactionStyle() == CompactionStyle.UNIVERSAL);
+
+      assert (options.memTableFactoryName().equals("SkipListFactory"));
+      options.setMemTableConfig(
+          new HashSkipListMemTableConfig()
+              .setHeight(4)
+              .setBranchingFactor(4)
+              .setBucketCount(2000000));
+      assert (options.memTableFactoryName().equals("HashSkipListRepFactory"));
+
+      options.setMemTableConfig(
+          new HashLinkedListMemTableConfig()
+              .setBucketCount(100000));
+      assert (options.memTableFactoryName().equals("HashLinkedListRepFactory"));
+
+      options.setMemTableConfig(
+          new VectorMemTableConfig().setReservedSize(10000));
+      assert (options.memTableFactoryName().equals("VectorRepFactory"));
+
+      options.setMemTableConfig(new SkipListMemTableConfig());
+      assert (options.memTableFactoryName().equals("SkipListFactory"));
+
+      options.setTableFormatConfig(new PlainTableConfig());
+      // Plain-Table requires mmap read
+      options.setAllowMmapReads(true);
+      assert (options.tableFactoryName().equals("PlainTable"));
+
+      options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000,
+          10000, 10));
+      options.setRateLimiterConfig(new GenericRateLimiterConfig(10000000));
+
+      final BlockBasedTableConfig table_options = new BlockBasedTableConfig();
+      table_options.setBlockCacheSize(64 * SizeUnit.KB)
+          .setFilter(bloomFilter)
+          .setCacheNumShardBits(6)
+          .setBlockSizeDeviation(5)
+          .setBlockRestartInterval(10)
+          .setCacheIndexAndFilterBlocks(true)
+          .setHashIndexAllowCollision(false)
+          .setBlockCacheCompressedSize(64 * SizeUnit.KB)
+          .setBlockCacheCompressedNumShardBits(10);
+
+      assert (table_options.blockCacheSize() == 64 * SizeUnit.KB);
+      assert (table_options.cacheNumShardBits() == 6);
+      assert (table_options.blockSizeDeviation() == 5);
+      assert (table_options.blockRestartInterval() == 10);
+      assert (table_options.cacheIndexAndFilterBlocks() == true);
+      assert (table_options.hashIndexAllowCollision() == false);
+      assert (table_options.blockCacheCompressedSize() == 64 * SizeUnit.KB);
+      assert (table_options.blockCacheCompressedNumShardBits() == 10);
+
+      options.setTableFormatConfig(table_options);
+      assert (options.tableFactoryName().equals("BlockBasedTable"));
+
+      try (final RocksDB db = RocksDB.open(options, db_path)) {
+        db.put("hello".getBytes(), "world".getBytes());
+        byte[] value = db.get("hello".getBytes());
+        assert ("world".equals(new String(value)));
+        String str = db.getProperty("rocksdb.stats");
+        assert (str != null && !str.equals(""));
+      } catch (RocksDBException e) {
+        System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
+        assert (false);
+      }
+
+      try (final RocksDB db = RocksDB.open(options, db_path)) {
+        db.put("hello".getBytes(), "world".getBytes());
+        byte[] value = db.get("hello".getBytes());
+        System.out.format("Get('hello') = %s\n",
+            new String(value));
+
+        for (int i = 1; i <= 9; ++i) {
+          for (int j = 1; j <= 9; ++j) {
+            db.put(String.format("%dx%d", i, j).getBytes(),
+                String.format("%d", i * j).getBytes());
+          }
+        }
+
+        for (int i = 1; i <= 9; ++i) {
+          for (int j = 1; j <= 9; ++j) {
+            System.out.format("%s ", new String(db.get(
+                String.format("%dx%d", i, j).getBytes())));
+          }
+          System.out.println("");
+        }
+
+        // write batch test
+        try (final WriteOptions writeOpt = new WriteOptions()) {
+          for (int i = 10; i <= 19; ++i) {
+            try (final WriteBatch batch = new WriteBatch()) {
+              for (int j = 10; j <= 19; ++j) {
+                batch.put(String.format("%dx%d", i, j).getBytes(),
                    String.format("%d", i * j).getBytes());
+              }
+              db.write(writeOpt, batch);
+            }
+          }
        }
-        db.write(writeOpt, batch);
-        batch.dispose();
-      }
-      for (int i = 10; i <= 19; ++i) {
-        for (int j = 10; j <= 19; ++j) {
-          assert(new String(
-              db.get(String.format("%dx%d", i, j).getBytes())).equals(
-                  String.format("%d", i * j)));
-          System.out.format("%s ", new String(db.get(
-              String.format("%dx%d", i, j).getBytes())));
+        for (int i = 10; i <= 19; ++i) {
+          for (int j = 10; j <= 19; ++j) {
+            assert (new String(
+                db.get(String.format("%dx%d", i, j).getBytes())).equals(
+                String.format("%d", i * j)));
+            System.out.format("%s ", new String(db.get(
+                String.format("%dx%d", i, j).getBytes())));
+          }
+          System.out.println("");
        }
-        System.out.println("");
-      }
-      writeOpt.dispose();

-      value = db.get("1x1".getBytes());
-      assert(value != null);
-      value = db.get("world".getBytes());
-      assert(value == null);
-      value = db.get(readOptions, "world".getBytes());
-      assert(value == null);
+        value = db.get("1x1".getBytes());
+        assert (value != null);
+        value = db.get("world".getBytes());
+        assert (value == null);
+        value = db.get(readOptions, "world".getBytes());
+        assert (value == null);

-      byte[] testKey = "asdf".getBytes();
-      byte[] testValue =
-          "asdfghjkl;'?><MNBVCXZQWERTYUIOP{+_)(*&^%$#@".getBytes();
-      db.put(testKey, testValue);
-      byte[] testResult = db.get(testKey);
-      assert(testResult != null);
-      assert(Arrays.equals(testValue, testResult));
-      assert(new String(testValue).equals(new String(testResult)));
-      testResult = db.get(readOptions, testKey);
-      assert(testResult != null);
-      assert(Arrays.equals(testValue, testResult));
-      assert(new String(testValue).equals(new String(testResult)));
+        byte[] testKey = "asdf".getBytes();
+        byte[] testValue =
+            "asdfghjkl;'?><MNBVCXZQWERTYUIOP{+_)(*&^%$#@".getBytes();
+        db.put(testKey, testValue);
+        byte[] testResult = db.get(testKey);
+        assert (testResult != null);
+        assert (Arrays.equals(testValue, testResult));
+        assert (new String(testValue).equals(new String(testResult)));
+        testResult = db.get(readOptions, testKey);
+        assert (testResult != null);
+        assert (Arrays.equals(testValue, testResult));
+        assert (new String(testValue).equals(new String(testResult)));

-      byte[] insufficientArray = new byte[10];
-      byte[] enoughArray = new byte[50];
-      int len;
-      len = db.get(testKey, insufficientArray);
-      assert(len > insufficientArray.length);
-      len = db.get("asdfjkl;".getBytes(), enoughArray);
-      assert(len == RocksDB.NOT_FOUND);
-      len = db.get(testKey, enoughArray);
-      assert(len == testValue.length);
+        byte[] insufficientArray = new byte[10];
+        byte[] enoughArray = new byte[50];
+        int len;
+        len = db.get(testKey, insufficientArray);
+        assert (len > insufficientArray.length);
+        len = db.get("asdfjkl;".getBytes(), enoughArray);
+        assert (len == RocksDB.NOT_FOUND);
+        len = db.get(testKey, enoughArray);
+        assert (len == testValue.length);

-      len = db.get(readOptions, testKey, insufficientArray);
-      assert(len > insufficientArray.length);
-      len = db.get(readOptions, "asdfjkl;".getBytes(), enoughArray);
-      assert(len == RocksDB.NOT_FOUND);
-      len = db.get(readOptions, testKey, enoughArray);
-      assert(len == testValue.length);
+        len = db.get(readOptions, testKey, insufficientArray);
+        assert (len > insufficientArray.length);
+        len = db.get(readOptions, "asdfjkl;".getBytes(), enoughArray);
+        assert (len == RocksDB.NOT_FOUND);
+        len = db.get(readOptions, testKey, enoughArray);
+        assert (len == testValue.length);

-      db.remove(testKey);
-      len = db.get(testKey, enoughArray);
-      assert(len == RocksDB.NOT_FOUND);
+        db.remove(testKey);
+        len = db.get(testKey, enoughArray);
+        assert (len == RocksDB.NOT_FOUND);

-      // repeat the test with WriteOptions
-      WriteOptions writeOpts = new WriteOptions();
-      writeOpts.setSync(true);
-      writeOpts.setDisableWAL(true);
-      db.put(writeOpts, testKey, testValue);
-      len = db.get(testKey, enoughArray);
-      assert(len == testValue.length);
-      assert(new String(testValue).equals(
-          new String(enoughArray, 0, len)));
-      writeOpts.dispose();
-
-      try {
-        for (TickerType statsType : TickerType.values()) {
-          stats.getTickerCount(statsType);
+        // repeat the test with WriteOptions
+        try (final WriteOptions writeOpts = new WriteOptions()) {
+          writeOpts.setSync(true);
+          writeOpts.setDisableWAL(true);
+          db.put(writeOpts, testKey, testValue);
+          len = db.get(testKey, enoughArray);
+          assert (len == testValue.length);
+          assert (new String(testValue).equals(
+              new String(enoughArray, 0, len)));
        }
-        System.out.println("getTickerCount() passed.");
-      } catch (Exception e) {
-        System.out.println("Failed in call to getTickerCount()");
-        assert(false); //Should never reach here.
-      }

-      try {
-        for (HistogramType histogramType : HistogramType.values()) {
-          HistogramData data = stats.geHistogramData(histogramType);
+        try {
+          for (TickerType statsType : TickerType.values()) {
+            stats.getTickerCount(statsType);
+          }
+          System.out.println("getTickerCount() passed.");
+        } catch (Exception e) {
+          System.out.println("Failed in call to getTickerCount()");
+          assert (false); //Should never reach here.
        }
-        System.out.println("geHistogramData() passed.");
-      } catch (Exception e) {
-        System.out.println("Failed in call to geHistogramData()");
-        assert(false); //Should never reach here.
+
+        try {
+          for (HistogramType histogramType : HistogramType.values()) {
+            HistogramData data = stats.geHistogramData(histogramType);
+          }
+          System.out.println("geHistogramData() passed.");
+        } catch (Exception e) {
+          System.out.println("Failed in call to geHistogramData()");
+          assert (false); //Should never reach here.
+        }
+
+        try (final RocksIterator iterator = db.newIterator()) {
+
+          boolean seekToFirstPassed = false;
+          for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {
+            iterator.status();
+            assert (iterator.key() != null);
+            assert (iterator.value() != null);
+            seekToFirstPassed = true;
+          }
+          if (seekToFirstPassed) {
+            System.out.println("iterator seekToFirst tests passed.");
+          }
+
+          boolean seekToLastPassed = false;
+          for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
+            iterator.status();
+            assert (iterator.key() != null);
+            assert (iterator.value() != null);
+            seekToLastPassed = true;
+          }
+
+          if (seekToLastPassed) {
+            System.out.println("iterator seekToLastPassed tests passed.");
+          }
+
+          iterator.seekToFirst();
+          iterator.seek(iterator.key());
+          assert (iterator.key() != null);
+          assert (iterator.value() != null);
+
+          System.out.println("iterator seek test passed.");
+
+        }
+        System.out.println("iterator tests passed.");
+
+        final List<byte[]> keys = new ArrayList<>();
+        try (final RocksIterator iterator = db.newIterator()) {
+          for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
+            keys.add(iterator.key());
+          }
+        }
+
+        Map<byte[], byte[]> values = db.multiGet(keys);
+        assert (values.size() == keys.size());
+        for (byte[] value1 : values.values()) {
+          assert (value1 != null);
+        }
+
+        values = db.multiGet(new ReadOptions(), keys);
+        assert (values.size() == keys.size());
+        for (byte[] value1 : values.values()) {
+          assert (value1 != null);
+        }
+      } catch (RocksDBException e) {
+        System.err.println(e);
      }
-
-      RocksIterator iterator = db.newIterator();
-
-      boolean seekToFirstPassed = false;
-      for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {
-        iterator.status();
-        assert(iterator.key() != null);
-        assert(iterator.value() != null);
-        seekToFirstPassed = true;
-      }
-      if(seekToFirstPassed) {
-        System.out.println("iterator seekToFirst tests passed.");
-      }
-
-      boolean seekToLastPassed = false;
-      for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
-        iterator.status();
-        assert(iterator.key() != null);
-        assert(iterator.value() != null);
-        seekToLastPassed = true;
-      }
-
-      if(seekToLastPassed) {
-        System.out.println("iterator seekToLastPassed tests passed.");
-      }
-
-      iterator.seekToFirst();
-      iterator.seek(iterator.key());
-      assert(iterator.key() != null);
-      assert(iterator.value() != null);
-
-      System.out.println("iterator seek test passed.");
-
-      iterator.dispose();
-      System.out.println("iterator tests passed.");
-
-      iterator = db.newIterator();
-      List<byte[]> keys = new ArrayList<byte[]>();
-      for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
-        keys.add(iterator.key());
-      }
-      iterator.dispose();
-
-      Map<byte[], byte[]> values = db.multiGet(keys);
-      assert(values.size() == keys.size());
-      for(byte[] value1 : values.values()) {
-        assert(value1 != null);
-      }
-
-      values = db.multiGet(new ReadOptions(), keys);
-      assert(values.size() == keys.size());
-      for(byte[] value1 : values.values()) {
-        assert(value1 != null);
-      }
-    } catch (RocksDBException e) {
-      System.err.println(e);
    }
-    if (db != null) {
-      db.close();
-    }
-    // be sure to dispose c++ pointers
-    options.dispose();
-    readOptions.dispose();
  }
 }
@@ -8,22 +8,23 @@ package org.rocksdb;
 * A CompactionFilter allows an application to modify/delete a key-value at
 * the time of compaction.
 *
- * At present we just permit an overriding Java class to wrap a C++ implementation
+ * At present we just permit an overriding Java class to wrap a C++
+ * implementation
 */
 public abstract class AbstractCompactionFilter<T extends AbstractSlice<?>>
    extends RocksObject {

-  /**
-   * Deletes underlying C++ comparator pointer.
-   *
-   * Note that this function should be called only after all
-   * RocksDB instances referencing the comparator are closed.
-   * Otherwise an undefined behavior will occur.
-   */
-  @Override protected void disposeInternal() {
-    assert(isInitialized());
-    disposeInternal(nativeHandle_);
+  protected AbstractCompactionFilter(final long nativeHandle) {
+    super(nativeHandle);
  }

-  private native void disposeInternal(long handle);
+  /**
+   * Deletes underlying C++ compaction pointer.
+   *
+   * Note that this function should be called only after all
+   * RocksDB instances referencing the compaction filter are closed.
+   * Otherwise an undefined behavior will occur.
+   */
+  @Override
+  protected final native void disposeInternal(final long handle);
 }
@@ -15,7 +15,11 @@ package org.rocksdb;
 *   @see org.rocksdb.DirectComparator
 */
 public abstract class AbstractComparator<T extends AbstractSlice<?>>
-    extends RocksObject {
+    extends AbstractImmutableNativeReference {
+
+  protected AbstractComparator() {
+    super(true);
+  }

  /**
   * The name of the comparator.  Used to check for comparator
@@ -91,10 +95,12 @@ public abstract class AbstractComparator<T extends AbstractSlice<?>>
   * RocksDB instances referencing the comparator are closed.
   * Otherwise an undefined behavior will occur.
   */
-  @Override protected void disposeInternal() {
-    assert(isInitialized());
-    disposeInternal(nativeHandle_);
+  @Override
+  protected void disposeInternal() {
+    disposeInternal(getNativeHandle());
  }

-  private native void disposeInternal(long handle);
+  protected abstract long getNativeHandle();
+
+  private native void disposeInternal(final long handle);
 }
@@ -0,0 +1,66 @@
+// Copyright (c) 2016, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Offers functionality for implementations of
+ * {@link AbstractNativeReference} which have an immutable reference to the
+ * underlying native C++ object
+ */
+public abstract class AbstractImmutableNativeReference
+    extends AbstractNativeReference {
+
+  /**
+   * A flag indicating whether the current {@code AbstractNativeReference} is
+   * responsible to free the underlying C++ object
+   */
+  private final AtomicBoolean owningHandle_;
+
+  protected AbstractImmutableNativeReference(final boolean owningHandle) {
+    this.owningHandle_ = new AtomicBoolean(owningHandle);
+  }
+
+  @Override
+  public boolean isOwningHandle() {
+    return owningHandle_.get();
+  }
+
+  /**
+   * Releases this {@code AbstractNativeReference} from  the responsibility of
+   * freeing the underlying native C++ object
+   * <p>
+   * This will prevent the object from attempting to delete the underlying
+   * native object in its finalizer. This must be used when another object
+   * takes over ownership of the native object or both will attempt to delete
+   * the underlying object when garbage collected.
+   * <p>
+   * When {@code disOwnNativeHandle()} is called, {@code dispose()} will
+   * subsequently take no action. As a result, incorrect use of this function
+   * may cause a memory leak.
+   * </p>
+   *
+   * @see #dispose()
+   */
+  protected final void disOwnNativeHandle() {
+    owningHandle_.set(false);
+  }
+
+  @Override
+  public void close() {
+    if (owningHandle_.compareAndSet(true, false)) {
+      disposeInternal();
+    }
+  }
+
+  /**
+   * The helper function of {@link AbstractImmutableNativeReference#dispose()}
+   * which all subclasses of {@code AbstractImmutableNativeReference} must
+   * implement to release their underlying native C++ objects.
+   */
+  protected abstract void disposeInternal();
+}