Compare commits

..
6 Commits
Author SHA1 Message Date
sdong 7e4a06c699 One more include<functional> 2019-10-31 14:52:10 -07:00
sdong 853247543f Disable warning as error 2019-10-31 14:52:09 -07:00
sdong a72f5f6f4f Add some include<functional> 2019-10-31 14:52:09 -07:00
sdong 9f22c0f9f4 [FB Internal] Point to the latest tool chain. 2019-10-31 14:52:09 -07:00
sdong 8f0ffa6afd [FB Only] use gcc-5 2017-07-17 21:13:44 -07:00
Igor Canadi 6c9f6a115f Fix signed/unsigned compile 2014-11-24 13:27:49 -05:00
214 changed files with 2498 additions and 12057 deletions

No files matched your search

+1 -1
View File
@@ -14,6 +14,6 @@ before_install:
- sudo dpkg -i libgflags-dev_2.0-1_amd64.deb
# Lousy hack to disable use and testing of fallocate, which doesn't behave quite
# as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
script: OPT=-DTRAVIS make unity && make clean && OPT=-DTRAVIS make check
script: OPT=-DTRAVIS make unity && make clean && OPT=-DTRAVIS make check -j8
notifications:
email: false
-17
View File
@@ -1,22 +1,5 @@
# Rocksdb Change Log
### 3.9.0 (12/8/2014)
### New Features
* Add rocksdb::GetThreadList(), which in the future will return the current status of all
rocksdb-related threads. We will have more code instruments in the following RocksDB
releases.
### Public API changes
* New API to create a checkpoint added. Given a directory name, creates a new
database which is an image of the existing database.
*New API LinkFile added to Env. If you implement your own Env class, an
implementation of the API LinkFile will have to be provided.
* MemTableRep takes MemTableAllocator instead of Arena
### Improvements
* RocksDBLite library now becomes smaller and will be compiled with -fno-exceptions flag.
## 3.8.0 (11/14/2014)
### Public API changes
+4 -33
View File
@@ -6,16 +6,8 @@
#-----------------------------------------------
CFLAGS += ${EXTRA_CFLAGS}
CXXFLAGS += ${EXTRA_CXXFLAGS}
LDFLAGS += $(EXTRA_LDFLAGS)
MACHINE ?= $(shell uname -m)
ifneq ($(MAKECMDGOALS),dbg)
OPT += -O2 -fno-omit-frame-pointer
ifneq ($(MACHINE),ppc64) # ppc64 doesn't support -momit-leaf-frame-pointer
OPT += -momit-leaf-frame-pointer
endif
OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
else
# intentionally left blank
endif
@@ -43,12 +35,6 @@ else
OPT += -DNDEBUG
endif
ifneq ($(filter -DROCKSDB_LITE,$(OPT)),)
# found
CFLAGS += -fno-exceptions
CXXFLAGS += -fno-exceptions
endif
# ASAN doesn't work well with jemalloc. If we're compiling with ASAN, we should use regular malloc.
ifdef COMPILE_WITH_ASAN
# ASAN compile flags
@@ -83,7 +69,6 @@ install:
@[ ! -e $(SHARED) ] || install -C -m 644 $(SHARED) $(INSTALL_PATH)/lib
#-------------------------------------------------
WARNING_FLAGS = -Wall -Werror -Wsign-compare -Wshadow
CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor
@@ -164,10 +149,7 @@ TESTS = \
flush_job_test \
wal_manager_test \
listener_test \
compaction_job_test \
thread_list_test
SUBSET := $(shell echo $(TESTS) |sed s/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/)
write_batch_with_index_test
TOOLS = \
sst_dump \
@@ -255,10 +237,6 @@ check: $(TESTS) ldb
for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
python tools/ldb_test.py
check_some: $(SUBSET) ldb
for t in $(SUBSET); do echo "***** Running $$t"; ./$$t || exit 1; done
python tools/ldb_test.py
ldb_tests: ldb
python tools/ldb_test.py
@@ -446,9 +424,6 @@ write_batch_with_index_test: utilities/write_batch_with_index/write_batch_with_i
flush_job_test: db/flush_job_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/flush_job_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
compaction_job_test: db/compaction_job_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/compaction_job_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
wal_manager_test: db/wal_manager_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/wal_manager_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
@@ -530,9 +505,6 @@ cuckoo_table_db_test: db/cuckoo_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
listener_test: db/listener_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/listener_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
thread_list_test: util/thread_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) util/thread_list_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
compactor_test: utilities/compaction/compactor_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) utilities/compaction/compactor_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
@@ -618,10 +590,9 @@ libsnappy.a:
rocksdbjavastatic: libz.a libbz2.a libsnappy.a
OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j
cd java;$(MAKE) javalib;
cd java;$(MAKE) java;
rm -f ./java/$(ROCKSDBJNILIB)
$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(LIBOBJECTS) $(COVERAGEFLAGS) libz.a libbz2.a libsnappy.a
cd java;strip -S -x $(ROCKSDBJNILIB)
cd java;jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class HISTORY*.md $(ROCKSDBJNILIB)
cd java/javadoc;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) *
cd java;jar -cf $(ROCKSDB_SOURCES_JAR) org
@@ -640,7 +611,7 @@ rocksdbjavastaticpublish: rocksdbjavastaticrelease
rocksdbjava:
OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j32
cd java;$(MAKE) javalib;
cd java;$(MAKE) java;
rm -f ./java/$(ROCKSDBJNILIB)
$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(LIBOBJECTS) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
cd java;jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class HISTORY*.md $(ROCKSDBJNILIB)
+19
View File
@@ -0,0 +1,19 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
+99 -56
View File
@@ -1,105 +1,148 @@
#!/bin/sh
#
# Set environment variables so that we can compile rocksdb using
# fbcode settings. It uses the latest g++ compiler and also
# fbcode settings. It uses the latest g++ and clang compilers and also
# uses jemalloc
# Environment variables that change the behavior of this script:
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
# location of libgcc
LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/7712e757d7355cb51292454ee0b7b46a467fdfed/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc"
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
# location of glibc
GLIBC_REV=6e40560b4e0b6d690fd1cf8c7a43ad7452b04cfa
GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/include"
GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/lib"
BASEDIR=`dirname $BASH_SOURCE`
source "$BASEDIR/dependencies.sh"
# location of snappy headers and libraries
SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/aef17f6c0b44b4fe408bd06f67c93701ab0a6ceb/1.0.3/gcc-4.8.1-glibc-2.17/43d84e2/include"
SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/aef17f6c0b44b4fe408bd06f67c93701ab0a6ceb/1.0.3/gcc-4.8.1-glibc-2.17/43d84e2/lib/libsnappy.a"
CFLAGS=""
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/25c6216928b4d77b59ddeca0990ff6fe9ac16b81/1.2.5/gcc-4.8.1-glibc-2.17/c3f970a/include"
ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/25c6216928b4d77b59ddeca0990ff6fe9ac16b81/1.2.5/gcc-4.8.1-glibc-2.17/c3f970a/lib/libz.a"
# libgcc
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
# location of bzip headers and libraries
BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/c9ef7629c2aa0024f7a416e87602f06eb88f5eac/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/include/"
BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/c9ef7629c2aa0024f7a416e87602f06eb88f5eac/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libbz2.a"
# glibc
GLIBC_INCLUDE="$GLIBC_BASE/include"
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
LZ4_REV=065ec7e38fe83329031f6668c43bef83eff5808b
LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/include"
LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a"
# snappy
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
if test -z $PIC_BUILD; then
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
else
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
fi
CFLAGS+=" -DSNAPPY"
if test -z $PIC_BUILD; then
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
CFLAGS+=" -DZLIB"
# location of bzip headers and libraries
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
CFLAGS+=" -DBZIP2"
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
CFLAGS+=" -DLZ4"
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
CFLAGS+=" -DZSTD"
fi
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/1ad047a6e6f6673991918ecadc670868205a243a/1.6/gcc-4.8.1-glibc-2.17/c3f970a/include/"
GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/1ad047a6e6f6673991918ecadc670868205a243a/1.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libgflags.a"
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
if test -z $PIC_BUILD; then
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
else
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
fi
CFLAGS+=" -DGFLAGS=gflags"
# location of jemalloc
JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/c60d854f7824f334195fe7fd34b2bc9057e3c1f9/3.6.0/gcc-4.8.1-glibc-2.17/4d53c6f/include"
JEMALLOC_LIB=" -Wl,--whole-archive /mnt/gvfs/third-party2/jemalloc/c60d854f7824f334195fe7fd34b2bc9057e3c1f9/3.6.0/gcc-4.8.1-glibc-2.17/4d53c6f/lib/libjemalloc.a"
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
# location of numa
NUMA_REV=829d10dac0230f99cd7e1778869d2adf3da24b65
NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/include/"
NUMA_LIB=" /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libnuma.a"
if test -z $PIC_BUILD; then
# location of numa
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
CFLAGS+=" -DNUMA"
# location of libunwind
LIBUNWIND_REV=2c060e64064559905d46fd194000d61592087bdc
LIBUNWIND="/mnt/gvfs/third-party2/libunwind/$LIBUNWIND_REV/1.1/gcc-4.8.1-glibc-2.17/675d945/lib/libunwind.a"
# location of libunwind
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
fi
# location of TBB
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
if test -z $PIC_BUILD; then
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
else
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
fi
CFLAGS+=" -DTBB"
# use Intel SSE support for checksum calculations
export USE_SSE=" -msse -msse4.2 "
BINUTILS="/mnt/gvfs/third-party2/binutils/2aff2e7b474cd3e6ab23495ad1224b7d214b9f8e/2.21.1/centos6-native/da39a3e/bin"
BINUTILS="$BINUTILS_BASE/bin"
AR="$BINUTILS/ar"
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
GCC_BASE="/mnt/gvfs/third-party2/gcc/1ec615e23800f0815d474478ba476a0adc3fe788/4.8.1/centos6-native/cc6c9dc"
STDLIBS="-L $GCC_BASE/lib64"
CLANG_BIN="$CLANG_BASE/bin"
CLANG_LIB="$CLANG_BASE/lib"
CLANG_SRC="$CLANG_BASE/../../src"
CLANG_ANALYZER="$CLANG_BIN/clang++"
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
if [ -z "$USE_CLANG" ]; then
# gcc
CC="$GCC_BASE/bin/gcc"
CXX="$GCC_BASE/bin/g++"
CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -B$BINUTILS/gold"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
CFLAGS+=" -isystem $GLIBC_INCLUDE"
JEMALLOC=1
else
# clang
CLANG_BASE="/mnt/gvfs/third-party2/clang/9ab68376f938992c4eb5946ca68f90c3185cffc8/3.4"
CLANG_INCLUDE="$CLANG_BASE/gcc-4.8.1-glibc-2.17/fb0f730/lib/clang/3.4/include"
CC="$CLANG_BASE/centos6-native/9cefd8a/bin/clang"
CXX="$CLANG_BASE/centos6-native/9cefd8a/bin/clang++"
# clang
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
CC="$CLANG_BIN/clang"
CXX="$CLANG_BIN/clang++"
KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/a683ed7135276731065a9d76d3016c9731f4e2f9/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.8.1-glibc-2.17/da39a3e/include/"
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib"
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux "
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
CFLAGS+=" -isystem $CLANG_INCLUDE"
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
CFLAGS+=" -Wno-expansion-to-defined "
CXXFLAGS="-nostdinc++"
fi
CFLAGS+=" $DEPS_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_FALLOCATE_PRESENT"
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DNUMA"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
CXXFLAGS+=" $CFLAGS"
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS $NUMA_LIB"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $LIBUNWIND"
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
EXEC_LDFLAGS+=" $LIBUNWIND"
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
# required by libtbb
EXEC_LDFLAGS+=" -ldl"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
VALGRIND_REV=b2a9f85e4b70cd03abc85a7f3027fbc4cef35bd0
VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/$VALGRIND_REV/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a/bin/"
VALGRIND_VER="$VALGRIND_BASE/bin/"
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
+3 -68
View File
@@ -56,7 +56,6 @@ using rocksdb::NewBloomFilterPolicy;
using rocksdb::NewLRUCache;
using rocksdb::Options;
using rocksdb::BlockBasedTableOptions;
using rocksdb::CuckooTableOptions;
using rocksdb::RandomAccessFile;
using rocksdb::Range;
using rocksdb::ReadOptions;
@@ -80,14 +79,10 @@ struct rocksdb_writebatch_t { WriteBatch rep; };
struct rocksdb_snapshot_t { const Snapshot* rep; };
struct rocksdb_flushoptions_t { FlushOptions rep; };
struct rocksdb_fifo_compaction_options_t { CompactionOptionsFIFO rep; };
struct rocksdb_readoptions_t {
ReadOptions rep;
Slice upper_bound; // stack variable to set pointer to in ReadOptions
};
struct rocksdb_readoptions_t { ReadOptions rep; };
struct rocksdb_writeoptions_t { WriteOptions rep; };
struct rocksdb_options_t { Options rep; };
struct rocksdb_block_based_table_options_t { BlockBasedTableOptions rep; };
struct rocksdb_cuckoo_table_options_t { CuckooTableOptions rep; };
struct rocksdb_seqfile_t { SequentialFile* rep; };
struct rocksdb_randomfile_t { RandomAccessFile* rep; };
struct rocksdb_writablefile_t { WritableFile* rep; };
@@ -1126,51 +1121,6 @@ void rocksdb_options_set_block_based_table_factory(
}
rocksdb_cuckoo_table_options_t*
rocksdb_cuckoo_options_create() {
return new rocksdb_cuckoo_table_options_t;
}
void rocksdb_cuckoo_options_destroy(
rocksdb_cuckoo_table_options_t* options) {
delete options;
}
void rocksdb_cuckoo_options_set_hash_ratio(
rocksdb_cuckoo_table_options_t* options, double v) {
options->rep.hash_table_ratio = v;
}
void rocksdb_cuckoo_options_set_max_search_depth(
rocksdb_cuckoo_table_options_t* options, uint32_t v) {
options->rep.max_search_depth = v;
}
void rocksdb_cuckoo_options_set_cuckoo_block_size(
rocksdb_cuckoo_table_options_t* options, uint32_t v) {
options->rep.cuckoo_block_size = v;
}
void rocksdb_cuckoo_options_set_identity_as_first_hash(
rocksdb_cuckoo_table_options_t* options, unsigned char v) {
options->rep.identity_as_first_hash = v;
}
void rocksdb_cuckoo_options_set_use_module_hash(
rocksdb_cuckoo_table_options_t* options, unsigned char v) {
options->rep.use_module_hash = v;
}
void rocksdb_options_set_cuckoo_table_factory(
rocksdb_options_t *opt,
rocksdb_cuckoo_table_options_t* table_options) {
if (table_options) {
opt->rep.table_factory.reset(
rocksdb::NewCuckooTableFactory(table_options->rep));
}
}
rocksdb_options_t* rocksdb_options_create() {
return new rocksdb_options_t;
}
@@ -1264,11 +1214,6 @@ void rocksdb_options_set_info_log_level(
opt->rep.info_log_level = static_cast<InfoLogLevel>(v);
}
void rocksdb_options_set_db_write_buffer_size(rocksdb_options_t* opt,
size_t s) {
opt->rep.db_write_buffer_size = s;
}
void rocksdb_options_set_write_buffer_size(rocksdb_options_t* opt, size_t s) {
opt->rep.write_buffer_size = s;
}
@@ -1277,10 +1222,6 @@ void rocksdb_options_set_max_open_files(rocksdb_options_t* opt, int n) {
opt->rep.max_open_files = n;
}
void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n) {
opt->rep.max_total_wal_size = n;
}
void rocksdb_options_set_target_file_size_base(
rocksdb_options_t* opt, uint64_t n) {
opt->rep.target_file_size_base = n;
@@ -1899,14 +1840,8 @@ void rocksdb_readoptions_set_snapshot(
void rocksdb_readoptions_set_iterate_upper_bound(
rocksdb_readoptions_t* opt,
const char* key, size_t keylen) {
if (key == nullptr) {
opt->upper_bound = Slice();
opt->rep.iterate_upper_bound = nullptr;
} else {
opt->upper_bound = Slice(key, keylen);
opt->rep.iterate_upper_bound = &opt->upper_bound;
}
Slice prefix = Slice(key, keylen);
opt->rep.iterate_upper_bound = &prefix;
}
void rocksdb_readoptions_set_read_tier(
-79
View File
@@ -801,87 +801,8 @@ int main(int argc, char** argv) {
rocksdb_iter_get_error(iter, &err);
CheckNoError(err);
rocksdb_iter_destroy(iter);
rocksdb_close(db);
rocksdb_destroy_db(options, dbname, &err);
}
StartPhase("cuckoo_options");
{
rocksdb_cuckoo_table_options_t* cuckoo_options;
cuckoo_options = rocksdb_cuckoo_options_create();
rocksdb_cuckoo_options_set_hash_ratio(cuckoo_options, 0.5);
rocksdb_cuckoo_options_set_max_search_depth(cuckoo_options, 200);
rocksdb_cuckoo_options_set_cuckoo_block_size(cuckoo_options, 10);
rocksdb_cuckoo_options_set_identity_as_first_hash(cuckoo_options, 1);
rocksdb_cuckoo_options_set_use_module_hash(cuckoo_options, 0);
rocksdb_options_set_cuckoo_table_factory(options, cuckoo_options);
db = rocksdb_open(options, dbname, &err);
CheckNoError(err);
rocksdb_cuckoo_options_destroy(cuckoo_options);
}
StartPhase("iterate_upper_bound");
{
// Create new empty database
rocksdb_close(db);
rocksdb_destroy_db(options, dbname, &err);
CheckNoError(err);
rocksdb_options_set_prefix_extractor(options, NULL);
db = rocksdb_open(options, dbname, &err);
CheckNoError(err);
rocksdb_put(db, woptions, "a", 1, "0", 1, &err); CheckNoError(err);
rocksdb_put(db, woptions, "foo", 3, "bar", 3, &err); CheckNoError(err);
rocksdb_put(db, woptions, "foo1", 4, "bar1", 4, &err); CheckNoError(err);
rocksdb_put(db, woptions, "g1", 2, "0", 1, &err); CheckNoError(err);
// testing basic case with no iterate_upper_bound and no prefix_extractor
{
rocksdb_readoptions_set_iterate_upper_bound(roptions, NULL, 0);
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
rocksdb_iter_seek(iter, "foo", 3);
CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "foo", "bar");
rocksdb_iter_next(iter);
CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "foo1", "bar1");
rocksdb_iter_next(iter);
CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "g1", "0");
rocksdb_iter_destroy(iter);
}
// testing iterate_upper_bound and forward iterator
// to make sure it stops at bound
{
// iterate_upper_bound points beyond the last expected entry
rocksdb_readoptions_set_iterate_upper_bound(roptions, "foo2", 4);
rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
rocksdb_iter_seek(iter, "foo", 3);
CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "foo", "bar");
rocksdb_iter_next(iter);
CheckCondition(rocksdb_iter_valid(iter));
CheckIter(iter, "foo1", "bar1");
rocksdb_iter_next(iter);
// should stop here...
CheckCondition(!rocksdb_iter_valid(iter));
rocksdb_iter_destroy(iter);
}
}
StartPhase("cleanup");
rocksdb_close(db);
+15 -26
View File
@@ -19,15 +19,12 @@
#include <algorithm>
#include <limits>
#include "db/compaction_picker.h"
#include "db/db_impl.h"
#include "db/job_context.h"
#include "db/version_set.h"
#include "db/writebuffer.h"
#include "db/internal_stats.h"
#include "db/job_context.h"
#include "db/compaction_picker.h"
#include "db/table_properties_collector.h"
#include "db/version_set.h"
#include "db/write_controller.h"
#include "util/autovector.h"
#include "util/hash_skiplist_rep.h"
@@ -226,7 +223,6 @@ void SuperVersionUnrefHandle(void* ptr) {
ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
Version* _dummy_versions,
Cache* _table_cache,
WriteBuffer* write_buffer,
const ColumnFamilyOptions& cf_options,
const DBOptions* db_options,
const EnvOptions& env_options,
@@ -241,7 +237,6 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
options_(*db_options, SanitizeOptions(&internal_comparator_, cf_options)),
ioptions_(options_),
mutable_cf_options_(options_, ioptions_),
write_buffer_(write_buffer),
mem_(nullptr),
imm_(options_.min_write_buffer_number_to_merge),
super_version_(nullptr),
@@ -258,13 +253,12 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
internal_stats_.reset(
new InternalStats(ioptions_.num_levels, db_options->env, this));
table_cache_.reset(new TableCache(ioptions_, env_options, _table_cache));
if (ioptions_.compaction_style == kCompactionStyleLevel) {
compaction_picker_.reset(
new LevelCompactionPicker(ioptions_, &internal_comparator_));
#ifndef ROCKSDB_LITE
} else if (ioptions_.compaction_style == kCompactionStyleUniversal) {
if (ioptions_.compaction_style == kCompactionStyleUniversal) {
compaction_picker_.reset(
new UniversalCompactionPicker(ioptions_, &internal_comparator_));
} else if (ioptions_.compaction_style == kCompactionStyleLevel) {
compaction_picker_.reset(
new LevelCompactionPicker(ioptions_, &internal_comparator_));
} else if (ioptions_.compaction_style == kCompactionStyleFIFO) {
compaction_picker_.reset(
new FIFOCompactionPicker(ioptions_, &internal_comparator_));
@@ -275,7 +269,6 @@ ColumnFamilyData::ColumnFamilyData(uint32_t id, const std::string& name,
"Column family %s does not use any background compaction. "
"Compactions can only be done via CompactFiles\n",
GetName().c_str());
#endif // !ROCKSDB_LITE
} else {
Log(InfoLogLevel::ERROR_LEVEL, ioptions_.info_log,
"Unable to recognize the specified compaction style %d. "
@@ -418,19 +411,13 @@ void ColumnFamilyData::SetCurrent(Version* current_version) {
current_ = current_version;
}
MemTable* ColumnFamilyData::ConstructNewMemtable(
const MutableCFOptions& mutable_cf_options) {
assert(current_ != nullptr);
return new MemTable(internal_comparator_, ioptions_,
mutable_cf_options, write_buffer_);
}
void ColumnFamilyData::CreateNewMemtable(
const MutableCFOptions& mutable_cf_options) {
assert(current_ != nullptr);
if (mem_ != nullptr) {
delete mem_->Unref();
}
SetMemtable(ConstructNewMemtable(mutable_cf_options));
mem_ = new MemTable(internal_comparator_, ioptions_, mutable_cf_options);
mem_->Ref();
}
@@ -611,10 +598,9 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
const DBOptions* db_options,
const EnvOptions& env_options,
Cache* table_cache,
WriteBuffer* write_buffer,
WriteController* write_controller)
: max_column_family_(0),
dummy_cfd_(new ColumnFamilyData(0, "", nullptr, nullptr, nullptr,
dummy_cfd_(new ColumnFamilyData(0, "", nullptr, nullptr,
ColumnFamilyOptions(), db_options,
env_options, nullptr)),
default_cfd_cache_(nullptr),
@@ -622,7 +608,6 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
db_options_(db_options),
env_options_(env_options),
table_cache_(table_cache),
write_buffer_(write_buffer),
write_controller_(write_controller),
spin_lock_(ATOMIC_FLAG_INIT) {
// initialize linked list
@@ -687,9 +672,8 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
const ColumnFamilyOptions& options) {
assert(column_families_.find(name) == column_families_.end());
ColumnFamilyData* new_cfd =
new ColumnFamilyData(id, name, dummy_versions, table_cache_,
write_buffer_, options, db_options_,
env_options_, this);
new ColumnFamilyData(id, name, dummy_versions, table_cache_, options,
db_options_, env_options_, this);
Lock();
column_families_.insert({name, id});
column_family_data_.insert({id, new_cfd});
@@ -768,6 +752,11 @@ MemTable* ColumnFamilyMemTablesImpl::GetMemTable() const {
return current_->mem();
}
const Options* ColumnFamilyMemTablesImpl::GetOptions() const {
assert(current_ != nullptr);
return current_->options();
}
ColumnFamilyHandle* ColumnFamilyMemTablesImpl::GetColumnFamilyHandle() {
assert(current_ != nullptr);
return &handle_;
+7 -10
View File
@@ -172,10 +172,8 @@ class ColumnFamilyData {
void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
uint64_t GetLogNumber() const { return log_number_; }
// !!! To be deprecated! Please don't not use this function anymore!
const Options* options() const { return &options_; }
// thread-safe
const Options* options() const { return &options_; }
const EnvOptions* soptions() const;
const ImmutableCFOptions* ioptions() const { return &ioptions_; }
// REQUIRES: DB mutex held
@@ -201,9 +199,8 @@ class ColumnFamilyData {
MemTable* mem() { return mem_; }
Version* current() { return current_; }
Version* dummy_versions() { return dummy_versions_; }
void SetCurrent(Version* current);
MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options);
void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
void SetCurrent(Version* current);
void CreateNewMemtable(const MutableCFOptions& mutable_cf_options);
TableCache* table_cache() const { return table_cache_.get(); }
@@ -265,7 +262,6 @@ class ColumnFamilyData {
friend class ColumnFamilySet;
ColumnFamilyData(uint32_t id, const std::string& name,
Version* dummy_versions, Cache* table_cache,
WriteBuffer* write_buffer,
const ColumnFamilyOptions& options,
const DBOptions* db_options, const EnvOptions& env_options,
ColumnFamilySet* column_family_set);
@@ -296,8 +292,6 @@ class ColumnFamilyData {
std::unique_ptr<InternalStats> internal_stats_;
WriteBuffer* write_buffer_;
MemTable* mem_;
MemTableList imm_;
SuperVersion* super_version_;
@@ -370,7 +364,7 @@ class ColumnFamilySet {
ColumnFamilySet(const std::string& dbname, const DBOptions* db_options,
const EnvOptions& env_options, Cache* table_cache,
WriteBuffer* write_buffer, WriteController* write_controller);
WriteController* write_controller);
~ColumnFamilySet();
ColumnFamilyData* GetDefault() const;
@@ -425,7 +419,6 @@ class ColumnFamilySet {
const DBOptions* const db_options_;
const EnvOptions env_options_;
Cache* table_cache_;
WriteBuffer* write_buffer_;
WriteController* write_controller_;
std::atomic_flag spin_lock_;
};
@@ -450,6 +443,10 @@ class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
// REQUIRES: Seek() called first
virtual MemTable* GetMemTable() const override;
// Returns options for selected column family
// REQUIRES: Seek() called first
virtual const Options* GetOptions() const override;
// Returns column family handle for the selected column family
virtual ColumnFamilyHandle* GetColumnFamilyHandle() override;
+6 -26
View File
@@ -218,7 +218,7 @@ class ColumnFamilyTest {
int NumTableFilesAtLevel(int level, int cf) {
return GetProperty(cf,
"rocksdb.num-files-at-level" + ToString(level));
"rocksdb.num-files-at-level" + std::to_string(level));
}
// Return spread of files per level
@@ -387,7 +387,7 @@ TEST(ColumnFamilyTest, DropTest) {
Open({"default"});
CreateColumnFamiliesAndReopen({"pikachu"});
for (int i = 0; i < 100; ++i) {
ASSERT_OK(Put(1, ToString(i), "bar" + ToString(i)));
ASSERT_OK(Put(1, std::to_string(i), "bar" + std::to_string(i)));
}
ASSERT_OK(Flush(1));
@@ -529,28 +529,8 @@ TEST(ColumnFamilyTest, FlushTest) {
ASSERT_OK(Put(1, "mirko", "v3"));
ASSERT_OK(Put(0, "foo", "v2"));
ASSERT_OK(Put(2, "fodor", "v5"));
for (int j = 0; j < 2; j++) {
ReadOptions ro;
std::vector<Iterator*> iterators;
// Hold super version.
if (j == 0) {
ASSERT_OK(db_->NewIterators(ro, handles_, &iterators));
}
for (int i = 0; i < 3; ++i) {
uint64_t max_total_in_memory_state =
dbfull()->TEST_max_total_in_memory_state();
Flush(i);
ASSERT_EQ(dbfull()->TEST_max_total_in_memory_state(),
max_total_in_memory_state);
}
ASSERT_OK(Put(1, "foofoo", "bar"));
ASSERT_OK(Put(0, "foofoo", "bar"));
for (auto* it : iterators) {
delete it;
}
for (int i = 0; i < 3; ++i) {
Flush(i);
}
Reopen();
@@ -794,14 +774,14 @@ TEST(ColumnFamilyTest, DifferentCompactionStyles) {
for (int i = 0; i < one.level0_file_num_compaction_trigger - 1; ++i) {
PutRandomData(1, 11, 10000);
WaitForFlush(1);
ASSERT_EQ(ToString(i + 1), FilesPerLevel(1));
ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(1));
}
// SETUP column family "two" -- level style with 4 levels
for (int i = 0; i < two.level0_file_num_compaction_trigger - 1; ++i) {
PutRandomData(2, 15, 10000);
WaitForFlush(2);
ASSERT_EQ(ToString(i + 1), FilesPerLevel(2));
ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(2));
}
// TRIGGER compaction "one"
-11
View File
@@ -328,15 +328,4 @@ uint64_t Compaction::OutputFilePreallocationSize(
return preallocation_size * 1.1;
}
Compaction* Compaction::TEST_NewCompaction(
int num_levels, int start_level, int out_level, uint64_t target_file_size,
uint64_t max_grandparent_overlap_bytes, uint32_t output_path_id,
CompressionType output_compression, bool seek_compaction,
bool deletion_compaction) {
return new Compaction(num_levels, start_level, out_level, target_file_size,
max_grandparent_overlap_bytes, output_path_id,
output_compression, seek_compaction,
deletion_compaction);
}
} // namespace rocksdb
-10
View File
@@ -183,16 +183,6 @@ class Compaction {
void SetupBottomMostLevel(VersionStorageInfo* vstorage, bool is_manual,
bool level0_only);
static Compaction* TEST_NewCompaction(
int num_levels, int start_level, int out_level, uint64_t target_file_size,
uint64_t max_grandparent_overlap_bytes, uint32_t output_path_id,
CompressionType output_compression, bool seek_compaction = false,
bool deletion_compaction = false);
CompactionInputFiles* TEST_GetInputFiles(int l) {
return &inputs_[l];
}
private:
friend class CompactionPicker;
friend class UniversalCompactionPicker;
+4 -11
View File
@@ -231,7 +231,6 @@ void CompactionJob::Prepare() {
// Generate file_levels_ for compaction berfore making Iterator
compact_->compaction->GenerateFileLevels();
ColumnFamilyData* cfd = compact_->compaction->column_family_data();
assert(cfd != nullptr);
LogToBuffer(
log_buffer_, "[%s] Compacting %d@%d + %d@%d files, score %.2f",
cfd->GetName().c_str(), compact_->compaction->num_input_files(0),
@@ -385,22 +384,17 @@ Status CompactionJob::Run() {
compact_->MergeKeyValueSliceBuffer(&cfd->internal_comparator());
status = ProcessKeyValueCompaction(&imm_micros, input.get(), true);
if (!status.ok()) {
break;
}
compact_->CleanupBatchBuffer();
compact_->CleanupMergedBuffer();
}
} // done processing all prefix batches
// finish the last batch
if (status.ok()) {
if (compact_->key_str_buf_.size() > 0) {
CallCompactionFilterV2(compaction_filter_v2);
}
compact_->MergeKeyValueSliceBuffer(&cfd->internal_comparator());
status = ProcessKeyValueCompaction(&imm_micros, input.get(), true);
if (compact_->key_str_buf_.size() > 0) {
CallCompactionFilterV2(compaction_filter_v2);
}
compact_->MergeKeyValueSliceBuffer(&cfd->internal_comparator());
status = ProcessKeyValueCompaction(&imm_micros, input.get(), true);
} // checking for compaction filter v2
if (status.ok() &&
@@ -996,7 +990,6 @@ Status CompactionJob::InstallCompactionResults(port::Mutex* db_mutex) {
inline SequenceNumber CompactionJob::findEarliestVisibleSnapshot(
SequenceNumber in, const std::vector<SequenceNumber>& snapshots,
SequenceNumber* prev_snapshot) {
assert(snapshots.size());
SequenceNumber prev __attribute__((unused)) = 0;
for (const auto cur : snapshots) {
assert(prev <= cur);
-182
View File
@@ -1,182 +0,0 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include <map>
#include <string>
#include "db/compaction_job.h"
#include "db/column_family.h"
#include "db/version_set.h"
#include "db/writebuffer.h"
#include "rocksdb/cache.h"
#include "rocksdb/options.h"
#include "rocksdb/db.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "table/mock_table.h"
namespace rocksdb {
// TODO(icanadi) Make it simpler once we mock out VersionSet
class CompactionJobTest {
public:
CompactionJobTest()
: env_(Env::Default()),
dbname_(test::TmpDir() + "/compaction_job_test"),
mutable_cf_options_(Options(), ImmutableCFOptions(Options())),
table_cache_(NewLRUCache(50000, 16, 8)),
write_buffer_(db_options_.db_write_buffer_size),
versions_(new VersionSet(dbname_, &db_options_, env_options_,
table_cache_.get(), &write_buffer_,
&write_controller_)),
shutting_down_(false),
mock_table_factory_(new mock::MockTableFactory()) {
ASSERT_OK(env_->CreateDirIfMissing(dbname_));
db_options_.db_paths.emplace_back(dbname_,
std::numeric_limits<uint64_t>::max());
NewDB();
std::vector<ColumnFamilyDescriptor> column_families;
cf_options_.table_factory = mock_table_factory_;
column_families.emplace_back(kDefaultColumnFamilyName, cf_options_);
ASSERT_OK(versions_->Recover(column_families, false));
}
std::string GenerateFileName(uint64_t file_number) {
FileMetaData meta;
std::vector<DbPath> db_paths;
db_paths.emplace_back(dbname_, std::numeric_limits<uint64_t>::max());
meta.fd = FileDescriptor(file_number, 0, 0);
return TableFileName(db_paths, meta.fd.GetNumber(), meta.fd.GetPathId());
}
// returns expected result after compaction
mock::MockFileContents CreateTwoFiles() {
mock::MockFileContents expected_results;
const int kKeysPerFile = 10000;
SequenceNumber sequence_number = 0;
for (int i = 0; i < 2; ++i) {
mock::MockFileContents contents;
SequenceNumber smallest_seqno = 0, largest_seqno = 0;
InternalKey smallest, largest;
for (int k = 0; k < kKeysPerFile; ++k) {
auto key = ToString(i * (kKeysPerFile / 2) + k);
auto value = ToString(i * kKeysPerFile + k);
InternalKey internal_key(key, ++sequence_number, kTypeValue);
if (k == 0) {
smallest = internal_key;
smallest_seqno = sequence_number;
} else if (k == kKeysPerFile - 1) {
largest = internal_key;
largest_seqno = sequence_number;
}
std::pair<std::string, std::string> key_value(
{internal_key.Encode().ToString(), value});
contents.insert(key_value);
if (i == 1 || k < kKeysPerFile / 2) {
expected_results.insert(key_value);
}
}
uint64_t file_number = versions_->NewFileNumber();
ASSERT_OK(mock_table_factory_->CreateMockTable(
env_, GenerateFileName(file_number), std::move(contents)));
VersionEdit edit;
edit.AddFile(0, file_number, 0, 10, smallest, largest, smallest_seqno,
largest_seqno);
mutex_.Lock();
versions_->LogAndApply(versions_->GetColumnFamilySet()->GetDefault(),
mutable_cf_options_, &edit, &mutex_);
mutex_.Unlock();
}
versions_->SetLastSequence(sequence_number);
return expected_results;
}
void NewDB() {
VersionEdit new_db;
new_db.SetLogNumber(0);
new_db.SetNextFile(2);
new_db.SetLastSequence(0);
const std::string manifest = DescriptorFileName(dbname_, 1);
unique_ptr<WritableFile> file;
Status s = env_->NewWritableFile(
manifest, &file, env_->OptimizeForManifestWrite(env_options_));
ASSERT_OK(s);
{
log::Writer log(std::move(file));
std::string record;
new_db.EncodeTo(&record);
s = log.AddRecord(record);
}
ASSERT_OK(s);
// Make "CURRENT" file that points to the new manifest file.
s = SetCurrentFile(env_, dbname_, 1, nullptr);
}
Env* env_;
std::string dbname_;
EnvOptions env_options_;
MutableCFOptions mutable_cf_options_;
std::shared_ptr<Cache> table_cache_;
WriteController write_controller_;
DBOptions db_options_;
ColumnFamilyOptions cf_options_;
WriteBuffer write_buffer_;
std::unique_ptr<VersionSet> versions_;
port::Mutex mutex_;
std::atomic<bool> shutting_down_;
std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
};
TEST(CompactionJobTest, Simple) {
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
auto expected_results = CreateTwoFiles();
auto files = cfd->current()->storage_info()->LevelFiles(0);
ASSERT_EQ(2U, files.size());
std::unique_ptr<Compaction> compaction(Compaction::TEST_NewCompaction(
7, 0, 1, 1024 * 1024, 10, 0, kNoCompression));
compaction->SetInputVersion(cfd->current());
auto compaction_input_files = compaction->TEST_GetInputFiles(0);
compaction_input_files->level = 0;
compaction_input_files->files.push_back(files[0]);
compaction_input_files->files.push_back(files[1]);
SnapshotList snapshots;
int yield_callback_called = 0;
std::function<uint64_t()> yield_callback = [&]() {
yield_callback_called++;
return 0;
};
LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
mutex_.Lock();
CompactionJob compaction_job(
compaction.get(), db_options_, *cfd->GetLatestMutableCFOptions(),
env_options_, versions_.get(), &shutting_down_, &log_buffer, nullptr,
nullptr, &snapshots, true, table_cache_, std::move(yield_callback));
compaction_job.Prepare();
mutex_.Unlock();
ASSERT_OK(compaction_job.Run());
mutex_.Lock();
Status s;
compaction_job.Install(&s, &mutex_);
ASSERT_OK(s);
mutex_.Unlock();
mock_table_factory_->AssertLatestFile(expected_results);
ASSERT_EQ(yield_callback_called, 20000);
}
} // namespace rocksdb
int main(int argc, char** argv) { return rocksdb::test::RunAllTests(); }
+4 -9
View File
@@ -19,7 +19,6 @@
#include "db/filename.h"
#include "util/log_buffer.h"
#include "util/statistics.h"
#include "util/string_util.h"
namespace rocksdb {
@@ -263,7 +262,7 @@ Status CompactionPicker::GetCompactionInputsFromFileNumbers(
"Cannot find matched SST files for the following file numbers:");
for (auto fn : *input_set) {
message += " ";
message += ToString(fn);
message += std::to_string(fn);
}
return Status::InvalidArgument(message);
}
@@ -387,7 +386,6 @@ Compaction* CompactionPicker::CompactRange(
begin = nullptr;
end = nullptr;
}
vstorage->GetOverlappingInputs(input_level, begin, end, &inputs);
if (inputs.empty()) {
return nullptr;
@@ -618,7 +616,7 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
return Status::InvalidArgument(
"Output level for column family " + cf_meta.name +
" must between [0, " +
ToString(cf_meta.levels[cf_meta.levels.size() - 1].level) +
std::to_string(cf_meta.levels[cf_meta.levels.size() - 1].level) +
"].");
}
@@ -626,7 +624,7 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
return Status::InvalidArgument(
"Exceed the maximum output level defined by "
"the current compaction algorithm --- " +
ToString(MaxOutputLevel()));
std::to_string(MaxOutputLevel()));
}
if (output_level < 0) {
@@ -677,7 +675,7 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
return Status::OK();
}
#endif // !ROCKSDB_LITE
#endif // ROCKSDB_LITE
bool LevelCompactionPicker::NeedsCompaction(const VersionStorageInfo* vstorage)
const {
@@ -843,7 +841,6 @@ Compaction* LevelCompactionPicker::PickCompactionBySize(
return c;
}
#ifndef ROCKSDB_LITE
bool UniversalCompactionPicker::NeedsCompaction(
const VersionStorageInfo* vstorage) const {
const int kLevel0 = 0;
@@ -1327,6 +1324,4 @@ Compaction* FIFOCompactionPicker::CompactRange(
return c;
}
#endif // !ROCKSDB_LITE
} // namespace rocksdb
+29 -31
View File
@@ -169,36 +169,6 @@ class CompactionPicker {
const InternalKeyComparator* const icmp_;
};
class LevelCompactionPicker : public CompactionPicker {
public:
LevelCompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp)
: CompactionPicker(ioptions, icmp) {}
virtual Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
LogBuffer* log_buffer) override;
// Returns current_num_levels - 2, meaning the last level cannot be
// compaction input level.
virtual int MaxInputLevel(int current_num_levels) const override {
return current_num_levels - 2;
}
virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
override;
private:
// For the specfied level, pick a compaction.
// Returns nullptr if there is no compaction to be done.
// If level is 0 and there is already a compaction on that level, this
// function will return nullptr.
Compaction* PickCompactionBySize(const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, int level,
double score);
};
#ifndef ROCKSDB_LITE
class UniversalCompactionPicker : public CompactionPicker {
public:
UniversalCompactionPicker(const ImmutableCFOptions& ioptions,
@@ -240,6 +210,35 @@ class UniversalCompactionPicker : public CompactionPicker {
uint64_t file_size);
};
class LevelCompactionPicker : public CompactionPicker {
public:
LevelCompactionPicker(const ImmutableCFOptions& ioptions,
const InternalKeyComparator* icmp)
: CompactionPicker(ioptions, icmp) {}
virtual Compaction* PickCompaction(const std::string& cf_name,
const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage,
LogBuffer* log_buffer) override;
// Returns current_num_levels - 2, meaning the last level cannot be
// compaction input level.
virtual int MaxInputLevel(int current_num_levels) const override {
return current_num_levels - 2;
}
virtual bool NeedsCompaction(const VersionStorageInfo* vstorage) const
override;
private:
// For the specfied level, pick a compaction.
// Returns nullptr if there is no compaction to be done.
// If level is 0 and there is already a compaction on that level, this
// function will return nullptr.
Compaction* PickCompactionBySize(const MutableCFOptions& mutable_cf_options,
VersionStorageInfo* vstorage, int level,
double score);
};
class FIFOCompactionPicker : public CompactionPicker {
public:
FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
@@ -307,7 +306,6 @@ class NullCompactionPicker : public CompactionPicker {
return false;
}
};
#endif // !ROCKSDB_LITE
// Utility function
extern uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files);
+6 -6
View File
@@ -192,8 +192,8 @@ TEST(CompactionPickerTest, NeedsCompactionLevel) {
// start a brand new version in each test.
NewVersionStorage(kLevels, kCompactionStyleLevel);
for (int i = 0; i < file_count; ++i) {
Add(level, i, ToString((i + 100) * 1000).c_str(),
ToString((i + 100) * 1000 + 999).c_str(),
Add(level, i, std::to_string((i + 100) * 1000).c_str(),
std::to_string((i + 100) * 1000 + 999).c_str(),
file_size, 0, i * 100, i * 100 + 99);
}
UpdateVersionStorageInfo();
@@ -217,8 +217,8 @@ TEST(CompactionPickerTest, NeedsCompactionUniversal) {
// verify the trigger given different number of L0 files.
for (int i = 1;
i <= mutable_cf_options_.level0_file_num_compaction_trigger * 2; ++i) {
Add(0, i, ToString((i + 100) * 1000).c_str(),
ToString((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
Add(0, i, std::to_string((i + 100) * 1000).c_str(),
std::to_string((i + 100) * 1000 + 999).c_str(), 1000000, 0, i * 100,
i * 100 + 99);
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
vstorage_->CompactionScore(0) >= 1);
@@ -243,8 +243,8 @@ TEST(CompactionPickerTest, NeedsCompactionFIFO) {
// size of L0 files.
uint64_t current_size = 0;
for (int i = 1; i <= kFileCount; ++i) {
Add(0, i, ToString((i + 100) * 1000).c_str(),
ToString((i + 100) * 1000 + 999).c_str(),
Add(0, i, std::to_string((i + 100) * 1000).c_str(),
std::to_string((i + 100) * 1000 + 999).c_str(),
kFileSize, 0, i * 100, i * 100 + 99);
current_size += kFileSize;
ASSERT_EQ(level_compaction_picker.NeedsCompaction(vstorage_.get()),
+1 -1
View File
@@ -377,7 +377,7 @@ TEST(ComparatorDBTest, DoubleComparator) {
for (uint32_t j = 0; j < divide_order; j++) {
to_divide *= 10.0;
}
source_strings.push_back(ToString(r / to_divide));
source_strings.push_back(std::to_string(r / to_divide));
}
DoRandomIteraratorTest(GetDB(), source_strings, &rnd, 200, 1000, 66);
+27
View File
@@ -228,6 +228,33 @@ TEST(CuckooTableDBTest, Uint64Comparator) {
ASSERT_EQ("v4", Get(Uint64Key(4)));
}
TEST(CuckooTableDBTest, CompactionTrigger) {
Options options = CurrentOptions();
options.write_buffer_size = 100 << 10; // 100KB
options.level0_file_num_compaction_trigger = 2;
Reopen(&options);
// Write 11 values, each 10016 B
for (int idx = 0; idx < 11; ++idx) {
ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + idx)));
}
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_EQ("1", FilesPerLevel());
// Generate one more file in level-0, and should trigger level-0 compaction
for (int idx = 11; idx < 22; ++idx) {
ASSERT_OK(Put(Key(idx), std::string(10000, 'a' + idx)));
}
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_EQ("2", FilesPerLevel());
dbfull()->TEST_CompactRange(0, nullptr, nullptr);
ASSERT_EQ("0,2", FilesPerLevel());
for (int idx = 0; idx < 22; ++idx) {
ASSERT_EQ(std::string(10000, 'a' + idx), Get(Key(idx)));
}
}
TEST(CuckooTableDBTest, CompactionIntoMultipleFiles) {
// Create a big L0 file and check it compacts into multiple files in L1.
Options options = CurrentOptions();
+10 -18
View File
@@ -198,9 +198,6 @@ DEFINE_bool(enable_numa, false,
"CPU and memory of same node. Use \"$numactl --hardware\" command "
"to see NUMA memory architecture.");
DEFINE_int64(db_write_buffer_size, rocksdb::Options().db_write_buffer_size,
"Number of bytes to buffer in all memtables before compacting");
DEFINE_int64(write_buffer_size, rocksdb::Options().write_buffer_size,
"Number of bytes to buffer in memtable before compacting");
@@ -1196,7 +1193,7 @@ class Benchmark {
while (start < s.size() && isspace(s[start])) {
start++;
}
unsigned int limit = static_cast<unsigned int>(s.size());
unsigned int limit = s.size();
while (limit > start && isspace(s[limit-1])) {
limit--;
}
@@ -1353,7 +1350,7 @@ class Benchmark {
}
std::string GetDbNameForMultiple(std::string base_name, size_t id) {
return base_name + ToString(id);
return base_name + std::to_string(id);
}
std::string ColumnFamilyName(int i) {
@@ -1367,13 +1364,11 @@ class Benchmark {
}
void Run() {
Options open_options; // keep options around to properly destroy db later
if (!SanityCheck()) {
exit(1);
}
PrintHeader();
Open(&open_options);
Open();
const char* benchmarks = FLAGS_benchmarks.c_str();
while (benchmarks != nullptr) {
const char* sep = strchr(benchmarks, ',');
@@ -1534,15 +1529,15 @@ class Benchmark {
delete db_.db;
db_.db = nullptr;
db_.cfh.clear();
DestroyDB(FLAGS_db, open_options);
DestroyDB(FLAGS_db, Options());
}
for (size_t i = 0; i < multi_dbs_.size(); i++) {
delete multi_dbs_[i].db;
DestroyDB(GetDbNameForMultiple(FLAGS_db, i), open_options);
DestroyDB(GetDbNameForMultiple(FLAGS_db, i), Options());
}
multi_dbs_.clear();
}
Open(&open_options); // use open_options for the last accessed
Open();
}
if (method != nullptr) {
@@ -1834,14 +1829,11 @@ class Benchmark {
}
}
void Open(Options* opts) {
Options& options = *opts;
void Open() {
assert(db_.db == nullptr);
Options options;
options.create_if_missing = !FLAGS_use_existing_db;
options.create_missing_column_families = FLAGS_num_column_families > 1;
options.db_write_buffer_size = FLAGS_db_write_buffer_size;
options.write_buffer_size = FLAGS_write_buffer_size;
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
options.min_write_buffer_number_to_merge =
@@ -3066,8 +3058,8 @@ int main(int argc, char** argv) {
dbstats = rocksdb::CreateDBStatistics();
}
std::vector<std::string> fanout = rocksdb::StringSplit(
FLAGS_max_bytes_for_level_multiplier_additional, ',');
std::vector<std::string> fanout =
rocksdb::stringSplit(FLAGS_max_bytes_for_level_multiplier_additional, ',');
for (unsigned int j= 0; j < fanout.size(); j++) {
FLAGS_max_bytes_for_level_multiplier_additional_v.push_back(
std::stoi(fanout[j]));
-4
View File
@@ -19,14 +19,12 @@
#include <stdint.h>
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/job_context.h"
#include "db/version_set.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "port/port.h"
#include "util/mutexlock.h"
#include "util/sync_point.h"
#include "util/file_util.h"
namespace rocksdb {
@@ -69,7 +67,6 @@ Status DBImpl::EnableFileDeletions(bool force) {
if (should_purge_files) {
PurgeObsoleteFiles(job_context);
}
job_context.Clean();
LogFlush(db_options_.info_log);
return Status::OK();
}
@@ -137,7 +134,6 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
return wal_manager_.GetSortedWalFiles(files);
}
}
#endif // ROCKSDB_LITE
+56 -144
View File
@@ -32,7 +32,6 @@
#include "db/db_iter.h"
#include "db/dbformat.h"
#include "db/filename.h"
#include "db/job_context.h"
#include "db/log_reader.h"
#include "db/log_writer.h"
#include "db/memtable.h"
@@ -44,7 +43,6 @@
#include "db/forward_iterator.h"
#include "db/transaction_log_impl.h"
#include "db/version_set.h"
#include "db/writebuffer.h"
#include "db/write_batch_internal.h"
#include "port/port.h"
#include "rocksdb/cache.h"
@@ -76,8 +74,6 @@
#include "util/iostats_context_imp.h"
#include "util/stop_watch.h"
#include "util/sync_point.h"
#include "util/string_util.h"
#include "util/thread_status_impl.h"
namespace rocksdb {
@@ -202,7 +198,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
default_cf_handle_(nullptr),
total_log_size_(0),
max_total_in_memory_state_(0),
write_buffer_(options.db_write_buffer_size),
tmp_batch_(),
bg_schedule_needed_(false),
bg_compaction_scheduled_(0),
@@ -233,8 +228,7 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
db_options_.table_cache_remove_scan_count_limit);
versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
table_cache_.get(), &write_buffer_,
&write_controller_));
table_cache_.get(), &write_controller_));
column_family_memtables_.reset(new ColumnFamilyMemTablesImpl(
versions_->GetColumnFamilySet(), &flush_scheduler_));
@@ -246,7 +240,6 @@ DBImpl::DBImpl(const DBOptions& options, const std::string& dbname)
}
DBImpl::~DBImpl() {
EraseThreadStatusDbInfo();
mutex_.Lock();
if (flush_on_destroy_) {
@@ -294,7 +287,6 @@ DBImpl::~DBImpl() {
if (job_context.HaveSomethingToDelete()) {
PurgeObsoleteFiles(job_context);
}
job_context.Clean();
}
// versions need to be destroyed before table_cache since it can hold
@@ -674,7 +666,6 @@ void DBImpl::DeleteObsoleteFiles() {
if (job_context.HaveSomethingToDelete()) {
PurgeObsoleteFiles(job_context);
}
job_context.Clean();
}
Status DBImpl::Recover(
@@ -1101,25 +1092,24 @@ Status DBImpl::FlushMemTableToOutputFile(
#ifndef ROCKSDB_LITE
if (s.ok()) {
// may temporarily unlock and lock the mutex.
NotifyOnFlushCompleted(cfd, file_number, mutable_cf_options);
NotifyOnFlushCompleted(cfd, file_number);
}
#endif // ROCKSDB_LITE
return s;
}
void DBImpl::NotifyOnFlushCompleted(
ColumnFamilyData* cfd, uint64_t file_number,
const MutableCFOptions& mutable_cf_options) {
ColumnFamilyData* cfd, uint64_t file_number) {
mutex_.AssertHeld();
if (shutting_down_.load(std::memory_order_acquire)) {
return;
}
bool triggered_flush_slowdown =
(cfd->current()->storage_info()->NumLevelFiles(0) >=
mutable_cf_options.level0_slowdown_writes_trigger);
cfd->options()->level0_slowdown_writes_trigger);
bool triggered_flush_stop =
(cfd->current()->storage_info()->NumLevelFiles(0) >=
mutable_cf_options.level0_stop_writes_trigger);
cfd->options()->level0_stop_writes_trigger);
notifying_events_++;
// release lock while notifying events
mutex_.Unlock();
@@ -1353,7 +1343,6 @@ Status DBImpl::CompactFilesImpl(
if (job_context.HaveSomethingToDelete()) {
PurgeObsoleteFiles(job_context);
}
job_context.Clean();
mutex_.Lock();
}
@@ -1819,7 +1808,6 @@ void DBImpl::BackgroundCallFlush() {
if (job_context.HaveSomethingToDelete()) {
PurgeObsoleteFiles(job_context);
}
job_context.Clean();
mutex_.Lock();
}
@@ -1896,7 +1884,6 @@ void DBImpl::BackgroundCallCompaction() {
if (job_context.HaveSomethingToDelete()) {
PurgeObsoleteFiles(job_context);
}
job_context.Clean();
mutex_.Lock();
}
@@ -2070,8 +2057,6 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, JobContext* job_context,
*c->mutable_cf_options());
VersionStorageInfo::LevelSummaryStorage tmp;
c->column_family_data()->internal_stats()->IncBytesMoved(
c->level() + 1, f->fd.GetFileSize());
LogToBuffer(log_buffer, "[%s] Moved #%" PRIu64 " to level-%d %" PRIu64
" bytes %s: %s\n",
c->column_family_data()->GetName().c_str(), f->fd.GetNumber(),
@@ -2205,7 +2190,6 @@ static void CleanupIteratorState(void* arg1, void* arg2) {
if (job_context.HaveSomethingToDelete()) {
state->db->PurgeObsoleteFiles(job_context);
}
job_context.Clean();
}
delete state;
@@ -2272,15 +2256,6 @@ SuperVersion* DBImpl::InstallSuperVersion(
ColumnFamilyData* cfd, SuperVersion* new_sv,
const MutableCFOptions& mutable_cf_options) {
mutex_.AssertHeld();
// Update max_total_in_memory_state_
size_t old_memtable_size = 0;
auto* old_sv = cfd->GetSuperVersion();
if (old_sv) {
old_memtable_size = old_sv->mutable_cf_options.write_buffer_size *
old_sv->mutable_cf_options.max_write_buffer_number;
}
auto* old = cfd->InstallSuperVersion(
new_sv ? new_sv : new SuperVersion(), &mutex_, mutable_cf_options);
@@ -2290,6 +2265,11 @@ SuperVersion* DBImpl::InstallSuperVersion(
MaybeScheduleFlushOrCompaction();
// Update max_total_in_memory_state_
size_t old_memtable_size = 0;
if (old) {
old_memtable_size = old->mutable_cf_options.write_buffer_size *
old->mutable_cf_options.max_write_buffer_number;
}
max_total_in_memory_state_ =
max_total_in_memory_state_ - old_memtable_size +
mutable_cf_options.write_buffer_size *
@@ -2465,50 +2445,40 @@ std::vector<Status> DBImpl::MultiGet(
Status DBImpl::CreateColumnFamily(const ColumnFamilyOptions& cf_options,
const std::string& column_family_name,
ColumnFamilyHandle** handle) {
Status s;
*handle = nullptr;
{
MutexLock l(&mutex_);
MutexLock l(&mutex_);
if (versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name) !=
nullptr) {
return Status::InvalidArgument("Column family already exists");
}
VersionEdit edit;
edit.AddColumnFamily(column_family_name);
uint32_t new_id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID();
edit.SetColumnFamily(new_id);
edit.SetLogNumber(logfile_number_);
edit.SetComparatorName(cf_options.comparator->Name());
if (versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name) !=
nullptr) {
return Status::InvalidArgument("Column family already exists");
}
VersionEdit edit;
edit.AddColumnFamily(column_family_name);
uint32_t new_id = versions_->GetColumnFamilySet()->GetNextColumnFamilyID();
edit.SetColumnFamily(new_id);
edit.SetLogNumber(logfile_number_);
edit.SetComparatorName(cf_options.comparator->Name());
// LogAndApply will both write the creation in MANIFEST and create
// ColumnFamilyData object
Options opt(db_options_, cf_options);
s = versions_->LogAndApply(nullptr,
MutableCFOptions(opt, ImmutableCFOptions(opt)),
&edit, &mutex_, db_directory_.get(), false, &cf_options);
if (s.ok()) {
single_column_family_mode_ = false;
auto* cfd =
versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
assert(cfd != nullptr);
delete InstallSuperVersion(
cfd, nullptr, *cfd->GetLatestMutableCFOptions());
*handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
"Created column family [%s] (ID %u)",
column_family_name.c_str(), (unsigned)cfd->GetID());
} else {
Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
"Creating column family [%s] FAILED -- %s",
column_family_name.c_str(), s.ToString().c_str());
}
} // MutexLock l(&mutex_)
// this is outside the mutex
// LogAndApply will both write the creation in MANIFEST and create
// ColumnFamilyData object
Options opt(db_options_, cf_options);
Status s = versions_->LogAndApply(nullptr,
MutableCFOptions(opt, ImmutableCFOptions(opt)),
&edit, &mutex_, db_directory_.get(), false, &cf_options);
if (s.ok()) {
NewThreadStatusCfInfo(
reinterpret_cast<ColumnFamilyHandleImpl*>(*handle)->cfd());
single_column_family_mode_ = false;
auto cfd =
versions_->GetColumnFamilySet()->GetColumnFamily(column_family_name);
assert(cfd != nullptr);
delete InstallSuperVersion(cfd, nullptr, *cfd->GetLatestMutableCFOptions());
*handle = new ColumnFamilyHandleImpl(cfd, this, &mutex_);
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
"Created column family [%s] (ID %u)",
column_family_name.c_str(), (unsigned)cfd->GetID());
} else {
Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
"Creating column family [%s] FAILED -- %s",
column_family_name.c_str(), s.ToString().c_str());
}
return s;
}
@@ -2542,10 +2512,6 @@ Status DBImpl::DropColumnFamily(ColumnFamilyHandle* column_family) {
}
if (s.ok()) {
// Note that here we erase the associated cf_info of the to-be-dropped
// cfd before its ref-count goes to zero to avoid having to erase cf_info
// later inside db_mutex.
EraseThreadStatusCfInfo(cfd);
assert(cfd->IsDropped());
auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
max_total_in_memory_state_ -= mutable_cf_options->write_buffer_size *
@@ -2722,13 +2688,10 @@ bool DBImpl::IsSnapshotSupported() const {
}
const Snapshot* DBImpl::GetSnapshot() {
int64_t unix_time = 0;
env_->GetCurrentTime(&unix_time); // Ignore error
MutexLock l(&mutex_);
// returns null if the underlying memtable does not support snapshot.
if (!IsSnapshotSupported()) return nullptr;
return snapshots_.New(versions_->LastSequence(), unix_time);
return snapshots_.New(versions_->LastSequence());
}
void DBImpl::ReleaseSnapshot(const Snapshot* s) {
@@ -2835,23 +2798,6 @@ Status DBImpl::Write(const WriteOptions& write_options, WriteBatch* my_batch) {
}
}
MaybeScheduleFlushOrCompaction();
} else if (UNLIKELY(write_buffer_.ShouldFlush())) {
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
"Flushing all column families. Write buffer is using %" PRIu64
" bytes out of a total of %" PRIu64 ".",
write_buffer_.memory_usage(), write_buffer_.buffer_size());
// no need to refcount because drop is happening in write thread, so can't
// happen while we're in the write thread
for (auto cfd : *versions_->GetColumnFamilySet()) {
if (!cfd->mem()->IsEmpty()) {
status = SetNewMemtableAndNewLogFile(cfd, &context);
if (!status.ok()) {
break;
}
cfd->imm()->FlushRequested();
}
}
MaybeScheduleFlushOrCompaction();
}
if (UNLIKELY(status.ok() && !bg_error_.ok())) {
@@ -3059,7 +3005,8 @@ Status DBImpl::SetNewMemtableAndNewLogFile(ColumnFamilyData* cfd,
}
if (s.ok()) {
new_mem = cfd->ConstructNewMemtable(mutable_cf_options);
new_mem = new MemTable(cfd->internal_comparator(), *cfd->ioptions(),
mutable_cf_options);
new_superversion = new SuperVersion();
}
}
@@ -3150,7 +3097,7 @@ bool DBImpl::GetProperty(ColumnFamilyHandle* column_family,
bool ret_value = GetIntPropertyInternal(column_family, property_type,
need_out_of_mutex, &int_value);
if (ret_value) {
*value = ToString(int_value);
*value = std::to_string(int_value);
}
return ret_value;
} else {
@@ -3314,7 +3261,6 @@ Status DBImpl::DeleteFile(std::string name) {
if (!status.ok()) {
Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
"DeleteFile %s failed. File not found\n", name.c_str());
job_context.Clean();
return Status::InvalidArgument("File not found");
}
assert(level < cfd->NumberLevels());
@@ -3323,7 +3269,6 @@ Status DBImpl::DeleteFile(std::string name) {
if (metadata->being_compacted) {
Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
"DeleteFile %s Skipped. File about to be compacted\n", name.c_str());
job_context.Clean();
return Status::OK();
}
@@ -3335,7 +3280,6 @@ Status DBImpl::DeleteFile(std::string name) {
if (vstoreage->NumLevelFiles(i) != 0) {
Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
"DeleteFile %s FAILED. File not in last level\n", name.c_str());
job_context.Clean();
return Status::InvalidArgument("File not in last level");
}
}
@@ -3345,7 +3289,6 @@ Status DBImpl::DeleteFile(std::string name) {
Log(InfoLogLevel::WARN_LEVEL, db_options_.info_log,
"DeleteFile %s failed ---"
" target file in level 0 must be the oldest.", name.c_str());
job_context.Clean();
return Status::InvalidArgument("File in level 0, but not oldest");
}
edit.SetColumnFamily(cfd->GetID());
@@ -3363,7 +3306,6 @@ Status DBImpl::DeleteFile(std::string name) {
if (job_context.HaveSomethingToDelete()) {
PurgeObsoleteFiles(job_context);
}
job_context.Clean();
{
MutexLock l(&mutex_);
// schedule flush if file deletion means we freed the space for flushes to
@@ -3407,8 +3349,8 @@ Status DBImpl::CheckConsistency() {
} else if (fsize != md.size) {
corruption_messages += "Sst file size mismatch: " + file_path +
". Size recorded in manifest " +
ToString(md.size) + ", actual size " +
ToString(fsize) + "\n";
std::to_string(md.size) + ", actual size " +
std::to_string(fsize) + "\n";
}
}
if (corruption_messages.size() == 0) {
@@ -3574,7 +3516,6 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
if (cfd != nullptr) {
handles->push_back(
new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_));
impl->NewThreadStatusCfInfo(cfd);
} else {
if (db_options.create_missing_column_families) {
// missing column family, create it
@@ -3639,8 +3580,17 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
if (s.ok()) {
impl->opened_successfully_ = true;
*dbptr = impl;
// TODO(yhchiang): Add NotifyOnDatabaseOpen() here.
// Since the column-family handles are only available after DB::Open(),
// typically developers will need to pass the returned ColumnFamilyHandles
// to their EventListeners in order to maintain the mapping between
// column-family-name to ColumnFamilyHandle. However, some database
// events might happen before the user passing those ColumnFamilyHandle to
// their Listeners. To address this, we should have NotifyOnDatabaseOpen()
// here which passes the created ColumnFamilyHandle to the Listeners
// as the first event after DB::Open().
} else {
for (auto* h : *handles) {
for (auto h : *handles) {
delete h;
}
handles->clear();
@@ -3739,44 +3689,6 @@ Status DestroyDB(const std::string& dbname, const Options& options) {
return result;
}
#if ROCKSDB_USING_THREAD_STATUS
void DBImpl::NewThreadStatusCfInfo(
ColumnFamilyData* cfd) const {
if (db_options_.enable_thread_tracking) {
ThreadStatusImpl::NewColumnFamilyInfo(
this, GetName(), cfd, cfd->GetName());
}
}
void DBImpl::EraseThreadStatusCfInfo(
ColumnFamilyData* cfd) const {
if (db_options_.enable_thread_tracking) {
ThreadStatusImpl::EraseColumnFamilyInfo(cfd);
}
}
void DBImpl::EraseThreadStatusDbInfo() const {
if (db_options_.enable_thread_tracking) {
ThreadStatusImpl::EraseDatabaseInfo(this);
}
}
Status GetThreadList(std::vector<ThreadStatus>* thread_list) {
return thread_local_status.GetThreadList(thread_list);
}
#else
void DBImpl::NewThreadStatusCfInfo(
ColumnFamilyData* cfd) const {
}
void DBImpl::EraseThreadStatusCfInfo(
ColumnFamilyData* cfd) const {
}
void DBImpl::EraseThreadStatusDbInfo() const {
}
#endif // ROCKSDB_USING_THREAD_STATUS
//
// A global method that can dump out the build version
void DumpRocksDBBuildVersion(Logger * log) {
+2 -19
View File
@@ -24,7 +24,6 @@
#include "db/column_family.h"
#include "db/version_edit.h"
#include "db/wal_manager.h"
#include "db/writebuffer.h"
#include "memtable_list.h"
#include "port/port.h"
#include "rocksdb/db.h"
@@ -35,11 +34,11 @@
#include "util/stop_watch.h"
#include "util/thread_local.h"
#include "util/scoped_arena_iterator.h"
#include "util/hash.h"
#include "db/internal_stats.h"
#include "db/write_controller.h"
#include "db/flush_scheduler.h"
#include "db/write_thread.h"
#include "db/job_context.h"
namespace rocksdb {
@@ -50,7 +49,6 @@ class VersionEdit;
class VersionSet;
class CompactionFilterV2;
class Arena;
struct JobContext;
class DBImpl : public DB {
public:
@@ -230,10 +228,6 @@ class DBImpl : public DB {
// REQUIRES: mutex locked
// pass the pointer that you got from TEST_BeginWrite()
void TEST_EndWrite(void* w);
uint64_t TEST_max_total_in_memory_state() {
return max_total_in_memory_state_;
}
#endif // ROCKSDB_LITE
// Returns the list of live files in 'live' and the list
@@ -252,8 +246,6 @@ class DBImpl : public DB {
ColumnFamilyHandle* DefaultColumnFamily() const;
const SnapshotList& snapshots() const { return snapshots_; }
protected:
Env* const env_;
const std::string dbname_;
@@ -264,14 +256,7 @@ class DBImpl : public DB {
Iterator* NewInternalIterator(const ReadOptions&, ColumnFamilyData* cfd,
SuperVersion* super_version, Arena* arena);
void NotifyOnFlushCompleted(ColumnFamilyData* cfd, uint64_t file_number,
const MutableCFOptions& mutable_cf_options);
void NewThreadStatusCfInfo(ColumnFamilyData* cfd) const;
void EraseThreadStatusCfInfo(ColumnFamilyData* cfd) const;
void EraseThreadStatusDbInfo() const;
void NotifyOnFlushCompleted(ColumnFamilyData* cfd, uint64_t file_number);
private:
friend class DB;
@@ -443,8 +428,6 @@ class DBImpl : public DB {
std::unique_ptr<Directory> db_directory_;
WriteBuffer write_buffer_;
WriteThread write_thread_;
WriteBatch tmp_batch_;
+2 -2
View File
@@ -81,8 +81,8 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
cfd = cfh->cfd();
}
int output_level =
(cfd->ioptions()->compaction_style == kCompactionStyleUniversal ||
cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
(cfd->options()->compaction_style == kCompactionStyleUniversal ||
cfd->options()->compaction_style == kCompactionStyleFIFO)
? level
: level + 1;
return RunManualCompaction(cfd, level, output_level, 0, begin, end);
+2 -23
View File
@@ -3,19 +3,15 @@
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#include "db/db_impl_readonly.h"
#include "utilities/compacted_db/compacted_db_impl.h"
#include "db/db_impl.h"
#include "db/merge_context.h"
#include "db/db_iter.h"
#include "util/perf_context_imp.h"
#include "util/thread_status_impl.h"
namespace rocksdb {
#ifndef ROCKSDB_LITE
DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
const std::string& dbname)
: DBImpl(db_options, dbname) {
@@ -100,10 +96,12 @@ Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
// Try to first open DB as fully compacted DB
Status s;
#ifndef ROCKSDB_LITE
s = CompactedDBImpl::Open(options, dbname, dbptr);
if (s.ok()) {
return s;
}
#endif
DBOptions db_options(options);
ColumnFamilyOptions cf_options(options);
@@ -154,10 +152,6 @@ Status DB::OpenForReadOnly(
impl->mutex_.Unlock();
if (s.ok()) {
*dbptr = impl;
for (auto* h : *handles) {
impl->NewThreadStatusCfInfo(
reinterpret_cast<ColumnFamilyHandleImpl*>(h)->cfd());
}
} else {
for (auto h : *handles) {
delete h;
@@ -168,20 +162,5 @@ Status DB::OpenForReadOnly(
return s;
}
#else // !ROCKSDB_LITE
Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
DB** dbptr, bool error_if_log_file_exist) {
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
}
Status DB::OpenForReadOnly(
const DBOptions& db_options, const std::string& dbname,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
bool error_if_log_file_exist) {
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
}
#endif // !ROCKSDB_LITE
} // namespace rocksdb
-5
View File
@@ -4,9 +4,6 @@
// of patent rights can be found in the PATENTS file in the same directory.
#pragma once
#ifndef ROCKSDB_LITE
#include "db/db_impl.h"
#include <vector>
#include <string>
@@ -103,5 +100,3 @@ class DBImplReadOnly : public DBImpl {
void operator=(const DBImplReadOnly&);
};
}
#endif // !ROCKSDB_LITE
+2 -3
View File
@@ -281,9 +281,8 @@ void DBIter::MergeValuesNewToOld() {
if (!user_merge_operator_) {
Log(InfoLogLevel::ERROR_LEVEL,
logger_, "Options::merge_operator is null.");
status_ = Status::InvalidArgument("user_merge_operator_ must be set.");
valid_ = false;
return;
throw std::logic_error("DBIter::MergeValuesNewToOld() with"
" Options::merge_operator null");
}
// Start the merge process by pushing the first operand
+7 -7
View File
@@ -366,7 +366,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
internal_iter->AddMerge("b", "merge_1");
internal_iter->AddMerge("a", "merge_2");
for (size_t k = 0; k < 200; ++k) {
internal_iter->AddPut("c", ToString(k));
internal_iter->AddPut("c", std::to_string(k));
}
internal_iter->Finish();
@@ -379,7 +379,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "c");
ASSERT_EQ(db_iter->value().ToString(), ToString(i));
ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
@@ -513,11 +513,11 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
internal_iter->AddMerge("b", "merge_1");
internal_iter->AddMerge("a", "merge_2");
for (size_t k = 0; k < 200; ++k) {
internal_iter->AddPut("d", ToString(k));
internal_iter->AddPut("d", std::to_string(k));
}
for (size_t k = 0; k < 200; ++k) {
internal_iter->AddPut("c", ToString(k));
internal_iter->AddPut("c", std::to_string(k));
}
internal_iter->Finish();
@@ -529,7 +529,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
ASSERT_TRUE(db_iter->Valid());
ASSERT_EQ(db_iter->key().ToString(), "d");
ASSERT_EQ(db_iter->value().ToString(), ToString(i));
ASSERT_EQ(db_iter->value().ToString(), std::to_string(i));
db_iter->Prev();
ASSERT_TRUE(db_iter->Valid());
@@ -552,7 +552,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
internal_iter->AddMerge("b", "b");
internal_iter->AddMerge("a", "a");
for (size_t k = 0; k < 200; ++k) {
internal_iter->AddMerge("c", ToString(k));
internal_iter->AddMerge("c", std::to_string(k));
}
internal_iter->Finish();
@@ -566,7 +566,7 @@ TEST(DBIteratorTest, DBIteratorUseSkip) {
ASSERT_EQ(db_iter->key().ToString(), "c");
std::string merge_result = "0";
for (size_t j = 1; j <= i; ++j) {
merge_result += "," + ToString(j);
merge_result += "," + std::to_string(j);
}
ASSERT_EQ(db_iter->value().ToString(), merge_result);
+26 -275
View File
@@ -19,7 +19,6 @@
#include "db/dbformat.h"
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/job_context.h"
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "rocksdb/cache.h"
@@ -33,9 +32,7 @@
#include "rocksdb/table.h"
#include "rocksdb/options.h"
#include "rocksdb/table_properties.h"
#include "rocksdb/thread_status.h"
#include "rocksdb/utilities/write_batch_with_index.h"
#include "rocksdb/utilities/checkpoint.h"
#include "table/block_based_table_factory.h"
#include "table/plain_table_factory.h"
#include "util/hash.h"
@@ -50,8 +47,6 @@
#include "util/sync_point.h"
#include "util/testutil.h"
#include "util/mock_env.h"
#include "util/string_util.h"
#include "util/thread_status_impl.h"
namespace rocksdb {
@@ -173,9 +168,7 @@ class SpecialEnv : public EnvWrapper {
std::function<void()>* table_write_callback_;
int64_t addon_time_;
explicit SpecialEnv(Env* base) : EnvWrapper(base), rnd_(301), addon_time_(0) {
explicit SpecialEnv(Env* base) : EnvWrapper(base), rnd_(301) {
delay_sstable_sync_.store(false, std::memory_order_release);
drop_writes_.store(false, std::memory_order_release);
no_space_.store(false, std::memory_order_release);
@@ -370,14 +363,6 @@ class SpecialEnv : public EnvWrapper {
sleep_counter_.Increment();
target()->SleepForMicroseconds(micros);
}
virtual Status GetCurrentTime(int64_t* unix_time) override {
Status s = target()->GetCurrentTime(unix_time);
if (s.ok()) {
*unix_time += addon_time_;
}
return s;
}
};
class DBTest {
@@ -824,19 +809,6 @@ class DBTest {
return result;
}
uint64_t GetNumSnapshots() {
uint64_t int_num;
ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.num-snapshots", &int_num));
return int_num;
}
uint64_t GetTimeOldestSnapshots() {
uint64_t int_num;
ASSERT_TRUE(
dbfull()->GetIntProperty("rocksdb.oldest-snapshot-time", &int_num));
return int_num;
}
// Return a string that contains all key,value pairs in order,
// formatted like "(k1->v1)(k2->v2)".
std::string Contents(int cf = 0) {
@@ -1518,7 +1490,7 @@ TEST(DBTest, GetPropertiesOfAllTablesTest) {
// Create 4 tables
for (int table = 0; table < 4; ++table) {
for (int i = 0; i < 10 + table; ++i) {
db_->Put(WriteOptions(), ToString(table * 100 + i), "val");
db_->Put(WriteOptions(), std::to_string(table * 100 + i), "val");
}
db_->Flush(FlushOptions());
}
@@ -1532,7 +1504,7 @@ TEST(DBTest, GetPropertiesOfAllTablesTest) {
// fetch key from 1st and 2nd table, which will internally place that table to
// the table cache.
for (int i = 0; i < 2; ++i) {
Get(ToString(i * 100 + 0));
Get(std::to_string(i * 100 + 0));
}
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
@@ -1542,7 +1514,7 @@ TEST(DBTest, GetPropertiesOfAllTablesTest) {
// fetch key from 1st and 2nd table, which will internally place that table to
// the table cache.
for (int i = 0; i < 4; ++i) {
Get(ToString(i * 100 + 0));
Get(std::to_string(i * 100 + 0));
}
VerifyTableProperties(db_, 10 + 11 + 12 + 13);
}
@@ -1634,63 +1606,6 @@ TEST(DBTest, GetSnapshot) {
} while (ChangeOptions(kSkipHashCuckoo));
}
TEST(DBTest, GetSnapshotLink) {
do {
Options options;
const std::string snapshot_name = test::TmpDir(env_) + "/snapshot";
DB* snapshotDB;
ReadOptions roptions;
std::string result;
Checkpoint* checkpoint;
options = CurrentOptions(options);
delete db_;
db_ = nullptr;
ASSERT_OK(DestroyDB(dbname_, options));
ASSERT_OK(DestroyDB(snapshot_name, options));
env_->DeleteDir(snapshot_name);
// Create a database
Status s;
options.create_if_missing = true;
ASSERT_OK(DB::Open(options, dbname_, &db_));
std::string key = std::string("foo");
ASSERT_OK(Put(key, "v1"));
// Take a snapshot
ASSERT_OK(Checkpoint::Create(db_, &checkpoint));
ASSERT_OK(checkpoint->CreateCheckpoint(snapshot_name));
ASSERT_OK(Put(key, "v2"));
ASSERT_EQ("v2", Get(key));
ASSERT_OK(Flush());
ASSERT_EQ("v2", Get(key));
// Open snapshot and verify contents while DB is running
options.create_if_missing = false;
ASSERT_OK(DB::Open(options, snapshot_name, &snapshotDB));
ASSERT_OK(snapshotDB->Get(roptions, key, &result));
ASSERT_EQ("v1", result);
delete snapshotDB;
snapshotDB = nullptr;
delete db_;
db_ = nullptr;
// Destroy original DB
ASSERT_OK(DestroyDB(dbname_, options));
// Open snapshot and verify contents
options.create_if_missing = false;
dbname_ = snapshot_name;
ASSERT_OK(DB::Open(options, dbname_, &db_));
ASSERT_EQ("v1", Get(key));
delete db_;
db_ = nullptr;
ASSERT_OK(DestroyDB(dbname_, options));
delete checkpoint;
// Restore DB name
dbname_ = test::TmpDir(env_) + "/db_test";
} while (ChangeOptions());
}
TEST(DBTest, GetLevel0Ordering) {
do {
CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
@@ -3468,7 +3383,7 @@ class ChangeFilterFactory : public CompactionFilterFactory {
// TODO(kailiu) The tests on UniversalCompaction has some issues:
// 1. A lot of magic numbers ("11" or "12").
// 2. Made assumption on the memtable flush conditions, which may change from
// 2. Made assumption on the memtable flush conidtions, which may change from
// time to time.
TEST(DBTest, UniversalCompactionTrigger) {
Options options;
@@ -3544,7 +3459,7 @@ TEST(DBTest, UniversalCompactionTrigger) {
}
dbfull()->TEST_WaitForCompact();
// Before compaction, we have 4 files at level 0, with size 4, 0.4, 1, 1.
// After compaction, we should have 2 files, with size 4, 2.4.
// After comapction, we should have 2 files, with size 4, 2.4.
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 2);
for (int i = 1; i < options.num_levels ; i++) {
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
@@ -3572,7 +3487,7 @@ TEST(DBTest, UniversalCompactionTrigger) {
}
dbfull()->TEST_WaitForCompact();
// Before compaction, we have 4 files at level 0, with size 4, 2.4, 1, 1.
// After compaction, we should have 3 files, with size 4, 2.4, 2.
// After comapction, we should have 3 files, with size 4, 2.4, 2.
ASSERT_EQ(NumTableFilesAtLevel(0, 1), 3);
for (int i = 1; i < options.num_levels ; i++) {
ASSERT_EQ(NumTableFilesAtLevel(i, 1), 0);
@@ -4771,7 +4686,7 @@ TEST(DBTest, CompactionFilterDeletesAll) {
// put some data
for (int table = 0; table < 4; ++table) {
for (int i = 0; i < 10 + table; ++i) {
Put(ToString(table * 100 + i), "val");
Put(std::to_string(table * 100 + i), "val");
}
Flush();
}
@@ -5452,25 +5367,13 @@ TEST(DBTest, Snapshot) {
CreateAndReopenWithCF({"pikachu"}, CurrentOptions());
Put(0, "foo", "0v1");
Put(1, "foo", "1v1");
const Snapshot* s1 = db_->GetSnapshot();
ASSERT_EQ(1U, GetNumSnapshots());
uint64_t time_snap1 = GetTimeOldestSnapshots();
ASSERT_GT(time_snap1, 0U);
Put(0, "foo", "0v2");
Put(1, "foo", "1v2");
env_->addon_time_++;
const Snapshot* s2 = db_->GetSnapshot();
ASSERT_EQ(2U, GetNumSnapshots());
ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
Put(0, "foo", "0v3");
Put(1, "foo", "1v3");
const Snapshot* s3 = db_->GetSnapshot();
ASSERT_EQ(3U, GetNumSnapshots());
ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
Put(0, "foo", "0v4");
Put(1, "foo", "1v4");
@@ -5484,8 +5387,6 @@ TEST(DBTest, Snapshot) {
ASSERT_EQ("1v4", Get(1, "foo"));
db_->ReleaseSnapshot(s3);
ASSERT_EQ(2U, GetNumSnapshots());
ASSERT_EQ(time_snap1, GetTimeOldestSnapshots());
ASSERT_EQ("0v1", Get(0, "foo", s1));
ASSERT_EQ("1v1", Get(1, "foo", s1));
ASSERT_EQ("0v2", Get(0, "foo", s2));
@@ -5498,11 +5399,8 @@ TEST(DBTest, Snapshot) {
ASSERT_EQ("1v2", Get(1, "foo", s2));
ASSERT_EQ("0v4", Get(0, "foo"));
ASSERT_EQ("1v4", Get(1, "foo"));
ASSERT_EQ(1U, GetNumSnapshots());
ASSERT_LT(time_snap1, GetTimeOldestSnapshots());
db_->ReleaseSnapshot(s2);
ASSERT_EQ(0U, GetNumSnapshots());
ASSERT_EQ("0v4", Get(0, "foo"));
ASSERT_EQ("1v4", Get(1, "foo"));
} while (ChangeOptions(kSkipHashCuckoo));
@@ -6842,86 +6740,6 @@ TEST(DBTest, RecoverCheckFileAmount) {
}
}
TEST(DBTest, SharedWriteBuffer) {
Options options;
options.db_write_buffer_size = 100000; // this is the real limit
options.write_buffer_size = 500000; // this is never hit
CreateAndReopenWithCF({"pikachu", "dobrynia", "nikitich"}, options);
// Trigger a flush on every CF
ASSERT_OK(Put(0, Key(1), DummyString(1)));
ASSERT_OK(Put(1, Key(1), DummyString(1)));
ASSERT_OK(Put(3, Key(1), DummyString(90000)));
ASSERT_OK(Put(2, Key(2), DummyString(20000)));
ASSERT_OK(Put(2, Key(1), DummyString(1)));
dbfull()->TEST_WaitForFlushMemTable(handles_[0]);
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
{
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
static_cast<uint64_t>(1));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
static_cast<uint64_t>(1));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
static_cast<uint64_t>(1));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
static_cast<uint64_t>(1));
}
// Flush 'dobrynia' and 'nikitich'
ASSERT_OK(Put(2, Key(2), DummyString(50000)));
ASSERT_OK(Put(3, Key(2), DummyString(40000)));
ASSERT_OK(Put(2, Key(3), DummyString(20000)));
ASSERT_OK(Put(3, Key(2), DummyString(40000)));
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
{
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
static_cast<uint64_t>(1));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
static_cast<uint64_t>(1));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
static_cast<uint64_t>(2));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
static_cast<uint64_t>(2));
}
// Make 'dobrynia' and 'nikitich' both take up 40% of space
// When 'pikachu' puts us over 100%, all 3 flush.
ASSERT_OK(Put(2, Key(2), DummyString(40000)));
ASSERT_OK(Put(1, Key(2), DummyString(20000)));
ASSERT_OK(Put(0, Key(1), DummyString(1)));
dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
dbfull()->TEST_WaitForFlushMemTable(handles_[3]);
{
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
static_cast<uint64_t>(1));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
static_cast<uint64_t>(2));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
static_cast<uint64_t>(3));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
static_cast<uint64_t>(3));
}
// Some remaining writes so 'default' and 'nikitich' flush on closure.
ASSERT_OK(Put(3, Key(1), DummyString(1)));
ReopenWithColumnFamilies({"default", "pikachu", "dobrynia", "nikitich"},
options);
{
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "default"),
static_cast<uint64_t>(2));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "pikachu"),
static_cast<uint64_t>(2));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "dobrynia"),
static_cast<uint64_t>(3));
ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_, "nikitich"),
static_cast<uint64_t>(4));
}
}
TEST(DBTest, PurgeInfoLogs) {
Options options = CurrentOptions();
options.keep_log_file_num = 5;
@@ -7115,7 +6933,7 @@ TEST(DBTest, TransactionLogIteratorCorruptedLog) {
Options options = OptionsForLogIterTest();
DestroyAndReopen(options);
for (int i = 0; i < 1024; i++) {
Put("key"+ToString(i), DummyString(10));
Put("key"+std::to_string(i), DummyString(10));
}
dbfull()->Flush(FlushOptions());
// Corrupt this log to create a gap
@@ -7183,20 +7001,20 @@ TEST(DBTest, TransactionLogIteratorBlobs) {
struct Handler : public WriteBatch::Handler {
std::string seen;
virtual Status PutCF(uint32_t cf, const Slice& key, const Slice& value) {
seen += "Put(" + ToString(cf) + ", " + key.ToString() + ", " +
ToString(value.size()) + ")";
seen += "Put(" + std::to_string(cf) + ", " + key.ToString() + ", " +
std::to_string(value.size()) + ")";
return Status::OK();
}
virtual Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) {
seen += "Merge(" + ToString(cf) + ", " + key.ToString() + ", " +
ToString(value.size()) + ")";
seen += "Merge(" + std::to_string(cf) + ", " + key.ToString() + ", " +
std::to_string(value.size()) + ")";
return Status::OK();
}
virtual void LogData(const Slice& blob) {
seen += "LogData(" + blob.ToString() + ")";
}
virtual Status DeleteCF(uint32_t cf, const Slice& key) {
seen += "Delete(" + ToString(cf) + ", " + key.ToString() + ")";
seen += "Delete(" + std::to_string(cf) + ", " + key.ToString() + ")";
return Status::OK();
}
} handler;
@@ -7324,7 +7142,7 @@ TEST(DBTest, MultiThreaded) {
do {
std::vector<std::string> cfs;
for (int i = 1; i < kColumnFamilies; ++i) {
cfs.push_back(ToString(i));
cfs.push_back(std::to_string(i));
}
CreateAndReopenWithCF(cfs, CurrentOptions());
// Initialize state
@@ -7377,7 +7195,7 @@ static void GCThreadBody(void* arg) {
WriteOptions wo;
for (int i = 0; i < kGCNumKeys; ++i) {
std::string kv(ToString(i + id * kGCNumKeys));
std::string kv(std::to_string(i + id * kGCNumKeys));
ASSERT_OK(db->Put(wo, kv, kv));
}
t->done = true;
@@ -7413,7 +7231,7 @@ TEST(DBTest, GroupCommitTest) {
std::vector<std::string> expected_db;
for (int i = 0; i < kGCNumThreads * kGCNumKeys; ++i) {
expected_db.push_back(ToString(i));
expected_db.push_back(std::to_string(i));
}
sort(expected_db.begin(), expected_db.end());
@@ -8297,7 +8115,7 @@ TEST(DBTest, FIFOCompactionTest) {
Random rnd(301);
for (int i = 0; i < 6; ++i) {
for (int j = 0; j < 100; ++j) {
ASSERT_OK(Put(ToString(i * 100 + j), RandomString(&rnd, 1024)));
ASSERT_OK(Put(std::to_string(i * 100 + j), RandomString(&rnd, 1024)));
}
// flush should happen here
}
@@ -8310,7 +8128,7 @@ TEST(DBTest, FIFOCompactionTest) {
ASSERT_EQ(NumTableFilesAtLevel(0), 5);
for (int i = 0; i < 50; ++i) {
// these keys should be deleted in previous compaction
ASSERT_EQ("NOT_FOUND", Get(ToString(i)));
ASSERT_EQ("NOT_FOUND", Get(std::to_string(i)));
}
}
}
@@ -8638,7 +8456,7 @@ TEST(DBTest, CompactFilesOnLevelCompaction) {
Random rnd(301);
for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) {
ASSERT_OK(Put(1, ToString(key), RandomString(&rnd, kTestValueSize)));
ASSERT_OK(Put(1, std::to_string(key), RandomString(&rnd, kTestValueSize)));
}
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
dbfull()->TEST_WaitForCompact();
@@ -8670,7 +8488,7 @@ TEST(DBTest, CompactFilesOnLevelCompaction) {
// make sure all key-values are still there.
for (int key = 64 * kEntriesPerBuffer; key >= 0; --key) {
ASSERT_NE(Get(1, ToString(key)), "NOT_FOUND");
ASSERT_NE(Get(1, std::to_string(key)), "NOT_FOUND");
}
}
@@ -8692,7 +8510,7 @@ TEST(DBTest, CompactFilesOnUniversalCompaction) {
ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
Random rnd(301);
for (int key = 1024 * kEntriesPerBuffer; key >= 0; --key) {
ASSERT_OK(Put(1, ToString(key), RandomString(&rnd, kTestValueSize)));
ASSERT_OK(Put(1, std::to_string(key), RandomString(&rnd, kTestValueSize)));
}
dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
dbfull()->TEST_WaitForCompact();
@@ -9104,73 +8922,6 @@ TEST(DBTest, DynamicMemtableOptions) {
sleeping_task_low3.WaitUntilDone();
}
#if ROCKSDB_USING_THREAD_STATUS
TEST(DBTest, GetThreadList) {
Options options;
options.env = env_;
options.enable_thread_tracking = true;
TryReopen(options);
std::vector<ThreadStatus> thread_list;
Status s = GetThreadList(&thread_list);
for (int i = 0; i < 2; ++i) {
// repeat the test with differet number of high / low priority threads
const int kTestCount = 3;
const unsigned int kHighPriCounts[kTestCount] = {3, 2, 5};
const unsigned int kLowPriCounts[kTestCount] = {10, 15, 3};
for (int test = 0; test < kTestCount; ++test) {
// Change the number of threads in high / low priority pool.
env_->SetBackgroundThreads(kHighPriCounts[test], Env::HIGH);
env_->SetBackgroundThreads(kLowPriCounts[test], Env::LOW);
// Wait to ensure the all threads has been registered
env_->SleepForMicroseconds(100000);
s = GetThreadList(&thread_list);
ASSERT_OK(s);
unsigned int thread_type_counts[ThreadStatus::ThreadType::TOTAL];
memset(thread_type_counts, 0, sizeof(thread_type_counts));
for (auto thread : thread_list) {
ASSERT_LT(thread.thread_type, ThreadStatus::ThreadType::TOTAL);
thread_type_counts[thread.thread_type]++;
}
// Verify the total number of threades
ASSERT_EQ(
thread_list.size(),
kHighPriCounts[test] + kLowPriCounts[test]);
// Verify the number of high-priority threads
ASSERT_EQ(
thread_type_counts[ThreadStatus::ThreadType::ROCKSDB_HIGH_PRIORITY],
kHighPriCounts[test]);
// Verify the number of low-priority threads
ASSERT_EQ(
thread_type_counts[ThreadStatus::ThreadType::ROCKSDB_LOW_PRIORITY],
kLowPriCounts[test]);
}
if (i == 0) {
// repeat the test with multiple column families
CreateAndReopenWithCF({"pikachu", "about-to-remove"}, options);
ThreadStatusImpl::TEST_VerifyColumnFamilyInfoMap(handles_, true);
}
}
db_->DropColumnFamily(handles_[2]);
delete handles_[2];
handles_.erase(handles_.begin() + 2);
ThreadStatusImpl::TEST_VerifyColumnFamilyInfoMap(handles_, true);
Close();
ThreadStatusImpl::TEST_VerifyColumnFamilyInfoMap(handles_, true);
}
TEST(DBTest, DisableThreadList) {
Options options;
options.env = env_;
options.enable_thread_tracking = false;
TryReopen(options);
CreateAndReopenWithCF({"pikachu", "about-to-remove"}, options);
// Verify non of the column family info exists
ThreadStatusImpl::TEST_VerifyColumnFamilyInfoMap(handles_, false);
}
#endif // ROCKSDB_USING_THREAD_STATUS
TEST(DBTest, DynamicCompactionOptions) {
// minimum write buffer size is enforced at 64KB
const uint64_t k32KB = 1 << 15;
@@ -9233,7 +8984,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// result in 2 32KB L1 files.
ASSERT_OK(dbfull()->SetOptions({
{"level0_file_num_compaction_trigger", "2"},
{"target_file_size_base", ToString(k32KB) }
{"target_file_size_base", std::to_string(k32KB) }
}));
gen_l0_kb(0, 64, 1);
@@ -9254,7 +9005,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// fill L1 and L2. L1 size should be around 256KB while L2 size should be
// around 256KB x 4.
ASSERT_OK(dbfull()->SetOptions({
{"max_bytes_for_level_base", ToString(k1MB) }
{"max_bytes_for_level_base", std::to_string(k1MB) }
}));
// writing 96 x 64KB => 6 * 1024KB
@@ -9276,7 +9027,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// reduces to 128KB from 256KB which was asserted previously. Same for L2.
ASSERT_OK(dbfull()->SetOptions({
{"max_bytes_for_level_multiplier", "2"},
{"max_bytes_for_level_base", ToString(k128KB) }
{"max_bytes_for_level_base", std::to_string(k128KB) }
}));
// writing 20 x 64KB = 10 x 128KB
@@ -9376,7 +9127,7 @@ TEST(DBTest, DynamicCompactionOptions) {
// L1 - L3. Then thrink max_bytes_for_level_base and disable auto compaction
// at the same time, we should see some level with score greater than 2.
ASSERT_OK(dbfull()->SetOptions({
{"max_bytes_for_level_base", ToString(k1MB) }
{"max_bytes_for_level_base", std::to_string(k1MB) }
}));
// writing 40 x 64KB = 10 x 256KB
// (L1 + L2 + L3) = (1 + 2 + 4) * 256KB
+1 -2
View File
@@ -34,7 +34,6 @@ class DeleteFileTest {
DeleteFileTest() {
db_ = nullptr;
env_ = Env::Default();
options_.enable_thread_tracking = true;
options_.max_background_flushes = 0;
options_.write_buffer_size = 1024*1024*1000;
options_.target_file_size_base = 1024*1024*1000;
@@ -79,7 +78,7 @@ class DeleteFileTest {
options.sync = false;
ReadOptions roptions;
for (int i = startkey; i < (numkeys + startkey) ; i++) {
std::string temp = ToString(i);
std::string temp = std::to_string(i);
Slice key(temp);
Slice value(temp);
ASSERT_OK(db_->Put(options, key, value));
+8 -12
View File
@@ -9,7 +9,6 @@
#include "db/flush_job.h"
#include "db/column_family.h"
#include "db/version_set.h"
#include "db/writebuffer.h"
#include "rocksdb/cache.h"
#include "util/testharness.h"
#include "util/testutil.h"
@@ -26,12 +25,10 @@ class FlushJobTest {
: env_(Env::Default()),
dbname_(test::TmpDir() + "/flush_job_test"),
table_cache_(NewLRUCache(50000, 16, 8)),
write_buffer_(db_options_.db_write_buffer_size),
versions_(new VersionSet(dbname_, &db_options_, env_options_,
table_cache_.get(), &write_buffer_,
&write_controller_)),
table_cache_.get(), &write_controller_)),
shutting_down_(false),
mock_table_factory_(new mock::MockTableFactory()) {
mock_table_factory_(new MockTableFactory()) {
ASSERT_OK(env_->CreateDirIfMissing(dbname_));
db_options_.db_paths.emplace_back(dbname_,
std::numeric_limits<uint64_t>::max());
@@ -72,12 +69,11 @@ class FlushJobTest {
std::shared_ptr<Cache> table_cache_;
WriteController write_controller_;
DBOptions db_options_;
WriteBuffer write_buffer_;
ColumnFamilyOptions cf_options_;
std::unique_ptr<VersionSet> versions_;
port::Mutex mutex_;
std::atomic<bool> shutting_down_;
std::shared_ptr<mock::MockTableFactory> mock_table_factory_;
std::shared_ptr<MockTableFactory> mock_table_factory_;
};
TEST(FlushJobTest, Empty) {
@@ -89,18 +85,19 @@ TEST(FlushJobTest, Empty) {
SequenceNumber(), &job_context, nullptr, nullptr,
kNoCompression, nullptr);
ASSERT_OK(flush_job.Run());
job_context.Clean();
}
TEST(FlushJobTest, NonEmpty) {
JobContext job_context;
auto cfd = versions_->GetColumnFamilySet()->GetDefault();
auto new_mem = cfd->ConstructNewMemtable(*cfd->GetLatestMutableCFOptions());
auto new_mem = new MemTable(cfd->internal_comparator(), *cfd->ioptions(),
*cfd->GetLatestMutableCFOptions());
new_mem->Ref();
std::map<std::string, std::string> inserted_keys;
for (int i = 1; i < 10000; ++i) {
std::string key(ToString(i));
std::string value("value" + ToString(i));
std::string key(std::to_string(i));
std::string value("value" + std::to_string(i));
new_mem->Add(SequenceNumber(i), kTypeValue, key, value);
InternalKey internal_key(key, SequenceNumber(i), kTypeValue);
inserted_keys.insert({internal_key.Encode().ToString(), value});
@@ -116,7 +113,6 @@ TEST(FlushJobTest, NonEmpty) {
ASSERT_OK(flush_job.Run());
mutex_.Unlock();
mock_table_factory_->AssertSingleFile(inserted_keys);
job_context.Clean();
}
} // namespace rocksdb
+1 -1
View File
@@ -119,7 +119,7 @@ ForwardIterator::ForwardIterator(DBImpl* db, const ReadOptions& read_options,
: db_(db),
read_options_(read_options),
cfd_(cfd),
prefix_extractor_(cfd->ioptions()->prefix_extractor),
prefix_extractor_(cfd->options()->prefix_extractor.get()),
user_comparator_(cfd->user_comparator()),
immutable_min_heap_(MinIterComparator(&cfd_->internal_comparator())),
sv_(current_sv),
+30 -38
View File
@@ -15,13 +15,10 @@
#include <inttypes.h>
#include <vector>
#include "db/column_family.h"
#include "db/db_impl.h"
#include "util/string_util.h"
namespace rocksdb {
#ifndef ROCKSDB_LITE
namespace {
const double kMB = 1048576.0;
const double kGB = kMB * 1024;
@@ -31,18 +28,18 @@ void PrintLevelStatsHeader(char* buf, size_t len, const std::string& cf_name) {
buf, len,
"\n** Compaction Stats [%s] **\n"
"Level Files Size(MB) Score Read(GB) Rn(GB) Rnp1(GB) "
"Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) "
"Comp(sec) Comp(cnt) Avg(sec) "
"Write(GB) Wnew(GB) RW-Amp W-Amp Rd(MB/s) Wr(MB/s) Rn(cnt) "
"Rnp1(cnt) Wnp1(cnt) Wnew(cnt) Comp(sec) Comp(cnt) Avg(sec) "
"Stall(sec) Stall(cnt) Avg(ms) RecordIn RecordDrop\n"
"--------------------------------------------------------------------"
"--------------------------------------------------------------------"
"----------------------------------------------------------\n",
"--------------------------------------------------------------------\n",
cf_name.c_str());
}
void PrintLevelStats(char* buf, size_t len, const std::string& name,
int num_files, int being_compacted, double total_file_size, double score,
double w_amp, double stall_us, uint64_t stalls,
double rw_amp, double w_amp, double stall_us, uint64_t stalls,
const InternalStats::CompactionStats& stats) {
uint64_t bytes_read = stats.bytes_readn + stats.bytes_readnp1;
uint64_t bytes_new = stats.bytes_written - stats.bytes_readnp1;
@@ -55,11 +52,15 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
"%8.1f " /* Rnp1(GB) */
"%9.1f " /* Write(GB) */
"%8.1f " /* Wnew(GB) */
"%9.1f " /* Moved(GB) */
"%6.1f " /* RW-Amp */
"%5.1f " /* W-Amp */
"%8.1f " /* Rd(MB/s) */
"%8.1f " /* Wr(MB/s) */
"%9.0f " /* Comp(sec) */
"%8d " /* Rn(cnt) */
"%9d " /* Rnp1(cnt) */
"%9d " /* Wnp1(cnt) */
"%9d " /* Wnew(cnt) */
"%10.0f " /* Comp(sec) */
"%9d " /* Comp(cnt) */
"%8.3f " /* Avg(sec) */
"%10.2f " /* Stall(sec) */
@@ -72,9 +73,10 @@ void PrintLevelStats(char* buf, size_t len, const std::string& name,
name.c_str(), num_files, being_compacted, total_file_size / kMB,
score, bytes_read / kGB, stats.bytes_readn / kGB,
stats.bytes_readnp1 / kGB, stats.bytes_written / kGB,
bytes_new / kGB, stats.bytes_moved / kGB,
w_amp, bytes_read / kMB / elapsed,
stats.bytes_written / kMB / elapsed,
bytes_new / kGB, rw_amp, w_amp, bytes_read / kMB / elapsed,
stats.bytes_written / kMB / elapsed, stats.files_in_leveln,
stats.files_in_levelnp1, stats.files_out_levelnp1,
stats.files_out_levelnp1 - stats.files_in_levelnp1,
stats.micros / 1000000.0, stats.count,
stats.count == 0 ? 0 : stats.micros / 1000000.0 / stats.count,
stall_us / 1000000.0, stalls,
@@ -136,10 +138,6 @@ DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
return kEstimatedUsageByTableReaders;
} else if (in == "is-file-deletions-enabled") {
return kIsFileDeletionEnabled;
} else if (in == "num-snapshots") {
return kNumSnapshots;
} else if (in == "oldest-snapshot-time") {
return kOldestSnapshotTime;
}
return kUnknown;
}
@@ -267,12 +265,6 @@ bool InternalStats::GetIntProperty(DBPropertyType property_type,
cfd_->imm()->current()->GetTotalNumEntries() +
vstorage->GetEstimatedActiveKeys();
return true;
case kNumSnapshots:
*value = db->snapshots().count();
return true;
case kOldestSnapshotTime:
*value = static_cast<uint64_t>(db->snapshots().GetOldestSnapshotTime());
return true;
#ifndef ROCKSDB_LITE
case kIsFileDeletionEnabled:
*value = db->IsFileDeletionsEnabled();
@@ -369,8 +361,8 @@ void InternalStats::DumpCFStats(std::string* value) {
const VersionStorageInfo* vstorage = cfd_->current()->storage_info();
int num_levels_to_check =
(cfd_->ioptions()->compaction_style != kCompactionStyleUniversal &&
cfd_->ioptions()->compaction_style != kCompactionStyleFIFO)
(cfd_->options()->compaction_style != kCompactionStyleUniversal &&
cfd_->options()->compaction_style != kCompactionStyleFIFO)
? vstorage->num_levels() - 1
: 1;
@@ -433,22 +425,29 @@ void InternalStats::DumpCFStats(std::string* value) {
total_slowdown_count_soft += stall_leveln_slowdown_count_soft_[level];
total_slowdown_hard += stall_leveln_slowdown_hard_[level];
total_slowdown_count_hard += stall_leveln_slowdown_count_hard_[level];
int64_t bytes_read = comp_stats_[level].bytes_readn +
comp_stats_[level].bytes_readnp1;
double rw_amp = (comp_stats_[level].bytes_readn == 0) ? 0.0
: (comp_stats_[level].bytes_written + bytes_read) /
static_cast<double>(comp_stats_[level].bytes_readn);
double w_amp = (comp_stats_[level].bytes_readn == 0) ? 0.0
: comp_stats_[level].bytes_written /
static_cast<double>(comp_stats_[level].bytes_readn);
PrintLevelStats(buf, sizeof(buf), "L" + ToString(level), files,
PrintLevelStats(buf, sizeof(buf), "L" + std::to_string(level), files,
files_being_compacted[level],
vstorage->NumLevelBytes(level), compaction_score[level],
w_amp, stall_us, stalls, comp_stats_[level]);
rw_amp, w_amp, stall_us, stalls, comp_stats_[level]);
value->append(buf);
}
}
uint64_t curr_ingest = cf_stats_value_[BYTES_FLUSHED];
// Cumulative summary
double rw_amp = (stats_sum.bytes_written + stats_sum.bytes_readn +
stats_sum.bytes_readnp1) / static_cast<double>(curr_ingest + 1);
double w_amp = stats_sum.bytes_written / static_cast<double>(curr_ingest + 1);
// Stats summary across levels
PrintLevelStats(buf, sizeof(buf), "Sum", total_files,
total_files_being_compacted, total_file_size, 0, w_amp,
total_files_being_compacted, total_file_size, 0, rw_amp, w_amp,
total_stall_us, total_stall_count, stats_sum);
value->append(buf);
// Interval summary
@@ -456,9 +455,12 @@ void InternalStats::DumpCFStats(std::string* value) {
curr_ingest - cf_stats_snapshot_.ingest_bytes + 1;
CompactionStats interval_stats(stats_sum);
interval_stats.Subtract(cf_stats_snapshot_.comp_stats);
rw_amp = (interval_stats.bytes_written +
interval_stats.bytes_readn + interval_stats.bytes_readnp1) /
static_cast<double>(interval_ingest);
w_amp = interval_stats.bytes_written / static_cast<double>(interval_ingest);
PrintLevelStats(buf, sizeof(buf), "Int", 0, 0, 0, 0,
w_amp, total_stall_us - cf_stats_snapshot_.stall_us,
rw_amp, w_amp, total_stall_us - cf_stats_snapshot_.stall_us,
total_stall_count - cf_stats_snapshot_.stall_count, interval_stats);
value->append(buf);
@@ -494,14 +496,4 @@ void InternalStats::DumpCFStats(std::string* value) {
cf_stats_snapshot_.stall_count = total_stall_count;
}
#else
DBPropertyType GetPropertyType(const Slice& property, bool* is_int_property,
bool* need_out_of_mutex) {
return kUnknown;
}
#endif // !ROCKSDB_LITE
} // namespace rocksdb
-88
View File
@@ -46,16 +46,12 @@ enum DBPropertyType : uint32_t {
kEstimatedUsageByTableReaders, // Estimated memory by table readers.
kIsFileDeletionEnabled, // Equals disable_delete_obsolete_files_,
// 0 means file deletions enabled
kNumSnapshots, // Number of snapshots in the system
kOldestSnapshotTime, // Unix timestamp of the first snapshot
};
extern DBPropertyType GetPropertyType(const Slice& property,
bool* is_int_property,
bool* need_out_of_mutex);
#ifndef ROCKSDB_LITE
class InternalStats {
public:
enum InternalCFStatsType {
@@ -121,9 +117,6 @@ class InternalStats {
// Total bytes written during compaction between levels N and N+1
uint64_t bytes_written;
// Total bytes moved to this level
uint64_t bytes_moved;
// Files read from level N during compaction between levels N and N+1
int files_in_leveln;
@@ -148,7 +141,6 @@ class InternalStats {
bytes_readn(0),
bytes_readnp1(0),
bytes_written(0),
bytes_moved(0),
files_in_leveln(0),
files_in_levelnp1(0),
files_out_levelnp1(0),
@@ -161,7 +153,6 @@ class InternalStats {
bytes_readn(c.bytes_readn),
bytes_readnp1(c.bytes_readnp1),
bytes_written(c.bytes_written),
bytes_moved(c.bytes_moved),
files_in_leveln(c.files_in_leveln),
files_in_levelnp1(c.files_in_levelnp1),
files_out_levelnp1(c.files_out_levelnp1),
@@ -174,7 +165,6 @@ class InternalStats {
this->bytes_readn += c.bytes_readn;
this->bytes_readnp1 += c.bytes_readnp1;
this->bytes_written += c.bytes_written;
this->bytes_moved += c.bytes_moved;
this->files_in_leveln += c.files_in_leveln;
this->files_in_levelnp1 += c.files_in_levelnp1;
this->files_out_levelnp1 += c.files_out_levelnp1;
@@ -188,7 +178,6 @@ class InternalStats {
this->bytes_readn -= c.bytes_readn;
this->bytes_readnp1 -= c.bytes_readnp1;
this->bytes_written -= c.bytes_written;
this->bytes_moved -= c.bytes_moved;
this->files_in_leveln -= c.files_in_leveln;
this->files_in_levelnp1 -= c.files_in_levelnp1;
this->files_out_levelnp1 -= c.files_out_levelnp1;
@@ -202,10 +191,6 @@ class InternalStats {
comp_stats_[level].Add(stats);
}
void IncBytesMoved(int level, uint64_t amount) {
comp_stats_[level].bytes_moved += amount;
}
void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {
if (soft) {
stall_leveln_slowdown_soft_[level] += micros;
@@ -311,77 +296,4 @@ class InternalStats {
const uint64_t started_at_;
};
#else
class InternalStats {
public:
enum InternalCFStatsType {
LEVEL0_SLOWDOWN,
MEMTABLE_COMPACTION,
LEVEL0_NUM_FILES,
WRITE_STALLS_ENUM_MAX,
BYTES_FLUSHED,
INTERNAL_CF_STATS_ENUM_MAX,
};
enum InternalDBStatsType {
WAL_FILE_BYTES,
WAL_FILE_SYNCED,
BYTES_WRITTEN,
NUMBER_KEYS_WRITTEN,
WRITE_DONE_BY_OTHER,
WRITE_DONE_BY_SELF,
WRITE_WITH_WAL,
INTERNAL_DB_STATS_ENUM_MAX,
};
InternalStats(int num_levels, Env* env, ColumnFamilyData* cfd) {}
struct CompactionStats {
uint64_t micros;
uint64_t bytes_readn;
uint64_t bytes_readnp1;
uint64_t bytes_written;
uint64_t bytes_moved;
int files_in_leveln;
int files_in_levelnp1;
int files_out_levelnp1;
uint64_t num_input_records;
uint64_t num_dropped_records;
int count;
explicit CompactionStats(int _count = 0) {}
explicit CompactionStats(const CompactionStats& c) {}
void Add(const CompactionStats& c) {}
void Subtract(const CompactionStats& c) {}
};
void AddCompactionStats(int level, const CompactionStats& stats) {}
void IncBytesMoved(int level, uint64_t amount) {}
void RecordLevelNSlowdown(int level, uint64_t micros, bool soft) {}
void AddCFStats(InternalCFStatsType type, uint64_t value) {}
void AddDBStats(InternalDBStatsType type, uint64_t value) {}
uint64_t GetBackgroundErrorCount() const { return 0; }
uint64_t BumpAndGetBackgroundErrorCount() { return 0; }
bool GetStringProperty(DBPropertyType property_type, const Slice& property,
std::string* value) { return false; }
bool GetIntProperty(DBPropertyType property_type, uint64_t* value,
DBImpl* db) const { return false; }
bool GetIntPropertyOutOfMutex(DBPropertyType property_type, Version* version,
uint64_t* value) const { return false; }
};
#endif // !ROCKSDB_LITE
} // namespace rocksdb
+2 -13
View File
@@ -21,8 +21,7 @@ class MemTable;
struct JobContext {
inline bool HaveSomethingToDelete() const {
return candidate_files.size() || sst_delete_files.size() ||
log_delete_files.size() || new_superversion != nullptr ||
superversions_to_free.size() > 0 || memtables_to_free.size() > 0;
log_delete_files.size();
}
// Structure to store information for candidate files to delete.
@@ -74,7 +73,7 @@ struct JobContext {
new_superversion = create_superversion ? new SuperVersion() : nullptr;
}
void Clean() {
~JobContext() {
// free pending memtables
for (auto m : memtables_to_free) {
delete m;
@@ -86,16 +85,6 @@ struct JobContext {
// if new_superversion was not used, it will be non-nullptr and needs
// to be freed here
delete new_superversion;
memtables_to_free.clear();
superversions_to_free.clear();
new_superversion = nullptr;
}
~JobContext() {
assert(memtables_to_free.size() == 0);
assert(superversions_to_free.size() == 0);
assert(new_superversion == nullptr);
}
};
+21 -25
View File
@@ -135,12 +135,10 @@ class EventListenerTest {
}
Status Flush(int cf = 0) {
FlushOptions opt = FlushOptions();
opt.wait = true;
if (cf == 0) {
return db_->Flush(opt);
return db_->Flush(FlushOptions());
} else {
return db_->Flush(opt, handles_[cf]);
return db_->Flush(FlushOptions(), handles_[cf]);
}
}
@@ -175,7 +173,6 @@ class TestFlushListener : public EventListener {
TEST(EventListenerTest, OnSingleDBFlushTest) {
Options options;
options.write_buffer_size = 100000;
TestFlushListener* listener = new TestFlushListener();
options.listeners.emplace_back(listener);
std::vector<std::string> cf_names = {
@@ -183,15 +180,15 @@ TEST(EventListenerTest, OnSingleDBFlushTest) {
"nikitich", "alyosha", "popovich"};
CreateAndReopenWithCF(cf_names, &options);
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
ASSERT_OK(Put(1, "pikachu", "pikachu"));
ASSERT_OK(Put(2, "ilya", "ilya"));
ASSERT_OK(Put(3, "muromec", "muromec"));
ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
ASSERT_OK(Put(5, "nikitich", "nikitich"));
ASSERT_OK(Put(6, "alyosha", "alyosha"));
ASSERT_OK(Put(7, "popovich", "popovich"));
for (size_t i = 1; i < 8; ++i) {
ASSERT_OK(Flush(static_cast<int>(i)));
Flush(static_cast<int>(i));
dbfull()->TEST_WaitForFlushMemTable();
ASSERT_EQ(listener->flushed_dbs_.size(), i);
ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
@@ -206,7 +203,6 @@ TEST(EventListenerTest, OnSingleDBFlushTest) {
TEST(EventListenerTest, MultiCF) {
Options options;
options.write_buffer_size = 100000;
TestFlushListener* listener = new TestFlushListener();
options.listeners.emplace_back(listener);
std::vector<std::string> cf_names = {
@@ -214,15 +210,15 @@ TEST(EventListenerTest, MultiCF) {
"nikitich", "alyosha", "popovich"};
CreateAndReopenWithCF(cf_names, &options);
ASSERT_OK(Put(1, "pikachu", std::string(90000, 'p')));
ASSERT_OK(Put(2, "ilya", std::string(90000, 'i')));
ASSERT_OK(Put(3, "muromec", std::string(90000, 'm')));
ASSERT_OK(Put(4, "dobrynia", std::string(90000, 'd')));
ASSERT_OK(Put(5, "nikitich", std::string(90000, 'n')));
ASSERT_OK(Put(6, "alyosha", std::string(90000, 'a')));
ASSERT_OK(Put(7, "popovich", std::string(90000, 'p')));
ASSERT_OK(Put(1, "pikachu", "pikachu"));
ASSERT_OK(Put(2, "ilya", "ilya"));
ASSERT_OK(Put(3, "muromec", "muromec"));
ASSERT_OK(Put(4, "dobrynia", "dobrynia"));
ASSERT_OK(Put(5, "nikitich", "nikitich"));
ASSERT_OK(Put(6, "alyosha", "alyosha"));
ASSERT_OK(Put(7, "popovich", "popovich"));
for (size_t i = 1; i < 8; ++i) {
ASSERT_OK(Flush(static_cast<int>(i)));
Flush(static_cast<int>(i));
ASSERT_EQ(listener->flushed_dbs_.size(), i);
ASSERT_EQ(listener->flushed_column_family_names_.size(), i);
}
@@ -258,10 +254,10 @@ TEST(EventListenerTest, MultiDBMultiListeners) {
std::vector<std::vector<ColumnFamilyHandle *>> vec_handles;
for (int d = 0; d < kNumDBs; ++d) {
ASSERT_OK(DestroyDB(dbname_ + ToString(d), options));
ASSERT_OK(DestroyDB(dbname_ + std::to_string(d), options));
DB* db;
std::vector<ColumnFamilyHandle*> handles;
ASSERT_OK(DB::Open(options, dbname_ + ToString(d), &db));
ASSERT_OK(DB::Open(options, dbname_ + std::to_string(d), &db));
for (size_t c = 0; c < cf_names.size(); ++c) {
ColumnFamilyHandle* handle;
db->CreateColumnFamily(cf_opts, cf_names[c], &handle);
@@ -331,7 +327,7 @@ TEST(EventListenerTest, DisableBGCompaction) {
db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
// keep writing until writes are forced to stop.
for (int i = 0; static_cast<int>(cf_meta.file_count) < kStopTrigger; ++i) {
Put(1, ToString(i), std::string(100000, 'x'), wopts);
Put(1, std::to_string(i), std::string(100000, 'x'), wopts);
db_->GetColumnFamilyMetaData(handles_[1], &cf_meta);
}
ASSERT_GE(listener->slowdown_count, kStopTrigger - kSlowdownTrigger);
+1 -3
View File
@@ -15,7 +15,6 @@
#include "util/benchharness.h"
#include "db/version_set.h"
#include "db/write_controller.h"
#include "db/writebuffer.h"
#include "util/mutexlock.h"
namespace rocksdb {
@@ -53,10 +52,9 @@ void BM_LogAndApply(int iters, int num_base_files) {
// Notice we are using the default options not through SanitizeOptions().
// We might want to initialize some options manually if needed.
options.db_paths.emplace_back(dbname, 0);
WriteBuffer wb(options.db_write_buffer_size);
// The parameter of table cache is passed in as null, so any file I/O
// operation is likely to fail.
vset = new VersionSet(dbname, &options, sopt, nullptr, &wb, &wc);
vset = new VersionSet(dbname, &options, sopt, nullptr, &wc);
std::vector<ColumnFamilyDescriptor> dummy;
dummy.push_back(ColumnFamilyDescriptor());
ASSERT_OK(vset->Recover(dummy));
+4 -8
View File
@@ -15,7 +15,6 @@
#include "db/dbformat.h"
#include "db/merge_context.h"
#include "db/writebuffer.h"
#include "rocksdb/comparator.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
@@ -53,17 +52,14 @@ MemTableOptions::MemTableOptions(
MemTable::MemTable(const InternalKeyComparator& cmp,
const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options,
WriteBuffer* write_buffer)
const MutableCFOptions& mutable_cf_options)
: comparator_(cmp),
moptions_(ioptions, mutable_cf_options),
refs_(0),
kArenaBlockSize(OptimizeBlockSize(moptions_.arena_block_size)),
arena_(moptions_.arena_block_size),
allocator_(&arena_, write_buffer),
table_(ioptions.memtable_factory->CreateMemTableRep(
comparator_, &allocator_, ioptions.prefix_extractor,
ioptions.info_log)),
comparator_, &arena_, ioptions.prefix_extractor, ioptions.info_log)),
num_entries_(0),
flush_in_progress_(false),
flush_completed_(false),
@@ -80,7 +76,7 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
assert(!should_flush_);
if (prefix_extractor_ && moptions_.memtable_prefix_bloom_bits > 0) {
prefix_bloom_.reset(new DynamicBloom(
&allocator_,
&arena_,
moptions_.memtable_prefix_bloom_bits, ioptions.bloom_locality,
moptions_.memtable_prefix_bloom_probes, nullptr,
moptions_.memtable_prefix_bloom_huge_page_tlb_size,
@@ -183,7 +179,7 @@ Slice MemTableRep::UserKey(const char* key) const {
}
KeyHandle MemTableRep::Allocate(const size_t len, char** buf) {
*buf = allocator_->Allocate(len);
*buf = arena_->Allocate(len);
return static_cast<KeyHandle>(*buf);
}
+5 -9
View File
@@ -19,17 +19,16 @@
#include "rocksdb/db.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/immutable_options.h"
#include "db/memtable_allocator.h"
#include "util/arena.h"
#include "util/dynamic_bloom.h"
#include "util/mutable_cf_options.h"
namespace rocksdb {
class Arena;
class Mutex;
class MemTableIterator;
class MergeContext;
class WriteBuffer;
struct MemTableOptions {
explicit MemTableOptions(
@@ -68,8 +67,7 @@ class MemTable {
// is zero and the caller must call Ref() at least once.
explicit MemTable(const InternalKeyComparator& comparator,
const ImmutableCFOptions& ioptions,
const MutableCFOptions& mutable_cf_options,
WriteBuffer* write_buffer);
const MutableCFOptions& mutable_cf_options);
~MemTable();
@@ -185,10 +183,7 @@ class MemTable {
void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
// Notify the underlying storage that no more items will be added
void MarkImmutable() {
table_->MarkReadOnly();
allocator_.DoneAllocating();
}
void MarkImmutable() { table_->MarkReadOnly(); }
// return true if the current MemTableRep supports merge operator.
bool IsMergeOperatorSupported() const {
@@ -205,6 +200,8 @@ class MemTable {
return comparator_.comparator;
}
const Arena& TEST_GetArena() const { return arena_; }
const MemTableOptions* GetMemTableOptions() const { return &moptions_; }
private:
@@ -220,7 +217,6 @@ class MemTable {
int refs_;
const size_t kArenaBlockSize;
Arena arena_;
MemTableAllocator allocator_;
unique_ptr<MemTableRep> table_;
uint64_t num_entries_;
-52
View File
@@ -1,52 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <assert.h>
#include "db/memtable_allocator.h"
#include "db/writebuffer.h"
#include "util/arena.h"
namespace rocksdb {
MemTableAllocator::MemTableAllocator(Arena* arena, WriteBuffer* write_buffer)
: arena_(arena), write_buffer_(write_buffer), bytes_allocated_(0) {
}
MemTableAllocator::~MemTableAllocator() {
DoneAllocating();
}
char* MemTableAllocator::Allocate(size_t bytes) {
assert(write_buffer_ != nullptr);
bytes_allocated_ += bytes;
write_buffer_->ReserveMem(bytes);
return arena_->Allocate(bytes);
}
char* MemTableAllocator::AllocateAligned(size_t bytes, size_t huge_page_size,
Logger* logger) {
assert(write_buffer_ != nullptr);
bytes_allocated_ += bytes;
write_buffer_->ReserveMem(bytes);
return arena_->AllocateAligned(bytes, huge_page_size, logger);
}
void MemTableAllocator::DoneAllocating() {
if (write_buffer_ != nullptr) {
write_buffer_->FreeMem(bytes_allocated_);
write_buffer_ = nullptr;
}
}
size_t MemTableAllocator::BlockSize() const {
return arena_->BlockSize();
}
} // namespace rocksdb
-47
View File
@@ -1,47 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
//
// This is used by the MemTable to allocate write buffer memory. It connects
// to WriteBuffer so we can track and enforce overall write buffer limits.
#pragma once
#include "util/allocator.h"
namespace rocksdb {
class Arena;
class Logger;
class WriteBuffer;
class MemTableAllocator : public Allocator {
public:
explicit MemTableAllocator(Arena* arena, WriteBuffer* write_buffer);
~MemTableAllocator();
// Allocator interface
char* Allocate(size_t bytes) override;
char* AllocateAligned(size_t bytes, size_t huge_page_size = 0,
Logger* logger = nullptr) override;
size_t BlockSize() const override;
// Call when we're finished allocating memory so we can free it from
// the write buffer's limit.
void DoneAllocating();
private:
Arena* arena_;
WriteBuffer* write_buffer_;
size_t bytes_allocated_;
// No copying allowed
MemTableAllocator(const MemTableAllocator&);
void operator=(const MemTableAllocator&);
};
} // namespace rocksdb
+15 -16
View File
@@ -14,7 +14,6 @@
#include "util/histogram.h"
#include "util/stop_watch.h"
#include "util/testharness.h"
#include "util/string_util.h"
bool FLAGS_random_key = false;
@@ -67,21 +66,21 @@ TEST(PerfContextTest, SeekIntoDeletion) {
ReadOptions read_options;
for (int i = 0; i < FLAGS_total_keys; ++i) {
std::string key = "k" + ToString(i);
std::string value = "v" + ToString(i);
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
db->Put(write_options, key, value);
}
for (int i = 0; i < FLAGS_total_keys -1 ; ++i) {
std::string key = "k" + ToString(i);
std::string key = "k" + std::to_string(i);
db->Delete(write_options, key);
}
HistogramImpl hist_get;
HistogramImpl hist_get_time;
for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
std::string key = "k" + ToString(i);
std::string key = "k" + std::to_string(i);
std::string value;
perf_context.Reset();
@@ -119,7 +118,7 @@ TEST(PerfContextTest, SeekIntoDeletion) {
HistogramImpl hist_seek;
for (int i = 0; i < FLAGS_total_keys; ++i) {
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
std::string key = "k" + ToString(i);
std::string key = "k" + std::to_string(i);
perf_context.Reset();
StopWatchNano timer(Env::Default(), true);
@@ -232,8 +231,8 @@ void ProfileQueries(bool enabled_time = false) {
db->Flush(fo);
continue;
}
std::string key = "k" + ToString(i);
std::string value = "v" + ToString(i);
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
std::vector<std::string> values;
@@ -246,8 +245,8 @@ void ProfileQueries(bool enabled_time = false) {
}
for (const int i : keys) {
std::string key = "k" + ToString(i);
std::string value = "v" + ToString(i);
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
std::vector<Slice> multiget_keys = {Slice(key)};
std::vector<std::string> values;
@@ -336,8 +335,8 @@ void ProfileQueries(bool enabled_time = false) {
hist_mget_num_memtable_checked.Clear();
for (const int i : keys) {
std::string key = "k" + ToString(i);
std::string value = "v" + ToString(i);
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
std::vector<Slice> multiget_keys = {Slice(key)};
std::vector<std::string> values;
@@ -452,8 +451,8 @@ TEST(PerfContextTest, SeekKeyComparison) {
SetPerfLevel(kEnableTime);
StopWatchNano timer(Env::Default());
for (const int i : keys) {
std::string key = "k" + ToString(i);
std::string value = "v" + ToString(i);
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
perf_context.Reset();
timer.Start();
@@ -472,8 +471,8 @@ TEST(PerfContextTest, SeekKeyComparison) {
HistogramImpl hist_next;
for (int i = 0; i < FLAGS_total_keys; ++i) {
std::string key = "k" + ToString(i);
std::string value = "v" + ToString(i);
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
perf_context.Reset();
+4 -4
View File
@@ -628,7 +628,7 @@ TEST(PlainTableDBTest, IteratorLargeKeys) {
};
for (size_t i = 0; i < 7; i++) {
ASSERT_OK(Put(key_list[i], ToString(i)));
ASSERT_OK(Put(key_list[i], std::to_string(i)));
}
dbfull()->TEST_FlushMemTable();
@@ -639,7 +639,7 @@ TEST(PlainTableDBTest, IteratorLargeKeys) {
for (size_t i = 0; i < 7; i++) {
ASSERT_TRUE(iter->Valid());
ASSERT_EQ(key_list[i], iter->key().ToString());
ASSERT_EQ(ToString(i), iter->value().ToString());
ASSERT_EQ(std::to_string(i), iter->value().ToString());
iter->Next();
}
@@ -676,7 +676,7 @@ TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
MakeLongKeyWithPrefix(26, '6')};
for (size_t i = 0; i < 7; i++) {
ASSERT_OK(Put(key_list[i], ToString(i)));
ASSERT_OK(Put(key_list[i], std::to_string(i)));
}
dbfull()->TEST_FlushMemTable();
@@ -687,7 +687,7 @@ TEST(PlainTableDBTest, IteratorLargeKeysWithPrefix) {
for (size_t i = 0; i < 7; i++) {
ASSERT_TRUE(iter->Valid());
ASSERT_EQ(key_list[i], iter->key().ToString());
ASSERT_EQ(ToString(i), iter->value().ToString());
ASSERT_EQ(std::to_string(i), iter->value().ToString());
iter->Next();
}
+1 -1
View File
@@ -442,7 +442,7 @@ TEST(PrefixTest, DynamicPrefixIterator) {
for (auto prefix : prefixes) {
TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
Slice key = TestKeyToSlice(test_key);
std::string value = "v" + ToString(0);
std::string value = "v" + std::to_string(0);
perf_context.Reset();
StopWatchNano timer(Env::Default(), true);
+2 -4
View File
@@ -45,7 +45,6 @@
#include "db/memtable.h"
#include "db/table_cache.h"
#include "db/version_edit.h"
#include "db/writebuffer.h"
#include "db/write_batch_internal.h"
#include "rocksdb/comparator.h"
#include "rocksdb/db.h"
@@ -221,10 +220,9 @@ class Repairer {
std::string scratch;
Slice record;
WriteBatch batch;
WriteBuffer wb(options_.db_write_buffer_size);
MemTable* mem = new MemTable(icmp_, ioptions_,
MutableCFOptions(options_, ioptions_), &wb);
auto cf_mems_default = new ColumnFamilyMemTablesDefault(mem);
MutableCFOptions(options_, ioptions_));
auto cf_mems_default = new ColumnFamilyMemTablesDefault(mem, &options_);
mem->Ref();
int counter = 0;
while (reader.ReadRecord(&record, &scratch)) {
+12 -12
View File
@@ -34,8 +34,9 @@
#include <assert.h>
#include <atomic>
#include <stdlib.h>
#include "util/arena.h"
#include "port/port.h"
#include "util/allocator.h"
#include "util/arena.h"
#include "util/random.h"
namespace rocksdb {
@@ -47,9 +48,9 @@ class SkipList {
public:
// Create a new SkipList object that will use "cmp" for comparing keys,
// and will allocate memory using "*allocator". Objects allocated in the
// allocator must remain allocated for the lifetime of the skiplist object.
explicit SkipList(Comparator cmp, Allocator* allocator,
// and will allocate memory using "*arena". Objects allocated in the arena
// must remain allocated for the lifetime of the skiplist object.
explicit SkipList(Comparator cmp, Arena* arena,
int32_t max_height = 12, int32_t branching_factor = 4);
// Insert key into the list.
@@ -109,7 +110,7 @@ class SkipList {
// Immutable after construction
Comparator const compare_;
Allocator* const allocator_; // Allocator used for allocations of nodes
Arena* const arena_; // Arena used for allocations of nodes
Node* const head_;
@@ -195,7 +196,7 @@ struct SkipList<Key, Comparator>::Node {
template<typename Key, class Comparator>
typename SkipList<Key, Comparator>::Node*
SkipList<Key, Comparator>::NewNode(const Key& key, int height) {
char* mem = allocator_->AllocateAligned(
char* mem = arena_->AllocateAligned(
sizeof(Node) + sizeof(std::atomic<Node*>) * (height - 1));
return new (mem) Node(key);
}
@@ -355,24 +356,23 @@ typename SkipList<Key, Comparator>::Node* SkipList<Key, Comparator>::FindLast()
}
template<typename Key, class Comparator>
SkipList<Key, Comparator>::SkipList(const Comparator cmp, Allocator* allocator,
SkipList<Key, Comparator>::SkipList(const Comparator cmp, Arena* arena,
int32_t max_height,
int32_t branching_factor)
: kMaxHeight_(max_height),
kBranching_(branching_factor),
compare_(cmp),
allocator_(allocator),
arena_(arena),
head_(NewNode(0 /* any key will do */, max_height)),
max_height_(1),
prev_height_(1),
rnd_(0xdeadbeef) {
assert(kMaxHeight_ > 0);
assert(kBranching_ > 0);
// Allocate the prev_ Node* array, directly from the passed-in allocator.
// Allocate the prev_ Node* array, directly from the passed-in arena.
// prev_ does not need to be freed, as its life cycle is tied up with
// the allocator as a whole.
prev_ = reinterpret_cast<Node**>(
allocator_->AllocateAligned(sizeof(Node*) * kMaxHeight_));
// the arena as a whole.
prev_ = (Node**) arena_->AllocateAligned(sizeof(Node*) * kMaxHeight_);
for (int i = 0; i < kMaxHeight_; i++) {
head_->SetNext(i, nullptr);
prev_[i] = head_;
+1 -18
View File
@@ -28,8 +28,6 @@ class SnapshotImpl : public Snapshot {
SnapshotImpl* next_;
SnapshotList* list_; // just for sanity checks
int64_t unix_time_;
};
class SnapshotList {
@@ -38,23 +36,20 @@ class SnapshotList {
list_.prev_ = &list_;
list_.next_ = &list_;
list_.number_ = 0xFFFFFFFFL; // placeholder marker, for debugging
count_ = 0;
}
bool empty() const { return list_.next_ == &list_; }
SnapshotImpl* oldest() const { assert(!empty()); return list_.next_; }
SnapshotImpl* newest() const { assert(!empty()); return list_.prev_; }
const SnapshotImpl* New(SequenceNumber seq, uint64_t unix_time) {
const SnapshotImpl* New(SequenceNumber seq) {
SnapshotImpl* s = new SnapshotImpl;
s->number_ = seq;
s->unix_time_ = unix_time;
s->list_ = this;
s->next_ = &list_;
s->prev_ = list_.prev_;
s->prev_->next_ = s;
s->next_->prev_ = s;
count_++;
return s;
}
@@ -62,7 +57,6 @@ class SnapshotList {
assert(s->list_ == this);
s->prev_->next_ = s->next_;
s->next_->prev_ = s->prev_;
count_--;
delete s;
}
@@ -84,20 +78,9 @@ class SnapshotList {
return newest()->number_;
}
int64_t GetOldestSnapshotTime() const {
if (empty()) {
return 0;
} else {
return oldest()->unix_time_;
}
}
uint64_t count() const { return count_; }
private:
// Dummy head of doubly-linked list of snapshots
SnapshotImpl list_;
uint64_t count_;
};
} // namespace rocksdb
+1 -2
View File
@@ -7,7 +7,6 @@
#include "db/dbformat.h"
#include "util/coding.h"
#include "util/string_util.h"
namespace rocksdb {
@@ -41,7 +40,7 @@ Status InternalKeyPropertiesCollector::Finish(
UserCollectedProperties
InternalKeyPropertiesCollector::GetReadableProperties() const {
return {
{ "kDeletedKeys", ToString(deleted_keys_) }
{ "kDeletedKeys", std::to_string(deleted_keys_) }
};
}
+2 -2
View File
@@ -135,8 +135,8 @@ TEST(VersionBuilderTest, EstimatedActiveKeys) {
const uint32_t kDeletionsPerFile = 100;
for (uint32_t i = 0; i < kNumFiles; ++i) {
Add(static_cast<int>(i / kFilesPerLevel), i + 1,
ToString((i + 100) * 1000).c_str(),
ToString((i + 100) * 1000 + 999).c_str(),
std::to_string((i + 100) * 1000).c_str(),
std::to_string((i + 100) * 1000 + 999).c_str(),
100U, 0, 100, 100,
kEntriesPerFile, kDeletionsPerFile,
(i < kTotalSamples));
+4 -12
View File
@@ -31,7 +31,6 @@
#include "db/table_cache.h"
#include "db/compaction.h"
#include "db/version_builder.h"
#include "db/writebuffer.h"
#include "rocksdb/env.h"
#include "rocksdb/merge_operator.h"
#include "table/table_reader.h"
@@ -1491,11 +1490,9 @@ struct VersionSet::ManifestWriter {
VersionSet::VersionSet(const std::string& dbname, const DBOptions* db_options,
const EnvOptions& storage_options, Cache* table_cache,
WriteBuffer* write_buffer,
WriteController* write_controller)
: column_family_set_(new ColumnFamilySet(
dbname, db_options, storage_options, table_cache,
write_buffer, write_controller)),
dbname, db_options, storage_options, table_cache, write_controller)),
env_(db_options->env),
dbname_(dbname),
db_options_(db_options),
@@ -2218,8 +2215,7 @@ Status VersionSet::ReduceNumberOfLevels(const std::string& dbname,
options->max_open_files - 10, options->table_cache_numshardbits,
options->table_cache_remove_scan_count_limit));
WriteController wc;
WriteBuffer wb(options->db_write_buffer_size);
VersionSet versions(dbname, options, env_options, tc.get(), &wb, &wc);
VersionSet versions(dbname, options, env_options, tc.get(), &wc);
Status status;
std::vector<ColumnFamilyDescriptor> dummy;
@@ -2440,6 +2436,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
v->storage_info()->num_levels() - 1);
cfd->compaction_picker()->SizeBeingCompacted(size_being_compacted);
v->PrepareApply(*cfd->GetLatestMutableCFOptions(), size_being_compacted);
delete builder;
printf("--------------- Column family \"%s\" (ID %u) --------------\n",
cfd->GetName().c_str(), (unsigned int)cfd->GetID());
@@ -2454,11 +2451,6 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
delete v;
}
// Free builders
for (auto& builder : builders) {
delete builder.second;
}
next_file_number_.store(next_file + 1);
last_sequence_ = last_sequence;
prev_log_number_ = previous_log_number;
@@ -2647,7 +2639,7 @@ Iterator* VersionSet::MakeInputIterator(Compaction* c) {
auto cfd = c->column_family_data();
ReadOptions read_options;
read_options.verify_checksums =
c->mutable_cf_options()->verify_checksums_in_compaction;
cfd->options()->verify_checksums_in_compaction;
read_options.fill_cache = false;
// Level-0 files have to be merged together. For other levels,
+1 -2
View File
@@ -50,7 +50,6 @@ class LookupKey;
class MemTable;
class Version;
class VersionSet;
class WriteBuffer;
class MergeContext;
class ColumnFamilyData;
class ColumnFamilySet;
@@ -476,7 +475,7 @@ class VersionSet {
public:
VersionSet(const std::string& dbname, const DBOptions* db_options,
const EnvOptions& env_options, Cache* table_cache,
WriteBuffer* write_buffer, WriteController* write_controller);
WriteController* write_controller);
~VersionSet();
// Apply *edit to the current version to form a new descriptor that
+2 -3
View File
@@ -31,7 +31,6 @@
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/sync_point.h"
#include "util/string_util.h"
namespace rocksdb {
@@ -356,9 +355,9 @@ Status WalManager::ReadFirstRecord(const WalFileType type,
SequenceNumber* sequence) {
if (type != kAliveLogFile && type != kArchivedLogFile) {
Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
"[WalManger] Unknown file type %s", ToString(type).c_str());
"[WalManger] Unknown file type %s", std::to_string(type).c_str());
return Status::NotSupported(
"File Type Not Known " + ToString(type));
"File Type Not Known " + std::to_string(type));
}
{
MutexLock l(&read_first_record_cache_mutex_);
+2 -6
View File
@@ -13,7 +13,6 @@
#include "db/log_writer.h"
#include "db/column_family.h"
#include "db/version_set.h"
#include "db/writebuffer.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "table/mock_table.h"
@@ -29,7 +28,6 @@ class WalManagerTest {
: env_(Env::Default()),
dbname_(test::TmpDir() + "/wal_manager_test"),
table_cache_(NewLRUCache(50000, 16, 8)),
write_buffer_(db_options_.db_write_buffer_size),
current_log_number_(0) {
DestroyDB(dbname_, Options());
}
@@ -42,8 +40,7 @@ class WalManagerTest {
db_options_.wal_dir = dbname_;
versions_.reset(new VersionSet(dbname_, &db_options_, env_options_,
table_cache_.get(), &write_buffer_,
&write_controller_));
table_cache_.get(), &write_controller_));
wal_manager_.reset(new WalManager(db_options_, env_options_));
}
@@ -76,7 +73,7 @@ class WalManagerTest {
for (int i = 1; i <= num_logs; ++i) {
RollTheLog(true);
for (int k = 0; k < entries_per_log; ++k) {
Put(ToString(k), std::string(1024, 'a'));
Put(std::to_string(k), std::string(1024, 'a'));
}
}
}
@@ -96,7 +93,6 @@ class WalManagerTest {
EnvOptions env_options_;
std::shared_ptr<Cache> table_cache_;
DBOptions db_options_;
WriteBuffer write_buffer_;
std::unique_ptr<VersionSet> versions_;
std::unique_ptr<WalManager> wal_manager_;
+14
View File
@@ -48,6 +48,20 @@ WriteBatch::~WriteBatch() { }
WriteBatch::Handler::~Handler() { }
void WriteBatch::Handler::Put(const Slice& key, const Slice& value) {
// you need to either implement Put or PutCF
throw std::runtime_error("Handler::Put not implemented!");
}
void WriteBatch::Handler::Merge(const Slice& key, const Slice& value) {
throw std::runtime_error("Handler::Merge not implemented!");
}
void WriteBatch::Handler::Delete(const Slice& key) {
// you need to either implement Delete or DeleteCF
throw std::runtime_error("Handler::Delete not implemented!");
}
void WriteBatch::Handler::LogData(const Slice& blob) {
// If the user has not specified something to do with blobs, then we ignore
// them.
+9 -2
View File
@@ -26,14 +26,15 @@ class ColumnFamilyMemTables {
// been processed)
virtual uint64_t GetLogNumber() const = 0;
virtual MemTable* GetMemTable() const = 0;
virtual const Options* GetOptions() const = 0;
virtual ColumnFamilyHandle* GetColumnFamilyHandle() = 0;
virtual void CheckMemtableFull() = 0;
};
class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
public:
explicit ColumnFamilyMemTablesDefault(MemTable* mem)
: ok_(false), mem_(mem) {}
ColumnFamilyMemTablesDefault(MemTable* mem, const Options* options)
: ok_(false), mem_(mem), options_(options) {}
bool Seek(uint32_t column_family_id) override {
ok_ = (column_family_id == 0);
@@ -47,6 +48,11 @@ class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
return mem_;
}
const Options* GetOptions() const override {
assert(ok_);
return options_;
}
ColumnFamilyHandle* GetColumnFamilyHandle() override { return nullptr; }
void CheckMemtableFull() override {}
@@ -54,6 +60,7 @@ class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
private:
bool ok_;
MemTable* mem_;
const Options* const options_;
};
// WriteBatchInternal provides static methods for manipulating a
+5 -40
View File
@@ -13,7 +13,6 @@
#include "db/memtable.h"
#include "db/column_family.h"
#include "db/write_batch_internal.h"
#include "db/writebuffer.h"
#include "rocksdb/env.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/utilities/write_batch_with_index.h"
@@ -29,12 +28,11 @@ static std::string PrintContents(WriteBatch* b) {
Options options;
options.memtable_factory = factory;
ImmutableCFOptions ioptions(options);
WriteBuffer wb(options.db_write_buffer_size);
MemTable* mem = new MemTable(cmp, ioptions,
MutableCFOptions(options, ioptions), &wb);
MutableCFOptions(options, ioptions));
mem->Ref();
std::string state;
ColumnFamilyMemTablesDefault cf_mems_default(mem);
ColumnFamilyMemTablesDefault cf_mems_default(mem, &options);
Status s = WriteBatchInternal::InsertInto(b, &cf_mems_default);
int count = 0;
Arena arena;
@@ -157,7 +155,7 @@ namespace {
if (column_family_id == 0) {
seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
} else {
seen += "PutCF(" + ToString(column_family_id) + ", " +
seen += "PutCF(" + std::to_string(column_family_id) + ", " +
key.ToString() + ", " + value.ToString() + ")";
}
return Status::OK();
@@ -167,7 +165,7 @@ namespace {
if (column_family_id == 0) {
seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
} else {
seen += "MergeCF(" + ToString(column_family_id) + ", " +
seen += "MergeCF(" + std::to_string(column_family_id) + ", " +
key.ToString() + ", " + value.ToString() + ")";
}
return Status::OK();
@@ -179,7 +177,7 @@ namespace {
if (column_family_id == 0) {
seen += "Delete(" + key.ToString() + ")";
} else {
seen += "DeleteCF(" + ToString(column_family_id) + ", " +
seen += "DeleteCF(" + std::to_string(column_family_id) + ", " +
key.ToString() + ")";
}
return Status::OK();
@@ -187,39 +185,6 @@ namespace {
};
}
TEST(WriteBatchTest, MergeNotImplemented) {
WriteBatch batch;
batch.Merge(Slice("foo"), Slice("bar"));
ASSERT_EQ(1, batch.Count());
ASSERT_EQ("Merge(foo, bar)@0",
PrintContents(&batch));
WriteBatch::Handler handler;
ASSERT_OK(batch.Iterate(&handler));
}
TEST(WriteBatchTest, PutNotImplemented) {
WriteBatch batch;
batch.Put(Slice("k1"), Slice("v1"));
ASSERT_EQ(1, batch.Count());
ASSERT_EQ("Put(k1, v1)@0",
PrintContents(&batch));
WriteBatch::Handler handler;
ASSERT_OK(batch.Iterate(&handler));
}
TEST(WriteBatchTest, DeleteNotImplemented) {
WriteBatch batch;
batch.Delete(Slice("k2"));
ASSERT_EQ(1, batch.Count());
ASSERT_EQ("Delete(k2)@0",
PrintContents(&batch));
WriteBatch::Handler handler;
ASSERT_OK(batch.Iterate(&handler));
}
TEST(WriteBatchTest, Blob) {
WriteBatch batch;
batch.Put(Slice("k1"), Slice("v1"));
-44
View File
@@ -1,44 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
//
// WriteBuffer is for managing memory allocation for one or more MemTables.
#pragma once
namespace rocksdb {
class WriteBuffer {
public:
explicit WriteBuffer(size_t _buffer_size)
: buffer_size_(_buffer_size), memory_used_(0) {}
~WriteBuffer() {}
size_t memory_usage() const { return memory_used_; }
size_t buffer_size() const { return buffer_size_; }
// Should only be called from write thread
bool ShouldFlush() const {
return buffer_size() > 0 && memory_usage() >= buffer_size();
}
// Should only be called from write thread
void ReserveMem(size_t mem) { memory_used_ += mem; }
void FreeMem(size_t mem) { memory_used_ -= mem; }
private:
const size_t buffer_size_;
size_t memory_used_;
// No copying allowed
WriteBuffer(const WriteBuffer&);
void operator=(const WriteBuffer&);
};
} // namespace rocksdb
+2 -2
View File
@@ -156,14 +156,14 @@ int main() {
// if background compaction is not working, write will stall
// because of options.level0_stop_writes_trigger
for (int i = 1000; i < 99999; ++i) {
db->Put(WriteOptions(), ToString(i),
db->Put(WriteOptions(), std::to_string(i),
std::string(500, 'a' + (i % 26)));
}
// verify the values are still there
std::string value;
for (int i = 1000; i < 99999; ++i) {
db->Get(ReadOptions(), ToString(i),
db->Get(ReadOptions(), std::to_string(i),
&value);
assert(value == std::string(500, 'a' + (i % 26)));
}
+1 -7
View File
@@ -93,8 +93,6 @@ class HdfsEnv : public Env {
virtual Status RenameFile(const std::string& src, const std::string& target);
virtual Status LinkFile(const std::string& src, const std::string& target);
virtual Status LockFile(const std::string& fname, FileLock** lock);
virtual Status UnlockFile(FileLock* lock);
@@ -238,7 +236,7 @@ class HdfsEnv : public Env {
explicit HdfsEnv(const std::string& fsname) {
fprintf(stderr, "You have not build rocksdb with HDFS support\n");
fprintf(stderr, "Please see hdfs/README for details\n");
abort();
throw std::exception();
}
virtual ~HdfsEnv() {
@@ -293,10 +291,6 @@ class HdfsEnv : public Env {
virtual Status RenameFile(const std::string& src, const std::string& target){return notsup;}
virtual Status LinkFile(const std::string& src, const std::string& target) {
return notsup;
}
virtual Status LockFile(const std::string& fname, FileLock** lock){return notsup;}
virtual Status UnlockFile(FileLock* lock){return notsup;}
-22
View File
@@ -77,8 +77,6 @@ typedef struct rocksdb_mergeoperator_t rocksdb_mergeoperator_t;
typedef struct rocksdb_options_t rocksdb_options_t;
typedef struct rocksdb_block_based_table_options_t
rocksdb_block_based_table_options_t;
typedef struct rocksdb_cuckoo_table_options_t
rocksdb_cuckoo_table_options_t;
typedef struct rocksdb_randomfile_t rocksdb_randomfile_t;
typedef struct rocksdb_readoptions_t rocksdb_readoptions_t;
typedef struct rocksdb_seqfile_t rocksdb_seqfile_t;
@@ -378,25 +376,6 @@ extern void rocksdb_block_based_options_set_whole_key_filtering(
extern void rocksdb_options_set_block_based_table_factory(
rocksdb_options_t *opt, rocksdb_block_based_table_options_t* table_options);
/* Cuckoo table options */
extern rocksdb_cuckoo_table_options_t*
rocksdb_cuckoo_options_create();
extern void rocksdb_cuckoo_options_destroy(
rocksdb_cuckoo_table_options_t* options);
extern void rocksdb_cuckoo_options_set_hash_ratio(
rocksdb_cuckoo_table_options_t* options, double v);
extern void rocksdb_cuckoo_options_set_max_search_depth(
rocksdb_cuckoo_table_options_t* options, uint32_t v);
extern void rocksdb_cuckoo_options_set_cuckoo_block_size(
rocksdb_cuckoo_table_options_t* options, uint32_t v);
extern void rocksdb_cuckoo_options_set_identity_as_first_hash(
rocksdb_cuckoo_table_options_t* options, unsigned char v);
extern void rocksdb_cuckoo_options_set_use_module_hash(
rocksdb_cuckoo_table_options_t* options, unsigned char v);
extern void rocksdb_options_set_cuckoo_table_factory(
rocksdb_options_t *opt, rocksdb_cuckoo_table_options_t* table_options);
/* Options */
extern rocksdb_options_t* rocksdb_options_create();
@@ -440,7 +419,6 @@ extern void rocksdb_options_set_info_log(rocksdb_options_t*, rocksdb_logger_t*);
extern void rocksdb_options_set_info_log_level(rocksdb_options_t*, int);
extern void rocksdb_options_set_write_buffer_size(rocksdb_options_t*, size_t);
extern void rocksdb_options_set_max_open_files(rocksdb_options_t*, int);
extern void rocksdb_options_set_max_total_wal_size(rocksdb_options_t* opt, uint64_t n);
extern void rocksdb_options_set_compression_options(
rocksdb_options_t*, int, int, int);
extern void rocksdb_options_set_prefix_extractor(
-13
View File
@@ -22,7 +22,6 @@
#include "rocksdb/types.h"
#include "rocksdb/transaction_log.h"
#include "rocksdb/listener.h"
#include "rocksdb/thread_status.h"
namespace rocksdb {
@@ -101,9 +100,6 @@ class DB {
// that modify data, like put/delete, will return error.
// If the db is opened in read only mode, then no compactions
// will happen.
//
// Not supported in ROCKSDB_LITE, in which case the function will
// return Status::NotSupported.
static Status OpenForReadOnly(const Options& options,
const std::string& name, DB** dbptr,
bool error_if_log_file_exist = false);
@@ -113,9 +109,6 @@ class DB {
// database that should be opened. However, you always need to specify default
// column family. The default column family name is 'default' and it's stored
// in rocksdb::kDefaultColumnFamilyName
//
// Not supported in ROCKSDB_LITE, in which case the function will
// return Status::NotSupported.
static Status OpenForReadOnly(
const DBOptions& db_options, const std::string& name,
const std::vector<ColumnFamilyDescriptor>& column_families,
@@ -548,12 +541,6 @@ Status DestroyDB(const std::string& name, const Options& options);
Status RepairDB(const std::string& dbname, const Options& options);
#endif
#if ROCKSDB_USING_THREAD_STATUS
// Obtain the status of all rocksdb-related threads.
Status GetThreadList(std::vector<ThreadStatus>* thread_list);
#endif
} // namespace rocksdb
#endif // STORAGE_ROCKSDB_INCLUDE_DB_H_
-10
View File
@@ -178,11 +178,6 @@ class Env {
virtual Status RenameFile(const std::string& src,
const std::string& target) = 0;
// Hard Link file src to target.
virtual Status LinkFile(const std::string& src, const std::string& target) {
return Status::NotSupported("LinkFile is not supported for this Env");
}
// Lock the specified file. Used to prevent concurrent access to
// the same db by multiple processes. On failure, stores nullptr in
// *lock and returns non-OK.
@@ -752,11 +747,6 @@ class EnvWrapper : public Env {
Status RenameFile(const std::string& s, const std::string& t) {
return target_->RenameFile(s, t);
}
Status LinkFile(const std::string& s, const std::string& t) {
return target_->LinkFile(s, t);
}
Status LockFile(const std::string& f, FileLock** l) {
return target_->LockFile(f, l);
}
+9 -13
View File
@@ -14,8 +14,8 @@
// (4) Items are never deleted.
// The liberal use of assertions is encouraged to enforce (1).
//
// The factory will be passed an MemTableAllocator object when a new MemTableRep
// is requested.
// The factory will be passed an Arena object when a new MemTableRep is
// requested. The API for this object is in rocksdb/arena.h.
//
// Users can implement their own memtable representations. We include three
// types built in:
@@ -41,7 +41,6 @@
namespace rocksdb {
class Arena;
class MemTableAllocator;
class LookupKey;
class Slice;
class SliceTransform;
@@ -66,7 +65,7 @@ class MemTableRep {
virtual ~KeyComparator() { }
};
explicit MemTableRep(MemTableAllocator* allocator) : allocator_(allocator) {}
explicit MemTableRep(Arena* arena) : arena_(arena) {}
// Allocate a buf of len size for storing key. The idea is that a specific
// memtable representation knows its underlying data structure better. By
@@ -102,7 +101,7 @@ class MemTableRep {
bool (*callback_func)(void* arg, const char* entry));
// Report an approximation of how much memory has been used other than memory
// that was allocated through the allocator.
// that was allocated through the arena.
virtual size_t ApproximateMemoryUsage() = 0;
virtual ~MemTableRep() { }
@@ -151,7 +150,7 @@ class MemTableRep {
// Return an iterator that has a special Seek semantics. The result of
// a Seek might only include keys with the same prefix as the target key.
// arena: If not null, the arena is used to allocate the Iterator.
// arena: If not null, the arena needs to be used to allocate the Iterator.
// When destroying the iterator, the caller will not call "delete"
// but Iterator::~Iterator() directly. The destructor needs to destroy
// all the states but those allocated in arena.
@@ -172,7 +171,7 @@ class MemTableRep {
// user key.
virtual Slice UserKey(const char* key) const;
MemTableAllocator* allocator_;
Arena* arena_;
};
// This is the base class for all factories that are used by RocksDB to create
@@ -181,8 +180,7 @@ class MemTableRepFactory {
public:
virtual ~MemTableRepFactory() {}
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
MemTableAllocator*,
const SliceTransform*,
Arena*, const SliceTransform*,
Logger* logger) = 0;
virtual const char* Name() const = 0;
};
@@ -199,8 +197,7 @@ class SkipListFactory : public MemTableRepFactory {
explicit SkipListFactory(size_t lookahead = 0) : lookahead_(lookahead) {}
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
MemTableAllocator*,
const SliceTransform*,
Arena*, const SliceTransform*,
Logger* logger) override;
virtual const char* Name() const override { return "SkipListFactory"; }
@@ -223,8 +220,7 @@ class VectorRepFactory : public MemTableRepFactory {
public:
explicit VectorRepFactory(size_t count = 0) : count_(count) { }
virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
MemTableAllocator*,
const SliceTransform*,
Arena*, const SliceTransform*,
Logger* logger) override;
virtual const char* Name() const override {
return "VectorRepFactory";
+8 -43
View File
@@ -56,18 +56,11 @@ enum CompressionType : char {
};
enum CompactionStyle : char {
// level based compaction style
kCompactionStyleLevel = 0x0,
// Universal compaction style
// Not supported in ROCKSDB_LITE.
kCompactionStyleUniversal = 0x1,
// FIFO compaction style
// Not supported in ROCKSDB_LITE
kCompactionStyleFIFO = 0x2,
// Disable background compaction. Compaction jobs are submitted
// via CompactFiles().
// Not supported in ROCKSDB_LITE
kCompactionStyleNone = 0x3,
kCompactionStyleLevel = 0x0, // level based compaction style
kCompactionStyleUniversal = 0x1, // Universal compaction style
kCompactionStyleFIFO = 0x2, // FIFO compaction style
kCompactionStyleNone = 0x3, // Disable background compaction. Compaction
// jobs are submitted via CompactFiles()
};
struct CompactionOptionsFIFO {
@@ -108,10 +101,9 @@ struct Options;
struct ColumnFamilyOptions {
// Some functions that make it easier to optimize RocksDB
#ifndef ROCKSDB_LITE
// Use this if you don't need to keep the data sorted, i.e. you'll never use
// an iterator, only Put() and Get() API calls
//
// Not supported in ROCKSDB_LITE
ColumnFamilyOptions* OptimizeForPointLookup(
uint64_t block_cache_size_mb);
@@ -129,12 +121,11 @@ struct ColumnFamilyOptions {
// biggest performance gains.
// Note: we might use more memory than memtable_memory_budget during high
// write rate period
//
// OptimizeUniversalStyleCompaction is not supported in ROCKSDB_LITE
ColumnFamilyOptions* OptimizeLevelStyleCompaction(
uint64_t memtable_memory_budget = 512 * 1024 * 1024);
ColumnFamilyOptions* OptimizeUniversalStyleCompaction(
uint64_t memtable_memory_budget = 512 * 1024 * 1024);
#endif // ROCKSDB_LITE
// -------------------
// Parameters that affect behavior
@@ -205,9 +196,6 @@ struct ColumnFamilyOptions {
// Also, a larger write buffer will result in a longer recovery time
// the next time the database is opened.
//
// Note that write_buffer_size is enforced per column family.
// See db_write_buffer_size for sharing memory across column families.
//
// Default: 4MB
//
// Dynamically changeable through SetOptions() API
@@ -322,7 +310,6 @@ struct ColumnFamilyOptions {
//
// Dynamically changeable through SetOptions() API
uint64_t target_file_size_base;
// By default target_file_size_multiplier is 1, which means
// by default files in different levels will have similar size.
//
@@ -342,7 +329,6 @@ struct ColumnFamilyOptions {
//
// Dynamically changeable through SetOptions() API
uint64_t max_bytes_for_level_base;
// Default: 10.
//
// Dynamically changeable through SetOptions() API
@@ -435,10 +421,7 @@ struct ColumnFamilyOptions {
// If true, compaction will verify checksum on every read that happens
// as part of compaction
//
// Default: true
//
// Dynamically changeable through SetOptions() API
bool verify_checksums_in_compaction;
// The options needed to support Universal Style compactions
@@ -686,7 +669,7 @@ struct DBOptions {
// column families whose memtables are backed by the oldest live WAL file
// (i.e. the ones that are causing all the space amplification). If set to 0
// (default), we will dynamically choose the WAL size limit to be
// [sum of all write_buffer_size * max_write_buffer_number] * 4
// [sum of all write_buffer_size * max_write_buffer_number] * 2
// Default: 0
uint64_t max_total_wal_size;
@@ -864,18 +847,6 @@ struct DBOptions {
// Default: true
bool advise_random_on_open;
// Amount of data to build up in memtables across all column
// families before writing to disk.
//
// This is distinct from write_buffer_size, which enforces a limit
// for a single memtable.
//
// This feature is disabled by default. Specify a non-zero value
// to enable it.
//
// Default: 0 (disabled)
size_t db_write_buffer_size;
// Specify the file access pattern once a compaction is started.
// It will be applied to all input files of a compaction.
// Default: NORMAL
@@ -910,12 +881,6 @@ struct DBOptions {
// When rate limiter is enabled, it automatically enables bytes_per_sync
// to 1MB.
uint64_t bytes_per_sync;
// If true, then the status of the threads involved in this DB will
// be tracked and available via GetThreadList() API.
//
// Default: false
bool enable_thread_tracking;
};
// Options to control the behavior of a database (passed to DB::Open)
+2 -3
View File
@@ -369,14 +369,13 @@ class TableFactory {
};
#ifndef ROCKSDB_LITE
// Create a special table factory that can open either of the supported
// table formats, based on setting inside the SST files. It should be used to
// Create a special table factory that can open both of block based table format
// and plain table, based on setting inside the SST files. It should be used to
// convert a DB from one table format to another.
// @table_factory_to_write: the table factory used when writing to new files.
// @block_based_table_factory: block based table factory to use. If NULL, use
// a default one.
// @plain_table_factory: plain table factory to use. If NULL, use a default one.
// @cuckoo_table_factory: cuckoo table factory to use. If NULL, use a default one.
extern TableFactory* NewAdaptiveTableFactory(
std::shared_ptr<TableFactory> table_factory_to_write = nullptr,
std::shared_ptr<TableFactory> block_based_table_factory = nullptr,
-67
View File
@@ -1,67 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
#pragma once
#include <cstddef>
#include <string>
#ifndef ROCKSDB_USING_THREAD_STATUS
#define ROCKSDB_USING_THREAD_STATUS \
!defined(ROCKSDB_LITE) && \
!defined(NROCKSDB_THREAD_STATUS) && \
!defined(OS_MACOSX) && \
!defined(IOS_CROSS_COMPILE)
#endif
namespace rocksdb {
// A structure that describes the current status of a thread.
// The status of active threads can be fetched using
// rocksdb::GetThreadList().
struct ThreadStatus {
enum ThreadType {
ROCKSDB_HIGH_PRIORITY = 0x0,
ROCKSDB_LOW_PRIORITY = 0x1,
USER_THREAD = 0x2,
TOTAL = 0x3
};
#if ROCKSDB_USING_THREAD_STATUS
ThreadStatus(const uint64_t _id,
const ThreadType _thread_type,
const std::string& _db_name,
const std::string& _cf_name,
const std::string& _event) :
thread_id(_id), thread_type(_thread_type),
db_name(_db_name),
cf_name(_cf_name),
event(_event) {}
// An unique ID for the thread.
const uint64_t thread_id;
// The type of the thread, it could be ROCKSDB_HIGH_PRIORITY,
// ROCKSDB_LOW_PRIORITY, and USER_THREAD
const ThreadType thread_type;
// The name of the DB instance where the thread is currently
// involved with. It would be set to empty string if the thread
// does not involve in any DB operation.
const std::string db_name;
// The name of the column family where the thread is currently
// It would be set to empty string if the thread does not involve
// in any column family.
const std::string cf_name;
// The event that the current thread is involved.
// It would be set to empty string if the information about event
// is not currently available.
const std::string event;
#endif // ROCKSDB_USING_THREAD_STATUS
};
} // namespace rocksdb
-34
View File
@@ -1,34 +0,0 @@
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// A checkpoint is an openable snapshot of a database at a point in time.
#pragma once
#include <string>
#include "rocksdb/status.h"
namespace rocksdb {
class DB;
class Checkpoint {
public:
// Creates a Checkpoint object to be used for creating openable sbapshots
static Status Create(DB* db, Checkpoint** checkpoint_ptr);
// Builds an openable snapshot of RocksDB on the same disk, which
// accepts an output directory on the same disk, and under the directory
// (1) hard-linked SST files pointing to existing live SST files
// SST files will be copied if output directory is on a different filesystem
// (2) a copied manifest files and other files
// The directory should not already exist and will be created by this API.
// The directory will be an absolute path
virtual Status CreateCheckpoint(const std::string& checkpoint_dir);
virtual ~Checkpoint() {}
};
} // namespace rocksdb
-144
View File
@@ -1,144 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#pragma once
#include <stddef.h>
namespace rocksdb {
class Cache;
class Comparator;
class Env;
class FilterPolicy;
class Logger;
struct Options;
class Snapshot;
enum CompressionType : char;
// Options to control the behavior of a database (passed to
// DB::Open). A LevelDBOptions object can be initialized as though
// it were a LevelDB Options object, and then it can be converted into
// a RocksDB Options object.
struct LevelDBOptions {
// -------------------
// Parameters that affect behavior
// Comparator used to define the order of keys in the table.
// Default: a comparator that uses lexicographic byte-wise ordering
//
// REQUIRES: The client must ensure that the comparator supplied
// here has the same name and orders keys *exactly* the same as the
// comparator provided to previous open calls on the same DB.
const Comparator* comparator;
// If true, the database will be created if it is missing.
// Default: false
bool create_if_missing;
// If true, an error is raised if the database already exists.
// Default: false
bool error_if_exists;
// If true, the implementation will do aggressive checking of the
// data it is processing and will stop early if it detects any
// errors. This may have unforeseen ramifications: for example, a
// corruption of one DB entry may cause a large number of entries to
// become unreadable or for the entire DB to become unopenable.
// Default: false
bool paranoid_checks;
// Use the specified object to interact with the environment,
// e.g. to read/write files, schedule background work, etc.
// Default: Env::Default()
Env* env;
// Any internal progress/error information generated by the db will
// be written to info_log if it is non-NULL, or to a file stored
// in the same directory as the DB contents if info_log is NULL.
// Default: NULL
Logger* info_log;
// -------------------
// Parameters that affect performance
// Amount of data to build up in memory (backed by an unsorted log
// on disk) before converting to a sorted on-disk file.
//
// Larger values increase performance, especially during bulk loads.
// Up to two write buffers may be held in memory at the same time,
// so you may wish to adjust this parameter to control memory usage.
// Also, a larger write buffer will result in a longer recovery time
// the next time the database is opened.
//
// Default: 4MB
size_t write_buffer_size;
// Number of open files that can be used by the DB. You may need to
// increase this if your database has a large working set (budget
// one open file per 2MB of working set).
//
// Default: 1000
int max_open_files;
// Control over blocks (user data is stored in a set of blocks, and
// a block is the unit of reading from disk).
// If non-NULL, use the specified cache for blocks.
// If NULL, leveldb will automatically create and use an 8MB internal cache.
// Default: NULL
Cache* block_cache;
// Approximate size of user data packed per block. Note that the
// block size specified here corresponds to uncompressed data. The
// actual size of the unit read from disk may be smaller if
// compression is enabled. This parameter can be changed dynamically.
//
// Default: 4K
size_t block_size;
// Number of keys between restart points for delta encoding of keys.
// This parameter can be changed dynamically. Most clients should
// leave this parameter alone.
//
// Default: 16
int block_restart_interval;
// Compress blocks using the specified compression algorithm. This
// parameter can be changed dynamically.
//
// Default: kSnappyCompression, which gives lightweight but fast
// compression.
//
// Typical speeds of kSnappyCompression on an Intel(R) Core(TM)2 2.4GHz:
// ~200-500MB/s compression
// ~400-800MB/s decompression
// Note that these speeds are significantly faster than most
// persistent storage speeds, and therefore it is typically never
// worth switching to kNoCompression. Even if the input data is
// incompressible, the kSnappyCompression implementation will
// efficiently detect that and will switch to uncompressed mode.
CompressionType compression;
// If non-NULL, use the specified filter policy to reduce disk reads.
// Many applications will benefit from passing the result of
// NewBloomFilterPolicy() here.
//
// Default: NULL
const FilterPolicy* filter_policy;
// Create a LevelDBOptions object with default values for all fields.
LevelDBOptions();
};
// Converts a LevelDBOptions object into a RocksDB Options object.
Options ConvertOptions(const LevelDBOptions& leveldb_options);
} // namespace rocksdb
+1 -1
View File
@@ -5,7 +5,7 @@
#pragma once
#define ROCKSDB_MAJOR 3
#define ROCKSDB_MINOR 9
#define ROCKSDB_MINOR 8
#define ROCKSDB_PATCH 0
// Do not use these. We made the mistake of declaring macros starting with
+4 -7
View File
@@ -105,11 +105,10 @@ class WriteBatch {
return Status::InvalidArgument(
"non-default column family and PutCF not implemented");
}
virtual void Put(const Slice& key, const Slice& value) {}
virtual void Put(const Slice& key, const Slice& value);
// Merge and LogData are not pure virtual. Otherwise, we would break
// existing clients of Handler on a source code level. The default
// implementation of Merge does nothing.
// implementation of Merge simply throws a runtime exception.
virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
const Slice& value) {
if (column_family_id == 0) {
@@ -119,8 +118,7 @@ class WriteBatch {
return Status::InvalidArgument(
"non-default column family and MergeCF not implemented");
}
virtual void Merge(const Slice& key, const Slice& value) {}
virtual void Merge(const Slice& key, const Slice& value);
// The default implementation of LogData does nothing.
virtual void LogData(const Slice& blob);
virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) {
@@ -131,8 +129,7 @@ class WriteBatch {
return Status::InvalidArgument(
"non-default column family and DeleteCF not implemented");
}
virtual void Delete(const Slice& key) {}
virtual void Delete(const Slice& key);
// Continue is called by WriteBatch::Iterate. If it returns false,
// iteration is halted. Otherwise, it continues iterating. The default
// implementation always returns true.
+34 -65
View File
@@ -4,7 +4,6 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractComparator\
org.rocksdb.BackupableDBOptions\
org.rocksdb.BlockBasedTableConfig\
org.rocksdb.BloomFilter\
org.rocksdb.Checkpoint\
org.rocksdb.ColumnFamilyHandle\
org.rocksdb.ColumnFamilyOptions\
org.rocksdb.Comparator\
@@ -33,9 +32,9 @@ NATIVE_JAVA_CLASSES = org.rocksdb.AbstractComparator\
org.rocksdb.StringAppendOperator\
org.rocksdb.WriteBatch\
org.rocksdb.WriteBatch.Handler\
org.rocksdb.test.WriteBatchInternal\
org.rocksdb.test.WriteBatchTest\
org.rocksdb.WriteOptions\
org.rocksdb.WriteBatchInternal\
org.rocksdb.WriteBatchTest\
org.rocksdb.WriteOptions\
ROCKSDB_MAJOR = $(shell egrep "ROCKSDB_MAJOR.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3)
ROCKSDB_MINOR = $(shell egrep "ROCKSDB_MINOR.[0-9]" ../include/rocksdb/version.h | cut -d ' ' -f 3)
@@ -44,68 +43,21 @@ ROCKSDB_PATCH = $(shell egrep "ROCKSDB_PATCH.[0-9]" ../include/rocksdb/version.h
NATIVE_INCLUDE = ./include
ARCH := $(shell getconf LONG_BIT)
ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux$(ARCH).jar
ifeq ($(PLATFORM), OS_MACOSX)
ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
endif
JAVA_TESTS = org.rocksdb.test.BackupableDBOptionsTest\
org.rocksdb.test.BackupableDBTest\
org.rocksdb.test.BlockBasedTableConfigTest\
org.rocksdb.test.CheckPointTest\
org.rocksdb.test.ColumnFamilyOptionsTest\
org.rocksdb.test.ColumnFamilyTest\
org.rocksdb.test.ComparatorOptionsTest\
org.rocksdb.test.ComparatorTest\
org.rocksdb.test.DBOptionsTest\
org.rocksdb.test.DirectComparatorTest\
org.rocksdb.test.EnvironmentTest\
org.rocksdb.test.FilterTest\
org.rocksdb.test.FlushTest\
org.rocksdb.test.InfoLogLevelTest\
org.rocksdb.test.KeyMayExistTest\
org.rocksdb.test.MemTableTest\
org.rocksdb.test.MergeTest\
org.rocksdb.test.MixedOptionsTest\
org.rocksdb.test.OptionsTest\
org.rocksdb.test.PlainTableConfigTest\
org.rocksdb.test.ReadOnlyTest\
org.rocksdb.test.ReadOptionsTest\
org.rocksdb.test.RocksDBTest\
org.rocksdb.test.RocksEnvTest\
org.rocksdb.test.RocksIteratorTest\
org.rocksdb.test.SizeUnitTest\
org.rocksdb.test.SnapshotTest\
org.rocksdb.test.StatisticsCollectorTest\
org.rocksdb.test.WriteBatchHandlerTest\
org.rocksdb.test.WriteBatchTest\
org.rocksdb.test.WriteOptionsTest\
JAVA_TEST_LIBDIR = ./test-libs/
JAVA_JUNIT_JAR = $(JAVA_TEST_LIBDIR)junit-4.12-beta-2.jar
JAVA_HAMCR_JAR = $(JAVA_TEST_LIBDIR)hamcrest-core-1.3.jar
JAVA_MOCKITO_JAR = $(JAVA_TEST_LIBDIR)mockito-all-1.9.5.jar
JAVA_CGLIB_JAR = $(JAVA_TEST_LIBDIR)cglib-2.2.2.jar
JAVA_ASSERTJ_JAR = $(JAVA_TEST_LIBDIR)assertj-core-1.7.0.jar
JAVA_TESTCLASSPATH = $(ROCKSDB_JAR):$(JAVA_JUNIT_JAR):$(JAVA_HAMCR_JAR):$(JAVA_MOCKITO_JAR):$(JAVA_CGLIB_JAR):$(JAVA_ASSERTJ_JAR):.:./*
clean:
-find . -name "*.class" -exec rm {} \;
rm -rf include/*
-find . -name "hs*.log" -exec rm {} \;
rm -rf javadoc/*
rm -rf test-libs/
rm -rf target
rm -rf librocksdbjni*
rm -f rocksdbjni*
javadocs:
mkdir -p javadoc; javadoc -d javadoc -sourcepath . -subpackages org -exclude org.rocksdb.test
mkdir -p javadoc; javadoc -d javadoc -sourcepath . -subpackages org
javalib: java javadocs
java: resolve_test_deps
java: javadocs
javac org/rocksdb/util/*.java org/rocksdb/*.java
javac -cp $(JAVA_TESTCLASSPATH) org/rocksdb/test/*.java
@cp ../HISTORY.md ./HISTORY-CPP.md
@rm -f ./HISTORY-CPP.md
javah -d $(NATIVE_INCLUDE) -jni $(NATIVE_JAVA_CLASSES)
@@ -124,16 +76,33 @@ column_family_sample: java
java -ea -Djava.library.path=.:../ -cp ".:./*" -Xcheck:jni RocksDBColumnFamilySample /tmp/rocksdbjni
@rm -rf /tmp/rocksdbjni
resolve_test_deps:
mkdir -p "$(JAVA_TEST_LIBDIR)"
test -s "$(JAVA_JUNIT_JAR)" || curl -k -L -o $(JAVA_JUNIT_JAR) http://search.maven.org/remotecontent?filepath=junit/junit/4.12-beta-2/junit-4.12-beta-2.jar
test -s "$(JAVA_HAMCR_JAR)" || curl -k -L -o $(JAVA_HAMCR_JAR) http://search.maven.org/remotecontent?filepath=org/hamcrest/hamcrest-core/1.3/hamcrest-core-1.3.jar
test -s "$(JAVA_MOCKITO_JAR)" || curl -k -L -o "$(JAVA_MOCKITO_JAR)" http://search.maven.org/remotecontent?filepath=org/mockito/mockito-all/1.9.5/mockito-all-1.9.5.jar
test -s "$(JAVA_CGLIB_JAR)" || curl -k -L -o "$(JAVA_CGLIB_JAR)" http://search.maven.org/remotecontent?filepath=cglib/cglib/2.2.2/cglib-2.2.2.jar
test -s "$(JAVA_ASSERTJ_JAR)" || curl -k -L -o "$(JAVA_ASSERTJ_JAR)" http://central.maven.org/maven2/org/assertj/assertj-core/1.7.0/assertj-core-1.7.0.jar
test: java resolve_test_deps
java -ea -Xcheck:jni -Djava.library.path=.:../ -cp "$(JAVA_TESTCLASSPATH)" org.rocksdb.test.RocksJunitRunner $(JAVA_TESTS)
test: java
@rm -rf /tmp/rocksdbjni_*
javac org/rocksdb/test/*.java
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.WriteBatchTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BackupableDBTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BlockBasedTableConfigTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.DBOptionsTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ColumnFamilyOptionsTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FilterTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.FlushTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.KeyMayExistTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MemTableTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MergeTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.MixedOptionsTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.PlainTableConfigTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOnlyTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.RocksIteratorTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.SnapshotTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.StatisticsCollectorTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ComparatorOptionsTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ComparatorTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.DirectComparatorTest
java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.WriteBatchHandlerTest
@rm -rf /tmp/rocksdbjni_*
db_bench: java
javac org/rocksdb/benchmark/*.java
-3
View File
@@ -44,9 +44,6 @@ public class RocksDBColumnFamilySample {
db.close();
db = null;
}
if (options != null) {
options.dispose();
}
}
// open DB with two column families
+1 -1
View File
@@ -119,7 +119,7 @@ public class RocksDBSample {
byte[] value = db.get("hello".getBytes());
assert("world".equals(new String(value)));
String str = db.getProperty("rocksdb.stats");
assert(str != null && !str.equals(""));
assert(str != null && str != "");
} catch (RocksDBException e) {
System.out.format("[ERROR] caught the unexpceted exception -- %s\n", e);
assert(db == null);
+16 -53
View File
@@ -8,23 +8,21 @@ package org.rocksdb;
import java.util.List;
/**
* <p>A subclass of RocksDB which supports
* backup-related operations.</p>
* A subclass of RocksDB which supports backup-related operations.
*
* @see org.rocksdb.BackupableDBOptions
*/
public class BackupableDB extends RocksDB {
/**
* <p>Open a {@code BackupableDB} under the specified path.
* Open a {@code BackupableDB} under the specified path.
* Note that the backup path should be set properly in the
* input BackupableDBOptions.</p>
* input BackupableDBOptions.
*
* @param opt {@link org.rocksdb.Options} to set for the database.
* @param bopt {@link org.rocksdb.BackupableDBOptions} to use.
* @param db_path Path to store data to. The path for storing the backup should be
* specified in the {@link org.rocksdb.BackupableDBOptions}.
*
* @return {@link BackupableDB} reference to the opened database.
* @return BackupableDB reference to the opened database.
*
* @throws RocksDBException thrown if error happens in underlying
* native library.
@@ -45,8 +43,8 @@ public class BackupableDB extends RocksDB {
}
/**
* <p>Captures the state of the database in the latest backup.
* Note that this function is not thread-safe.</p>
* Captures the state of the database in the latest backup.
* Note that this function is not thread-safe.
*
* @param flushBeforeBackup if true, then all data will be flushed
* before creating backup.
@@ -56,12 +54,11 @@ public class BackupableDB extends RocksDB {
*/
public void createNewBackup(boolean flushBeforeBackup)
throws RocksDBException {
assert(isInitialized());
createNewBackup(nativeHandle_, flushBeforeBackup);
}
/**
* <p>Deletes old backups, keeping latest numBackupsToKeep alive.</p>
* Deletes old backups, keeping latest numBackupsToKeep alive.
*
* @param numBackupsToKeep Number of latest backups to keep.
*
@@ -70,12 +67,11 @@ public class BackupableDB extends RocksDB {
*/
public void purgeOldBackups(int numBackupsToKeep)
throws RocksDBException {
assert(isInitialized());
purgeOldBackups(nativeHandle_, numBackupsToKeep);
}
/**
* <p>Deletes a specific backup.</p>
* Deletes a specific backup.
*
* @param backupId of backup to delete.
*
@@ -83,54 +79,25 @@ public class BackupableDB extends RocksDB {
* native library.
*/
public void deleteBackup(int backupId) throws RocksDBException {
assert(isInitialized());
deleteBackup0(nativeHandle_, backupId);
}
/**
* <p>Returns a list of {@link BackupInfo} instances, which describe
* already made backups.</p>
* Returns a list of {@link BackupInfo} instances, which describe
* already made backups.
*
* @return List of {@link BackupInfo} instances.
*/
public List<BackupInfo> getBackupInfos() {
assert(isInitialized());
return getBackupInfo(nativeHandle_);
}
/**
* <p>Returns a list of corrupted backup ids. If there
* is no corrupted backup the method will return an
* empty list.</p>
* Close the BackupableDB instance and release resource.
*
* @return array of backup ids as int ids.
*/
public int[] getCorruptedBackups() {
assert(isInitialized());
return getCorruptedBackups(nativeHandle_);
}
/**
* <p>Will delete all the files we don't need anymore. It will
* do the full scan of the files/ directory and delete all the
* files that are not referenced.</p>
*
* @throws RocksDBException thrown if error happens in underlying
* native library.
*/
public void garbageCollect() throws RocksDBException {
assert(isInitialized());
garbageCollect(nativeHandle_);
}
/**
* <p>Close the BackupableDB instance and release resource.</p>
*
* <p>Internally, {@link BackupableDB} owns the {@code rocksdb::DB}
* pointer to its associated {@link org.rocksdb.RocksDB}.
* The release of that RocksDB pointer is handled in the destructor
* of the c++ {@code rocksdb::BackupableDB} and should be transparent
* to Java developers.</p>
* Internally, BackupableDB owns the {@code rocksdb::DB} pointer to its associated
* {@link org.rocksdb.RocksDB}. The release of that RocksDB pointer is handled in the destructor
* of the c++ {@code rocksdb::BackupableDB} and should be transparent to Java developers.
*/
@Override public synchronized void close() {
if (isInitialized()) {
@@ -139,9 +106,8 @@ public class BackupableDB extends RocksDB {
}
/**
* <p>A protected construction that will be used in the static
* factory method {@link #open(Options, BackupableDBOptions, String)}.
* </p>
* A protected construction that will be used in the static factory
* method {@link #open(Options, BackupableDBOptions, String)}.
*/
protected BackupableDB() {
super();
@@ -160,7 +126,4 @@ public class BackupableDB extends RocksDB {
private native void deleteBackup0(long nativeHandle, int backupId)
throws RocksDBException;
protected native List<BackupInfo> getBackupInfo(long handle);
private native int[] getCorruptedBackups(long handle);
private native void garbageCollect(long handle)
throws RocksDBException;
}
+40 -230
View File
@@ -5,38 +5,57 @@
package org.rocksdb;
import java.io.File;
import java.nio.file.Path;
/**
* <p>BackupableDBOptions to control the behavior of a backupable database.
* BackupableDBOptions to control the behavior of a backupable database.
* It will be used during the creation of a {@link org.rocksdb.BackupableDB}.
* </p>
* <p>Note that dispose() must be called before an Options instance
* become out-of-scope to release the allocated memory in c++.</p>
*
* Note that dispose() must be called before an Options instance
* become out-of-scope to release the allocated memory in c++.
*
* @see org.rocksdb.BackupableDB
*/
public class BackupableDBOptions extends RocksObject {
/**
* <p>BackupableDBOptions constructor.</p>
* BackupableDBOptions constructor
*
* @param path Where to keep the backup files. Has to be different than db name.
* Best to set this to {@code db name_ + "/backups"}
* @throws java.lang.IllegalArgumentException if illegal path is used.
* Best to set this to {@code db name_ + "/backups"}
* @param shareTableFiles If {@code share_table_files == true}, backup will assume
* that table files with same name have the same contents. This enables incremental
* backups and avoids unnecessary data copies. If {@code share_table_files == false},
* each backup will be on its own and will not share any data with other backups.
* Default: true
* @param sync If {@code sync == true}, we can guarantee you'll get consistent backup
* even on a machine crash/reboot. Backup process is slower with sync enabled.
* If {@code sync == false}, we don't guarantee anything on machine reboot.
* However,chances are some of the backups are consistent.
* Default: true
* @param destroyOldData If true, it will delete whatever backups there are already.
* Default: false
* @param backupLogFiles If false, we won't backup log files. This option can be
* useful for backing up in-memory databases where log file are persisted,but table
* files are in memory.
* Default: true
* @param backupRateLimit Max bytes that can be transferred in a second during backup.
* If 0 or negative, then go as fast as you can. Default: 0
* @param restoreRateLimit Max bytes that can be transferred in a second during restore.
* If 0 or negative, then go as fast as you can. Default: 0
*/
public BackupableDBOptions(String path) {
public BackupableDBOptions(String path, boolean shareTableFiles, boolean sync,
boolean destroyOldData, boolean backupLogFiles, long backupRateLimit,
long restoreRateLimit) {
super();
File backupPath = path == null ? null : new File(path);
if (backupPath == null || !backupPath.isDirectory() || !backupPath.canWrite()) {
throw new IllegalArgumentException("Illegal path provided.");
}
newBackupableDBOptions(path);
backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit;
restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit;
newBackupableDBOptions(path, shareTableFiles, sync, destroyOldData,
backupLogFiles, backupRateLimit, restoreRateLimit);
}
/**
* <p>Returns the path to the BackupableDB directory.</p>
* Returns the path to the BackupableDB directory.
*
* @return the path to the BackupableDB directory.
*/
@@ -45,227 +64,18 @@ public class BackupableDBOptions extends RocksObject {
return backupDir(nativeHandle_);
}
/**
* <p>Share table files between backups.</p>
*
* @param shareTableFiles If {@code share_table_files == true}, backup will assume
* that table files with same name have the same contents. This enables incremental
* backups and avoids unnecessary data copies. If {@code share_table_files == false},
* each backup will be on its own and will not share any data with other backups.
*
* <p>Default: true</p>
*
* @return instance of current BackupableDBOptions.
*/
public BackupableDBOptions setShareTableFiles(boolean shareTableFiles) {
assert(isInitialized());
setShareTableFiles(nativeHandle_, shareTableFiles);
return this;
}
/**
* <p>Share table files between backups.</p>
*
* @return boolean value indicating if SST files will be shared between
* backups.
*/
public boolean shareTableFiles() {
assert(isInitialized());
return shareTableFiles(nativeHandle_);
}
/**
* <p>Set synchronous backups.</p>
*
* @param sync If {@code sync == true}, we can guarantee you'll get consistent backup
* even on a machine crash/reboot. Backup process is slower with sync enabled.
* If {@code sync == false}, we don't guarantee anything on machine reboot.
* However,chances are some of the backups are consistent.
*
* <p>Default: true</p>
*
* @return instance of current BackupableDBOptions.
*/
public BackupableDBOptions setSync(boolean sync) {
assert(isInitialized());
setSync(nativeHandle_, sync);
return this;
}
/**
* <p>Are synchronous backups activated.</p>
*
* @return boolean value if synchronous backups are configured.
*/
public boolean sync() {
assert(isInitialized());
return sync(nativeHandle_);
}
/**
* <p>Set if old data will be destroyed.</p>
*
* @param destroyOldData If true, it will delete whatever backups there are already.
*
* <p>Default: false</p>
*
* @return instance of current BackupableDBOptions.
*/
public BackupableDBOptions setDestroyOldData(boolean destroyOldData) {
assert(isInitialized());
setDestroyOldData(nativeHandle_, destroyOldData);
return this;
}
/**
* <p>Returns if old data will be destroyed will performing new backups.</p>
*
* @return boolean value indicating if old data will be destroyed.
*/
public boolean destroyOldData() {
assert(isInitialized());
return destroyOldData(nativeHandle_);
}
/**
* <p>Set if log files shall be persisted.</p>
*
* @param backupLogFiles If false, we won't backup log files. This option can be
* useful for backing up in-memory databases where log file are persisted,but table
* files are in memory.
*
* <p>Default: true</p>
*
* @return instance of current BackupableDBOptions.
*/
public BackupableDBOptions setBackupLogFiles(boolean backupLogFiles) {
assert(isInitialized());
setBackupLogFiles(nativeHandle_, backupLogFiles);
return this;
}
/**
* <p>Return information if log files shall be persisted.</p>
*
* @return boolean value indicating if log files will be persisted.
*/
public boolean backupLogFiles() {
assert(isInitialized());
return backupLogFiles(nativeHandle_);
}
/**
* <p>Set backup rate limit.</p>
*
* @param backupRateLimit Max bytes that can be transferred in a second during backup.
* If 0 or negative, then go as fast as you can.
*
* <p>Default: 0</p>
*
* @return instance of current BackupableDBOptions.
*/
public BackupableDBOptions setBackupRateLimit(long backupRateLimit) {
assert(isInitialized());
backupRateLimit = (backupRateLimit <= 0) ? 0 : backupRateLimit;
setBackupRateLimit(nativeHandle_, backupRateLimit);
return this;
}
/**
* <p>Return backup rate limit which described the max bytes that can be transferred in a
* second during backup.</p>
*
* @return numerical value describing the backup transfer limit in bytes per second.
*/
public long backupRateLimit() {
assert(isInitialized());
return backupRateLimit(nativeHandle_);
}
/**
* <p>Set restore rate limit.</p>
*
* @param restoreRateLimit Max bytes that can be transferred in a second during restore.
* If 0 or negative, then go as fast as you can.
*
* <p>Default: 0</p>
*
* @return instance of current BackupableDBOptions.
*/
public BackupableDBOptions setRestoreRateLimit(long restoreRateLimit) {
assert(isInitialized());
restoreRateLimit = (restoreRateLimit <= 0) ? 0 : restoreRateLimit;
setRestoreRateLimit(nativeHandle_, restoreRateLimit);
return this;
}
/**
* <p>Return restore rate limit which described the max bytes that can be transferred in a
* second during restore.</p>
*
* @return numerical value describing the restore transfer limit in bytes per second.
*/
public long restoreRateLimit() {
assert(isInitialized());
return restoreRateLimit(nativeHandle_);
}
/**
* <p>Only used if share_table_files is set to true. If true, will consider that
* backups can come from different databases, hence a sst is not uniquely
* identified by its name, but by the triple (file name, crc32, file length)</p>
*
* @param shareFilesWithChecksum boolean value indicating if SST files are stored
* using the triple (file name, crc32, file length) and not its name.
*
* <p>Note: this is an experimental option, and you'll need to set it manually
* turn it on only if you know what you're doing*</p>
*
* <p>Default: false</p>
*
* @return instance of current BackupableDBOptions.
*/
public BackupableDBOptions setShareFilesWithChecksum(
boolean shareFilesWithChecksum) {
assert(isInitialized());
setShareFilesWithChecksum(nativeHandle_, shareFilesWithChecksum);
return this;
}
/**
* <p>Return of share files with checksum is active.</p>
*
* @return boolean value indicating if share files with checksum
* is active.
*/
public boolean shareFilesWithChecksum() {
assert(isInitialized());
return shareFilesWithChecksum(nativeHandle_);
}
/**
* Release the memory allocated for the current instance
* in the c++ side.
*/
@Override protected void disposeInternal() {
assert(isInitialized());
disposeInternal(nativeHandle_);
}
private native void newBackupableDBOptions(String path);
private native void newBackupableDBOptions(String path,
boolean shareTableFiles, boolean sync, boolean destroyOldData,
boolean backupLogFiles, long backupRateLimit, long restoreRateLimit);
private native String backupDir(long handle);
private native void setShareTableFiles(long handle, boolean flag);
private native boolean shareTableFiles(long handle);
private native void setSync(long handle, boolean flag);
private native boolean sync(long handle);
private native void setDestroyOldData(long handle, boolean flag);
private native boolean destroyOldData(long handle);
private native void setBackupLogFiles(long handle, boolean flag);
private native boolean backupLogFiles(long handle);
private native void setBackupRateLimit(long handle, long rateLimit);
private native long backupRateLimit(long handle);
private native void setRestoreRateLimit(long handle, long rateLimit);
private native long restoreRateLimit(long handle);
private native void setShareFilesWithChecksum(long handle, boolean flag);
private native boolean shareFilesWithChecksum(long handle);
private native void disposeInternal(long handle);
}
-72
View File
@@ -1,72 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb;
/**
* Provides Checkpoint functionality. Checkpoints
* provide persistent snapshots of RocksDB databases.
*/
public class Checkpoint extends RocksObject {
/**
* Creates a Checkpoint object to be used for creating open-able
* snapshots.
*
* @param db {@link RocksDB} instance.
* @return a Checkpoint instance.
*
* @throws java.lang.IllegalArgumentException if {@link RocksDB}
* instance is null.
* @throws java.lang.IllegalStateException if {@link RocksDB}
* instance is not initialized.
*/
public static Checkpoint create(RocksDB db) {
if (db == null) {
throw new IllegalArgumentException(
"RocksDB instance shall not be null.");
} else if (!db.isInitialized()) {
throw new IllegalStateException(
"RocksDB instance must be initialized.");
}
Checkpoint checkpoint = new Checkpoint(db);
return checkpoint;
}
/**
* <p>Builds an open-able snapshot of RocksDB on the same disk, which
* accepts an output directory on the same disk, and under the directory
* (1) hard-linked SST files pointing to existing live SST files
* (2) a copied manifest files and other files</p>
*
* @param checkpointPath path to the folder where the snapshot is going
* to be stored.
* @throws RocksDBException thrown if an error occurs within the native
* part of the library.
*/
public void createCheckpoint(String checkpointPath)
throws RocksDBException {
createCheckpoint(nativeHandle_, checkpointPath);
}
@Override
protected void disposeInternal() {
disposeInternal(nativeHandle_);
}
private Checkpoint(RocksDB db) {
super();
nativeHandle_ = newCheckpoint(db.nativeHandle_);
db_ = db;
}
RocksDB db_;
private static native long newCheckpoint(long dbHandle);
private native void disposeInternal(long handle);
private native void createCheckpoint(long handle, String checkpointPath)
throws RocksDBException;
}
+4 -6
View File
@@ -30,15 +30,13 @@ public class ColumnFamilyHandle extends RocksObject {
* before freeing the native handle.</p>
*/
@Override protected void disposeInternal() {
synchronized (rocksDB_) {
assert (isInitialized());
if (rocksDB_.isInitialized()) {
disposeInternal(nativeHandle_);
}
assert(isInitialized());
if (rocksDB_.isInitialized()) {
disposeInternal(nativeHandle_);
}
}
private native void disposeInternal(long handle);
private final RocksDB rocksDB_;
private RocksDB rocksDB_;
}
-62
View File
@@ -5,8 +5,6 @@
package org.rocksdb;
import java.util.Properties;
/**
* ColumnFamilyOptions to control the behavior of a database. It will be used
* during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()).
@@ -31,47 +29,6 @@ public class ColumnFamilyOptions extends RocksObject
newColumnFamilyOptions();
}
/**
* <p>Method to get a options instance by using pre-configured
* property values. If one or many values are undefined in
* the context of RocksDB the method will return a null
* value.</p>
*
* <p><strong>Note</strong>: Property keys can be derived from
* getter methods within the options class. Example: the method
* {@code writeBufferSize()} has a property key:
* {@code write_buffer_size}.</p>
*
* @param properties {@link java.util.Properties} instance.
*
* @return {@link org.rocksdb.ColumnFamilyOptions instance}
* or null.
*
* @throws java.lang.IllegalArgumentException if null or empty
* {@link Properties} instance is passed to the method call.
*/
public static ColumnFamilyOptions getColumnFamilyOptionsFromProps(
Properties properties) {
if (properties == null || properties.size() == 0) {
throw new IllegalArgumentException(
"Properties value must contain at least one value.");
}
ColumnFamilyOptions columnFamilyOptions = null;
StringBuilder stringBuilder = new StringBuilder();
for (final String name : properties.stringPropertyNames()){
stringBuilder.append(name);
stringBuilder.append("=");
stringBuilder.append(properties.getProperty(name));
stringBuilder.append(";");
}
long handle = getColumnFamilyOptionsFromProps(
stringBuilder.toString());
if (handle != 0){
columnFamilyOptions = new ColumnFamilyOptions(handle);
}
return columnFamilyOptions;
}
@Override
public ColumnFamilyOptions optimizeForPointLookup(
long blockCacheSizeMb) {
@@ -127,11 +84,6 @@ public class ColumnFamilyOptions extends RocksObject
@Override
public ColumnFamilyOptions setMergeOperatorName(String name) {
assert (isInitialized());
if (name == null) {
throw new IllegalArgumentException(
"Merge operator name must not be null.");
}
setMergeOperatorName(nativeHandle_, name);
return this;
}
@@ -570,20 +522,6 @@ public class ColumnFamilyOptions extends RocksObject
disposeInternal(nativeHandle_);
}
/**
* <p>Private constructor to be used by
* {@link #getColumnFamilyOptionsFromProps(java.util.Properties)}</p>
*
* @param handle native handle to ColumnFamilyOptions instance.
*/
private ColumnFamilyOptions(long handle) {
super();
nativeHandle_ = handle;
}
private static native long getColumnFamilyOptionsFromProps(
String optString);
private native void newColumnFamilyOptions();
private native void disposeInternal(long handle);
+10 -44
View File
@@ -14,59 +14,25 @@ package org.rocksdb;
* compression method (if any) is used to compress a block.</p>
*/
public enum CompressionType {
NO_COMPRESSION((byte) 0),
SNAPPY_COMPRESSION((byte) 1),
ZLIB_COMPRESSION((byte) 2),
BZLIB2_COMPRESSION((byte) 3),
LZ4_COMPRESSION((byte) 4),
LZ4HC_COMPRESSION((byte) 5);
NO_COMPRESSION((byte) 0, null),
SNAPPY_COMPRESSION((byte) 1, "snappy"),
ZLIB_COMPRESSION((byte) 2, "z"),
BZLIB2_COMPRESSION((byte) 3, "bzip2"),
LZ4_COMPRESSION((byte) 4, "lz4"),
LZ4HC_COMPRESSION((byte) 5, "lz4hc");
private final byte value_;
/**
* <p>Get the CompressionType enumeration value by
* passing the library name to this method.</p>
*
* <p>If library cannot be found the enumeration
* value {@code NO_COMPRESSION} will be returned.</p>
*
* @return CompressionType instance.
*/
public static CompressionType getCompressionType(String libraryName) {
if (libraryName != null) {
for (CompressionType compressionType : CompressionType.values()) {
if (compressionType.getLibraryName() != null &&
compressionType.getLibraryName().equals(libraryName)) {
return compressionType;
}
}
}
return CompressionType.NO_COMPRESSION;
private CompressionType(byte value) {
value_ = value;
}
/**
* <p>Returns the byte value of the enumerations value.</p>
* Returns the byte value of the enumerations value
*
* @return byte representation
*/
public byte getValue() {
return value_;
}
/**
* <p>Returns the library name of the compression type
* identified by the enumeration value.</p>
*
* @return library name
*/
public String getLibraryName() {
return libraryName_;
}
private CompressionType(byte value, final String libraryName) {
value_ = value;
libraryName_ = libraryName;
}
private final byte value_;
private final String libraryName_;
}
-74
View File
@@ -5,8 +5,6 @@
package org.rocksdb;
import java.util.Properties;
/**
* DBOptions to control the behavior of a database. It will be used
* during the creation of a {@link org.rocksdb.RocksDB} (i.e., RocksDB.open()).
@@ -31,47 +29,6 @@ public class DBOptions extends RocksObject implements DBOptionsInterface {
newDBOptions();
}
/**
* <p>Method to get a options instance by using pre-configured
* property values. If one or many values are undefined in
* the context of RocksDB the method will return a null
* value.</p>
*
* <p><strong>Note</strong>: Property keys can be derived from
* getter methods within the options class. Example: the method
* {@code allowMmapReads()} has a property key:
* {@code allow_mmap_reads}.</p>
*
* @param properties {@link java.util.Properties} instance.
*
* @return {@link org.rocksdb.DBOptions instance}
* or null.
*
* @throws java.lang.IllegalArgumentException if null or empty
* {@link java.util.Properties} instance is passed to the method call.
*/
public static DBOptions getDBOptionsFromProps(
Properties properties) {
if (properties == null || properties.size() == 0) {
throw new IllegalArgumentException(
"Properties value must contain at least one value.");
}
DBOptions dbOptions = null;
StringBuilder stringBuilder = new StringBuilder();
for (final String name : properties.stringPropertyNames()){
stringBuilder.append(name);
stringBuilder.append("=");
stringBuilder.append(properties.getProperty(name));
stringBuilder.append(";");
}
long handle = getDBOptionsFromProps(
stringBuilder.toString());
if (handle != 0){
dbOptions = new DBOptions(handle);
}
return dbOptions;
}
@Override
public DBOptions setCreateIfMissing(boolean flag) {
assert(isInitialized());
@@ -126,26 +83,11 @@ public class DBOptions extends RocksObject implements DBOptionsInterface {
@Override
public DBOptions setRateLimiterConfig(RateLimiterConfig config) {
assert(isInitialized());
rateLimiterConfig_ = config;
setRateLimiter(nativeHandle_, config.newRateLimiterHandle());
return this;
}
@Override
public DBOptions setInfoLogLevel(InfoLogLevel infoLogLevel) {
assert(isInitialized());
setInfoLogLevel(nativeHandle_, infoLogLevel.getValue());
return this;
}
@Override
public InfoLogLevel infoLogLevel() {
assert(isInitialized());
return InfoLogLevel.getInfoLogLevel(
infoLogLevel(nativeHandle_));
}
@Override
public DBOptions setMaxOpenFiles(int maxOpenFiles) {
assert(isInitialized());
@@ -530,20 +472,6 @@ public class DBOptions extends RocksObject implements DBOptionsInterface {
static final int DEFAULT_NUM_SHARD_BITS = -1;
/**
* <p>Private constructor to be used by
* {@link #getDBOptionsFromProps(java.util.Properties)}</p>
*
* @param handle native handle to DBOptions instance.
*/
private DBOptions(long handle) {
super();
nativeHandle_ = handle;
}
private static native long getDBOptionsFromProps(
String optString);
private native void newDBOptions();
private native void disposeInternal(long handle);
@@ -559,8 +487,6 @@ public class DBOptions extends RocksObject implements DBOptionsInterface {
private native boolean paranoidChecks(long handle);
private native void setRateLimiter(long handle,
long rateLimiterHandle);
private native void setInfoLogLevel(long handle, byte logLevel);
private native byte infoLogLevel(long handle);
private native void setMaxOpenFiles(long handle, int maxOpenFiles);
private native int maxOpenFiles(long handle);
private native void setMaxTotalWalSize(long handle,
-14
View File
@@ -113,20 +113,6 @@ public interface DBOptionsInterface {
*/
Object setRateLimiterConfig(RateLimiterConfig config);
/**
* <p>Sets the RocksDB log level. Default level is INFO</p>
*
* @param infoLogLevel log level to set.
* @return the instance of the current Object.
*/
Object setInfoLogLevel(InfoLogLevel infoLogLevel);
/**
* <p>Returns currently set log level.</p>
* @return {@link org.rocksdb.InfoLogLevel} instance.
*/
InfoLogLevel infoLogLevel();
/**
* Number of open files that can be used by the DB. You may need to
* increase this if your database has a large working set. Value -1 means
-47
View File
@@ -1,47 +0,0 @@
package org.rocksdb;
/**
* RocksDB log levels.
*/
public enum InfoLogLevel {
DEBUG_LEVEL((byte)0),
INFO_LEVEL((byte)1),
WARN_LEVEL((byte)2),
ERROR_LEVEL((byte)3),
FATAL_LEVEL((byte)4),
NUM_INFO_LOG_LEVELS((byte)5);
private final byte value_;
private InfoLogLevel(byte value) {
value_ = value;
}
/**
* Returns the byte value of the enumerations value
*
* @return byte representation
*/
public byte getValue() {
return value_;
}
/**
* Get InfoLogLevel by byte value.
*
* @param value byte representation of InfoLogLevel.
*
* @return {@link org.rocksdb.InfoLogLevel} instance or null.
* @throws java.lang.IllegalArgumentException if an invalid
* value is provided.
*/
public static InfoLogLevel getInfoLogLevel(byte value) {
for (InfoLogLevel infoLogLevel : InfoLogLevel.values()) {
if (infoLogLevel.getValue() == value){
return infoLogLevel;
}
}
throw new IllegalArgumentException(
"Illegal value provided for InfoLogLevel.");
}
}
+27 -46
View File
@@ -1,9 +1,6 @@
package org.rocksdb;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import org.rocksdb.util.Environment;
/**
@@ -11,58 +8,42 @@ import org.rocksdb.util.Environment;
* The shared library is extracted to a temp folder and loaded from there.
*/
public class NativeLibraryLoader {
//singleton
private static final NativeLibraryLoader instance = new NativeLibraryLoader();
private static String sharedLibraryName = Environment.getJniLibraryName("rocksdb");
private static String tempFileSuffix = "." + Environment.getJniLibraryExtension();
private static final String sharedLibraryName = Environment.getJniLibraryName("rocksdb");
private static final String tempFilePrefix = "librocksdbjni";
private static final String tempFileSuffix = "." + Environment.getJniLibraryExtension();
/**
* Get a reference to the NativeLibraryLoader
*
* @return The NativeLibraryLoader
*/
public static NativeLibraryLoader getInstance() {
return instance;
}
/**
* Attempts to extract the native RocksDB library
* from the classpath and load it
*
* @param tmpDir A temporary directory to use
* to copy the native library to. If null,
* or the empty string, we rely on Java's
* {@link java.io.File#createTempFile(String, String)}
* function to provide a temporary location.
* The temporary file will be registered for deletion
* on exit.
*
* @throws java.io.IOException if a filesystem operation fails.
*/
public void loadLibraryFromJar(final String tmpDir)
public static void loadLibraryFromJar(String tmpDir)
throws IOException {
final File temp;
if(tmpDir == null || tmpDir.equals("")) {
File temp;
String tempFilePrefix = "librocksdbjni";
if(tmpDir == null || tmpDir.equals(""))
temp = File.createTempFile(tempFilePrefix, tempFileSuffix);
} else {
temp = new File(tmpDir, sharedLibraryName);
}
else
temp = new File(tmpDir + "/" + sharedLibraryName);
temp.deleteOnExit();
if (!temp.exists()) {
throw new RuntimeException("File " + temp.getAbsolutePath() + " does not exist.");
} else {
temp.deleteOnExit();
}
// attempt to copy the library from the Jar file to the temp destination
try(final InputStream is = getClass().getClassLoader().getResourceAsStream(sharedLibraryName)) {
if (is == null) {
throw new RuntimeException(sharedLibraryName + " was not found inside JAR.");
} else {
Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING);
byte[] buffer = new byte[102400];
int readBytes;
InputStream is = ClassLoader.getSystemClassLoader().getResourceAsStream(sharedLibraryName);
if (is == null) {
throw new RuntimeException(sharedLibraryName + " was not found inside JAR.");
}
OutputStream os = null;
try {
os = new FileOutputStream(temp);
while ((readBytes = is.read(buffer)) != -1) {
os.write(buffer, 0, readBytes);
}
} finally {
if(os != null)
os.close();
is.close();
}
System.load(temp.getAbsolutePath());
+1 -21
View File
@@ -165,11 +165,6 @@ public class Options extends RocksObject
@Override
public Options setMergeOperatorName(String name) {
assert (isInitialized());
if (name == null) {
throw new IllegalArgumentException(
"Merge operator name must not be null.");
}
setMergeOperatorName(nativeHandle_, name);
return this;
}
@@ -621,20 +616,6 @@ public class Options extends RocksObject
return this;
}
@Override
public Options setInfoLogLevel(InfoLogLevel infoLogLevel) {
assert(isInitialized());
setInfoLogLevel(nativeHandle_, infoLogLevel.getValue());
return this;
}
@Override
public InfoLogLevel infoLogLevel() {
assert(isInitialized());
return InfoLogLevel.getInfoLogLevel(
infoLogLevel(nativeHandle_));
}
@Override
public String memTableFactoryName() {
assert(isInitialized());
@@ -1029,6 +1010,7 @@ public class Options extends RocksObject
long cfOptHandle);
private native void disposeInternal(long handle);
private native void setEnv(long optHandle, long envHandle);
private native long getEnvHandle(long handle);
private native void prepareForBulkLoad(long handle);
// DB native handles
@@ -1044,8 +1026,6 @@ public class Options extends RocksObject
private native boolean paranoidChecks(long handle);
private native void setRateLimiter(long handle,
long rateLimiterHandle);
private native void setInfoLogLevel(long handle, byte logLevel);
private native byte infoLogLevel(long handle);
private native void setMaxOpenFiles(long handle, int maxOpenFiles);
private native int maxOpenFiles(long handle);
private native void setMaxTotalWalSize(long handle,
+1
View File
@@ -155,6 +155,7 @@ public class ReadOptions extends RocksObject {
@Override protected void disposeInternal() {
assert(isInitialized());
disposeInternal(nativeHandle_);
}
private native void disposeInternal(long handle);
+22 -58
View File
@@ -8,17 +8,15 @@ package org.rocksdb;
import java.util.List;
/**
* <p>This class is used to access information about backups and
* restore from them.</p>
* This class is used to access information about backups and restore from them.
*
* <p>Note: {@code dispose()} must be called before this instance
* become out-of-scope to release the allocated
* memory in c++.</p>
* Note that dispose() must be called before this instance become out-of-scope
* to release the allocated memory in c++.
*
*/
public class RestoreBackupableDB extends RocksObject {
/**
* <p>Construct new estoreBackupableDB instance.</p>
* Constructor
*
* @param options {@link org.rocksdb.BackupableDBOptions} instance
*/
@@ -28,18 +26,16 @@ public class RestoreBackupableDB extends RocksObject {
}
/**
* <p>Restore from backup with backup_id.</p>
* Restore from backup with backup_id
* IMPORTANT -- if options_.share_table_files == true and you restore DB
* from some backup that is not the latest, and you start creating new
* backups from the new DB, they will probably fail.
*
* <p><strong>Important</strong>: If options_.share_table_files == true
* and you restore DB from some backup that is not the latest, and you
* start creating new backups from the new DB, they will probably
* fail.</p>
*
* <p><strong>Example</strong>: Let's say you have backups 1, 2, 3, 4, 5
* and you restore 3. If you add new data to the DB and try creating a new
* backup now, the database will diverge from backups 4 and 5 and the new
* backup will fail. If you want to create new backup, you will first have
* to delete backups 4 and 5.</p>
* Example: Let's say you have backups 1, 2, 3, 4, 5 and you restore 3.
* If you add new data to the DB and try creating a new backup now, the
* database will diverge from backups 4 and 5 and the new backup will fail.
* If you want to create new backup, you will first have to delete backups 4
* and 5.
*
* @param backupId id pointing to backup
* @param dbDir database directory to restore to
@@ -51,13 +47,12 @@ public class RestoreBackupableDB extends RocksObject {
*/
public void restoreDBFromBackup(long backupId, String dbDir, String walDir,
RestoreOptions restoreOptions) throws RocksDBException {
assert(isInitialized());
restoreDBFromBackup0(nativeHandle_, backupId, dbDir, walDir,
restoreOptions.nativeHandle_);
}
/**
* <p>Restore from the latest backup.</p>
* Restore from the latest backup.
*
* @param dbDir database directory to restore to
* @param walDir directory where wal files are located
@@ -68,13 +63,12 @@ public class RestoreBackupableDB extends RocksObject {
*/
public void restoreDBFromLatestBackup(String dbDir, String walDir,
RestoreOptions restoreOptions) throws RocksDBException {
assert(isInitialized());
restoreDBFromLatestBackup0(nativeHandle_, dbDir, walDir,
restoreOptions.nativeHandle_);
}
/**
* <p>Deletes old backups, keeping latest numBackupsToKeep alive.</p>
* Deletes old backups, keeping latest numBackupsToKeep alive.
*
* @param numBackupsToKeep of latest backups to keep
*
@@ -82,12 +76,11 @@ public class RestoreBackupableDB extends RocksObject {
* native library.
*/
public void purgeOldBackups(int numBackupsToKeep) throws RocksDBException {
assert(isInitialized());
purgeOldBackups0(nativeHandle_, numBackupsToKeep);
}
/**
* <p>Deletes a specific backup.</p>
* Deletes a specific backup.
*
* @param backupId of backup to delete.
*
@@ -95,51 +88,25 @@ public class RestoreBackupableDB extends RocksObject {
* native library.
*/
public void deleteBackup(int backupId) throws RocksDBException {
assert(isInitialized());
deleteBackup0(nativeHandle_, backupId);
}
/**
* <p>Returns a list of {@link BackupInfo} instances, which describe
* already made backups.</p>
* Returns a list of {@link BackupInfo} instances, which describe
* already made backups.
*
* @return List of {@link BackupInfo} instances.
*/
public List<BackupInfo> getBackupInfos() {
assert(isInitialized());
return getBackupInfo(nativeHandle_);
}
/**
* <p>Returns a list of corrupted backup ids. If there
* is no corrupted backup the method will return an
* empty list.</p>
*
* @return array of backup ids as int ids.
*/
public int[] getCorruptedBackups() {
assert(isInitialized());
return getCorruptedBackups(nativeHandle_);
}
/**
* <p>Will delete all the files we don't need anymore. It will
* do the full scan of the files/ directory and delete all the
* files that are not referenced.</p>
*
* @throws RocksDBException thrown if error happens in underlying
* native library.
*/
public void garbageCollect() throws RocksDBException {
assert(isInitialized());
garbageCollect(nativeHandle_);
}
/**
* <p>Release the memory allocated for the current instance
* in the c++ side.</p>
* Release the memory allocated for the current instance
* in the c++ side.
*/
@Override public synchronized void disposeInternal() {
assert(isInitialized());
dispose(nativeHandle_);
}
@@ -154,9 +121,6 @@ public class RestoreBackupableDB extends RocksObject {
throws RocksDBException;
private native void deleteBackup0(long nativeHandle, int backupId)
throws RocksDBException;
private native List<BackupInfo> getBackupInfo(long handle);
private native int[] getCorruptedBackups(long handle);
private native void garbageCollect(long handle)
throws RocksDBException;
protected native List<BackupInfo> getBackupInfo(long handle);
private native void dispose(long nativeHandle);
}
+9 -78
View File
@@ -18,6 +18,8 @@ import org.rocksdb.util.Environment;
public class RocksDB extends RocksObject {
public static final String DEFAULT_COLUMN_FAMILY = "default";
public static final int NOT_FOUND = -1;
private static final String[] compressionLibs_ = {
"snappy", "z", "bzip2", "lz4", "lz4hc"};
static {
RocksDB.loadLibrary();
@@ -33,18 +35,16 @@ public class RocksDB extends RocksObject {
public static synchronized void loadLibrary() {
String tmpDir = System.getenv("ROCKSDB_SHAREDLIB_DIR");
// loading possibly necessary libraries.
for (CompressionType compressionType : CompressionType.values()) {
for (String lib : compressionLibs_) {
try {
if (compressionType.getLibraryName() != null) {
System.loadLibrary(compressionType.getLibraryName());
}
System.loadLibrary(lib);
} catch (UnsatisfiedLinkError e) {
// since it may be optional, we ignore its loading failure here.
}
}
try
{
NativeLibraryLoader.getInstance().loadLibraryFromJar(tmpDir);
NativeLibraryLoader.loadLibraryFromJar(tmpDir);
}
catch (IOException e)
{
@@ -60,14 +60,10 @@ public class RocksDB extends RocksObject {
* of a library.
*/
public static synchronized void loadLibrary(List<String> paths) {
for (CompressionType compressionType : CompressionType.values()) {
if (compressionType.equals(CompressionType.NO_COMPRESSION)) {
continue;
}
for (String lib : compressionLibs_) {
for (String path : paths) {
try {
System.load(path + "/" + Environment.getSharedLibraryName(
compressionType.getLibraryName()));
System.load(path + "/" + Environment.getSharedLibraryName(lib));
break;
} catch (UnsatisfiedLinkError e) {
// since they are optional, we ignore loading fails.
@@ -107,7 +103,6 @@ public class RocksDB extends RocksObject {
// This allows to use the rocksjni default Options instead of
// the c++ one.
Options options = new Options();
options.setCreateIfMissing(true);
return open(options, path);
}
@@ -368,10 +363,8 @@ public class RocksDB extends RocksObject {
}
@Override protected void disposeInternal() {
synchronized (this) {
assert (isInitialized());
disposeInternal(nativeHandle_);
}
assert(isInitialized());
disposeInternal(nativeHandle_);
}
/**
@@ -1036,62 +1029,6 @@ public class RocksDB extends RocksObject {
return getProperty0(nativeHandle_, property, property.length());
}
/**
* <p> Similar to GetProperty(), but only works for a subset of properties whose
* return value is a numerical value. Return the value as long.</p>
*
* <p><strong>Note</strong>: As the returned property is of type
* {@code uint64_t} on C++ side the returning value can be negative
* because Java supports in Java 7 only signed long values.</p>
*
* <p><strong>Java 7</strong>: To mitigate the problem of the non
* existent unsigned long tpye, values should be encapsulated using
* {@link java.math.BigInteger} to reflect the correct value. The correct
* behavior is guaranteed if {@code 2^64} is added to negative values.</p>
*
* <p><strong>Java 8</strong>: In Java 8 the value should be treated as
* unsigned long using provided methods of type {@link Long}.</p>
*
* @param property to be fetched.
*
* @return numerical property value.
*
* @throws RocksDBException if an error happens in the underlying native code.
*/
public long getLongProperty(String property) throws RocksDBException {
return getLongProperty(nativeHandle_, property, property.length());
}
/**
* <p> Similar to GetProperty(), but only works for a subset of properties whose
* return value is a numerical value. Return the value as long.</p>
*
* <p><strong>Note</strong>: As the returned property is of type
* {@code uint64_t} on C++ side the returning value can be negative
* because Java supports in Java 7 only signed long values.</p>
*
* <p><strong>Java 7</strong>: To mitigate the problem of the non
* existent unsigned long tpye, values should be encapsulated using
* {@link java.math.BigInteger} to reflect the correct value. The correct
* behavior is guaranteed if {@code 2^64} is added to negative values.</p>
*
* <p><strong>Java 8</strong>: In Java 8 the value should be treated as
* unsigned long using provided methods of type {@link Long}.</p>
*
* @param columnFamilyHandle {@link org.rocksdb.ColumnFamilyHandle}
* instance
* @param property to be fetched.
*
* @return numerical property value
*
* @throws RocksDBException if an error happens in the underlying native code.
*/
public long getLongProperty(ColumnFamilyHandle columnFamilyHandle, String property)
throws RocksDBException {
return getLongProperty(nativeHandle_, columnFamilyHandle.nativeHandle_, property,
property.length());
}
/**
* Return a heap-allocated iterator over the contents of the database.
* The result of newIterator() is initially invalid (caller must
@@ -1213,8 +1150,6 @@ public class RocksDB extends RocksObject {
throws RocksDBException, IllegalArgumentException {
// throws RocksDBException if something goes wrong
dropColumnFamily(nativeHandle_, columnFamilyHandle.nativeHandle_);
// After the drop the native handle is not valid anymore
columnFamilyHandle.nativeHandle_ = 0;
}
/**
@@ -1357,10 +1292,6 @@ public class RocksDB extends RocksObject {
String property, int propertyLength) throws RocksDBException;
protected native String getProperty0(long nativeHandle, long cfHandle,
String property, int propertyLength) throws RocksDBException;
protected native long getLongProperty(long nativeHandle,
String property, int propertyLength) throws RocksDBException;
protected native long getLongProperty(long nativeHandle, long cfHandle,
String property, int propertyLength) throws RocksDBException;
protected native long iterator0(long handle);
protected native long iterator0(long handle, long cfHandle);
protected native long[] iterators(long handle,
+1 -2
View File
@@ -18,7 +18,6 @@ public class RocksEnv extends RocksObject {
static {
default_env_ = new RocksEnv(getDefaultEnvInternal());
}
private static native long getDefaultEnvInternal();
@@ -102,7 +101,7 @@ public class RocksEnv extends RocksObject {
* {@link RocksObject} must implement to release their associated C++
* resource.
*/
@Override protected void disposeInternal() {
protected void disposeInternal() {
disposeInternal(nativeHandle_);
}
private native void disposeInternal(long handle);
+4 -6
View File
@@ -143,11 +143,9 @@ public class RocksIterator extends RocksObject {
* before freeing the native handle.</p>
*/
@Override protected void disposeInternal() {
synchronized (rocksDB_) {
assert (isInitialized());
if (rocksDB_.isInitialized()) {
disposeInternal(nativeHandle_);
}
assert(isInitialized());
if (rocksDB_.isInitialized()) {
disposeInternal(nativeHandle_);
}
}
@@ -162,5 +160,5 @@ public class RocksIterator extends RocksObject {
private native void seek0(long handle, byte[] target, int targetLen);
private native void status0(long handle);
final RocksDB rocksDB_;
RocksDB rocksDB_;
}
+10
View File
@@ -221,3 +221,13 @@ public class WriteBatch extends RocksObject {
private native void disposeInternal(long handle);
}
}
/**
* Package-private class which provides java api to access
* c++ WriteBatchInternal.
*/
class WriteBatchInternal {
static native void setSequence(WriteBatch batch, long sn);
static native long sequence(WriteBatch batch);
static native void append(WriteBatch b1, WriteBatch b2);
}
+124
View File
@@ -0,0 +1,124 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
//
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
package org.rocksdb;
import java.util.*;
import java.io.UnsupportedEncodingException;
/**
* This class mimics the db/write_batch_test.cc in the c++ rocksdb library.
*/
public class WriteBatchTest {
static {
RocksDB.loadLibrary();
}
public static void main(String args[]) {
System.out.println("Testing WriteBatchTest.Empty ===");
Empty();
System.out.println("Testing WriteBatchTest.Multiple ===");
Multiple();
System.out.println("Testing WriteBatchTest.Append ===");
Append();
System.out.println("Testing WriteBatchTest.Blob ===");
Blob();
// The following tests have not yet ported.
// Continue();
// PutGatherSlices();
System.out.println("Passed all WriteBatchTest!");
}
static void Empty() {
WriteBatch batch = new WriteBatch();
assert(batch.count() == 0);
}
static void Multiple() {
try {
WriteBatch batch = new WriteBatch();
batch.put("foo".getBytes("US-ASCII"), "bar".getBytes("US-ASCII"));
batch.remove("box".getBytes("US-ASCII"));
batch.put("baz".getBytes("US-ASCII"), "boo".getBytes("US-ASCII"));
WriteBatchInternal.setSequence(batch, 100);
assert(100 == WriteBatchInternal.sequence(batch));
assert(3 == batch.count());
assert(("Put(baz, boo)@102" +
"Delete(box)@101" +
"Put(foo, bar)@100")
.equals(new String(getContents(batch), "US-ASCII")));
} catch (UnsupportedEncodingException e) {
System.err.println(e);
assert(false);
}
}
static void Append() {
WriteBatch b1 = new WriteBatch();
WriteBatch b2 = new WriteBatch();
WriteBatchInternal.setSequence(b1, 200);
WriteBatchInternal.setSequence(b2, 300);
WriteBatchInternal.append(b1, b2);
assert(getContents(b1).length == 0);
assert(b1.count() == 0);
try {
b2.put("a".getBytes("US-ASCII"), "va".getBytes("US-ASCII"));
WriteBatchInternal.append(b1, b2);
assert("Put(a, va)@200".equals(new String(getContents(b1), "US-ASCII")));
assert(1 == b1.count());
b2.clear();
b2.put("b".getBytes("US-ASCII"), "vb".getBytes("US-ASCII"));
WriteBatchInternal.append(b1, b2);
assert(("Put(a, va)@200" +
"Put(b, vb)@201")
.equals(new String(getContents(b1), "US-ASCII")));
assert(2 == b1.count());
b2.remove("foo".getBytes("US-ASCII"));
WriteBatchInternal.append(b1, b2);
assert(("Put(a, va)@200" +
"Put(b, vb)@202" +
"Put(b, vb)@201" +
"Delete(foo)@203")
.equals(new String(getContents(b1), "US-ASCII")));
assert(4 == b1.count());
} catch (UnsupportedEncodingException e) {
System.err.println(e);
assert(false);
}
}
static void Blob() {
WriteBatch batch = new WriteBatch();
try {
batch.put("k1".getBytes("US-ASCII"), "v1".getBytes("US-ASCII"));
batch.put("k2".getBytes("US-ASCII"), "v2".getBytes("US-ASCII"));
batch.put("k3".getBytes("US-ASCII"), "v3".getBytes("US-ASCII"));
batch.putLogData("blob1".getBytes("US-ASCII"));
batch.remove("k2".getBytes("US-ASCII"));
batch.putLogData("blob2".getBytes("US-ASCII"));
batch.merge("foo".getBytes("US-ASCII"), "bar".getBytes("US-ASCII"));
assert(5 == batch.count());
assert(("Merge(foo, bar)@4" +
"Put(k1, v1)@0" +
"Delete(k2)@3" +
"Put(k2, v2)@1" +
"Put(k3, v3)@2")
.equals(new String(getContents(batch), "US-ASCII")));
} catch (UnsupportedEncodingException e) {
System.err.println(e);
assert(false);
}
}
static native byte[] getContents(WriteBatch batch);
}
+88 -95
View File
@@ -163,6 +163,15 @@ public class DbBenchmark {
EXISTING
}
enum CompressionType {
NONE,
SNAPPY,
ZLIB,
BZIP2,
LZ4,
LZ4HC
}
static {
RocksDB.loadLibrary();
}
@@ -448,16 +457,18 @@ public class DbBenchmark {
// options.setPrefixSize((Integer)flags_.get(Flag.prefix_size));
// options.setKeysPerPrefix((Long)flags_.get(Flag.keys_per_prefix));
compressionType_ = (String) flags.get(Flag.compression_type);
compression_ = CompressionType.NO_COMPRESSION;
compression_ = CompressionType.NONE;
try {
if (compressionType_!=null) {
final CompressionType compressionType =
CompressionType.getCompressionType(compressionType_);
if (compressionType != null &&
compressionType != CompressionType.NO_COMPRESSION) {
System.loadLibrary(compressionType.getLibraryName());
}
if (compressionType_.equals("snappy")) {
System.loadLibrary("snappy");
} else if (compressionType_.equals("zlib")) {
System.loadLibrary("z");
} else if (compressionType_.equals("bzip2")) {
System.loadLibrary("bzip2");
} else if (compressionType_.equals("lz4")) {
System.loadLibrary("lz4");
} else if (compressionType_.equals("lz4hc")) {
System.loadLibrary("lz4hc");
}
} catch (UnsatisfiedLinkError e) {
System.err.format("Unable to load %s library:%s%n" +
@@ -484,32 +495,26 @@ public class DbBenchmark {
} else {
options.setCreateIfMissing(false);
}
switch (memtable_) {
case "skip_list":
options.setMemTableConfig(new SkipListMemTableConfig());
break;
case "vector":
options.setMemTableConfig(new VectorMemTableConfig());
break;
case "hash_linkedlist":
options.setMemTableConfig(
new HashLinkedListMemTableConfig()
.setBucketCount(hashBucketCount_));
options.useFixedLengthPrefixExtractor(prefixSize_);
break;
case "hash_skiplist":
case "prefix_hash":
options.setMemTableConfig(
new HashSkipListMemTableConfig()
.setBucketCount(hashBucketCount_));
options.useFixedLengthPrefixExtractor(prefixSize_);
break;
default:
System.err.format(
"unable to detect the specified memtable, " +
"use the default memtable factory %s%n",
options.memTableFactoryName());
break;
if (memtable_.equals("skip_list")) {
options.setMemTableConfig(new SkipListMemTableConfig());
} else if (memtable_.equals("vector")) {
options.setMemTableConfig(new VectorMemTableConfig());
} else if (memtable_.equals("hash_linkedlist")) {
options.setMemTableConfig(
new HashLinkedListMemTableConfig()
.setBucketCount(hashBucketCount_));
options.useFixedLengthPrefixExtractor(prefixSize_);
} else if (memtable_.equals("hash_skiplist") ||
memtable_.equals("prefix_hash")) {
options.setMemTableConfig(
new HashSkipListMemTableConfig()
.setBucketCount(hashBucketCount_));
options.useFixedLengthPrefixExtractor(prefixSize_);
} else {
System.err.format(
"unable to detect the specified memtable, " +
"use the default memtable factory %s%n",
options.memTableFactoryName());
}
if (usePlainTable_) {
options.setTableFormatConfig(
@@ -640,65 +645,53 @@ public class DbBenchmark {
int currentTaskId = 0;
boolean known = true;
switch (benchmark) {
case "fillseq":
tasks.add(new WriteSequentialTask(
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
break;
case "fillbatch":
tasks.add(new WriteRandomTask(
currentTaskId++, randSeed_, num_ / 1000, num_, writeOpt, 1000));
break;
case "fillrandom":
tasks.add(new WriteRandomTask(
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
break;
case "filluniquerandom":
tasks.add(new WriteUniqueRandomTask(
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
break;
case "fillsync":
writeOpt.setSync(true);
tasks.add(new WriteRandomTask(
currentTaskId++, randSeed_, num_ / 1000, num_ / 1000,
writeOpt, 1));
break;
case "readseq":
for (int t = 0; t < threadNum_; ++t) {
tasks.add(new ReadSequentialTask(
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
}
break;
case "readrandom":
for (int t = 0; t < threadNum_; ++t) {
tasks.add(new ReadRandomTask(
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
}
break;
case "readwhilewriting":
WriteTask writeTask = new WriteRandomTask(
-1, randSeed_, Long.MAX_VALUE, num_, writeOpt, 1, writesPerSeconds_);
writeTask.stats_.setExcludeFromMerge();
bgTasks.add(writeTask);
for (int t = 0; t < threadNum_; ++t) {
tasks.add(new ReadRandomTask(
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
}
break;
case "readhot":
for (int t = 0; t < threadNum_; ++t) {
tasks.add(new ReadRandomTask(
currentTaskId++, randSeed_, reads_ / threadNum_, num_ / 100));
}
break;
case "delete":
destroyDb();
open(options);
break;
default:
known = false;
System.err.println("Unknown benchmark: " + benchmark);
break;
if (benchmark.equals("fillseq")) {
tasks.add(new WriteSequentialTask(
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
} else if (benchmark.equals("fillbatch")) {
tasks.add(new WriteRandomTask(
currentTaskId++, randSeed_, num_ / 1000, num_, writeOpt, 1000));
} else if (benchmark.equals("fillrandom")) {
tasks.add(new WriteRandomTask(
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
} else if (benchmark.equals("filluniquerandom")) {
tasks.add(new WriteUniqueRandomTask(
currentTaskId++, randSeed_, num_, num_, writeOpt, 1));
} else if (benchmark.equals("fillsync")) {
writeOpt.setSync(true);
tasks.add(new WriteRandomTask(
currentTaskId++, randSeed_, num_ / 1000, num_ / 1000,
writeOpt, 1));
} else if (benchmark.equals("readseq")) {
for (int t = 0; t < threadNum_; ++t) {
tasks.add(new ReadSequentialTask(
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
}
} else if (benchmark.equals("readrandom")) {
for (int t = 0; t < threadNum_; ++t) {
tasks.add(new ReadRandomTask(
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
}
} else if (benchmark.equals("readwhilewriting")) {
WriteTask writeTask = new WriteRandomTask(
-1, randSeed_, Long.MAX_VALUE, num_, writeOpt, 1, writesPerSeconds_);
writeTask.stats_.setExcludeFromMerge();
bgTasks.add(writeTask);
for (int t = 0; t < threadNum_; ++t) {
tasks.add(new ReadRandomTask(
currentTaskId++, randSeed_, reads_ / threadNum_, num_));
}
} else if (benchmark.equals("readhot")) {
for (int t = 0; t < threadNum_; ++t) {
tasks.add(new ReadRandomTask(
currentTaskId++, randSeed_, reads_ / threadNum_, num_ / 100));
}
} else if (benchmark.equals("delete")) {
destroyDb();
open(options);
} else {
known = false;
System.err.println("Unknown benchmark: " + benchmark);
}
if (known) {
ExecutorService executor = Executors.newCachedThreadPool();
@@ -807,7 +800,7 @@ public class DbBenchmark {
System.out.printf(
"%-16s : %11.5f micros/op; %6.1f MB/s;%s %d / %d task(s) finished.\n",
benchmark, elapsedSeconds / stats.done_ * 1e6,
benchmark, (double) elapsedSeconds / stats.done_ * 1e6,
(stats.bytes_ / 1048576.0) / elapsedSeconds, extra,
taskFinishedCount, concurrentThreads);
}
@@ -9,11 +9,9 @@ import org.rocksdb.*;
import java.io.IOException;
import java.nio.file.*;
import java.util.ArrayList;
import java.util.List;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
import static org.rocksdb.test.Types.byteToInt;
import static org.rocksdb.test.Types.intToByte;
@@ -41,7 +39,7 @@ public abstract class AbstractComparatorTest {
*
* @throws java.io.IOException if IO error happens.
*/
public void testRoundtrip(final Path db_path) throws IOException, RocksDBException {
public void testRoundtrip(final Path db_path) throws IOException {
Options opt = null;
RocksDB db = null;
@@ -66,6 +64,7 @@ public abstract class AbstractComparatorTest {
}
db.close();
// re-open db and read from start to end
// integer keys should be in ascending
// order as defined by SimpleIntComparator
@@ -76,14 +75,17 @@ public abstract class AbstractComparatorTest {
int count = 0;
for (it.seekToFirst(); it.isValid(); it.next()) {
final int thisKey = byteToInt(it.key());
assertThat(thisKey).isGreaterThan(lastKey);
assert(thisKey > lastKey);
lastKey = thisKey;
count++;
}
db.close();
assertThat(count).isEqualTo(ITERATIONS);
assert(count == ITERATIONS);
} catch (final RocksDBException e) {
System.err.format("[ERROR]: %s%n", e);
e.printStackTrace();
} finally {
if (db != null) {
db.close();
@@ -92,97 +94,8 @@ public abstract class AbstractComparatorTest {
if (opt != null) {
opt.dispose();
}
}
}
/**
* Test which stores random keys into a column family
* in the database
* using an @see getAscendingIntKeyComparator
* it then checks that these keys are read back in
* ascending order
*
* @param db_path A path where we can store database
* files temporarily
*
* @throws java.io.IOException if IO error happens.
*/
public void testRoundtripCf(final Path db_path) throws IOException,
RocksDBException {
DBOptions opt = null;
RocksDB db = null;
List<ColumnFamilyDescriptor> cfDescriptors =
new ArrayList<>();
cfDescriptors.add(new ColumnFamilyDescriptor(
RocksDB.DEFAULT_COLUMN_FAMILY));
cfDescriptors.add(new ColumnFamilyDescriptor("new_cf",
new ColumnFamilyOptions().setComparator(
getAscendingIntKeyComparator())));
List<ColumnFamilyHandle> cfHandles = new ArrayList<>();
try {
opt = new DBOptions().
setCreateIfMissing(true).
setCreateMissingColumnFamilies(true);
// store 10,000 random integer keys
final int ITERATIONS = 10000;
db = RocksDB.open(opt, db_path.toString(), cfDescriptors, cfHandles);
assertThat(cfDescriptors.size()).isEqualTo(2);
assertThat(cfHandles.size()).isEqualTo(2);
final Random random = new Random();
for (int i = 0; i < ITERATIONS; i++) {
final byte key[] = intToByte(random.nextInt());
if (i > 0 && db.get(cfHandles.get(1), key) != null) {
// does key already exist (avoid duplicates)
i--; // generate a different key
} else {
db.put(cfHandles.get(1), key, "value".getBytes());
}
}
for (ColumnFamilyHandle handle : cfHandles) {
handle.dispose();
}
cfHandles.clear();
db.close();
// re-open db and read from start to end
// integer keys should be in ascending
// order as defined by SimpleIntComparator
db = RocksDB.open(opt, db_path.toString(), cfDescriptors, cfHandles);
assertThat(cfDescriptors.size()).isEqualTo(2);
assertThat(cfHandles.size()).isEqualTo(2);
final RocksIterator it = db.newIterator(cfHandles.get(1));
it.seekToFirst();
int lastKey = Integer.MIN_VALUE;
int count = 0;
for (it.seekToFirst(); it.isValid(); it.next()) {
final int thisKey = byteToInt(it.key());
assertThat(thisKey).isGreaterThan(lastKey);
lastKey = thisKey;
count++;
}
for (ColumnFamilyHandle handle : cfHandles) {
handle.dispose();
}
cfHandles.clear();
db.close();
assertThat(count).isEqualTo(ITERATIONS);
} finally {
for (ColumnFamilyHandle handle : cfHandles) {
handle.dispose();
}
if (db != null) {
db.close();
}
if (opt != null) {
opt.dispose();
}
removeDb(db_path); // cleanup after ourselves!
}
}
@@ -213,4 +126,43 @@ public abstract class AbstractComparatorTest {
return result;
}
/**
* Utility method for deleting database files
*
* @param db_path The path to the database to remove
* from the filesystem
*/
private static void removeDb(final Path db_path) throws IOException {
Files.walkFileTree(db_path, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs)
throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(final Path file, IOException exc)
throws IOException {
// try to delete the file anyway, even if its attributes
// could not be read, since delete-only access is
// theoretically possible
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(final Path dir, IOException exc)
throws IOException {
if (exc == null) {
Files.delete(dir);
return FileVisitResult.CONTINUE;
} else {
// directory iteration failed; propagate exception
throw exc;
}
}
});
}
}
@@ -1,284 +0,0 @@
// Copyright (c) 2014, Facebook, Inc. All rights reserved.
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree. An additional grant
// of patent rights can be found in the PATENTS file in the same directory.
package org.rocksdb.test;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.rocksdb.BackupableDBOptions;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
public class BackupableDBOptionsTest {
private final static String ARBITRARY_PATH = "/tmp";
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
@Rule
public ExpectedException exception = ExpectedException.none();
public static final Random rand = PlatformRandomHelper.
getPlatformSpecificRandomFactory();
@Test
public void backupDir() {
BackupableDBOptions backupableDBOptions = null;
try {
backupableDBOptions = new BackupableDBOptions(ARBITRARY_PATH);
assertThat(backupableDBOptions.backupDir()).
isEqualTo(ARBITRARY_PATH);
} finally {
if (backupableDBOptions != null) {
backupableDBOptions.dispose();
}
}
}
@Test
public void shareTableFiles() {
BackupableDBOptions backupableDBOptions = null;
try {
backupableDBOptions = new BackupableDBOptions(ARBITRARY_PATH);
boolean value = rand.nextBoolean();
backupableDBOptions.setShareTableFiles(value);
assertThat(backupableDBOptions.shareTableFiles()).
isEqualTo(value);
} finally {
if (backupableDBOptions != null) {
backupableDBOptions.dispose();
}
}
}
@Test
public void sync() {
BackupableDBOptions backupableDBOptions = null;
try {
backupableDBOptions = new BackupableDBOptions(ARBITRARY_PATH);
boolean value = rand.nextBoolean();
backupableDBOptions.setSync(value);
assertThat(backupableDBOptions.sync()).isEqualTo(value);
} finally {
if (backupableDBOptions != null) {
backupableDBOptions.dispose();
}
}
}
@Test
public void destroyOldData() {
BackupableDBOptions backupableDBOptions = null;
try {
backupableDBOptions = new BackupableDBOptions(ARBITRARY_PATH);
boolean value = rand.nextBoolean();
backupableDBOptions.setDestroyOldData(value);
assertThat(backupableDBOptions.destroyOldData()).
isEqualTo(value);
} finally {
if (backupableDBOptions != null) {
backupableDBOptions.dispose();
}
}
}
@Test
public void backupLogFiles() {
BackupableDBOptions backupableDBOptions = null;
try {
backupableDBOptions = new BackupableDBOptions(ARBITRARY_PATH);
boolean value = rand.nextBoolean();
backupableDBOptions.setBackupLogFiles(value);
assertThat(backupableDBOptions.backupLogFiles()).
isEqualTo(value);
} finally {
if (backupableDBOptions != null) {
backupableDBOptions.dispose();
}
}
}
@Test
public void backupRateLimit() {
BackupableDBOptions backupableDBOptions = null;
try {
backupableDBOptions = new BackupableDBOptions(ARBITRARY_PATH);
long value = Math.abs(rand.nextLong());
backupableDBOptions.setBackupRateLimit(value);
assertThat(backupableDBOptions.backupRateLimit()).
isEqualTo(value);
// negative will be mapped to 0
backupableDBOptions.setBackupRateLimit(-1);
assertThat(backupableDBOptions.backupRateLimit()).
isEqualTo(0);
} finally {
if (backupableDBOptions != null) {
backupableDBOptions.dispose();
}
}
}
@Test
public void restoreRateLimit() {
BackupableDBOptions backupableDBOptions = null;
try {
backupableDBOptions = new BackupableDBOptions(ARBITRARY_PATH);
long value = Math.abs(rand.nextLong());
backupableDBOptions.setRestoreRateLimit(value);
assertThat(backupableDBOptions.restoreRateLimit()).
isEqualTo(value);
// negative will be mapped to 0
backupableDBOptions.setRestoreRateLimit(-1);
assertThat(backupableDBOptions.restoreRateLimit()).
isEqualTo(0);
} finally {
if (backupableDBOptions != null) {
backupableDBOptions.dispose();
}
}
}
@Test
public void shareFilesWithChecksum() {
BackupableDBOptions backupableDBOptions = null;
try {
backupableDBOptions = new BackupableDBOptions(ARBITRARY_PATH);
boolean value = rand.nextBoolean();
backupableDBOptions.setShareFilesWithChecksum(value);
assertThat(backupableDBOptions.shareFilesWithChecksum()).
isEqualTo(value);
} finally {
if (backupableDBOptions != null) {
backupableDBOptions.dispose();
}
}
}
@Test
public void failBackupDirIsNull() {
exception.expect(IllegalArgumentException.class);
new BackupableDBOptions(null);
}
@Test
public void failBackupDirIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.backupDir();
}
@Test
public void failSetShareTableFilesIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.setShareTableFiles(true);
}
@Test
public void failShareTableFilesIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.shareTableFiles();
}
@Test
public void failSetSyncIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.setSync(true);
}
@Test
public void failSyncIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.sync();
}
@Test
public void failSetDestroyOldDataIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.setDestroyOldData(true);
}
@Test
public void failDestroyOldDataIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.destroyOldData();
}
@Test
public void failSetBackupLogFilesIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.setBackupLogFiles(true);
}
@Test
public void failBackupLogFilesIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.backupLogFiles();
}
@Test
public void failSetBackupRateLimitIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.setBackupRateLimit(1);
}
@Test
public void failBackupRateLimitIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.backupRateLimit();
}
@Test
public void failSetRestoreRateLimitIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.setRestoreRateLimit(1);
}
@Test
public void failRestoreRateLimitIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.restoreRateLimit();
}
@Test
public void failSetShareFilesWithChecksumIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.setShareFilesWithChecksum(true);
}
@Test
public void failShareFilesWithChecksumIfDisposed(){
BackupableDBOptions options = setupUninitializedBackupableDBOptions(
exception);
options.shareFilesWithChecksum();
}
private BackupableDBOptions setupUninitializedBackupableDBOptions(
ExpectedException exception) {
BackupableDBOptions backupableDBOptions =
new BackupableDBOptions(ARBITRARY_PATH);
backupableDBOptions.dispose();
exception.expect(AssertionError.class);
return backupableDBOptions;
}
}
+97 -398
View File
@@ -5,422 +5,121 @@
package org.rocksdb.test;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.*;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class BackupableDBTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
@Rule
public TemporaryFolder dbFolder = new TemporaryFolder();
@Rule
public TemporaryFolder backupFolder = new TemporaryFolder();
@Test
public void backupDb() throws RocksDBException {
Options opt = null;
BackupableDBOptions bopt = null;
BackupableDB bdb = null;
try {
opt = new Options().setCreateIfMissing(true);
bopt = new BackupableDBOptions(
backupFolder.getRoot().getAbsolutePath());
assertThat(bopt.backupDir()).isEqualTo(
backupFolder.getRoot().getAbsolutePath());
// Open empty database.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Fill database with some test values
prepareDatabase(bdb);
// Create two backups
bdb.createNewBackup(false);
bdb.createNewBackup(true);
verifyNumberOfValidBackups(bdb, 2);
} finally {
if (bdb != null) {
bdb.close();
}
if (bopt != null) {
bopt.dispose();
}
if (opt != null) {
opt.dispose();
}
}
static final String db_path = "/tmp/rocksdbjni_backupable_db_test";
static final String backup_path = "/tmp/rocksdbjni_backupable_db_backup_test";
static {
RocksDB.loadLibrary();
}
public static void main(String[] args) {
@Test
public void deleteBackup() throws RocksDBException {
Options opt = null;
BackupableDBOptions bopt = null;
BackupableDB bdb = null;
try {
opt = new Options().setCreateIfMissing(true);
bopt = new BackupableDBOptions(
backupFolder.getRoot().getAbsolutePath());
assertThat(bopt.backupDir()).isEqualTo(
backupFolder.getRoot().getAbsolutePath());
// Open empty database.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Fill database with some test values
prepareDatabase(bdb);
// Create two backups
bdb.createNewBackup(false);
bdb.createNewBackup(true);
List<BackupInfo> backupInfo =
verifyNumberOfValidBackups(bdb, 2);
// Delete the first backup
bdb.deleteBackup(backupInfo.get(0).backupId());
List<BackupInfo> newBackupInfo =
verifyNumberOfValidBackups(bdb, 1);
// The second backup must remain.
assertThat(newBackupInfo.get(0).backupId()).
isEqualTo(backupInfo.get(1).backupId());
} finally {
if (bdb != null) {
bdb.close();
}
if (bopt != null) {
bopt.dispose();
}
if (opt != null) {
opt.dispose();
}
}
}
Options opt = new Options();
opt.setCreateIfMissing(true);
@Test
public void deleteBackupWithRestoreBackupableDB()
throws RocksDBException {
Options opt = null;
BackupableDBOptions bopt = null;
BackupableDBOptions bopt = new BackupableDBOptions(backup_path, false,
true, false, true, 0, 0);
BackupableDB bdb = null;
RestoreBackupableDB rdb = null;
List<BackupInfo> backupInfos;
List<BackupInfo> restoreInfos;
try {
opt = new Options().setCreateIfMissing(true);
bopt = new BackupableDBOptions(
backupFolder.getRoot().getAbsolutePath());
assertThat(bopt.backupDir()).isEqualTo(
backupFolder.getRoot().getAbsolutePath());
// Open empty database.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Fill database with some test values
prepareDatabase(bdb);
// Create two backups
bdb.createNewBackup(false);
bdb.createNewBackup(true);
List<BackupInfo> backupInfo =
verifyNumberOfValidBackups(bdb, 2);
// init RestoreBackupableDB
rdb = new RestoreBackupableDB(bopt);
// Delete the first backup
rdb.deleteBackup(backupInfo.get(0).backupId());
// Fetch backup info using RestoreBackupableDB
List<BackupInfo> newBackupInfo = verifyNumberOfValidBackups(rdb, 1);
// The second backup must remain.
assertThat(newBackupInfo.get(0).backupId()).
isEqualTo(backupInfo.get(1).backupId());
} finally {
if (bdb != null) {
bdb.close();
}
if (rdb != null) {
rdb.dispose();
}
if (bopt != null) {
bopt.dispose();
}
if (opt != null) {
opt.dispose();
}
}
}
bdb = BackupableDB.open(opt, bopt, db_path);
@Test
public void purgeOldBackups() throws RocksDBException {
Options opt = null;
BackupableDBOptions bopt = null;
BackupableDB bdb = null;
try {
opt = new Options().setCreateIfMissing(true);
bopt = new BackupableDBOptions(
backupFolder.getRoot().getAbsolutePath());
assertThat(bopt.backupDir()).isEqualTo(
backupFolder.getRoot().getAbsolutePath());
// Open empty database.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Fill database with some test values
prepareDatabase(bdb);
// Create two backups
bdb.createNewBackup(false);
bdb.createNewBackup(true);
bdb.createNewBackup(true);
bdb.createNewBackup(true);
List<BackupInfo> backupInfo =
verifyNumberOfValidBackups(bdb, 4);
// Delete everything except the latest backup
bdb.purgeOldBackups(1);
List<BackupInfo> newBackupInfo =
verifyNumberOfValidBackups(bdb, 1);
// The latest backup must remain.
assertThat(newBackupInfo.get(0).backupId()).
isEqualTo(backupInfo.get(3).backupId());
} finally {
if (bdb != null) {
bdb.close();
}
if (bopt != null) {
bopt.dispose();
}
if (opt != null) {
opt.dispose();
}
}
}
bdb.put("abc".getBytes(), "def".getBytes());
bdb.put("ghi".getBytes(), "jkl".getBytes());
backupInfos = bdb.getBackupInfos();
assert(backupInfos.size() == 0);
@Test
public void purgeOldBackupsWithRestoreBackupableDb()
throws RocksDBException {
Options opt = null;
BackupableDBOptions bopt = null;
BackupableDB bdb = null;
RestoreBackupableDB rdb = null;
try {
opt = new Options().setCreateIfMissing(true);
bopt = new BackupableDBOptions(
backupFolder.getRoot().getAbsolutePath());
assertThat(bopt.backupDir()).isEqualTo(
backupFolder.getRoot().getAbsolutePath());
// Open empty database.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Fill database with some test values
prepareDatabase(bdb);
// Create two backups
bdb.createNewBackup(false);
bdb.createNewBackup(true);
bdb.createNewBackup(true);
bdb.createNewBackup(true);
List<BackupInfo> infos = verifyNumberOfValidBackups(bdb, 4);
assertThat(infos.get(1).size()).
isEqualTo(infos.get(2).size());
assertThat(infos.get(1).numberFiles()).
isEqualTo(infos.get(2).numberFiles());
long maxTimeBeforePurge = Long.MIN_VALUE;
for (BackupInfo backupInfo : infos) {
if (maxTimeBeforePurge < backupInfo.timestamp()) {
maxTimeBeforePurge = backupInfo.timestamp();
}
}
// init RestoreBackupableDB
rdb = new RestoreBackupableDB(bopt);
// the same number of backups must
// exist using RestoreBackupableDB.
verifyNumberOfValidBackups(rdb, 4);
backupInfos = bdb.getBackupInfos();
assert(backupInfos.size() == 1);
// Retrieving backup infos twice shall not
// lead to different results
List<BackupInfo> tmpBackupInfo = bdb.getBackupInfos();
assert(tmpBackupInfo.get(0).backupId() ==
backupInfos.get(0).backupId());
assert(tmpBackupInfo.get(0).timestamp() ==
backupInfos.get(0).timestamp());
assert(tmpBackupInfo.get(0).size() ==
backupInfos.get(0).size());
assert(tmpBackupInfo.get(0).numberFiles() ==
backupInfos.get(0).numberFiles());
// delete record after backup
bdb.remove("abc".getBytes());
byte[] value = bdb.get("abc".getBytes());
assert(value == null);
bdb.close();
// restore from backup
RestoreOptions ropt = new RestoreOptions(false);
RestoreBackupableDB rdb = new RestoreBackupableDB(bopt);
// getting backup infos from restorable db should
// lead to the same infos as from backupable db
restoreInfos = rdb.getBackupInfos();
assert(restoreInfos.size() == backupInfos.size());
assert(restoreInfos.get(0).backupId() ==
backupInfos.get(0).backupId());
assert(restoreInfos.get(0).timestamp() ==
backupInfos.get(0).timestamp());
assert(restoreInfos.get(0).size() ==
backupInfos.get(0).size());
assert(restoreInfos.get(0).numberFiles() ==
backupInfos.get(0).numberFiles());
rdb.restoreDBFromLatestBackup(db_path, db_path,
ropt);
// do nothing because there is only one backup
rdb.purgeOldBackups(1);
infos = verifyNumberOfValidBackups(rdb, 1);
assertThat(infos.get(0).timestamp()).
isEqualTo(maxTimeBeforePurge);
restoreInfos = rdb.getBackupInfos();
assert(restoreInfos.size() == 1);
rdb.dispose();
ropt.dispose();
// verify that backed up data contains deleted record
bdb = BackupableDB.open(opt, bopt, db_path);
value = bdb.get("abc".getBytes());
assert(new String(value).equals("def"));
bdb.createNewBackup(false);
// after new backup there must be two backup infos
backupInfos = bdb.getBackupInfos();
assert(backupInfos.size() == 2);
// deleting the backup must be possible using the
// id provided by backup infos
bdb.deleteBackup(backupInfos.get(1).backupId());
// after deletion there should only be one info
backupInfos = bdb.getBackupInfos();
assert(backupInfos.size() == 1);
bdb.createNewBackup(false);
bdb.createNewBackup(false);
bdb.createNewBackup(false);
backupInfos = bdb.getBackupInfos();
assert(backupInfos.size() == 4);
// purge everything and keep two
bdb.purgeOldBackups(2);
// backup infos need to be two
backupInfos = bdb.getBackupInfos();
assert(backupInfos.size() == 2);
assert(backupInfos.get(0).backupId() == 4);
assert(backupInfos.get(1).backupId() == 5);
System.out.println("Backup and restore test passed");
} catch (RocksDBException e) {
System.err.format("[ERROR]: %s%n", e);
e.printStackTrace();
} finally {
opt.dispose();
bopt.dispose();
if (bdb != null) {
bdb.close();
}
if (rdb != null) {
rdb.dispose();
}
if (bopt != null) {
bopt.dispose();
}
if (opt != null) {
opt.dispose();
}
}
}
@Test
public void restoreLatestBackup()
throws RocksDBException {
Options opt = null;
BackupableDBOptions bopt = null;
BackupableDB bdb = null;
RestoreBackupableDB rdb = null;
try {
opt = new Options().setCreateIfMissing(true);
bopt = new BackupableDBOptions(
backupFolder.getRoot().getAbsolutePath());
assertThat(bopt.backupDir()).isEqualTo(
backupFolder.getRoot().getAbsolutePath());
// Open empty database.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Fill database with some test values
prepareDatabase(bdb);
bdb.createNewBackup(true);
verifyNumberOfValidBackups(bdb, 1);
bdb.put("key1".getBytes(), "valueV2".getBytes());
bdb.put("key2".getBytes(), "valueV2".getBytes());
bdb.createNewBackup(true);
verifyNumberOfValidBackups(bdb, 2);
bdb.put("key1".getBytes(), "valueV3".getBytes());
bdb.put("key2".getBytes(), "valueV3".getBytes());
assertThat(new String(bdb.get("key1".getBytes()))).endsWith("V3");
assertThat(new String(bdb.get("key2".getBytes()))).endsWith("V3");
bdb.close();
// init RestoreBackupableDB
rdb = new RestoreBackupableDB(bopt);
verifyNumberOfValidBackups(rdb, 2);
// restore db from latest backup
rdb.restoreDBFromLatestBackup(dbFolder.getRoot().getAbsolutePath(),
dbFolder.getRoot().getAbsolutePath(),
new RestoreOptions(false));
// Open database again.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Values must have suffix V2 because of restoring latest backup.
assertThat(new String(bdb.get("key1".getBytes()))).endsWith("V2");
assertThat(new String(bdb.get("key2".getBytes()))).endsWith("V2");
} finally {
if (bdb != null) {
bdb.close();
}
if (rdb != null) {
rdb.dispose();
}
if (bopt != null) {
bopt.dispose();
}
if (opt != null) {
opt.dispose();
}
}
}
@Test
public void restoreFromBackup()
throws RocksDBException {
Options opt = null;
BackupableDBOptions bopt = null;
BackupableDB bdb = null;
RestoreBackupableDB rdb = null;
try {
opt = new Options().setCreateIfMissing(true);
bopt = new BackupableDBOptions(
backupFolder.getRoot().getAbsolutePath());
assertThat(bopt.backupDir()).isEqualTo(
backupFolder.getRoot().getAbsolutePath());
// Open empty database.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Fill database with some test values
prepareDatabase(bdb);
bdb.createNewBackup(true);
verifyNumberOfValidBackups(bdb, 1);
bdb.put("key1".getBytes(), "valueV2".getBytes());
bdb.put("key2".getBytes(), "valueV2".getBytes());
bdb.createNewBackup(true);
verifyNumberOfValidBackups(bdb, 2);
bdb.put("key1".getBytes(), "valueV3".getBytes());
bdb.put("key2".getBytes(), "valueV3".getBytes());
assertThat(new String(bdb.get("key1".getBytes()))).endsWith("V3");
assertThat(new String(bdb.get("key2".getBytes()))).endsWith("V3");
bdb.close();
// init RestoreBackupableDB
rdb = new RestoreBackupableDB(bopt);
List<BackupInfo> backupInfo = verifyNumberOfValidBackups(rdb, 2);
// restore db from first backup
rdb.restoreDBFromBackup(backupInfo.get(0).backupId(),
dbFolder.getRoot().getAbsolutePath(),
dbFolder.getRoot().getAbsolutePath(),
new RestoreOptions(false));
// Open database again.
bdb = BackupableDB.open(opt, bopt,
dbFolder.getRoot().getAbsolutePath());
// Values must have suffix V2 because of restoring latest backup.
assertThat(new String(bdb.get("key1".getBytes()))).endsWith("V1");
assertThat(new String(bdb.get("key2".getBytes()))).endsWith("V1");
} finally {
if (bdb != null) {
bdb.close();
}
if (rdb != null) {
rdb.dispose();
}
if (bopt != null) {
bopt.dispose();
}
if (opt != null) {
opt.dispose();
}
}
}
/**
* Verify backups.
*
* @param bdb {@link BackupableDB} instance.
* @param expectedNumberOfBackups numerical value
* @throws RocksDBException thrown if an error occurs within the native
* part of the library.
*/
private List<BackupInfo> verifyNumberOfValidBackups(BackupableDB bdb,
int expectedNumberOfBackups) throws RocksDBException {
// Verify that backups exist
assertThat(bdb.getCorruptedBackups().length).
isEqualTo(0);
bdb.garbageCollect();
List<BackupInfo> backupInfo = bdb.getBackupInfos();
assertThat(backupInfo.size()).
isEqualTo(expectedNumberOfBackups);
return backupInfo;
}
/**
* Verify backups.
*
* @param rdb {@link RestoreBackupableDB} instance.
* @param expectedNumberOfBackups numerical value
* @throws RocksDBException thrown if an error occurs within the native
* part of the library.
*/
private List<BackupInfo> verifyNumberOfValidBackups(
RestoreBackupableDB rdb, int expectedNumberOfBackups)
throws RocksDBException {
// Verify that backups exist
assertThat(rdb.getCorruptedBackups().length).
isEqualTo(0);
rdb.garbageCollect();
List<BackupInfo> backupInfo = rdb.getBackupInfos();
assertThat(backupInfo.size()).
isEqualTo(expectedNumberOfBackups);
return backupInfo;
}
/**
* Fill database with some test values.
*
* @param db {@link RocksDB} instance.
* @throws RocksDBException thrown if an error occurs within the native
* part of the library.
*/
private void prepareDatabase(RocksDB db)
throws RocksDBException {
db.put("key1".getBytes(), "valueV1".getBytes());
db.put("key2".getBytes(), "valueV1".getBytes());
}
}
@@ -5,161 +5,43 @@
package org.rocksdb.test;
import org.junit.ClassRule;
import org.junit.Test;
import org.rocksdb.*;
import static org.assertj.core.api.Assertions.assertThat;
import org.rocksdb.BlockBasedTableConfig;
import org.rocksdb.ChecksumType;
import org.rocksdb.IndexType;
public class BlockBasedTableConfigTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
@Test
public void noBlockCache() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
public static void main(String[] args) {
BlockBasedTableConfig blockBasedTableConfig =
new BlockBasedTableConfig();
blockBasedTableConfig.setNoBlockCache(true);
assertThat(blockBasedTableConfig.noBlockCache()).isTrue();
}
@Test
public void blockCacheSize() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
blockBasedTableConfig.setBlockCacheSize(8 * 1024);
assertThat(blockBasedTableConfig.blockCacheSize()).
isEqualTo(8 * 1024);
}
@Test
public void blockSizeDeviation() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assert(blockBasedTableConfig.noBlockCache());
blockBasedTableConfig.setBlockCacheSize(8*1024);
assert(blockBasedTableConfig.blockCacheSize() == (8*1024));
blockBasedTableConfig.setBlockSizeDeviation(12);
assertThat(blockBasedTableConfig.blockSizeDeviation()).
isEqualTo(12);
}
@Test
public void blockRestartInterval() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assert(blockBasedTableConfig.blockSizeDeviation() == 12);
blockBasedTableConfig.setBlockRestartInterval(15);
assertThat(blockBasedTableConfig.blockRestartInterval()).
isEqualTo(15);
}
@Test
public void wholeKeyFiltering() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assert(blockBasedTableConfig.blockRestartInterval() == 15);
blockBasedTableConfig.setWholeKeyFiltering(false);
assertThat(blockBasedTableConfig.wholeKeyFiltering()).
isFalse();
}
@Test
public void cacheIndexAndFilterBlocks() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assert(!blockBasedTableConfig.wholeKeyFiltering());
blockBasedTableConfig.setCacheIndexAndFilterBlocks(true);
assertThat(blockBasedTableConfig.cacheIndexAndFilterBlocks()).
isTrue();
}
@Test
public void hashIndexAllowCollision() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assert(blockBasedTableConfig.cacheIndexAndFilterBlocks());
blockBasedTableConfig.setHashIndexAllowCollision(false);
assertThat(blockBasedTableConfig.hashIndexAllowCollision()).
isFalse();
}
@Test
public void blockCacheCompressedSize() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assert(!blockBasedTableConfig.hashIndexAllowCollision());
blockBasedTableConfig.setBlockCacheCompressedSize(40);
assertThat(blockBasedTableConfig.blockCacheCompressedSize()).
isEqualTo(40);
}
@Test
public void checksumType() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assertThat(ChecksumType.values().length).isEqualTo(3);
assertThat(ChecksumType.valueOf("kxxHash")).
isEqualTo(ChecksumType.kxxHash);
assert(blockBasedTableConfig.blockCacheCompressedSize() == 40);
blockBasedTableConfig.setChecksumType(ChecksumType.kNoChecksum);
blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash);
assertThat(blockBasedTableConfig.checksumType().equals(
assert(blockBasedTableConfig.checksumType().equals(
ChecksumType.kxxHash));
}
@Test
public void indexType() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assertThat(IndexType.values().length).isEqualTo(2);
blockBasedTableConfig.setIndexType(IndexType.kHashSearch);
assertThat(blockBasedTableConfig.indexType().equals(
assert(blockBasedTableConfig.indexType().equals(
IndexType.kHashSearch));
assertThat(IndexType.valueOf("kBinarySearch")).isNotNull();
blockBasedTableConfig.setIndexType(IndexType.valueOf("kBinarySearch"));
assertThat(blockBasedTableConfig.indexType().equals(
IndexType.kBinarySearch));
}
@Test
public void blockCacheCompressedNumShardBits() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
blockBasedTableConfig.setBlockCacheCompressedNumShardBits(4);
assertThat(blockBasedTableConfig.blockCacheCompressedNumShardBits()).
isEqualTo(4);
}
@Test
public void cacheNumShardBits() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assert(blockBasedTableConfig.blockCacheCompressedNumShardBits()
== 4);
blockBasedTableConfig.setCacheNumShardBits(5);
assertThat(blockBasedTableConfig.cacheNumShardBits()).
isEqualTo(5);
}
@Test
public void blockSize() {
BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
blockBasedTableConfig.setBlockSize(10);
assertThat(blockBasedTableConfig.blockSize()).isEqualTo(10);
}
@Test
public void blockBasedTableWithFilter() {
Options options = null;
try {
options = new Options();
options.setTableFormatConfig(
new BlockBasedTableConfig().setFilter(
new BloomFilter(10)));
assertThat(options.tableFactoryName()).
isEqualTo("BlockBasedTable");
} finally {
if (options != null) {
options.dispose();
}
}
}
@Test
public void blockBasedTableWithoutFilter() {
Options options = null;
try {
options = new Options();
options.setTableFormatConfig(
new BlockBasedTableConfig().setFilter(null));
assertThat(options.tableFactoryName()).
isEqualTo("BlockBasedTable");
} finally {
if (options != null) {
options.dispose();
}
}
assert(blockBasedTableConfig.cacheNumShardBits() == 5);
System.out.println("BlockBasedTableConfig test passed");
}
}
-101
View File
@@ -1,101 +0,0 @@
package org.rocksdb.test;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.Checkpoint;
import org.rocksdb.Options;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import static org.assertj.core.api.Assertions.assertThat;
public class CheckPointTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
@Rule
public TemporaryFolder dbFolder = new TemporaryFolder();
@Rule
public TemporaryFolder checkpointFolder = new TemporaryFolder();
@Test
public void checkPoint() throws RocksDBException {
RocksDB db = null;
Options options = null;
Checkpoint checkpoint = null;
try {
options = new Options().
setCreateIfMissing(true);
db = RocksDB.open(options,
dbFolder.getRoot().getAbsolutePath());
db.put("key".getBytes(), "value".getBytes());
checkpoint = Checkpoint.create(db);
checkpoint.createCheckpoint(checkpointFolder.
getRoot().getAbsolutePath() + "/snapshot1");
db.put("key2".getBytes(), "value2".getBytes());
checkpoint.createCheckpoint(checkpointFolder.
getRoot().getAbsolutePath() + "/snapshot2");
db.close();
db = RocksDB.open(options,
checkpointFolder.getRoot().getAbsolutePath() +
"/snapshot1");
assertThat(new String(db.get("key".getBytes()))).
isEqualTo("value");
assertThat(db.get("key2".getBytes())).isNull();
db.close();
db = RocksDB.open(options,
checkpointFolder.getRoot().getAbsolutePath() +
"/snapshot2");
assertThat(new String(db.get("key".getBytes()))).
isEqualTo("value");
assertThat(new String(db.get("key2".getBytes()))).
isEqualTo("value2");
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
if (checkpoint != null) {
checkpoint.dispose();
}
}
}
@Test(expected = IllegalArgumentException.class)
public void failIfDbIsNull() {
Checkpoint.create(null);
}
@Test(expected = IllegalStateException.class)
public void failIfDbNotInitialized() throws RocksDBException {
RocksDB db = RocksDB.open(dbFolder.getRoot().getAbsolutePath());
db.dispose();
Checkpoint.create(db);
}
@Test(expected = RocksDBException.class)
public void failWithIllegalPath() throws RocksDBException {
RocksDB db = null;
Checkpoint checkpoint = null;
try {
db = RocksDB.open(dbFolder.getRoot().getAbsolutePath());
checkpoint = Checkpoint.create(db);
checkpoint.createCheckpoint("/Z:///:\\C:\\TZ/-");
} finally {
if (db != null) {
db.close();
}
if (checkpoint != null) {
checkpoint.dispose();
}
}
}
}
@@ -5,636 +5,225 @@
package org.rocksdb.test;
import org.junit.ClassRule;
import org.junit.Test;
import org.rocksdb.*;
import java.util.Properties;
import java.util.Random;
import static org.assertj.core.api.Assertions.assertThat;
public class ColumnFamilyOptionsTest {
static {
RocksDB.loadLibrary();
}
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
public static final Random rand = PlatformRandomHelper.
getPlatformSpecificRandomFactory();
@Test
public void getColumnFamilyOptionsFromProps() {
ColumnFamilyOptions opt = null;
try {
// setup sample properties
Properties properties = new Properties();
properties.put("write_buffer_size", "112");
properties.put("max_write_buffer_number", "13");
opt = ColumnFamilyOptions.
getColumnFamilyOptionsFromProps(properties);
assertThat(opt).isNotNull();
assertThat(String.valueOf(opt.writeBufferSize())).
isEqualTo(properties.get("write_buffer_size"));
assertThat(String.valueOf(opt.maxWriteBufferNumber())).
isEqualTo(properties.get("max_write_buffer_number"));
} finally {
if (opt != null) {
opt.dispose();
public static void testCFOptions(ColumnFamilyOptionsInterface opt) {
Random rand = PlatformRandomHelper.
getPlatformSpecificRandomFactory();
{ // WriteBufferSize test
try {
long longValue = rand.nextLong();
opt.setWriteBufferSize(longValue);
assert(opt.writeBufferSize() == longValue);
} catch (RocksDBException e) {
assert(false);
}
}
}
@Test
public void failColumnFamilyOptionsFromPropsWithIllegalValue() {
ColumnFamilyOptions opt = null;
try {
// setup sample properties
Properties properties = new Properties();
properties.put("tomato", "1024");
properties.put("burger", "2");
opt = ColumnFamilyOptions.
getColumnFamilyOptionsFromProps(properties);
assertThat(opt).isNull();
} finally {
if (opt != null) {
opt.dispose();
}
}
}
@Test(expected = IllegalArgumentException.class)
public void failColumnFamilyOptionsFromPropsWithNullValue() {
ColumnFamilyOptions.getColumnFamilyOptionsFromProps(null);
}
@Test(expected = IllegalArgumentException.class)
public void failColumnFamilyOptionsFromPropsWithEmptyProps() {
ColumnFamilyOptions.getColumnFamilyOptionsFromProps(
new Properties());
}
@Test
public void writeBufferSize() throws RocksDBException {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
long longValue = rand.nextLong();
opt.setWriteBufferSize(longValue);
assertThat(opt.writeBufferSize()).isEqualTo(longValue);
} finally {
if (opt != null) {
opt.dispose();
}
}
}
@Test
public void maxWriteBufferNumber() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // MaxWriteBufferNumber test
int intValue = rand.nextInt();
opt.setMaxWriteBufferNumber(intValue);
assertThat(opt.maxWriteBufferNumber()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.maxWriteBufferNumber() == intValue);
}
}
@Test
public void minWriteBufferNumberToMerge() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // MinWriteBufferNumberToMerge test
int intValue = rand.nextInt();
opt.setMinWriteBufferNumberToMerge(intValue);
assertThat(opt.minWriteBufferNumberToMerge()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.minWriteBufferNumberToMerge() == intValue);
}
}
@Test
public void numLevels() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // NumLevels test
int intValue = rand.nextInt();
opt.setNumLevels(intValue);
assertThat(opt.numLevels()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.numLevels() == intValue);
}
}
@Test
public void levelZeroFileNumCompactionTrigger() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // LevelFileNumCompactionTrigger test
int intValue = rand.nextInt();
opt.setLevelZeroFileNumCompactionTrigger(intValue);
assertThat(opt.levelZeroFileNumCompactionTrigger()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.levelZeroFileNumCompactionTrigger() == intValue);
}
}
@Test
public void levelZeroSlowdownWritesTrigger() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // LevelSlowdownWritesTrigger test
int intValue = rand.nextInt();
opt.setLevelZeroSlowdownWritesTrigger(intValue);
assertThat(opt.levelZeroSlowdownWritesTrigger()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.levelZeroSlowdownWritesTrigger() == intValue);
}
}
@Test
public void levelZeroStopWritesTrigger() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // LevelStopWritesTrigger test
int intValue = rand.nextInt();
opt.setLevelZeroStopWritesTrigger(intValue);
assertThat(opt.levelZeroStopWritesTrigger()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.levelZeroStopWritesTrigger() == intValue);
}
}
@Test
public void maxMemCompactionLevel() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // MaxMemCompactionLevel test
int intValue = rand.nextInt();
opt.setMaxMemCompactionLevel(intValue);
assertThat(opt.maxMemCompactionLevel()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.maxMemCompactionLevel() == intValue);
}
}
@Test
public void targetFileSizeBase() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // TargetFileSizeBase test
long longValue = rand.nextLong();
opt.setTargetFileSizeBase(longValue);
assertThat(opt.targetFileSizeBase()).isEqualTo(longValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.targetFileSizeBase() == longValue);
}
}
@Test
public void targetFileSizeMultiplier() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // TargetFileSizeMultiplier test
int intValue = rand.nextInt();
opt.setTargetFileSizeMultiplier(intValue);
assertThat(opt.targetFileSizeMultiplier()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.targetFileSizeMultiplier() == intValue);
}
}
@Test
public void maxBytesForLevelBase() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // MaxBytesForLevelBase test
long longValue = rand.nextLong();
opt.setMaxBytesForLevelBase(longValue);
assertThat(opt.maxBytesForLevelBase()).isEqualTo(longValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.maxBytesForLevelBase() == longValue);
}
}
@Test
public void maxBytesForLevelMultiplier() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // MaxBytesForLevelMultiplier test
int intValue = rand.nextInt();
opt.setMaxBytesForLevelMultiplier(intValue);
assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.maxBytesForLevelMultiplier() == intValue);
}
}
@Test
public void expandedCompactionFactor() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // ExpandedCompactionFactor test
int intValue = rand.nextInt();
opt.setExpandedCompactionFactor(intValue);
assertThat(opt.expandedCompactionFactor()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.expandedCompactionFactor() == intValue);
}
}
@Test
public void sourceCompactionFactor() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // SourceCompactionFactor test
int intValue = rand.nextInt();
opt.setSourceCompactionFactor(intValue);
assertThat(opt.sourceCompactionFactor()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.sourceCompactionFactor() == intValue);
}
}
@Test
public void maxGrandparentOverlapFactor() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // MaxGrandparentOverlapFactor test
int intValue = rand.nextInt();
opt.setMaxGrandparentOverlapFactor(intValue);
assertThat(opt.maxGrandparentOverlapFactor()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.maxGrandparentOverlapFactor() == intValue);
}
}
@Test
public void softRateLimit() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // SoftRateLimit test
double doubleValue = rand.nextDouble();
opt.setSoftRateLimit(doubleValue);
assertThat(opt.softRateLimit()).isEqualTo(doubleValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.softRateLimit() == doubleValue);
}
}
@Test
public void hardRateLimit() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // HardRateLimit test
double doubleValue = rand.nextDouble();
opt.setHardRateLimit(doubleValue);
assertThat(opt.hardRateLimit()).isEqualTo(doubleValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.hardRateLimit() == doubleValue);
}
}
@Test
public void rateLimitDelayMaxMilliseconds() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // RateLimitDelayMaxMilliseconds test
int intValue = rand.nextInt();
opt.setRateLimitDelayMaxMilliseconds(intValue);
assertThat(opt.rateLimitDelayMaxMilliseconds()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
assert(opt.rateLimitDelayMaxMilliseconds() == intValue);
}
{ // ArenaBlockSize test
try {
long longValue = rand.nextLong();
opt.setArenaBlockSize(longValue);
assert(opt.arenaBlockSize() == longValue);
} catch (RocksDBException e) {
assert(false);
}
}
}
@Test
public void arenaBlockSize() throws RocksDBException {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
long longValue = rand.nextLong();
opt.setArenaBlockSize(longValue);
assertThat(opt.arenaBlockSize()).isEqualTo(longValue);
} finally {
if (opt != null) {
opt.dispose();
}
}
}
@Test
public void disableAutoCompactions() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // DisableAutoCompactions test
boolean boolValue = rand.nextBoolean();
opt.setDisableAutoCompactions(boolValue);
assertThat(opt.disableAutoCompactions()).isEqualTo(boolValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.disableAutoCompactions() == boolValue);
}
}
@Test
public void purgeRedundantKvsWhileFlush() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // PurgeRedundantKvsWhileFlush test
boolean boolValue = rand.nextBoolean();
opt.setPurgeRedundantKvsWhileFlush(boolValue);
assertThat(opt.purgeRedundantKvsWhileFlush()).isEqualTo(boolValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.purgeRedundantKvsWhileFlush() == boolValue);
}
}
@Test
public void verifyChecksumsInCompaction() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // VerifyChecksumsInCompaction test
boolean boolValue = rand.nextBoolean();
opt.setVerifyChecksumsInCompaction(boolValue);
assertThat(opt.verifyChecksumsInCompaction()).isEqualTo(boolValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.verifyChecksumsInCompaction() == boolValue);
}
}
@Test
public void filterDeletes() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // FilterDeletes test
boolean boolValue = rand.nextBoolean();
opt.setFilterDeletes(boolValue);
assertThat(opt.filterDeletes()).isEqualTo(boolValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.filterDeletes() == boolValue);
}
}
@Test
public void maxSequentialSkipInIterations() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // MaxSequentialSkipInIterations test
long longValue = rand.nextLong();
opt.setMaxSequentialSkipInIterations(longValue);
assertThat(opt.maxSequentialSkipInIterations()).isEqualTo(longValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.maxSequentialSkipInIterations() == longValue);
}
}
@Test
public void inplaceUpdateSupport() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // InplaceUpdateSupport test
boolean boolValue = rand.nextBoolean();
opt.setInplaceUpdateSupport(boolValue);
assertThat(opt.inplaceUpdateSupport()).isEqualTo(boolValue);
} finally {
if (opt != null) {
opt.dispose();
assert(opt.inplaceUpdateSupport() == boolValue);
}
{ // InplaceUpdateNumLocks test
try {
long longValue = rand.nextLong();
opt.setInplaceUpdateNumLocks(longValue);
assert(opt.inplaceUpdateNumLocks() == longValue);
} catch (RocksDBException e) {
assert(false);
}
}
}
@Test
public void inplaceUpdateNumLocks() throws RocksDBException {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
long longValue = rand.nextLong();
opt.setInplaceUpdateNumLocks(longValue);
assertThat(opt.inplaceUpdateNumLocks()).isEqualTo(longValue);
} finally {
if (opt != null) {
opt.dispose();
}
}
}
@Test
public void memtablePrefixBloomBits() {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
{ // MemtablePrefixBloomBits test
int intValue = rand.nextInt();
opt.setMemtablePrefixBloomBits(intValue);
assertThat(opt.memtablePrefixBloomBits()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.memtablePrefixBloomBits() == intValue);
}
}
@Test
public void memtablePrefixBloomProbes() {
ColumnFamilyOptions opt = null;
try {
{ // MemtablePrefixBloomProbes test
int intValue = rand.nextInt();
opt = new ColumnFamilyOptions();
opt.setMemtablePrefixBloomProbes(intValue);
assertThat(opt.memtablePrefixBloomProbes()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.memtablePrefixBloomProbes() == intValue);
}
}
@Test
public void bloomLocality() {
ColumnFamilyOptions opt = null;
try {
{ // BloomLocality test
int intValue = rand.nextInt();
opt = new ColumnFamilyOptions();
opt.setBloomLocality(intValue);
assertThat(opt.bloomLocality()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
assert(opt.bloomLocality() == intValue);
}
{ // MaxSuccessiveMerges test
try {
long longValue = rand.nextLong();
opt.setMaxSuccessiveMerges(longValue);
assert(opt.maxSuccessiveMerges() == longValue);
} catch (RocksDBException e){
assert(false);
}
}
}
@Test
public void maxSuccessiveMerges() throws RocksDBException {
ColumnFamilyOptions opt = null;
try {
long longValue = rand.nextLong();
opt = new ColumnFamilyOptions();
opt.setMaxSuccessiveMerges(longValue);
assertThat(opt.maxSuccessiveMerges()).isEqualTo(longValue);
} finally {
if (opt != null) {
opt.dispose();
}
}
}
@Test
public void minPartialMergeOperands() {
ColumnFamilyOptions opt = null;
try {
{ // MinPartialMergeOperands test
int intValue = rand.nextInt();
opt = new ColumnFamilyOptions();
opt.setMinPartialMergeOperands(intValue);
assertThat(opt.minPartialMergeOperands()).isEqualTo(intValue);
} finally {
if (opt != null) {
opt.dispose();
}
assert(opt.minPartialMergeOperands() == intValue);
}
}
@Test
public void memTable() throws RocksDBException {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
opt.setMemTableConfig(new HashLinkedListMemTableConfig());
assertThat(opt.memTableFactoryName()).
isEqualTo("HashLinkedListRepFactory");
} finally {
if (opt != null) {
opt.dispose();
}
}
}
@Test
public void comparator() throws RocksDBException {
ColumnFamilyOptions opt = null;
try {
opt = new ColumnFamilyOptions();
opt.setComparator(BuiltinComparator.BYTEWISE_COMPARATOR);
} finally {
if (opt != null) {
opt.dispose();
}
}
}
@Test
public void linkageOfPrepMethods() {
ColumnFamilyOptions options = null;
try {
options = new ColumnFamilyOptions();
options.optimizeUniversalStyleCompaction();
options.optimizeUniversalStyleCompaction(4000);
options.optimizeLevelStyleCompaction();
options.optimizeLevelStyleCompaction(3000);
options.optimizeForPointLookup(10);
} finally {
if (options != null) {
options.dispose();
}
}
}
@Test
public void shouldSetTestPrefixExtractor() {
ColumnFamilyOptions options = null;
try {
options = new ColumnFamilyOptions();
options.useFixedLengthPrefixExtractor(100);
options.useFixedLengthPrefixExtractor(10);
} finally {
if (options != null) {
options.dispose();
}
}
}
@Test
public void compressionTypes() {
ColumnFamilyOptions ColumnFamilyOptions = null;
try {
ColumnFamilyOptions = new ColumnFamilyOptions();
for (CompressionType compressionType :
CompressionType.values()) {
ColumnFamilyOptions.setCompressionType(compressionType);
assertThat(ColumnFamilyOptions.compressionType()).
isEqualTo(compressionType);
assertThat(CompressionType.valueOf("NO_COMPRESSION")).
isEqualTo(CompressionType.NO_COMPRESSION);
}
} finally {
if (ColumnFamilyOptions != null) {
ColumnFamilyOptions.dispose();
}
}
}
@Test
public void compactionStyles() {
ColumnFamilyOptions ColumnFamilyOptions = null;
try {
ColumnFamilyOptions = new ColumnFamilyOptions();
for (CompactionStyle compactionStyle :
CompactionStyle.values()) {
ColumnFamilyOptions.setCompactionStyle(compactionStyle);
assertThat(ColumnFamilyOptions.compactionStyle()).
isEqualTo(compactionStyle);
assertThat(CompactionStyle.valueOf("FIFO")).
isEqualTo(CompactionStyle.FIFO);
}
} finally {
if (ColumnFamilyOptions != null) {
ColumnFamilyOptions.dispose();
}
}
public static void main(String[] args) {
ColumnFamilyOptions opt = new ColumnFamilyOptions();
testCFOptions(opt);
opt.dispose();
System.out.println("Passed DBOptionsTest");
}
}
+180 -459
View File
@@ -9,104 +9,79 @@ import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.*;
import static org.assertj.core.api.Assertions.assertThat;
public class ColumnFamilyTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
@Rule
public TemporaryFolder dbFolder = new TemporaryFolder();
@Test
public void listColumnFamilies() throws RocksDBException {
RocksDB db = null;
Options options = null;
try {
options = new Options();
options.setCreateIfMissing(true);
DBOptions dbOptions = new DBOptions();
dbOptions.setCreateIfMissing(true);
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath());
// Test listColumnFamilies
List<byte[]> columnFamilyNames;
columnFamilyNames = RocksDB.listColumnFamilies(options, dbFolder.getRoot().getAbsolutePath());
assertThat(columnFamilyNames).isNotNull();
assertThat(columnFamilyNames.size()).isGreaterThan(0);
assertThat(columnFamilyNames.size()).isEqualTo(1);
assertThat(new String(columnFamilyNames.get(0))).isEqualTo("default");
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
}
static final String db_path = "/tmp/rocksdbjni_columnfamily_test";
static {
RocksDB.loadLibrary();
}
@Test
public void createColumnFamily() throws RocksDBException {
public static void main(String[] args) {
RocksDB db = null;
Options options = null;
Options options = new Options();
options.setCreateIfMissing(true);
DBOptions dbOptions = new DBOptions();
dbOptions.setCreateIfMissing(true);
try {
options = new Options();
options.setCreateIfMissing(true);
db = RocksDB.open(options, db_path);
} catch (RocksDBException e) {
assert(false);
}
// Test listColumnFamilies
List<byte[]> columnFamilyNames;
try {
columnFamilyNames = RocksDB.listColumnFamilies(options, db_path);
if (columnFamilyNames != null && columnFamilyNames.size() > 0) {
assert(columnFamilyNames.size() == 1);
assert(new String(columnFamilyNames.get(0)).equals("default"));
} else {
assert(false);
}
} catch (RocksDBException e) {
assert(false);
}
DBOptions dbOptions = new DBOptions();
dbOptions.setCreateIfMissing(true);
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath());
// Test createColumnFamily
try {
db.createColumnFamily(new ColumnFamilyDescriptor("new_cf",
new ColumnFamilyOptions()));
db.close();
List<byte[]> columnFamilyNames;
columnFamilyNames = RocksDB.listColumnFamilies(options, dbFolder.getRoot().getAbsolutePath());
assertThat(columnFamilyNames).isNotNull();
assertThat(columnFamilyNames.size()).isGreaterThan(0);
assertThat(columnFamilyNames.size()).isEqualTo(2);
assertThat(new String(columnFamilyNames.get(0))).isEqualTo("default");
assertThat(new String(columnFamilyNames.get(1))).isEqualTo("new_cf");
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
} catch (RocksDBException e) {
assert(false);
}
}
@Test
public void openWithColumnFamilies() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
if (db != null) {
db.close();
}
// Test listColumnFamilies after create "new_cf"
try {
options = new DBOptions();
options.setCreateIfMissing(true);
options.setCreateMissingColumnFamilies(true);
// Test open database with column family names
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
columnFamilyNames = RocksDB.listColumnFamilies(options, db_path);
if (columnFamilyNames != null && columnFamilyNames.size() > 0) {
assert(columnFamilyNames.size() == 2);
assert(new String(columnFamilyNames.get(0)).equals("default"));
assert(new String(columnFamilyNames.get(1)).equals("new_cf"));
} else {
assert(false);
}
} catch (RocksDBException e) {
assert(false);
}
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
assertThat(columnFamilyHandleList.size()).isEqualTo(2);
// Test open database with column family names
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor("default"));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
try {
db = RocksDB.open(dbOptions, db_path, cfNames, columnFamilyHandleList);
assert(columnFamilyHandleList.size() == 2);
db.put("dfkey1".getBytes(), "dfvalue".getBytes());
db.put(columnFamilyHandleList.get(0), "dfkey2".getBytes(),
"dfvalue".getBytes());
@@ -115,122 +90,57 @@ public class ColumnFamilyTest {
String retVal = new String(db.get(columnFamilyHandleList.get(1),
"newcfkey1".getBytes()));
assertThat(retVal).isEqualTo("newcfvalue");
assertThat((db.get(columnFamilyHandleList.get(1),
"dfkey1".getBytes()))).isNull();
assert(retVal.equals("newcfvalue"));
assert( (db.get(columnFamilyHandleList.get(1),
"dfkey1".getBytes())) == null);
db.remove(columnFamilyHandleList.get(1), "newcfkey1".getBytes());
assertThat((db.get(columnFamilyHandleList.get(1),
"newcfkey1".getBytes()))).isNull();
db.remove(columnFamilyHandleList.get(0), new WriteOptions(),
"dfkey2".getBytes());
assertThat(db.get(columnFamilyHandleList.get(0), new ReadOptions(),
"dfkey2".getBytes())).isNull();
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
assert( (db.get(columnFamilyHandleList.get(1),
"newcfkey1".getBytes())) == null);
db.remove("dfkey2".getBytes());
assert( (db.get(columnFamilyHandleList.get(0),
"dfkey2".getBytes())) == null);
} catch (RocksDBException e) {
assert(false);
}
}
@Test
public void getWithOutValueAndCf() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
try {
options = new DBOptions();
options.setCreateIfMissing(true);
options.setCreateMissingColumnFamilies(true);
// Test open database with column family names
List<ColumnFamilyDescriptor> cfDescriptors =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfDescriptors, columnFamilyHandleList);
db.put(columnFamilyHandleList.get(0), new WriteOptions(),
"key1".getBytes(), "value".getBytes());
db.put("key2".getBytes(), "12345678".getBytes());
byte[] outValue = new byte[5];
// not found value
int getResult = db.get("keyNotFound".getBytes(), outValue);
assertThat(getResult).isEqualTo(RocksDB.NOT_FOUND);
// found value which fits in outValue
getResult = db.get(columnFamilyHandleList.get(0), "key1".getBytes(), outValue);
assertThat(getResult).isNotEqualTo(RocksDB.NOT_FOUND);
assertThat(outValue).isEqualTo("value".getBytes());
// found value which fits partially
getResult = db.get(columnFamilyHandleList.get(0), new ReadOptions(),
"key2".getBytes(), outValue);
assertThat(getResult).isNotEqualTo(RocksDB.NOT_FOUND);
assertThat(outValue).isEqualTo("12345".getBytes());
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
}
}
@Test
public void createWriteDropColumnFamily() throws RocksDBException {
RocksDB db = null;
DBOptions opt = null;
// Test create write to and drop ColumnFamily
ColumnFamilyHandle tmpColumnFamilyHandle = null;
try {
opt = new DBOptions();
opt.setCreateIfMissing(true);
opt.setCreateMissingColumnFamilies(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
tmpColumnFamilyHandle = db.createColumnFamily(
new ColumnFamilyDescriptor("tmpCF", new ColumnFamilyOptions()));
db.put(tmpColumnFamilyHandle, "key".getBytes(), "value".getBytes());
db.dropColumnFamily(tmpColumnFamilyHandle);
tmpColumnFamilyHandle.dispose();
} finally {
if (tmpColumnFamilyHandle != null) {
tmpColumnFamilyHandle.dispose();
}
if (db != null) {
db.close();
}
if (opt != null) {
opt.dispose();
}
} catch (Exception e) {
assert(false);
}
}
@Test
public void writeBatch() throws RocksDBException {
RocksDB db = null;
DBOptions opt = null;
// Put to disposed column family tmpColumnFamilyHandle must fail
try {
opt = new DBOptions();
opt.setCreateIfMissing(true);
opt.setCreateMissingColumnFamilies(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db.put(tmpColumnFamilyHandle, "key".getBytes(), "value".getBytes());
assert(false);
} catch (RocksDBException e) {
assert(true);
}
db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
// Remove to disposed column family tmpColumnFamilyHandle must fail
try {
db.remove(tmpColumnFamilyHandle, "key".getBytes());
assert(false);
} catch (RocksDBException e) {
assert(true);
}
// Get on a disposed column family tmpColumnFamilyHandle must fail
try {
db.get(tmpColumnFamilyHandle, "key".getBytes());
assert(false);
} catch (RocksDBException e) {
assert(true);
}
// Test WriteBatch
try {
WriteBatch writeBatch = new WriteBatch();
WriteOptions writeOpt = new WriteOptions();
writeBatch.put("key".getBytes(), "value".getBytes());
@@ -242,325 +152,136 @@ public class ColumnFamilyTest {
writeBatch.remove(columnFamilyHandleList.get(1), "xyz".getBytes());
db.write(writeOpt, writeBatch);
writeBatch.dispose();
assertThat(db.get(columnFamilyHandleList.get(1),
assert(db.get(columnFamilyHandleList.get(1),
"xyz".getBytes()) == null);
assertThat(new String(db.get(columnFamilyHandleList.get(1),
"newcfkey".getBytes()))).isEqualTo("value");
assertThat(new String(db.get(columnFamilyHandleList.get(1),
"newcfkey2".getBytes()))).isEqualTo("value2");
assertThat(new String(db.get("key".getBytes()))).isEqualTo("value");
} finally {
if (db != null) {
db.close();
}
if (opt != null) {
opt.dispose();
}
assert(new String(db.get(columnFamilyHandleList.get(1),
"newcfkey".getBytes())).equals("value"));
assert(new String(db.get(columnFamilyHandleList.get(1),
"newcfkey2".getBytes())).equals("value2"));
assert(new String(db.get("key".getBytes())).equals("value"));
} catch (Exception e) {
e.printStackTrace();
assert(false);
}
}
@Test
public void iteratorOnColumnFamily() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
RocksIterator rocksIterator = null;
// Test iterator on column family
try {
options = new DBOptions();
options.setCreateIfMissing(true);
options.setCreateMissingColumnFamilies(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
db.put(columnFamilyHandleList.get(1), "newcfkey".getBytes(),
"value".getBytes());
db.put(columnFamilyHandleList.get(1), "newcfkey2".getBytes(),
"value2".getBytes());
rocksIterator = db.newIterator(
RocksIterator rocksIterator = db.newIterator(
columnFamilyHandleList.get(1));
rocksIterator.seekToFirst();
Map<String, String> refMap = new HashMap<>();
Map<String, String> refMap = new HashMap<String, String>();
refMap.put("newcfkey", "value");
refMap.put("newcfkey2", "value2");
int i = 0;
while (rocksIterator.isValid()) {
while(rocksIterator.isValid()) {
i++;
assertThat(refMap.get(new String(rocksIterator.key()))).
isEqualTo(new String(rocksIterator.value()));
refMap.get(new String(rocksIterator.key())).equals(
new String(rocksIterator.value()));
rocksIterator.next();
}
assertThat(i).isEqualTo(2);
assert(i == 2);
rocksIterator.dispose();
} finally {
if (rocksIterator != null) {
rocksIterator.dispose();
}
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
} catch(Exception e) {
assert(false);
}
}
@Test
public void multiGet() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
// Test property handling on column families
try {
options = new DBOptions();
options.setCreateIfMissing(true);
options.setCreateMissingColumnFamilies(true);
List<ColumnFamilyDescriptor> cfDescriptors =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfDescriptors.add(new ColumnFamilyDescriptor("new_cf"));
assert(db.getProperty("rocksdb.estimate-num-keys") != null);
assert(db.getProperty("rocksdb.stats") != null);
assert(db.getProperty(columnFamilyHandleList.get(0),
"rocksdb.sstables") != null);
assert(db.getProperty(columnFamilyHandleList.get(1),
"rocksdb.estimate-num-keys") != null);
assert(db.getProperty(columnFamilyHandleList.get(1),
"rocksdb.stats") != null);
assert(db.getProperty(columnFamilyHandleList.get(1),
"rocksdb.sstables") != null);
} catch(Exception e) {
assert(false);
}
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfDescriptors, columnFamilyHandleList);
db.put(columnFamilyHandleList.get(0), "key".getBytes(), "value".getBytes());
db.put(columnFamilyHandleList.get(1), "newcfkey".getBytes(), "value".getBytes());
List<byte[]> keys = new ArrayList<>();
// MultiGet test
List<ColumnFamilyHandle> cfCustomList = new ArrayList<ColumnFamilyHandle>();
try {
List<byte[]> keys = new ArrayList<byte[]>();
keys.add("key".getBytes());
keys.add("newcfkey".getBytes());
Map<byte[], byte[]> retValues = db.multiGet(columnFamilyHandleList, keys);
assertThat(retValues.size()).isEqualTo(2);
assertThat(new String(retValues.get(keys.get(0))))
.isEqualTo("value");
assertThat(new String(retValues.get(keys.get(1))))
.isEqualTo("value");
retValues = db.multiGet(new ReadOptions(), columnFamilyHandleList, keys);
assertThat(retValues.size()).isEqualTo(2);
assertThat(new String(retValues.get(keys.get(0))))
.isEqualTo("value");
assertThat(new String(retValues.get(keys.get(1))))
.isEqualTo("value");
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
Map<byte[], byte[]> retValues = db.multiGet(columnFamilyHandleList,keys);
assert(retValues.size() == 2);
assert(new String(retValues.get(keys.get(0)))
.equals("value"));
assert(new String(retValues.get(keys.get(1)))
.equals("value"));
cfCustomList.add(columnFamilyHandleList.get(0));
cfCustomList.add(columnFamilyHandleList.get(0));
retValues = db.multiGet(cfCustomList, keys);
assert(retValues.size() == 1);
assert(new String(retValues.get(keys.get(0)))
.equals("value"));
} catch (RocksDBException e) {
assert(false);
}
}
@Test
public void properties() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
// Test multiget without correct number of column
// families
try {
options = new DBOptions();
options.setCreateIfMissing(true);
options.setCreateMissingColumnFamilies(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
assertThat(db.getProperty("rocksdb.estimate-num-keys")).
isNotNull();
assertThat(db.getLongProperty(columnFamilyHandleList.get(0),
"rocksdb.estimate-num-keys")).isGreaterThanOrEqualTo(0);
assertThat(db.getProperty("rocksdb.stats")).isNotNull();
assertThat(db.getProperty(columnFamilyHandleList.get(0),
"rocksdb.sstables")).isNotNull();
assertThat(db.getProperty(columnFamilyHandleList.get(1),
"rocksdb.estimate-num-keys")).isNotNull();
assertThat(db.getProperty(columnFamilyHandleList.get(1),
"rocksdb.stats")).isNotNull();
assertThat(db.getProperty(columnFamilyHandleList.get(1),
"rocksdb.sstables")).isNotNull();
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
List<byte[]> keys = new ArrayList<byte[]>();
keys.add("key".getBytes());
keys.add("newcfkey".getBytes());
cfCustomList.remove(1);
db.multiGet(cfCustomList, keys);
assert(false);
} catch (RocksDBException e) {
assert(false);
} catch (IllegalArgumentException e) {
assert(true);
}
}
@Test
public void iterators() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
try {
options = new DBOptions();
options.setCreateIfMissing(true);
options.setCreateMissingColumnFamilies(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
// iterate over default key/value pairs
List<RocksIterator> iterators =
db.newIterators(columnFamilyHandleList);
assertThat(iterators.size()).isEqualTo(2);
assert(iterators.size() == 2);
RocksIterator iter = iterators.get(0);
iter.seekToFirst();
Map<String, String> defRefMap = new HashMap<>();
Map<String,String> defRefMap = new HashMap<String, String>();
defRefMap.put("dfkey1", "dfvalue");
defRefMap.put("key", "value");
while (iter.isValid()) {
assertThat(defRefMap.get(new String(iter.key()))).
isEqualTo(new String(iter.value()));
defRefMap.get(new String(iter.key())).equals(
new String(iter.value()));
iter.next();
}
// iterate over new_cf key/value pairs
Map<String, String> cfRefMap = new HashMap<>();
Map<String,String> cfRefMap = new HashMap<String, String>();
cfRefMap.put("newcfkey", "value");
cfRefMap.put("newcfkey2", "value2");
iter = iterators.get(1);
iter.seekToFirst();
while (iter.isValid()) {
assertThat(cfRefMap.get(new String(iter.key()))).
isEqualTo(new String(iter.value()));
cfRefMap.get(new String(iter.key())).equals(
new String(iter.value()));
iter.next();
}
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
// free iterators
for (RocksIterator iterator : iterators) {
iterator.dispose();
}
assert(true);
} catch (RocksDBException e) {
assert(false);
}
}
@Test(expected = RocksDBException.class)
public void failPutDisposedCF() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
try {
options = new DBOptions();
options.setCreateIfMissing(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
db.dropColumnFamily(columnFamilyHandleList.get(1));
db.put(columnFamilyHandleList.get(1), "key".getBytes(), "value".getBytes());
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
System.out.println("Passed ColumnFamilyTest");
// free cf handles before database close
for (ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) {
columnFamilyHandle.dispose();
}
// close database
db.close();
// be sure to dispose c++ pointers
options.dispose();
}
@Test(expected = RocksDBException.class)
public void failRemoveDisposedCF() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
try {
options = new DBOptions();
options.setCreateIfMissing(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
db.dropColumnFamily(columnFamilyHandleList.get(1));
db.remove(columnFamilyHandleList.get(1), "key".getBytes());
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
}
}
@Test(expected = RocksDBException.class)
public void failGetDisposedCF() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
try {
options = new DBOptions();
options.setCreateIfMissing(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
db.dropColumnFamily(columnFamilyHandleList.get(1));
db.get(columnFamilyHandleList.get(1), "key".getBytes());
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
}
}
@Test(expected = RocksDBException.class)
public void failMultiGetWithoutCorrectNumberOfCF() throws RocksDBException {
RocksDB db = null;
DBOptions options = null;
try {
options = new DBOptions();
options.setCreateIfMissing(true);
List<ColumnFamilyDescriptor> cfNames =
new ArrayList<>();
List<ColumnFamilyHandle> columnFamilyHandleList =
new ArrayList<>();
cfNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY));
cfNames.add(new ColumnFamilyDescriptor("new_cf"));
db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath(),
cfNames, columnFamilyHandleList);
List<byte[]> keys = new ArrayList<>();
keys.add("key".getBytes());
keys.add("newcfkey".getBytes());
List<ColumnFamilyHandle> cfCustomList = new ArrayList<>();
db.multiGet(cfCustomList, keys);
} finally {
if (db != null) {
db.close();
}
if (options != null) {
options.dispose();
}
}
}
}
@@ -5,32 +5,30 @@
package org.rocksdb.test;
import org.junit.ClassRule;
import org.junit.Test;
import org.rocksdb.ComparatorOptions;
import org.rocksdb.RocksDB;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Random;
public class ComparatorOptionsTest {
@ClassRule
public static final RocksMemoryResource rocksMemoryResource =
new RocksMemoryResource();
static {
RocksDB.loadLibrary();
}
@Test
public void comparatorOptions() {
public static void main(String[] args) {
final ComparatorOptions copt = new ComparatorOptions();
assertThat(copt).isNotNull();
Random rand = new Random();
{ // UseAdaptiveMutex test
copt.setUseAdaptiveMutex(true);
assertThat(copt.useAdaptiveMutex()).isTrue();
assert(copt.useAdaptiveMutex() == true);
copt.setUseAdaptiveMutex(false);
assertThat(copt.useAdaptiveMutex()).isFalse();
assert(copt.useAdaptiveMutex() == false);
}
copt.dispose();
System.out.println("Passed ComparatorOptionsTest");
}
}
Loaded 100 of 214 files, more files were not shown because too many files have changed in this diff. Show more