Compare commits

..
8 Commits
Author SHA1 Message Date
sdong fe637231e8 [FB Internal] Point to the latest tool chain. 2019-10-31 15:41:33 -07:00
sdong 66e57a5735 fb internal tool chain upgrade to gcc-5 2018-06-06 16:08:03 -07:00
Siying Dong 98368ee181 Fix the previous commit 2018-05-29 10:35:40 -07:00
Siying Dong 701e9db415 Set env_options.allow_mmap_writes = false if not from DB Options 2018-05-29 10:33:05 -07:00
Siying Dong 677ca7d806 LDB to use allow_mmap_writes = false 2018-05-29 10:23:12 -07:00
sdong 981082769e Miss a file in previous commit 2017-07-17 18:21:52 -07:00
sdong f854a8516a [FB Internal] Use gcc-5 2017-07-17 14:57:11 -07:00
sdong 9761a06c1d fb internal: Allow to compile with GCC 4.8.1
Have to disable some errors to make the build pass.
2016-10-13 14:57:05 -07:00
245 changed files with 2322 additions and 5588 deletions

No files matched your search

-1
View File
@@ -21,4 +21,3 @@ util/build_version.cc
build_tools/VALGRIND_LOGS/
coverage/COVERAGE_REPORT
util/build_version.cc.tmp
.gdbhistory
+5 -9
View File
@@ -14,11 +14,11 @@ OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
#-----------------------------------------------
# detect what platform we're building on
$(shell (export ROCKSDB_ROOT=$(CURDIR); $(CURDIR)/build_tools/build_detect_platform $(CURDIR)/build_config.mk))
$(shell (cd build_tools/; ROCKSDB_ROOT=.. ./build_detect_platform ../build_config.mk))
# this file is generated by the previous line to set build flags and sources
include build_config.mk
WARNING_FLAGS = -Wall -Werror
WARNING_FLAGS = -Wall -Wmaybe-uninitialized
CFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
CXXFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -std=gnu++0x -Woverloaded-virtual
@@ -64,8 +64,7 @@ TESTS = \
ttl_test \
version_edit_test \
version_set_test \
write_batch_test\
deletefile_test
write_batch_test
TOOLS = \
sst_dump \
@@ -118,11 +117,11 @@ all: $(SHARED) $(LIBRARY) $(PROGRAMS)
release:
$(MAKE) clean
OPT=-DNDEBUG $(MAKE) -j32
OPT=-DNDEBUG $(MAKE) $(LIBRARY) $(TOOLS) db_bench -j32
coverage:
$(MAKE) clean
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) all check
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" $(MAKE) all check
(cd coverage; ./coverage_test.sh)
# Delete intermediate files
find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
@@ -267,9 +266,6 @@ write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
$(CXX) db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
$(MEMENVLIBRARY) : $(MEMENVOBJECTS)
rm -f $@
$(AR) -rs $@ $(MEMENVOBJECTS)
+17 -17
View File
@@ -1,6 +1,6 @@
rocksdb: A persistent key-value store for flash storage
Authors: * The Facebook Database Engineering Team
* Build on earlier work on leveldb by Sanjay Ghemawat
* Build on earlier work on leveldb by Sanjay Ghemawat
(sanjay@google.com) and Jeff Dean (jeff@google.com)
This code is a library that forms the core building block for a fast
@@ -19,64 +19,64 @@ persistent key/value store.
See doc/index.html for more explanation.
See doc/impl.html for a brief overview of the implementation.
The public interface is in include/*. Callers should not include or
The public interface is in include/*.h. Callers should not include or
rely on the details of any other header files in this package. Those
internal APIs may be changed without warning.
Guide to header files:
include/rocksdb/db.h
include/db.h
Main interface to the DB: Start here
include/rocksdb/options.h
include/options.h
Control over the behavior of an entire database, and also
control over the behavior of individual reads and writes.
include/rocksdb/comparator.h
include/comparator.h
Abstraction for user-specified comparison function. If you want
just bytewise comparison of keys, you can use the default comparator,
but clients can write their own comparator implementations if they
want custom ordering (e.g. to handle different character
encodings, etc.)
include/rocksdb/iterator.h
include/iterator.h
Interface for iterating over data. You can get an iterator
from a DB object.
include/rocksdb/write_batch.h
include/write_batch.h
Interface for atomically applying multiple updates to a database.
include/rocksdb/slice.h
include/slice.h
A simple module for maintaining a pointer and a length into some
other byte array.
include/rocksdb/status.h
include/status.h
Status is returned from many of the public interfaces and is used
to report success and various kinds of errors.
include/rocksdb/env.h
include/env.h
Abstraction of the OS environment. A posix implementation of
this interface is in util/env_posix.cc
include/rocksdb/table_builder.h
include/table_builder.h
Lower-level modules that most clients probably won't use directly
include/rocksdb/cache.h
include/cache.h
An API for the block cache.
include/rocksdb/compaction_filter.h
include/compaction_filter.h
An API for a application filter invoked on every compaction.
include/rocksdb/filter_policy.h
include/filter_policy.h
An API for configuring a bloom filter.
include/rocksdb/memtablerep.h
include/memtablerep.h
An API for implementing a memtable.
include/rocksdb/statistics.h
include/statistics.h
An API to retrieve various database statistics.
include/rocksdb/transaction_log.h
include/transaction_log_iterator.h
An API to retrieve transaction logs from a database.
+1 -1
View File
@@ -1,3 +1,3 @@
* Detailed instructions on how to compile using fbcode and jemalloc
* Latest release is 2.4.fb
* Latest release is 2.1.fb
+4 -8
View File
@@ -30,11 +30,7 @@ fi
# Default to fbcode gcc on internal fb machines
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
if [ -z "$USE_CLANG" ]; then
source $PWD/build_tools/fbcode.gcc471.sh
else
source $PWD/build_tools/fbcode.clang31.sh
fi
source "$PWD/fbcode_config.sh"
fi
# Delete existing output, if it exists
@@ -65,7 +61,7 @@ PLATFORM_SHARED_CFLAGS="-fPIC"
PLATFORM_SHARED_VERSIONED=true
# generic port files (working on all platform by #ifdef) go directly in /port
GENERIC_PORT_FILES=`find $ROCKSDB_ROOT/port -name '*.cc' | tr "\n" " "`
GENERIC_PORT_FILES=`cd $ROCKSDB_ROOT; find port -name '*.cc' | tr "\n" " "`
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
case "$TARGET_OS" in
@@ -82,7 +78,7 @@ case "$TARGET_OS" in
if [ -z "$USE_CLANG" ]; then
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
fi
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
# PORT_FILES=port/linux/linux_specific.cc
;;
SunOS)
@@ -127,7 +123,7 @@ case "$TARGET_OS" in
exit 1
esac
$PWD/build_tools/build_detect_version
./build_detect_version
# We want to make a list of all cc files within util, db, table, and helpers
# except for the test and benchmark files. By default, find will output a list
+3 -3
View File
@@ -19,15 +19,15 @@ if [ "$?" = 0 ]; then
awk '
BEGIN {
print "#include \"build_version.h\"\n"
}
}
{ print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";" }
' > ${VFILE}
else
echo "git not found" |
awk '
BEGIN {
print "#include \"build_version.h\""
}
print "#include \"build_version.h\""
}
{ print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";" }
' > ${VFILE}
fi
+19
View File
@@ -0,0 +1,19 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
+78
View File
@@ -0,0 +1,78 @@
#!/bin/sh
#
# Set environment variables so that we can compile rocksdb using
# fbcode settings. It uses the latest g++ compiler and also
# uses jemalloc
GCC_BASE=/mnt/gvfs/third-party2/gcc/8219ec1bcedf8ad9da05e121e193364de2cc4f61/5.x/centos6-native/c447969
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/64d8d58e3d84f8bde7a029763d4f5baf39d0d5b9/stable/centos6-native/6aaf4de
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ba9be983c81de7299b59fe71950c664a84dcb5f8/5.x/gcc-5-glibc-2.23/339d858
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f20197cf3d4bd50339c9777aaa0b2ccadad9e2cb/2.23/gcc-5-glibc-2.23/ca1d1c0
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/6427ce8c7496e4ab06c2da81543b94c0de8be3d0/1.1.3/gcc-5-glibc-2.23/9bc6787
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/8f1e8b867d26efef93eac2fabbdb2e1d512665d7/1.2.8/gcc-5-glibc-2.23/9bc6787
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/70471c0571559fe0af7db6d7e8860b93a7eadfe1/1.0.6/gcc-5-glibc-2.23/9bc6787
LZ4_BASE=/mnt/gvfs/third-party2/lz4/453c89d6f0e68cdf1c151c769197fabedad9cac8/r131/gcc-5-glibc-2.23/9bc6787
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/00a40fa5f8bd2cd0622f2e868552793aef37ccf4/1.3.0/gcc-5-glibc-2.23/03859b5
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/47eef08f9acb77de982fbda6047c26d330739538/2.2.0/gcc-5-glibc-2.23/9bc6787
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/4414ddc78df8008b35cc4adac23590ad29148584/master/gcc-5-glibc-2.23/d506c82
NUMA_BASE=/mnt/gvfs/third-party2/numa/9d7ae2693d05d62f9a579cb21e6b717cf257a75d/2.0.11/gcc-5-glibc-2.23/9bc6787
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/2b2dd58e3a52ccf2c1d827def59e5f740de0ad15/1.2/gcc-5-glibc-2.23/b443de1
TBB_BASE=/mnt/gvfs/third-party2/tbb/379addf7ab2468a2b4293b47456cfcd1c9cb318d/4.3/gcc-5-glibc-2.23/9bc6787
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/3f68f5fe65a85b7c2d3e66852268fbd1efdb3151/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b9fab0aec99d9c36408e810b2677e91c12807afd/2.28/centos6-native/da39a3e
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/423431d61786b20bcc3bde8972901130cb29e6b3/3.11.0/gcc-5-glibc-2.23/9bc6787
LUA_BASE=/mnt/gvfs/third-party2/lua/3b0bb3bd9a0f690a069c479fcc0f7424fc7456d2/5.2.3/gcc-5-glibc-2.23/65372bd
# location of libgcc
LIBGCC_INCLUDE=" -I $LIBGCC_BASE/include"
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
# location of glibc
GLIBC_INCLUDE=" -I $GLIBC_BASE/include"
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
# location of snappy headers and libraries
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
# location of jemalloc
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
JEMALLOC_LIB=" -Wl,--whole-archive $JEMALLOC_BASE/lib/libjemalloc.a"
# use Intel SSE support for checksum calculations
export USE_SSE=" -msse -msse4.2 "
CC="$GCC_BASE/bin/gcc"
CXX="$GCC_BASE/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE"
AR="$BINUTILS_BASE/bin/ar"
RANLIB="$BINUTILS_BASE/bin/ranlib"
CFLAGS="-B$BINUTILS/bin/gold -m64 -mtune=generic"
CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
CFLAGS+=" -DSNAPPY -DZLIB -DLZ4"
EXEC_LDFLAGS+=" -B$BINUTILS_BASE/bin/gold"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-5-glibc-2.23/lib/ld.so"
EXEC_LDFLAGS+=" $LIBUNWIND"
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-5-glibc-2.23/lib"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS "
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $LZ4_LIBS $GFLAGS_LIBS"
VALGRIND_VER="$VALGRIND_BASE/bin/"
export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE
+118
View File
@@ -0,0 +1,118 @@
#!/bin/sh
#
# Set environment variables so that we can compile rocksdb using
# fbcode settings. It uses the latest g++ and clang compilers and also
# uses jemalloc
# Environment variables that change the behavior of this script:
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
BASEDIR=`dirname $BASH_SOURCE`
source "$BASEDIR/dependencies.sh"
CFLAGS=""
# libgcc
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
# glibc
GLIBC_INCLUDE="$GLIBC_BASE/include"
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
# snappy
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
if test -z $PIC_BUILD; then
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
else
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
fi
CFLAGS+=" -DSNAPPY"
if test -z $PIC_BUILD; then
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
CFLAGS+=" -DZLIB"
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
CFLAGS+=" -DLZ4"
fi
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
if test -z $PIC_BUILD; then
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
else
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
fi
CFLAGS+=" -DGFLAGS=gflags"
# location of jemalloc
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
if test -z $PIC_BUILD; then
# location of numa
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
CFLAGS+=" -DNUMA"
# location of libunwind
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
fi
# location of TBB
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
if test -z $PIC_BUILD; then
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
else
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
fi
CFLAGS+=" -DTBB"
# use Intel SSE support for checksum calculations
export USE_SSE=" -msse -msse4.2 "
BINUTILS="$BINUTILS_BASE/bin"
AR="$BINUTILS/ar"
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE"
STDLIBS="-L $GCC_BASE/lib64"
CLANG_BIN="$CLANG_BASE/bin"
CLANG_LIB="$CLANG_BASE/lib"
CLANG_SRC="$CLANG_BASE/../../src"
CLANG_ANALYZER="$CLANG_BIN/clang++"
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
CC="$GCC_BASE/bin/gcc"
CXX="$GCC_BASE/bin/g++"
CFLAGS+=" -B$BINUTILS/gold"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
CFLAGS+=" -isystem $GLIBC_INCLUDE"
JEMALLOC=1
CFLAGS+=" $DEPS_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
CXXFLAGS+=" $CFLAGS"
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
EXEC_LDFLAGS+=" $LIBUNWIND"
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
# required by libtbb
EXEC_LDFLAGS+=" -ldl"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
VALGRIND_VER="$VALGRIND_BASE/bin/"
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
+2 -9
View File
@@ -55,19 +55,12 @@ then
exit 0
fi
LCOV_VERSION=$(lcov -v | grep 1.1 || true)
if [ $LCOV_VERSION ]
then
echo "Not supported lcov version. Expect lcov 1.1."
exit 0
fi
(cd $ROOT; lcov --no-external \
--capture \
--directory $PWD \
--gcov-tool $GCOV \
--output-file $COVERAGE_DIR/coverage.info)
--output-file $COVERAGE_DIR/coverage.info &>/dev/null)
genhtml $COVERAGE_DIR/coverage.info -o $COVERAGE_DIR
genhtml $COVERAGE_DIR/coverage.info -o $COVERAGE_DIR &>/dev/null
echo "HTML Coverage report is generated in $COVERAGE_DIR"
+5 -5
View File
@@ -9,12 +9,12 @@
#include "db/merge_helper.h"
#include "db/table_cache.h"
#include "db/version_edit.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/iterator.h"
#include "util/stop_watch.h"
namespace rocksdb {
namespace leveldb {
Status BuildTable(const std::string& dbname,
Env* env,
@@ -204,4 +204,4 @@ Status BuildTable(const std::string& dbname,
return s;
}
} // namespace rocksdb
} // namespace leveldb
+5 -5
View File
@@ -5,11 +5,11 @@
#ifndef STORAGE_LEVELDB_DB_BUILDER_H_
#define STORAGE_LEVELDB_DB_BUILDER_H_
#include "rocksdb/comparator.h"
#include "rocksdb/status.h"
#include "rocksdb/types.h"
#include "leveldb/comparator.h"
#include "leveldb/status.h"
#include "leveldb/types.h"
namespace rocksdb {
namespace leveldb {
struct Options;
struct FileMetaData;
@@ -36,6 +36,6 @@ extern Status BuildTable(const std::string& dbname,
const SequenceNumber newest_snapshot,
const SequenceNumber earliest_seqno_in_memtable);
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_BUILDER_H_
+32 -32
View File
@@ -2,42 +2,42 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/c.h"
#include "leveldb/c.h"
#include <stdlib.h>
#include <unistd.h>
#include "rocksdb/cache.h"
#include "rocksdb/comparator.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/iterator.h"
#include "rocksdb/options.h"
#include "rocksdb/status.h"
#include "rocksdb/write_batch.h"
#include "leveldb/cache.h"
#include "leveldb/comparator.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/filter_policy.h"
#include "leveldb/iterator.h"
#include "leveldb/options.h"
#include "leveldb/status.h"
#include "leveldb/write_batch.h"
using rocksdb::Cache;
using rocksdb::Comparator;
using rocksdb::CompressionType;
using rocksdb::DB;
using rocksdb::Env;
using rocksdb::FileLock;
using rocksdb::FilterPolicy;
using rocksdb::Iterator;
using rocksdb::Logger;
using rocksdb::NewBloomFilterPolicy;
using rocksdb::NewLRUCache;
using rocksdb::Options;
using rocksdb::RandomAccessFile;
using rocksdb::Range;
using rocksdb::ReadOptions;
using rocksdb::SequentialFile;
using rocksdb::Slice;
using rocksdb::Snapshot;
using rocksdb::Status;
using rocksdb::WritableFile;
using rocksdb::WriteBatch;
using rocksdb::WriteOptions;
using leveldb::Cache;
using leveldb::Comparator;
using leveldb::CompressionType;
using leveldb::DB;
using leveldb::Env;
using leveldb::FileLock;
using leveldb::FilterPolicy;
using leveldb::Iterator;
using leveldb::Logger;
using leveldb::NewBloomFilterPolicy;
using leveldb::NewLRUCache;
using leveldb::Options;
using leveldb::RandomAccessFile;
using leveldb::Range;
using leveldb::ReadOptions;
using leveldb::SequentialFile;
using leveldb::Slice;
using leveldb::Snapshot;
using leveldb::Status;
using leveldb::WritableFile;
using leveldb::WriteBatch;
using leveldb::WriteOptions;
using std::shared_ptr;
+1 -1
View File
@@ -2,7 +2,7 @@
Use of this source code is governed by a BSD-style license that can be
found in the LICENSE file. See the AUTHORS file for names of contributors. */
#include "rocksdb/c.h"
#include "leveldb/c.h"
#include <stddef.h>
#include <stdio.h>
+8 -8
View File
@@ -2,15 +2,15 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/db.h"
#include "leveldb/db.h"
#include <errno.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include "rocksdb/cache.h"
#include "rocksdb/env.h"
#include "rocksdb/write_batch.h"
#include "leveldb/cache.h"
#include "leveldb/env.h"
#include "leveldb/write_batch.h"
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/log_format.h"
@@ -20,7 +20,7 @@
#include "util/testharness.h"
#include "util/testutil.h"
namespace rocksdb {
namespace leveldb {
static const int kValueSize = 1000;
@@ -68,7 +68,7 @@ class CorruptionTest {
void RepairDB() {
delete db_;
db_ = nullptr;
ASSERT_OK(::rocksdb::RepairDB(dbname_, options_));
ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
}
void Build(int n) {
@@ -354,8 +354,8 @@ TEST(CorruptionTest, UnrelatedKeys) {
ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
}
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
+143 -357
View File
File diff suppressed because it is too large. Load diff
+30 -31
View File
@@ -8,12 +8,12 @@
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/version_set.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "port/port.h"
#include "util/mutexlock.h"
namespace rocksdb {
namespace leveldb {
Status DBImpl::DisableFileDeletions() {
MutexLock l(&mutex_);
@@ -30,19 +30,16 @@ Status DBImpl::EnableFileDeletions() {
}
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
uint64_t* manifest_file_size,
bool flush_memtable) {
uint64_t* manifest_file_size) {
*manifest_file_size = 0;
if (flush_memtable) {
// flush all dirty data to disk.
Status status = Flush(FlushOptions());
if (!status.ok()) {
Log(options_.info_log, "Cannot Flush data %s\n",
status.ToString().c_str());
return status;
}
// flush all dirty data to disk.
Status status = Flush(FlushOptions());
if (!status.ok()) {
Log(options_.info_log, "Cannot Flush data %s\n",
status.ToString().c_str());
return status;
}
MutexLock l(&mutex_);
@@ -51,17 +48,18 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
std::set<uint64_t> live;
versions_->AddLiveFilesCurrentVersion(&live);
ret.clear();
ret.reserve(live.size() + 2); //*.sst + CURRENT + MANIFEST
ret.resize(live.size() + 2); //*.sst + CURRENT + MANIFEST
// create names of the live files. The names are not absolute
// paths, instead they are relative to dbname_;
for (auto live_file : live) {
ret.push_back(TableFileName("", live_file));
std::set<uint64_t>::iterator it = live.begin();
for (unsigned int i = 0; i < live.size(); i++, it++) {
ret[i] = TableFileName("", *it);
}
ret.push_back(CurrentFileName(""));
ret.push_back(DescriptorFileName("", versions_->ManifestFileNumber()));
ret[live.size()] = CurrentFileName("");
ret[live.size()+1] = DescriptorFileName("",
versions_->ManifestFileNumber());
// find length of manifest file while holding the mutex lock
*manifest_file_size = versions_->ManifestFileSize();
@@ -73,7 +71,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
// First get sorted files in archive dir, then append sorted files from main
// dir to maintain sorted order
// list wal files in archive dir.
// list wal files in archive dir.
Status s;
std::string archivedir = ArchivalDirectory(dbname_);
if (env_->FileExists(archivedir)) {
@@ -83,7 +81,11 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
}
}
// list wal files in main db dir.
return AppendSortedWalsOfType(dbname_, files, kAliveLogFile);
s = AppendSortedWalsOfType(dbname_, files, kAliveLogFile);
if (!s.ok()) {
return s;
}
return s;
}
Status DBImpl::DeleteWalFiles(const VectorLogPtr& files) {
@@ -91,17 +93,14 @@ Status DBImpl::DeleteWalFiles(const VectorLogPtr& files) {
std::string archivedir = ArchivalDirectory(dbname_);
std::string files_not_deleted;
for (const auto& wal : files) {
/* Try deleting in the dir that pathname points to for the logfile.
This may fail if we try to delete a log file which was live when captured
but is archived now. Try deleting it from archive also
/* Try deleting in archive dir. If fails, try deleting in main db dir.
* This is efficient because all except for very few wal files will be in
* archive. Checking for WalType is not much helpful because alive wal could
be archived now.
*/
Status st = env_->DeleteFile(dbname_ + "/" + wal->PathName());
if (!st.ok()) {
if (wal->Type() == kAliveLogFile &&
env_->DeleteFile(LogFileName(archivedir, wal->LogNumber())).ok()) {
continue;
}
files_not_deleted.append(wal->PathName());
if (!env_->DeleteFile(archivedir + "/" + wal->Filename()).ok() &&
!env_->DeleteFile(dbname_ + "/" + wal->Filename()).ok()) {
files_not_deleted.append(wal->Filename());
}
}
if (!files_not_deleted.empty()) {
+74 -283
View File
@@ -28,13 +28,13 @@
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "db/transaction_log_impl.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/statistics.h"
#include "rocksdb/status.h"
#include "rocksdb/table_builder.h"
#include "leveldb/compaction_filter.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/merge_operator.h"
#include "leveldb/statistics.h"
#include "leveldb/status.h"
#include "leveldb/table_builder.h"
#include "port/port.h"
#include "table/block.h"
#include "table/merger.h"
@@ -47,7 +47,7 @@
#include "util/mutexlock.h"
#include "util/stop_watch.h"
namespace rocksdb {
namespace leveldb {
void dumpLeveldbBuildVersion(Logger * log);
@@ -163,22 +163,6 @@ Options SanitizeOptions(const std::string& dbname,
result.compaction_filter_factory->CreateCompactionFilter().get()) {
Log(result.info_log, "Both filter and factory specified. Using filter");
}
if (result.prefix_extractor) {
// If a prefix extractor has been supplied and a PrefixHashRepFactory is
// being used, make sure that the latter uses the former as its transform
// function.
auto factory = dynamic_cast<PrefixHashRepFactory*>(
result.memtable_factory.get());
if (factory &&
factory->GetTransform() != result.prefix_extractor) {
Log(result.info_log, "A prefix hash representation factory was supplied "
"whose prefix extractor does not match options.prefix_extractor. "
"Falling back to skip list representation factory");
result.memtable_factory = std::make_shared<SkipListFactory>();
} else if (factory) {
Log(result.info_log, "Prefix hash memtable rep is in use.");
}
}
return result;
}
@@ -200,7 +184,6 @@ DBImpl::DBImpl(const Options& options, const std::string& dbname)
logfile_number_(0),
tmp_batch_(),
bg_compaction_scheduled_(0),
bg_flush_scheduled_(0),
bg_logstats_scheduled_(false),
manual_compaction_(nullptr),
logger_(nullptr),
@@ -268,9 +251,7 @@ DBImpl::~DBImpl() {
}
mutex_.Lock();
shutting_down_.Release_Store(this); // Any non-nullptr value is ok
while (bg_compaction_scheduled_ ||
bg_flush_scheduled_ ||
bg_logstats_scheduled_) {
while (bg_compaction_scheduled_ || bg_logstats_scheduled_) {
bg_cv_.Wait();
}
mutex_.Unlock();
@@ -290,17 +271,13 @@ void DBImpl::TEST_Destroy_DBImpl() {
// wait till all background compactions are done.
mutex_.Lock();
while (bg_compaction_scheduled_ ||
bg_flush_scheduled_ ||
bg_logstats_scheduled_) {
while (bg_compaction_scheduled_ || bg_logstats_scheduled_) {
bg_cv_.Wait();
}
// Prevent new compactions from occuring.
bg_work_gate_closed_ = true;
const int LargeNumber = 10000000;
bg_compaction_scheduled_ += LargeNumber;
mutex_.Unlock();
// force release the lock file.
@@ -429,26 +406,6 @@ void DBImpl::FindObsoleteFiles(DeletionState& deletion_state) {
deletion_state.prevlognumber = versions_->PrevLogNumber();
}
Status DBImpl::DeleteLogFile(uint64_t number) {
Status s;
auto filename = LogFileName(dbname_, number);
if (options_.WAL_ttl_seconds > 0) {
s = env_->RenameFile(filename,
ArchivedLogFileName(dbname_, number));
if (!s.ok()) {
Log(options_.info_log, "RenameFile logfile #%lu FAILED", number);
}
} else {
s = env_->DeleteFile(filename);
if(!s.ok()) {
Log(options_.info_log, "Delete logfile #%lu FAILED", number);
}
}
return s;
}
// Diffs the files listed in filenames and those that do not
// belong to live files are posibly removed. If the removed file
// is a sst file, then it returns the file number in files_to_evict.
@@ -503,9 +460,19 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
state.files_to_evict.push_back(number);
}
Log(options_.info_log, "Delete type=%d #%lu", int(type), number);
if (type == kLogFile && options_.WAL_ttl_seconds > 0) {
Status st = env_->RenameFile(
LogFileName(dbname_, number),
ArchivedLogFileName(dbname_, number)
);
if (type == kLogFile) {
DeleteLogFile(number);
if (!st.ok()) {
Log(
options_.info_log, "RenameFile type=%d #%lu FAILED",
int(type),
number
);
}
} else {
Status st = env_->DeleteFile(dbname_ + "/" + state.allfiles[i]);
if (!st.ok()) {
@@ -518,7 +485,7 @@ void DBImpl::PurgeObsoleteFiles(DeletionState& state) {
}
}
// Delete old info log files.
// Delete old log files.
size_t old_log_file_count = old_log_files.size();
// NOTE: Currently we only support log purge when options_.db_log_dir is
// located in `dbname` directory.
@@ -923,6 +890,7 @@ Status DBImpl::CompactMemTable(bool* madeProgress) {
}
// Save the contents of the earliest memtable as a new Table
// This will release and re-acquire the mutex.
uint64_t file_number;
std::vector<MemTable*> mems;
imm_.PickMemtablesToFlush(&mems);
@@ -936,10 +904,8 @@ Status DBImpl::CompactMemTable(bool* madeProgress) {
MemTable* m = mems[0];
VersionEdit* edit = m->GetEdits();
edit->SetPrevLogNumber(0);
edit->SetLogNumber(m->GetNextLogNumber()); // Earlier logs no longer needed
auto to_delete = m->GetLogNumber();
edit->SetLogNumber(m->GetLogNumber()); // Earlier logs no longer needed
// This will release and re-acquire the mutex.
Status s = WriteLevel0Table(mems, edit, &file_number);
if (s.ok() && shutting_down_.Acquire_Load()) {
@@ -957,25 +923,16 @@ Status DBImpl::CompactMemTable(bool* madeProgress) {
if (madeProgress) {
*madeProgress = 1;
}
MaybeScheduleLogDBDeployStats();
// TODO: if log deletion failed for any reason, we probably
// should store the file number in the shared state, and retry
// However, for now, PurgeObsoleteFiles will take care of that
// anyways.
if (options_.purge_log_after_memtable_flush &&
!disable_delete_obsolete_files_ &&
to_delete > 0) {
mutex_.Unlock();
DeleteLogFile(to_delete);
mutex_.Lock();
}
// we could have deleted obsolete files here, but it is not
// absolutely necessary because it could be also done as part
// of other background compaction
}
return s;
}
void DBImpl::CompactRange(const Slice* begin, const Slice* end,
bool reduce_level, int target_level) {
bool reduce_level) {
int max_level_with_files = 1;
{
MutexLock l(&mutex_);
@@ -992,7 +949,7 @@ void DBImpl::CompactRange(const Slice* begin, const Slice* end,
}
if (reduce_level) {
ReFitLevel(max_level_with_files, target_level);
ReFitLevel(max_level_with_files);
}
}
@@ -1012,7 +969,7 @@ int DBImpl::FindMinimumEmptyLevelFitting(int level) {
return minimum_level;
}
void DBImpl::ReFitLevel(int level, int target_level) {
void DBImpl::ReFitLevel(int level) {
assert(level < NumberLevels());
MutexLock l(&mutex_);
@@ -1026,18 +983,15 @@ void DBImpl::ReFitLevel(int level, int target_level) {
// wait for all background threads to stop
bg_work_gate_closed_ = true;
while (bg_compaction_scheduled_ > 0 || bg_flush_scheduled_) {
while (bg_compaction_scheduled_ > 0) {
Log(options_.info_log,
"RefitLevel: waiting for background threads to stop: %d %d",
bg_compaction_scheduled_, bg_flush_scheduled_);
"RefitLevel: waiting for background threads to stop: %d",
bg_compaction_scheduled_);
bg_cv_.Wait();
}
// move to a smaller level
int to_level = target_level;
if (target_level < 0) {
to_level = FindMinimumEmptyLevelFitting(level);
}
int to_level = FindMinimumEmptyLevelFitting(level);
assert(to_level <= level);
@@ -1092,7 +1046,6 @@ SequenceNumber DBImpl::GetLatestSequenceNumber() {
Status DBImpl::GetUpdatesSince(SequenceNumber seq,
unique_ptr<TransactionLogIterator>* iter) {
RecordTick(options_.statistics, GET_UPDATES_SINCE_CALLS);
if (seq > last_flushed_sequence_) {
return Status::IOError("Requested sequence not yet written in the db");
}
@@ -1243,12 +1196,6 @@ Status DBImpl::AppendSortedWalsOfType(const std::string& path,
return status;
}
log_files.reserve(log_files.size() + all_files.size());
VectorLogPtr::iterator pos_start;
if (!log_files.empty()) {
pos_start = log_files.end() - 1;
} else {
pos_start = log_files.begin();
}
for (const auto& f : all_files) {
uint64_t number;
FileType type;
@@ -1274,7 +1221,7 @@ Status DBImpl::AppendSortedWalsOfType(const std::string& path,
}
}
CompareLogByPointer compare_log_files;
std::sort(pos_start, log_files.end(), compare_log_files);
std::sort(log_files.begin(), log_files.end(), compare_log_files);
return status;
}
@@ -1381,8 +1328,7 @@ Status DBImpl::TEST_WaitForCompactMemTable() {
Status DBImpl::TEST_WaitForCompact() {
// Wait until the compaction completes
MutexLock l(&mutex_);
while ((bg_compaction_scheduled_ || bg_flush_scheduled_) &&
bg_error_.ok()) {
while (bg_compaction_scheduled_ && bg_error_.ok()) {
bg_cv_.Wait();
}
return bg_error_;
@@ -1392,80 +1338,29 @@ void DBImpl::MaybeScheduleCompaction() {
mutex_.AssertHeld();
if (bg_work_gate_closed_) {
// gate closed for backgrond work
} else if (bg_compaction_scheduled_ >= options_.max_background_compactions) {
// Already scheduled
} else if (shutting_down_.Acquire_Load()) {
// DB is being deleted; no more background compactions
} else if (!imm_.IsFlushPending(options_.min_write_buffer_number_to_merge) &&
manual_compaction_ == nullptr &&
!versions_->NeedsCompaction()) {
// No work to be done
} else {
bool is_flush_pending =
imm_.IsFlushPending(options_.min_write_buffer_number_to_merge);
if (is_flush_pending &&
(bg_flush_scheduled_ < options_.max_background_flushes)) {
// memtable flush needed
bg_flush_scheduled_++;
env_->Schedule(&DBImpl::BGWorkFlush, this, Env::Priority::HIGH);
}
if ((manual_compaction_ ||
versions_->NeedsCompaction() ||
(is_flush_pending && (options_.max_background_flushes <= 0))) &&
bg_compaction_scheduled_ < options_.max_background_compactions) {
// compaction needed, or memtable flush needed but HIGH pool not enabled.
bg_compaction_scheduled_++;
env_->Schedule(&DBImpl::BGWorkCompaction, this, Env::Priority::LOW);
}
bg_compaction_scheduled_++;
env_->Schedule(&DBImpl::BGWork, this);
}
}
void DBImpl::BGWorkFlush(void* db) {
reinterpret_cast<DBImpl*>(db)->BackgroundCallFlush();
void DBImpl::BGWork(void* db) {
reinterpret_cast<DBImpl*>(db)->BackgroundCall();
}
void DBImpl::BGWorkCompaction(void* db) {
reinterpret_cast<DBImpl*>(db)->BackgroundCallCompaction();
}
Status DBImpl::BackgroundFlush() {
Status stat;
while (stat.ok() &&
imm_.IsFlushPending(options_.min_write_buffer_number_to_merge)) {
Log(options_.info_log,
"BackgroundCallFlush doing CompactMemTable, flush slots available %d",
options_.max_background_flushes - bg_flush_scheduled_);
stat = CompactMemTable();
}
return stat;
}
void DBImpl::BackgroundCallFlush() {
assert(bg_flush_scheduled_);
MutexLock l(&mutex_);
if (!shutting_down_.Acquire_Load()) {
Status s = BackgroundFlush();
if (!s.ok()) {
// Wait a little bit before retrying background compaction in
// case this is an environmental problem and we do not want to
// chew up resources for failed compactions for the duration of
// the problem.
bg_cv_.SignalAll(); // In case a waiter can proceed despite the error
Log(options_.info_log, "Waiting after background flush error: %s",
s.ToString().c_str());
mutex_.Unlock();
env_->SleepForMicroseconds(1000000);
mutex_.Lock();
}
}
bg_flush_scheduled_--;
bg_cv_.SignalAll();
}
void DBImpl::TEST_PurgeObsoleteteWAL() {
PurgeObsoleteWALFiles();
}
void DBImpl::BackgroundCallCompaction() {
void DBImpl::BackgroundCall() {
bool madeProgress = false;
DeletionState deletion_state;
@@ -1518,7 +1413,6 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
*madeProgress = false;
mutex_.AssertHeld();
// TODO: remove memtable flush from formal compaction
while (imm_.IsFlushPending(options_.min_write_buffer_number_to_merge)) {
Log(options_.info_log,
"BackgroundCompaction doing CompactMemTable, compaction slots available %d",
@@ -1895,7 +1789,6 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
}
for (; input->Valid() && !shutting_down_.Acquire_Load(); ) {
// Prioritize immutable compaction work
// TODO: remove memtable flush from normal compaction work
if (imm_.imm_flush_needed.NoBarrier_Load() != nullptr) {
const uint64_t imm_start = env_->NowMicros();
mutex_.Lock();
@@ -2192,7 +2085,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
}
mutex_.Lock();
stats_[compact->compaction->output_level()].Add(stats);
stats_[compact->compaction->level() + 1].Add(stats);
// if there were any unused file number (mostly in case of
// compaction error), free up the entry from pending_putputs
@@ -2204,8 +2097,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
VersionSet::LevelSummaryStorage tmp;
Log(options_.info_log,
"compacted to: %s, %.1f MB/sec, level %d, files in(%d, %d) out(%d) "
"MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
"write-amplify(%.1f) %s\n",
"MB in(%.1f, %.1f) out(%.1f), amplify(%.1f) %s\n",
versions_->LevelSummary(&tmp),
(stats.bytes_readn + stats.bytes_readnp1 + stats.bytes_written) /
(double) stats.micros,
@@ -2214,9 +2106,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
stats.bytes_readn / 1048576.0,
stats.bytes_readnp1 / 1048576.0,
stats.bytes_written / 1048576.0,
(stats.bytes_written + stats.bytes_readnp1 + stats.bytes_readn) /
(stats.bytes_written + stats.bytes_readnp1) /
(double) stats.bytes_readn,
stats.bytes_written / (double) stats.bytes_readn,
status.ToString().c_str());
return status;
@@ -2250,8 +2141,7 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
// Collect together all needed child iterators for mem
std::vector<Iterator*> list;
mem_->Ref();
list.push_back(mem_->NewIterator(options.prefix));
list.push_back(mem_->NewIterator());
cleanup->mem.push_back(mem_);
// Collect together all needed child iterators for imm_
@@ -2260,7 +2150,7 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
for (unsigned int i = 0; i < immutables.size(); i++) {
MemTable* m = immutables[i];
m->Ref();
list.push_back(m->NewIterator(options.prefix));
list.push_back(m->NewIterator());
cleanup->mem.push_back(m);
}
@@ -2297,12 +2187,13 @@ Status DBImpl::Get(const ReadOptions& options,
Status DBImpl::GetImpl(const ReadOptions& options,
const Slice& key,
std::string* value,
const bool no_io,
bool* value_found) {
Status s;
StopWatch sw(env_, options_.statistics, DB_GET);
SequenceNumber snapshot;
mutex_.Lock();
MutexLock l(&mutex_);
if (options.snapshot != nullptr) {
snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
} else {
@@ -2347,9 +2238,6 @@ Status DBImpl::GetImpl(const ReadOptions& options,
mem->Unref();
imm.UnrefAll();
current->Unref();
mutex_.Unlock();
// Note, tickers are atomic now - no lock protection needed any more.
RecordTick(options_.statistics, NUMBER_KEYS_READ);
RecordTick(options_.statistics, BYTES_READ, value->size());
return s;
@@ -2361,7 +2249,7 @@ std::vector<Status> DBImpl::MultiGet(const ReadOptions& options,
StopWatch sw(env_, options_.statistics, DB_MULTIGET);
SequenceNumber snapshot;
mutex_.Lock();
MutexLock l(&mutex_);
if (options.snapshot != nullptr) {
snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
} else {
@@ -2425,8 +2313,6 @@ std::vector<Status> DBImpl::MultiGet(const ReadOptions& options,
mem->Unref();
imm.UnrefAll();
current->Unref();
mutex_.Unlock();
RecordTick(options_.statistics, NUMBER_MULTIGET_CALLS);
RecordTick(options_.statistics, NUMBER_MULTIGET_KEYS_READ, numKeys);
RecordTick(options_.statistics, NUMBER_MULTIGET_BYTES_READ, bytesRead);
@@ -2441,9 +2327,7 @@ bool DBImpl::KeyMayExist(const ReadOptions& options,
if (value_found != nullptr) {
*value_found = true; // falsify later if key-may-exist but can't fetch value
}
ReadOptions roptions = options;
roptions.read_tier = kBlockCacheTier; // read from block cache only
return GetImpl(roptions, key, value, value_found).ok();
return GetImpl(options, key, value, true, value_found).ok();
}
Iterator* DBImpl::NewIterator(const ReadOptions& options) {
@@ -2513,28 +2397,22 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
uint64_t last_sequence = versions_->LastSequence();
Writer* last_writer = &w;
if (status.ok() && my_batch != nullptr) { // nullptr batch is for compactions
// TODO: BuildBatchGroup physically concatenate/copy all write batches into
// a new one. Mem copy is done with the lock held. Ideally, we only need
// the lock to obtain the last_writer and the references to all batches.
// Creation (copy) of the merged batch could have been done outside of the
// lock protected region.
WriteBatch* updates = BuildBatchGroup(&last_writer);
const SequenceNumber current_sequence = last_sequence + 1;
WriteBatchInternal::SetSequence(updates, current_sequence);
int my_batch_count = WriteBatchInternal::Count(updates);
last_sequence += my_batch_count;
// Record statistics
RecordTick(options_.statistics, NUMBER_KEYS_WRITTEN, my_batch_count);
RecordTick(options_.statistics,
BYTES_WRITTEN,
WriteBatchInternal::ByteSize(updates));
// Add to log and apply to memtable. We can release the lock
// during this phase since &w is currently responsible for logging
// and protects against concurrent loggers and concurrent writes
// into mem_.
{
mutex_.Unlock();
const SequenceNumber current_sequence = last_sequence + 1;
WriteBatchInternal::SetSequence(updates, current_sequence);
int my_batch_count = WriteBatchInternal::Count(updates);
last_sequence += my_batch_count;
// Record statistics
RecordTick(options_.statistics, NUMBER_KEYS_WRITTEN, my_batch_count);
RecordTick(options_.statistics,
BYTES_WRITTEN,
WriteBatchInternal::ByteSize(updates));
if (options.disableWAL) {
flush_on_destroy_ = true;
}
@@ -2561,13 +2439,11 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
// have succeeded in memtable but Status reports error for all writes.
throw std::runtime_error("In memory WriteBatch corruption!");
}
SetTickerCount(options_.statistics, SEQUENCE_NUMBER, last_sequence);
}
mutex_.Lock();
if (status.ok()) {
RecordTick(options_.statistics, SEQUENCE_NUMBER, my_batch_count);
versions_->SetLastSequence(last_sequence);
last_flushed_sequence_ = current_sequence;
}
mutex_.Lock();
}
if (updates == &tmp_batch_) tmp_batch_.Clear();
}
@@ -2829,15 +2705,11 @@ Status DBImpl::MakeRoomForWrite(bool force) {
lfile->SetPreallocationBlockSize(1.1 * options_.write_buffer_size);
logfile_number_ = new_log_number;
log_.reset(new log::Writer(std::move(lfile)));
mem_->SetNextLogNumber(logfile_number_);
mem_->SetLogNumber(logfile_number_);
imm_.Add(mem_);
if (force) {
imm_.FlushRequested();
}
mem_ = new MemTable(internal_comparator_, mem_rep_factory_,
NumberLevels(), options_);
mem_->Ref();
mem_->SetLogNumber(logfile_number_);
force = false; // Do not force another compaction if have room
MaybeScheduleCompaction();
}
@@ -2901,7 +2773,7 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
// Pardon the long line but I think it is easier to read this way.
snprintf(buf, sizeof(buf),
" Compactions\n"
"Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt\n"
"Level Files Size(MB) Score Time(sec) Read(MB) Write(MB) Rn(MB) Rnp1(MB) Wnew(MB) Amplify Read(MB/s) Write(MB/s) Rn Rnp1 Wnp1 NewW Count Ln-stall Stall-cnt\n"
"--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
);
value->append(buf);
@@ -2914,9 +2786,7 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
stats_[level].bytes_readnp1;
double amplify = (stats_[level].bytes_readn == 0)
? 0.0
: (stats_[level].bytes_written +
stats_[level].bytes_readnp1 +
stats_[level].bytes_readn) /
: (stats_[level].bytes_written + stats_[level].bytes_readnp1) /
(double) stats_[level].bytes_readn;
total_bytes_read += bytes_read;
@@ -2924,12 +2794,12 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
snprintf(
buf, sizeof(buf),
"%3d %8d %8.0f %5.1f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %10.1f %9.1f %11.1f %8d %8d %8d %8d %8d %9.1f %9lu\n",
"%3d %8d %8.0f %5.1f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %7.1f %9.1f %11.1f %8d %8d %8d %8d %8d %9.1f %9lu\n",
level,
files,
versions_->NumLevelBytes(level) / 1048576.0,
versions_->NumLevelBytes(level) /
versions_->MaxBytesForLevel(level),
versions_->MaxBytesForLevel(level),
stats_[level].micros / 1e6,
bytes_read / 1048576.0,
stats_[level].bytes_written / 1048576.0,
@@ -3081,71 +2951,6 @@ inline void DBImpl::DelayLoggingAndReset() {
}
}
Status DBImpl::DeleteFile(std::string name) {
uint64_t number;
FileType type;
if (!ParseFileName(name, &number, &type) ||
(type != kTableFile)) {
Log(options_.info_log, "DeleteFile #%lld FAILED. Invalid file name\n",
static_cast<unsigned long long>(number));
return Status::InvalidArgument("Invalid file name");
}
int level;
FileMetaData metadata;
int maxlevel = NumberLevels();
VersionEdit edit(maxlevel);
DeletionState deletion_state;
Status status;
{
MutexLock l(&mutex_);
status = versions_->GetMetadataForFile(number, &level, &metadata);
if (!status.ok()) {
Log(options_.info_log, "DeleteFile #%lld FAILED. File not found\n",
static_cast<unsigned long long>(number));
return Status::InvalidArgument("File not found");
}
assert((level > 0) && (level < maxlevel));
// If the file is being compacted no need to delete.
if (metadata.being_compacted) {
Log(options_.info_log,
"DeleteFile #%lld Skipped. File about to be compacted\n",
static_cast<unsigned long long>(number));
return Status::OK();
}
// Only the files in the last level can be deleted externally.
// This is to make sure that any deletion tombstones are not
// lost. Check that the level passed is the last level.
for (int i = level + 1; i < maxlevel; i++) {
if (versions_->NumLevelFiles(i) != 0) {
Log(options_.info_log,
"DeleteFile #%lld FAILED. File not in last level\n",
static_cast<unsigned long long>(number));
return Status::InvalidArgument("File not in last level");
}
}
edit.DeleteFile(level, number);
status = versions_->LogAndApply(&edit, &mutex_);
if (status.ok()) {
FindObsoleteFiles(deletion_state);
}
} // lock released here
if (status.ok()) {
// remove files outside the db-lock
PurgeObsoleteFiles(deletion_state);
EvictObsoleteFiles(deletion_state);
}
return status;
}
void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata) {
MutexLock l(&mutex_);
return versions_->GetLiveFilesMetaData(metadata);
}
// Default implementations of convenience methods that subclasses of DB
// can call if they wish
Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) {
@@ -3200,26 +3005,12 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
s = impl->versions_->LogAndApply(&edit, &impl->mutex_);
}
if (s.ok()) {
impl->mem_->SetLogNumber(impl->logfile_number_);
impl->DeleteObsoleteFiles();
impl->MaybeScheduleCompaction();
impl->MaybeScheduleLogDBDeployStats();
}
}
impl->mutex_.Unlock();
if (options.compaction_style == kCompactionStyleUniversal) {
int num_files;
for (int i = 1; i < impl->NumberLevels(); i++) {
num_files = impl->versions_->NumLevelFiles(i);
if (num_files > 0) {
s = Status::InvalidArgument("Not all files are at level 0. Cannot "
"open with universal compaction style.");
break;
}
}
}
if (s.ok()) {
*dbptr = impl;
} else {
@@ -3294,4 +3085,4 @@ void dumpLeveldbBuildVersion(Logger * log) {
leveldb_build_compile_time, leveldb_build_compile_date);
}
} // namespace rocksdb
} // namespace leveldb
+14 -27
View File
@@ -12,10 +12,10 @@
#include "db/dbformat.h"
#include "db/log_writer.h"
#include "db/snapshot.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/transaction_log.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/memtablerep.h"
#include "leveldb/transaction_log.h"
#include "port/port.h"
#include "util/stats_logger.h"
#include "memtablelist.h"
@@ -24,7 +24,7 @@
#include "scribe/scribe_logger.h"
#endif
namespace rocksdb {
namespace leveldb {
class MemTable;
class TableCache;
@@ -64,7 +64,7 @@ class DBImpl : public DB {
virtual bool GetProperty(const Slice& property, std::string* value);
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
virtual void CompactRange(const Slice* begin, const Slice* end,
bool reduce_level = false, int target_level = -1);
bool reduce_level = false);
virtual int NumberLevels();
virtual int MaxMemCompactionLevel();
virtual int Level0StopWriteTrigger();
@@ -72,17 +72,12 @@ class DBImpl : public DB {
virtual Status DisableFileDeletions();
virtual Status EnableFileDeletions();
virtual Status GetLiveFiles(std::vector<std::string>&,
uint64_t* manifest_file_size,
bool flush_memtable = true);
uint64_t* manifest_file_size);
virtual Status GetSortedWalFiles(VectorLogPtr& files);
virtual Status DeleteWalFiles(const VectorLogPtr& files);
virtual SequenceNumber GetLatestSequenceNumber();
virtual Status GetUpdatesSince(SequenceNumber seq_number,
unique_ptr<TransactionLogIterator>* iter);
virtual Status DeleteFile(std::string name);
virtual void GetLiveFilesMetaData(
std::vector<LiveFileMetaData> *metadata);
// Extra methods (for testing) that are not in the public DB interface
@@ -188,12 +183,9 @@ class DBImpl : public DB {
void LogDBDeployStats();
void MaybeScheduleCompaction();
static void BGWorkCompaction(void* db);
static void BGWorkFlush(void* db);
void BackgroundCallCompaction();
void BackgroundCallFlush();
static void BGWork(void* db);
void BackgroundCall();
Status BackgroundCompaction(bool* madeProgress,DeletionState& deletion_state);
Status BackgroundFlush();
void CleanupCompaction(CompactionState* compact);
Status DoCompactionWork(CompactionState* compact);
@@ -216,8 +208,6 @@ class DBImpl : public DB {
// Removes the file listed in files_to_evict from the table_cache
void EvictObsoleteFiles(DeletionState& deletion_state);
Status DeleteLogFile(uint64_t number);
void PurgeObsoleteWALFiles();
Status AppendSortedWalsOfType(const std::string& path,
@@ -247,10 +237,9 @@ class DBImpl : public DB {
// input level. Return the input level, if such level could not be found.
int FindMinimumEmptyLevelFitting(int level);
// Move the files in the input level to the target level.
// If target_level < 0, automatically calculate the minimum level that could
// hold the data set.
void ReFitLevel(int level, int target_level = -1);
// Move the files in the input level to the minimum level that could hold
// the data set.
void ReFitLevel(int level);
// Constant after construction
const InternalFilterPolicy internal_filter_policy_;
@@ -287,9 +276,6 @@ class DBImpl : public DB {
// count how many background compaction been scheduled or is running?
int bg_compaction_scheduled_;
// number of background memtable flush jobs, submitted to the HIGH pool
int bg_flush_scheduled_;
// Has a background stats log thread scheduled?
bool bg_logstats_scheduled_;
@@ -434,6 +420,7 @@ class DBImpl : public DB {
Status GetImpl(const ReadOptions& options,
const Slice& key,
std::string* value,
const bool no_io = false,
bool* value_found = nullptr);
};
@@ -444,6 +431,6 @@ extern Options SanitizeOptions(const std::string& db,
const InternalFilterPolicy* ipolicy,
const Options& src);
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_DB_IMPL_H_
+5 -5
View File
@@ -21,10 +21,10 @@
#include "db/table_cache.h"
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "rocksdb/table_builder.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/status.h"
#include "leveldb/table_builder.h"
#include "port/port.h"
#include "table/block.h"
#include "table/merger.h"
@@ -34,7 +34,7 @@
#include "util/logging.h"
#include "util/build_version.h"
namespace rocksdb {
namespace leveldb {
DBImplReadOnly::DBImplReadOnly(const Options& options,
const std::string& dbname)
+5 -6
View File
@@ -12,8 +12,8 @@
#include "db/dbformat.h"
#include "db/log_writer.h"
#include "db/snapshot.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "port/port.h"
#include "util/stats_logger.h"
@@ -21,7 +21,7 @@
#include "scribe/scribe_logger.h"
#endif
namespace rocksdb {
namespace leveldb {
class DBImplReadOnly : public DBImpl {
public:
@@ -51,7 +51,7 @@ public:
return Status::NotSupported("Not supported operation in read only mode.");
}
virtual void CompactRange(const Slice* begin, const Slice* end,
bool reduce_level = false, int target_level = -1) {
bool reduce_level = false) {
}
virtual Status DisableFileDeletions() {
return Status::NotSupported("Not supported operation in read only mode.");
@@ -60,8 +60,7 @@ public:
return Status::NotSupported("Not supported operation in read only mode.");
}
virtual Status GetLiveFiles(std::vector<std::string>&,
uint64_t* manifest_file_size,
bool flush_memtable = true) {
uint64_t* manifest_file_size) {
return Status::NotSupported("Not supported operation in read only mode.");
}
virtual Status Flush(const FlushOptions& options) {
+9 -45
View File
@@ -8,16 +8,15 @@
#include "db/filename.h"
#include "db/dbformat.h"
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/iterator.h"
#include "rocksdb/merge_operator.h"
#include "leveldb/env.h"
#include "leveldb/options.h"
#include "leveldb/iterator.h"
#include "leveldb/merge_operator.h"
#include "port/port.h"
#include "util/logging.h"
#include "util/mutexlock.h"
#include "util/perf_context_imp.h"
namespace rocksdb {
namespace leveldb {
#if 0
static void DumpInternalIter(Iterator* iter) {
@@ -66,7 +65,6 @@ class DBIter: public Iterator {
current_entry_is_merged_(false),
statistics_(options.statistics) {
RecordTick(statistics_, NO_ITERATORS, 1);
max_skip_ = options.max_sequential_skip_in_iterations;
}
virtual ~DBIter() {
RecordTick(statistics_, NO_ITERATORS, -1);
@@ -131,7 +129,6 @@ class DBIter: public Iterator {
bool valid_;
bool current_entry_is_merged_;
std::shared_ptr<Statistics> statistics_;
uint64_t max_skip_;
// No copying allowed
DBIter(const DBIter&);
@@ -191,14 +188,12 @@ void DBIter::FindNextUserEntry(bool skipping) {
assert(iter_->Valid());
assert(direction_ == kForward);
current_entry_is_merged_ = false;
uint64_t num_skipped = 0;
do {
ParsedInternalKey ikey;
if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
if (skipping &&
user_comparator_->Compare(ikey.user_key, saved_key_) <= 0) {
num_skipped++; // skip this entry
BumpPerfCount(&perf_context.internal_key_skipped_count);
// skip this entry
} else {
skipping = false;
switch (ikey.type) {
@@ -207,8 +202,6 @@ void DBIter::FindNextUserEntry(bool skipping) {
// they are hidden by this deletion.
SaveKey(ikey.user_key, &saved_key_);
skipping = true;
num_skipped = 0;
BumpPerfCount(&perf_context.internal_delete_skipped_count);
break;
case kTypeValue:
valid_ = true;
@@ -227,20 +220,7 @@ void DBIter::FindNextUserEntry(bool skipping) {
}
}
}
// If we have sequentially iterated via numerous keys and still not
// found the next user-key, then it is better to seek so that we can
// avoid too many key comparisons. We seek to the last occurence of
// our current key by looking for sequence number 0.
if (skipping && num_skipped > max_skip_) {
num_skipped = 0;
std::string last_key;
AppendInternalKey(&last_key,
ParsedInternalKey(Slice(saved_key_), 0, kValueTypeForSeek));
iter_->Seek(last_key);
RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
} else {
iter_->Next();
}
iter_->Next();
} while (iter_->Valid());
valid_ = false;
}
@@ -362,7 +342,6 @@ void DBIter::Prev() {
void DBIter::FindPrevUserEntry() {
assert(direction_ == kReverse);
uint64_t num_skipped = 0;
ValueType value_type = kTypeDeletion;
if (iter_->Valid()) {
@@ -388,22 +367,7 @@ void DBIter::FindPrevUserEntry() {
saved_value_.assign(raw_value.data(), raw_value.size());
}
}
num_skipped++;
// If we have sequentially iterated via numerous keys and still not
// found the prev user-key, then it is better to seek so that we can
// avoid too many key comparisons. We seek to the first occurence of
// our current key by looking for max sequence number.
if (num_skipped > max_skip_) {
num_skipped = 0;
std::string last_key;
AppendInternalKey(&last_key,
ParsedInternalKey(Slice(saved_key_), kMaxSequenceNumber,
kValueTypeForSeek));
iter_->Seek(last_key);
RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
} else {
iter_->Prev();
}
iter_->Prev();
} while (iter_->Valid());
}
@@ -471,4 +435,4 @@ Iterator* NewDBIterator(
internal_iter, sequence);
}
} // namespace rocksdb
} // namespace leveldb
+3 -3
View File
@@ -6,10 +6,10 @@
#define STORAGE_LEVELDB_DB_DB_ITER_H_
#include <stdint.h>
#include "rocksdb/db.h"
#include "leveldb/db.h"
#include "db/dbformat.h"
namespace rocksdb {
namespace leveldb {
// Return a new iterator that converts internal keys (yielded by
// "*internal_iter") that were live at the specified "sequence" number
@@ -22,6 +22,6 @@ extern Iterator* NewDBIterator(
Iterator* internal_iter,
const SequenceNumber& sequence);
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_DB_ITER_H_
+5 -3
View File
@@ -10,13 +10,13 @@
#include <vector>
#include <memory>
#include "rocksdb/statistics.h"
#include "leveldb/statistics.h"
#include "util/histogram.h"
#include "port/port.h"
#include "util/mutexlock.h"
namespace rocksdb {
namespace leveldb {
class DBStatistics: public Statistics {
public:
@@ -59,6 +59,8 @@ std::shared_ptr<Statistics> CreateDBStatistics() {
return std::make_shared<DBStatistics>();
}
} // namespace rocksdb
} // namespace leveldb
#endif // LEVELDB_STORAGE_DB_DB_STATISTICS_H_
+3 -3
View File
@@ -7,12 +7,12 @@
#include <stdint.h>
#include <stdio.h>
#include "db/version_set.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "port/port.h"
#include "util/mutexlock.h"
namespace rocksdb {
namespace leveldb {
void DBImpl::MaybeScheduleLogDBDeployStats() {
+34 -460
View File
@@ -5,16 +5,15 @@
#include <algorithm>
#include <set>
#include "rocksdb/db.h"
#include "rocksdb/filter_policy.h"
#include "leveldb/db.h"
#include "leveldb/filter_policy.h"
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "db/db_statistics.h"
#include "rocksdb/cache.h"
#include "rocksdb/compaction_filter.h"
#include "rocksdb/env.h"
#include "leveldb/cache.h"
#include "leveldb/compaction_filter.h"
#include "leveldb/env.h"
#include "table/table.h"
#include "util/hash.h"
#include "util/logging.h"
@@ -23,7 +22,7 @@
#include "util/testutil.h"
#include "utilities/merge_operators.h"
namespace rocksdb {
namespace leveldb {
static bool SnappyCompressionSupported(const CompressionOptions& options) {
std::string out;
@@ -69,7 +68,6 @@ class AtomicCounter {
count_ = 0;
}
};
}
// Special Env used to delay background operations
@@ -209,12 +207,9 @@ class DBTest {
private:
const FilterPolicy* filter_policy_;
protected:
// Sequence of option configurations to try
enum OptionConfig {
kDefault,
kVectorRep,
kUnsortedRep,
kMergePut,
kFilter,
kUncompressed,
@@ -224,7 +219,6 @@ class DBTest {
kCompactOnFlush,
kPerfOptions,
kDeletesFilterFirst,
kPrefixHashRep,
kUniversalCompaction,
kEnd
};
@@ -299,10 +293,6 @@ class DBTest {
Options CurrentOptions() {
Options options;
switch (option_config_) {
case kPrefixHashRep:
options.memtable_factory.reset(new
PrefixHashRepFactory(NewFixedPrefixTransform(1)));
break;
case kMergePut:
options.merge_operator = MergeOperators::CreatePutOperator();
break;
@@ -331,12 +321,6 @@ class DBTest {
case kDeletesFilterFirst:
options.filter_deletes = true;
break;
case kUnsortedRep:
options.memtable_factory.reset(new UnsortedRepFactory);
break;
case kVectorRep:
options.memtable_factory.reset(new VectorRepFactory(100));
break;
case kUniversalCompaction:
options.compaction_style = kCompactionStyleUniversal;
break;
@@ -831,7 +815,6 @@ TEST(DBTest, KeyMayExist) {
std::string value;
Options options = CurrentOptions();
options.filter_policy = NewBloomFilterPolicy(20);
options.statistics = rocksdb::CreateDBStatistics();
Reopen(&options);
ASSERT_TRUE(!db_->KeyMayExist(ropts, "a", &value));
@@ -844,114 +827,25 @@ TEST(DBTest, KeyMayExist) {
dbfull()->Flush(FlushOptions());
value.clear();
long numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
long cache_miss =
options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
value_found = false;
ASSERT_TRUE(db_->KeyMayExist(ropts, "a", &value, &value_found));
ASSERT_TRUE(!value_found);
// assert that no new files were opened and no new blocks were
// read into block cache.
ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
ASSERT_EQ(cache_miss,
options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
ASSERT_TRUE(value_found);
ASSERT_EQ("b", value);
ASSERT_OK(db_->Delete(WriteOptions(), "a"));
numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
cache_miss = options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
ASSERT_TRUE(!db_->KeyMayExist(ropts, "a", &value));
ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
ASSERT_EQ(cache_miss,
options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
dbfull()->Flush(FlushOptions());
dbfull()->CompactRange(nullptr, nullptr);
numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
cache_miss = options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
ASSERT_TRUE(!db_->KeyMayExist(ropts, "a", &value));
ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
ASSERT_EQ(cache_miss,
options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
ASSERT_OK(db_->Delete(WriteOptions(), "c"));
numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
cache_miss = options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
ASSERT_TRUE(!db_->KeyMayExist(ropts, "c", &value));
ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
ASSERT_EQ(cache_miss,
options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
delete options.filter_policy;
} while (ChangeOptions());
}
TEST(DBTest, NonBlockingIteration) {
do {
ReadOptions non_blocking_opts, regular_opts;
Options options = CurrentOptions();
options.statistics = rocksdb::CreateDBStatistics();
non_blocking_opts.read_tier = kBlockCacheTier;
Reopen(&options);
// write one kv to the database.
ASSERT_OK(db_->Put(WriteOptions(), "a", "b"));
// scan using non-blocking iterator. We should find it because
// it is in memtable.
Iterator* iter = db_->NewIterator(non_blocking_opts);
int count = 0;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ASSERT_TRUE(iter->status().ok());
count++;
}
ASSERT_EQ(count, 1);
delete iter;
// flush memtable to storage. Now, the key should not be in the
// memtable neither in the block cache.
dbfull()->Flush(FlushOptions());
// verify that a non-blocking iterator does not find any
// kvs. Neither does it do any IOs to storage.
long numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
long cache_miss =
options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
iter = db_->NewIterator(non_blocking_opts);
count = 0;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
count++;
}
ASSERT_EQ(count, 0);
ASSERT_TRUE(iter->status().IsIncomplete());
ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
ASSERT_EQ(cache_miss,
options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
delete iter;
// read in the specified block via a regular get
ASSERT_EQ(Get("a"), "b");
// verify that we can find it via a non-blocking scan
numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
cache_miss = options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
iter = db_->NewIterator(non_blocking_opts);
count = 0;
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
ASSERT_TRUE(iter->status().ok());
count++;
}
ASSERT_EQ(count, 1);
ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
ASSERT_EQ(cache_miss,
options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
delete iter;
} while (ChangeOptions());
}
// A delete is skipped for key if KeyMayExist(key) returns False
// Tests Writebatch consistency and proper delete behaviour
TEST(DBTest, FilterDeletes) {
@@ -1134,95 +1028,6 @@ TEST(DBTest, IterMulti) {
} while (ChangeCompactOptions());
}
// Check that we can skip over a run of user keys
// by using reseek rather than sequential scan
TEST(DBTest, IterReseek) {
Options options = CurrentOptions();
options.max_sequential_skip_in_iterations = 3;
options.create_if_missing = true;
options.statistics = rocksdb::CreateDBStatistics();
DestroyAndReopen(&options);
// insert two keys with same userkey and verify that
// reseek is not invoked. For each of these test cases,
// verify that we can find the next key "b".
ASSERT_OK(Put("a", "one"));
ASSERT_OK(Put("a", "two"));
ASSERT_OK(Put("b", "bone"));
Iterator* iter = db_->NewIterator(ReadOptions());
iter->SeekToFirst();
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), 0);
ASSERT_EQ(IterStatus(iter), "a->two");
iter->Next();
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), 0);
ASSERT_EQ(IterStatus(iter), "b->bone");
delete iter;
// insert a total of three keys with same userkey and verify
// that reseek is still not invoked.
ASSERT_OK(Put("a", "three"));
iter = db_->NewIterator(ReadOptions());
iter->SeekToFirst();
ASSERT_EQ(IterStatus(iter), "a->three");
iter->Next();
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), 0);
ASSERT_EQ(IterStatus(iter), "b->bone");
delete iter;
// insert a total of four keys with same userkey and verify
// that reseek is invoked.
ASSERT_OK(Put("a", "four"));
iter = db_->NewIterator(ReadOptions());
iter->SeekToFirst();
ASSERT_EQ(IterStatus(iter), "a->four");
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), 0);
iter->Next();
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), 1);
ASSERT_EQ(IterStatus(iter), "b->bone");
delete iter;
// Testing reverse iterator
// At this point, we have three versions of "a" and one version of "b".
// The reseek statistics is already at 1.
int num_reseeks = (int)options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION);
// Insert another version of b and assert that reseek is not invoked
ASSERT_OK(Put("b", "btwo"));
iter = db_->NewIterator(ReadOptions());
iter->SeekToLast();
ASSERT_EQ(IterStatus(iter), "b->btwo");
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), num_reseeks);
iter->Prev();
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), num_reseeks+1);
ASSERT_EQ(IterStatus(iter), "a->four");
delete iter;
// insert two more versions of b. This makes a total of 4 versions
// of b and 4 versions of a.
ASSERT_OK(Put("b", "bthree"));
ASSERT_OK(Put("b", "bfour"));
iter = db_->NewIterator(ReadOptions());
iter->SeekToLast();
ASSERT_EQ(IterStatus(iter), "b->bfour");
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), num_reseeks + 2);
iter->Prev();
// the previous Prev call should have invoked reseek
ASSERT_EQ(options.statistics.get()->getTickerCount(
NUMBER_OF_RESEEKS_IN_ITERATION), num_reseeks + 3);
ASSERT_EQ(IterStatus(iter), "a->four");
delete iter;
}
TEST(DBTest, IterSmallAndLargeMix) {
do {
ASSERT_OK(Put("a", "va"));
@@ -1283,44 +1088,6 @@ TEST(DBTest, IterMultiWithDelete) {
} while (ChangeOptions());
}
TEST(DBTest, IterWithSnapshot) {
do {
ASSERT_OK(Put("key1", "val1"));
ASSERT_OK(Put("key2", "val2"));
ASSERT_OK(Put("key3", "val3"));
ASSERT_OK(Put("key4", "val4"));
ASSERT_OK(Put("key5", "val5"));
const Snapshot *snapshot = db_->GetSnapshot();
ReadOptions options;
options.snapshot = snapshot;
Iterator* iter = db_->NewIterator(options);
// Put more values after the snapshot
ASSERT_OK(Put("key100", "val100"));
ASSERT_OK(Put("key101", "val101"));
iter->Seek("key5");
ASSERT_EQ(IterStatus(iter), "key5->val5");
if (!CurrentOptions().merge_operator) {
// TODO: merge operator does not support backward iteration yet
iter->Prev();
ASSERT_EQ(IterStatus(iter), "key4->val4");
iter->Prev();
ASSERT_EQ(IterStatus(iter), "key3->val3");
iter->Next();
ASSERT_EQ(IterStatus(iter), "key4->val4");
iter->Next();
ASSERT_EQ(IterStatus(iter), "key5->val5");
iter->Next();
ASSERT_TRUE(!iter->Valid());
}
db_->ReleaseSnapshot(snapshot);
delete iter;
} while (ChangeOptions());
}
TEST(DBTest, Recover) {
do {
ASSERT_OK(Put("foo", "v1"));
@@ -1404,24 +1171,6 @@ TEST(DBTest, CheckLock) {
} while (ChangeCompactOptions());
}
TEST(DBTest, FlushMultipleMemtable) {
do {
Options options = CurrentOptions();
WriteOptions writeOpt = WriteOptions();
writeOpt.disableWAL = true;
options.max_write_buffer_number = 4;
options.min_write_buffer_number_to_merge = 3;
Reopen(&options);
ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v1"));
dbfull()->Flush(FlushOptions());
ASSERT_OK(dbfull()->Put(writeOpt, "bar", "v1"));
ASSERT_EQ("v1", Get("foo"));
ASSERT_EQ("v1", Get("bar"));
dbfull()->Flush(FlushOptions());
} while (ChangeCompactOptions());
}
TEST(DBTest, FLUSH) {
do {
Options options = CurrentOptions();
@@ -1637,8 +1386,8 @@ TEST(DBTest, UniversalCompactionTrigger) {
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 100<<10; //100KB
// trigger compaction if there are >= 4 files
options.level0_file_num_compaction_trigger = 4;
// trigger compaction if there are > 3 files
options.level0_file_num_compaction_trigger = 3;
Reopen(&options);
Random rnd(301);
@@ -1648,7 +1397,7 @@ TEST(DBTest, UniversalCompactionTrigger) {
// Generate a set of files at level 0, but don't trigger level-0
// compaction.
for (int num = 0;
num < options.level0_file_num_compaction_trigger-1;
num < options.level0_file_num_compaction_trigger;
num++) {
// Write 120KB (12 values, each 10K)
for (int i = 0; i < 12; i++) {
@@ -1683,7 +1432,7 @@ TEST(DBTest, UniversalCompactionTrigger) {
// data amount).
dbfull()->Flush(FlushOptions());
for (int num = 0;
num < options.level0_file_num_compaction_trigger-3;
num < options.level0_file_num_compaction_trigger-2;
num++) {
// Write 120KB (12 values, each 10K)
for (int i = 0; i < 12; i++) {
@@ -1712,7 +1461,7 @@ TEST(DBTest, UniversalCompactionTrigger) {
// Now we have 2 files at level 0, with size 4 and 2.4. Continue
// generating new files at level 0.
for (int num = 0;
num < options.level0_file_num_compaction_trigger-3;
num < options.level0_file_num_compaction_trigger-2;
num++) {
// Write 120KB (12 values, each 10K)
for (int i = 0; i < 12; i++) {
@@ -1766,170 +1515,6 @@ TEST(DBTest, UniversalCompactionTrigger) {
}
}
TEST(DBTest, UniversalCompactionSizeAmplification) {
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 100<<10; //100KB
options.level0_file_num_compaction_trigger = 3;
// Trigger compaction if size amplification exceeds 110%
options.compaction_options_universal.
max_size_amplification_percent = 110;
Reopen(&options);
Random rnd(301);
int key_idx = 0;
// Generate two files in Level 0. Both files are approx the same size.
for (int num = 0;
num < options.level0_file_num_compaction_trigger-1;
num++) {
// Write 120KB (12 values, each 10K)
for (int i = 0; i < 12; i++) {
ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
key_idx++;
}
dbfull()->TEST_WaitForCompactMemTable();
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
}
ASSERT_EQ(NumTableFilesAtLevel(0), 2);
// Flush whatever is remaining in memtable. This is typically
// small, which should not trigger size ratio based compaction
// but will instead trigger size amplification.
dbfull()->Flush(FlushOptions());
// Verify that size amplification did occur
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
}
TEST(DBTest, UniversalCompactionOptions) {
Options options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 100<<10; //100KB
options.level0_file_num_compaction_trigger = 4;
options.num_levels = 1;
Reopen(&options);
Random rnd(301);
int key_idx = 0;
for (int num = 0;
num < options.level0_file_num_compaction_trigger;
num++) {
// Write 120KB (12 values, each 10K)
for (int i = 0; i < 12; i++) {
ASSERT_OK(Put(Key(key_idx), RandomString(&rnd, 10000)));
key_idx++;
}
dbfull()->TEST_WaitForCompactMemTable();
if (num < options.level0_file_num_compaction_trigger - 1) {
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
}
}
dbfull()->TEST_WaitForCompact();
ASSERT_EQ(NumTableFilesAtLevel(0), 1);
for (int i = 1; i < options.num_levels ; i++) {
ASSERT_EQ(NumTableFilesAtLevel(i), 0);
}
}
TEST(DBTest, ConvertCompactionStyle) {
Random rnd(301);
int max_key_level_insert = 200;
int max_key_universal_insert = 600;
// Stage 1: generate a db with level compaction
Options options = CurrentOptions();
options.write_buffer_size = 100<<10; //100KB
options.num_levels = 4;
options.level0_file_num_compaction_trigger = 3;
options.max_bytes_for_level_base = 500<<10; // 500KB
options.max_bytes_for_level_multiplier = 1;
options.target_file_size_base = 200<<10; // 200KB
options.target_file_size_multiplier = 1;
Reopen(&options);
for (int i = 0; i <= max_key_level_insert; i++) {
// each value is 10K
ASSERT_OK(Put(Key(i), RandomString(&rnd, 10000)));
}
dbfull()->Flush(FlushOptions());
dbfull()->TEST_WaitForCompact();
ASSERT_GT(TotalTableFiles(), 1);
int non_level0_num_files = 0;
for (int i = 1; i < dbfull()->NumberLevels(); i++) {
non_level0_num_files += NumTableFilesAtLevel(i);
}
ASSERT_GT(non_level0_num_files, 0);
// Stage 2: reopen with universal compaction - should fail
options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
Status s = TryReopen(&options);
ASSERT_TRUE(s.IsInvalidArgument());
// Stage 3: compact into a single file and move the file to level 0
options = CurrentOptions();
options.disable_auto_compactions = true;
options.target_file_size_base = INT_MAX;
options.target_file_size_multiplier = 1;
options.max_bytes_for_level_base = INT_MAX;
options.max_bytes_for_level_multiplier = 1;
Reopen(&options);
dbfull()->CompactRange(nullptr, nullptr,
true /* reduce level */,
0 /* reduce to level 0 */);
for (int i = 0; i < dbfull()->NumberLevels(); i++) {
int num = NumTableFilesAtLevel(i);
if (i == 0) {
ASSERT_EQ(num, 1);
} else {
ASSERT_EQ(num, 0);
}
}
// Stage 4: re-open in universal compaction style and do some db operations
options = CurrentOptions();
options.compaction_style = kCompactionStyleUniversal;
options.write_buffer_size = 100<<10; //100KB
options.level0_file_num_compaction_trigger = 3;
Reopen(&options);
for (int i = max_key_level_insert / 2; i <= max_key_universal_insert; i++) {
ASSERT_OK(Put(Key(i), RandomString(&rnd, 10000)));
}
dbfull()->Flush(FlushOptions());
dbfull()->TEST_WaitForCompact();
for (int i = 1; i < dbfull()->NumberLevels(); i++) {
ASSERT_EQ(NumTableFilesAtLevel(i), 0);
}
// verify keys inserted in both level compaction style and universal
// compaction style
std::string keys_in_db;
Iterator* iter = dbfull()->NewIterator(ReadOptions());
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
keys_in_db.append(iter->key().ToString());
keys_in_db.push_back(',');
}
delete iter;
std::string expected_keys;
for (int i = 0; i <= max_key_universal_insert; i++) {
expected_keys.append(Key(i));
expected_keys.push_back(',');
}
ASSERT_EQ(keys_in_db, expected_keys);
}
void MinLevelHelper(DBTest* self, Options& options) {
Random rnd(301);
@@ -3840,7 +3425,7 @@ class ModelDB: public DB {
}
}
virtual void CompactRange(const Slice* start, const Slice* end,
bool reduce_level, int target_level) {
bool reduce_level ) {
}
virtual int NumberLevels()
@@ -3858,7 +3443,7 @@ class ModelDB: public DB {
return -1;
}
virtual Status Flush(const rocksdb::FlushOptions& options) {
virtual Status Flush(const leveldb::FlushOptions& options) {
Status ret;
return ret;
}
@@ -3869,8 +3454,7 @@ class ModelDB: public DB {
virtual Status EnableFileDeletions() {
return Status::OK();
}
virtual Status GetLiveFiles(std::vector<std::string>&, uint64_t* size,
bool flush_memtable = true) {
virtual Status GetLiveFiles(std::vector<std::string>&, uint64_t* size) {
return Status::OK();
}
@@ -3885,8 +3469,8 @@ class ModelDB: public DB {
virtual SequenceNumber GetLatestSequenceNumber() {
return 0;
}
virtual Status GetUpdatesSince(rocksdb::SequenceNumber,
unique_ptr<rocksdb::TransactionLogIterator>*) {
virtual Status GetUpdatesSince(leveldb::SequenceNumber,
unique_ptr<leveldb::TransactionLogIterator>*) {
return Status::NotSupported("Not supported in Model DB");
}
@@ -3925,13 +3509,10 @@ class ModelDB: public DB {
KVMap map_;
};
static std::string RandomKey(Random* rnd, int minimum = 0) {
int len;
do {
len = (rnd->OneIn(3)
? 1 // Short sometimes to encourage collisions
: (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
} while (len < minimum);
static std::string RandomKey(Random* rnd) {
int len = (rnd->OneIn(3)
? 1 // Short sometimes to encourage collisions
: (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
return test::RandomKey(rnd, len);
}
@@ -3993,12 +3574,8 @@ TEST(DBTest, Randomized) {
for (int step = 0; step < N; step++) {
// TODO(sanjay): Test Get() works
int p = rnd.Uniform(100);
int minimum = 0;
if (option_config_ == kPrefixHashRep) {
minimum = 1;
}
if (p < 45) { // Put
k = RandomKey(&rnd, minimum);
k = RandomKey(&rnd);
v = RandomString(&rnd,
rnd.OneIn(20)
? 100 + rnd.Uniform(100)
@@ -4007,7 +3584,7 @@ TEST(DBTest, Randomized) {
ASSERT_OK(db_->Put(WriteOptions(), k, v));
} else if (p < 90) { // Delete
k = RandomKey(&rnd, minimum);
k = RandomKey(&rnd);
ASSERT_OK(model.Delete(WriteOptions(), k));
ASSERT_OK(db_->Delete(WriteOptions(), k));
@@ -4017,7 +3594,7 @@ TEST(DBTest, Randomized) {
const int num = rnd.Uniform(8);
for (int i = 0; i < num; i++) {
if (i == 0 || !rnd.OneIn(10)) {
k = RandomKey(&rnd, minimum);
k = RandomKey(&rnd);
} else {
// Periodically re-use the same key from the previous iter, so
// we have multiple entries in the write batch for the same key
@@ -4173,9 +3750,6 @@ TEST(DBTest, PrefixScan) {
snprintf(buf, sizeof(buf), "03______:");
prefix = Slice(buf, 8);
key = Slice(buf, 9);
auto prefix_extractor = NewFixedPrefixTransform(8);
auto memtable_factory =
std::make_shared<PrefixHashRepFactory>(prefix_extractor);
// db configs
env_->count_random_reads_ = true;
@@ -4183,13 +3757,12 @@ TEST(DBTest, PrefixScan) {
options.env = env_;
options.block_cache = NewLRUCache(0); // Prevent cache hits
options.filter_policy = NewBloomFilterPolicy(10);
options.prefix_extractor = prefix_extractor;
options.prefix_extractor = NewFixedPrefixTransform(8);
options.whole_key_filtering = false;
options.disable_auto_compactions = true;
options.max_background_compactions = 2;
options.create_if_missing = true;
options.disable_seek_compaction = true;
options.memtable_factory = memtable_factory;
// prefix specified, with blooms: 2 RAND I/Os
// SeekToFirst
@@ -4243,6 +3816,7 @@ TEST(DBTest, PrefixScan) {
ASSERT_EQ(env_->random_read_counter_.Read(), 11);
Close();
delete options.filter_policy;
delete options.prefix_extractor;
}
std::string MakeKey(unsigned int num) {
@@ -4303,16 +3877,16 @@ void BM_LogAndApply(int iters, int num_base_files) {
buf, iters, us, ((float)us) / iters);
}
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
if (argc > 1 && std::string(argv[1]) == "--benchmark") {
rocksdb::BM_LogAndApply(1000, 1);
rocksdb::BM_LogAndApply(1000, 100);
rocksdb::BM_LogAndApply(1000, 10000);
rocksdb::BM_LogAndApply(100, 100000);
leveldb::BM_LogAndApply(1000, 1);
leveldb::BM_LogAndApply(1000, 100);
leveldb::BM_LogAndApply(1000, 10000);
leveldb::BM_LogAndApply(100, 100000);
return 0;
}
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
+4 -4
View File
@@ -6,9 +6,9 @@
#include "db/dbformat.h"
#include "port/port.h"
#include "util/coding.h"
#include "util/perf_context_imp.h"
#include "include/leveldb/perf_context.h"
namespace rocksdb {
namespace leveldb {
static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
assert(seq <= kMaxSequenceNumber);
@@ -54,7 +54,7 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
// decreasing sequence number
// decreasing type (though sequence# should be enough to disambiguate)
int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
BumpPerfCount(&perf_context.user_key_comparison_count);
perf_context.user_key_comparison_count++;
if (r == 0) {
const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
@@ -139,4 +139,4 @@ LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
end_ = dst;
}
} // namespace rocksdb
} // namespace leveldb
+8 -8
View File
@@ -6,16 +6,16 @@
#define STORAGE_LEVELDB_DB_FORMAT_H_
#include <stdio.h>
#include "rocksdb/comparator.h"
#include "rocksdb/db.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/slice.h"
#include "rocksdb/table_builder.h"
#include "rocksdb/types.h"
#include "leveldb/comparator.h"
#include "leveldb/db.h"
#include "leveldb/filter_policy.h"
#include "leveldb/slice.h"
#include "leveldb/table_builder.h"
#include "leveldb/types.h"
#include "util/coding.h"
#include "util/logging.h"
namespace rocksdb {
namespace leveldb {
class InternalKey;
@@ -223,6 +223,6 @@ inline LookupKey::~LookupKey() {
if (start_ != space_) delete[] start_;
}
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_FORMAT_H_
+3 -3
View File
@@ -6,7 +6,7 @@
#include "util/logging.h"
#include "util/testharness.h"
namespace rocksdb {
namespace leveldb {
static std::string IKey(const std::string& user_key,
uint64_t seq,
@@ -105,8 +105,8 @@ TEST(FormatTest, InternalKeyShortestSuccessor) {
ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
}
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
-188
View File
@@ -1,188 +0,0 @@
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/db.h"
#include "db/db_impl.h"
#include "db/filename.h"
#include "db/version_set.h"
#include "db/write_batch_internal.h"
#include "util/testharness.h"
#include "util/testutil.h"
#include "boost/lexical_cast.hpp"
#include "rocksdb/env.h"
#include <vector>
#include <boost/algorithm/string.hpp>
#include <stdlib.h>
#include <map>
namespace rocksdb {
class DeleteFileTest {
public:
std::string dbname_;
Options options_;
DB* db_;
Env* env_;
int numlevels_;
DeleteFileTest() {
db_ = nullptr;
env_ = Env::Default();
options_.write_buffer_size = 1024*1024*1000;
options_.target_file_size_base = 1024*1024*1000;
options_.max_bytes_for_level_base = 1024*1024*1000;
dbname_ = test::TmpDir() + "/deletefile_test";
DestroyDB(dbname_, options_);
numlevels_ = 7;
ASSERT_OK(ReopenDB(true));
}
Status ReopenDB(bool create) {
delete db_;
if (create) {
DestroyDB(dbname_, options_);
}
db_ = nullptr;
options_.create_if_missing = create;
return DB::Open(options_, dbname_, &db_);
}
void CloseDB() {
delete db_;
}
void AddKeys(int numkeys, int startkey = 0) {
WriteOptions options;
options.sync = false;
ReadOptions roptions;
for (int i = startkey; i < (numkeys + startkey) ; i++) {
std::string temp = boost::lexical_cast<std::string>(i);
Slice key(temp);
Slice value(temp);
ASSERT_OK(db_->Put(options, key, value));
}
}
int numKeysInLevels(
std::vector<LiveFileMetaData> &metadata,
std::vector<int> *keysperlevel = nullptr) {
if (keysperlevel != nullptr) {
keysperlevel->resize(numlevels_);
}
int numKeys = 0;
for (size_t i = 0; i < metadata.size(); i++) {
int startkey = atoi(metadata[i].smallestkey.c_str());
int endkey = atoi(metadata[i].largestkey.c_str());
int numkeysinfile = (endkey - startkey + 1);
numKeys += numkeysinfile;
if (keysperlevel != nullptr) {
(*keysperlevel)[(int)metadata[i].level] += numkeysinfile;
}
fprintf(stderr, "level %d name %s smallest %s largest %s\n",
metadata[i].level, metadata[i].name.c_str(),
metadata[i].smallestkey.c_str(),
metadata[i].largestkey.c_str());
}
return numKeys;
}
void CreateTwoLevels() {
AddKeys(50000, 10000);
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
ASSERT_OK(dbi->TEST_CompactMemTable());
ASSERT_OK(dbi->TEST_WaitForCompactMemTable());
AddKeys(50000, 10000);
ASSERT_OK(dbi->TEST_CompactMemTable());
ASSERT_OK(dbi->TEST_WaitForCompactMemTable());
}
};
TEST(DeleteFileTest, AddKeysAndQueryLevels) {
CreateTwoLevels();
std::vector<LiveFileMetaData> metadata;
std::vector<int> keysinlevel;
db_->GetLiveFilesMetaData(&metadata);
std::string level1file = "";
int level1keycount = 0;
std::string level2file = "";
int level2keycount = 0;
int level1index = 0;
int level2index = 1;
ASSERT_EQ((int)metadata.size(), 2);
if (metadata[0].level == 2) {
level1index = 1;
level2index = 0;
}
level1file = metadata[level1index].name;
int startkey = atoi(metadata[level1index].smallestkey.c_str());
int endkey = atoi(metadata[level1index].largestkey.c_str());
level1keycount = (endkey - startkey + 1);
level2file = metadata[level2index].name;
startkey = atoi(metadata[level2index].smallestkey.c_str());
endkey = atoi(metadata[level2index].largestkey.c_str());
level2keycount = (endkey - startkey + 1);
// COntrolled setup. Levels 1 and 2 should both have 50K files.
// This is a little fragile as it depends on the current
// compaction heuristics.
ASSERT_EQ(level1keycount, 50000);
ASSERT_EQ(level2keycount, 50000);
Status status = db_->DeleteFile("0.sst");
ASSERT_TRUE(status.IsInvalidArgument());
// intermediate level files cannot be deleted.
status = db_->DeleteFile(level1file);
ASSERT_TRUE(status.IsInvalidArgument());
// Lowest level file deletion should succeed.
ASSERT_OK(db_->DeleteFile(level2file));
CloseDB();
}
TEST(DeleteFileTest, DeleteFileWithIterator) {
CreateTwoLevels();
ReadOptions options;
Iterator* it = db_->NewIterator(options);
std::vector<LiveFileMetaData> metadata;
db_->GetLiveFilesMetaData(&metadata);
std::string level2file = "";
ASSERT_EQ((int)metadata.size(), 2);
if (metadata[0].level == 1) {
level2file = metadata[1].name;
} else {
level2file = metadata[0].name;
}
Status status = db_->DeleteFile(level2file);
fprintf(stdout, "Deletion status %s: %s\n",
level2file.c_str(), status.ToString().c_str());
ASSERT_TRUE(status.ok());
it->SeekToFirst();
int numKeysIterated = 0;
while(it->Valid()) {
numKeysIterated++;
it->Next();
}
ASSERT_EQ(numKeysIterated, 50000);
delete it;
CloseDB();
}
} //namespace rocksdb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
}
+9 -10
View File
@@ -7,10 +7,10 @@
#include <ctype.h>
#include <stdio.h>
#include "db/dbformat.h"
#include "rocksdb/env.h"
#include "leveldb/env.h"
#include "util/logging.h"
namespace rocksdb {
namespace leveldb {
// Given a path, flatten the path name by replacing all chars not in
// {[0-9,a-z,A-Z,-,_,.]} with _. And append '\0' at the end.
@@ -46,10 +46,13 @@ extern Status WriteStringToFileSync(Env* env, const Slice& data,
static std::string MakeFileName(const std::string& name, uint64_t number,
const char* suffix) {
char buf[100];
snprintf(buf, sizeof(buf), "/%06llu.%s",
snprintf(buf, sizeof(buf), "%06llu.%s",
static_cast<unsigned long long>(number),
suffix);
return name + buf;
if (name.empty()) {
return buf;
}
return name + "/" + buf;
}
std::string LogFileName(const std::string& name, uint64_t number) {
@@ -62,7 +65,7 @@ std::string ArchivalDirectory(const std::string& dbname) {
}
std::string ArchivedLogFileName(const std::string& name, uint64_t number) {
assert(number > 0);
return MakeFileName(name + "/" + ARCHIVAL_DIR, number, "log");
return MakeFileName(name + "/archive", number, "log");
}
std::string TableFileName(const std::string& name, uint64_t number) {
@@ -130,14 +133,10 @@ std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
// dbname/MANIFEST-[0-9]+
// dbname/[0-9]+.(log|sst)
// dbname/METADB-[0-9]+
// Disregards / at the beginning
bool ParseFileName(const std::string& fname,
uint64_t* number,
FileType* type) {
Slice rest(fname);
if (fname.length() > 1 && fname[0] == '/') {
rest.remove_prefix(1);
}
if (rest == "CURRENT") {
*number = 0;
*type = kCurrentFile;
@@ -218,4 +217,4 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
return s;
}
} // namespace rocksdb
} // namespace leveldb
+4 -4
View File
@@ -9,11 +9,11 @@
#include <stdint.h>
#include <string>
#include "rocksdb/slice.h"
#include "rocksdb/status.h"
#include "leveldb/slice.h"
#include "leveldb/status.h"
#include "port/port.h"
namespace rocksdb {
namespace leveldb {
class Env;
@@ -92,6 +92,6 @@ extern Status SetCurrentFile(Env* env, const std::string& dbname,
uint64_t descriptor_number);
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_FILENAME_H_
+3 -3
View File
@@ -9,7 +9,7 @@
#include "util/logging.h"
#include "util/testharness.h"
namespace rocksdb {
namespace leveldb {
class FileNameTest { };
@@ -128,8 +128,8 @@ TEST(FileNameTest, Construction) {
ASSERT_EQ(kMetaDatabase, type);
}
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
+2 -2
View File
@@ -8,7 +8,7 @@
#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
namespace rocksdb {
namespace leveldb {
namespace log {
enum RecordType {
@@ -30,6 +30,6 @@ static const unsigned int kBlockSize = 32768;
static const int kHeaderSize = 4 + 1 + 2;
} // namespace log
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_
+3 -3
View File
@@ -5,11 +5,11 @@
#include "db/log_reader.h"
#include <stdio.h>
#include "rocksdb/env.h"
#include "leveldb/env.h"
#include "util/coding.h"
#include "util/crc32c.h"
namespace rocksdb {
namespace leveldb {
namespace log {
Reader::Reporter::~Reporter() {
@@ -256,4 +256,4 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
}
} // namespace log
} // namespace rocksdb
} // namespace leveldb
+4 -4
View File
@@ -9,10 +9,10 @@
#include <stdint.h>
#include "db/log_format.h"
#include "rocksdb/slice.h"
#include "rocksdb/status.h"
#include "leveldb/slice.h"
#include "leveldb/status.h"
namespace rocksdb {
namespace leveldb {
class SequentialFile;
using std::unique_ptr;
@@ -118,6 +118,6 @@ class Reader {
};
} // namespace log
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_LOG_READER_H_
+4 -4
View File
@@ -4,13 +4,13 @@
#include "db/log_reader.h"
#include "db/log_writer.h"
#include "rocksdb/env.h"
#include "leveldb/env.h"
#include "util/coding.h"
#include "util/crc32c.h"
#include "util/random.h"
#include "util/testharness.h"
namespace rocksdb {
namespace leveldb {
namespace log {
// Construct a string of the specified length made out of the supplied
@@ -516,8 +516,8 @@ TEST(LogTest, ReadPastEnd) {
}
} // namespace log
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
+3 -3
View File
@@ -5,11 +5,11 @@
#include "db/log_writer.h"
#include <stdint.h>
#include "rocksdb/env.h"
#include "leveldb/env.h"
#include "util/coding.h"
#include "util/crc32c.h"
namespace rocksdb {
namespace leveldb {
namespace log {
Writer::Writer(unique_ptr<WritableFile>&& dest)
@@ -100,4 +100,4 @@ Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
}
} // namespace log
} // namespace rocksdb
} // namespace leveldb
+4 -4
View File
@@ -8,10 +8,10 @@
#include <memory>
#include <stdint.h>
#include "db/log_format.h"
#include "rocksdb/slice.h"
#include "rocksdb/status.h"
#include "leveldb/slice.h"
#include "leveldb/status.h"
namespace rocksdb {
namespace leveldb {
class WritableFile;
@@ -49,6 +49,6 @@ class Writer {
};
} // namespace log
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_
+17 -26
View File
@@ -7,14 +7,20 @@
#include <memory>
#include "db/dbformat.h"
#include "rocksdb/comparator.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "rocksdb/merge_operator.h"
#include "leveldb/comparator.h"
#include "leveldb/env.h"
#include "leveldb/iterator.h"
#include "leveldb/merge_operator.h"
#include "util/coding.h"
#include "util/murmurhash.h"
namespace rocksdb {
namespace leveldb {
static Slice GetLengthPrefixedSlice(const char* data) {
uint32_t len;
const char* p = data;
p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted
return Slice(p, len);
}
MemTable::MemTable(const InternalKeyComparator& cmp,
std::shared_ptr<MemTableRepFactory> table_factory,
@@ -29,7 +35,6 @@ MemTable::MemTable(const InternalKeyComparator& cmp,
file_number_(0),
edit_(numlevel),
first_seqno_(0),
mem_next_logfile_number_(0),
mem_logfile_number_(0) { }
MemTable::~MemTable() {
@@ -37,8 +42,7 @@ MemTable::~MemTable() {
}
size_t MemTable::ApproximateMemoryUsage() {
return arena_impl_.ApproximateMemoryUsage() +
table_->ApproximateMemoryUsage();
return arena_impl_.ApproximateMemoryUsage();
}
int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
@@ -49,11 +53,6 @@ int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
return comparator.Compare(a, b);
}
Slice MemTableRep::UserKey(const char* key) const {
Slice slice = GetLengthPrefixedSlice(key);
return Slice(slice.data(), slice.size() - 8);
}
// Encode a suitable internal key target for "target" and return it.
// Uses *scratch as scratch space, and the returned pointer will point
// into this scratch space.
@@ -69,9 +68,6 @@ class MemTableIterator: public Iterator {
explicit MemTableIterator(MemTableRep* table)
: iter_(table->GetIterator()) { }
MemTableIterator(MemTableRep* table, const Slice* prefix)
: iter_(table->GetPrefixIterator(*prefix)) { }
virtual bool Valid() const { return iter_->Valid(); }
virtual void Seek(const Slice& k) { iter_->Seek(EncodeKey(&tmp_, k)); }
virtual void SeekToFirst() { iter_->SeekToFirst(); }
@@ -97,12 +93,8 @@ class MemTableIterator: public Iterator {
void operator=(const MemTableIterator&);
};
Iterator* MemTable::NewIterator(const Slice* prefix) {
if (prefix) {
return new MemTableIterator(table_.get(), prefix);
} else {
return new MemTableIterator(table_.get());
}
Iterator* MemTable::NewIterator() {
return new MemTableIterator(table_.get());
}
void MemTable::Add(SequenceNumber s, ValueType type,
@@ -140,8 +132,7 @@ void MemTable::Add(SequenceNumber s, ValueType type,
bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
std::deque<std::string>* operands, const Options& options) {
Slice memkey = key.memtable_key();
std::shared_ptr<MemTableRep::Iterator> iter(
table_->GetIterator(key.user_key()));
std::shared_ptr<MemTableRep::Iterator> iter(table_.get()->GetIterator());
iter->Seek(memkey.data());
// It is the caller's responsibility to allocate/delete operands list
@@ -238,4 +229,4 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
return false;
}
} // namespace rocksdb
} // namespace leveldb
+5 -24
View File
@@ -11,11 +11,11 @@
#include "db/dbformat.h"
#include "db/skiplist.h"
#include "db/version_set.h"
#include "rocksdb/db.h"
#include "rocksdb/memtablerep.h"
#include "leveldb/db.h"
#include "leveldb/memtablerep.h"
#include "util/arena_impl.h"
namespace rocksdb {
namespace leveldb {
class Mutex;
class MemTableIterator;
@@ -61,11 +61,7 @@ class MemTable {
// while the returned iterator is live. The keys returned by this
// iterator are internal keys encoded by AppendInternalKey in the
// db/dbformat.{h,cc} module.
//
// If a prefix is supplied, it is passed to the underlying MemTableRep as a
// hint that the iterator only need to support access to keys with that
// prefix.
Iterator* NewIterator(const Slice* prefix = nullptr);
Iterator* NewIterator();
// Add an entry into memtable that maps key to value at the
// specified sequence number and with the specified type.
@@ -92,14 +88,6 @@ class MemTable {
// into the memtable
SequenceNumber GetFirstSequenceNumber() { return first_seqno_; }
// Returns the next active logfile number when this memtable is about to
// be flushed to storage
uint64_t GetNextLogNumber() { return mem_next_logfile_number_; }
// Sets the next active logfile number when this memtable is about to
// be flushed to storage
void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
// Returns the logfile number that can be safely deleted when this
// memstore is flushed to storage
uint64_t GetLogNumber() { return mem_logfile_number_; }
@@ -108,9 +96,6 @@ class MemTable {
// memstore is flushed to storage
void SetLogNumber(uint64_t num) { mem_logfile_number_ = num; }
// Notify the underlying storage that no more items will be added
void MarkImmutable() { table_->MarkReadOnly(); }
private:
~MemTable(); // Private since only Unref() should be used to delete it
friend class MemTableIterator;
@@ -135,10 +120,6 @@ class MemTable {
SequenceNumber first_seqno_;
// The log files earlier than this number can be deleted.
uint64_t mem_next_logfile_number_;
// The log file that backs this memtable (to be deleted when
// memtable flush is done)
uint64_t mem_logfile_number_;
// No copying allowed
@@ -146,6 +127,6 @@ class MemTable {
void operator=(const MemTable&);
};
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_MEMTABLE_H_
+6 -9
View File
@@ -4,13 +4,13 @@
#include "db/memtablelist.h"
#include <string>
#include "rocksdb/db.h"
#include "leveldb/db.h"
#include "db/memtable.h"
#include "rocksdb/env.h"
#include "rocksdb/iterator.h"
#include "leveldb/env.h"
#include "leveldb/iterator.h"
#include "util/coding.h"
namespace rocksdb {
namespace leveldb {
class InternalKeyComparator;
class Mutex;
@@ -42,8 +42,7 @@ int MemTableList::size() {
// Returns true if there is at least one memtable on which flush has
// not yet started.
bool MemTableList::IsFlushPending(int min_write_buffer_number_to_merge) {
if ((flush_requested_ && num_flush_not_started_ >= 1) ||
(num_flush_not_started_ >= min_write_buffer_number_to_merge)) {
if (num_flush_not_started_ >= min_write_buffer_number_to_merge) {
assert(imm_flush_needed.NoBarrier_Load() != nullptr);
return true;
}
@@ -64,7 +63,6 @@ void MemTableList::PickMemtablesToFlush(std::vector<MemTable*>* ret) {
ret->push_back(m);
}
}
flush_requested_ = false; // start-flush request is complete
}
// Record a successful flush in the manifest file
@@ -174,7 +172,6 @@ void MemTableList::Add(MemTable* m) {
assert(size_ >= num_flush_not_started_);
size_++;
memlist_.push_front(m);
m->MarkImmutable();
num_flush_not_started_++;
if (num_flush_not_started_ == 1) {
imm_flush_needed.Release_Store((void *)1);
@@ -210,4 +207,4 @@ void MemTableList::GetMemTables(std::vector<MemTable*>* output) {
}
}
} // namespace rocksdb
} // namespace leveldb
+4 -11
View File
@@ -6,12 +6,12 @@
#include <string>
#include <list>
#include <deque>
#include "rocksdb/db.h"
#include "leveldb/db.h"
#include "db/dbformat.h"
#include "db/skiplist.h"
#include "memtable.h"
namespace rocksdb {
namespace leveldb {
class InternalKeyComparator;
class Mutex;
@@ -29,8 +29,7 @@ class MemTableList {
public:
// A list of memtables.
MemTableList() : size_(0), num_flush_not_started_(0),
commit_in_progress_(false),
flush_requested_(false) {
commit_in_progress_(false) {
imm_flush_needed.Release_Store(nullptr);
}
~MemTableList() {};
@@ -77,9 +76,6 @@ class MemTableList {
// Returns the list of underlying memtables.
void GetMemTables(std::vector<MemTable*>* list);
// Request a flush of all existing memtables to storage
void FlushRequested() { flush_requested_ = true; }
// Copying allowed
// MemTableList(const MemTableList&);
// void operator=(const MemTableList&);
@@ -94,11 +90,8 @@ class MemTableList {
// committing in progress
bool commit_in_progress_;
// Requested a flush of all memtables to storage
bool flush_requested_;
};
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_MEMTABLELIST_H_
+5 -5
View File
@@ -1,12 +1,12 @@
#include "merge_helper.h"
#include "db/dbformat.h"
#include "rocksdb/comparator.h"
#include "rocksdb/db.h"
#include "rocksdb/merge_operator.h"
#include "leveldb/comparator.h"
#include "leveldb/db.h"
#include "leveldb/merge_operator.h"
#include <string>
#include <stdio.h>
namespace rocksdb {
namespace leveldb {
// PRE: iter points to the first merge type entry
// POST: iter points to the first entry beyond the merge process (or the end)
@@ -189,4 +189,4 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
}
}
} // namespace rocksdb
} // namespace leveldb
+4 -4
View File
@@ -2,12 +2,12 @@
#define MERGE_HELPER_H
#include "db/dbformat.h"
#include "rocksdb/slice.h"
#include "rocksdb/statistics.h"
#include "leveldb/slice.h"
#include "leveldb/statistics.h"
#include <string>
#include <deque>
namespace rocksdb {
namespace leveldb {
class Comparator;
class Iterator;
@@ -93,6 +93,6 @@ class MergeHelper {
bool success_;
};
} // namespace rocksdb
} // namespace leveldb
#endif
+3 -3
View File
@@ -5,9 +5,9 @@
* Copyright 2013 Facebook
*/
#include "rocksdb/merge_operator.h"
#include "leveldb/merge_operator.h"
namespace rocksdb {
namespace leveldb {
// Given a "real" merge from the library, call the user's
// associative merge function one-by-one on each of the operands.
@@ -48,4 +48,4 @@ bool AssociativeMergeOperator::PartialMerge(
return Merge(key, &left_operand, right_operand, new_value, logger);
}
} // namespace rocksdb
} // namespace leveldb
+6 -6
View File
@@ -2,11 +2,11 @@
#include <memory>
#include <iostream>
#include "rocksdb/cache.h"
#include "rocksdb/comparator.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "rocksdb/merge_operator.h"
#include "leveldb/cache.h"
#include "leveldb/comparator.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "leveldb/merge_operator.h"
#include "db/dbformat.h"
#include "db/db_impl.h"
#include "utilities/merge_operators.h"
@@ -14,7 +14,7 @@
#include "utilities/utility_db.h"
using namespace std;
using namespace rocksdb;
using namespace leveldb;
std::shared_ptr<DB> OpenDb(const string& dbname, const bool ttl = false) {
+12
View File
@@ -0,0 +1,12 @@
#include "include/leveldb/perf_context.h"
namespace leveldb {
void PerfContext::Reset() {
user_key_comparison_count = 0;
}
__thread PerfContext perf_context;
}
+29 -268
View File
@@ -1,52 +1,20 @@
#include <algorithm>
#include <iostream>
#include <vector>
#include "/usr/include/valgrind/callgrind.h"
#include "rocksdb/db.h"
#include "rocksdb/perf_context.h"
#include "util/histogram.h"
#include "util/stop_watch.h"
#include "leveldb/db.h"
#include "leveldb/perf_context.h"
#include "util/testharness.h"
bool FLAGS_random_key = false;
bool FLAGS_use_set_based_memetable = false;
int FLAGS_total_keys = 100;
int FLAGS_write_buffer_size = 1000000000;
int FLAGS_max_write_buffer_number = 8;
int FLAGS_min_write_buffer_number_to_merge = 7;
namespace leveldb {
// Path to the database on file system
const std::string kDbName = rocksdb::test::TmpDir() + "/perf_context_test";
void SeekToFirst(rocksdb::Iterator* iter) {
// std::cout << "Press a key to continue:";
// std::string s;
// std::cin >> s;
iter->SeekToFirst();
// std::cout << "Press a key to continue:";
// std::string s2;
// std::cin >> s2;
}
namespace rocksdb {
const std::string kDbName = test::TmpDir() + "/perf_context_test";
std::shared_ptr<DB> OpenDb() {
DB* db;
Options options;
options.create_if_missing = true;
options.write_buffer_size = FLAGS_write_buffer_size;
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
options.min_write_buffer_number_to_merge =
FLAGS_min_write_buffer_number_to_merge;
if (FLAGS_use_set_based_memetable) {
auto prefix_extractor = rocksdb::NewFixedPrefixTransform(0);
options.memtable_factory =
std::make_shared<rocksdb::PrefixHashRepFactory>(prefix_extractor);
}
options.write_buffer_size = 1000000000; // give it a big memtable
Status s = DB::Open(options, kDbName, &db);
ASSERT_OK(s);
return std::shared_ptr<DB>(db);
@@ -54,125 +22,10 @@ std::shared_ptr<DB> OpenDb() {
class PerfContextTest { };
TEST(PerfContextTest, SeekIntoDeletion) {
DestroyDB(kDbName, Options());
auto db = OpenDb();
WriteOptions write_options;
ReadOptions read_options;
int kTotalKeys = 100;
for (int i = 0; i < FLAGS_total_keys; ++i) {
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
TEST(PerfContextTest, KeyComparisonCount) {
db->Put(write_options, key, value);
}
for (int i = 0; i < FLAGS_total_keys -1 ; ++i) {
std::string key = "k" + std::to_string(i);
db->Delete(write_options, key);
}
HistogramImpl hist_get;
HistogramImpl hist_get_time;
for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
std::string key = "k" + std::to_string(i);
std::string value;
perf_context.Reset();
StopWatchNano timer(Env::Default(), true);
auto status = db->Get(read_options, key, &value);
auto elapsed_nanos = timer.ElapsedNanos();
ASSERT_TRUE(status.IsNotFound());
hist_get.Add(perf_context.user_key_comparison_count);
hist_get_time.Add(elapsed_nanos);
}
std::cout << "Get uesr key comparison: \n" << hist_get.ToString()
<< "Get time: \n" << hist_get_time.ToString();
HistogramImpl hist_seek_to_first;
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
perf_context.Reset();
StopWatchNano timer(Env::Default(), true);
//CALLGRIND_ZERO_STATS;
SeekToFirst(iter.get());
//iter->SeekToFirst();
//CALLGRIND_DUMP_STATS;
hist_seek_to_first.Add(perf_context.user_key_comparison_count);
auto elapsed_nanos = timer.ElapsedNanos();
std::cout << "SeekToFirst uesr key comparison: \n" << hist_seek_to_first.ToString()
<< "ikey skipped: " << perf_context.internal_key_skipped_count << "\n"
<< "idelete skipped: " << perf_context.internal_delete_skipped_count << "\n"
<< "elapsed: " << elapsed_nanos << "\n";
HistogramImpl hist_seek;
for (int i = 0; i < FLAGS_total_keys; ++i) {
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
std::string key = "k" + std::to_string(i);
perf_context.Reset();
StopWatchNano timer(Env::Default(), true);
iter->Seek(key);
auto elapsed_nanos = timer.ElapsedNanos();
hist_seek.Add(perf_context.user_key_comparison_count);
std::cout << "seek cmp: " << perf_context.user_key_comparison_count
<< " ikey skipped " << perf_context.internal_key_skipped_count
<< " idelete skipped " << perf_context.internal_delete_skipped_count
<< " elapsed: " << elapsed_nanos << "ns\n";
perf_context.Reset();
ASSERT_TRUE(iter->Valid());
StopWatchNano timer2(Env::Default(), true);
iter->Next();
auto elapsed_nanos2 = timer2.ElapsedNanos();
std::cout << "next cmp: " << perf_context.user_key_comparison_count
<< "elapsed: " << elapsed_nanos2 << "ns\n";
}
std::cout << "Seek uesr key comparison: \n" << hist_seek.ToString();
}
TEST(PerfContextTest, StopWatchNanoOverhead) {
// profile the timer cost by itself!
const int kTotalIterations = 1000000;
std::vector<uint64_t> timings(kTotalIterations);
StopWatchNano timer(Env::Default(), true);
for (auto& timing : timings) {
timing = timer.ElapsedNanos(true /* reset */);
}
HistogramImpl histogram;
for (const auto timing : timings) {
histogram.Add(timing);
}
std::cout << histogram.ToString();
}
TEST(PerfContextTest, StopWatchOverhead) {
// profile the timer cost by itself!
const int kTotalIterations = 1000000;
std::vector<uint64_t> timings(kTotalIterations);
StopWatch timer(Env::Default());
for (auto& timing : timings) {
timing = timer.ElapsedMicros();
}
HistogramImpl histogram;
uint64_t prev_timing = 0;
for (const auto timing : timings) {
histogram.Add(timing - prev_timing);
prev_timing = timing;
}
std::cout << histogram.ToString();
}
void ProfileKeyComparison() {
DestroyDB(kDbName, Options()); // Start this test with a fresh DB
auto db = OpenDb();
@@ -180,141 +33,49 @@ void ProfileKeyComparison() {
WriteOptions write_options;
ReadOptions read_options;
HistogramImpl hist_put;
HistogramImpl hist_get;
uint64_t total_user_key_comparison_get = 0;
uint64_t total_user_key_comparison_put = 0;
uint64_t max_user_key_comparison_get = 0;
std::cout << "Inserting " << FLAGS_total_keys << " key/value pairs\n...\n";
std::cout << "Inserting " << kTotalKeys << " key/value pairs\n...\n";
std::vector<int> keys;
for (int i = 0; i < FLAGS_total_keys; ++i) {
keys.push_back(i);
}
if (FLAGS_random_key) {
std::random_shuffle(keys.begin(), keys.end());
}
for (const int i : keys) {
for (int i = 0; i < kTotalKeys; ++i) {
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
perf_context.Reset();
db->Put(write_options, key, value);
hist_put.Add(perf_context.user_key_comparison_count);
total_user_key_comparison_put += perf_context.user_key_comparison_count;
perf_context.Reset();
db->Get(read_options, key, &value);
hist_get.Add(perf_context.user_key_comparison_count);
total_user_key_comparison_get += perf_context.user_key_comparison_count;
max_user_key_comparison_get =
std::max(max_user_key_comparison_get,
perf_context.user_key_comparison_count);
}
std::cout << "Put uesr key comparison: \n" << hist_put.ToString()
<< "Get uesr key comparison: \n" << hist_get.ToString();
std::cout << "total user key comparison get: "
<< total_user_key_comparison_get << "\n"
<< "total user key comparison put: "
<< total_user_key_comparison_put << "\n"
<< "max user key comparison get: "
<< max_user_key_comparison_get << "\n"
<< "avg user key comparison get:"
<< total_user_key_comparison_get/kTotalKeys << "\n";
}
TEST(PerfContextTest, KeyComparisonCount) {
SetPerfLevel(kEnableCount);
ProfileKeyComparison();
SetPerfLevel(kDisable);
ProfileKeyComparison();
SetPerfLevel(kEnableTime);
ProfileKeyComparison();
}
// make perf_context_test
// export LEVELDB_TESTS=PerfContextTest.SeekKeyComparison
// For one memtable:
// ./perf_context_test --write_buffer_size=500000 --total_keys=10000
// For two memtables:
// ./perf_context_test --write_buffer_size=250000 --total_keys=10000
// Specify --random_key=1 to shuffle the key before insertion
// Results show that, for sequential insertion, worst-case Seek Key comparison
// is close to the total number of keys (linear), when there is only one
// memtable. When there are two memtables, even the avg Seek Key comparison
// starts to become linear to the input size.
TEST(PerfContextTest, SeekKeyComparison) {
DestroyDB(kDbName, Options());
auto db = OpenDb();
WriteOptions write_options;
ReadOptions read_options;
std::cout << "Inserting " << FLAGS_total_keys << " key/value pairs\n...\n";
std::vector<int> keys;
for (int i = 0; i < FLAGS_total_keys; ++i) {
keys.push_back(i);
}
if (FLAGS_random_key) {
std::random_shuffle(keys.begin(), keys.end());
}
for (const int i : keys) {
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
db->Put(write_options, key, value);
}
HistogramImpl hist_seek;
HistogramImpl hist_next;
for (int i = 0; i < FLAGS_total_keys; ++i) {
std::string key = "k" + std::to_string(i);
std::string value = "v" + std::to_string(i);
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
perf_context.Reset();
iter->Seek(key);
ASSERT_TRUE(iter->Valid());
ASSERT_EQ(iter->value().ToString(), value);
hist_seek.Add(perf_context.user_key_comparison_count);
}
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
for (iter->SeekToFirst(); iter->Valid();) {
perf_context.Reset();
iter->Next();
hist_next.Add(perf_context.user_key_comparison_count);
}
std::cout << "Seek:\n" << hist_seek.ToString()
<< "Next:\n" << hist_next.ToString();
}
}
int main(int argc, char** argv) {
for (int i = 1; i < argc; i++) {
int n;
char junk;
if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
FLAGS_write_buffer_size = n;
}
if (sscanf(argv[i], "--total_keys=%d%c", &n, &junk) == 1) {
FLAGS_total_keys = n;
}
if (sscanf(argv[i], "--random_key=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
FLAGS_random_key = n;
}
if (sscanf(argv[i], "--use_set_based_memetable=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
FLAGS_use_set_based_memetable = n;
}
if (argc > 1) {
leveldb::kTotalKeys = std::stoi(argv[1]);
}
std::cout << kDbName << "\n";
rocksdb::test::RunAllTests();
leveldb::test::RunAllTests();
return 0;
}
+3 -3
View File
@@ -12,9 +12,9 @@
#ifndef STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
#define STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
#include "rocksdb/iterator.h"
#include "leveldb/iterator.h"
namespace rocksdb {
namespace leveldb {
class PrefixFilterIterator : public Iterator {
private:
@@ -71,6 +71,6 @@ class PrefixFilterIterator : public Iterator {
}
};
} // namespace rocksdb
} // namespace leveldb
#endif
+5 -5
View File
@@ -34,11 +34,11 @@
#include "db/table_cache.h"
#include "db/version_edit.h"
#include "db/write_batch_internal.h"
#include "rocksdb/comparator.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "leveldb/comparator.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
namespace rocksdb {
namespace leveldb {
namespace {
@@ -383,4 +383,4 @@ Status RepairDB(const std::string& dbname, const Options& options) {
return repairer.Run();
}
} // namespace rocksdb
} // namespace leveldb
+2 -12
View File
@@ -33,7 +33,7 @@
#include "port/port.h"
#include "util/random.h"
namespace rocksdb {
namespace leveldb {
template<typename Key, class Comparator>
class SkipList {
@@ -107,7 +107,6 @@ class SkipList {
// Used for optimizing sequential insert patterns
Node* prev_[kMaxHeight];
int prev_height_;
inline int GetMaxHeight() const {
return static_cast<int>(
@@ -268,10 +267,6 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
Node* x = prev[0];
Node* next = x->Next(0);
if ((x == head_) || KeyIsAfterNode(key, x)) {
// Adjust all relevant insertion points to the previous entry
for (int i = 1; i < prev_height_; i++) {
prev[i] = x;
}
return next;
}
}
@@ -280,9 +275,6 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
int level = GetMaxHeight() - 1;
while (true) {
Node* next = x->Next(level);
// Make sure the lists are sorted.
// If x points to head_ or next points nullptr, it is trivially satisfied.
assert((x == head_) || (next == nullptr) || KeyIsAfterNode(next->key, x));
if (KeyIsAfterNode(key, next)) {
// Keep searching in this list
x = next;
@@ -345,7 +337,6 @@ SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
arena_(arena),
head_(NewNode(0 /* any key will do */, kMaxHeight)),
max_height_(reinterpret_cast<void*>(1)),
prev_height_(1),
rnd_(0xdeadbeef) {
for (int i = 0; i < kMaxHeight; i++) {
head_->SetNext(i, nullptr);
@@ -387,7 +378,6 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
prev_[i]->SetNext(i, x);
}
prev_[0] = x;
prev_height_ = height;
}
template<typename Key, class Comparator>
@@ -400,6 +390,6 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
}
}
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_
+4 -4
View File
@@ -4,13 +4,13 @@
#include "db/skiplist.h"
#include <set>
#include "rocksdb/env.h"
#include "leveldb/env.h"
#include "util/arena_impl.h"
#include "util/hash.h"
#include "util/random.h"
#include "util/testharness.h"
namespace rocksdb {
namespace leveldb {
typedef uint64_t Key;
@@ -371,8 +371,8 @@ TEST(SkipTest, Concurrent3) { RunConcurrent(3); }
TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
+33 -30
View File
@@ -1,9 +1,14 @@
#include "rocksdb/memtablerep.h"
#ifndef STORAGE_LEVELDB_DB_SKIPLISTREP_H_
#define STORAGE_LEVELDB_DB_SKIPLISTREP_H_
#include "leveldb/memtablerep.h"
#include "db/memtable.h"
#include "db/skiplist.h"
namespace rocksdb {
namespace {
namespace leveldb {
class Arena;
class SkipListRep : public MemTableRep {
SkipList<const char*, MemTableRep::KeyComparator&> skip_list_;
public:
@@ -13,21 +18,16 @@ public:
// Insert key into the list.
// REQUIRES: nothing that compares equal to key is currently in the list.
virtual void Insert(const char* key) override {
virtual void Insert(const char* key) {
skip_list_.Insert(key);
}
// Returns true iff an entry that compares equal to key is in the list.
virtual bool Contains(const char* key) const override {
virtual bool Contains(const char* key) const {
return skip_list_.Contains(key);
}
virtual size_t ApproximateMemoryUsage() override {
// All memory is allocated through arena; nothing to report here
return 0;
}
virtual ~SkipListRep() override { }
virtual ~SkipListRep() { }
// Iteration over the contents of a skip list
class Iterator : public MemTableRep::Iterator {
@@ -39,61 +39,64 @@ public:
const SkipList<const char*, MemTableRep::KeyComparator&>* list
) : iter_(list) { }
virtual ~Iterator() override { }
virtual ~Iterator() { }
// Returns true iff the iterator is positioned at a valid node.
virtual bool Valid() const override {
virtual bool Valid() const {
return iter_.Valid();
}
// Returns the key at the current position.
// REQUIRES: Valid()
virtual const char* key() const override {
virtual const char* key() const {
return iter_.key();
}
// Advances to the next position.
// REQUIRES: Valid()
virtual void Next() override {
virtual void Next() {
iter_.Next();
}
// Advances to the previous position.
// REQUIRES: Valid()
virtual void Prev() override {
virtual void Prev() {
iter_.Prev();
}
// Advance to the first entry with a key >= target
virtual void Seek(const char* target) override {
virtual void Seek(const char* target) {
iter_.Seek(target);
}
// Position at the first entry in list.
// Final state of iterator is Valid() iff list is not empty.
virtual void SeekToFirst() override {
virtual void SeekToFirst() {
iter_.SeekToFirst();
}
// Position at the last entry in list.
// Final state of iterator is Valid() iff list is not empty.
virtual void SeekToLast() override {
virtual void SeekToLast() {
iter_.SeekToLast();
}
};
// Unhide default implementations of GetIterator
using MemTableRep::GetIterator;
virtual std::shared_ptr<MemTableRep::Iterator> GetIterator() override {
return std::make_shared<SkipListRep::Iterator>(&skip_list_);
virtual std::shared_ptr<MemTableRep::Iterator> GetIterator() {
return std::shared_ptr<MemTableRep::Iterator>(
new SkipListRep::Iterator(&skip_list_)
);
}
};
class SkipListFactory : public MemTableRepFactory {
public:
virtual std::shared_ptr<MemTableRep> CreateMemTableRep (
MemTableRep::KeyComparator& compare, Arena* arena) {
return std::shared_ptr<MemTableRep>(new SkipListRep(compare, arena));
}
};
}
std::shared_ptr<MemTableRep> SkipListFactory::CreateMemTableRep (
MemTableRep::KeyComparator& compare, Arena* arena) {
return std::shared_ptr<MemTableRep>(new SkipListRep(compare, arena));
}
} // namespace rocksdb
#endif // STORAGE_LEVELDB_DB_SKIPLISTREP_H_
+3 -3
View File
@@ -5,9 +5,9 @@
#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
#define STORAGE_LEVELDB_DB_SNAPSHOT_H_
#include "rocksdb/db.h"
#include "leveldb/db.h"
namespace rocksdb {
namespace leveldb {
class SnapshotList;
@@ -80,6 +80,6 @@ class SnapshotList {
SnapshotImpl list_;
};
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_
+10 -29
View File
@@ -6,12 +6,12 @@
#include "db/filename.h"
#include "rocksdb/statistics.h"
#include "leveldb/statistics.h"
#include "table/table.h"
#include "util/coding.h"
#include "util/stop_watch.h"
namespace rocksdb {
namespace leveldb {
static void DeleteEntry(const Slice& key, void* value) {
Table* table = reinterpret_cast<Table*>(value);
@@ -48,7 +48,7 @@ Status TableCache::FindTable(const EnvOptions& toptions,
*handle = cache_->Lookup(key);
if (*handle == nullptr) {
if (no_io) { // Dont do IO and return a not-found status
return Status::Incomplete("Table not found in table_cache, no_io is set");
return Status::NotFound("Table not found in table_cache, no_io is set");
}
if (table_io != nullptr) {
*table_io = true; // we had to do IO from storage
@@ -90,8 +90,7 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
}
Cache::Handle* handle = nullptr;
Status s = FindTable(toptions, file_number, file_size, &handle,
nullptr, options.read_tier == kBlockCacheTier);
Status s = FindTable(toptions, file_number, file_size, &handle);
if (!s.ok()) {
return NewErrorIterator(s);
}
@@ -118,17 +117,17 @@ Status TableCache::Get(const ReadOptions& options,
void* arg,
bool (*saver)(void*, const Slice&, const Slice&, bool),
bool* table_io,
void (*mark_key_may_exist)(void*)) {
void (*mark_key_may_exist)(void*),
const bool no_io) {
Cache::Handle* handle = nullptr;
Status s = FindTable(storage_options_, file_number, file_size,
&handle, table_io,
options.read_tier == kBlockCacheTier);
&handle, table_io, no_io);
if (s.ok()) {
Table* t =
reinterpret_cast<Table*>(cache_->Value(handle));
s = t->InternalGet(options, k, arg, saver, mark_key_may_exist);
s = t->InternalGet(options, k, arg, saver, mark_key_may_exist, no_io);
cache_->Release(handle);
} else if (options.read_tier && s.IsIncomplete()) {
} else if (no_io && s.IsNotFound()) {
// Couldnt find Table in cache but treat as kFound if no_io set
(*mark_key_may_exist)(arg);
return Status::OK();
@@ -136,28 +135,10 @@ Status TableCache::Get(const ReadOptions& options,
return s;
}
bool TableCache::PrefixMayMatch(const ReadOptions& options,
uint64_t file_number,
uint64_t file_size,
const Slice& internal_prefix,
bool* table_io) {
Cache::Handle* handle = nullptr;
Status s = FindTable(storage_options_, file_number,
file_size, &handle, table_io);
bool may_match = true;
if (s.ok()) {
Table* t =
reinterpret_cast<Table*>(cache_->Value(handle));
may_match = t->PrefixMayMatch(internal_prefix);
cache_->Release(handle);
}
return may_match;
}
void TableCache::Evict(uint64_t file_number) {
char buf[sizeof(file_number)];
EncodeFixed64(buf, file_number);
cache_->Erase(Slice(buf, sizeof(buf)));
}
} // namespace rocksdb
} // namespace leveldb
+6 -11
View File
@@ -10,12 +10,12 @@
#include <string>
#include <stdint.h>
#include "db/dbformat.h"
#include "rocksdb/env.h"
#include "rocksdb/cache.h"
#include "leveldb/env.h"
#include "leveldb/cache.h"
#include "port/port.h"
#include "table/table.h"
namespace rocksdb {
namespace leveldb {
class Env;
@@ -49,13 +49,8 @@ class TableCache {
void* arg,
bool (*handle_result)(void*, const Slice&, const Slice&, bool),
bool* table_io,
void (*mark_key_may_exist)(void*) = nullptr);
// Determine whether the table may contain the specified prefix. If
// the table index of blooms are not in memory, this may cause an I/O
bool PrefixMayMatch(const ReadOptions& options, uint64_t file_number,
uint64_t file_size, const Slice& internal_prefix,
bool* table_io);
void (*mark_key_may_exist)(void*) = nullptr,
const bool no_io = false);
// Evict any entry for the specified file number
void Evict(uint64_t file_number);
@@ -72,6 +67,6 @@ class TableCache {
const bool no_io = false);
};
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
+2 -2
View File
@@ -1,7 +1,7 @@
#include "db/transaction_log_impl.h"
#include "db/write_batch_internal.h"
namespace rocksdb {
namespace leveldb {
TransactionLogIteratorImpl::TransactionLogIteratorImpl(
const std::string& dbname,
@@ -168,4 +168,4 @@ Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) {
);
return Status::OK();
}
} // namespace rocksdb
} // namespace leveldb
+7 -12
View File
@@ -4,14 +4,14 @@
#include <vector>
#include "rocksdb/env.h"
#include "rocksdb/options.h"
#include "rocksdb/types.h"
#include "rocksdb/transaction_log.h"
#include "leveldb/env.h"
#include "leveldb/options.h"
#include "leveldb/types.h"
#include "leveldb/transaction_log.h"
#include "db/log_reader.h"
#include "db/filename.h"
namespace rocksdb {
namespace leveldb {
struct LogReporter : public log::Reader::Reporter {
Env* env;
@@ -31,12 +31,7 @@ class LogFileImpl : public LogFile {
sizeFileBytes_(sizeBytes) {
}
std::string PathName() const {
if (type_ == kArchivedLogFile) {
return ArchivedLogFileName("", logNumber_);
}
return LogFileName("", logNumber_);
}
std::string Filename() const { return LogFileName("", logNumber_); }
uint64_t LogNumber() const { return logNumber_; }
@@ -99,5 +94,5 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
+2 -2
View File
@@ -7,7 +7,7 @@
#include "db/version_set.h"
#include "util/coding.h"
namespace rocksdb {
namespace leveldb {
// Tag numbers for serialized VersionEdit. These numbers are written to
// disk and should not be changed.
@@ -293,4 +293,4 @@ std::string VersionEdit::DebugString(bool hex_key) const {
return r;
}
} // namespace rocksdb
} // namespace leveldb
+2 -2
View File
@@ -10,7 +10,7 @@
#include <vector>
#include "db/dbformat.h"
namespace rocksdb {
namespace leveldb {
class VersionSet;
@@ -122,6 +122,6 @@ class VersionEdit {
std::vector< std::pair<int, FileMetaData> > new_files_;
};
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_
+3 -3
View File
@@ -5,7 +5,7 @@
#include "db/version_edit.h"
#include "util/testharness.h"
namespace rocksdb {
namespace leveldb {
static void TestEncodeDecode(const VersionEdit& edit) {
std::string encoded, encoded2;
@@ -41,8 +41,8 @@ TEST(VersionEditTest, EncodeDecode) {
TestEncodeDecode(edit);
}
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
+136 -310
View File
@@ -12,16 +12,16 @@
#include "db/log_writer.h"
#include "db/memtable.h"
#include "db/table_cache.h"
#include "rocksdb/env.h"
#include "rocksdb/merge_operator.h"
#include "rocksdb/table_builder.h"
#include "leveldb/env.h"
#include "leveldb/merge_operator.h"
#include "leveldb/table_builder.h"
#include "table/merger.h"
#include "table/two_level_iterator.h"
#include "util/coding.h"
#include "util/logging.h"
#include "util/stop_watch.h"
namespace rocksdb {
namespace leveldb {
static uint64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
uint64_t sum = 0;
@@ -189,14 +189,7 @@ static Iterator* GetFileIterator(void* arg,
return NewErrorIterator(
Status::Corruption("FileReader invoked with unexpected value"));
} else {
ReadOptions options_copy;
if (options.prefix) {
// suppress prefix filtering since we have already checked the
// filters once at this point
options_copy = options;
options_copy.prefix = nullptr;
}
return cache->NewIterator(options.prefix ? options_copy : options,
return cache->NewIterator(options,
soptions,
DecodeFixed64(file_value.data()),
DecodeFixed64(file_value.data() + 8),
@@ -205,55 +198,22 @@ static Iterator* GetFileIterator(void* arg,
}
}
bool Version::PrefixMayMatch(const ReadOptions& options,
const EnvOptions& soptions,
const Slice& internal_prefix,
Iterator* level_iter) const {
bool may_match = true;
level_iter->Seek(internal_prefix);
if (!level_iter->Valid()) {
// we're past end of level
may_match = false;
} else if (ExtractUserKey(level_iter->key()).starts_with(
ExtractUserKey(internal_prefix))) {
// TODO(tylerharter): do we need this case? Or are we guaranteed
// key() will always be the biggest value for this SST?
may_match = true;
} else {
may_match = vset_->table_cache_->PrefixMayMatch(
options,
DecodeFixed64(level_iter->value().data()),
DecodeFixed64(level_iter->value().data() + 8),
internal_prefix, nullptr);
}
return may_match;
}
Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
const EnvOptions& soptions,
int level) const {
Iterator* level_iter = new LevelFileNumIterator(vset_->icmp_, &files_[level]);
if (options.prefix) {
InternalKey internal_prefix(*options.prefix, 0, kTypeValue);
if (!PrefixMayMatch(options, soptions,
internal_prefix.Encode(), level_iter)) {
delete level_iter;
// nothing in this level can match the prefix
return NewEmptyIterator();
}
}
return NewTwoLevelIterator(level_iter, &GetFileIterator,
vset_->table_cache_, options, soptions);
return NewTwoLevelIterator(
new LevelFileNumIterator(vset_->icmp_, &files_[level]),
&GetFileIterator, vset_->table_cache_, options, soptions);
}
void Version::AddIterators(const ReadOptions& options,
const EnvOptions& soptions,
std::vector<Iterator*>* iters) {
// Merge all level zero files together since they may overlap
for (const FileMetaData* file : files_[0]) {
for (size_t i = 0; i < files_[0].size(); i++) {
iters->push_back(
vset_->table_cache_->NewIterator(
options, soptions, file->number, file->file_size));
options, soptions, files_[0][i]->number, files_[0][i]->file_size));
}
// For levels > 0, we can use a concatenating iterator that sequentially
@@ -397,7 +357,6 @@ static bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b) {
Version::Version(VersionSet* vset, uint64_t version_number)
: vset_(vset), next_(this), prev_(this), refs_(0),
files_(new std::vector<FileMetaData*>[vset->NumberLevels()]),
files_by_size_(vset->NumberLevels()),
next_file_to_compact_by_size_(vset->NumberLevels()),
file_to_compact_(nullptr),
@@ -406,6 +365,7 @@ Version::Version(VersionSet* vset, uint64_t version_number)
compaction_level_(vset->NumberLevels()),
offset_manifest_file_(0),
version_number_(version_number) {
files_ = new std::vector<FileMetaData*>[vset->NumberLevels()];
}
void Version::Get(const ReadOptions& options,
@@ -415,6 +375,7 @@ void Version::Get(const ReadOptions& options,
std::deque<std::string>* operands,
GetStats* stats,
const Options& db_options,
const bool no_io,
bool* value_found) {
Slice ikey = k.internal_key();
Slice user_key = k.user_key();
@@ -424,6 +385,9 @@ void Version::Get(const ReadOptions& options,
auto logger = db_options.info_log;
assert(status->ok() || status->IsMergeInProgress());
if (no_io) {
assert(status->ok());
}
Saver saver;
saver.state = status->ok()? kNotFound : kMerge;
saver.ucmp = ucmp;
@@ -512,7 +476,7 @@ void Version::Get(const ReadOptions& options,
bool tableIO = false;
*status = vset_->table_cache_->Get(options, f->number, f->file_size,
ikey, &saver, SaveValue, &tableIO,
MarkKeyMayExist);
MarkKeyMayExist, no_io);
// TODO: examine the behavior for corrupted key
if (!status->ok()) {
return;
@@ -2159,215 +2123,17 @@ void VersionSet::SizeBeingCompacted(std::vector<uint64_t>& sizes) {
}
}
//
// Look at overall size amplification. If size amplification
// exceeeds the configured value, then do a compaction
// of the candidate files all the way upto the earliest
// base file (overrides configured values of file-size ratios,
// min_merge_width and max_merge_width).
//
Compaction* VersionSet::PickCompactionUniversalSizeAmp(
int level, double score) {
Compaction* VersionSet::PickCompactionUniversal(int level, double score) {
assert (level == 0);
// percentage flexibilty while reducing size amplification
uint64_t ratio = options_->compaction_options_universal.
max_size_amplification_percent;
// The files are sorted from newest first to oldest last.
std::vector<int>& file_by_time = current_->files_by_size_[level];
assert(file_by_time.size() == current_->files_[level].size());
unsigned int candidate_count = 0;
uint64_t candidate_size = 0;
unsigned int start_index = 0;
FileMetaData* f = nullptr;
// Skip files that are already being compacted
for (unsigned int loop = 0; loop < file_by_time.size() - 1; loop++) {
int index = file_by_time[loop];
f = current_->files_[level][index];
if (!f->being_compacted) {
start_index = loop; // Consider this as the first candidate.
break;
}
Log(options_->info_log, "Universal: skipping file %ld[%d] compacted %s",
f->number, loop, " cannot be a candidate to reduce size amp.\n");
f = nullptr;
}
if (f == nullptr) {
return nullptr; // no candidate files
}
Log(options_->info_log, "Universal: First candidate file %ld[%d] %s",
f->number, start_index, " to reduce size amp.\n");
// keep adding up all the remaining files
for (unsigned int loop = start_index; loop < file_by_time.size() - 1;
loop++) {
int index = file_by_time[loop];
f = current_->files_[level][index];
if (f->being_compacted) {
Log(options_->info_log,
"Universal: Possible candidate file %ld[%d] %s.", f->number, loop,
" is already being compacted. No size amp reduction possible.\n");
return nullptr;
}
candidate_size += f->file_size;
candidate_count++;
}
if (candidate_count == 0) {
return nullptr;
}
// size of earliest file
int index = file_by_time[file_by_time.size() - 1];
uint64_t earliest_file_size = current_->files_[level][index]->file_size;
// size amplification = percentage of additional size
if (candidate_size * 100 < ratio * earliest_file_size) {
Log(options_->info_log,
"Universal: size amp not needed. newer-files-total-size %ld "
"earliest-file-size %ld",
candidate_size, earliest_file_size);
return nullptr;
} else {
Log(options_->info_log,
"Universal: size amp needed. newer-files-total-size %ld "
"earliest-file-size %ld",
candidate_size, earliest_file_size);
}
assert(start_index >= 0 && start_index < file_by_time.size() - 1);
// create a compaction request
Compaction* c = new Compaction(level, level, MaxFileSizeForLevel(level),
LLONG_MAX, NumberLevels());
c->score_ = score;
for (unsigned int loop = start_index; loop < file_by_time.size(); loop++) {
int index = file_by_time[loop];
f = current_->files_[level][index];
c->inputs_[0].push_back(f);
Log(options_->info_log,
"Universal: size amp picking file %ld[%d] with size %ld",
f->number, index, f->file_size);
}
return c;
}
//
// Consider compaction files based on their size differences with
// the next file in time order.
//
Compaction* VersionSet::PickCompactionUniversalReadAmp(
int level, double score, unsigned int ratio,
unsigned int max_number_of_files_to_compact) {
// percentage flexibilty while comparing file sizes
uint64_t ratio = options_->compaction_options_universal.size_ratio;
unsigned int min_merge_width =
options_->compaction_options_universal.min_merge_width;
unsigned int max_merge_width =
options_->compaction_options_universal.max_merge_width;
// The files are sorted from newest first to oldest last.
std::vector<int>& file_by_time = current_->files_by_size_[level];
FileMetaData* f = nullptr;
bool done = false;
int start_index = 0;
unsigned int candidate_count;
assert(file_by_time.size() == current_->files_[level].size());
unsigned int max_files_to_compact = std::min(max_merge_width,
max_number_of_files_to_compact);
min_merge_width = std::max(min_merge_width, 2U);
// Considers a candidate file only if it is smaller than the
// total size accumulated so far.
for (unsigned int loop = 0; loop < file_by_time.size(); loop++) {
candidate_count = 0;
// Skip files that are already being compacted
for (f = nullptr; loop < file_by_time.size(); loop++) {
int index = file_by_time[loop];
f = current_->files_[level][index];
if (!f->being_compacted) {
candidate_count = 1;
break;
}
Log(options_->info_log,
"Universal: file %ld[%d] being compacted, skipping",
f->number, loop);
f = nullptr;
}
// This file is not being compacted. Consider it as the
// first candidate to be compacted.
uint64_t candidate_size = f != nullptr? f->file_size : 0;
if (f != nullptr) {
Log(options_->info_log, "Universal: Possible candidate file %ld[%d].",
f->number, loop);
}
// Check if the suceeding files need compaction.
for (unsigned int i = loop+1;
candidate_count < max_files_to_compact && i < file_by_time.size();
i++) {
int index = file_by_time[i];
FileMetaData* f = current_->files_[level][index];
if (f->being_compacted) {
break;
}
// pick files if the total candidate file size (increased by the
// specified ratio) is still larger than the next candidate file.
uint64_t sz = (candidate_size * (100L + ratio)) /100;
if (sz < f->file_size) {
break;
}
candidate_count++;
candidate_size += f->file_size;
}
// Found a series of consecutive files that need compaction.
if (candidate_count >= (unsigned int)min_merge_width) {
start_index = loop;
done = true;
break;
} else {
for (unsigned int i = loop;
i < loop + candidate_count && i < file_by_time.size(); i++) {
int index = file_by_time[i];
FileMetaData* f = current_->files_[level][index];
Log(options_->info_log,
"Universal: Skipping file %ld[%d] with size %ld %d\n",
f->number, i, f->file_size, f->being_compacted);
}
}
}
if (!done || candidate_count <= 1) {
return nullptr;
}
Compaction* c = new Compaction(level, level, MaxFileSizeForLevel(level),
LLONG_MAX, NumberLevels());
c->score_ = score;
for (unsigned int i = start_index; i < start_index + candidate_count; i++) {
int index = file_by_time[i];
FileMetaData* f = current_->files_[level][index];
c->inputs_[0].push_back(f);
Log(options_->info_log, "Universal: Picking file %ld[%d] with size %ld\n",
f->number, i, f->file_size);
}
return c;
}
//
// Universal style of compaction. Pick files that are contiguous in
// time-range to compact.
//
Compaction* VersionSet::PickCompactionUniversal(int level, double score) {
assert (level == 0);
if ((current_->files_[level].size() <
if ((current_->files_[level].size() <=
(unsigned int)options_->level0_file_num_compaction_trigger)) {
Log(options_->info_log, "Universal: nothing to do\n");
return nullptr;
@@ -2377,29 +2143,127 @@ Compaction* VersionSet::PickCompactionUniversal(int level, double score) {
current_->files_[level].size(),
LevelFileSummary(&tmp, 0));
// Check for size amplification first.
Compaction* c = PickCompactionUniversalSizeAmp(level, score);
if (c == nullptr) {
Compaction* c = nullptr;
c = new Compaction(level, level, MaxFileSizeForLevel(level),
LLONG_MAX, NumberLevels());
c->score_ = score;
// Size amplification is within limits. Try reducing read
// amplification while maintaining file size ratios.
unsigned int ratio = options_->compaction_options_universal.size_ratio;
c = PickCompactionUniversalReadAmp(level, score, ratio, UINT_MAX);
// The files are sorted from newest first to oldest last.
std::vector<int>& file_by_time = current_->files_by_size_[level];
FileMetaData* f = nullptr;
bool done = false;
assert(file_by_time.size() == current_->files_[level].size());
// Size amplification and file size ratios are within configured limits.
// If max read amplification is exceeding configured limits, then force
// compaction without looking at filesize ratios and try to reduce
// the number of files to fewer than level0_file_num_compaction_trigger.
if (c == nullptr) {
unsigned int num_files = current_->files_[level].size() -
options_->level0_file_num_compaction_trigger;
c = PickCompactionUniversalReadAmp(level, score, UINT_MAX, num_files);
unsigned int max_files_to_compact = std::min(max_merge_width, UINT_MAX);
// Make two pass. The first pass considers a candidate file
// only if it is smaller than the total size accumulated so far.
// The second pass does not look at the slope of the
// file-size curve to decide what to pick for compaction.
for (int iter = 0; !done && iter < 2; iter++) {
for (unsigned int loop = 0; loop < file_by_time.size(); ) {
// Skip files that are already being compacted
for (f = nullptr; loop < file_by_time.size(); loop++) {
int index = file_by_time[loop];
f = current_->files_[level][index];
if (!f->being_compacted) {
break;
}
Log(options_->info_log, "Universal: file %ld[%d] being compacted, skipping",
f->number, loop);
f = nullptr;
}
// This file is not being compacted. Consider it as the
// first candidate to be compacted.
unsigned int candidate_count = 1;
uint64_t candidate_size = f != nullptr? f->file_size : 0;
if (f != nullptr) {
Log(options_->info_log, "Universal: Possible candidate file %ld[%d] %s.",
f->number, loop, iter == 0? "" : "forced ");
}
// Check if the suceeding files need compaction.
for (unsigned int i = loop+1;
candidate_count < max_files_to_compact && i < file_by_time.size();
i++) {
int index = file_by_time[i];
FileMetaData* f = current_->files_[level][index];
if (f->being_compacted) {
break;
}
// If this is the first iteration, then we pick files if the
// total candidate file size (increased by the specified ratio)
// is still larger than the next candidate file.
if (iter == 0) {
uint64_t sz = (candidate_size * (100 + ratio)) /100;
if (sz < f->file_size) {
break;
}
}
candidate_count++;
candidate_size += f->file_size;
}
// Found a series of consecutive files that need compaction.
if (candidate_count >= (unsigned int)min_merge_width) {
for (unsigned int i = loop; i < loop + candidate_count; i++) {
int index = file_by_time[i];
FileMetaData* f = current_->files_[level][index];
c->inputs_[0].push_back(f);
Log(options_->info_log, "Universal: Picking file %ld[%d] with size %ld %s",
f->number, i, f->file_size,
(iter == 0 ? "" : "forced"));
}
done = true;
break;
} else {
for (unsigned int i = loop;
i < loop + candidate_count && i < file_by_time.size(); i++) {
int index = file_by_time[i];
FileMetaData* f = current_->files_[level][index];
Log(options_->info_log, "Universal: Skipping file %ld[%d] with size %ld %d %s",
f->number, i, f->file_size, f->being_compacted,
(iter == 0 ? "" : "forced"));
}
}
loop += candidate_count;
}
assert(done || c->inputs_[0].size() == 0);
// If we are unable to find a normal compaction run and we are still
// above the compaction threshold, iterate again to pick compaction
// candidates, this time without considering their size differences.
if (!done) {
int files_not_in_compaction = 0;
for (unsigned int i = 0; i < current_->files_[level].size(); i++) {
f = current_->files_[level][i];
if (!f->being_compacted) {
files_not_in_compaction++;
}
}
int expected_num_files = files_not_in_compaction +
compactions_in_progress_[level].size();
if (expected_num_files <=
options_->level0_file_num_compaction_trigger + 1) {
done = true; // nothing more to do
} else {
max_files_to_compact = std::min((int)max_merge_width,
expected_num_files - options_->level0_file_num_compaction_trigger);
Log(options_->info_log, "Universal: second loop with maxfiles %d",
max_files_to_compact);
}
}
}
if (c == nullptr) {
if (c->inputs_[0].size() <= 1) {
Log(options_->info_log, "Universal: only %ld files, nothing to do.\n",
c->inputs_[0].size());
delete c;
return nullptr;
}
assert(c->inputs_[0].size() > 1);
// validate that all the chosen files are non overlapping in time
FileMetaData* newerfile __attribute__((unused)) = nullptr;
@@ -2411,9 +2275,6 @@ Compaction* VersionSet::PickCompactionUniversal(int level, double score) {
newerfile = f;
}
// The files are sorted from newest first to oldest last.
std::vector<int>& file_by_time = current_->files_by_size_[level];
// Is the earliest file part of this compaction?
int last_index = file_by_time[file_by_time.size()-1];
FileMetaData* last_file = current_->files_[level][last_index];
@@ -2776,41 +2637,6 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
c->edit_->SetCompactPointer(level, largest);
}
Status VersionSet::GetMetadataForFile(
uint64_t number,
int *filelevel,
FileMetaData *meta) {
for (int level = 0; level < NumberLevels(); level++) {
const std::vector<FileMetaData*>& files = current_->files_[level];
for (size_t i = 0; i < files.size(); i++) {
if (files[i]->number == number) {
*meta = *files[i];
*filelevel = level;
return Status::OK();
}
}
}
return Status::NotFound("File not present in any level");
}
void VersionSet::GetLiveFilesMetaData(
std::vector<LiveFileMetaData> * metadata) {
for (int level = 0; level < NumberLevels(); level++) {
const std::vector<FileMetaData*>& files = current_->files_[level];
for (size_t i = 0; i < files.size(); i++) {
LiveFileMetaData filemetadata;
filemetadata.name = TableFileName("", files[i]->number);
filemetadata.level = level;
filemetadata.size = files[i]->file_size;
filemetadata.smallestkey = files[i]->smallest.user_key().ToString();
filemetadata.largestkey = files[i]->largest.user_key().ToString();
filemetadata.smallest_seqno = files[i]->smallest_seqno;
filemetadata.largest_seqno = files[i]->largest_seqno;
metadata->push_back(filemetadata);
}
}
}
Compaction* VersionSet::CompactRange(
int level,
const InternalKey* begin,
@@ -3055,4 +2881,4 @@ void Compaction::Summary(char* output, int len) {
level_low_summary, level_up_summary);
}
} // namespace rocksdb
} // namespace leveldb
+3 -18
View File
@@ -25,7 +25,7 @@
#include "port/port.h"
#include "db/table_cache.h"
namespace rocksdb {
namespace leveldb {
namespace log { class Writer; }
@@ -76,7 +76,7 @@ class Version {
};
void Get(const ReadOptions&, const LookupKey& key, std::string* val,
Status* status, std::deque<std::string>* operands, GetStats* stats,
const Options& db_option,
const Options& db_option, const bool no_io = false,
bool* value_found = nullptr);
// Adds "stats" into the current state. Returns true if a new
@@ -152,8 +152,6 @@ class Version {
Iterator* NewConcatenatingIterator(const ReadOptions&,
const EnvOptions& soptions,
int level) const;
bool PrefixMayMatch(const ReadOptions& options, const EnvOptions& soptions,
const Slice& internal_prefix, Iterator* level_iter) const;
VersionSet* vset_; // VersionSet to which this Version belongs
Version* next_; // Next version in linked list
@@ -383,13 +381,6 @@ class VersionSet {
// Pick files to compact in Universal mode
Compaction* PickCompactionUniversal(int level, double score);
// Pick Universal compaction to limit read amplification
Compaction* PickCompactionUniversalReadAmp(int level, double score,
unsigned int ratio, unsigned int num_files);
// Pick Universal compaction to limit space amplification.
Compaction* PickCompactionUniversalSizeAmp(int level, double score);
// Free up the files that were participated in a compaction
void ReleaseCompactionFiles(Compaction* c, Status status);
@@ -414,12 +405,6 @@ class VersionSet {
double MaxBytesForLevel(int level);
Status GetMetadataForFile(
uint64_t number, int *filelevel, FileMetaData *metadata);
void GetLiveFilesMetaData(
std::vector<LiveFileMetaData> *metadata);
private:
class Builder;
struct ManifestWriter;
@@ -627,6 +612,6 @@ class Compaction {
void ResetNextCompactionIndex();
};
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_VERSION_SET_H_
+1 -1
View File
@@ -10,7 +10,7 @@
#include "db/log_writer.h"
#include "util/logging.h"
namespace rocksdb {
namespace leveldb {
Status VersionSet::ReduceNumberOfLevels(int new_levels, port::Mutex* mu) {
+3 -3
View File
@@ -7,7 +7,7 @@
#include "util/testharness.h"
#include "util/testutil.h"
namespace rocksdb {
namespace leveldb {
class FindFileTest {
public:
@@ -172,8 +172,8 @@ TEST(FindFileTest, OverlappingFiles) {
ASSERT_TRUE(Overlaps("600", "700"));
}
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
+6 -10
View File
@@ -14,10 +14,10 @@
// len: varint32
// data: uint8[len]
#include "rocksdb/write_batch.h"
#include "leveldb/write_batch.h"
#include "rocksdb/options.h"
#include "rocksdb/statistics.h"
#include "leveldb/options.h"
#include "leveldb/statistics.h"
#include "db/dbformat.h"
#include "db/db_impl.h"
#include "db/memtable.h"
@@ -26,7 +26,7 @@
#include "util/coding.h"
#include <stdexcept>
namespace rocksdb {
namespace leveldb {
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
static const size_t kHeader = 12;
@@ -48,10 +48,6 @@ void WriteBatch::Handler::LogData(const Slice& blob) {
// them.
}
bool WriteBatch::Handler::Continue() {
return true;
}
void WriteBatch::Clear() {
rep_.clear();
rep_.resize(kHeader);
@@ -70,7 +66,7 @@ Status WriteBatch::Iterate(Handler* handler) const {
input.remove_prefix(kHeader);
Slice key, value, blob;
int found = 0;
while (!input.empty() && handler->Continue()) {
while (!input.empty()) {
char tag = input[0];
input.remove_prefix(1);
switch (tag) {
@@ -227,4 +223,4 @@ void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) {
dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader);
}
} // namespace rocksdb
} // namespace leveldb
+6 -6
View File
@@ -5,12 +5,12 @@
#ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
#define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
#include "rocksdb/types.h"
#include "rocksdb/write_batch.h"
#include "rocksdb/db.h"
#include "rocksdb/options.h"
#include "leveldb/types.h"
#include "leveldb/write_batch.h"
#include "leveldb/db.h"
#include "leveldb/options.h"
namespace rocksdb {
namespace leveldb {
class MemTable;
@@ -51,7 +51,7 @@ class WriteBatchInternal {
static void Append(WriteBatch* dst, const WriteBatch* src);
};
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
+21 -64
View File
@@ -2,17 +2,17 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include "rocksdb/db.h"
#include "leveldb/db.h"
#include <memory>
#include "db/skiplistrep.h"
#include "db/memtable.h"
#include "db/write_batch_internal.h"
#include "rocksdb/env.h"
#include "rocksdb/memtablerep.h"
#include "leveldb/env.h"
#include "util/logging.h"
#include "util/testharness.h"
namespace rocksdb {
namespace leveldb {
static std::string PrintContents(WriteBatch* b) {
InternalKeyComparator cmp(BytewiseComparator());
@@ -134,24 +134,6 @@ TEST(WriteBatchTest, Append) {
ASSERT_EQ(4, b1.Count());
}
namespace {
struct TestHandler : public WriteBatch::Handler {
std::string seen;
virtual void Put(const Slice& key, const Slice& value) {
seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
}
virtual void Merge(const Slice& key, const Slice& value) {
seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
}
virtual void LogData(const Slice& blob) {
seen += "LogData(" + blob.ToString() + ")";
}
virtual void Delete(const Slice& key) {
seen += "Delete(" + key.ToString() + ")";
}
};
}
TEST(WriteBatchTest, Blob) {
WriteBatch batch;
batch.Put(Slice("k1"), Slice("v1"));
@@ -169,7 +151,21 @@ TEST(WriteBatchTest, Blob) {
"Put(k3, v3)@2",
PrintContents(&batch));
TestHandler handler;
struct Handler : public WriteBatch::Handler {
std::string seen;
virtual void Put(const Slice& key, const Slice& value) {
seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
}
virtual void Merge(const Slice& key, const Slice& value) {
seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
}
virtual void LogData(const Slice& blob) {
seen += "LogData(" + blob.ToString() + ")";
}
virtual void Delete(const Slice& key) {
seen += "Delete(" + key.ToString() + ")";
}
} handler;
batch.Iterate(&handler);
ASSERT_EQ(
"Put(k1, v1)"
@@ -182,47 +178,8 @@ TEST(WriteBatchTest, Blob) {
handler.seen);
}
TEST(WriteBatchTest, Continue) {
WriteBatch batch;
struct Handler : public TestHandler {
int num_seen = 0;
virtual void Put(const Slice& key, const Slice& value) {
++num_seen;
TestHandler::Put(key, value);
}
virtual void Merge(const Slice& key, const Slice& value) {
++num_seen;
TestHandler::Merge(key, value);
}
virtual void LogData(const Slice& blob) {
++num_seen;
TestHandler::LogData(blob);
}
virtual void Delete(const Slice& key) {
++num_seen;
TestHandler::Delete(key);
}
virtual bool Continue() override {
return num_seen < 3;
}
} handler;
batch.Put(Slice("k1"), Slice("v1"));
batch.PutLogData(Slice("blob1"));
batch.Delete(Slice("k1"));
batch.PutLogData(Slice("blob2"));
batch.Merge(Slice("foo"), Slice("bar"));
batch.Iterate(&handler);
ASSERT_EQ(
"Put(k1, v1)"
"LogData(blob1)"
"Delete(k1)",
handler.seen);
}
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
+6 -6
View File
@@ -111,7 +111,7 @@ static void WalCheckpoint(sqlite3* db_) {
}
}
namespace rocksdb {
namespace leveldb {
// Helper for quickly generating random data.
namespace {
@@ -664,7 +664,7 @@ class Benchmark {
};
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
std::string default_db_path;
@@ -672,7 +672,7 @@ int main(int argc, char** argv) {
double d;
int n;
char junk;
if (rocksdb::Slice(argv[i]).starts_with("--benchmarks=")) {
if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
} else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
@@ -688,7 +688,7 @@ int main(int argc, char** argv) {
FLAGS_reads = n;
} else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
FLAGS_value_size = n;
} else if (rocksdb::Slice(argv[i]) == rocksdb::Slice("--no_transaction")) {
} else if (leveldb::Slice(argv[i]) == leveldb::Slice("--no_transaction")) {
FLAGS_transaction = false;
} else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) {
FLAGS_page_size = n;
@@ -707,12 +707,12 @@ int main(int argc, char** argv) {
// Choose a location for the test database if none given with --db=<path>
if (FLAGS_db == NULL) {
rocksdb::Env::Default()->GetTestDirectory(&default_db_path);
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
default_db_path += "/dbbench";
FLAGS_db = default_db_path.c_str();
}
rocksdb::Benchmark benchmark;
leveldb::Benchmark benchmark;
benchmark.Run();
return 0;
}
+5 -5
View File
@@ -80,7 +80,7 @@ static void DBSynchronize(kyotocabinet::TreeDB* db_)
}
}
namespace rocksdb {
namespace leveldb {
// Helper for quickly generating random data.
namespace {
@@ -479,7 +479,7 @@ class Benchmark {
}
};
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
std::string default_db_path;
@@ -487,7 +487,7 @@ int main(int argc, char** argv) {
double d;
int n;
char junk;
if (rocksdb::Slice(argv[i]).starts_with("--benchmarks=")) {
if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
} else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
FLAGS_compression_ratio = d;
@@ -517,12 +517,12 @@ int main(int argc, char** argv) {
// Choose a location for the test database if none given with --db=<path>
if (FLAGS_db == NULL) {
rocksdb::Env::Default()->GetTestDirectory(&default_db_path);
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
default_db_path += "/dbbench";
FLAGS_db = default_db_path.c_str();
}
rocksdb::Benchmark benchmark;
leveldb::Benchmark benchmark;
benchmark.Run();
return 0;
}
+52 -52
View File
@@ -25,27 +25,27 @@ creating it if necessary:
#include &lt;assert&gt;
#include "leveldb/db.h"
rocksdb::DB* db;
rocksdb::Options options;
leveldb::DB* db;
leveldb::Options options;
options.create_if_missing = true;
rocksdb::Status status = rocksdb::DB::Open(options, "/tmp/testdb", &amp;db);
leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &amp;db);
assert(status.ok());
...
</pre>
If you want to raise an error if the database already exists, add
the following line before the <code>rocksdb::DB::Open</code> call:
the following line before the <code>leveldb::DB::Open</code> call:
<pre>
options.error_if_exists = true;
</pre>
<h1>Status</h1>
<p>
You may have noticed the <code>rocksdb::Status</code> type above. Values of this
You may have noticed the <code>leveldb::Status</code> type above. Values of this
type are returned by most functions in <code>leveldb</code> that may encounter an
error. You can check if such a result is ok, and also print an
associated error message:
<p>
<pre>
rocksdb::Status s = ...;
leveldb::Status s = ...;
if (!s.ok()) cerr &lt;&lt; s.ToString() &lt;&lt; endl;
</pre>
<h1>Closing A Database</h1>
@@ -65,9 +65,9 @@ modify/query the database. For example, the following code
moves the value stored under key1 to key2.
<pre>
std::string value;
rocksdb::Status s = db-&gt;Get(rocksdb::ReadOptions(), key1, &amp;value);
if (s.ok()) s = db-&gt;Put(rocksdb::WriteOptions(), key2, value);
if (s.ok()) s = db-&gt;Delete(rocksdb::WriteOptions(), key1);
leveldb::Status s = db-&gt;Get(leveldb::ReadOptions(), key1, &amp;value);
if (s.ok()) s = db-&gt;Put(leveldb::WriteOptions(), key2, value);
if (s.ok()) s = db-&gt;Delete(leveldb::WriteOptions(), key1);
</pre>
<h1>Atomic Updates</h1>
@@ -81,12 +81,12 @@ atomically apply a set of updates:
#include "leveldb/write_batch.h"
...
std::string value;
rocksdb::Status s = db-&gt;Get(rocksdb::ReadOptions(), key1, &amp;value);
leveldb::Status s = db-&gt;Get(leveldb::ReadOptions(), key1, &amp;value);
if (s.ok()) {
rocksdb::WriteBatch batch;
leveldb::WriteBatch batch;
batch.Delete(key1);
batch.Put(key2, value);
s = db-&gt;Write(rocksdb::WriteOptions(), &amp;batch);
s = db-&gt;Write(leveldb::WriteOptions(), &amp;batch);
}
</pre>
The <code>WriteBatch</code> holds a sequence of edits to be made to the database,
@@ -109,7 +109,7 @@ persistent storage. (On Posix systems, this is implemented by calling
either <code>fsync(...)</code> or <code>fdatasync(...)</code> or
<code>msync(..., MS_SYNC)</code> before the write operation returns.)
<pre>
rocksdb::WriteOptions write_options;
leveldb::WriteOptions write_options;
write_options.sync = true;
db-&gt;Put(write_options, ...);
</pre>
@@ -144,7 +144,7 @@ the batch.
A database may only be opened by one process at a time.
The <code>leveldb</code> implementation acquires a lock from the
operating system to prevent misuse. Within a single process, the
same <code>rocksdb::DB</code> object may be safely shared by multiple
same <code>leveldb::DB</code> object may be safely shared by multiple
concurrent threads. I.e., different threads may write into or fetch
iterators or call <code>Get</code> on the same database without any
external synchronization (the leveldb implementation will
@@ -160,7 +160,7 @@ The following example demonstrates how to print all key,value pairs
in a database.
<p>
<pre>
rocksdb::Iterator* it = db-&gt;NewIterator(rocksdb::ReadOptions());
leveldb::Iterator* it = db-&gt;NewIterator(leveldb::ReadOptions());
for (it-&gt;SeekToFirst(); it-&gt;Valid(); it-&gt;Next()) {
cout &lt;&lt; it-&gt;key().ToString() &lt;&lt; ": " &lt;&lt; it-&gt;value().ToString() &lt;&lt; endl;
}
@@ -196,10 +196,10 @@ implicit snapshot of the current state.
Snapshots are created by the DB::GetSnapshot() method:
<p>
<pre>
rocksdb::ReadOptions options;
leveldb::ReadOptions options;
options.snapshot = db-&gt;GetSnapshot();
... apply some updates to db ...
rocksdb::Iterator* iter = db-&gt;NewIterator(options);
leveldb::Iterator* iter = db-&gt;NewIterator(options);
... read using iter to view the state when the snapshot was created ...
delete iter;
db-&gt;ReleaseSnapshot(options.snapshot);
@@ -211,7 +211,7 @@ support reading as of that snapshot.
<h1>Slice</h1>
<p>
The return value of the <code>it->key()</code> and <code>it->value()</code> calls above
are instances of the <code>rocksdb::Slice</code> type. <code>Slice</code> is a simple
are instances of the <code>leveldb::Slice</code> type. <code>Slice</code> is a simple
structure that contains a length and a pointer to an external byte
array. Returning a <code>Slice</code> is a cheaper alternative to returning a
<code>std::string</code> since we do not need to copy potentially large keys and
@@ -223,10 +223,10 @@ C++ strings and null-terminated C-style strings can be easily converted
to a Slice:
<p>
<pre>
rocksdb::Slice s1 = "hello";
leveldb::Slice s1 = "hello";
std::string str("world");
rocksdb::Slice s2 = str;
leveldb::Slice s2 = str;
</pre>
A Slice can be easily converted back to a C++ string:
<pre>
@@ -238,7 +238,7 @@ the external byte array into which the Slice points remains live while
the Slice is in use. For example, the following is buggy:
<p>
<pre>
rocksdb::Slice slice;
leveldb::Slice slice;
if (...) {
std::string str = ...;
slice = str;
@@ -255,16 +255,16 @@ which orders bytes lexicographically. You can however supply a custom
comparator when opening a database. For example, suppose each
database key consists of two numbers and we should sort by the first
number, breaking ties by the second number. First, define a proper
subclass of <code>rocksdb::Comparator</code> that expresses these rules:
subclass of <code>leveldb::Comparator</code> that expresses these rules:
<p>
<pre>
class TwoPartComparator : public rocksdb::Comparator {
class TwoPartComparator : public leveldb::Comparator {
public:
// Three-way comparison function:
// if a &lt; b: negative result
// if a &gt; b: positive result
// else: zero result
int Compare(const rocksdb::Slice&amp; a, const rocksdb::Slice&amp; b) const {
int Compare(const leveldb::Slice&amp; a, const leveldb::Slice&amp; b) const {
int a1, a2, b1, b2;
ParseKey(a, &amp;a1, &amp;a2);
ParseKey(b, &amp;b1, &amp;b2);
@@ -277,7 +277,7 @@ subclass of <code>rocksdb::Comparator</code> that expresses these rules:
// Ignore the following methods for now:
const char* Name() const { return "TwoPartComparator"; }
void FindShortestSeparator(std::string*, const rocksdb::Slice&amp;) const { }
void FindShortestSeparator(std::string*, const leveldb::Slice&amp;) const { }
void FindShortSuccessor(std::string*) const { }
};
</pre>
@@ -285,18 +285,18 @@ Now create a database using this custom comparator:
<p>
<pre>
TwoPartComparator cmp;
rocksdb::DB* db;
rocksdb::Options options;
leveldb::DB* db;
leveldb::Options options;
options.create_if_missing = true;
options.comparator = &amp;cmp;
rocksdb::Status status = rocksdb::DB::Open(options, "/tmp/testdb", &amp;db);
leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &amp;db);
...
</pre>
<h2>Backwards compatibility</h2>
<p>
The result of the comparator's <code>Name</code> method is attached to the
database when it is created, and is checked on every subsequent
database open. If the name changes, the <code>rocksdb::DB::Open</code> call will
database open. If the name changes, the <code>leveldb::DB::Open</code> call will
fail. Therefore, change the name if and only if the new key format
and comparison function are incompatible with existing databases, and
it is ok to discard the contents of all existing databases.
@@ -339,9 +339,9 @@ compression entirely, but should only do so if benchmarks show a
performance improvement:
<p>
<pre>
rocksdb::Options options;
options.compression = rocksdb::kNoCompression;
... rocksdb::DB::Open(options, name, ...) ....
leveldb::Options options;
options.compression = leveldb::kNoCompression;
... leveldb::DB::Open(options, name, ...) ....
</pre>
<h2>Cache</h2>
<p>
@@ -353,10 +353,10 @@ uncompressed block contents.
<pre>
#include "leveldb/cache.h"
rocksdb::Options options;
options.cache = rocksdb::NewLRUCache(100 * 1048576); // 100MB cache
rocksdb::DB* db;
rocksdb::DB::Open(options, name, &db);
leveldb::Options options;
options.cache = leveldb::NewLRUCache(100 * 1048576); // 100MB cache
leveldb::DB* db;
leveldb::DB::Open(options, name, &db);
... use the db ...
delete db
delete options.cache;
@@ -373,9 +373,9 @@ displacing most of the cached contents. A per-iterator option can be
used to achieve this:
<p>
<pre>
rocksdb::ReadOptions options;
leveldb::ReadOptions options;
options.fill_cache = false;
rocksdb::Iterator* it = db-&gt;NewIterator(options);
leveldb::Iterator* it = db-&gt;NewIterator(options);
for (it-&gt;SeekToFirst(); it-&gt;Valid(); it-&gt;Next()) {
...
}
@@ -407,10 +407,10 @@ a single <code>Get()</code> call may involve multiple reads from disk.
The optional <code>FilterPolicy</code> mechanism can be used to reduce
the number of disk reads substantially.
<pre>
rocksdb::Options options;
leveldb::Options options;
options.filter_policy = NewBloomFilter(10);
rocksdb::DB* db;
rocksdb::DB::Open(options, "/tmp/testdb", &amp;db);
leveldb::DB* db;
leveldb::DB::Open(options, "/tmp/testdb", &amp;db);
... use the database ...
delete db;
delete options.filter_policy;
@@ -434,7 +434,7 @@ consider a comparator that ignores trailing spaces when comparing keys.
Instead, the application should provide a custom filter policy that
also ignores trailing spaces. For example:
<pre>
class CustomFilterPolicy : public rocksdb::FilterPolicy {
class CustomFilterPolicy : public leveldb::FilterPolicy {
private:
FilterPolicy* builtin_policy_;
public:
@@ -484,7 +484,7 @@ checksums are verified:
parts of its persistent storage have been corrupted.
<p>
If a database is corrupted (perhaps it cannot be opened when
paranoid checking is turned on), the <code>rocksdb::RepairDB</code> function
paranoid checking is turned on), the <code>leveldb::RepairDB</code> function
may be used to recover as much of the data as possible
<p>
</ul>
@@ -494,11 +494,11 @@ The <code>GetApproximateSizes</code> method can used to get the approximate
number of bytes of file system space used by one or more key ranges.
<p>
<pre>
rocksdb::Range ranges[2];
ranges[0] = rocksdb::Range("a", "c");
ranges[1] = rocksdb::Range("x", "z");
leveldb::Range ranges[2];
ranges[0] = leveldb::Range("a", "c");
ranges[1] = leveldb::Range("x", "z");
uint64_t sizes[2];
rocksdb::Status s = db-&gt;GetApproximateSizes(ranges, 2, sizes);
leveldb::Status s = db-&gt;GetApproximateSizes(ranges, 2, sizes);
</pre>
The preceding call will set <code>sizes[0]</code> to the approximate number of
bytes of file system space used by the key range <code>[a..c)</code> and
@@ -508,21 +508,21 @@ bytes of file system space used by the key range <code>[a..c)</code> and
<h1>Environment</h1>
<p>
All file operations (and other operating system calls) issued by the
<code>leveldb</code> implementation are routed through a <code>rocksdb::Env</code> object.
<code>leveldb</code> implementation are routed through a <code>leveldb::Env</code> object.
Sophisticated clients may wish to provide their own <code>Env</code>
implementation to get better control. For example, an application may
introduce artificial delays in the file IO paths to limit the impact
of <code>leveldb</code> on other activities in the system.
<p>
<pre>
class SlowEnv : public rocksdb::Env {
class SlowEnv : public leveldb::Env {
.. implementation of the Env interface ...
};
SlowEnv env;
rocksdb::Options options;
leveldb::Options options;
options.env = &amp;env;
Status s = rocksdb::DB::Open(options, ...);
Status s = leveldb::DB::Open(options, ...);
</pre>
<h1>Porting</h1>
<p>
@@ -531,7 +531,7 @@ specific implementations of the types/methods/functions exported by
<code>leveldb/port/port.h</code>. See <code>leveldb/port/port_example.h</code> for more
details.
<p>
In addition, the new platform may need a new default <code>rocksdb::Env</code>
In addition, the new platform may need a new default <code>leveldb::Env</code>
implementation. See <code>leveldb/util/env_posix.h</code> for an example.
<h1>Other Information</h1>
+11 -13
View File
@@ -25,13 +25,13 @@
#include <sys/time.h>
#include <time.h>
#include <iostream>
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "leveldb/env.h"
#include "leveldb/status.h"
#ifdef USE_HDFS
#include "hdfs/hdfs.h"
namespace rocksdb {
namespace leveldb {
static const std::string kProto = "hdfs://";
static const std::string pathsep = "/";
@@ -107,9 +107,8 @@ class HdfsEnv : public Env {
virtual Status NewLogger(const std::string& fname, Logger** result);
virtual void Schedule(void (*function)(void* arg), void* arg,
Priority pri = LOW) {
posixEnv->Schedule(function, arg, pri);
virtual void Schedule( void (*function)(void* arg), void* arg) {
posixEnv->Schedule(function, arg);
}
virtual void StartThread(void (*function)(void* arg), void* arg) {
@@ -141,8 +140,8 @@ class HdfsEnv : public Env {
return posixEnv->GetAbsolutePath(db_path, output_path);
}
virtual void SetBackgroundThreads(int number, Priority pri = LOW) {
posixEnv->SetBackgroundThreads(number, pri);
virtual void SetBackgroundThreads(int number) {
posixEnv->SetBackgroundThreads(number);
}
virtual std::string TimeToString(uint64_t number) {
@@ -214,12 +213,12 @@ class HdfsEnv : public Env {
}
};
} // namespace rocksdb
} // namespace leveldb
#else // USE_HDFS
namespace rocksdb {
namespace leveldb {
static const Status notsup;
@@ -280,8 +279,7 @@ class HdfsEnv : public Env {
virtual Status NewLogger(const std::string& fname,
shared_ptr<Logger>* result){return notsup;}
virtual void Schedule(void (*function)(void* arg), void* arg,
Priority pri = LOW) {}
virtual void Schedule( void (*function)(void* arg), void* arg) {}
virtual void StartThread(void (*function)(void* arg), void* arg) {}
@@ -298,7 +296,7 @@ class HdfsEnv : public Env {
virtual Status GetAbsolutePath(const std::string& db_path,
std::string* outputpath) {return notsup;}
virtual void SetBackgroundThreads(int number, Priority pri = LOW) {}
virtual void SetBackgroundThreads(int number) {}
virtual std::string TimeToString(uint64_t number) { return "";}
};
+4 -4
View File
@@ -4,8 +4,8 @@
#include "helpers/memenv/memenv.h"
#include "rocksdb/env.h"
#include "rocksdb/status.h"
#include "leveldb/env.h"
#include "leveldb/status.h"
#include "port/port.h"
#include "util/mutexlock.h"
#include <map>
@@ -13,7 +13,7 @@
#include <string>
#include <vector>
namespace rocksdb {
namespace leveldb {
namespace {
@@ -383,4 +383,4 @@ Env* NewMemEnv(Env* base_env) {
return new InMemoryEnv(base_env);
}
} // namespace rocksdb
} // namespace leveldb
+6 -5
View File
@@ -2,9 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
#define STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
namespace rocksdb {
#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
namespace leveldb {
class Env;
@@ -14,6 +15,6 @@ class Env;
// *base_env must remain live while the result is in use.
Env* NewMemEnv(Env* base_env);
} // namespace rocksdb
} // namespace leveldb
#endif // STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
#endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
+5 -5
View File
@@ -5,14 +5,14 @@
#include "helpers/memenv/memenv.h"
#include "db/db_impl.h"
#include "rocksdb/db.h"
#include "rocksdb/env.h"
#include "leveldb/db.h"
#include "leveldb/env.h"
#include "util/testharness.h"
#include <memory>
#include <string>
#include <vector>
namespace rocksdb {
namespace leveldb {
class MemEnvTest {
public:
@@ -226,8 +226,8 @@ TEST(MemEnvTest, DBTest) {
delete db;
}
} // namespace rocksdb
} // namespace leveldb
int main(int argc, char** argv) {
return rocksdb::test::RunAllTests();
return leveldb::test::RunAllTests();
}
@@ -5,13 +5,10 @@
// Arena class defines memory allocation methods. It's used by memtable and
// skiplist.
#ifndef STORAGE_ROCKSDB_INCLUDE_ARENA_H_
#define STORAGE_ROCKSDB_INCLUDE_ARENA_H_
#ifndef STORAGE_LEVELDB_INCLUDE_ARENA_H_
#define STORAGE_LEVELDB_INCLUDE_ARENA_H_
#include <limits>
#include <memory>
namespace rocksdb {
namespace leveldb {
class Arena {
public:
@@ -36,8 +33,6 @@ class Arena {
void operator=(const Arena&);
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_ARENA_H_
#endif // STORAGE_LEVELDB_INCLUDE_ARENA_H_
+3 -3
View File
@@ -37,8 +37,8 @@
(5) All of the pointer arguments must be non-NULL.
*/
#ifndef STORAGE_ROCKSDB_INCLUDE_C_H_
#define STORAGE_ROCKSDB_INCLUDE_C_H_
#ifndef STORAGE_LEVELDB_INCLUDE_C_H_
#define STORAGE_LEVELDB_INCLUDE_C_H_
#ifdef __cplusplus
extern "C" {
@@ -278,4 +278,4 @@ extern void leveldb_env_destroy(leveldb_env_t*);
} /* end extern "C" */
#endif
#endif /* STORAGE_ROCKSDB_INCLUDE_C_H_ */
#endif /* STORAGE_LEVELDB_INCLUDE_C_H_ */
@@ -15,14 +15,14 @@
// they want something more sophisticated (like scan-resistance, a
// custom eviction policy, variable cache sizing, etc.)
#ifndef STORAGE_ROCKSDB_INCLUDE_CACHE_H_
#define STORAGE_ROCKSDB_INCLUDE_CACHE_H_
#ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_
#define STORAGE_LEVELDB_INCLUDE_CACHE_H_
#include <memory>
#include <stdint.h>
#include "rocksdb/slice.h"
#include "leveldb/slice.h"
namespace rocksdb {
namespace leveldb {
using std::shared_ptr;
@@ -101,8 +101,6 @@ class Cache {
void operator=(const Cache&);
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_UTIL_CACHE_H_
#endif // STORAGE_LEVELDB_UTIL_CACHE_H_
@@ -2,12 +2,12 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_ROCKSDB_INCLUDE_COMPACTION_FILTER_H_
#define STORAGE_ROCKSDB_INCLUDE_COMPACTION_FILTER_H_
#ifndef STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
#define STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
#include <string>
namespace rocksdb {
namespace leveldb {
class Slice;
@@ -66,8 +66,6 @@ class DefaultCompactionFilterFactory : public CompactionFilterFactory {
}
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_COMPACTION_FILTER_H_
#endif // STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
@@ -2,12 +2,12 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
#define STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
#ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
#define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
#include <string>
namespace rocksdb {
namespace leveldb {
class Slice;
@@ -58,8 +58,6 @@ class Comparator {
// must not be deleted.
extern const Comparator* BytewiseComparator();
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
#endif // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
+18 -54
View File
@@ -2,19 +2,19 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_ROCKSDB_INCLUDE_DB_H_
#define STORAGE_ROCKSDB_INCLUDE_DB_H_
#ifndef STORAGE_LEVELDB_INCLUDE_DB_H_
#define STORAGE_LEVELDB_INCLUDE_DB_H_
#include <stdint.h>
#include <stdio.h>
#include <memory>
#include <vector>
#include "rocksdb/iterator.h"
#include "rocksdb/options.h"
#include "rocksdb/types.h"
#include "rocksdb/transaction_log.h"
#include "leveldb/iterator.h"
#include "leveldb/options.h"
#include "leveldb/types.h"
#include "leveldb/transaction_log.h"
namespace rocksdb {
namespace leveldb {
using std::unique_ptr;
@@ -28,17 +28,6 @@ struct WriteOptions;
struct FlushOptions;
class WriteBatch;
// Metadata associated with each SST file.
struct LiveFileMetaData {
std::string name; // Name of the file
int level; // Level at which this file resides.
size_t size; // File size in bytes.
std::string smallestkey; // Smallest user defined key in the file.
std::string largestkey; // Largest user defined key in the file.
SequenceNumber smallest_seqno; // smallest seqno in file
SequenceNumber largest_seqno; // largest seqno in file
};
// Abstract handle to particular state of a DB.
// A Snapshot is an immutable object and can therefore be safely
// accessed from multiple threads without any external synchronization.
@@ -209,10 +198,9 @@ class DB {
// after compaction is reduced, that level might not be appropriate for
// hosting all the files. In this case, client could set reduce_level
// to true, to move the files back to the minimum level capable of holding
// the data set or a given level (specified by non-negative target_level).
// the data set.
virtual void CompactRange(const Slice* begin, const Slice* end,
bool reduce_level = false,
int target_level = -1) = 0;
bool reduce_level = false) = 0;
// Number of levels used for this DB.
virtual int NumberLevels() = 0;
@@ -235,30 +223,21 @@ class DB {
// Allow compactions to delete obselete files.
virtual Status EnableFileDeletions() = 0;
// GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup
// THIS METHOD IS DEPRECATED. Use the GetTableMetaData to get more
// detailed information on the live files.
// Retrieve the list of all files in the database. The files are
// relative to the dbname and are not absolute paths. The valid size of the
// manifest file is returned in manifest_file_size. The manifest file is an
// ever growing file, but only the portion specified by manifest_file_size is
// valid for this snapshot.
// Setting flush_memtable to true does Flush before recording the live files.
// Setting flush_memtable to false is useful when we don't want to wait for
// flush which may have to wait for compaction to complete taking an
// indeterminate time. But this will have to use GetSortedWalFiles after
// GetLiveFiles to compensate for memtables missed in this snapshot due to the
// absence of Flush, by WAL files to recover the database consistently later
// relative to the dbname and are not absolute paths. This list
// can be used to generate a backup. The valid size of the manifest
// file is returned in manifest_file_size. The manifest file is
// an ever growing file, but only the portion specified
// by manifest_file_size is valid for this snapshot.
virtual Status GetLiveFiles(std::vector<std::string>&,
uint64_t* manifest_file_size,
bool flush_memtable = true) = 0;
uint64_t* manifest_file_size) = 0;
// Retrieve the sorted list of all wal files with earliest file first
virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0;
// Delete wal files in files. These can be either live or archived.
// Returns Status::OK if all files could be deleted, otherwise Status::IOError
// which contains information about files that could not be deleted.
virtual Status DeleteWalFiles(const VectorLogPtr& files) = 0;
// The sequence number of the most recent transaction.
@@ -277,19 +256,6 @@ class DB {
// an update is read.
virtual Status GetUpdatesSince(SequenceNumber seq_number,
unique_ptr<TransactionLogIterator>* iter) = 0;
// Delete the file name from the db directory and update the internal
// state to reflect that.
virtual Status DeleteFile(std::string name) {
return Status::OK();
}
// Returns a list of all table files with their level, start key
// and end key
virtual void GetLiveFilesMetaData(
std::vector<LiveFileMetaData> *metadata) {
}
private:
// No copying allowed
DB(const DB&);
@@ -306,8 +272,6 @@ Status DestroyDB(const std::string& name, const Options& options);
// on a database that contains important information.
Status RepairDB(const std::string& dbname, const Options& options);
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_DB_H_
#endif // STORAGE_LEVELDB_INCLUDE_DB_H_
+17 -53
View File
@@ -10,17 +10,17 @@
// All Env implementations are safe for concurrent access from
// multiple threads without any external synchronization.
#ifndef STORAGE_ROCKSDB_INCLUDE_ENV_H_
#define STORAGE_ROCKSDB_INCLUDE_ENV_H_
#ifndef STORAGE_LEVELDB_INCLUDE_ENV_H_
#define STORAGE_LEVELDB_INCLUDE_ENV_H_
#include <cstdarg>
#include <string>
#include <memory>
#include <vector>
#include <stdint.h>
#include "rocksdb/status.h"
#include "leveldb/status.h"
namespace rocksdb {
namespace leveldb {
class FileLock;
class Logger;
@@ -41,7 +41,7 @@ struct EnvOptions {
EnvOptions();
// construct from Options
explicit EnvOptions(const Options& options);
EnvOptions(const Options& options);
// If true, then allow caching of data in environment buffers
bool use_os_buffer;
@@ -162,20 +162,15 @@ class Env {
// REQUIRES: lock has not already been unlocked.
virtual Status UnlockFile(FileLock* lock) = 0;
enum Priority { LOW, HIGH, TOTAL };
// Arrange to run "(*function)(arg)" once in a background thread, in
// the thread pool specified by pri. By default, jobs go to the 'LOW'
// priority thread pool.
// Arrange to run "(*function)(arg)" once in a background thread.
//
// "function" may run in an unspecified thread. Multiple functions
// added to the same Env may run concurrently in different threads.
// I.e., the caller may not assume that background work items are
// serialized.
virtual void Schedule(
void (*function)(void* arg),
void* arg,
Priority pri = LOW) = 0;
void* arg) = 0;
// Start a new thread, invoking "function(arg)" within the new thread.
// When "function(arg)" returns, the thread will be destroyed.
@@ -195,13 +190,6 @@ class Env {
// useful for computing deltas of time.
virtual uint64_t NowMicros() = 0;
// Returns the number of nano-seconds since some fixed point in time. Only
// useful for computing deltas of time in one run.
// Default implementation simply relies on NowMicros
virtual uint64_t NowNanos() {
return NowMicros() * 1000;
}
// Sleep/delay the thread for the perscribed number of micro-seconds.
virtual void SleepForMicroseconds(int micros) = 0;
@@ -215,10 +203,9 @@ class Env {
virtual Status GetAbsolutePath(const std::string& db_path,
std::string* output_path) = 0;
// The number of background worker threads of a specific thread pool
// for this environment. 'LOW' is the default pool.
// default number: 1
virtual void SetBackgroundThreads(int number, Priority pri = LOW) = 0;
// The number of background worker threads for this environment.
// default: 1
virtual void SetBackgroundThreads(int number) = 0;
// Converts seconds-since-Jan-01-1970 to a printable string
virtual std::string TimeToString(uint64_t time) = 0;
@@ -253,13 +240,6 @@ class SequentialFile {
//
// REQUIRES: External synchronization
virtual Status Skip(uint64_t n) = 0;
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
virtual Status InvalidateCache(size_t offset, size_t length) {
return Status::NotSupported("InvalidateCache not supported.");
}
};
// A file abstraction for randomly reading the contents of a file.
@@ -305,12 +285,6 @@ class RandomAccessFile {
virtual void Hint(AccessPattern pattern) {}
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
virtual Status InvalidateCache(size_t offset, size_t length) {
return Status::NotSupported("InvalidateCache not supported.");
}
};
// A file abstraction for sequential writing. The implementation
@@ -360,14 +334,6 @@ class WritableFile {
*block_size = preallocation_block_size_;
}
// Remove any kind of caching of data from the offset to offset+length
// of this file. If the length is 0, then it refers to the end of file.
// If the system is not caching the file contents, then this is a noop.
// This call has no effect on dirty pages in the cache.
virtual Status InvalidateCache(size_t offset, size_t length) {
return Status::NotSupported("InvalidateCache not supported.");
}
protected:
// PrepareWrite performs any necessary preparation for a write
// before the write actually occurs. This allows for pre-allocation
@@ -523,8 +489,8 @@ class EnvWrapper : public Env {
return target_->LockFile(f, l);
}
Status UnlockFile(FileLock* l) { return target_->UnlockFile(l); }
void Schedule(void (*f)(void*), void* a, Priority pri) {
return target_->Schedule(f, a, pri);
void Schedule(void (*f)(void*), void* a) {
return target_->Schedule(f, a);
}
void StartThread(void (*f)(void*), void* a) {
return target_->StartThread(f, a);
@@ -552,8 +518,8 @@ class EnvWrapper : public Env {
std::string* output_path) {
return target_->GetAbsolutePath(db_path, output_path);
}
void SetBackgroundThreads(int num, Priority pri) {
return target_->SetBackgroundThreads(num, pri);
void SetBackgroundThreads(int num) {
return target_->SetBackgroundThreads(num);
}
std::string TimeToString(uint64_t time) {
return target_->TimeToString(time);
@@ -563,8 +529,6 @@ class EnvWrapper : public Env {
Env* target_;
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_ENV_H_
#endif // STORAGE_LEVELDB_INCLUDE_ENV_H_
@@ -13,12 +13,12 @@
// Most people will want to use the builtin bloom filter support (see
// NewBloomFilterPolicy() below).
#ifndef STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
#define STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
#ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
#include <string>
namespace rocksdb {
namespace leveldb {
class Slice;
@@ -66,6 +66,5 @@ class FilterPolicy {
extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key);
}
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
#endif // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
@@ -12,13 +12,13 @@
// non-const method, all threads accessing the same Iterator must use
// external synchronization.
#ifndef STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
#define STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
#ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
#define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
#include "rocksdb/slice.h"
#include "rocksdb/status.h"
#include "leveldb/slice.h"
#include "leveldb/status.h"
namespace rocksdb {
namespace leveldb {
class Iterator {
public:
@@ -65,8 +65,6 @@ class Iterator {
virtual Slice value() const = 0;
// If an error has occurred, return it. Else return an ok status.
// If non-blocking IO is requested and this operation cannot be
// satisfied without doing some IO, then this returns Status::Incomplete().
virtual Status status() const = 0;
// Clients are allowed to register function/arg1/arg2 triples that
@@ -97,8 +95,6 @@ extern Iterator* NewEmptyIterator();
// Return an empty iterator with the specified status.
extern Iterator* NewErrorIterator(const Status& status);
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
#endif // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2008-present Facebook. All Rights Reserved.
#ifndef STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H
#define STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H
#include "leveldb/options.h"
namespace leveldb {
class LDBTool {
public:
void Run(int argc, char** argv, Options = Options());
};
} // namespace leveldb
#endif // STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H
+88
View File
@@ -0,0 +1,88 @@
// This file contains the interface that must be implemented by any collection
// to be used as the backing store for a MemTable. Such a collection must
// satisfy the following properties:
// (1) It does not store duplicate items.
// (2) It uses MemTableRep::KeyComparator to compare items for iteration and
// equality.
// (3) It can be accessed concurrently by multiple readers but need not support
// concurrent writes.
// (4) Items are never deleted.
// The liberal use of assertions is encouraged to enforce (1).
#ifndef STORAGE_LEVELDB_DB_TABLE_H_
#define STORAGE_LEVELDB_DB_TABLE_H_
#include <memory>
#include "leveldb/arena.h"
namespace leveldb {
class MemTableRep {
public:
// KeyComparator(a, b) returns a negative value if a is less than b, 0 if they
// are equal, and a positive value if b is greater than a
class KeyComparator {
public:
virtual int operator()(const char* a, const char* b) const = 0;
virtual ~KeyComparator() { }
};
// Insert key into the collection. (The caller will pack key and value into a
// single buffer and pass that in as the parameter to Insert)
// REQUIRES: nothing that compares equal to key is currently in the
// collection.
virtual void Insert(const char* key) = 0;
// Returns true iff an entry that compares equal to key is in the collection.
virtual bool Contains(const char* key) const = 0;
virtual ~MemTableRep() { }
// Iteration over the contents of a skip collection
class Iterator {
public:
// Initialize an iterator over the specified collection.
// The returned iterator is not valid.
// explicit Iterator(const MemTableRep* collection);
virtual ~Iterator() { };
// Returns true iff the iterator is positioned at a valid node.
virtual bool Valid() const = 0;
// Returns the key at the current position.
// REQUIRES: Valid()
virtual const char* key() const = 0;
// Advances to the next position.
// REQUIRES: Valid()
virtual void Next() = 0;
// Advances to the previous position.
// REQUIRES: Valid()
virtual void Prev() = 0;
// Advance to the first entry with a key >= target
virtual void Seek(const char* target) = 0;
// Position at the first entry in collection.
// Final state of iterator is Valid() iff collection is not empty.
virtual void SeekToFirst() = 0;
// Position at the last entry in collection.
// Final state of iterator is Valid() iff collection is not empty.
virtual void SeekToLast() = 0;
};
virtual std::shared_ptr<Iterator> GetIterator() = 0;
};
class MemTableRepFactory {
public:
virtual ~MemTableRepFactory() { };
virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
MemTableRep::KeyComparator&, Arena* arena) = 0;
};
}
#endif // STORAGE_LEVELDB_DB_TABLE_H_
@@ -2,14 +2,14 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_
#define STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_
#ifndef STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_
#define STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_
#include <string>
#include <deque>
#include "rocksdb/slice.h" // TODO: Remove this when migration is done;
#include "leveldb/slice.h" // TODO: Remove this when migration is done;
namespace rocksdb {
namespace leveldb {
class Slice;
class Logger;
@@ -142,8 +142,6 @@ class AssociativeMergeOperator : public MergeOperator {
Logger* logger) const override;
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_
#endif // STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_
@@ -2,21 +2,22 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
#define STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
#ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
#include <stddef.h>
#include <string>
#include <memory>
#include <vector>
#include <stdint.h>
#include "rocksdb/slice.h"
#include "rocksdb/statistics.h"
#include "rocksdb/universal_compaction.h"
#include "rocksdb/memtablerep.h"
#include "rocksdb/slice_transform.h"
#include "leveldb/slice.h"
#include "leveldb/statistics.h"
#include "leveldb/universal_compaction.h"
#include "leveldb/memtablerep.h"
namespace rocksdb {
#include "leveldb/slice_transform.h"
namespace leveldb {
class Cache;
class Comparator;
@@ -369,23 +370,10 @@ struct Options {
// every compaction run.
uint64_t delete_obsolete_files_period_micros;
// Maximum number of concurrent background jobs, submitted to
// the default LOW priority thread pool
// Maximum number of concurrent background compactions.
// Default: 1
int max_background_compactions;
// Maximum number of concurrent background memtable flush jobs, submitted to
// the HIGH priority thread pool.
// By default, all background jobs (major compaction and memtable flush) go
// to the LOW priority pool. If this option is set to a positive number,
// memtable flush jobs will be submitted to the HIGH priority pool.
// It is important when the same Env is shared by multiple db instances.
// Without a separate pool, long running major compaction jobs could
// potentially block memtable flush jobs of other db instances, leading to
// unnecessary Put stalls.
// Default: 0
int max_background_flushes;
// Specify the maximal size of the info log file. If the log file
// is larger than `max_log_file_size`, a new info log file will
// be created.
@@ -545,13 +533,6 @@ struct Options {
// Default: false
bool filter_deletes;
// An iteration->Next() sequentially skips over keys with the same
// user-key unless this option is set. This number specifies the number
// of keys (with the same userkey) that will be sequentially
// skipped before a reseek is issued.
// Default: 8
uint64_t max_sequential_skip_in_iterations;
// This is a factory that provides MemTableRep objects.
// Default: a factory that provides a skip-list-based implementation of
// MemTableRep.
@@ -561,23 +542,6 @@ struct Options {
// an application to modify/delete a key-value during background compaction.
// Default: a factory that doesn't provide any object
std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
// Remove the log file immediately after the corresponding memtable is flushed
// to data file.
// Default: true
bool purge_log_after_memtable_flush;
};
//
// An application can issue a read request (via Get/Iterators) and specify
// if that read should process data that ALREADY resides on a specified cache
// level. For example, if an application specifies kBlockCacheTier then the
// Get call will process data that is already processed in the memtable or
// the block cache. It will not page in data from the OS cache or data that
// resides in storage.
enum ReadTier {
kReadAllTier = 0x0, // data in memtable, block cache, OS cache or storage
kBlockCacheTier = 0x1 // data in memtable or block cache
};
// Options that control read operations
@@ -612,23 +576,15 @@ struct ReadOptions {
// Default: nullptr
const Slice* prefix;
// Specify if this read request should process data that ALREADY
// resides on a particular cache. If the required data is not
// found at the specified cache, then Status::Incomplete is returned.
// Default: kReadAllTier
ReadTier read_tier;
ReadOptions()
: verify_checksums(false),
fill_cache(true),
snapshot(nullptr),
prefix(nullptr),
read_tier(kReadAllTier) {
prefix(nullptr) {
}
ReadOptions(bool cksum, bool cache) :
verify_checksums(cksum), fill_cache(cache),
snapshot(nullptr), prefix(nullptr),
read_tier(kReadAllTier) {
snapshot(nullptr), prefix(nullptr) {
}
};
@@ -673,8 +629,6 @@ struct FlushOptions {
}
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
#endif // STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
+24
View File
@@ -0,0 +1,24 @@
#ifndef STORAGE_LEVELDB_INCLUDE_PERF_CONTEXT_H
#define STORAGE_LEVELDB_INCLUDE_PERF_CONTEXT_H
#include <stdint.h>
namespace leveldb {
// A thread local context for gathering performance counter efficiently
// and transparently.
struct PerfContext {
void Reset(); // reset all performance counters to zero
uint64_t user_key_comparison_count; // total number of user key comparisons
};
extern __thread PerfContext perf_context;
}
#endif
@@ -12,15 +12,15 @@
// non-const method, all threads accessing the same Slice must use
// external synchronization.
#ifndef STORAGE_ROCKSDB_INCLUDE_SLICE_H_
#define STORAGE_ROCKSDB_INCLUDE_SLICE_H_
#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_
#define STORAGE_LEVELDB_INCLUDE_SLICE_H_
#include <assert.h>
#include <stddef.h>
#include <string.h>
#include <string>
namespace rocksdb {
namespace leveldb {
class Slice {
public:
@@ -31,11 +31,9 @@ class Slice {
Slice(const char* d, size_t n) : data_(d), size_(n) { }
// Create a slice that refers to the contents of "s"
/* implicit */
Slice(const std::string& s) : data_(s.data()), size_(s.size()) { }
// Create a slice that refers to s[0,strlen(s)-1]
/* implicit */
Slice(const char* s) : data_(s), size_(strlen(s)) { }
// Return a pointer to the beginning of the referenced data
@@ -117,8 +115,7 @@ inline int Slice::compare(const Slice& b) const {
return r;
}
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_SLICE_H_
#endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_
@@ -8,12 +8,12 @@
// define InDomain and InRange to determine which slices are in either
// of these sets respectively.
#ifndef STORAGE_ROCKSDB_INCLUDE_SLICE_TRANSFORM_H_
#define STORAGE_ROCKSDB_INCLUDE_SLICE_TRANSFORM_H_
#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
#define STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
#include <string>
namespace rocksdb {
namespace leveldb {
class Slice;
@@ -36,10 +36,6 @@ class SliceTransform {
extern const SliceTransform* NewFixedPrefixTransform(size_t prefix_len);
extern const SliceTransform* NewNoopTransform();
}
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_SLICE_TRANSFORM_H_
#endif // STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
@@ -2,8 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#ifndef STORAGE_ROCKSDB_INCLUDE_STATISTICS_H_
#define STORAGE_ROCKSDB_INCLUDE_STATISTICS_H_
#ifndef STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
#define STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
#include <atomic>
#include <cassert>
@@ -13,7 +13,7 @@
#include <memory>
#include <vector>
namespace rocksdb {
namespace leveldb {
/**
* Keep adding ticker's here.
@@ -23,68 +23,48 @@ namespace rocksdb {
* And incrementing TICKER_ENUM_MAX.
*/
enum Tickers {
BLOCK_CACHE_MISS,
BLOCK_CACHE_HIT,
BLOOM_FILTER_USEFUL, // no. of times bloom filter has avoided file reads.
BLOCK_CACHE_MISS = 0,
BLOCK_CACHE_HIT = 1,
BLOOM_FILTER_USEFUL = 2, // no. of times bloom filter has avoided file reads.
/**
* COMPACTION_KEY_DROP_* count the reasons for key drop during compaction
* There are 3 reasons currently.
*/
COMPACTION_KEY_DROP_NEWER_ENTRY, // key was written with a newer value.
COMPACTION_KEY_DROP_OBSOLETE, // The key is obsolete.
COMPACTION_KEY_DROP_USER, // user compaction function has dropped the key.
COMPACTION_KEY_DROP_NEWER_ENTRY = 3, // key was written with a newer value.
COMPACTION_KEY_DROP_OBSOLETE = 4, // The key is obsolete.
COMPACTION_KEY_DROP_USER = 5, // user compaction function has dropped the key.
// Number of keys written to the database via the Put and Write call's
NUMBER_KEYS_WRITTEN,
NUMBER_KEYS_WRITTEN = 6,
// Number of Keys read,
NUMBER_KEYS_READ,
NUMBER_KEYS_READ = 7,
// Bytes written / read
BYTES_WRITTEN,
BYTES_READ,
NO_FILE_CLOSES,
NO_FILE_OPENS,
NO_FILE_ERRORS,
BYTES_WRITTEN = 8,
BYTES_READ = 9,
NO_FILE_CLOSES = 10,
NO_FILE_OPENS = 11,
NO_FILE_ERRORS = 12,
// Time system had to wait to do LO-L1 compactions
STALL_L0_SLOWDOWN_MICROS,
STALL_L0_SLOWDOWN_MICROS = 13,
// Time system had to wait to move memtable to L1.
STALL_MEMTABLE_COMPACTION_MICROS,
STALL_MEMTABLE_COMPACTION_MICROS = 14,
// write throttle because of too many files in L0
STALL_L0_NUM_FILES_MICROS,
RATE_LIMIT_DELAY_MILLIS,
STALL_L0_NUM_FILES_MICROS = 15,
RATE_LIMIT_DELAY_MILLIS = 16,
NO_ITERATORS, // number of iterators currently open
NO_ITERATORS = 17, // number of iterators currently open
// Number of MultiGet calls, keys read, and bytes read
NUMBER_MULTIGET_CALLS,
NUMBER_MULTIGET_KEYS_READ,
NUMBER_MULTIGET_BYTES_READ,
NUMBER_MULTIGET_CALLS = 18,
NUMBER_MULTIGET_KEYS_READ = 19,
NUMBER_MULTIGET_BYTES_READ = 20,
// Number of deletes records that were not required to be
// written to storage because key does not exist
NUMBER_FILTERED_DELETES,
NUMBER_MERGE_FAILURES,
SEQUENCE_NUMBER,
NUMBER_FILTERED_DELETES = 21,
NUMBER_MERGE_FAILURES = 22,
SEQUENCE_NUMBER = 23,
// number of times bloom was checked before creating iterator on a
// file, and the number of times the check was useful in avoiding
// iterator creation (and thus likely IOPs).
BLOOM_FILTER_PREFIX_CHECKED,
BLOOM_FILTER_PREFIX_USEFUL,
// Number of times we had to reseek inside an iteration to skip
// over large number of keys with same userkey.
NUMBER_OF_RESEEKS_IN_ITERATION,
// Record the number of calls to GetUpadtesSince. Useful to keep track of
// transaction log iterator refreshes
GET_UPDATES_SINCE_CALLS,
TICKER_ENUM_MAX
TICKER_ENUM_MAX = 24
};
// The order of items listed in Tickers should be the same as
// the order listed in TickersNameMap
const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
{ BLOCK_CACHE_MISS, "rocksdb.block.cache.miss" },
{ BLOCK_CACHE_HIT, "rocksdb.block.cache.hit" },
@@ -109,11 +89,7 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
{ NUMBER_MULTIGET_BYTES_READ, "rocksdb.number.multiget.bytes.read" },
{ NUMBER_FILTERED_DELETES, "rocksdb.number.deletes.filtered" },
{ NUMBER_MERGE_FAILURES, "rocksdb.number.merge.failures" },
{ SEQUENCE_NUMBER, "rocksdb.sequence.number" },
{ BLOOM_FILTER_PREFIX_CHECKED, "rocksdb.bloom.filter.prefix.checked" },
{ BLOOM_FILTER_PREFIX_USEFUL, "rocksdb.bloom.filter.prefix.useful" },
{ NUMBER_OF_RESEEKS_IN_ITERATION, "rocksdb.number.reseeks.iteration" },
{ GET_UPDATES_SINCE_CALLS, "rocksdb.getupdatessince.calls" }
{ SEQUENCE_NUMBER, "rocksdb.sequence.number" }
};
/**
@@ -124,27 +100,27 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
* And increment HISTOGRAM_ENUM_MAX
*/
enum Histograms {
DB_GET,
DB_WRITE,
COMPACTION_TIME,
TABLE_SYNC_MICROS,
COMPACTION_OUTFILE_SYNC_MICROS,
WAL_FILE_SYNC_MICROS,
MANIFEST_FILE_SYNC_MICROS,
DB_GET = 0,
DB_WRITE = 1,
COMPACTION_TIME = 2,
TABLE_SYNC_MICROS = 3,
COMPACTION_OUTFILE_SYNC_MICROS = 4,
WAL_FILE_SYNC_MICROS = 5,
MANIFEST_FILE_SYNC_MICROS = 6,
// TIME SPENT IN IO DURING TABLE OPEN
TABLE_OPEN_IO_MICROS,
DB_MULTIGET,
READ_BLOCK_COMPACTION_MICROS,
READ_BLOCK_GET_MICROS,
WRITE_RAW_BLOCK_MICROS,
TABLE_OPEN_IO_MICROS = 7,
DB_MULTIGET = 8,
READ_BLOCK_COMPACTION_MICROS = 9,
READ_BLOCK_GET_MICROS = 10,
WRITE_RAW_BLOCK_MICROS = 11,
STALL_L0_SLOWDOWN_COUNT,
STALL_MEMTABLE_COMPACTION_COUNT,
STALL_L0_NUM_FILES_COUNT,
HARD_RATE_LIMIT_DELAY_COUNT,
SOFT_RATE_LIMIT_DELAY_COUNT,
NUM_FILES_IN_SINGLE_COMPACTION,
HISTOGRAM_ENUM_MAX
STALL_L0_SLOWDOWN_COUNT = 12,
STALL_MEMTABLE_COMPACTION_COUNT = 13,
STALL_L0_NUM_FILES_COUNT = 14,
HARD_RATE_LIMIT_DELAY_COUNT = 15,
SOFT_RATE_LIMIT_DELAY_COUNT = 16,
NUM_FILES_IN_SINGLE_COMPACTION = 17,
HISTOGRAM_ENUM_MAX = 18
};
const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
@@ -258,8 +234,6 @@ inline void SetTickerCount(std::shared_ptr<Statistics> statistics,
}
}
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_STATISTICS_H_
#endif // STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
@@ -10,13 +10,13 @@
// non-const method, all threads accessing the same Status must use
// external synchronization.
#ifndef STORAGE_ROCKSDB_INCLUDE_STATUS_H_
#define STORAGE_ROCKSDB_INCLUDE_STATUS_H_
#ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_
#define STORAGE_LEVELDB_INCLUDE_STATUS_H_
#include <string>
#include "rocksdb/slice.h"
#include "leveldb/slice.h"
namespace rocksdb {
namespace leveldb {
class Status {
public:
@@ -50,9 +50,6 @@ class Status {
static Status MergeInProgress(const Slice& msg, const Slice& msg2 = Slice()) {
return Status(kMergeInProgress, msg, msg2);
}
static Status Incomplete(const Slice& msg, const Slice& msg2 = Slice()) {
return Status(kIncomplete, msg, msg2);
}
// Returns true iff the status indicates success.
bool ok() const { return (state_ == nullptr); }
@@ -75,9 +72,6 @@ class Status {
// Returns true iff the status indicates an MergeInProgress.
bool IsMergeInProgress() const { return code() == kMergeInProgress; }
// Returns true iff the status indicates Incomplete
bool IsIncomplete() const { return code() == kIncomplete; }
// Return a string representation of this status suitable for printing.
// Returns the string "OK" for success.
std::string ToString() const;
@@ -97,8 +91,7 @@ class Status {
kNotSupported = 3,
kInvalidArgument = 4,
kIOError = 5,
kMergeInProgress = 6,
kIncomplete = 7
kMergeInProgress = 6
};
Code code() const {
@@ -121,8 +114,6 @@ inline void Status::operator=(const Status& s) {
}
}
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_STATUS_H_
#endif // STORAGE_LEVELDB_INCLUDE_STATUS_H_
@@ -10,14 +10,14 @@
// non-const method, all threads accessing the same TableBuilder must use
// external synchronization.
#ifndef STORAGE_ROCKSDB_INCLUDE_TABLE_BUILDER_H_
#define STORAGE_ROCKSDB_INCLUDE_TABLE_BUILDER_H_
#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
#define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
#include <stdint.h>
#include "rocksdb/options.h"
#include "rocksdb/status.h"
#include "leveldb/options.h"
#include "leveldb/status.h"
namespace rocksdb {
namespace leveldb {
class BlockBuilder;
class BlockHandle;
@@ -90,8 +90,6 @@ class TableBuilder {
void operator=(const TableBuilder&);
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_TABLE_BUILDER_H_
#endif // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
@@ -1,12 +1,12 @@
// Copyright 2008-present Facebook. All Rights Reserved.
#ifndef STORAGE_ROCKSDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
#define STORAGE_ROCKSDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
#ifndef STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
#define STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
#include "rocksdb/status.h"
#include "rocksdb/types.h"
#include "rocksdb/write_batch.h"
#include "leveldb/status.h"
#include "leveldb/types.h"
#include "leveldb/write_batch.h"
namespace rocksdb {
namespace leveldb {
class LogFile;
typedef std::vector<std::unique_ptr<LogFile>> VectorLogPtr;
@@ -27,11 +27,8 @@ class LogFile {
LogFile() {}
virtual ~LogFile() {}
// Returns log file's pathname relative to the main db dir
// Eg. For a live-log-file = /000003.log
// For an archived-log-file = /archive/000003.log
virtual std::string PathName() const = 0;
// Returns log file's name excluding the db path
virtual std::string Filename() const = 0;
// Primary identifier for log file.
// This is directly proportional to creation time of the log file
@@ -76,8 +73,6 @@ class TransactionLogIterator {
// ONLY use if Valid() is true and status() is OK.
virtual BatchResult GetBatch() = 0;
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
#endif // STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
+14
View File
@@ -0,0 +1,14 @@
#ifndef STORAGE_LEVELDB_INCLUDE_TYPES_H_
#define STORAGE_LEVELDB_INCLUDE_TYPES_H_
#include <stdint.h>
namespace leveldb {
// Define all public custom types here.
// Represents a sequence number in a WAL file.
typedef uint64_t SequenceNumber;
} // namespace leveldb
#endif // STORAGE_LEVELDB_INCLUDE_TYPES_H_
@@ -11,10 +11,10 @@
#include <vector>
#include <stdint.h>
#include <climits>
#include "rocksdb/slice.h"
#include "rocksdb/statistics.h"
#include "leveldb/slice.h"
#include "leveldb/statistics.h"
namespace rocksdb {
namespace leveldb {
//
// Algorithm used to make a compaction request stop picking new files
@@ -36,21 +36,9 @@ class CompactionOptionsUniversal {
// The minimum number of files in a single compaction run. Default: 2
unsigned int min_merge_width;
// The maximum number of files in a single compaction run. Default: UINT_MAX
// The maximum number of files in a single compaction run. Default: INT_MAX
unsigned int max_merge_width;
// The size amplification is defined as the amount (in percentage) of
// additional storage needed to store a single byte of data in the database.
// For example, a size amplification of 2% means that a database that
// contains 100 bytes of user-data may occupy upto 102 bytes of
// physical storage. By this definition, a fully compacted database has
// a size amplification of 0%. Rocksdb uses the following heuristic
// to calculate size amplification: it assumes that all files excluding
// the earliest file contribute to the size amplification.
// Default: 200, which means that a 100 byte database could require upto
// 300 bytes of storage.
unsigned int max_size_amplification_percent;
// The algorithm used to stop picking files into a single compaction run
// Default: kCompactionStopStyleTotalSize
CompactionStopStyle stop_style;
@@ -60,13 +48,10 @@ class CompactionOptionsUniversal {
size_ratio(1),
min_merge_width(2),
max_merge_width(UINT_MAX),
max_size_amplification_percent(200),
stop_style(kCompactionStopStyleTotalSize) {
}
};
} // namespace rocksdb
#include "rocksdb/rocksdb_to_leveldb.h"
} // namespace leveldb
#endif // STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
@@ -18,13 +18,13 @@
// non-const method, all threads accessing the same WriteBatch must use
// external synchronization.
#ifndef STORAGE_ROCKSDB_INCLUDE_WRITE_BATCH_H_
#define STORAGE_ROCKSDB_INCLUDE_WRITE_BATCH_H_
#ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
#define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
#include <string>
#include "rocksdb/status.h"
#include "leveldb/status.h"
namespace rocksdb {
namespace leveldb {
class Slice;
@@ -43,13 +43,12 @@ class WriteBatch {
// If the database contains a mapping for "key", erase it. Else do nothing.
void Delete(const Slice& key);
// Append a blob of arbitrary size to the records in this batch. The blob will
// be stored in the transaction log but not in any other file. In particular,
// it will not be persisted to the SST files. When iterating over this
// Append a blob of arbitrary to the records in this batch. The blob will be
// stored in the transaction log but not in any other file. In particular, it
// will not be persisted to the SST files. When iterating over this
// WriteBatch, WriteBatch::Handler::LogData will be called with the contents
// of the blob as it is encountered. Blobs, puts, deletes, and merges will be
// encountered in the same order in thich they were inserted. The blob will
// NOT consume sequence number(s) and will NOT increase the count of the batch
// encountered in the same order in thich they were inserted.
//
// Example application: add timestamps to the transaction log for use in
// replication.
@@ -70,10 +69,6 @@ class WriteBatch {
// The default implementation of LogData does nothing.
virtual void LogData(const Slice& blob);
virtual void Delete(const Slice& key) = 0;
// Continue is called by WriteBatch::Iterate. If it returns false,
// iteration is halted. Otherwise, it continues iterating. The default
// implementation always returns true.
virtual bool Continue();
};
Status Iterate(Handler* handler) const;
@@ -94,8 +89,6 @@ class WriteBatch {
// Intentionally copyable
};
} // namespace rocksdb
} // namespace leveldb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_WRITE_BATCH_H_
#endif // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
-20
View File
@@ -1,20 +0,0 @@
// Copyright 2008-present Facebook. All Rights Reserved.
#ifndef STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
#define STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
#include "rocksdb/options.h"
namespace rocksdb {
class LDBTool {
public:
void Run(int argc, char** argv, Options = Options());
};
namespace leveldb = rocksdb;
} // namespace rocksdb
#include "rocksdb/rocksdb_to_leveldb.h"
#endif // STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
Loaded 100 of 245 files, more files were not shown because too many files have changed in this diff. Show more