mirror of
https://github.com/facebook/rocksdb.git
synced 2026-11-07 11:32:34 -05:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6cc7241206 |
No files matched your search
+8
-13
@@ -1,23 +1,18 @@
|
||||
build_config.mk
|
||||
|
||||
*.a
|
||||
*.arc
|
||||
*.d
|
||||
*.dylib*
|
||||
*.gcda
|
||||
*.gcno
|
||||
*.o
|
||||
*.dylib*
|
||||
*.so
|
||||
*.so.*
|
||||
*_test
|
||||
*_bench
|
||||
*_stress
|
||||
|
||||
*.arc
|
||||
*.d
|
||||
db_bench
|
||||
db_stress
|
||||
ldb
|
||||
leveldb_server
|
||||
leveldb_shell
|
||||
manifest_dump
|
||||
sst_dump
|
||||
util/build_version.cc
|
||||
build_tools/VALGRIND_LOGS/
|
||||
coverage/COVERAGE_REPORT
|
||||
util/build_version.cc.tmp
|
||||
.gdbhistory
|
||||
db_repl_stress
|
||||
@@ -1,11 +1,3 @@
|
||||
BSD License
|
||||
|
||||
For rocksdb software
|
||||
|
||||
Copyright (c) 2013, Facebook, Inc.
|
||||
All rights reserved.
|
||||
---------------------------------------------------------------------
|
||||
|
||||
Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
|
||||
@@ -14,13 +14,13 @@ OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
|
||||
#-----------------------------------------------
|
||||
|
||||
# detect what platform we're building on
|
||||
$(shell (export ROCKSDB_ROOT=$(CURDIR); $(CURDIR)/build_tools/build_detect_platform $(CURDIR)/build_config.mk))
|
||||
$(shell ./build_detect_platform build_config.mk)
|
||||
# this file is generated by the previous line to set build flags and sources
|
||||
include build_config.mk
|
||||
|
||||
WARNING_FLAGS = -Wall -Werror
|
||||
WARNING_FLAGS = -Wall -Werror -Wno-unused-parameter -Wno-sign-compare
|
||||
CFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
|
||||
CXXFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -std=gnu++0x -Woverloaded-virtual
|
||||
CXXFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -std=gnu++0x
|
||||
|
||||
LDFLAGS += $(PLATFORM_LDFLAGS)
|
||||
|
||||
@@ -31,78 +31,65 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
|
||||
TESTUTIL = ./util/testutil.o
|
||||
TESTHARNESS = ./util/testharness.o $(TESTUTIL)
|
||||
VALGRIND_ERROR = 2
|
||||
VALGRIND_DIR = build_tools/VALGRIND_LOGS
|
||||
VALGRIND_DIR = VALGRIND_LOGS
|
||||
VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
|
||||
VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
|
||||
|
||||
TESTS = \
|
||||
table_stats_collector_test \
|
||||
arena_test \
|
||||
auto_roll_logger_test \
|
||||
block_test \
|
||||
bloom_test \
|
||||
c_test \
|
||||
cache_test \
|
||||
coding_test \
|
||||
histogram_test \
|
||||
corruption_test \
|
||||
crc32c_test \
|
||||
db_test \
|
||||
dbformat_test \
|
||||
env_test \
|
||||
blob_store_test \
|
||||
filelock_test \
|
||||
filename_test \
|
||||
filter_block_test \
|
||||
histogram_test \
|
||||
log_test \
|
||||
manual_compaction_test \
|
||||
memenv_test \
|
||||
merge_test \
|
||||
redis_test \
|
||||
reduce_levels_test \
|
||||
simple_table_db_test \
|
||||
skiplist_test \
|
||||
stringappend_test \
|
||||
table_test \
|
||||
ttl_test \
|
||||
block_test \
|
||||
version_edit_test \
|
||||
version_set_test \
|
||||
write_batch_test\
|
||||
deletefile_test
|
||||
reduce_levels_test \
|
||||
write_batch_test \
|
||||
auto_roll_logger_test \
|
||||
filelock_test
|
||||
|
||||
|
||||
TOOLS = \
|
||||
manifest_dump \
|
||||
sst_dump \
|
||||
db_stress \
|
||||
ldb \
|
||||
db_repl_stress \
|
||||
blob_store_bench
|
||||
db_repl_stress
|
||||
|
||||
PROGRAMS = db_bench signal_test $(TESTS) $(TOOLS)
|
||||
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db table_reader_bench
|
||||
PROGRAMS = db_bench $(TESTS) $(TOOLS)
|
||||
BENCHMARKS = db_bench_sqlite3 db_bench_tree_db
|
||||
|
||||
# The library name is configurable since we are maintaining libraries of both
|
||||
# debug/release mode.
|
||||
LIBNAME = librocksdb
|
||||
LIBRARY = ${LIBNAME}.a
|
||||
LIBRARY = libleveldb.a
|
||||
MEMENVLIBRARY = libmemenv.a
|
||||
|
||||
default: all
|
||||
|
||||
#-----------------------------------------------
|
||||
# Create platform independent shared libraries.
|
||||
#-----------------------------------------------
|
||||
# Should we build shared libraries?
|
||||
ifneq ($(PLATFORM_SHARED_EXT),)
|
||||
|
||||
ifneq ($(PLATFORM_SHARED_VERSIONED),true)
|
||||
SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT)
|
||||
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
|
||||
SHARED2 = $(SHARED1)
|
||||
SHARED3 = $(SHARED1)
|
||||
SHARED = $(SHARED1)
|
||||
else
|
||||
# Update db.h if you change these.
|
||||
SHARED_MAJOR = 2
|
||||
SHARED_MINOR = 0
|
||||
SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT)
|
||||
SHARED_MAJOR = 1
|
||||
SHARED_MINOR = 5
|
||||
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
|
||||
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
|
||||
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
|
||||
SHARED = $(SHARED1) $(SHARED2) $(SHARED3)
|
||||
@@ -113,25 +100,15 @@ $(SHARED2): $(SHARED3)
|
||||
endif
|
||||
|
||||
$(SHARED3):
|
||||
$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(COVERAGEFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
|
||||
$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
|
||||
|
||||
endif # PLATFORM_SHARED_EXT
|
||||
|
||||
all: $(LIBRARY) $(PROGRAMS)
|
||||
|
||||
.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests \
|
||||
release tags valgrind_check whitebox_crash_test
|
||||
all: $(SHARED) $(LIBRARY) $(PROGRAMS)
|
||||
|
||||
release:
|
||||
$(MAKE) clean
|
||||
OPT=-DNDEBUG $(MAKE) -j32
|
||||
|
||||
coverage:
|
||||
$(MAKE) clean
|
||||
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) all check
|
||||
(cd coverage; ./coverage_test.sh)
|
||||
# Delete intermediate files
|
||||
find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
|
||||
make clean
|
||||
OPT=-DNDEBUG make -j32
|
||||
|
||||
check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests
|
||||
for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
|
||||
@@ -139,16 +116,7 @@ check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests
|
||||
ldb_tests: all $(PROGRAMS) $(TOOLS)
|
||||
python tools/ldb_test.py
|
||||
|
||||
crash_test: blackbox_crash_test whitebox_crash_test
|
||||
|
||||
blackbox_crash_test: db_stress
|
||||
python -u tools/db_crashtest.py
|
||||
|
||||
whitebox_crash_test: db_stress
|
||||
python -u tools/db_crashtest2.py
|
||||
|
||||
valgrind_check: all $(PROGRAMS) $(TESTS)
|
||||
mkdir -p $(VALGRIND_DIR)
|
||||
echo TESTS THAT HAVE VALGRIND ERRORS > $(VALGRIND_DIR)/valgrind_failed_tests; \
|
||||
echo TIMES in seconds TAKEN BY TESTS ON VALGRIND > $(VALGRIND_DIR)/valgrind_tests_times; \
|
||||
for t in $(filter-out skiplist_test,$(TESTS)); do \
|
||||
@@ -162,169 +130,119 @@ valgrind_check: all $(PROGRAMS) $(TESTS)
|
||||
done
|
||||
|
||||
clean:
|
||||
-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) build_config.mk
|
||||
-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk
|
||||
-rm -rf ios-x86/* ios-arm/*
|
||||
-find . -name "*.[od]" | xargs --no-run-if-empty rm
|
||||
-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
|
||||
tags:
|
||||
ctags * -R
|
||||
cscope -b `find . -name '*.cc'` `find . -name '*.h'`
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests and tools
|
||||
# ---------------------------------------------------------------------------
|
||||
$(LIBRARY): $(LIBOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(LIBOBJECTS)
|
||||
|
||||
|
||||
db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
db_stress: tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
db_repl_stress: tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
blob_store_bench: tools/blob_store_bench.o $(LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) tools/blob_store_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lsqlite3 $(COVERAGEFLAGS)
|
||||
$(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lsqlite3
|
||||
|
||||
db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
|
||||
$(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lkyotocabinet $(COVERAGEFLAGS)
|
||||
|
||||
signal_test: util/signal_test.o $(LIBOBJECTS)
|
||||
$(CXX) util/signal_test.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lkyotocabinet
|
||||
|
||||
arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
table_stats_collector_test: db/table_stats_collector_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/table_stats_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
blob_store_test: util/blob_store_test.o $(LIBOBJECTS) $(TESTHARNESS) $(TESTUTIL)
|
||||
$(CXX) util/blob_store_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
histogram_test: util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS)
|
||||
|
||||
corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
simple_table_db_test: db/simple_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/simple_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
table_reader_bench: table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
perf_context_test: db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
prefix_test: db/prefix_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/prefix_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
block_test: table/block_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) table/block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) table/block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
reduce_levels_test: tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
$(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
$(MEMENVLIBRARY) : $(MEMENVOBJECTS)
|
||||
rm -f $@
|
||||
$(AR) -rs $@ $(MEMENVOBJECTS)
|
||||
|
||||
memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
|
||||
$(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
manual_compaction_test: util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
leveldb_shell: tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o tools/shell/ShellContext.h tools/shell/ShellState.h tools/shell/DBClientProxy.h $(LIBOBJECTS)
|
||||
$(CXX) tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
rocksdb_shell: tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o tools/shell/ShellContext.h tools/shell/ShellState.h tools/shell/DBClientProxy.h $(LIBOBJECTS)
|
||||
$(CXX) tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
DBClientProxy_test: tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY)
|
||||
$(CXX) tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) $(EXEC_LDFLAGS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
DBClientProxy_test: tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY)
|
||||
$(CXX) tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) $(EXEC_LDFLAGS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
manifest_dump: tools/manifest_dump.o $(LIBOBJECTS)
|
||||
$(CXX) tools/manifest_dump.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
auto_roll_logger_test: util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
sst_dump: tools/sst_dump.o $(LIBOBJECTS)
|
||||
$(CXX) tools/sst_dump.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(CXX) tools/sst_dump.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
ldb: tools/ldb.o $(LIBOBJECTS)
|
||||
$(CXX) tools/ldb.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Platform-specific compilation
|
||||
# ---------------------------------------------------------------------------
|
||||
$(CXX) tools/ldb.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
|
||||
|
||||
ifeq ($(PLATFORM), IOS)
|
||||
# For iOS, create universal object files to be used on both the simulator and
|
||||
@@ -336,9 +254,9 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/versionCFBun
|
||||
|
||||
.cc.o:
|
||||
mkdir -p ios-x86/$(dir $@)
|
||||
$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ $(COVERAGEFLAGS)
|
||||
$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
|
||||
mkdir -p ios-arm/$(dir $@)
|
||||
$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ $(COVERAGEFLAGS)
|
||||
$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
|
||||
lipo ios-x86/$@ ios-arm/$@ -create -output $@
|
||||
|
||||
.c.o:
|
||||
@@ -350,26 +268,17 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/versionCFBun
|
||||
|
||||
else
|
||||
.cc.o:
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS)
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
|
||||
|
||||
.c.o:
|
||||
$(CC) $(CFLAGS) -c $< -o $@
|
||||
$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
|
||||
endif
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source files dependencies detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Add proper dependency support so changing a .h file forces a .cc file to
|
||||
# rebuild.
|
||||
|
||||
# The .d file indicates .cc file's dependencies on .h files. We generate such
|
||||
# dependency by g++'s -MM option, whose output is a make dependency rule.
|
||||
# The sed command makes sure the "target" file in the generated .d file has
|
||||
# the correct path prefix.
|
||||
%.d: %.cc
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $@
|
||||
@sed -i -e 's|.*:|$*.o:|' $@
|
||||
$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $*.d
|
||||
@cp -f $*.d $*.d.tmp
|
||||
@sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d
|
||||
@rm -f $*.d.tmp
|
||||
|
||||
DEPFILES = $(filter-out util/build_version.d,$(SOURCES:.cc=.d))
|
||||
|
||||
@@ -378,3 +287,4 @@ depend: $(DEPFILES)
|
||||
ifneq ($(MAKECMDGOALS),clean)
|
||||
-include $(DEPFILES)
|
||||
endif
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
Additional Grant of Patent Rights
|
||||
|
||||
“Software” means the rocksdb software distributed by Facebook, Inc.
|
||||
|
||||
Facebook hereby grants you a perpetual, worldwide, royalty-free,
|
||||
non-exclusive, irrevocable (subject to the termination provision below)
|
||||
license under any rights in any patent claims owned by Facebook, to make,
|
||||
have made, use, sell, offer to sell, import, and otherwise transfer the
|
||||
Software. For avoidance of doubt, no license is granted under Facebook’s
|
||||
rights in any patent claims that are infringed by (i) modifications to the
|
||||
Software made by you or a third party, or (ii) the Software in combination
|
||||
with any software or other technology provided by you or a third party.
|
||||
|
||||
The license granted hereunder will terminate, automatically and without
|
||||
notice, for anyone that makes any claim (including by filing any lawsuit,
|
||||
assertion or other action) alleging (a) direct, indirect, or contributory
|
||||
infringement or inducement to infringe any patent: (i) by Facebook or any
|
||||
of its subsidiaries or affiliates, whether or not such claim is related
|
||||
to the Software, (ii) by any party if such claim arises in whole or in
|
||||
part from any software, product or service of Facebook or any of its
|
||||
subsidiaries or affiliates, whether or not such claim is related to the
|
||||
Software, or (iii) by any party relating to the Software; or (b) that
|
||||
any right in any patent claim of Facebook is invalid or unenforceable.
|
||||
@@ -1,7 +1,5 @@
|
||||
rocksdb: A persistent key-value store for flash storage
|
||||
Authors: * The Facebook Database Engineering Team
|
||||
* Build on earlier work on leveldb by Sanjay Ghemawat
|
||||
(sanjay@google.com) and Jeff Dean (jeff@google.com)
|
||||
Authors: The Facebook Database Engineering Team
|
||||
|
||||
This code is a library that forms the core building block for a fast
|
||||
key value server, especially suited for storing data on flash drives.
|
||||
@@ -19,64 +17,45 @@ persistent key/value store.
|
||||
See doc/index.html for more explanation.
|
||||
See doc/impl.html for a brief overview of the implementation.
|
||||
|
||||
The public interface is in include/*. Callers should not include or
|
||||
The public interface is in include/*.h. Callers should not include or
|
||||
rely on the details of any other header files in this package. Those
|
||||
internal APIs may be changed without warning.
|
||||
|
||||
Guide to header files:
|
||||
|
||||
include/rocksdb/db.h
|
||||
include/db.h
|
||||
Main interface to the DB: Start here
|
||||
|
||||
include/rocksdb/options.h
|
||||
include/options.h
|
||||
Control over the behavior of an entire database, and also
|
||||
control over the behavior of individual reads and writes.
|
||||
|
||||
include/rocksdb/comparator.h
|
||||
include/comparator.h
|
||||
Abstraction for user-specified comparison function. If you want
|
||||
just bytewise comparison of keys, you can use the default comparator,
|
||||
but clients can write their own comparator implementations if they
|
||||
want custom ordering (e.g. to handle different character
|
||||
encodings, etc.)
|
||||
|
||||
include/rocksdb/iterator.h
|
||||
include/iterator.h
|
||||
Interface for iterating over data. You can get an iterator
|
||||
from a DB object.
|
||||
|
||||
include/rocksdb/write_batch.h
|
||||
include/write_batch.h
|
||||
Interface for atomically applying multiple updates to a database.
|
||||
|
||||
include/rocksdb/slice.h
|
||||
include/slice.h
|
||||
A simple module for maintaining a pointer and a length into some
|
||||
other byte array.
|
||||
|
||||
include/rocksdb/status.h
|
||||
include/status.h
|
||||
Status is returned from many of the public interfaces and is used
|
||||
to report success and various kinds of errors.
|
||||
|
||||
include/rocksdb/env.h
|
||||
include/env.h
|
||||
Abstraction of the OS environment. A posix implementation of
|
||||
this interface is in util/env_posix.cc
|
||||
|
||||
include/rocksdb/table_builder.h
|
||||
include/table.h
|
||||
include/table_builder.h
|
||||
Lower-level modules that most clients probably won't use directly
|
||||
|
||||
include/rocksdb/cache.h
|
||||
An API for the block cache.
|
||||
|
||||
include/rocksdb/compaction_filter.h
|
||||
An API for a application filter invoked on every compaction.
|
||||
|
||||
include/rocksdb/filter_policy.h
|
||||
An API for configuring a bloom filter.
|
||||
|
||||
include/rocksdb/memtablerep.h
|
||||
An API for implementing a memtable.
|
||||
|
||||
include/rocksdb/statistics.h
|
||||
An API to retrieve various database statistics.
|
||||
|
||||
include/rocksdb/transaction_log.h
|
||||
An API to retrieve transaction logs from a database.
|
||||
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
* Detailed instructions on how to compile using fbcode and jemalloc
|
||||
|
||||
* Latest release is 2.5.fb
|
||||
* Latest release is 1.5.8.fb
|
||||
@@ -0,0 +1,4 @@
|
||||
This directory stores the tests that failed valgrind and the times associated
|
||||
with the failed runs.
|
||||
"make valgrind_check" can be invoked to call valgrind on the rocksdb tests and
|
||||
generate files in this directory
|
||||
@@ -31,9 +31,9 @@ fi
|
||||
# Default to fbcode gcc on internal fb machines
|
||||
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
source $PWD/build_tools/fbcode.gcc471.sh
|
||||
source fbcode.gcc471.sh
|
||||
else
|
||||
source $PWD/build_tools/fbcode.clang31.sh
|
||||
source fbcode.clang31.sh
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -64,9 +64,6 @@ PLATFORM_SHARED_LDFLAGS="${EXEC_LDFLAGS_SHARED} -shared -Wl,-soname -Wl,"
|
||||
PLATFORM_SHARED_CFLAGS="-fPIC"
|
||||
PLATFORM_SHARED_VERSIONED=true
|
||||
|
||||
# generic port files (working on all platform by #ifdef) go directly in /port
|
||||
GENERIC_PORT_FILES=`find $ROCKSDB_ROOT/port -name '*.cc' | tr "\n" " "`
|
||||
|
||||
# On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
|
||||
case "$TARGET_OS" in
|
||||
Darwin)
|
||||
@@ -74,52 +71,52 @@ case "$TARGET_OS" in
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -DOS_MACOSX"
|
||||
PLATFORM_SHARED_EXT=dylib
|
||||
PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
|
||||
# PORT_FILES=port/darwin/darwin_specific.cc
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
Linux)
|
||||
PLATFORM=OS_LINUX
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX -fPIC"
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
|
||||
fi
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
|
||||
# PORT_FILES=port/linux/linux_specific.cc
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
SunOS)
|
||||
PLATFORM=OS_SOLARIS
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
|
||||
# PORT_FILES=port/sunos/sunos_specific.cc
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
FreeBSD)
|
||||
PLATFORM=OS_FREEBSD
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
|
||||
# PORT_FILES=port/freebsd/freebsd_specific.cc
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
NetBSD)
|
||||
PLATFORM=OS_NETBSD
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lgcc_s"
|
||||
# PORT_FILES=port/netbsd/netbsd_specific.cc
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
OpenBSD)
|
||||
PLATFORM=OS_OPENBSD
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
|
||||
# PORT_FILES=port/openbsd/openbsd_specific.cc
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
DragonFly)
|
||||
PLATFORM=OS_DRAGONFLYBSD
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_DRAGONFLYBSD"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
|
||||
# PORT_FILES=port/dragonfly/dragonfly_specific.cc
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
OS_ANDROID_CROSSCOMPILE)
|
||||
PLATFORM=OS_ANDROID
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DLEVELDB_PLATFORM_POSIX"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS " # All pthread features are in the Android C library
|
||||
# PORT_FILES=port/android/android.cc
|
||||
PORT_FILE=port/port_posix.cc
|
||||
CROSS_COMPILE=true
|
||||
;;
|
||||
*)
|
||||
@@ -127,24 +124,32 @@ case "$TARGET_OS" in
|
||||
exit 1
|
||||
esac
|
||||
|
||||
$PWD/build_tools/build_detect_version
|
||||
./build_detect_version
|
||||
|
||||
# We want to make a list of all cc files within util, db, table, and helpers
|
||||
# except for the test and benchmark files. By default, find will output a list
|
||||
# of all files matching either rule, so we need to append -print to make the
|
||||
# prune take effect.
|
||||
DIRS="util db table utilities"
|
||||
DIRS="util db table"
|
||||
if test "$USE_THRIFT"; then
|
||||
DIRS="$DIRS thrift/server_utils.cpp thrift/gen-cpp "
|
||||
THRIFTSERVER=leveldb_server
|
||||
fi
|
||||
|
||||
if test "$USE_SCRIBE"; then
|
||||
DIRS="$DIRS scribe "
|
||||
fi
|
||||
|
||||
set -f # temporarily disable globbing so that our patterns arent expanded
|
||||
PRUNE_TEST="-name *test*.cc -prune"
|
||||
PRUNE_BENCH="-name *_bench.cc -prune"
|
||||
PORTABLE_FILES=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
|
||||
PORTABLE_CPP=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
|
||||
PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
|
||||
PORTABLE_CPP=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
|
||||
set +f # re-enable globbing
|
||||
|
||||
# The sources consist of the portable files, plus the platform-specific port
|
||||
# file.
|
||||
echo "SOURCES=$PORTABLE_FILES $GENERIC_PORT_FILES $PORT_FILES" >> $OUTPUT
|
||||
echo "SOURCES=$PORTABLE_FILES $PORT_FILE" >> $OUTPUT
|
||||
echo "SOURCESCPP=$PORTABLE_CPP" >> $OUTPUT
|
||||
echo "MEMENV_SOURCES=helpers/memenv/memenv.cc" >> $OUTPUT
|
||||
|
||||
@@ -175,17 +180,6 @@ EOF
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS ${SNAPPY_LDFLAGS:-./snappy/libs/libsnappy.a}"
|
||||
fi
|
||||
|
||||
|
||||
# Test whether gflags library is installed
|
||||
# http://code.google.com/p/gflags/
|
||||
$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
||||
#include <gflags/gflags.h>
|
||||
int main() {}
|
||||
EOF
|
||||
if [ "$?" = 0 ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS"
|
||||
fi
|
||||
|
||||
# Test whether zlib library is installed
|
||||
$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
|
||||
#include <zlib.h>
|
||||
@@ -230,6 +224,14 @@ if test "$USE_HDFS"; then
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS"
|
||||
fi
|
||||
|
||||
# shall we build thrift server or scribe logger
|
||||
if test "$USE_THRIFT" || test "$USE_SCRIBE" ; then
|
||||
THRIFT_CCFLAGS=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp -I/usr/include -std=gnu++0x"
|
||||
THRIFT_LDFLAGS=" -lexample -lserver -lthrift_base -ltransport -lthrift_exception -lutil -L./thrift/libs "
|
||||
COMMON_FLAGS="$COMMON_FLAGS $THRIFT_CCFLAGS"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $THRIFT_LDFLAGS"
|
||||
fi
|
||||
|
||||
# if Intel SSE instruction set is supported, set USE_SSE=" -msse -msse4.2 "
|
||||
COMMON_FLAGS="$COMMON_FLAGS $USE_SSE"
|
||||
|
||||
@@ -249,4 +251,5 @@ echo "PLATFORM_SHARED_CFLAGS=$PLATFORM_SHARED_CFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_EXT=$PLATFORM_SHARED_EXT" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_LDFLAGS=$PLATFORM_SHARED_LDFLAGS" >> $OUTPUT
|
||||
echo "PLATFORM_SHARED_VERSIONED=$PLATFORM_SHARED_VERSIONED" >> $OUTPUT
|
||||
echo "THRIFTSERVER=$THRIFTSERVER" >> $OUTPUT
|
||||
echo "EXEC_LDFLAGS=$EXEC_LDFLAGS" >> $OUTPUT
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Record the version of the source that we are compiling.
|
||||
# We keep a record of the git revision in util/version.cc. This source file
|
||||
# is then built as a regular source file as part of the compilation process.
|
||||
# One can run "strings executable_filename | grep _build_" to find the version of
|
||||
# the source that we used to build the executable file.
|
||||
#
|
||||
|
||||
# create git version file
|
||||
VFILE=$(mktemp)
|
||||
trap "rm $VFILE" EXIT
|
||||
|
||||
# check to see if git is in the path
|
||||
which git > /dev/null
|
||||
|
||||
if [ "$?" = 0 ]; then
|
||||
env -i git rev-parse HEAD | awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";"} END {}' > ${VFILE}
|
||||
else
|
||||
echo "git not found"| awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";"} END {}' > ${VFILE}
|
||||
fi
|
||||
|
||||
date | awk 'BEGIN {} {print "const char * leveldb_build_git_datetime = \"leveldb_build_git_datetime:"$0"\";"} END {} ' >> ${VFILE}
|
||||
echo "const char * leveldb_build_compile_date = __DATE__;" >> ${VFILE}
|
||||
echo "const char * leveldb_build_compile_time = __TIME__;" >> ${VFILE}
|
||||
|
||||
OUTFILE=util/build_version.cc
|
||||
if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
|
||||
cp $VFILE $OUTFILE
|
||||
fi
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/bash -e
|
||||
function print_usage {
|
||||
echo "Usage: $0 [build | release | bump_version version ]"
|
||||
}
|
||||
|
||||
function check_env {
|
||||
if [ -z $LEVELDB_HOME ] ; then
|
||||
echo "Expect LEVELDB_HOME to be SET"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z $JAVA_HOME ] ; then
|
||||
export JAVA_HOME=/usr/local/jdk-6u14-64/
|
||||
echo "JAVA_HOME not set. Assuming JAVA_HOME=$JAVA_HOME"
|
||||
fi
|
||||
if [ -z $LEVELDBJNI_HOME ] ; then
|
||||
export LEVELDBJNI_HOME=$LEVELDB_HOME/java/leveldbjni/
|
||||
echo "LEVELDBJNI_HOME not set. Assuming LEVELDBJNI_HOME=$LEVELDBJNI_HOME"
|
||||
fi
|
||||
if [ -z $SNAPPY_HOME ] ; then
|
||||
export SNAPPY_HOME="/home/dhruba/snappy-1.0.5"
|
||||
echo "SNAPPY_HOME not set. Assuming SNAPPY_HOME=$SNAPPY_HOME"
|
||||
fi
|
||||
if [ -z $LEVELDB_PATCH ] ; then
|
||||
LEVELDB_PATCH=$LEVELDB_HOME/java/leveldbjni/db.h.patch
|
||||
echo "LEVELDB_PATCH not set. Assuming LEVELDB_PATCH=$LEVELDB_PATCH"
|
||||
fi
|
||||
}
|
||||
|
||||
function build {
|
||||
cd $LEVELDB_HOME
|
||||
git apply $LEVELDB_PATCH
|
||||
make clean libleveldb.a
|
||||
cd $LEVELDB_HOME/java/leveldb/leveldb-api
|
||||
mvn clean package
|
||||
cd $LEVELDBJNI_HOME
|
||||
mvn clean install -P linux64
|
||||
cd $LEVELDB_HOME
|
||||
git checkout $LEVELDB_HOME/include/leveldb/db.h
|
||||
}
|
||||
|
||||
function release {
|
||||
cd $LEVELDB_HOME
|
||||
git apply $LEVELDB_PATCH
|
||||
make clean libleveldb.a
|
||||
cd $LEVELDB_HOME/java/leveldb/leveldb-api
|
||||
mvn clean package
|
||||
cd $LEVELDBJNI_HOME
|
||||
mvn deploy -P linux64 -DskipTests
|
||||
cd $LEVELDB_HOME
|
||||
git checkout $LEVELDB_HOME/include/leveldb/db.h
|
||||
}
|
||||
|
||||
CMD=$1
|
||||
if [ -z $CMD ]; then
|
||||
print_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$CMD" in
|
||||
build)
|
||||
check_env
|
||||
build
|
||||
;;
|
||||
bump_version)
|
||||
if [ -z $2 ] ; then
|
||||
echo "bump_version requires a version.no parameter at the end"
|
||||
exit 1
|
||||
fi
|
||||
check_env
|
||||
VERSION=$2
|
||||
pushd $LEVELDBJNI_HOME
|
||||
mvn versions:set -DnewVersion="$VERSION"
|
||||
popd
|
||||
echo "VERSION SET TO $VERSION"
|
||||
;;
|
||||
release)
|
||||
check_env
|
||||
pushd $LEVELDBJNI_HOME
|
||||
release
|
||||
popd
|
||||
;;
|
||||
*)
|
||||
print_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@@ -1,42 +0,0 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Record the version of the source that we are compiling.
|
||||
# We keep a record of the git revision in util/version.cc. This source file
|
||||
# is then built as a regular source file as part of the compilation process.
|
||||
# One can run "strings executable_filename | grep _build_" to find the version of
|
||||
# the source that we used to build the executable file.
|
||||
#
|
||||
|
||||
# create git version file
|
||||
VFILE=$ROCKSDB_ROOT/util/build_version.cc.tmp
|
||||
trap "rm $VFILE" EXIT
|
||||
|
||||
# check to see if git is in the path
|
||||
which git > /dev/null
|
||||
|
||||
if [ "$?" = 0 ]; then
|
||||
env -i git rev-parse HEAD |
|
||||
awk '
|
||||
BEGIN {
|
||||
print "#include \"build_version.h\"\n"
|
||||
}
|
||||
{ print "const char* rocksdb_build_git_sha = \"rocksdb_build_git_sha:" $0"\";" }
|
||||
' > ${VFILE}
|
||||
else
|
||||
echo "git not found" |
|
||||
awk '
|
||||
BEGIN {
|
||||
print "#include \"build_version.h\""
|
||||
}
|
||||
{ print "const char* rocksdb_build_git_sha = \"rocksdb_build_git_sha:git not found\";" }
|
||||
' > ${VFILE}
|
||||
fi
|
||||
|
||||
echo "const char* rocksdb_build_git_datetime = \"rocksdb_build_git_datetime:$(date)\";" >> ${VFILE}
|
||||
echo "const char* rocksdb_build_compile_date = __DATE__;" >> ${VFILE}
|
||||
echo "const char* rocksdb_build_compile_time = __TIME__;" >> ${VFILE}
|
||||
|
||||
OUTFILE=$ROCKSDB_ROOT/util/build_version.cc
|
||||
if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
|
||||
cp $VFILE $OUTFILE
|
||||
fi
|
||||
@@ -1,73 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Exit on error.
|
||||
set -e
|
||||
|
||||
if [ -n "$USE_CLANG" ]; then
|
||||
echo "Error: Coverage test is supported only for gcc."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ROOT=".."
|
||||
# Fetch right version of gcov
|
||||
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
|
||||
source $ROOT/build_tools/fbcode.gcc471.sh
|
||||
GCOV=$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1/cc6c9dc/bin/gcov
|
||||
else
|
||||
GCOV=$(which gcov)
|
||||
fi
|
||||
|
||||
COVERAGE_DIR="$PWD/COVERAGE_REPORT"
|
||||
mkdir -p $COVERAGE_DIR
|
||||
|
||||
# Find all gcno files to generate the coverage report
|
||||
|
||||
GCNO_FILES=`find $ROOT -name "*.gcno"`
|
||||
$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
|
||||
# Parse the raw gcov report to more human readable form.
|
||||
python $ROOT/coverage/parse_gcov_output.py |
|
||||
# Write the output to both stdout and report file.
|
||||
tee $COVERAGE_DIR/coverage_report_all.txt &&
|
||||
echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n"
|
||||
|
||||
# TODO: we also need to get the files of the latest commits.
|
||||
# Get the most recently committed files.
|
||||
LATEST_FILES=`
|
||||
git show --pretty="format:" --name-only HEAD |
|
||||
grep -v "^$" |
|
||||
paste -s -d,`
|
||||
RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt
|
||||
|
||||
echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT
|
||||
$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
|
||||
python $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
|
||||
tee -a $RECENT_REPORT &&
|
||||
echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
|
||||
|
||||
# Generate the html report. If we cannot find lcov in this machine, we'll simply
|
||||
# skip this step.
|
||||
echo "Generating the html coverage report..."
|
||||
|
||||
LCOV=$(which lcov || true 2>/dev/null)
|
||||
if [ -z $LCOV ]
|
||||
then
|
||||
echo "Skip: Cannot find lcov to generate the html report."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
LCOV_VERSION=$(lcov -v | grep 1.1 || true)
|
||||
if [ $LCOV_VERSION ]
|
||||
then
|
||||
echo "Not supported lcov version. Expect lcov 1.1."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
(cd $ROOT; lcov --no-external \
|
||||
--capture \
|
||||
--directory $PWD \
|
||||
--gcov-tool $GCOV \
|
||||
--output-file $COVERAGE_DIR/coverage.info)
|
||||
|
||||
genhtml $COVERAGE_DIR/coverage.info -o $COVERAGE_DIR
|
||||
|
||||
echo "HTML Coverage report is generated in $COVERAGE_DIR"
|
||||
@@ -1,118 +0,0 @@
|
||||
import optparse
|
||||
import re
|
||||
import sys
|
||||
|
||||
from optparse import OptionParser
|
||||
|
||||
# the gcov report follows certain pattern. Each file will have two lines
|
||||
# of report, from which we can extract the file name, total lines and coverage
|
||||
# percentage.
|
||||
def parse_gcov_report(gcov_input):
|
||||
per_file_coverage = {}
|
||||
total_coverage = None
|
||||
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
|
||||
# --First line of the coverage report (with file name in it)?
|
||||
match_obj = re.match("^File '(.*)'$", line)
|
||||
if match_obj:
|
||||
# fetch the file name from the first line of the report.
|
||||
current_file = match_obj.group(1)
|
||||
continue
|
||||
|
||||
# -- Second line of the file report (with coverage percentage)
|
||||
match_obj = re.match("^Lines executed:(.*)% of (.*)", line)
|
||||
|
||||
if match_obj:
|
||||
coverage = float(match_obj.group(1))
|
||||
lines = int(match_obj.group(2))
|
||||
|
||||
if current_file is not None:
|
||||
per_file_coverage[current_file] = (coverage, lines)
|
||||
current_file = None
|
||||
else:
|
||||
# If current_file is not set, we reach the last line of report,
|
||||
# which contains the summarized coverage percentage.
|
||||
total_coverage = (coverage, lines)
|
||||
continue
|
||||
|
||||
# If the line's pattern doesn't fall into the above categories. We
|
||||
# can simply ignore them since they're either empty line or doesn't
|
||||
# find executable lines of the given file.
|
||||
current_file = None
|
||||
|
||||
return per_file_coverage, total_coverage
|
||||
|
||||
def get_option_parser():
|
||||
usage = "Parse the gcov output and generate more human-readable code " +\
|
||||
"coverage report."
|
||||
parser = OptionParser(usage)
|
||||
|
||||
parser.add_option(
|
||||
"--interested-files", "-i",
|
||||
dest="filenames",
|
||||
help="Comma separated files names. if specified, we will display " +
|
||||
"the coverage report only for interested source files. " +
|
||||
"Otherwise we will display the coverage report for all " +
|
||||
"source files."
|
||||
)
|
||||
return parser
|
||||
|
||||
def display_file_coverage(per_file_coverage, total_coverage):
|
||||
# To print out auto-adjustable column, we need to know the longest
|
||||
# length of file names.
|
||||
max_file_name_length = max(
|
||||
len(fname) for fname in per_file_coverage.keys()
|
||||
)
|
||||
|
||||
# -- Print header
|
||||
# size of separator is determined by 3 column sizes:
|
||||
# file name, coverage percentage and lines.
|
||||
header_template = \
|
||||
"%" + str(max_file_name_length) + "s\t%s\t%s"
|
||||
separator = "-" * (max_file_name_length + 10 + 20)
|
||||
print header_template % ("Filename", "Coverage", "Lines")
|
||||
print separator
|
||||
|
||||
# -- Print body
|
||||
# template for printing coverage report for each file.
|
||||
record_template = "%" + str(max_file_name_length) + "s\t%5.2f%%\t%10d"
|
||||
|
||||
for fname, coverage_info in per_file_coverage.items():
|
||||
coverage, lines = coverage_info
|
||||
print record_template % (fname, coverage, lines)
|
||||
|
||||
# -- Print footer
|
||||
if total_coverage:
|
||||
print separator
|
||||
print record_template % ("Total", total_coverage[0], total_coverage[1])
|
||||
|
||||
def report_coverage():
|
||||
parser = get_option_parser()
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
interested_files = set()
|
||||
if options.filenames is not None:
|
||||
interested_files = set(f.strip() for f in options.filenames.split(','))
|
||||
|
||||
# To make things simple, right now we only read gcov report from the input
|
||||
per_file_coverage, total_coverage = parse_gcov_report(sys.stdin)
|
||||
|
||||
# Check if we need to display coverage info for interested files.
|
||||
if len(interested_files):
|
||||
per_file_coverage = dict(
|
||||
(fname, per_file_coverage[fname]) for fname in interested_files
|
||||
if fname in per_file_coverage
|
||||
)
|
||||
# If we only interested in several files, it makes no sense to report
|
||||
# the total_coverage
|
||||
total_coverage = None
|
||||
|
||||
if not len(per_file_coverage):
|
||||
print >> sys.stderr, "Cannot find coverage info for the given files."
|
||||
return
|
||||
display_file_coverage(per_file_coverage, total_coverage)
|
||||
|
||||
if __name__ == "__main__":
|
||||
report_coverage()
|
||||
+28
-113
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -11,41 +6,25 @@
|
||||
|
||||
#include "db/filename.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/merge_helper.h"
|
||||
#include "db/table_cache.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/iterator.h"
|
||||
#include "rocksdb/options.h"
|
||||
#include "table/block_based_table_builder.h"
|
||||
#include "util/stop_watch.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class TableFactory;
|
||||
|
||||
TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
|
||||
CompressionType compression_type) {
|
||||
return options.table_factory->GetTableBuilder(options, file,
|
||||
compression_type);
|
||||
}
|
||||
namespace leveldb {
|
||||
|
||||
Status BuildTable(const std::string& dbname,
|
||||
Env* env,
|
||||
const Options& options,
|
||||
const EnvOptions& soptions,
|
||||
TableCache* table_cache,
|
||||
Iterator* iter,
|
||||
FileMetaData* meta,
|
||||
const Comparator* user_comparator,
|
||||
const SequenceNumber newest_snapshot,
|
||||
const SequenceNumber earliest_seqno_in_memtable,
|
||||
const bool enable_compression) {
|
||||
const SequenceNumber earliest_seqno_in_memtable) {
|
||||
Status s;
|
||||
meta->file_size = 0;
|
||||
meta->smallest_seqno = meta->largest_seqno = 0;
|
||||
iter->SeekToFirst();
|
||||
|
||||
// If the sequence number of the smallest entry in the memtable is
|
||||
@@ -59,117 +38,56 @@ Status BuildTable(const std::string& dbname,
|
||||
std::string fname = TableFileName(dbname, meta->number);
|
||||
if (iter->Valid()) {
|
||||
unique_ptr<WritableFile> file;
|
||||
s = env->NewWritableFile(fname, &file, soptions);
|
||||
s = env->NewWritableFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
TableBuilder* builder = GetTableBuilder(options, file.get(),
|
||||
options.compression);
|
||||
TableBuilder* builder = new TableBuilder(options, file.get(), 0);
|
||||
|
||||
// the first key is the smallest key
|
||||
Slice key = iter->key();
|
||||
meta->smallest.DecodeFrom(key);
|
||||
meta->smallest_seqno = GetInternalKeySeqno(key);
|
||||
meta->largest_seqno = meta->smallest_seqno;
|
||||
|
||||
MergeHelper merge(user_comparator, options.merge_operator.get(),
|
||||
options.info_log.get(),
|
||||
true /* internal key corruption is not ok */);
|
||||
|
||||
if (purge) {
|
||||
// Ugly walkaround to avoid compiler error for release build
|
||||
bool ok __attribute__((unused)) = true;
|
||||
|
||||
// Will write to builder if current key != prev key
|
||||
ParsedInternalKey prev_ikey;
|
||||
std::string prev_value;
|
||||
std::string prev_key;
|
||||
bool is_first_key = true; // Also write if this is the very first key
|
||||
|
||||
while (iter->Valid()) {
|
||||
bool iterator_at_next = false;
|
||||
// store first key-value
|
||||
prev_key.assign(key.data(), key.size());
|
||||
prev_value.assign(iter->value().data(), iter->value().size());
|
||||
ParseInternalKey(Slice(prev_key), &prev_ikey);
|
||||
assert(prev_ikey.sequence >= earliest_seqno_in_memtable);
|
||||
|
||||
// Get current key
|
||||
for (iter->Next(); iter->Valid(); iter->Next()) {
|
||||
ParsedInternalKey this_ikey;
|
||||
Slice key = iter->key();
|
||||
Slice value = iter->value();
|
||||
|
||||
// In-memory key corruption is not ok;
|
||||
// TODO: find a clean way to treat in memory key corruption
|
||||
ok = ParseInternalKey(key, &this_ikey);
|
||||
assert(ok);
|
||||
ParseInternalKey(key, &this_ikey);
|
||||
assert(this_ikey.sequence >= earliest_seqno_in_memtable);
|
||||
|
||||
// If the key is the same as the previous key (and it is not the
|
||||
// first key), then we skip it, since it is an older version.
|
||||
// Otherwise we output the key and mark it as the "new" previous key.
|
||||
if (!is_first_key && !user_comparator->Compare(prev_ikey.user_key,
|
||||
this_ikey.user_key)) {
|
||||
if (user_comparator->Compare(prev_ikey.user_key, this_ikey.user_key)) {
|
||||
// This key is different from previous key.
|
||||
// Output prev key and remember current key
|
||||
builder->Add(Slice(prev_key), Slice(prev_value));
|
||||
prev_key.assign(key.data(), key.size());
|
||||
prev_value.assign(iter->value().data(), iter->value().size());
|
||||
ParseInternalKey(Slice(prev_key), &prev_ikey);
|
||||
} else {
|
||||
// seqno within the same key are in decreasing order
|
||||
assert(this_ikey.sequence < prev_ikey.sequence);
|
||||
} else {
|
||||
is_first_key = false;
|
||||
|
||||
if (this_ikey.type == kTypeMerge) {
|
||||
// Handle merge-type keys using the MergeHelper
|
||||
merge.MergeUntil(iter, 0 /* don't worry about snapshot */);
|
||||
iterator_at_next = true;
|
||||
if (merge.IsSuccess()) {
|
||||
// Merge completed correctly.
|
||||
// Add the resulting merge key/value and continue to next
|
||||
builder->Add(merge.key(), merge.value());
|
||||
prev_key.assign(merge.key().data(), merge.key().size());
|
||||
ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
|
||||
assert(ok);
|
||||
} else {
|
||||
// Merge did not find a Put/Delete.
|
||||
// Can not compact these merges into a kValueType.
|
||||
// Write them out one-by-one. (Proceed back() to front())
|
||||
const std::deque<std::string>& keys = merge.keys();
|
||||
const std::deque<std::string>& values = merge.values();
|
||||
assert(keys.size() == values.size() && keys.size() >= 1);
|
||||
std::deque<std::string>::const_reverse_iterator key_iter;
|
||||
std::deque<std::string>::const_reverse_iterator value_iter;
|
||||
for (key_iter=keys.rbegin(), value_iter = values.rbegin();
|
||||
key_iter != keys.rend() && value_iter != values.rend();
|
||||
++key_iter, ++value_iter) {
|
||||
|
||||
builder->Add(Slice(*key_iter), Slice(*value_iter));
|
||||
}
|
||||
|
||||
// Sanity check. Both iterators should end at the same time
|
||||
assert(key_iter == keys.rend() && value_iter == values.rend());
|
||||
|
||||
prev_key.assign(keys.front());
|
||||
ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
|
||||
assert(ok);
|
||||
}
|
||||
} else {
|
||||
// Handle Put/Delete-type keys by simply writing them
|
||||
builder->Add(key, value);
|
||||
prev_key.assign(key.data(), key.size());
|
||||
ok = ParseInternalKey(Slice(prev_key), &prev_ikey);
|
||||
assert(ok);
|
||||
}
|
||||
// This key is an earlier version of the same key in prev_key.
|
||||
// Skip current key.
|
||||
}
|
||||
|
||||
if (!iterator_at_next) iter->Next();
|
||||
}
|
||||
|
||||
// The last key is the largest key
|
||||
// output last key
|
||||
builder->Add(Slice(prev_key), Slice(prev_value));
|
||||
meta->largest.DecodeFrom(Slice(prev_key));
|
||||
SequenceNumber seqno = GetInternalKeySeqno(Slice(prev_key));
|
||||
meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
|
||||
meta->largest_seqno = std::max(meta->largest_seqno, seqno);
|
||||
|
||||
} else {
|
||||
for (; iter->Valid(); iter->Next()) {
|
||||
Slice key = iter->key();
|
||||
meta->largest.DecodeFrom(key);
|
||||
builder->Add(key, iter->value());
|
||||
SequenceNumber seqno = GetInternalKeySeqno(key);
|
||||
meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
|
||||
meta->largest_seqno = std::max(meta->largest_seqno, seqno);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,10 +106,8 @@ Status BuildTable(const std::string& dbname,
|
||||
// Finish and check for file errors
|
||||
if (s.ok() && !options.disableDataSync) {
|
||||
if (options.use_fsync) {
|
||||
StopWatch sw(env, options.statistics, TABLE_SYNC_MICROS);
|
||||
s = file->Fsync();
|
||||
} else {
|
||||
StopWatch sw(env, options.statistics, TABLE_SYNC_MICROS);
|
||||
s = file->Sync();
|
||||
}
|
||||
}
|
||||
@@ -202,7 +118,6 @@ Status BuildTable(const std::string& dbname,
|
||||
if (s.ok()) {
|
||||
// Verify that the table is usable
|
||||
Iterator* it = table_cache->NewIterator(ReadOptions(),
|
||||
soptions,
|
||||
meta->number,
|
||||
meta->file_size);
|
||||
s = it->status();
|
||||
@@ -223,4 +138,4 @@ Status BuildTable(const std::string& dbname,
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+12
-21
@@ -1,32 +1,23 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#pragma once
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "rocksdb/types.h"
|
||||
#include "rocksdb/options.h"
|
||||
|
||||
namespace rocksdb {
|
||||
#ifndef STORAGE_LEVELDB_DB_BUILDER_H_
|
||||
#define STORAGE_LEVELDB_DB_BUILDER_H_
|
||||
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/types.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
struct Options;
|
||||
struct FileMetaData;
|
||||
|
||||
class Env;
|
||||
class EnvOptions;
|
||||
class Iterator;
|
||||
class TableCache;
|
||||
class VersionEdit;
|
||||
class TableBuilder;
|
||||
class WritableFile;
|
||||
|
||||
|
||||
extern TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
|
||||
CompressionType compression_type);
|
||||
|
||||
// Build a Table file from the contents of *iter. The generated file
|
||||
// will be named according to meta->number. On success, the rest of
|
||||
@@ -36,13 +27,13 @@ extern TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
|
||||
extern Status BuildTable(const std::string& dbname,
|
||||
Env* env,
|
||||
const Options& options,
|
||||
const EnvOptions& soptions,
|
||||
TableCache* table_cache,
|
||||
Iterator* iter,
|
||||
FileMetaData* meta,
|
||||
const Comparator* user_comparator,
|
||||
const SequenceNumber newest_snapshot,
|
||||
const SequenceNumber earliest_seqno_in_memtable,
|
||||
const bool enable_compression);
|
||||
const SequenceNumber earliest_seqno_in_memtable);
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_BUILDER_H_
|
||||
@@ -1,48 +1,43 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "rocksdb/c.h"
|
||||
#include "leveldb/c.h"
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include "rocksdb/cache.h"
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/filter_policy.h"
|
||||
#include "rocksdb/iterator.h"
|
||||
#include "rocksdb/options.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "rocksdb/write_batch.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
|
||||
using rocksdb::Cache;
|
||||
using rocksdb::Comparator;
|
||||
using rocksdb::CompressionType;
|
||||
using rocksdb::DB;
|
||||
using rocksdb::Env;
|
||||
using rocksdb::FileLock;
|
||||
using rocksdb::FilterPolicy;
|
||||
using rocksdb::Iterator;
|
||||
using rocksdb::Logger;
|
||||
using rocksdb::NewBloomFilterPolicy;
|
||||
using rocksdb::NewLRUCache;
|
||||
using rocksdb::Options;
|
||||
using rocksdb::RandomAccessFile;
|
||||
using rocksdb::Range;
|
||||
using rocksdb::ReadOptions;
|
||||
using rocksdb::SequentialFile;
|
||||
using rocksdb::Slice;
|
||||
using rocksdb::Snapshot;
|
||||
using rocksdb::Status;
|
||||
using rocksdb::WritableFile;
|
||||
using rocksdb::WriteBatch;
|
||||
using rocksdb::WriteOptions;
|
||||
using leveldb::Cache;
|
||||
using leveldb::Comparator;
|
||||
using leveldb::CompressionType;
|
||||
using leveldb::DB;
|
||||
using leveldb::Env;
|
||||
using leveldb::FileLock;
|
||||
using leveldb::FilterPolicy;
|
||||
using leveldb::Iterator;
|
||||
using leveldb::Logger;
|
||||
using leveldb::NewBloomFilterPolicy;
|
||||
using leveldb::NewLRUCache;
|
||||
using leveldb::Options;
|
||||
using leveldb::RandomAccessFile;
|
||||
using leveldb::Range;
|
||||
using leveldb::ReadOptions;
|
||||
using leveldb::SequentialFile;
|
||||
using leveldb::Slice;
|
||||
using leveldb::Snapshot;
|
||||
using leveldb::Status;
|
||||
using leveldb::WritableFile;
|
||||
using leveldb::WriteBatch;
|
||||
using leveldb::WriteOptions;
|
||||
|
||||
using std::shared_ptr;
|
||||
|
||||
@@ -554,11 +549,6 @@ void leveldb_options_set_WAL_ttl_seconds(leveldb_options_t* opt, uint64_t ttl) {
|
||||
opt->rep.WAL_ttl_seconds = ttl;
|
||||
}
|
||||
|
||||
void leveldb_options_set_WAL_size_limit_MB(
|
||||
leveldb_options_t* opt, uint64_t limit) {
|
||||
opt->rep.WAL_size_limit_MB = limit;
|
||||
}
|
||||
|
||||
leveldb_comparator_t* leveldb_comparator_create(
|
||||
void* state,
|
||||
void (*destructor)(void*),
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@
|
||||
Use of this source code is governed by a BSD-style license that can be
|
||||
found in the LICENSE file. See the AUTHORS file for names of contributors. */
|
||||
|
||||
#include "rocksdb/c.h"
|
||||
#include "leveldb/c.h"
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdio.h>
|
||||
@@ -296,7 +296,7 @@ int main(int argc, char** argv) {
|
||||
{
|
||||
char* prop = leveldb_property_value(db, "nosuchprop");
|
||||
CheckCondition(prop == NULL);
|
||||
prop = leveldb_property_value(db, "rocksdb.stats");
|
||||
prop = leveldb_property_value(db, "leveldb.stats");
|
||||
CheckCondition(prop != NULL);
|
||||
Free(&prop);
|
||||
}
|
||||
|
||||
+20
-40
@@ -1,22 +1,17 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "leveldb/db.h"
|
||||
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include "rocksdb/cache.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "rocksdb/write_batch.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/log_format.h"
|
||||
@@ -25,7 +20,7 @@
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
static const int kValueSize = 1000;
|
||||
|
||||
@@ -45,7 +40,6 @@ class CorruptionTest {
|
||||
|
||||
db_ = nullptr;
|
||||
options_.create_if_missing = true;
|
||||
options_.block_size_deviation = 0; // make unit test pass for now
|
||||
Reopen();
|
||||
options_.create_if_missing = false;
|
||||
}
|
||||
@@ -61,8 +55,6 @@ class CorruptionTest {
|
||||
Options opt = (options ? *options : options_);
|
||||
opt.env = &env_;
|
||||
opt.block_cache = tiny_cache_;
|
||||
opt.block_size_deviation = 0;
|
||||
opt.arena_block_size = 4096;
|
||||
return DB::Open(opt, dbname_, &db_);
|
||||
}
|
||||
|
||||
@@ -73,7 +65,7 @@ class CorruptionTest {
|
||||
void RepairDB() {
|
||||
delete db_;
|
||||
db_ = nullptr;
|
||||
ASSERT_OK(::rocksdb::RepairDB(dbname_, options_));
|
||||
ASSERT_OK(::leveldb::RepairDB(dbname_, options_));
|
||||
}
|
||||
|
||||
void Build(int n) {
|
||||
@@ -221,15 +213,10 @@ TEST(CorruptionTest, NewFileErrorDuringWrite) {
|
||||
const int num = 3 + (Options().write_buffer_size / kValueSize);
|
||||
std::string value_storage;
|
||||
Status s;
|
||||
bool failed = false;
|
||||
for (int i = 0; i < num; i++) {
|
||||
for (int i = 0; s.ok() && i < num; i++) {
|
||||
WriteBatch batch;
|
||||
batch.Put("a", Value(100, &value_storage));
|
||||
s = db_->Write(WriteOptions(), &batch);
|
||||
if (!s.ok()) {
|
||||
failed = true;
|
||||
}
|
||||
ASSERT_TRUE(!failed || !s.ok());
|
||||
}
|
||||
ASSERT_TRUE(!s.ok());
|
||||
ASSERT_GE(env_.num_writable_file_errors_, 1);
|
||||
@@ -240,7 +227,7 @@ TEST(CorruptionTest, NewFileErrorDuringWrite) {
|
||||
TEST(CorruptionTest, TableFile) {
|
||||
Build(100);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_FlushMemTable();
|
||||
dbi->TEST_CompactMemTable();
|
||||
dbi->TEST_CompactRange(0, nullptr, nullptr);
|
||||
dbi->TEST_CompactRange(1, nullptr, nullptr);
|
||||
|
||||
@@ -251,7 +238,7 @@ TEST(CorruptionTest, TableFile) {
|
||||
TEST(CorruptionTest, TableFileIndexData) {
|
||||
Build(10000); // Enough to build multiple Tables
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_FlushMemTable();
|
||||
dbi->TEST_CompactMemTable();
|
||||
|
||||
Corrupt(kTableFile, -2000, 500);
|
||||
Reopen();
|
||||
@@ -289,7 +276,7 @@ TEST(CorruptionTest, SequenceNumberRecovery) {
|
||||
TEST(CorruptionTest, CorruptedDescriptor) {
|
||||
ASSERT_OK(db_->Put(WriteOptions(), "foo", "hello"));
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_FlushMemTable();
|
||||
dbi->TEST_CompactMemTable();
|
||||
dbi->TEST_CompactRange(0, nullptr, nullptr);
|
||||
|
||||
Corrupt(kDescriptorFile, 0, 1000);
|
||||
@@ -306,9 +293,9 @@ TEST(CorruptionTest, CorruptedDescriptor) {
|
||||
TEST(CorruptionTest, CompactionInputError) {
|
||||
Build(10);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_FlushMemTable();
|
||||
dbi->TEST_CompactMemTable();
|
||||
const int last = dbi->MaxMemCompactionLevel();
|
||||
ASSERT_EQ(1, Property("rocksdb.num-files-at-level" + NumberToString(last)));
|
||||
ASSERT_EQ(1, Property("leveldb.num-files-at-level" + NumberToString(last)));
|
||||
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
Check(9, 9);
|
||||
@@ -329,13 +316,12 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
|
||||
for (int level = 1; level < dbi->NumberLevels(); level++) {
|
||||
dbi->Put(WriteOptions(), "", "begin");
|
||||
dbi->Put(WriteOptions(), "~", "end");
|
||||
dbi->TEST_FlushMemTable();
|
||||
dbi->TEST_CompactMemTable();
|
||||
}
|
||||
|
||||
Build(10);
|
||||
dbi->TEST_FlushMemTable();
|
||||
dbi->TEST_WaitForCompact();
|
||||
ASSERT_EQ(1, Property("rocksdb.num-files-at-level0"));
|
||||
dbi->TEST_CompactMemTable();
|
||||
ASSERT_EQ(1, Property("leveldb.num-files-at-level0"));
|
||||
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
Check(9, 9);
|
||||
@@ -343,14 +329,8 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
|
||||
// Write must eventually fail because of corrupted table
|
||||
Status s;
|
||||
std::string tmp1, tmp2;
|
||||
bool failed = false;
|
||||
for (int i = 0; i < 10000 && s.ok(); i++) {
|
||||
s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2));
|
||||
if (!s.ok()) {
|
||||
failed = true;
|
||||
}
|
||||
// if one write failed, every subsequent write must fail, too
|
||||
ASSERT_TRUE(!failed || !s.ok()) << "write did not fail in a corrupted db";
|
||||
}
|
||||
ASSERT_TRUE(!s.ok()) << "write did not fail in corrupted paranoid db";
|
||||
}
|
||||
@@ -358,7 +338,7 @@ TEST(CorruptionTest, CompactionInputErrorParanoid) {
|
||||
TEST(CorruptionTest, UnrelatedKeys) {
|
||||
Build(10);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
dbi->TEST_FlushMemTable();
|
||||
dbi->TEST_CompactMemTable();
|
||||
Corrupt(kTableFile, 100, 1);
|
||||
|
||||
std::string tmp1, tmp2;
|
||||
@@ -366,13 +346,13 @@ TEST(CorruptionTest, UnrelatedKeys) {
|
||||
std::string v;
|
||||
ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
|
||||
ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
|
||||
dbi->TEST_FlushMemTable();
|
||||
dbi->TEST_CompactMemTable();
|
||||
ASSERT_OK(db_->Get(ReadOptions(), Key(1000, &tmp1), &v));
|
||||
ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
+562
-1332
File diff suppressed because it is too large.
Load diff
+19
-46
@@ -1,53 +1,42 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2012 Facebook.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <stdint.h>
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include <string>
|
||||
#include <stdint.h>
|
||||
#include "db/version_set.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "port/port.h"
|
||||
#include "util/mutexlock.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
Status DBImpl::DisableFileDeletions() {
|
||||
MutexLock l(&mutex_);
|
||||
disable_delete_obsolete_files_ = true;
|
||||
Log(options_.info_log, "File Deletions Disabled");
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status DBImpl::EnableFileDeletions() {
|
||||
MutexLock l(&mutex_);
|
||||
disable_delete_obsolete_files_ = false;
|
||||
Log(options_.info_log, "File Deletions Enabled");
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
||||
uint64_t* manifest_file_size,
|
||||
bool flush_memtable) {
|
||||
uint64_t* manifest_file_size) {
|
||||
|
||||
*manifest_file_size = 0;
|
||||
|
||||
if (flush_memtable) {
|
||||
// flush all dirty data to disk.
|
||||
Status status = Flush(FlushOptions());
|
||||
if (!status.ok()) {
|
||||
Log(options_.info_log, "Cannot Flush data %s\n",
|
||||
status.ToString().c_str());
|
||||
return status;
|
||||
}
|
||||
// flush all dirty data to disk.
|
||||
Status status = Flush(FlushOptions());
|
||||
if (!status.ok()) {
|
||||
Log(options_.info_log, "Cannot Flush data %s\n",
|
||||
status.ToString().c_str());
|
||||
return status;
|
||||
}
|
||||
|
||||
MutexLock l(&mutex_);
|
||||
@@ -56,17 +45,18 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
||||
std::set<uint64_t> live;
|
||||
versions_->AddLiveFilesCurrentVersion(&live);
|
||||
|
||||
ret.clear();
|
||||
ret.reserve(live.size() + 2); //*.sst + CURRENT + MANIFEST
|
||||
ret.resize(live.size() + 2); //*.sst + CURRENT + MANIFEST
|
||||
|
||||
// create names of the live files. The names are not absolute
|
||||
// paths, instead they are relative to dbname_;
|
||||
for (auto live_file : live) {
|
||||
ret.push_back(TableFileName("", live_file));
|
||||
std::set<uint64_t>::iterator it = live.begin();
|
||||
for (unsigned int i = 0; i < live.size(); i++, it++) {
|
||||
ret[i] = TableFileName("", *it);
|
||||
}
|
||||
|
||||
ret.push_back(CurrentFileName(""));
|
||||
ret.push_back(DescriptorFileName("", versions_->ManifestFileNumber()));
|
||||
ret[live.size()] = CurrentFileName("");
|
||||
ret[live.size()+1] = DescriptorFileName("",
|
||||
versions_->ManifestFileNumber());
|
||||
|
||||
// find length of manifest file while holding the mutex lock
|
||||
*manifest_file_size = versions_->ManifestFileSize();
|
||||
@@ -74,21 +64,4 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
|
||||
// First get sorted files in archive dir, then append sorted files from main
|
||||
// dir to maintain sorted order
|
||||
|
||||
// list wal files in archive dir.
|
||||
Status s;
|
||||
std::string archivedir = ArchivalDirectory(options_.wal_dir);
|
||||
if (env_->FileExists(archivedir)) {
|
||||
s = AppendSortedWalsOfType(archivedir, files, kArchivedLogFile);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
// list wal files in main db dir.
|
||||
return AppendSortedWalsOfType(options_.wal_dir, files, kAliveLogFile);
|
||||
}
|
||||
|
||||
}
|
||||
+409
-1539
File diff suppressed because it is too large.
Load diff
+51
-161
@@ -1,28 +1,27 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#pragma once
|
||||
#include <atomic>
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_DB_IMPL_H_
|
||||
#define STORAGE_LEVELDB_DB_DB_IMPL_H_
|
||||
|
||||
#include <deque>
|
||||
#include <set>
|
||||
#include <vector>
|
||||
#include "db/dbformat.h"
|
||||
#include "db/log_file.h"
|
||||
#include "db/log_writer.h"
|
||||
#include "db/snapshot.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/memtablerep.h"
|
||||
#include "rocksdb/transaction_log.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "port/port.h"
|
||||
#include "util/stats_logger.h"
|
||||
#include "memtablelist.h"
|
||||
|
||||
namespace rocksdb {
|
||||
#ifdef USE_SCRIBE
|
||||
#include "scribe/scribe_logger.h"
|
||||
#endif
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class MemTable;
|
||||
class TableCache;
|
||||
@@ -37,32 +36,17 @@ class DBImpl : public DB {
|
||||
|
||||
// Implementations of the DB interface
|
||||
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
|
||||
virtual Status Merge(const WriteOptions&, const Slice& key,
|
||||
const Slice& value);
|
||||
virtual Status Delete(const WriteOptions&, const Slice& key);
|
||||
virtual Status Write(const WriteOptions& options, WriteBatch* updates);
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value);
|
||||
virtual std::vector<Status> MultiGet(const ReadOptions& options,
|
||||
const std::vector<Slice>& keys,
|
||||
std::vector<std::string>* values);
|
||||
|
||||
// Returns false if key doesn't exist in the database and true if it may.
|
||||
// If value_found is not passed in as null, then return the value if found in
|
||||
// memory. On return, if value was found, then value_found will be set to true
|
||||
// , otherwise false.
|
||||
virtual bool KeyMayExist(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value,
|
||||
bool* value_found = nullptr);
|
||||
virtual Iterator* NewIterator(const ReadOptions&);
|
||||
virtual const Snapshot* GetSnapshot();
|
||||
virtual void ReleaseSnapshot(const Snapshot* snapshot);
|
||||
virtual bool GetProperty(const Slice& property, std::string* value);
|
||||
virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end,
|
||||
bool reduce_level = false, int target_level = -1);
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end);
|
||||
virtual int NumberLevels();
|
||||
virtual int MaxMemCompactionLevel();
|
||||
virtual int Level0StopWriteTrigger();
|
||||
@@ -70,27 +54,21 @@ class DBImpl : public DB {
|
||||
virtual Status DisableFileDeletions();
|
||||
virtual Status EnableFileDeletions();
|
||||
virtual Status GetLiveFiles(std::vector<std::string>&,
|
||||
uint64_t* manifest_file_size,
|
||||
bool flush_memtable = true);
|
||||
virtual Status GetSortedWalFiles(VectorLogPtr& files);
|
||||
virtual SequenceNumber GetLatestSequenceNumber() const;
|
||||
uint64_t* manifest_file_size);
|
||||
virtual SequenceNumber GetLatestSequenceNumber();
|
||||
virtual Status GetUpdatesSince(SequenceNumber seq_number,
|
||||
unique_ptr<TransactionLogIterator>* iter);
|
||||
virtual Status DeleteFile(std::string name);
|
||||
|
||||
virtual void GetLiveFilesMetaData(
|
||||
std::vector<LiveFileMetaData> *metadata);
|
||||
|
||||
// Extra methods (for testing) that are not in the public DB interface
|
||||
|
||||
// Compact any files in the named level that overlap [*begin, *end]
|
||||
// Compact any files in the named level that overlap [*begin,*end]
|
||||
void TEST_CompactRange(int level, const Slice* begin, const Slice* end);
|
||||
|
||||
// Force current memtable contents to be flushed.
|
||||
Status TEST_FlushMemTable();
|
||||
// Force current memtable contents to be compacted.
|
||||
Status TEST_CompactMemTable();
|
||||
|
||||
// Wait for memtable compaction
|
||||
Status TEST_WaitForFlushMemTable();
|
||||
Status TEST_WaitForCompactMemTable();
|
||||
|
||||
// Wait for any compaction
|
||||
Status TEST_WaitForCompact();
|
||||
@@ -109,18 +87,6 @@ class DBImpl : public DB {
|
||||
|
||||
// Return the current manifest file no.
|
||||
uint64_t TEST_Current_Manifest_FileNo();
|
||||
|
||||
// Trigger's a background call for testing.
|
||||
void TEST_PurgeObsoleteteWAL();
|
||||
|
||||
// get total level0 file size. Only for testing.
|
||||
uint64_t TEST_GetLevel0TotalSize() { return versions_->NumLevelBytes(0);}
|
||||
|
||||
void TEST_SetDefaultTimeToCheck(uint64_t default_interval_to_delete_obsolete_WAL)
|
||||
{
|
||||
default_interval_to_delete_obsolete_WAL_ = default_interval_to_delete_obsolete_WAL;
|
||||
}
|
||||
|
||||
protected:
|
||||
Env* const env_;
|
||||
const std::string dbname_;
|
||||
@@ -160,9 +126,9 @@ class DBImpl : public DB {
|
||||
// Delete any unneeded files and stale in-memory entries.
|
||||
void DeleteObsoleteFiles();
|
||||
|
||||
// Flush the in-memory write buffer to storage. Switches to a new
|
||||
// Compact the in-memory write buffer to disk. Switches to a new
|
||||
// log-file/memtable and writes a new descriptor iff successful.
|
||||
Status FlushMemTableToOutputFile(bool* madeProgress = nullptr);
|
||||
Status CompactMemTable(bool* madeProgress = nullptr);
|
||||
|
||||
Status RecoverLogFile(uint64_t log_number,
|
||||
VersionEdit* edit,
|
||||
@@ -175,31 +141,27 @@ class DBImpl : public DB {
|
||||
// for the entire period. The second method WriteLevel0Table supports
|
||||
// concurrent flush memtables to storage.
|
||||
Status WriteLevel0TableForRecovery(MemTable* mem, VersionEdit* edit);
|
||||
Status WriteLevel0Table(std::vector<MemTable*> &mems, VersionEdit* edit,
|
||||
Status WriteLevel0Table(MemTable* mem, VersionEdit* edit,
|
||||
uint64_t* filenumber);
|
||||
|
||||
uint64_t SlowdownAmount(int n, int top, int bottom);
|
||||
Status MakeRoomForWrite(bool force /* compact even if there is room? */);
|
||||
WriteBatch* BuildBatchGroup(Writer** last_writer);
|
||||
|
||||
// Force current memtable contents to be flushed.
|
||||
Status FlushMemTable(const FlushOptions& options);
|
||||
|
||||
// Wait for memtable flushed
|
||||
Status WaitForFlushMemTable();
|
||||
// Wait for memtable compaction
|
||||
Status WaitForCompactMemTable();
|
||||
|
||||
void MaybeScheduleLogDBDeployStats();
|
||||
static void BGLogDBDeployStats(void* db);
|
||||
void LogDBDeployStats();
|
||||
|
||||
void MaybeScheduleFlushOrCompaction();
|
||||
static void BGWorkCompaction(void* db);
|
||||
static void BGWorkFlush(void* db);
|
||||
void BackgroundCallCompaction();
|
||||
void BackgroundCallFlush();
|
||||
Status BackgroundCompaction(bool* madeProgress,DeletionState& deletion_state);
|
||||
Status BackgroundFlush(bool* madeProgress);
|
||||
void CleanupCompaction(CompactionState* compact, Status status);
|
||||
void MaybeScheduleCompaction();
|
||||
static void BGWork(void* db);
|
||||
void BackgroundCall();
|
||||
Status BackgroundCompaction(bool* madeProgress, DeletionState& deletion_state);
|
||||
void CleanupCompaction(CompactionState* compact);
|
||||
Status DoCompactionWork(CompactionState* compact);
|
||||
|
||||
Status OpenCompactionOutputFile(CompactionState* compact);
|
||||
@@ -221,42 +183,23 @@ class DBImpl : public DB {
|
||||
// Removes the file listed in files_to_evict from the table_cache
|
||||
void EvictObsoleteFiles(DeletionState& deletion_state);
|
||||
|
||||
Status DeleteLogFile(uint64_t number);
|
||||
|
||||
void PurgeObsoleteWALFiles();
|
||||
|
||||
Status AppendSortedWalsOfType(const std::string& path,
|
||||
VectorLogPtr& log_files,
|
||||
WalFileType type);
|
||||
Status ListAllWALFiles(const std::string& path,
|
||||
std::vector<LogFile>* logFiles,
|
||||
WalFileType type);
|
||||
|
||||
// Requires: all_logs should be sorted with earliest log file first
|
||||
// Retains all log files in all_logs which contain updates with seq no.
|
||||
// Greater Than or Equal to the requested SequenceNumber.
|
||||
Status RetainProbableWalFiles(VectorLogPtr& all_logs,
|
||||
const SequenceNumber target);
|
||||
// return true if
|
||||
bool CheckWalFileExistsAndEmpty(const WalFileType type,
|
||||
const uint64_t number);
|
||||
// Find's all the log files which contain updates with seq no.
|
||||
// Greater Than or Equal to the requested SequenceNumber
|
||||
Status FindProbableWALFiles(std::vector<LogFile>* const allLogs,
|
||||
std::vector<LogFile>* const result,
|
||||
const SequenceNumber target);
|
||||
|
||||
|
||||
Status ReadFirstRecord(const LogFile& file, WriteBatch* const result);
|
||||
|
||||
Status ReadFirstRecord(const WalFileType type, const uint64_t number,
|
||||
WriteBatch* const result);
|
||||
|
||||
Status ReadFirstLine(const std::string& fname, WriteBatch* const batch);
|
||||
|
||||
void PrintStatistics();
|
||||
|
||||
// dump rocksdb.stats to LOG
|
||||
void MaybeDumpStats();
|
||||
|
||||
// Return the minimum empty level that could hold the total data in the
|
||||
// input level. Return the input level, if such level could not be found.
|
||||
int FindMinimumEmptyLevelFitting(int level);
|
||||
|
||||
// Move the files in the input level to the target level.
|
||||
// If target_level < 0, automatically calculate the minimum level that could
|
||||
// hold the data set.
|
||||
void ReFitLevel(int level, int target_level = -1);
|
||||
|
||||
// Constant after construction
|
||||
const InternalFilterPolicy internal_filter_policy_;
|
||||
bool owns_info_log_;
|
||||
@@ -271,7 +214,6 @@ class DBImpl : public DB {
|
||||
port::Mutex mutex_;
|
||||
port::AtomicPointer shutting_down_;
|
||||
port::CondVar bg_cv_; // Signalled when background work finishes
|
||||
std::shared_ptr<MemTableRepFactory> mem_rep_factory_;
|
||||
MemTable* mem_;
|
||||
MemTableList imm_; // Memtable that are not changing
|
||||
uint64_t logfile_number_;
|
||||
@@ -281,7 +223,7 @@ class DBImpl : public DB {
|
||||
|
||||
// Queue of writers.
|
||||
std::deque<Writer*> writers_;
|
||||
WriteBatch tmp_batch_;
|
||||
WriteBatch* tmp_batch_;
|
||||
|
||||
SnapshotList snapshots_;
|
||||
|
||||
@@ -292,9 +234,6 @@ class DBImpl : public DB {
|
||||
// count how many background compaction been scheduled or is running?
|
||||
int bg_compaction_scheduled_;
|
||||
|
||||
// number of background memtable flush jobs, submitted to the HIGH pool
|
||||
int bg_flush_scheduled_;
|
||||
|
||||
// Has a background stats log thread scheduled?
|
||||
bool bg_logstats_scheduled_;
|
||||
|
||||
@@ -312,7 +251,7 @@ class DBImpl : public DB {
|
||||
// Have we encountered a background error in paranoid mode?
|
||||
Status bg_error_;
|
||||
|
||||
std::unique_ptr<StatsLogger> logger_;
|
||||
StatsLogger* logger_;
|
||||
|
||||
int64_t volatile last_log_ts;
|
||||
|
||||
@@ -322,25 +261,11 @@ class DBImpl : public DB {
|
||||
// last time when DeleteObsoleteFiles was invoked
|
||||
uint64_t delete_obsolete_files_last_run_;
|
||||
|
||||
// last time when PurgeObsoleteWALFiles ran.
|
||||
uint64_t purge_wal_files_last_run_;
|
||||
|
||||
// last time stats were dumped to LOG
|
||||
std::atomic<uint64_t> last_stats_dump_time_microsec_;
|
||||
|
||||
// obsolete files will be deleted every this seconds if ttl deletion is
|
||||
// enabled and archive size_limit is disabled.
|
||||
uint64_t default_interval_to_delete_obsolete_WAL_;
|
||||
|
||||
// These count the number of microseconds for which MakeRoomForWrite stalls.
|
||||
uint64_t stall_level0_slowdown_;
|
||||
uint64_t stall_memtable_compaction_;
|
||||
uint64_t stall_level0_num_files_;
|
||||
std::vector<uint64_t> stall_leveln_slowdown_;
|
||||
uint64_t stall_level0_slowdown_count_;
|
||||
uint64_t stall_memtable_compaction_count_;
|
||||
uint64_t stall_level0_num_files_count_;
|
||||
std::vector<uint64_t> stall_leveln_slowdown_count_;
|
||||
|
||||
// Time at which this instance was started.
|
||||
const uint64_t started_at_;
|
||||
@@ -390,20 +315,7 @@ class DBImpl : public DB {
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<CompactionStats> stats_;
|
||||
|
||||
// Used to compute per-interval statistics
|
||||
struct StatsSnapshot {
|
||||
uint64_t bytes_read_;
|
||||
uint64_t bytes_written_;
|
||||
uint64_t bytes_new_;
|
||||
double seconds_up_;
|
||||
|
||||
StatsSnapshot() : bytes_read_(0), bytes_written_(0),
|
||||
bytes_new_(0), seconds_up_(0) {}
|
||||
};
|
||||
|
||||
StatsSnapshot last_stats_;
|
||||
CompactionStats* stats_;
|
||||
|
||||
static const int KEEP_LOG_FILE_NUM = 1000;
|
||||
std::string db_absolute_path_;
|
||||
@@ -411,14 +323,9 @@ class DBImpl : public DB {
|
||||
// count of the number of contiguous delaying writes
|
||||
int delayed_writes_;
|
||||
|
||||
// The options to access storage files
|
||||
const EnvOptions storage_options_;
|
||||
|
||||
// A value of true temporarily disables scheduling of background work
|
||||
bool bg_work_gate_closed_;
|
||||
|
||||
// Guard against multiple concurrent refitting
|
||||
bool refitting_level_;
|
||||
// store the last flushed sequence.
|
||||
// Used by transaction log iterator.
|
||||
SequenceNumber last_flushed_sequence_;
|
||||
|
||||
// No copying allowed
|
||||
DBImpl(const DBImpl&);
|
||||
@@ -427,19 +334,9 @@ class DBImpl : public DB {
|
||||
// dump the delayed_writes_ to the log file and reset counter.
|
||||
void DelayLoggingAndReset();
|
||||
|
||||
// Return the earliest snapshot where seqno is visible.
|
||||
// Store the snapshot right before that, if any, in prev_snapshot
|
||||
inline SequenceNumber findEarliestVisibleSnapshot(
|
||||
SequenceNumber in,
|
||||
std::vector<SequenceNumber>& snapshots,
|
||||
SequenceNumber* prev_snapshot);
|
||||
|
||||
// Function that Get and KeyMayExist call with no_io true or false
|
||||
// Note: 'value_found' from KeyMayExist propagates here
|
||||
Status GetImpl(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value,
|
||||
bool* value_found = nullptr);
|
||||
// find the earliest snapshot where seqno is visible
|
||||
inline SequenceNumber findEarliestVisibleSnapshot(SequenceNumber in,
|
||||
std::vector<SequenceNumber>& snapshots);
|
||||
};
|
||||
|
||||
// Sanitize db options. The caller should delete result.info_log if
|
||||
@@ -449,13 +346,6 @@ extern Options SanitizeOptions(const std::string& db,
|
||||
const InternalFilterPolicy* ipolicy,
|
||||
const Options& src);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
// Determine compression type, based on user options, level of the output
|
||||
// file and whether compression is disabled.
|
||||
// If enable_compression is false, then compression is always disabled no
|
||||
// matter what the values of the other two parameters are.
|
||||
// Otherwise, the compression type is determined based on options and level.
|
||||
CompressionType GetCompressionType(const Options& options, int level,
|
||||
const bool enable_compression);
|
||||
|
||||
} // namespace rocksdb
|
||||
#endif // STORAGE_LEVELDB_DB_DB_IMPL_H_
|
||||
+9
-14
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2012 Facebook. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
@@ -26,10 +21,11 @@
|
||||
#include "db/table_cache.h"
|
||||
#include "db/version_set.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/table_builder.h"
|
||||
#include "port/port.h"
|
||||
#include "table/block.h"
|
||||
#include "table/merger.h"
|
||||
@@ -38,7 +34,7 @@
|
||||
#include "util/logging.h"
|
||||
#include "util/build_version.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
DBImplReadOnly::DBImplReadOnly(const Options& options,
|
||||
const std::string& dbname)
|
||||
@@ -57,12 +53,11 @@ Status DBImplReadOnly::Get(const ReadOptions& options,
|
||||
MemTable* mem = GetMemTable();
|
||||
Version* current = versions_->current();
|
||||
SequenceNumber snapshot = versions_->LastSequence();
|
||||
std::deque<std::string> merge_operands;
|
||||
LookupKey lkey(key, snapshot);
|
||||
if (mem->Get(lkey, value, &s, &merge_operands, options_)) {
|
||||
if (mem->Get(lkey, value, &s)) {
|
||||
} else {
|
||||
Version::GetStats stats;
|
||||
current->Get(options, lkey, value, &s, &merge_operands, &stats, options_);
|
||||
s = current->Get(options, lkey, value, &stats);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
@@ -71,7 +66,7 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& options) {
|
||||
SequenceNumber latest_snapshot;
|
||||
Iterator* internal_iter = NewInternalIterator(options, &latest_snapshot);
|
||||
return NewDBIterator(
|
||||
&dbname_, env_, options_, user_comparator(),internal_iter,
|
||||
&dbname_, env_, user_comparator(), internal_iter,
|
||||
(options.snapshot != nullptr
|
||||
? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
|
||||
: latest_snapshot));
|
||||
|
||||
+14
-20
@@ -1,13 +1,10 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2012 Facebook. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_DB_IMPL_READONLY_H_
|
||||
#define STORAGE_LEVELDB_DB_DB_IMPL_READONLY_H_
|
||||
|
||||
#include "db/db_impl.h"
|
||||
|
||||
#include <deque>
|
||||
@@ -15,12 +12,16 @@
|
||||
#include "db/dbformat.h"
|
||||
#include "db/log_writer.h"
|
||||
#include "db/snapshot.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "port/port.h"
|
||||
#include "util/stats_logger.h"
|
||||
|
||||
namespace rocksdb {
|
||||
#ifdef USE_SCRIBE
|
||||
#include "scribe/scribe_logger.h"
|
||||
#endif
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class DBImplReadOnly : public DBImpl {
|
||||
public:
|
||||
@@ -31,26 +32,18 @@ public:
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value);
|
||||
|
||||
// TODO: Implement ReadOnly MultiGet?
|
||||
|
||||
virtual Iterator* NewIterator(const ReadOptions&);
|
||||
|
||||
virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value) {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
virtual Status Merge(const WriteOptions&, const Slice& key,
|
||||
const Slice& value) {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
virtual Status Delete(const WriteOptions&, const Slice& key) {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
virtual Status Write(const WriteOptions& options, WriteBatch* updates) {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end,
|
||||
bool reduce_level = false, int target_level = -1) {
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end) {
|
||||
}
|
||||
virtual Status DisableFileDeletions() {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
@@ -59,8 +52,7 @@ public:
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
virtual Status GetLiveFiles(std::vector<std::string>&,
|
||||
uint64_t* manifest_file_size,
|
||||
bool flush_memtable = true) {
|
||||
uint64_t* manifest_file_size) {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
virtual Status Flush(const FlushOptions& options) {
|
||||
@@ -76,3 +68,5 @@ private:
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
+39
-221
@@ -1,28 +1,18 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/db_iter.h"
|
||||
#include <stdexcept>
|
||||
#include <deque>
|
||||
|
||||
#include "db/filename.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/options.h"
|
||||
#include "rocksdb/iterator.h"
|
||||
#include "rocksdb/merge_operator.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "port/port.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/perf_context_imp.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
#if 0
|
||||
static void DumpInternalIter(Iterator* iter) {
|
||||
@@ -46,7 +36,6 @@ namespace {
|
||||
// numbers, deletion markers, overwrites, etc.
|
||||
class DBIter: public Iterator {
|
||||
public:
|
||||
// The following is grossly complicated. TODO: clean it up
|
||||
// Which direction is the iterator currently moving?
|
||||
// (1) When moving forward, the internal iterator is positioned at
|
||||
// the exact entry that yields this->key(), this->value()
|
||||
@@ -57,35 +46,27 @@ class DBIter: public Iterator {
|
||||
kReverse
|
||||
};
|
||||
|
||||
DBIter(const std::string* dbname, Env* env, const Options& options,
|
||||
DBIter(const std::string* dbname, Env* env,
|
||||
const Comparator* cmp, Iterator* iter, SequenceNumber s)
|
||||
: dbname_(dbname),
|
||||
env_(env),
|
||||
logger_(options.info_log),
|
||||
user_comparator_(cmp),
|
||||
user_merge_operator_(options.merge_operator.get()),
|
||||
iter_(iter),
|
||||
sequence_(s),
|
||||
direction_(kForward),
|
||||
valid_(false),
|
||||
current_entry_is_merged_(false),
|
||||
statistics_(options.statistics) {
|
||||
RecordTick(statistics_, NO_ITERATORS, 1);
|
||||
max_skip_ = options.max_sequential_skip_in_iterations;
|
||||
valid_(false) {
|
||||
}
|
||||
virtual ~DBIter() {
|
||||
RecordTick(statistics_, NO_ITERATORS, -1);
|
||||
delete iter_;
|
||||
}
|
||||
virtual bool Valid() const { return valid_; }
|
||||
virtual Slice key() const {
|
||||
assert(valid_);
|
||||
return saved_key_;
|
||||
return (direction_ == kForward) ? ExtractUserKey(iter_->key()) : saved_key_;
|
||||
}
|
||||
virtual Slice value() const {
|
||||
assert(valid_);
|
||||
return (direction_ == kForward && !current_entry_is_merged_) ?
|
||||
iter_->value() : saved_value_;
|
||||
return (direction_ == kForward) ? iter_->value() : saved_value_;
|
||||
}
|
||||
virtual Status status() const {
|
||||
if (status_.ok()) {
|
||||
@@ -102,10 +83,9 @@ class DBIter: public Iterator {
|
||||
virtual void SeekToLast();
|
||||
|
||||
private:
|
||||
void FindNextUserEntry(bool skipping);
|
||||
void FindNextUserEntry(bool skipping, std::string* skip);
|
||||
void FindPrevUserEntry();
|
||||
bool ParseKey(ParsedInternalKey* key);
|
||||
void MergeValuesNewToOld();
|
||||
|
||||
inline void SaveKey(const Slice& k, std::string* dst) {
|
||||
dst->assign(k.data(), k.size());
|
||||
@@ -122,21 +102,15 @@ class DBIter: public Iterator {
|
||||
|
||||
const std::string* const dbname_;
|
||||
Env* const env_;
|
||||
shared_ptr<Logger> logger_;
|
||||
const Comparator* const user_comparator_;
|
||||
const MergeOperator* const user_merge_operator_;
|
||||
Iterator* const iter_;
|
||||
SequenceNumber const sequence_;
|
||||
|
||||
Status status_;
|
||||
std::string saved_key_; // == current key when direction_==kReverse
|
||||
std::string saved_value_; // == current raw value when direction_==kReverse
|
||||
std::string skip_key_;
|
||||
Direction direction_;
|
||||
bool valid_;
|
||||
bool current_entry_is_merged_;
|
||||
std::shared_ptr<Statistics> statistics_;
|
||||
uint64_t max_skip_;
|
||||
|
||||
// No copying allowed
|
||||
DBIter(const DBIter&);
|
||||
@@ -146,8 +120,6 @@ class DBIter: public Iterator {
|
||||
inline bool DBIter::ParseKey(ParsedInternalKey* ikey) {
|
||||
if (!ParseInternalKey(iter_->key(), ikey)) {
|
||||
status_ = Status::Corruption("corrupted internal key in DBIter");
|
||||
Log(logger_, "corrupted internal key in DBIter: %s",
|
||||
iter_->key().ToString(true).c_str());
|
||||
return false;
|
||||
} else {
|
||||
return true;
|
||||
@@ -174,173 +146,47 @@ void DBIter::Next() {
|
||||
}
|
||||
}
|
||||
|
||||
// If the current value is merged, we might already hit end of iter_
|
||||
if (!iter_->Valid()) {
|
||||
valid_ = false;
|
||||
return;
|
||||
}
|
||||
FindNextUserEntry(true /* skipping the current user key */);
|
||||
// Temporarily use saved_key_ as storage for key to skip.
|
||||
std::string* skip = &saved_key_;
|
||||
SaveKey(ExtractUserKey(iter_->key()), skip);
|
||||
FindNextUserEntry(true, skip);
|
||||
}
|
||||
|
||||
|
||||
// PRE: saved_key_ has the current user key if skipping
|
||||
// POST: saved_key_ should have the next user key if valid_,
|
||||
// if the current entry is a result of merge
|
||||
// current_entry_is_merged_ => true
|
||||
// saved_value_ => the merged value
|
||||
//
|
||||
// NOTE: In between, saved_key_ can point to a user key that has
|
||||
// a delete marker
|
||||
void DBIter::FindNextUserEntry(bool skipping) {
|
||||
void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {
|
||||
// Loop until we hit an acceptable entry to yield
|
||||
assert(iter_->Valid());
|
||||
assert(direction_ == kForward);
|
||||
current_entry_is_merged_ = false;
|
||||
uint64_t num_skipped = 0;
|
||||
do {
|
||||
ParsedInternalKey ikey;
|
||||
if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
|
||||
if (skipping &&
|
||||
user_comparator_->Compare(ikey.user_key, saved_key_) <= 0) {
|
||||
num_skipped++; // skip this entry
|
||||
BumpPerfCount(&perf_context.internal_key_skipped_count);
|
||||
} else {
|
||||
skipping = false;
|
||||
switch (ikey.type) {
|
||||
case kTypeDeletion:
|
||||
// Arrange to skip all upcoming entries for this key since
|
||||
// they are hidden by this deletion.
|
||||
SaveKey(ikey.user_key, &saved_key_);
|
||||
skipping = true;
|
||||
num_skipped = 0;
|
||||
BumpPerfCount(&perf_context.internal_delete_skipped_count);
|
||||
break;
|
||||
case kTypeValue:
|
||||
valid_ = true;
|
||||
SaveKey(ikey.user_key, &saved_key_);
|
||||
return;
|
||||
case kTypeMerge:
|
||||
// By now, we are sure the current ikey is going to yield a value
|
||||
SaveKey(ikey.user_key, &saved_key_);
|
||||
current_entry_is_merged_ = true;
|
||||
valid_ = true;
|
||||
MergeValuesNewToOld(); // Go to a different state machine
|
||||
return;
|
||||
case kTypeLogData:
|
||||
assert(false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// If we have sequentially iterated via numerous keys and still not
|
||||
// found the next user-key, then it is better to seek so that we can
|
||||
// avoid too many key comparisons. We seek to the last occurence of
|
||||
// our current key by looking for sequence number 0.
|
||||
if (skipping && num_skipped > max_skip_) {
|
||||
num_skipped = 0;
|
||||
std::string last_key;
|
||||
AppendInternalKey(&last_key,
|
||||
ParsedInternalKey(Slice(saved_key_), 0, kValueTypeForSeek));
|
||||
iter_->Seek(last_key);
|
||||
RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
|
||||
} else {
|
||||
iter_->Next();
|
||||
}
|
||||
} while (iter_->Valid());
|
||||
valid_ = false;
|
||||
}
|
||||
|
||||
// Merge values of the same user key starting from the current iter_ position
|
||||
// Scan from the newer entries to older entries.
|
||||
// PRE: iter_->key() points to the first merge type entry
|
||||
// saved_key_ stores the user key
|
||||
// POST: saved_value_ has the merged value for the user key
|
||||
// iter_ points to the next entry (or invalid)
|
||||
void DBIter::MergeValuesNewToOld() {
|
||||
if (!user_merge_operator_) {
|
||||
Log(logger_, "Options::merge_operator is null.");
|
||||
throw std::logic_error("DBIter::MergeValuesNewToOld() with"
|
||||
" Options::merge_operator null");
|
||||
}
|
||||
|
||||
// Start the merge process by pushing the first operand
|
||||
std::deque<std::string> operands;
|
||||
operands.push_front(iter_->value().ToString());
|
||||
|
||||
std::string merge_result; // Temporary string to hold merge result later
|
||||
ParsedInternalKey ikey;
|
||||
for (iter_->Next(); iter_->Valid(); iter_->Next()) {
|
||||
if (!ParseKey(&ikey)) {
|
||||
// skip corrupted key
|
||||
continue;
|
||||
}
|
||||
|
||||
if (user_comparator_->Compare(ikey.user_key, saved_key_) != 0) {
|
||||
// hit the next user key, stop right here
|
||||
break;
|
||||
}
|
||||
|
||||
if (kTypeDeletion == ikey.type) {
|
||||
// hit a delete with the same user key, stop right here
|
||||
// iter_ is positioned after delete
|
||||
iter_->Next();
|
||||
break;
|
||||
}
|
||||
|
||||
if (kTypeValue == ikey.type) {
|
||||
// hit a put, merge the put value with operands and store the
|
||||
// final result in saved_value_. We are done!
|
||||
// ignore corruption if there is any.
|
||||
const Slice value = iter_->value();
|
||||
user_merge_operator_->FullMerge(ikey.user_key, &value, operands,
|
||||
&saved_value_, logger_.get());
|
||||
// iter_ is positioned after put
|
||||
iter_->Next();
|
||||
return;
|
||||
}
|
||||
|
||||
if (kTypeMerge == ikey.type) {
|
||||
// hit a merge, add the value as an operand and run associative merge.
|
||||
// when complete, add result to operands and continue.
|
||||
const Slice& value = iter_->value();
|
||||
operands.push_front(value.ToString());
|
||||
while(operands.size() >= 2) {
|
||||
// Call user associative-merge until it returns false
|
||||
if (user_merge_operator_->PartialMerge(ikey.user_key,
|
||||
Slice(operands[0]),
|
||||
Slice(operands[1]),
|
||||
&merge_result,
|
||||
logger_.get())) {
|
||||
operands.pop_front();
|
||||
swap(operands.front(), merge_result);
|
||||
} else {
|
||||
// Associative merge returns false ==> stack the operands
|
||||
switch (ikey.type) {
|
||||
case kTypeDeletion:
|
||||
// Arrange to skip all upcoming entries for this key since
|
||||
// they are hidden by this deletion.
|
||||
SaveKey(ikey.user_key, skip);
|
||||
skipping = true;
|
||||
break;
|
||||
case kTypeValue:
|
||||
if (skipping &&
|
||||
user_comparator_->Compare(ikey.user_key, *skip) <= 0) {
|
||||
// Entry hidden
|
||||
} else {
|
||||
valid_ = true;
|
||||
saved_key_.clear();
|
||||
return;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// we either exhausted all internal keys under this user key, or hit
|
||||
// a deletion marker.
|
||||
// feed null as the existing value to the merge operator, such that
|
||||
// client can differentiate this scenario and do things accordingly.
|
||||
user_merge_operator_->FullMerge(saved_key_, nullptr, operands,
|
||||
&saved_value_, logger_.get());
|
||||
iter_->Next();
|
||||
} while (iter_->Valid());
|
||||
saved_key_.clear();
|
||||
valid_ = false;
|
||||
}
|
||||
|
||||
void DBIter::Prev() {
|
||||
assert(valid_);
|
||||
|
||||
// Throw an exception now if merge_operator is provided
|
||||
// TODO: support backward iteration
|
||||
if (user_merge_operator_) {
|
||||
Log(logger_, "Prev not supported yet if merge_operator is provided");
|
||||
throw std::logic_error("DBIter::Prev backward iteration not supported"
|
||||
" if merge_operator is provided");
|
||||
}
|
||||
|
||||
if (direction_ == kForward) { // Switch directions?
|
||||
// iter_ is pointing at the current entry. Scan backwards until
|
||||
// the key changes so we can use the normal reverse scanning code.
|
||||
@@ -367,13 +213,11 @@ void DBIter::Prev() {
|
||||
|
||||
void DBIter::FindPrevUserEntry() {
|
||||
assert(direction_ == kReverse);
|
||||
uint64_t num_skipped = 0;
|
||||
|
||||
ValueType value_type = kTypeDeletion;
|
||||
if (iter_->Valid()) {
|
||||
do {
|
||||
ParsedInternalKey ikey;
|
||||
bool saved_key_cleared = false;
|
||||
if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
|
||||
if ((value_type != kTypeDeletion) &&
|
||||
user_comparator_->Compare(ikey.user_key, saved_key_) < 0) {
|
||||
@@ -384,7 +228,6 @@ void DBIter::FindPrevUserEntry() {
|
||||
if (value_type == kTypeDeletion) {
|
||||
saved_key_.clear();
|
||||
ClearSavedValue();
|
||||
saved_key_cleared = true;
|
||||
} else {
|
||||
Slice raw_value = iter_->value();
|
||||
if (saved_value_.capacity() > raw_value.size() + 1048576) {
|
||||
@@ -395,22 +238,7 @@ void DBIter::FindPrevUserEntry() {
|
||||
saved_value_.assign(raw_value.data(), raw_value.size());
|
||||
}
|
||||
}
|
||||
num_skipped++;
|
||||
// If we have sequentially iterated via numerous keys and still not
|
||||
// found the prev user-key, then it is better to seek so that we can
|
||||
// avoid too many key comparisons. We seek to the first occurence of
|
||||
// our current key by looking for max sequence number.
|
||||
if (!saved_key_cleared && num_skipped > max_skip_) {
|
||||
num_skipped = 0;
|
||||
std::string last_key;
|
||||
AppendInternalKey(&last_key,
|
||||
ParsedInternalKey(Slice(saved_key_), kMaxSequenceNumber,
|
||||
kValueTypeForSeek));
|
||||
iter_->Seek(last_key);
|
||||
RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
|
||||
} else {
|
||||
iter_->Prev();
|
||||
}
|
||||
iter_->Prev();
|
||||
} while (iter_->Valid());
|
||||
}
|
||||
|
||||
@@ -433,7 +261,7 @@ void DBIter::Seek(const Slice& target) {
|
||||
&saved_key_, ParsedInternalKey(target, sequence_, kValueTypeForSeek));
|
||||
iter_->Seek(saved_key_);
|
||||
if (iter_->Valid()) {
|
||||
FindNextUserEntry(false /*not skipping */);
|
||||
FindNextUserEntry(false, &saved_key_ /* temporary storage */);
|
||||
} else {
|
||||
valid_ = false;
|
||||
}
|
||||
@@ -444,21 +272,13 @@ void DBIter::SeekToFirst() {
|
||||
ClearSavedValue();
|
||||
iter_->SeekToFirst();
|
||||
if (iter_->Valid()) {
|
||||
FindNextUserEntry(false /* not skipping */);
|
||||
FindNextUserEntry(false, &saved_key_ /* temporary storage */);
|
||||
} else {
|
||||
valid_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
void DBIter::SeekToLast() {
|
||||
// Throw an exception for now if merge_operator is provided
|
||||
// TODO: support backward iteration
|
||||
if (user_merge_operator_) {
|
||||
Log(logger_, "SeekToLast not supported yet if merge_operator is provided");
|
||||
throw std::logic_error("DBIter::SeekToLast: backward iteration not"
|
||||
" supported if merge_operator is provided");
|
||||
}
|
||||
|
||||
direction_ = kReverse;
|
||||
ClearSavedValue();
|
||||
iter_->SeekToLast();
|
||||
@@ -470,12 +290,10 @@ void DBIter::SeekToLast() {
|
||||
Iterator* NewDBIterator(
|
||||
const std::string* dbname,
|
||||
Env* env,
|
||||
const Options& options,
|
||||
const Comparator *user_key_comparator,
|
||||
const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter,
|
||||
const SequenceNumber& sequence) {
|
||||
return new DBIter(dbname, env, options, user_key_comparator,
|
||||
internal_iter, sequence);
|
||||
return new DBIter(dbname, env, user_key_comparator, internal_iter, sequence);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+9
-11
@@ -1,18 +1,15 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_DB_ITER_H_
|
||||
#define STORAGE_LEVELDB_DB_DB_ITER_H_
|
||||
|
||||
#include <stdint.h>
|
||||
#include "rocksdb/db.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "db/dbformat.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
// Return a new iterator that converts internal keys (yielded by
|
||||
// "*internal_iter") that were live at the specified "sequence" number
|
||||
@@ -20,9 +17,10 @@ namespace rocksdb {
|
||||
extern Iterator* NewDBIterator(
|
||||
const std::string* dbname,
|
||||
Env* env,
|
||||
const Options& options,
|
||||
const Comparator *user_key_comparator,
|
||||
const Comparator* user_key_comparator,
|
||||
Iterator* internal_iter,
|
||||
const SequenceNumber& sequence);
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_DB_ITER_H_
|
||||
+13
-18
@@ -1,25 +1,21 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#ifndef LEVELDB_STORAGE_DB_DB_STATISTICS_H_
|
||||
#define LEVELDB_STORAGE_DB_DB_STATISTICS_H_
|
||||
|
||||
#include <cassert>
|
||||
#include <stdlib.h>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
|
||||
#include "rocksdb/statistics.h"
|
||||
#include "leveldb/statistics.h"
|
||||
#include "util/histogram.h"
|
||||
#include "port/port.h"
|
||||
#include "util/mutexlock.h"
|
||||
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class DBStatistics: public Statistics {
|
||||
public:
|
||||
@@ -33,11 +29,6 @@ class DBStatistics: public Statistics {
|
||||
return allTickers_[tickerType].getCount();
|
||||
}
|
||||
|
||||
virtual void setTickerCount(Tickers tickerType, uint64_t count) {
|
||||
assert(tickerType < TICKER_ENUM_MAX);
|
||||
allTickers_[tickerType].setTickerCount(count);
|
||||
}
|
||||
|
||||
virtual void recordTick(Tickers tickerType, uint64_t count) {
|
||||
assert(tickerType < TICKER_ENUM_MAX);
|
||||
allTickers_[tickerType].recordTick(count);
|
||||
@@ -48,6 +39,11 @@ class DBStatistics: public Statistics {
|
||||
allHistograms_[histogramType].Add(value);
|
||||
}
|
||||
|
||||
virtual void measureTime(Histograms histogramType, double value) {
|
||||
assert(histogramType < HISTOGRAM_ENUM_MAX);
|
||||
allHistograms_[histogramType].Add(value);
|
||||
}
|
||||
|
||||
virtual void histogramData(Histograms histogramType,
|
||||
HistogramData * const data) {
|
||||
assert(histogramType < HISTOGRAM_ENUM_MAX);
|
||||
@@ -57,9 +53,8 @@ class DBStatistics: public Statistics {
|
||||
std::vector<Ticker> allTickers_;
|
||||
std::vector<HistogramImpl> allHistograms_;
|
||||
};
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // LEVELDB_STORAGE_DB_DB_STATISTICS_H_
|
||||
|
||||
std::shared_ptr<Statistics> CreateDBStatistics() {
|
||||
return std::make_shared<DBStatistics>();
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -12,12 +7,12 @@
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include "db/version_set.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "port/port.h"
|
||||
#include "util/mutexlock.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
void DBImpl::MaybeScheduleLogDBDeployStats() {
|
||||
|
||||
|
||||
+896
-2610
File diff suppressed because it is too large.
Load diff
+3
-10
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -11,9 +6,8 @@
|
||||
#include "db/dbformat.h"
|
||||
#include "port/port.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/perf_context_imp.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
static uint64_t PackSequenceAndType(uint64_t seq, ValueType t) {
|
||||
assert(seq <= kMaxSequenceNumber);
|
||||
@@ -50,7 +44,7 @@ std::string InternalKey::DebugString(bool hex) const {
|
||||
}
|
||||
|
||||
const char* InternalKeyComparator::Name() const {
|
||||
return name_.c_str();
|
||||
return "leveldb.InternalKeyComparator";
|
||||
}
|
||||
|
||||
int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
|
||||
@@ -59,7 +53,6 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
|
||||
// decreasing sequence number
|
||||
// decreasing type (though sequence# should be enough to disambiguate)
|
||||
int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
|
||||
BumpPerfCount(&perf_context.user_key_comparison_count);
|
||||
if (r == 0) {
|
||||
const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
|
||||
const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
|
||||
@@ -144,4 +137,4 @@ LookupKey::LookupKey(const Slice& user_key, SequenceNumber s) {
|
||||
end_ = dst;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+17
-34
@@ -1,24 +1,21 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
|
||||
#include <stdio.h>
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/filter_policy.h"
|
||||
#include "rocksdb/slice.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "rocksdb/types.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/filter_policy.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/table_builder.h"
|
||||
#include "leveldb/types.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class InternalKey;
|
||||
|
||||
@@ -27,9 +24,7 @@ class InternalKey;
|
||||
// data structures.
|
||||
enum ValueType {
|
||||
kTypeDeletion = 0x0,
|
||||
kTypeValue = 0x1,
|
||||
kTypeMerge = 0x2,
|
||||
kTypeLogData = 0x3
|
||||
kTypeValue = 0x1
|
||||
};
|
||||
// kValueTypeForSeek defines the ValueType that should be passed when
|
||||
// constructing a ParsedInternalKey object for seeking to a particular
|
||||
@@ -37,7 +32,7 @@ enum ValueType {
|
||||
// and the value type is embedded as the low 8 bits in the sequence
|
||||
// number in internal keys, we need to use the highest-numbered
|
||||
// ValueType, not the lowest).
|
||||
static const ValueType kValueTypeForSeek = kTypeMerge;
|
||||
static const ValueType kValueTypeForSeek = kTypeValue;
|
||||
|
||||
// We leave eight bits empty at the bottom so a type and sequence#
|
||||
// can be packed together into 64-bits.
|
||||
@@ -90,13 +85,8 @@ inline ValueType ExtractValueType(const Slice& internal_key) {
|
||||
class InternalKeyComparator : public Comparator {
|
||||
private:
|
||||
const Comparator* user_comparator_;
|
||||
std::string name_;
|
||||
public:
|
||||
explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c),
|
||||
name_("rocksdb.InternalKeyComparator:" +
|
||||
std::string(user_comparator_->Name())) {
|
||||
}
|
||||
|
||||
explicit InternalKeyComparator(const Comparator* c) : user_comparator_(c) { }
|
||||
virtual const char* Name() const;
|
||||
virtual int Compare(const Slice& a, const Slice& b) const;
|
||||
virtual void FindShortestSeparator(
|
||||
@@ -164,7 +154,7 @@ inline bool ParseInternalKey(const Slice& internal_key,
|
||||
result->sequence = num >> 8;
|
||||
result->type = static_cast<ValueType>(c);
|
||||
result->user_key = Slice(internal_key.data(), n - 8);
|
||||
return (c <= static_cast<unsigned char>(kValueTypeForSeek));
|
||||
return (c <= static_cast<unsigned char>(kTypeValue));
|
||||
}
|
||||
|
||||
// Update the sequence number in the internal key
|
||||
@@ -177,15 +167,6 @@ inline void UpdateInternalKey(char* internal_key,
|
||||
EncodeFixed64(seqtype, newval);
|
||||
}
|
||||
|
||||
// Get the sequence number from the internal key
|
||||
inline uint64_t GetInternalKeySeqno(const Slice& internal_key) {
|
||||
const size_t n = internal_key.size();
|
||||
assert(n >= 8);
|
||||
uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
|
||||
return num >> 8;
|
||||
}
|
||||
|
||||
|
||||
// A helper class useful for DBImpl::Get()
|
||||
class LookupKey {
|
||||
public:
|
||||
@@ -226,4 +207,6 @@ inline LookupKey::~LookupKey() {
|
||||
if (start_ != space_) delete[] start_;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
+3
-8
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -11,7 +6,7 @@
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
static std::string IKey(const std::string& user_key,
|
||||
uint64_t seq,
|
||||
@@ -110,8 +105,8 @@ TEST(FormatTest, InternalKeyShortestSuccessor) {
|
||||
ShortSuccessor(IKey("\xff\xff", 100, kTypeValue)));
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/version_set.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/transaction_log.h"
|
||||
#include <vector>
|
||||
#include <stdlib.h>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class DeleteFileTest {
|
||||
public:
|
||||
std::string dbname_;
|
||||
Options options_;
|
||||
DB* db_;
|
||||
Env* env_;
|
||||
int numlevels_;
|
||||
|
||||
DeleteFileTest() {
|
||||
db_ = nullptr;
|
||||
env_ = Env::Default();
|
||||
options_.write_buffer_size = 1024*1024*1000;
|
||||
options_.target_file_size_base = 1024*1024*1000;
|
||||
options_.max_bytes_for_level_base = 1024*1024*1000;
|
||||
options_.WAL_ttl_seconds = 300; // Used to test log files
|
||||
options_.WAL_size_limit_MB = 1024; // Used to test log files
|
||||
dbname_ = test::TmpDir() + "/deletefile_test";
|
||||
DestroyDB(dbname_, options_);
|
||||
numlevels_ = 7;
|
||||
ASSERT_OK(ReopenDB(true));
|
||||
}
|
||||
|
||||
Status ReopenDB(bool create) {
|
||||
delete db_;
|
||||
if (create) {
|
||||
DestroyDB(dbname_, options_);
|
||||
}
|
||||
db_ = nullptr;
|
||||
options_.create_if_missing = create;
|
||||
return DB::Open(options_, dbname_, &db_);
|
||||
}
|
||||
|
||||
void CloseDB() {
|
||||
delete db_;
|
||||
}
|
||||
|
||||
void AddKeys(int numkeys, int startkey = 0) {
|
||||
WriteOptions options;
|
||||
options.sync = false;
|
||||
ReadOptions roptions;
|
||||
for (int i = startkey; i < (numkeys + startkey) ; i++) {
|
||||
std::string temp = std::to_string(i);
|
||||
Slice key(temp);
|
||||
Slice value(temp);
|
||||
ASSERT_OK(db_->Put(options, key, value));
|
||||
}
|
||||
}
|
||||
|
||||
int numKeysInLevels(
|
||||
std::vector<LiveFileMetaData> &metadata,
|
||||
std::vector<int> *keysperlevel = nullptr) {
|
||||
|
||||
if (keysperlevel != nullptr) {
|
||||
keysperlevel->resize(numlevels_);
|
||||
}
|
||||
|
||||
int numKeys = 0;
|
||||
for (size_t i = 0; i < metadata.size(); i++) {
|
||||
int startkey = atoi(metadata[i].smallestkey.c_str());
|
||||
int endkey = atoi(metadata[i].largestkey.c_str());
|
||||
int numkeysinfile = (endkey - startkey + 1);
|
||||
numKeys += numkeysinfile;
|
||||
if (keysperlevel != nullptr) {
|
||||
(*keysperlevel)[(int)metadata[i].level] += numkeysinfile;
|
||||
}
|
||||
fprintf(stderr, "level %d name %s smallest %s largest %s\n",
|
||||
metadata[i].level, metadata[i].name.c_str(),
|
||||
metadata[i].smallestkey.c_str(),
|
||||
metadata[i].largestkey.c_str());
|
||||
}
|
||||
return numKeys;
|
||||
}
|
||||
|
||||
void CreateTwoLevels() {
|
||||
AddKeys(50000, 10000);
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
ASSERT_OK(dbi->TEST_FlushMemTable());
|
||||
ASSERT_OK(dbi->TEST_WaitForFlushMemTable());
|
||||
|
||||
AddKeys(50000, 10000);
|
||||
ASSERT_OK(dbi->TEST_FlushMemTable());
|
||||
ASSERT_OK(dbi->TEST_WaitForFlushMemTable());
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
TEST(DeleteFileTest, AddKeysAndQueryLevels) {
|
||||
CreateTwoLevels();
|
||||
std::vector<LiveFileMetaData> metadata;
|
||||
std::vector<int> keysinlevel;
|
||||
db_->GetLiveFilesMetaData(&metadata);
|
||||
|
||||
std::string level1file = "";
|
||||
int level1keycount = 0;
|
||||
std::string level2file = "";
|
||||
int level2keycount = 0;
|
||||
int level1index = 0;
|
||||
int level2index = 1;
|
||||
|
||||
ASSERT_EQ((int)metadata.size(), 2);
|
||||
if (metadata[0].level == 2) {
|
||||
level1index = 1;
|
||||
level2index = 0;
|
||||
}
|
||||
|
||||
level1file = metadata[level1index].name;
|
||||
int startkey = atoi(metadata[level1index].smallestkey.c_str());
|
||||
int endkey = atoi(metadata[level1index].largestkey.c_str());
|
||||
level1keycount = (endkey - startkey + 1);
|
||||
level2file = metadata[level2index].name;
|
||||
startkey = atoi(metadata[level2index].smallestkey.c_str());
|
||||
endkey = atoi(metadata[level2index].largestkey.c_str());
|
||||
level2keycount = (endkey - startkey + 1);
|
||||
|
||||
// COntrolled setup. Levels 1 and 2 should both have 50K files.
|
||||
// This is a little fragile as it depends on the current
|
||||
// compaction heuristics.
|
||||
ASSERT_EQ(level1keycount, 50000);
|
||||
ASSERT_EQ(level2keycount, 50000);
|
||||
|
||||
Status status = db_->DeleteFile("0.sst");
|
||||
ASSERT_TRUE(status.IsInvalidArgument());
|
||||
|
||||
// intermediate level files cannot be deleted.
|
||||
status = db_->DeleteFile(level1file);
|
||||
ASSERT_TRUE(status.IsInvalidArgument());
|
||||
|
||||
// Lowest level file deletion should succeed.
|
||||
ASSERT_OK(db_->DeleteFile(level2file));
|
||||
|
||||
CloseDB();
|
||||
}
|
||||
|
||||
TEST(DeleteFileTest, DeleteFileWithIterator) {
|
||||
CreateTwoLevels();
|
||||
ReadOptions options;
|
||||
Iterator* it = db_->NewIterator(options);
|
||||
std::vector<LiveFileMetaData> metadata;
|
||||
db_->GetLiveFilesMetaData(&metadata);
|
||||
|
||||
std::string level2file = "";
|
||||
|
||||
ASSERT_EQ((int)metadata.size(), 2);
|
||||
if (metadata[0].level == 1) {
|
||||
level2file = metadata[1].name;
|
||||
} else {
|
||||
level2file = metadata[0].name;
|
||||
}
|
||||
|
||||
Status status = db_->DeleteFile(level2file);
|
||||
fprintf(stdout, "Deletion status %s: %s\n",
|
||||
level2file.c_str(), status.ToString().c_str());
|
||||
ASSERT_TRUE(status.ok());
|
||||
it->SeekToFirst();
|
||||
int numKeysIterated = 0;
|
||||
while(it->Valid()) {
|
||||
numKeysIterated++;
|
||||
it->Next();
|
||||
}
|
||||
ASSERT_EQ(numKeysIterated, 50000);
|
||||
delete it;
|
||||
CloseDB();
|
||||
}
|
||||
|
||||
TEST(DeleteFileTest, DeleteLogFiles) {
|
||||
AddKeys(10, 0);
|
||||
VectorLogPtr logfiles;
|
||||
db_->GetSortedWalFiles(logfiles);
|
||||
ASSERT_GT(logfiles.size(), 0UL);
|
||||
// Take the last log file which is expected to be alive and try to delete it
|
||||
// Should not succeed because live logs are not allowed to be deleted
|
||||
std::unique_ptr<LogFile> alive_log = std::move(logfiles.back());
|
||||
ASSERT_EQ(alive_log->Type(), kAliveLogFile);
|
||||
ASSERT_TRUE(env_->FileExists(dbname_ + "/" + alive_log->PathName()));
|
||||
fprintf(stdout, "Deleting alive log file %s\n",
|
||||
alive_log->PathName().c_str());
|
||||
ASSERT_TRUE(!db_->DeleteFile(alive_log->PathName()).ok());
|
||||
ASSERT_TRUE(env_->FileExists(dbname_ + "/" + alive_log->PathName()));
|
||||
logfiles.clear();
|
||||
|
||||
// Call Flush to bring about a new working log file and add more keys
|
||||
// Call Flush again to flush out memtable and move alive log to archived log
|
||||
// and try to delete the archived log file
|
||||
FlushOptions fopts;
|
||||
db_->Flush(fopts);
|
||||
AddKeys(10, 0);
|
||||
db_->Flush(fopts);
|
||||
db_->GetSortedWalFiles(logfiles);
|
||||
ASSERT_GT(logfiles.size(), 0UL);
|
||||
std::unique_ptr<LogFile> archived_log = std::move(logfiles.front());
|
||||
ASSERT_EQ(archived_log->Type(), kArchivedLogFile);
|
||||
ASSERT_TRUE(env_->FileExists(dbname_ + "/" + archived_log->PathName()));
|
||||
fprintf(stdout, "Deleting archived log file %s\n",
|
||||
archived_log->PathName().c_str());
|
||||
ASSERT_OK(db_->DeleteFile(archived_log->PathName()));
|
||||
ASSERT_TRUE(!env_->FileExists(dbname_ + "/" + archived_log->PathName()));
|
||||
CloseDB();
|
||||
}
|
||||
|
||||
} //namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
}
|
||||
|
||||
+9
-58
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -12,10 +7,10 @@
|
||||
#include <ctype.h>
|
||||
#include <stdio.h>
|
||||
#include "db/dbformat.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
// Given a path, flatten the path name by replacing all chars not in
|
||||
// {[0-9,a-z,A-Z,-,_,.]} with _. And append '\0' at the end.
|
||||
@@ -62,12 +57,12 @@ std::string LogFileName(const std::string& name, uint64_t number) {
|
||||
return MakeFileName(name, number, "log");
|
||||
}
|
||||
|
||||
std::string ArchivalDirectory(const std::string& dir) {
|
||||
return dir + "/" + ARCHIVAL_DIR;
|
||||
std::string ArchivalDirectory(const std::string& dbname) {
|
||||
return dbname + "/" + ARCHIVAL_DIR;
|
||||
}
|
||||
std::string ArchivedLogFileName(const std::string& name, uint64_t number) {
|
||||
assert(number > 0);
|
||||
return MakeFileName(name + "/" + ARCHIVAL_DIR, number, "log");
|
||||
return MakeFileName(name + "/archive", number, "log");
|
||||
}
|
||||
|
||||
std::string TableFileName(const std::string& name, uint64_t number) {
|
||||
@@ -92,7 +87,7 @@ std::string LockFileName(const std::string& dbname) {
|
||||
}
|
||||
|
||||
std::string TempFileName(const std::string& dbname, uint64_t number) {
|
||||
assert(number >= 0);
|
||||
assert(number > 0);
|
||||
return MakeFileName(dbname, number, "dbtmp");
|
||||
}
|
||||
|
||||
@@ -127,12 +122,7 @@ std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
|
||||
return dbname + buf;
|
||||
}
|
||||
|
||||
std::string IdentityFileName(const std::string& dbname) {
|
||||
return dbname + "/IDENTITY";
|
||||
}
|
||||
|
||||
// Owned filenames have the form:
|
||||
// dbname/IDENTITY
|
||||
// dbname/CURRENT
|
||||
// dbname/LOCK
|
||||
// dbname/LOG
|
||||
@@ -140,19 +130,11 @@ std::string IdentityFileName(const std::string& dbname) {
|
||||
// dbname/MANIFEST-[0-9]+
|
||||
// dbname/[0-9]+.(log|sst)
|
||||
// dbname/METADB-[0-9]+
|
||||
// Disregards / at the beginning
|
||||
bool ParseFileName(const std::string& fname,
|
||||
uint64_t* number,
|
||||
FileType* type,
|
||||
WalFileType* log_type) {
|
||||
FileType* type) {
|
||||
Slice rest(fname);
|
||||
if (fname.length() > 1 && fname[0] == '/') {
|
||||
rest.remove_prefix(1);
|
||||
}
|
||||
if (rest == "IDENTITY") {
|
||||
*number = 0;
|
||||
*type = kIdentityFile;
|
||||
} else if (rest == "CURRENT") {
|
||||
if (rest == "CURRENT") {
|
||||
*number = 0;
|
||||
*type = kCurrentFile;
|
||||
} else if (rest == "LOCK") {
|
||||
@@ -195,17 +177,6 @@ bool ParseFileName(const std::string& fname,
|
||||
} else {
|
||||
// Avoid strtoull() to keep filename format independent of the
|
||||
// current locale
|
||||
bool archive_dir_found = false;
|
||||
if (rest.starts_with(ARCHIVAL_DIR)) {
|
||||
if (rest.size() <= ARCHIVAL_DIR.size()) {
|
||||
return false;
|
||||
}
|
||||
rest.remove_prefix(ARCHIVAL_DIR.size() + 1); // Add 1 to remove / also
|
||||
if (log_type) {
|
||||
*log_type = kArchivedLogFile;
|
||||
}
|
||||
archive_dir_found = true;
|
||||
}
|
||||
uint64_t num;
|
||||
if (!ConsumeDecimalNumber(&rest, &num)) {
|
||||
return false;
|
||||
@@ -213,11 +184,6 @@ bool ParseFileName(const std::string& fname,
|
||||
Slice suffix = rest;
|
||||
if (suffix == Slice(".log")) {
|
||||
*type = kLogFile;
|
||||
if (log_type && !archive_dir_found) {
|
||||
*log_type = kAliveLogFile;
|
||||
}
|
||||
} else if (archive_dir_found) {
|
||||
return false; // Archive dir can contain only log files
|
||||
} else if (suffix == Slice(".sst")) {
|
||||
*type = kTableFile;
|
||||
} else if (suffix == Slice(".dbtmp")) {
|
||||
@@ -248,19 +214,4 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
|
||||
return s;
|
||||
}
|
||||
|
||||
Status SetIdentityFile(Env* env, const std::string& dbname) {
|
||||
std::string id = env->GenerateUniqueId();
|
||||
assert(!id.empty());
|
||||
// Reserve the filename dbname/000000.dbtmp for the temporary identity file
|
||||
std::string tmp = TempFileName(dbname, 0);
|
||||
Status s = WriteStringToFileSync(env, id, tmp);
|
||||
if (s.ok()) {
|
||||
s = env->RenameFile(tmp, IdentityFileName(dbname));
|
||||
}
|
||||
if (!s.ok()) {
|
||||
env->DeleteFile(tmp);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+12
-23
@@ -1,23 +1,19 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// File names used by DB code
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_FILENAME_H_
|
||||
#define STORAGE_LEVELDB_DB_FILENAME_H_
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
#include "rocksdb/slice.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "rocksdb/transaction_log.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "port/port.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class Env;
|
||||
|
||||
@@ -29,8 +25,7 @@ enum FileType {
|
||||
kCurrentFile,
|
||||
kTempFile,
|
||||
kInfoLogFile, // Either the current one, or an old one
|
||||
kMetaDatabase,
|
||||
kIdentityFile
|
||||
kMetaDatabase
|
||||
};
|
||||
|
||||
// Return the name of the log file with the specified number
|
||||
@@ -84,25 +79,19 @@ extern std::string OldInfoLogFileName(const std::string& dbname, uint64_t ts,
|
||||
extern std::string MetaDatabaseName(const std::string& dbname,
|
||||
uint64_t number);
|
||||
|
||||
// Return the name of the Identity file which stores a unique number for the db
|
||||
// that will get regenerated if the db loses all its data and is recreated fresh
|
||||
// either from a backup-image or empty
|
||||
extern std::string IdentityFileName(const std::string& dbname);
|
||||
|
||||
// If filename is a rocksdb file, store the type of the file in *type.
|
||||
// If filename is a leveldb file, store the type of the file in *type.
|
||||
// The number encoded in the filename is stored in *number. If the
|
||||
// filename was successfully parsed, returns true. Else return false.
|
||||
extern bool ParseFileName(const std::string& filename,
|
||||
uint64_t* number,
|
||||
FileType* type,
|
||||
WalFileType* log_type = nullptr);
|
||||
FileType* type);
|
||||
|
||||
// Make the CURRENT file point to the descriptor file with the
|
||||
// specified number.
|
||||
extern Status SetCurrentFile(Env* env, const std::string& dbname,
|
||||
uint64_t descriptor_number);
|
||||
|
||||
// Make the IDENTITY file for the db
|
||||
extern Status SetIdentityFile(Env* env, const std::string& dbname);
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_FILENAME_H_
|
||||
+3
-8
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -14,7 +9,7 @@
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class FileNameTest { };
|
||||
|
||||
@@ -133,8 +128,8 @@ TEST(FileNameTest, Construction) {
|
||||
ASSERT_EQ(kMetaDatabase, type);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2008-present Facebook. All Rights Reserved.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_FILE_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_FILE_H_
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
enum WalFileType {
|
||||
kArchivedLogFile = 0,
|
||||
kAliveLogFile = 1
|
||||
} ;
|
||||
|
||||
class LogFile {
|
||||
|
||||
public:
|
||||
uint64_t logNumber;
|
||||
WalFileType type;
|
||||
|
||||
LogFile(uint64_t logNum,WalFileType logType) :
|
||||
logNumber(logNum),
|
||||
type(logType) {}
|
||||
|
||||
LogFile(const LogFile& that) {
|
||||
logNumber = that.logNumber;
|
||||
type = that.type;
|
||||
}
|
||||
|
||||
bool operator < (const LogFile& that) const {
|
||||
return logNumber < that.logNumber;
|
||||
}
|
||||
|
||||
std::string ToString() const {
|
||||
char response[100];
|
||||
const char* typeOfLog;
|
||||
if (type == kAliveLogFile) {
|
||||
typeOfLog = "Alive Log";
|
||||
} else {
|
||||
typeOfLog = "Archived Log";
|
||||
}
|
||||
sprintf(response,
|
||||
"LogNumber : %ld LogType : %s",
|
||||
logNumber,
|
||||
typeOfLog);
|
||||
return std::string(response);
|
||||
}
|
||||
};
|
||||
} // namespace leveldb
|
||||
#endif // STORAGE_LEVELDB_DB_LOG_FILE_H_
|
||||
+8
-9
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -10,8 +5,10 @@
|
||||
// Log format information shared by reader and writer.
|
||||
// See ../doc/log_format.txt for more detail.
|
||||
|
||||
#pragma once
|
||||
namespace rocksdb {
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_FORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_FORMAT_H_
|
||||
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
enum RecordType {
|
||||
@@ -27,10 +24,12 @@ enum RecordType {
|
||||
};
|
||||
static const int kMaxRecordType = kLastType;
|
||||
|
||||
static const unsigned int kBlockSize = 32768;
|
||||
static const int kBlockSize = 32768;
|
||||
|
||||
// Header is checksum (4 bytes), type (1 byte), length (2 bytes).
|
||||
static const int kHeaderSize = 4 + 1 + 2;
|
||||
|
||||
} // namespace log
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_LOG_FORMAT_H_
|
||||
+3
-8
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -10,11 +5,11 @@
|
||||
#include "db/log_reader.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/crc32c.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
Reader::Reporter::~Reporter() {
|
||||
@@ -261,4 +256,4 @@ unsigned int Reader::ReadPhysicalRecord(Slice* result) {
|
||||
}
|
||||
|
||||
} // namespace log
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+9
-21
@@ -1,21 +1,18 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_READER_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_READER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "db/log_format.h"
|
||||
#include "rocksdb/slice.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class SequentialFile;
|
||||
using std::unique_ptr;
|
||||
@@ -62,17 +59,6 @@ class Reader {
|
||||
// Undefined before the first call to ReadRecord.
|
||||
uint64_t LastRecordOffset();
|
||||
|
||||
// returns true if the reader has encountered an eof condition.
|
||||
bool IsEOF() {
|
||||
return eof_;
|
||||
}
|
||||
|
||||
// when we know more data has been written to the file. we can use this
|
||||
// function to force the reader to look again in the file.
|
||||
void UnmarkEOF() {
|
||||
eof_ = false;
|
||||
}
|
||||
|
||||
SequentialFile* file() { return file_.get(); }
|
||||
|
||||
private:
|
||||
@@ -121,4 +107,6 @@ class Reader {
|
||||
};
|
||||
|
||||
} // namespace log
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_LOG_READER_H_
|
||||
+6
-11
@@ -1,21 +1,16 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/log_reader.h"
|
||||
#include "db/log_writer.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/crc32c.h"
|
||||
#include "util/random.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
// Construct a string of the specified length made out of the supplied
|
||||
@@ -455,13 +450,13 @@ TEST(LogTest, ErrorJoinsRecords) {
|
||||
Write("correct");
|
||||
|
||||
// Wipe the middle block
|
||||
for (unsigned int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
|
||||
for (int offset = kBlockSize; offset < 2*kBlockSize; offset++) {
|
||||
SetByte(offset, 'x');
|
||||
}
|
||||
|
||||
ASSERT_EQ("correct", Read());
|
||||
ASSERT_EQ("EOF", Read());
|
||||
const unsigned int dropped = DroppedBytes();
|
||||
const int dropped = DroppedBytes();
|
||||
ASSERT_LE(dropped, 2*kBlockSize + 100);
|
||||
ASSERT_GE(dropped, 2*kBlockSize);
|
||||
}
|
||||
@@ -521,8 +516,8 @@ TEST(LogTest, ReadPastEnd) {
|
||||
}
|
||||
|
||||
} // namespace log
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
+3
-8
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -10,11 +5,11 @@
|
||||
#include "db/log_writer.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/crc32c.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
namespace log {
|
||||
|
||||
Writer::Writer(unique_ptr<WritableFile>&& dest)
|
||||
@@ -105,4 +100,4 @@ Status Writer::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n) {
|
||||
}
|
||||
|
||||
} // namespace log
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+9
-10
@@ -1,20 +1,17 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_LOG_WRITER_H_
|
||||
#define STORAGE_LEVELDB_DB_LOG_WRITER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <stdint.h>
|
||||
#include "db/log_format.h"
|
||||
#include "rocksdb/slice.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class WritableFile;
|
||||
|
||||
@@ -52,4 +49,6 @@ class Writer {
|
||||
};
|
||||
|
||||
} // namespace log
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_LOG_WRITER_H_
|
||||
+41
-215
@@ -1,63 +1,39 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/memtable.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/iterator.h"
|
||||
#include "rocksdb/merge_operator.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/murmurhash.h"
|
||||
|
||||
namespace std {
|
||||
template <>
|
||||
struct hash<rocksdb::Slice> {
|
||||
size_t operator()(const rocksdb::Slice& slice) const {
|
||||
return MurmurHash(slice.data(), slice.size(), 0);
|
||||
}
|
||||
};
|
||||
namespace leveldb {
|
||||
|
||||
static Slice GetLengthPrefixedSlice(const char* data) {
|
||||
uint32_t len;
|
||||
const char* p = data;
|
||||
p = GetVarint32Ptr(p, p + 5, &len); // +5: we assume "p" is not corrupted
|
||||
return Slice(p, len);
|
||||
}
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
MemTable::MemTable(const InternalKeyComparator& cmp,
|
||||
std::shared_ptr<MemTableRepFactory> table_factory,
|
||||
int numlevel,
|
||||
const Options& options)
|
||||
MemTable::MemTable(const InternalKeyComparator& cmp, int numlevel)
|
||||
: comparator_(cmp),
|
||||
refs_(0),
|
||||
arena_impl_(options.arena_block_size),
|
||||
table_(table_factory->CreateMemTableRep(comparator_, &arena_impl_)),
|
||||
table_(comparator_, &arena_),
|
||||
flush_in_progress_(false),
|
||||
flush_completed_(false),
|
||||
file_number_(0),
|
||||
edit_(numlevel),
|
||||
first_seqno_(0),
|
||||
mem_next_logfile_number_(0),
|
||||
mem_logfile_number_(0),
|
||||
locks_(options.inplace_update_support
|
||||
? options.inplace_update_num_locks
|
||||
: 0) { }
|
||||
first_seqno_(0) {
|
||||
}
|
||||
|
||||
MemTable::~MemTable() {
|
||||
assert(refs_ == 0);
|
||||
}
|
||||
|
||||
size_t MemTable::ApproximateMemoryUsage() {
|
||||
return arena_impl_.ApproximateMemoryUsage() +
|
||||
table_->ApproximateMemoryUsage();
|
||||
}
|
||||
size_t MemTable::ApproximateMemoryUsage() { return arena_.MemoryUsage(); }
|
||||
|
||||
int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
|
||||
const {
|
||||
@@ -67,11 +43,6 @@ int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
|
||||
return comparator.Compare(a, b);
|
||||
}
|
||||
|
||||
Slice MemTableRep::UserKey(const char* key) const {
|
||||
Slice slice = GetLengthPrefixedSlice(key);
|
||||
return Slice(slice.data(), slice.size() - 8);
|
||||
}
|
||||
|
||||
// Encode a suitable internal key target for "target" and return it.
|
||||
// Uses *scratch as scratch space, and the returned pointer will point
|
||||
// into this scratch space.
|
||||
@@ -84,35 +55,24 @@ static const char* EncodeKey(std::string* scratch, const Slice& target) {
|
||||
|
||||
class MemTableIterator: public Iterator {
|
||||
public:
|
||||
MemTableIterator(MemTableRep* table, const ReadOptions& options)
|
||||
: iter_() {
|
||||
if (options.prefix) {
|
||||
iter_ = table->GetPrefixIterator(*options.prefix);
|
||||
} else if (options.prefix_seek) {
|
||||
iter_ = table->GetDynamicPrefixIterator();
|
||||
} else {
|
||||
iter_ = table->GetIterator();
|
||||
}
|
||||
}
|
||||
explicit MemTableIterator(MemTable::Table* table) : iter_(table) { }
|
||||
|
||||
virtual bool Valid() const { return iter_->Valid(); }
|
||||
virtual void Seek(const Slice& k) { iter_->Seek(EncodeKey(&tmp_, k)); }
|
||||
virtual void SeekToFirst() { iter_->SeekToFirst(); }
|
||||
virtual void SeekToLast() { iter_->SeekToLast(); }
|
||||
virtual void Next() { iter_->Next(); }
|
||||
virtual void Prev() { iter_->Prev(); }
|
||||
virtual Slice key() const {
|
||||
return GetLengthPrefixedSlice(iter_->key());
|
||||
}
|
||||
virtual bool Valid() const { return iter_.Valid(); }
|
||||
virtual void Seek(const Slice& k) { iter_.Seek(EncodeKey(&tmp_, k)); }
|
||||
virtual void SeekToFirst() { iter_.SeekToFirst(); }
|
||||
virtual void SeekToLast() { iter_.SeekToLast(); }
|
||||
virtual void Next() { iter_.Next(); }
|
||||
virtual void Prev() { iter_.Prev(); }
|
||||
virtual Slice key() const { return GetLengthPrefixedSlice(iter_.key()); }
|
||||
virtual Slice value() const {
|
||||
Slice key_slice = GetLengthPrefixedSlice(iter_->key());
|
||||
Slice key_slice = GetLengthPrefixedSlice(iter_.key());
|
||||
return GetLengthPrefixedSlice(key_slice.data() + key_slice.size());
|
||||
}
|
||||
|
||||
virtual Status status() const { return Status::OK(); }
|
||||
|
||||
private:
|
||||
std::shared_ptr<MemTableRep::Iterator> iter_;
|
||||
MemTable::Table::Iterator iter_;
|
||||
std::string tmp_; // For passing to EncodeKey
|
||||
|
||||
// No copying allowed
|
||||
@@ -120,12 +80,8 @@ class MemTableIterator: public Iterator {
|
||||
void operator=(const MemTableIterator&);
|
||||
};
|
||||
|
||||
Iterator* MemTable::NewIterator(const ReadOptions& options) {
|
||||
return new MemTableIterator(table_.get(), options);
|
||||
}
|
||||
|
||||
port::RWMutex* MemTable::GetLock(const Slice& key) {
|
||||
return &locks_[std::hash<Slice>()(key) % locks_.size()];
|
||||
Iterator* MemTable::NewIterator() {
|
||||
return new MemTableIterator(&table_);
|
||||
}
|
||||
|
||||
void MemTable::Add(SequenceNumber s, ValueType type,
|
||||
@@ -142,7 +98,7 @@ void MemTable::Add(SequenceNumber s, ValueType type,
|
||||
const size_t encoded_len =
|
||||
VarintLength(internal_key_size) + internal_key_size +
|
||||
VarintLength(val_size) + val_size;
|
||||
char* buf = arena_impl_.Allocate(encoded_len);
|
||||
char* buf = arena_.Allocate(encoded_len);
|
||||
char* p = EncodeVarint32(buf, internal_key_size);
|
||||
memcpy(p, key.data(), key_size);
|
||||
p += key_size;
|
||||
@@ -151,7 +107,7 @@ void MemTable::Add(SequenceNumber s, ValueType type,
|
||||
p = EncodeVarint32(p, val_size);
|
||||
memcpy(p, value.data(), val_size);
|
||||
assert((p + val_size) - buf == (unsigned)encoded_len);
|
||||
table_->Insert(buf);
|
||||
table_.Insert(buf);
|
||||
|
||||
// The first sequence number inserted into the memtable
|
||||
assert(first_seqno_ == 0 || s > first_seqno_);
|
||||
@@ -160,22 +116,11 @@ void MemTable::Add(SequenceNumber s, ValueType type,
|
||||
}
|
||||
}
|
||||
|
||||
bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
|
||||
std::deque<std::string>* operands, const Options& options) {
|
||||
bool MemTable::Get(const LookupKey& key, std::string* value, Status* s) {
|
||||
Slice memkey = key.memtable_key();
|
||||
std::shared_ptr<MemTableRep::Iterator> iter(
|
||||
table_->GetIterator(key.user_key()));
|
||||
iter->Seek(memkey.data());
|
||||
|
||||
// It is the caller's responsibility to allocate/delete operands list
|
||||
assert(operands != nullptr);
|
||||
|
||||
bool merge_in_progress = s->IsMergeInProgress();
|
||||
auto merge_operator = options.merge_operator.get();
|
||||
auto logger = options.info_log;
|
||||
std::string merge_result;
|
||||
|
||||
for (; iter->Valid(); iter->Next()) {
|
||||
Table::Iterator iter(&table_);
|
||||
iter.Seek(memkey.data());
|
||||
if (iter.Valid()) {
|
||||
// entry format is:
|
||||
// klength varint32
|
||||
// userkey char[klength-8]
|
||||
@@ -185,146 +130,27 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
|
||||
// Check that it belongs to same user key. We do not check the
|
||||
// sequence number since the Seek() call above should have skipped
|
||||
// all entries with overly large sequence numbers.
|
||||
const char* entry = iter->key();
|
||||
const char* entry = iter.key();
|
||||
uint32_t key_length;
|
||||
const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
|
||||
const char* key_ptr = GetVarint32Ptr(entry, entry+5, &key_length);
|
||||
if (comparator_.comparator.user_comparator()->Compare(
|
||||
Slice(key_ptr, key_length - 8), key.user_key()) == 0) {
|
||||
Slice(key_ptr, key_length - 8),
|
||||
key.user_key()) == 0) {
|
||||
// Correct user key
|
||||
const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
|
||||
switch (static_cast<ValueType>(tag & 0xff)) {
|
||||
case kTypeValue: {
|
||||
if (options.inplace_update_support) {
|
||||
GetLock(key.user_key())->ReadLock();
|
||||
}
|
||||
Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
|
||||
*s = Status::OK();
|
||||
if (merge_in_progress) {
|
||||
assert(merge_operator);
|
||||
if (!merge_operator->FullMerge(key.user_key(), &v, *operands,
|
||||
value, logger.get())) {
|
||||
RecordTick(options.statistics, NUMBER_MERGE_FAILURES);
|
||||
*s = Status::Corruption("Error: Could not perform merge.");
|
||||
}
|
||||
} else {
|
||||
value->assign(v.data(), v.size());
|
||||
}
|
||||
if (options.inplace_update_support) {
|
||||
GetLock(key.user_key())->Unlock();
|
||||
}
|
||||
value->assign(v.data(), v.size());
|
||||
return true;
|
||||
}
|
||||
case kTypeDeletion: {
|
||||
if (merge_in_progress) {
|
||||
assert(merge_operator);
|
||||
*s = Status::OK();
|
||||
if (!merge_operator->FullMerge(key.user_key(), nullptr, *operands,
|
||||
value, logger.get())) {
|
||||
RecordTick(options.statistics, NUMBER_MERGE_FAILURES);
|
||||
*s = Status::Corruption("Error: Could not perform merge.");
|
||||
}
|
||||
} else {
|
||||
*s = Status::NotFound(Slice());
|
||||
}
|
||||
case kTypeDeletion:
|
||||
*s = Status::NotFound(Slice());
|
||||
return true;
|
||||
}
|
||||
case kTypeMerge: {
|
||||
Slice v = GetLengthPrefixedSlice(key_ptr + key_length);
|
||||
merge_in_progress = true;
|
||||
operands->push_front(v.ToString());
|
||||
while(operands->size() >= 2) {
|
||||
// Attempt to associative merge. (Returns true if successful)
|
||||
if (merge_operator->PartialMerge(key.user_key(),
|
||||
Slice((*operands)[0]),
|
||||
Slice((*operands)[1]),
|
||||
&merge_result,
|
||||
logger.get())) {
|
||||
operands->pop_front();
|
||||
swap(operands->front(), merge_result);
|
||||
} else {
|
||||
// Stack them because user can't associative merge
|
||||
break;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case kTypeLogData:
|
||||
assert(false);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// exit loop if user key does not match
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// No change to value, since we have not yet found a Put/Delete
|
||||
|
||||
if (merge_in_progress) {
|
||||
*s = Status::MergeInProgress("");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool MemTable::Update(SequenceNumber seq, ValueType type,
|
||||
const Slice& key,
|
||||
const Slice& value) {
|
||||
LookupKey lkey(key, seq);
|
||||
Slice memkey = lkey.memtable_key();
|
||||
|
||||
std::shared_ptr<MemTableRep::Iterator> iter(
|
||||
table_.get()->GetIterator(lkey.user_key()));
|
||||
iter->Seek(memkey.data());
|
||||
|
||||
if (iter->Valid()) {
|
||||
// entry format is:
|
||||
// klength varint32
|
||||
// userkey char[klength-8]
|
||||
// tag uint64
|
||||
// vlength varint32
|
||||
// value char[vlength]
|
||||
// Check that it belongs to same user key. We do not check the
|
||||
// sequence number since the Seek() call above should have skipped
|
||||
// all entries with overly large sequence numbers.
|
||||
const char* entry = iter->key();
|
||||
uint32_t key_length;
|
||||
const char* key_ptr = GetVarint32Ptr(entry, entry + 5, &key_length);
|
||||
if (comparator_.comparator.user_comparator()->Compare(
|
||||
Slice(key_ptr, key_length - 8), lkey.user_key()) == 0) {
|
||||
// Correct user key
|
||||
const uint64_t tag = DecodeFixed64(key_ptr + key_length - 8);
|
||||
switch (static_cast<ValueType>(tag & 0xff)) {
|
||||
case kTypeValue: {
|
||||
uint32_t vlength;
|
||||
GetVarint32Ptr(key_ptr + key_length,
|
||||
key_ptr + key_length+5, &vlength);
|
||||
// Update value, if newValue size <= curValue size
|
||||
if (value.size() <= vlength) {
|
||||
char* p = EncodeVarint32(const_cast<char*>(key_ptr) + key_length,
|
||||
value.size());
|
||||
WriteLock wl(GetLock(lkey.user_key()));
|
||||
memcpy(p, value.data(), value.size());
|
||||
assert(
|
||||
(p + value.size()) - entry ==
|
||||
(unsigned) (VarintLength(key_length) +
|
||||
key_length +
|
||||
VarintLength(value.size()) +
|
||||
value.size())
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
default:
|
||||
// If the latest value is kTypeDeletion, kTypeMerge or kTypeLogData
|
||||
// then we probably don't have enough space to update in-place
|
||||
// Maybe do something later
|
||||
// Return false, and do normal Add()
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Key doesn't exist
|
||||
return false;
|
||||
}
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+24
-83
@@ -1,43 +1,29 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_MEMTABLE_H_
|
||||
#define STORAGE_LEVELDB_DB_MEMTABLE_H_
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <deque>
|
||||
#include "leveldb/db.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/skiplist.h"
|
||||
#include "db/version_set.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/memtablerep.h"
|
||||
#include "util/arena_impl.h"
|
||||
#include "util/arena.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class InternalKeyComparator;
|
||||
class Mutex;
|
||||
class MemTableIterator;
|
||||
|
||||
class MemTable {
|
||||
public:
|
||||
struct KeyComparator : public MemTableRep::KeyComparator {
|
||||
const InternalKeyComparator comparator;
|
||||
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
|
||||
virtual int operator()(const char* a, const char* b) const;
|
||||
};
|
||||
|
||||
// MemTables are reference counted. The initial reference count
|
||||
// is zero and the caller must call Ref() at least once.
|
||||
explicit MemTable(
|
||||
const InternalKeyComparator& comparator,
|
||||
std::shared_ptr<MemTableRepFactory> table_factory,
|
||||
int numlevel = 7,
|
||||
const Options& options = Options());
|
||||
explicit MemTable(const InternalKeyComparator& comparator,
|
||||
int numlevel = 7);
|
||||
|
||||
// Increase reference count.
|
||||
void Ref() { ++refs_; }
|
||||
@@ -64,15 +50,7 @@ class MemTable {
|
||||
// while the returned iterator is live. The keys returned by this
|
||||
// iterator are internal keys encoded by AppendInternalKey in the
|
||||
// db/dbformat.{h,cc} module.
|
||||
//
|
||||
// If options.prefix is supplied, it is passed to the underlying MemTableRep
|
||||
// as a hint that the iterator only need to support access to keys with that
|
||||
// specific prefix.
|
||||
// If options.prefix is not supplied and options.prefix_seek is set, the
|
||||
// iterator is not bound to a specific prefix. However, the semantics of
|
||||
// Seek is changed - the result might only include keys with the same prefix
|
||||
// as the seek-key.
|
||||
Iterator* NewIterator(const ReadOptions& options = ReadOptions());
|
||||
Iterator* NewIterator();
|
||||
|
||||
// Add an entry into memtable that maps key to value at the
|
||||
// specified sequence number and with the specified type.
|
||||
@@ -84,23 +62,8 @@ class MemTable {
|
||||
// If memtable contains a value for key, store it in *value and return true.
|
||||
// If memtable contains a deletion for key, store a NotFound() error
|
||||
// in *status and return true.
|
||||
// If memtable contains Merge operation as the most recent entry for a key,
|
||||
// and the merge process does not stop (not reaching a value or delete),
|
||||
// prepend the current merge operand to *operands.
|
||||
// store MergeInProgress in s, and return false.
|
||||
// Else, return false.
|
||||
bool Get(const LookupKey& key, std::string* value, Status* s,
|
||||
std::deque<std::string>* operands, const Options& options);
|
||||
|
||||
// Update the value and return status ok,
|
||||
// if key exists in current memtable
|
||||
// if new sizeof(new_value) <= sizeof(old_value) &&
|
||||
// old_value for that key is a put i.e. kTypeValue
|
||||
// else return false, and status - NotUpdatable()
|
||||
// else return false, and status - NotFound()
|
||||
bool Update(SequenceNumber seq, ValueType type,
|
||||
const Slice& key,
|
||||
const Slice& value);
|
||||
bool Get(const LookupKey& key, std::string* value, Status* s);
|
||||
|
||||
// Returns the edits area that is needed for flushing the memtable
|
||||
VersionEdit* GetEdits() { return &edit_; }
|
||||
@@ -109,35 +72,24 @@ class MemTable {
|
||||
// into the memtable
|
||||
SequenceNumber GetFirstSequenceNumber() { return first_seqno_; }
|
||||
|
||||
// Returns the next active logfile number when this memtable is about to
|
||||
// be flushed to storage
|
||||
uint64_t GetNextLogNumber() { return mem_next_logfile_number_; }
|
||||
|
||||
// Sets the next active logfile number when this memtable is about to
|
||||
// be flushed to storage
|
||||
void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
|
||||
|
||||
// Returns the logfile number that can be safely deleted when this
|
||||
// memstore is flushed to storage
|
||||
uint64_t GetLogNumber() { return mem_logfile_number_; }
|
||||
|
||||
// Sets the logfile number that can be safely deleted when this
|
||||
// memstore is flushed to storage
|
||||
void SetLogNumber(uint64_t num) { mem_logfile_number_ = num; }
|
||||
|
||||
// Notify the underlying storage that no more items will be added
|
||||
void MarkImmutable() { table_->MarkReadOnly(); }
|
||||
|
||||
private:
|
||||
~MemTable(); // Private since only Unref() should be used to delete it
|
||||
|
||||
struct KeyComparator {
|
||||
const InternalKeyComparator comparator;
|
||||
explicit KeyComparator(const InternalKeyComparator& c) : comparator(c) { }
|
||||
int operator()(const char* a, const char* b) const;
|
||||
};
|
||||
friend class MemTableIterator;
|
||||
friend class MemTableBackwardIterator;
|
||||
friend class MemTableList;
|
||||
|
||||
typedef SkipList<const char*, KeyComparator> Table;
|
||||
|
||||
KeyComparator comparator_;
|
||||
int refs_;
|
||||
ArenaImpl arena_impl_;
|
||||
shared_ptr<MemTableRep> table_;
|
||||
Arena arena_;
|
||||
Table table_;
|
||||
|
||||
// These are used to manage memtable flushes to storage
|
||||
bool flush_in_progress_; // started the flush
|
||||
@@ -151,22 +103,11 @@ class MemTable {
|
||||
// The sequence number of the kv that was inserted first
|
||||
SequenceNumber first_seqno_;
|
||||
|
||||
// The log files earlier than this number can be deleted.
|
||||
uint64_t mem_next_logfile_number_;
|
||||
|
||||
// The log file that backs this memtable (to be deleted when
|
||||
// memtable flush is done)
|
||||
uint64_t mem_logfile_number_;
|
||||
|
||||
// rw locks for inplace updates
|
||||
std::vector<port::RWMutex> locks_;
|
||||
|
||||
// No copying allowed
|
||||
MemTable(const MemTable&);
|
||||
void operator=(const MemTable&);
|
||||
|
||||
// Get the lock associated for the key
|
||||
port::RWMutex* GetLock(const Slice& key);
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_MEMTABLE_H_
|
||||
+76
-95
@@ -1,18 +1,16 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2012 Facebook.
|
||||
|
||||
|
||||
#include "db/memtablelist.h"
|
||||
|
||||
#include <string>
|
||||
#include "rocksdb/db.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "db/memtable.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/iterator.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class InternalKeyComparator;
|
||||
class Mutex;
|
||||
@@ -23,15 +21,17 @@ using std::list;
|
||||
|
||||
// Increase reference count on all underling memtables
|
||||
void MemTableList::RefAll() {
|
||||
for (auto &memtable : memlist_) {
|
||||
memtable->Ref();
|
||||
for (list<MemTable*>::iterator it = memlist_.begin();
|
||||
it != memlist_.end() ; ++it) {
|
||||
(*it)->Ref();
|
||||
}
|
||||
}
|
||||
|
||||
// Drop reference count on all underling memtables
|
||||
void MemTableList::UnrefAll() {
|
||||
for (auto &memtable : memlist_) {
|
||||
memtable->Unref();
|
||||
for (list<MemTable*>::iterator it = memlist_.begin();
|
||||
it != memlist_.end() ; ++it) {
|
||||
(*it)->Unref();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,18 +43,19 @@ int MemTableList::size() {
|
||||
|
||||
// Returns true if there is at least one memtable on which flush has
|
||||
// not yet started.
|
||||
bool MemTableList::IsFlushPending(int min_write_buffer_number_to_merge) {
|
||||
if ((flush_requested_ && num_flush_not_started_ >= 1) ||
|
||||
(num_flush_not_started_ >= min_write_buffer_number_to_merge)) {
|
||||
bool MemTableList::IsFlushPending() {
|
||||
if (num_flush_not_started_ > 0) {
|
||||
assert(imm_flush_needed.NoBarrier_Load() != nullptr);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Returns the memtables that need to be flushed.
|
||||
void MemTableList::PickMemtablesToFlush(std::vector<MemTable*>* ret) {
|
||||
for (auto it = memlist_.rbegin(); it != memlist_.rend(); it++) {
|
||||
// Returns the earliest memtable that needs to be flushed.
|
||||
// Returns null, if no such memtable exist.
|
||||
MemTable* MemTableList::PickMemtableToFlush() {
|
||||
for (list<MemTable*>::reverse_iterator it = memlist_.rbegin();
|
||||
it != memlist_.rend(); it++) {
|
||||
MemTable* m = *it;
|
||||
if (!m->flush_in_progress_) {
|
||||
assert(!m->flush_completed_);
|
||||
@@ -63,46 +64,37 @@ void MemTableList::PickMemtablesToFlush(std::vector<MemTable*>* ret) {
|
||||
imm_flush_needed.Release_Store(nullptr);
|
||||
}
|
||||
m->flush_in_progress_ = true; // flushing will start very soon
|
||||
ret->push_back(m);
|
||||
return m;
|
||||
}
|
||||
}
|
||||
flush_requested_ = false; // start-flush request is complete
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Record a successful flush in the manifest file
|
||||
Status MemTableList::InstallMemtableFlushResults(
|
||||
const std::vector<MemTable*> &mems,
|
||||
Status MemTableList::InstallMemtableFlushResults(MemTable* m,
|
||||
VersionSet* vset, Status flushStatus,
|
||||
port::Mutex* mu, Logger* info_log,
|
||||
uint64_t file_number,
|
||||
std::set<uint64_t>& pending_outputs) {
|
||||
mu->AssertHeld();
|
||||
assert(m->flush_in_progress_);
|
||||
assert(m->file_number_ == 0);
|
||||
|
||||
// If the flush was not successful, then just reset state.
|
||||
// Maybe a suceeding attempt to flush will be successful.
|
||||
if (!flushStatus.ok()) {
|
||||
for (MemTable* m : mems) {
|
||||
assert(m->flush_in_progress_);
|
||||
assert(m->file_number_ == 0);
|
||||
|
||||
m->flush_in_progress_ = false;
|
||||
m->flush_completed_ = false;
|
||||
m->edit_.Clear();
|
||||
num_flush_not_started_++;
|
||||
imm_flush_needed.Release_Store((void *)1);
|
||||
pending_outputs.erase(file_number);
|
||||
}
|
||||
m->flush_in_progress_ = false;
|
||||
m->flush_completed_ = false;
|
||||
m->edit_.Clear();
|
||||
num_flush_not_started_++;
|
||||
imm_flush_needed.Release_Store((void *)1);
|
||||
pending_outputs.erase(file_number);
|
||||
return flushStatus;
|
||||
}
|
||||
|
||||
// flush was sucessful
|
||||
for (size_t i = 0; i < mems.size(); ++i) {
|
||||
// All the edits are associated with the first memtable of this batch.
|
||||
assert(i == 0 || mems[i]->GetEdits()->NumEntries() == 0);
|
||||
|
||||
mems[i]->flush_completed_ = true;
|
||||
mems[i]->file_number_ = file_number;
|
||||
}
|
||||
m->flush_completed_ = true;
|
||||
m->file_number_ = file_number;
|
||||
|
||||
// if some other thread is already commiting, then return
|
||||
Status s;
|
||||
@@ -114,56 +106,46 @@ Status MemTableList::InstallMemtableFlushResults(
|
||||
commit_in_progress_ = true;
|
||||
|
||||
// scan all memtables from the earliest, and commit those
|
||||
// (in that order) that have finished flushing. Memetables
|
||||
// are always committed in the order that they were created.
|
||||
while (!memlist_.empty() && s.ok()) {
|
||||
MemTable* m = memlist_.back(); // get the last element
|
||||
// (in that order) that have finished flushing.
|
||||
while (!memlist_.empty()) {
|
||||
m = memlist_.back(); // get the last element
|
||||
if (!m->flush_completed_) {
|
||||
break;
|
||||
}
|
||||
|
||||
Log(info_log, "Level-0 commit table #%lu started", m->file_number_);
|
||||
Log(info_log,
|
||||
"Level-0 commit table #%llu: started",
|
||||
(unsigned long long)m->file_number_);
|
||||
|
||||
// this can release and reacquire the mutex.
|
||||
s = vset->LogAndApply(&m->edit_, mu);
|
||||
|
||||
// All the later memtables that have the same filenum
|
||||
// are part of the same batch. They can be committed now.
|
||||
uint64_t mem_id = 1; // how many memtables has been flushed.
|
||||
do {
|
||||
if (s.ok()) { // commit new state
|
||||
Log(info_log,
|
||||
"Level-0 commit table #%lu: memtable #%lu done",
|
||||
m->file_number_,
|
||||
mem_id);
|
||||
memlist_.remove(m);
|
||||
assert(m->file_number_ > 0);
|
||||
if (s.ok()) { // commit new state
|
||||
Log(info_log, "Level-0 commit table #%llu: done",
|
||||
(unsigned long long)m->file_number_);
|
||||
memlist_.remove(m);
|
||||
assert(m->file_number_ > 0);
|
||||
|
||||
// pending_outputs can be cleared only after the newly created file
|
||||
// has been written to a committed version so that other concurrently
|
||||
// executing compaction threads do not mistakenly assume that this
|
||||
// file is not live.
|
||||
pending_outputs.erase(m->file_number_);
|
||||
m->Unref();
|
||||
size_--;
|
||||
} else {
|
||||
//commit failed. setup state so that we can flush again.
|
||||
Log(info_log,
|
||||
"Level-0 commit table #%lu: memtable #%lu failed",
|
||||
m->file_number_,
|
||||
mem_id);
|
||||
m->flush_completed_ = false;
|
||||
m->flush_in_progress_ = false;
|
||||
m->edit_.Clear();
|
||||
num_flush_not_started_++;
|
||||
pending_outputs.erase(m->file_number_);
|
||||
m->file_number_ = 0;
|
||||
imm_flush_needed.Release_Store((void *)1);
|
||||
s = Status::IOError("Unable to commit flushed memtable");
|
||||
}
|
||||
++mem_id;
|
||||
} while (!memlist_.empty() && (m = memlist_.back()) &&
|
||||
m->file_number_ == file_number);
|
||||
// pending_outputs can be cleared only after the newly created file
|
||||
// has been written to a committed version so that other concurrently
|
||||
// executing compaction threads do not mistakenly assume that this
|
||||
// file is not live.
|
||||
pending_outputs.erase(m->file_number_);
|
||||
m->Unref();
|
||||
size_--;
|
||||
} else {
|
||||
//commit failed. setup state so that we can flush again.
|
||||
Log(info_log, "Level-0 commit table #%llu: failed",
|
||||
(unsigned long long)m->file_number_);
|
||||
m->flush_completed_ = false;
|
||||
m->flush_in_progress_ = false;
|
||||
m->edit_.Clear();
|
||||
num_flush_not_started_++;
|
||||
pending_outputs.erase(m->file_number_);
|
||||
m->file_number_ = 0;
|
||||
imm_flush_needed.Release_Store((void *)1);
|
||||
s = Status::IOError("Unable to commit flushed memtable");
|
||||
break;
|
||||
}
|
||||
}
|
||||
commit_in_progress_ = false;
|
||||
return s;
|
||||
@@ -174,7 +156,6 @@ void MemTableList::Add(MemTable* m) {
|
||||
assert(size_ >= num_flush_not_started_);
|
||||
size_++;
|
||||
memlist_.push_front(m);
|
||||
m->MarkImmutable();
|
||||
num_flush_not_started_++;
|
||||
if (num_flush_not_started_ == 1) {
|
||||
imm_flush_needed.Release_Store((void *)1);
|
||||
@@ -184,20 +165,19 @@ void MemTableList::Add(MemTable* m) {
|
||||
// Returns an estimate of the number of bytes of data in use.
|
||||
size_t MemTableList::ApproximateMemoryUsage() {
|
||||
size_t size = 0;
|
||||
for (auto &memtable : memlist_) {
|
||||
size += memtable->ApproximateMemoryUsage();
|
||||
for (list<MemTable*>::iterator it = memlist_.begin();
|
||||
it != memlist_.end(); ++it) {
|
||||
size += (*it)->ApproximateMemoryUsage();
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
// Search all the memtables starting from the most recent one.
|
||||
// Return the most recent value found, if any.
|
||||
// Operands stores the list of merge operations to apply, so far.
|
||||
bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s,
|
||||
std::deque<std::string>* operands,
|
||||
const Options& options) {
|
||||
for (auto &memtable : memlist_) {
|
||||
if (memtable->Get(key, value, s, operands, options)) {
|
||||
bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s) {
|
||||
for (list<MemTable*>::iterator it = memlist_.begin();
|
||||
it != memlist_.end(); ++it) {
|
||||
if ((*it)->Get(key, value, s)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -205,9 +185,10 @@ bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s,
|
||||
}
|
||||
|
||||
void MemTableList::GetMemTables(std::vector<MemTable*>* output) {
|
||||
for (auto &memtable : memlist_) {
|
||||
output->push_back(memtable);
|
||||
for (list<MemTable*>::iterator it = memlist_.begin();
|
||||
it != memlist_.end(); ++it) {
|
||||
output->push_back(*it);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+18
-26
@@ -1,26 +1,24 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2012 Facebook.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_MEMTABLELIST_H_
|
||||
#define STORAGE_LEVELDB_DB_MEMTABLELIST_H_
|
||||
|
||||
#pragma once
|
||||
#include <string>
|
||||
#include <list>
|
||||
#include <deque>
|
||||
#include "rocksdb/db.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/skiplist.h"
|
||||
#include "util/arena.h"
|
||||
#include "memtable.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class InternalKeyComparator;
|
||||
class Mutex;
|
||||
class MemTableListIterator;
|
||||
|
||||
//
|
||||
// This class stores references to all the immutable memtables.
|
||||
// This class stores refeernces to all the immutable memtables.
|
||||
// The memtables are flushed to L0 as soon as possible and in
|
||||
// any order. If there are more than one immutable memtable, their
|
||||
// flushes can occur concurrently. However, they are 'committed'
|
||||
@@ -31,8 +29,7 @@ class MemTableList {
|
||||
public:
|
||||
// A list of memtables.
|
||||
MemTableList() : size_(0), num_flush_not_started_(0),
|
||||
commit_in_progress_(false),
|
||||
flush_requested_(false) {
|
||||
commit_in_progress_(false) {
|
||||
imm_flush_needed.Release_Store(nullptr);
|
||||
}
|
||||
~MemTableList() {};
|
||||
@@ -52,14 +49,14 @@ class MemTableList {
|
||||
|
||||
// Returns true if there is at least one memtable on which flush has
|
||||
// not yet started.
|
||||
bool IsFlushPending(int min_write_buffer_number_to_merge);
|
||||
bool IsFlushPending();
|
||||
|
||||
// Returns the earliest memtables that needs to be flushed. The returned
|
||||
// memtables are guaranteed to be in the ascending order of created time.
|
||||
void PickMemtablesToFlush(std::vector<MemTable*>* mems);
|
||||
// Returns the earliest memtable that needs to be flushed.
|
||||
// Returns null, if no such memtable exist.
|
||||
MemTable* PickMemtableToFlush();
|
||||
|
||||
// Commit a successful flush in the manifest file
|
||||
Status InstallMemtableFlushResults(const std::vector<MemTable*> &m,
|
||||
Status InstallMemtableFlushResults(MemTable* m,
|
||||
VersionSet* vset, Status flushStatus,
|
||||
port::Mutex* mu, Logger* info_log,
|
||||
uint64_t file_number,
|
||||
@@ -74,15 +71,11 @@ class MemTableList {
|
||||
|
||||
// Search all the memtables starting from the most recent one.
|
||||
// Return the most recent value found, if any.
|
||||
bool Get(const LookupKey& key, std::string* value, Status* s,
|
||||
std::deque<std::string>* operands, const Options& options);
|
||||
bool Get(const LookupKey& key, std::string* value, Status* s);
|
||||
|
||||
// Returns the list of underlying memtables.
|
||||
void GetMemTables(std::vector<MemTable*>* list);
|
||||
|
||||
// Request a flush of all existing memtables to storage
|
||||
void FlushRequested() { flush_requested_ = true; }
|
||||
|
||||
// Copying allowed
|
||||
// MemTableList(const MemTableList&);
|
||||
// void operator=(const MemTableList&);
|
||||
@@ -97,9 +90,8 @@ class MemTableList {
|
||||
// committing in progress
|
||||
bool commit_in_progress_;
|
||||
|
||||
// Requested a flush of all memtables to storage
|
||||
bool flush_requested_;
|
||||
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_MEMTABLELIST_H_
|
||||
@@ -1,197 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
#include "merge_helper.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/merge_operator.h"
|
||||
#include <string>
|
||||
#include <stdio.h>
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
// PRE: iter points to the first merge type entry
|
||||
// POST: iter points to the first entry beyond the merge process (or the end)
|
||||
// keys_, operands_ are updated to reflect the merge result.
|
||||
// keys_ stores the list of keys encountered while merging.
|
||||
// operands_ stores the list of merge operands encountered while merging.
|
||||
// keys_[i] corresponds to operands_[i] for each i.
|
||||
void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
|
||||
bool at_bottom, shared_ptr<Statistics> stats) {
|
||||
// Get a copy of the internal key, before it's invalidated by iter->Next()
|
||||
// Also maintain the list of merge operands seen.
|
||||
keys_.clear();
|
||||
operands_.clear();
|
||||
keys_.push_front(iter->key().ToString());
|
||||
operands_.push_front(iter->value().ToString());
|
||||
|
||||
success_ = false; // Will become true if we hit Put/Delete or bottom
|
||||
|
||||
// We need to parse the internal key again as the parsed key is
|
||||
// backed by the internal key!
|
||||
// Assume no internal key corruption as it has been successfully parsed
|
||||
// by the caller.
|
||||
// Invariant: keys_.back() will not change. Hence, orig_ikey is always valid.
|
||||
ParsedInternalKey orig_ikey;
|
||||
ParseInternalKey(keys_.back(), &orig_ikey);
|
||||
|
||||
bool hit_the_next_user_key = false;
|
||||
ParsedInternalKey ikey;
|
||||
std::string merge_result; // Temporary value for merge results
|
||||
for (iter->Next(); iter->Valid(); iter->Next()) {
|
||||
assert(operands_.size() >= 1); // Should be invariants!
|
||||
assert(keys_.size() == operands_.size());
|
||||
|
||||
if (!ParseInternalKey(iter->key(), &ikey)) {
|
||||
// stop at corrupted key
|
||||
if (assert_valid_internal_key_) {
|
||||
assert(!"corrupted internal key is not expected");
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if (user_comparator_->Compare(ikey.user_key, orig_ikey.user_key) != 0) {
|
||||
// hit a different user key, stop right here
|
||||
hit_the_next_user_key = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (stop_before && ikey.sequence <= stop_before) {
|
||||
// hit an entry that's visible by the previous snapshot, can't touch that
|
||||
break;
|
||||
}
|
||||
|
||||
// At this point we are guaranteed that we need to process this key.
|
||||
|
||||
if (kTypeDeletion == ikey.type) {
|
||||
// hit a delete
|
||||
// => merge nullptr with operands_
|
||||
// => store result in operands_.back() (and update keys_.back())
|
||||
// => change the entry type to kTypeValue for keys_.back()
|
||||
// We are done! Return a success if the merge passes.
|
||||
success_ = user_merge_operator_->FullMerge(ikey.user_key, nullptr,
|
||||
operands_, &merge_result,
|
||||
logger_);
|
||||
|
||||
// We store the result in keys_.back() and operands_.back()
|
||||
// if nothing went wrong (i.e.: no operand corruption on disk)
|
||||
if (success_) {
|
||||
std::string& key = keys_.back(); // The original key encountered
|
||||
orig_ikey.type = kTypeValue;
|
||||
UpdateInternalKey(&key[0], key.size(),
|
||||
orig_ikey.sequence, orig_ikey.type);
|
||||
swap(operands_.back(), merge_result);
|
||||
} else {
|
||||
RecordTick(stats, NUMBER_MERGE_FAILURES);
|
||||
}
|
||||
|
||||
// move iter to the next entry (before doing anything else)
|
||||
iter->Next();
|
||||
return;
|
||||
}
|
||||
|
||||
if (kTypeValue == ikey.type) {
|
||||
// hit a put
|
||||
// => merge the put value with operands_
|
||||
// => store result in operands_.back() (and update keys_.back())
|
||||
// => change the entry type to kTypeValue for keys_.back()
|
||||
// We are done! Success!
|
||||
const Slice value = iter->value();
|
||||
success_ = user_merge_operator_->FullMerge(ikey.user_key, &value,
|
||||
operands_, &merge_result,
|
||||
logger_);
|
||||
|
||||
// We store the result in keys_.back() and operands_.back()
|
||||
// if nothing went wrong (i.e.: no operand corruption on disk)
|
||||
if (success_) {
|
||||
std::string& key = keys_.back(); // The original key encountered
|
||||
orig_ikey.type = kTypeValue;
|
||||
UpdateInternalKey(&key[0], key.size(),
|
||||
orig_ikey.sequence, orig_ikey.type);
|
||||
swap(operands_.back(), merge_result);
|
||||
} else {
|
||||
RecordTick(stats, NUMBER_MERGE_FAILURES);
|
||||
}
|
||||
|
||||
// move iter to the next entry
|
||||
iter->Next();
|
||||
return;
|
||||
}
|
||||
|
||||
if (kTypeMerge == ikey.type) {
|
||||
// hit a merge
|
||||
// => merge the operand into the front of the operands_ list
|
||||
// => use the user's associative merge function to determine how.
|
||||
// => then continue because we haven't yet seen a Put/Delete.
|
||||
assert(!operands_.empty()); // Should have at least one element in it
|
||||
|
||||
keys_.push_front(iter->key().ToString());
|
||||
operands_.push_front(iter->value().ToString());
|
||||
while (operands_.size() >= 2) {
|
||||
// Returns false when the merge_operator can no longer process it
|
||||
if (user_merge_operator_->PartialMerge(ikey.user_key,
|
||||
Slice(operands_[0]),
|
||||
Slice(operands_[1]),
|
||||
&merge_result,
|
||||
logger_)) {
|
||||
// Merging of operands (associative merge) was successful.
|
||||
// Replace these frontmost two operands with the merge result
|
||||
keys_.pop_front();
|
||||
operands_.pop_front();
|
||||
swap(operands_.front(), merge_result);
|
||||
} else {
|
||||
// Merging of operands (associative merge) returned false.
|
||||
// The user merge_operator does not know how to merge these operands.
|
||||
// So we just stack them up until we find a Put/Delete or end of key.
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// We are sure we have seen this key's entire history if we are at the
|
||||
// last level and exhausted all internal keys of this user key.
|
||||
// NOTE: !iter->Valid() does not necessarily mean we hit the
|
||||
// beginning of a user key, as versions of a user key might be
|
||||
// split into multiple files (even files on the same level)
|
||||
// and some files might not be included in the compaction/merge.
|
||||
//
|
||||
// There are also cases where we have seen the root of history of this
|
||||
// key without being sure of it. Then, we simply miss the opportunity
|
||||
// to combine the keys. Since VersionSet::SetupOtherInputs() always makes
|
||||
// sure that all merge-operands on the same level get compacted together,
|
||||
// this will simply lead to these merge operands moving to the next level.
|
||||
//
|
||||
// So, we only perform the following logic (to merge all operands together
|
||||
// without a Put/Delete) if we are certain that we have seen the end of key.
|
||||
bool surely_seen_the_beginning = hit_the_next_user_key && at_bottom;
|
||||
if (surely_seen_the_beginning) {
|
||||
// do a final merge with nullptr as the existing value and say
|
||||
// bye to the merge type (it's now converted to a Put)
|
||||
assert(kTypeMerge == orig_ikey.type);
|
||||
assert(operands_.size() >= 1);
|
||||
assert(operands_.size() == keys_.size());
|
||||
success_ = user_merge_operator_->FullMerge(ikey.user_key, nullptr,
|
||||
operands_, &merge_result,
|
||||
logger_);
|
||||
|
||||
if (success_) {
|
||||
std::string& key = keys_.back(); // The original key encountered
|
||||
orig_ikey.type = kTypeValue;
|
||||
UpdateInternalKey(&key[0], key.size(),
|
||||
orig_ikey.sequence, orig_ikey.type);
|
||||
|
||||
// The final value() is always stored in operands_.back()
|
||||
swap(operands_.back(),merge_result);
|
||||
} else {
|
||||
RecordTick(stats, NUMBER_MERGE_FAILURES);
|
||||
// Do nothing if not success_. Leave keys() and operands() as they are.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -1,103 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
#ifndef MERGE_HELPER_H
|
||||
#define MERGE_HELPER_H
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "rocksdb/slice.h"
|
||||
#include "rocksdb/statistics.h"
|
||||
#include <string>
|
||||
#include <deque>
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class Comparator;
|
||||
class Iterator;
|
||||
class Logger;
|
||||
class MergeOperator;
|
||||
|
||||
class MergeHelper {
|
||||
public:
|
||||
MergeHelper(const Comparator* user_comparator,
|
||||
const MergeOperator* user_merge_operator,
|
||||
Logger* logger,
|
||||
bool assert_valid_internal_key)
|
||||
: user_comparator_(user_comparator),
|
||||
user_merge_operator_(user_merge_operator),
|
||||
logger_(logger),
|
||||
assert_valid_internal_key_(assert_valid_internal_key),
|
||||
keys_(),
|
||||
operands_(),
|
||||
success_(false) {}
|
||||
|
||||
// Merge entries until we hit
|
||||
// - a corrupted key
|
||||
// - a Put/Delete,
|
||||
// - a different user key,
|
||||
// - a specific sequence number (snapshot boundary),
|
||||
// or - the end of iteration
|
||||
// iter: (IN) points to the first merge type entry
|
||||
// (OUT) points to the first entry not included in the merge process
|
||||
// stop_before: (IN) a sequence number that merge should not cross.
|
||||
// 0 means no restriction
|
||||
// at_bottom: (IN) true if the iterator covers the bottem level, which means
|
||||
// we could reach the start of the history of this user key.
|
||||
void MergeUntil(Iterator* iter, SequenceNumber stop_before = 0,
|
||||
bool at_bottom = false, shared_ptr<Statistics> stats=nullptr);
|
||||
|
||||
// Query the merge result
|
||||
// These are valid until the next MergeUntil call
|
||||
// If the merging was successful:
|
||||
// - IsSuccess() will be true
|
||||
// - key() will have the latest sequence number of the merges.
|
||||
// The type will be Put or Merge. See IMPORTANT 1 note, below.
|
||||
// - value() will be the result of merging all the operands together
|
||||
// - The user should ignore keys() and values().
|
||||
//
|
||||
// IMPORTANT 1: the key type could change after the MergeUntil call.
|
||||
// Put/Delete + Merge + ... + Merge => Put
|
||||
// Merge + ... + Merge => Merge
|
||||
//
|
||||
// If the merge operator is not associative, and if a Put/Delete is not found
|
||||
// then the merging will be unsuccessful. In this case:
|
||||
// - IsSuccess() will be false
|
||||
// - keys() contains the list of internal keys seen in order of iteration.
|
||||
// - values() contains the list of values (merges) seen in the same order.
|
||||
// values() is parallel to keys() so that the first entry in
|
||||
// keys() is the key associated with the first entry in values()
|
||||
// and so on. These lists will be the same length.
|
||||
// All of these pairs will be merges over the same user key.
|
||||
// See IMPORTANT 2 note below.
|
||||
// - The user should ignore key() and value().
|
||||
//
|
||||
// IMPORTANT 2: The entries were traversed in order from BACK to FRONT.
|
||||
// So keys().back() was the first key seen by iterator.
|
||||
// TODO: Re-style this comment to be like the first one
|
||||
bool IsSuccess() { return success_; }
|
||||
Slice key() { assert(success_); return Slice(keys_.back()); }
|
||||
Slice value() { assert(success_); return Slice(operands_.back()); }
|
||||
const std::deque<std::string>& keys() { assert(!success_); return keys_; }
|
||||
const std::deque<std::string>& values() {
|
||||
assert(!success_); return operands_;
|
||||
}
|
||||
|
||||
private:
|
||||
const Comparator* user_comparator_;
|
||||
const MergeOperator* user_merge_operator_;
|
||||
Logger* logger_;
|
||||
Iterator* iter_; // in: the internal iterator, positioned at the first merge entry
|
||||
bool assert_valid_internal_key_; // enforce no internal key corruption?
|
||||
|
||||
// the scratch area that holds the result of MergeUntil
|
||||
// valid up to the next MergeUntil call
|
||||
std::deque<std::string> keys_; // Keeps track of the sequence of keys seen
|
||||
std::deque<std::string> operands_; // Parallel with keys_; stores the values
|
||||
bool success_;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
#endif
|
||||
@@ -1,53 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
/**
|
||||
* Back-end implementation details specific to the Merge Operator.
|
||||
*/
|
||||
|
||||
#include "rocksdb/merge_operator.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
// Given a "real" merge from the library, call the user's
|
||||
// associative merge function one-by-one on each of the operands.
|
||||
// NOTE: It is assumed that the client's merge-operator will handle any errors.
|
||||
bool AssociativeMergeOperator::FullMerge(
|
||||
const Slice& key,
|
||||
const Slice* existing_value,
|
||||
const std::deque<std::string>& operand_list,
|
||||
std::string* new_value,
|
||||
Logger* logger) const {
|
||||
|
||||
// Simply loop through the operands
|
||||
Slice temp_existing;
|
||||
std::string temp_value;
|
||||
for (const auto& operand : operand_list) {
|
||||
Slice value(operand);
|
||||
if (!Merge(key, existing_value, value, &temp_value, logger)) {
|
||||
return false;
|
||||
}
|
||||
swap(temp_value, *new_value);
|
||||
temp_existing = Slice(*new_value);
|
||||
existing_value = &temp_existing;
|
||||
}
|
||||
|
||||
// The result will be in *new_value. All merges succeeded.
|
||||
return true;
|
||||
}
|
||||
|
||||
// Call the user defined simple merge on the operands;
|
||||
// NOTE: It is assumed that the client's merge-operator will handle any errors.
|
||||
bool AssociativeMergeOperator::PartialMerge(
|
||||
const Slice& key,
|
||||
const Slice& left_operand,
|
||||
const Slice& right_operand,
|
||||
std::string* new_value,
|
||||
Logger* logger) const {
|
||||
|
||||
return Merge(key, &left_operand, right_operand, new_value, logger);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -1,275 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
#include <assert.h>
|
||||
#include <memory>
|
||||
#include <iostream>
|
||||
|
||||
#include "rocksdb/cache.h"
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/merge_operator.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "utilities/merge_operators.h"
|
||||
#include "util/testharness.h"
|
||||
#include "utilities/utility_db.h"
|
||||
|
||||
using namespace std;
|
||||
using namespace rocksdb;
|
||||
|
||||
|
||||
std::shared_ptr<DB> OpenDb(const string& dbname, const bool ttl = false) {
|
||||
DB* db;
|
||||
StackableDB* sdb;
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.merge_operator = MergeOperators::CreateUInt64AddOperator();
|
||||
Status s;
|
||||
DestroyDB(dbname, Options());
|
||||
if (ttl) {
|
||||
cout << "Opening database with TTL\n";
|
||||
s = UtilityDB::OpenTtlDB(options, dbname, &sdb);
|
||||
db = sdb;
|
||||
} else {
|
||||
s = DB::Open(options, dbname, &db);
|
||||
}
|
||||
if (!s.ok()) {
|
||||
cerr << s.ToString() << endl;
|
||||
assert(false);
|
||||
}
|
||||
return std::shared_ptr<DB>(db);
|
||||
}
|
||||
|
||||
// Imagine we are maintaining a set of uint64 counters.
|
||||
// Each counter has a distinct name. And we would like
|
||||
// to support four high level operations:
|
||||
// set, add, get and remove
|
||||
// This is a quick implementation without a Merge operation.
|
||||
class Counters {
|
||||
|
||||
protected:
|
||||
std::shared_ptr<DB> db_;
|
||||
|
||||
WriteOptions put_option_;
|
||||
ReadOptions get_option_;
|
||||
WriteOptions delete_option_;
|
||||
|
||||
uint64_t default_;
|
||||
|
||||
public:
|
||||
explicit Counters(std::shared_ptr<DB> db, uint64_t defaultCount = 0)
|
||||
: db_(db),
|
||||
put_option_(),
|
||||
get_option_(),
|
||||
delete_option_(),
|
||||
default_(defaultCount) {
|
||||
assert(db_);
|
||||
}
|
||||
|
||||
virtual ~Counters() {}
|
||||
|
||||
// public interface of Counters.
|
||||
// All four functions return false
|
||||
// if the underlying level db operation failed.
|
||||
|
||||
// mapped to a levedb Put
|
||||
bool set(const string& key, uint64_t value) {
|
||||
// just treat the internal rep of int64 as the string
|
||||
Slice slice((char *)&value, sizeof(value));
|
||||
auto s = db_->Put(put_option_, key, slice);
|
||||
|
||||
if (s.ok()) {
|
||||
return true;
|
||||
} else {
|
||||
cerr << s.ToString() << endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// mapped to a rocksdb Delete
|
||||
bool remove(const string& key) {
|
||||
auto s = db_->Delete(delete_option_, key);
|
||||
|
||||
if (s.ok()) {
|
||||
return true;
|
||||
} else {
|
||||
cerr << s.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// mapped to a rocksdb Get
|
||||
bool get(const string& key, uint64_t *value) {
|
||||
string str;
|
||||
auto s = db_->Get(get_option_, key, &str);
|
||||
|
||||
if (s.IsNotFound()) {
|
||||
// return default value if not found;
|
||||
*value = default_;
|
||||
return true;
|
||||
} else if (s.ok()) {
|
||||
// deserialization
|
||||
if (str.size() != sizeof(uint64_t)) {
|
||||
cerr << "value corruption\n";
|
||||
return false;
|
||||
}
|
||||
*value = DecodeFixed64(&str[0]);
|
||||
return true;
|
||||
} else {
|
||||
cerr << s.ToString() << std::endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// 'add' is implemented as get -> modify -> set
|
||||
// An alternative is a single merge operation, see MergeBasedCounters
|
||||
virtual bool add(const string& key, uint64_t value) {
|
||||
uint64_t base = default_;
|
||||
return get(key, &base) && set(key, base + value);
|
||||
}
|
||||
|
||||
|
||||
// convenience functions for testing
|
||||
void assert_set(const string& key, uint64_t value) {
|
||||
assert(set(key, value));
|
||||
}
|
||||
|
||||
void assert_remove(const string& key) {
|
||||
assert(remove(key));
|
||||
}
|
||||
|
||||
uint64_t assert_get(const string& key) {
|
||||
uint64_t value = default_;
|
||||
assert(get(key, &value));
|
||||
return value;
|
||||
}
|
||||
|
||||
void assert_add(const string& key, uint64_t value) {
|
||||
assert(add(key, value));
|
||||
}
|
||||
};
|
||||
|
||||
// Implement 'add' directly with the new Merge operation
|
||||
class MergeBasedCounters : public Counters {
|
||||
private:
|
||||
WriteOptions merge_option_; // for merge
|
||||
|
||||
public:
|
||||
explicit MergeBasedCounters(std::shared_ptr<DB> db, uint64_t defaultCount = 0)
|
||||
: Counters(db, defaultCount),
|
||||
merge_option_() {
|
||||
}
|
||||
|
||||
// mapped to a rocksdb Merge operation
|
||||
virtual bool add(const string& key, uint64_t value) override {
|
||||
char encoded[sizeof(uint64_t)];
|
||||
EncodeFixed64(encoded, value);
|
||||
Slice slice(encoded, sizeof(uint64_t));
|
||||
auto s = db_->Merge(merge_option_, key, slice);
|
||||
|
||||
if (s.ok()) {
|
||||
return true;
|
||||
} else {
|
||||
cerr << s.ToString() << endl;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
void dumpDb(DB* db) {
|
||||
auto it = unique_ptr<Iterator>(db->NewIterator(ReadOptions()));
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
uint64_t value = DecodeFixed64(it->value().data());
|
||||
cout << it->key().ToString() << ": " << value << endl;
|
||||
}
|
||||
assert(it->status().ok()); // Check for any errors found during the scan
|
||||
}
|
||||
|
||||
void testCounters(Counters& counters, DB* db, bool test_compaction) {
|
||||
|
||||
FlushOptions o;
|
||||
o.wait = true;
|
||||
|
||||
counters.assert_set("a", 1);
|
||||
|
||||
if (test_compaction) db->Flush(o);
|
||||
|
||||
assert(counters.assert_get("a") == 1);
|
||||
|
||||
counters.assert_remove("b");
|
||||
|
||||
// defaut value is 0 if non-existent
|
||||
assert(counters.assert_get("b") == 0);
|
||||
|
||||
counters.assert_add("a", 2);
|
||||
|
||||
if (test_compaction) db->Flush(o);
|
||||
|
||||
// 1+2 = 3
|
||||
assert(counters.assert_get("a")== 3);
|
||||
|
||||
dumpDb(db);
|
||||
|
||||
std::cout << "1\n";
|
||||
|
||||
// 1+...+49 = ?
|
||||
uint64_t sum = 0;
|
||||
for (int i = 1; i < 50; i++) {
|
||||
counters.assert_add("b", i);
|
||||
sum += i;
|
||||
}
|
||||
assert(counters.assert_get("b") == sum);
|
||||
|
||||
std::cout << "2\n";
|
||||
dumpDb(db);
|
||||
|
||||
std::cout << "3\n";
|
||||
|
||||
if (test_compaction) {
|
||||
db->Flush(o);
|
||||
|
||||
cout << "Compaction started ...\n";
|
||||
db->CompactRange(nullptr, nullptr);
|
||||
cout << "Compaction ended\n";
|
||||
|
||||
dumpDb(db);
|
||||
|
||||
assert(counters.assert_get("a")== 3);
|
||||
assert(counters.assert_get("b") == sum);
|
||||
}
|
||||
}
|
||||
|
||||
void runTest(int argc, const string& dbname, const bool use_ttl = false) {
|
||||
auto db = OpenDb(dbname, use_ttl);
|
||||
|
||||
{
|
||||
cout << "Test read-modify-write counters... \n";
|
||||
Counters counters(db, 0);
|
||||
testCounters(counters, db.get(), true);
|
||||
}
|
||||
|
||||
bool compact = false;
|
||||
if (argc > 1) {
|
||||
compact = true;
|
||||
cout << "Turn on Compaction\n";
|
||||
}
|
||||
|
||||
{
|
||||
cout << "Test merge-based counters... \n";
|
||||
MergeBasedCounters counters(db, 0);
|
||||
testCounters(counters, db.get(), compact);
|
||||
}
|
||||
|
||||
DestroyDB(dbname, Options());
|
||||
}
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
//TODO: Make this test like a general rocksdb unit-test
|
||||
runTest(argc, test::TmpDir() + "/merge_testdb");
|
||||
runTest(argc, test::TmpDir() + "/merge_testdbttl", true); // Run test on TTL database
|
||||
return 0;
|
||||
}
|
||||
@@ -1,328 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
#include "/usr/include/valgrind/callgrind.h"
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/perf_context.h"
|
||||
#include "util/histogram.h"
|
||||
#include "util/stop_watch.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
|
||||
bool FLAGS_random_key = false;
|
||||
bool FLAGS_use_set_based_memetable = false;
|
||||
int FLAGS_total_keys = 100;
|
||||
int FLAGS_write_buffer_size = 1000000000;
|
||||
int FLAGS_max_write_buffer_number = 8;
|
||||
int FLAGS_min_write_buffer_number_to_merge = 7;
|
||||
|
||||
// Path to the database on file system
|
||||
const std::string kDbName = rocksdb::test::TmpDir() + "/perf_context_test";
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
std::shared_ptr<DB> OpenDb() {
|
||||
DB* db;
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.write_buffer_size = FLAGS_write_buffer_size;
|
||||
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
|
||||
options.min_write_buffer_number_to_merge =
|
||||
FLAGS_min_write_buffer_number_to_merge;
|
||||
|
||||
if (FLAGS_use_set_based_memetable) {
|
||||
auto prefix_extractor = rocksdb::NewFixedPrefixTransform(0);
|
||||
options.memtable_factory =
|
||||
std::make_shared<rocksdb::PrefixHashRepFactory>(prefix_extractor);
|
||||
}
|
||||
|
||||
Status s = DB::Open(options, kDbName, &db);
|
||||
ASSERT_OK(s);
|
||||
return std::shared_ptr<DB>(db);
|
||||
}
|
||||
|
||||
class PerfContextTest { };
|
||||
|
||||
TEST(PerfContextTest, SeekIntoDeletion) {
|
||||
DestroyDB(kDbName, Options());
|
||||
auto db = OpenDb();
|
||||
WriteOptions write_options;
|
||||
ReadOptions read_options;
|
||||
|
||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||
std::string key = "k" + std::to_string(i);
|
||||
std::string value = "v" + std::to_string(i);
|
||||
|
||||
db->Put(write_options, key, value);
|
||||
}
|
||||
|
||||
for (int i = 0; i < FLAGS_total_keys -1 ; ++i) {
|
||||
std::string key = "k" + std::to_string(i);
|
||||
db->Delete(write_options, key);
|
||||
}
|
||||
|
||||
HistogramImpl hist_get;
|
||||
HistogramImpl hist_get_time;
|
||||
for (int i = 0; i < FLAGS_total_keys - 1; ++i) {
|
||||
std::string key = "k" + std::to_string(i);
|
||||
std::string value;
|
||||
|
||||
perf_context.Reset();
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
auto status = db->Get(read_options, key, &value);
|
||||
auto elapsed_nanos = timer.ElapsedNanos();
|
||||
ASSERT_TRUE(status.IsNotFound());
|
||||
hist_get.Add(perf_context.user_key_comparison_count);
|
||||
hist_get_time.Add(elapsed_nanos);
|
||||
}
|
||||
|
||||
std::cout << "Get uesr key comparison: \n" << hist_get.ToString()
|
||||
<< "Get time: \n" << hist_get_time.ToString();
|
||||
|
||||
HistogramImpl hist_seek_to_first;
|
||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||
|
||||
perf_context.Reset();
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
iter->SeekToFirst();
|
||||
hist_seek_to_first.Add(perf_context.user_key_comparison_count);
|
||||
auto elapsed_nanos = timer.ElapsedNanos();
|
||||
|
||||
std::cout << "SeekToFirst uesr key comparison: \n" << hist_seek_to_first.ToString()
|
||||
<< "ikey skipped: " << perf_context.internal_key_skipped_count << "\n"
|
||||
<< "idelete skipped: " << perf_context.internal_delete_skipped_count << "\n"
|
||||
<< "elapsed: " << elapsed_nanos << "\n";
|
||||
|
||||
HistogramImpl hist_seek;
|
||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||
std::string key = "k" + std::to_string(i);
|
||||
|
||||
perf_context.Reset();
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
iter->Seek(key);
|
||||
auto elapsed_nanos = timer.ElapsedNanos();
|
||||
hist_seek.Add(perf_context.user_key_comparison_count);
|
||||
std::cout << "seek cmp: " << perf_context.user_key_comparison_count
|
||||
<< " ikey skipped " << perf_context.internal_key_skipped_count
|
||||
<< " idelete skipped " << perf_context.internal_delete_skipped_count
|
||||
<< " elapsed: " << elapsed_nanos << "ns\n";
|
||||
|
||||
perf_context.Reset();
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
StopWatchNano timer2(Env::Default(), true);
|
||||
iter->Next();
|
||||
auto elapsed_nanos2 = timer2.ElapsedNanos();
|
||||
std::cout << "next cmp: " << perf_context.user_key_comparison_count
|
||||
<< "elapsed: " << elapsed_nanos2 << "ns\n";
|
||||
}
|
||||
|
||||
std::cout << "Seek uesr key comparison: \n" << hist_seek.ToString();
|
||||
}
|
||||
|
||||
TEST(PerfContextTest, StopWatchNanoOverhead) {
|
||||
// profile the timer cost by itself!
|
||||
const int kTotalIterations = 1000000;
|
||||
std::vector<uint64_t> timings(kTotalIterations);
|
||||
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
for (auto& timing : timings) {
|
||||
timing = timer.ElapsedNanos(true /* reset */);
|
||||
}
|
||||
|
||||
HistogramImpl histogram;
|
||||
for (const auto timing : timings) {
|
||||
histogram.Add(timing);
|
||||
}
|
||||
|
||||
std::cout << histogram.ToString();
|
||||
}
|
||||
|
||||
TEST(PerfContextTest, StopWatchOverhead) {
|
||||
// profile the timer cost by itself!
|
||||
const int kTotalIterations = 1000000;
|
||||
std::vector<uint64_t> timings(kTotalIterations);
|
||||
|
||||
StopWatch timer(Env::Default());
|
||||
for (auto& timing : timings) {
|
||||
timing = timer.ElapsedMicros();
|
||||
}
|
||||
|
||||
HistogramImpl histogram;
|
||||
uint64_t prev_timing = 0;
|
||||
for (const auto timing : timings) {
|
||||
histogram.Add(timing - prev_timing);
|
||||
prev_timing = timing;
|
||||
}
|
||||
|
||||
std::cout << histogram.ToString();
|
||||
}
|
||||
|
||||
void ProfileKeyComparison() {
|
||||
DestroyDB(kDbName, Options()); // Start this test with a fresh DB
|
||||
|
||||
auto db = OpenDb();
|
||||
|
||||
WriteOptions write_options;
|
||||
ReadOptions read_options;
|
||||
|
||||
HistogramImpl hist_put;
|
||||
HistogramImpl hist_get;
|
||||
|
||||
std::cout << "Inserting " << FLAGS_total_keys << " key/value pairs\n...\n";
|
||||
|
||||
std::vector<int> keys;
|
||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||
keys.push_back(i);
|
||||
}
|
||||
|
||||
if (FLAGS_random_key) {
|
||||
std::random_shuffle(keys.begin(), keys.end());
|
||||
}
|
||||
|
||||
for (const int i : keys) {
|
||||
std::string key = "k" + std::to_string(i);
|
||||
std::string value = "v" + std::to_string(i);
|
||||
|
||||
perf_context.Reset();
|
||||
db->Put(write_options, key, value);
|
||||
hist_put.Add(perf_context.user_key_comparison_count);
|
||||
|
||||
perf_context.Reset();
|
||||
db->Get(read_options, key, &value);
|
||||
hist_get.Add(perf_context.user_key_comparison_count);
|
||||
}
|
||||
|
||||
std::cout << "Put uesr key comparison: \n" << hist_put.ToString()
|
||||
<< "Get uesr key comparison: \n" << hist_get.ToString();
|
||||
|
||||
}
|
||||
|
||||
TEST(PerfContextTest, KeyComparisonCount) {
|
||||
SetPerfLevel(kEnableCount);
|
||||
ProfileKeyComparison();
|
||||
|
||||
SetPerfLevel(kDisable);
|
||||
ProfileKeyComparison();
|
||||
|
||||
SetPerfLevel(kEnableTime);
|
||||
ProfileKeyComparison();
|
||||
}
|
||||
|
||||
// make perf_context_test
|
||||
// export ROCKSDB_TESTS=PerfContextTest.SeekKeyComparison
|
||||
// For one memtable:
|
||||
// ./perf_context_test --write_buffer_size=500000 --total_keys=10000
|
||||
// For two memtables:
|
||||
// ./perf_context_test --write_buffer_size=250000 --total_keys=10000
|
||||
// Specify --random_key=1 to shuffle the key before insertion
|
||||
// Results show that, for sequential insertion, worst-case Seek Key comparison
|
||||
// is close to the total number of keys (linear), when there is only one
|
||||
// memtable. When there are two memtables, even the avg Seek Key comparison
|
||||
// starts to become linear to the input size.
|
||||
|
||||
TEST(PerfContextTest, SeekKeyComparison) {
|
||||
DestroyDB(kDbName, Options());
|
||||
auto db = OpenDb();
|
||||
WriteOptions write_options;
|
||||
ReadOptions read_options;
|
||||
|
||||
std::cout << "Inserting " << FLAGS_total_keys << " key/value pairs\n...\n";
|
||||
|
||||
std::vector<int> keys;
|
||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||
keys.push_back(i);
|
||||
}
|
||||
|
||||
if (FLAGS_random_key) {
|
||||
std::random_shuffle(keys.begin(), keys.end());
|
||||
}
|
||||
|
||||
HistogramImpl hist_put_time;
|
||||
HistogramImpl hist_wal_time;
|
||||
HistogramImpl hist_time_diff;
|
||||
|
||||
SetPerfLevel(kEnableTime);
|
||||
StopWatchNano timer(Env::Default());
|
||||
for (const int i : keys) {
|
||||
std::string key = "k" + std::to_string(i);
|
||||
std::string value = "v" + std::to_string(i);
|
||||
|
||||
perf_context.Reset();
|
||||
timer.Start();
|
||||
db->Put(write_options, key, value);
|
||||
auto put_time = timer.ElapsedNanos();
|
||||
hist_put_time.Add(put_time);
|
||||
hist_wal_time.Add(perf_context.wal_write_time);
|
||||
hist_time_diff.Add(put_time - perf_context.wal_write_time);
|
||||
}
|
||||
|
||||
std::cout << "Put time:\n" << hist_put_time.ToString()
|
||||
<< "WAL time:\n" << hist_wal_time.ToString()
|
||||
<< "time diff:\n" << hist_time_diff.ToString();
|
||||
|
||||
HistogramImpl hist_seek;
|
||||
HistogramImpl hist_next;
|
||||
|
||||
for (int i = 0; i < FLAGS_total_keys; ++i) {
|
||||
std::string key = "k" + std::to_string(i);
|
||||
std::string value = "v" + std::to_string(i);
|
||||
|
||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||
perf_context.Reset();
|
||||
iter->Seek(key);
|
||||
ASSERT_TRUE(iter->Valid());
|
||||
ASSERT_EQ(iter->value().ToString(), value);
|
||||
hist_seek.Add(perf_context.user_key_comparison_count);
|
||||
}
|
||||
|
||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||
for (iter->SeekToFirst(); iter->Valid();) {
|
||||
perf_context.Reset();
|
||||
iter->Next();
|
||||
hist_next.Add(perf_context.user_key_comparison_count);
|
||||
}
|
||||
|
||||
std::cout << "Seek:\n" << hist_seek.ToString()
|
||||
<< "Next:\n" << hist_next.ToString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
int n;
|
||||
char junk;
|
||||
|
||||
if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_write_buffer_size = n;
|
||||
}
|
||||
|
||||
if (sscanf(argv[i], "--total_keys=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_total_keys = n;
|
||||
}
|
||||
|
||||
if (sscanf(argv[i], "--random_key=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_random_key = n;
|
||||
}
|
||||
|
||||
if (sscanf(argv[i], "--use_set_based_memetable=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
FLAGS_use_set_based_memetable = n;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
std::cout << kDbName << "\n";
|
||||
|
||||
rocksdb::test::RunAllTests();
|
||||
return 0;
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Wrap an underlying iterator, but exclude any results not starting
|
||||
// with a given prefix. Seeking to keys not beginning with the prefix
|
||||
// is invalid, and SeekToLast is not implemented (that would be
|
||||
// non-trivial), but otherwise this iterator will behave just like the
|
||||
// underlying iterator would if there happened to be no non-matching
|
||||
// keys in the dataset.
|
||||
|
||||
#pragma once
|
||||
#include "rocksdb/iterator.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class PrefixFilterIterator : public Iterator {
|
||||
private:
|
||||
Iterator* iter_;
|
||||
const Slice &prefix_;
|
||||
const SliceTransform *prefix_extractor_;
|
||||
Status status_;
|
||||
|
||||
public:
|
||||
PrefixFilterIterator(Iterator* iter,
|
||||
const Slice &prefix,
|
||||
const SliceTransform* prefix_extractor)
|
||||
: iter_(iter), prefix_(prefix),
|
||||
prefix_extractor_(prefix_extractor),
|
||||
status_(Status::OK()) {
|
||||
if (prefix_extractor == nullptr) {
|
||||
status_ = Status::InvalidArgument("A prefix filter may not be used "
|
||||
"unless a function is also defined "
|
||||
"for extracting prefixes");
|
||||
} else if (!prefix_extractor_->InRange(prefix)) {
|
||||
status_ = Status::InvalidArgument("Must provide a slice for prefix which"
|
||||
"is a prefix for some key");
|
||||
}
|
||||
}
|
||||
~PrefixFilterIterator() {
|
||||
delete iter_;
|
||||
}
|
||||
Slice key() const { return iter_->key(); }
|
||||
Slice value() const { return iter_->value(); }
|
||||
Status status() const {
|
||||
if (!status_.ok()) {
|
||||
return status_;
|
||||
}
|
||||
return iter_->status();
|
||||
}
|
||||
void Next() { iter_->Next(); }
|
||||
void Prev() { iter_->Prev(); }
|
||||
void Seek(const Slice& k) {
|
||||
if (prefix_extractor_->Transform(k) == prefix_) {
|
||||
iter_->Seek(k);
|
||||
} else {
|
||||
status_ = Status::InvalidArgument("Seek must begin with target prefix");
|
||||
}
|
||||
}
|
||||
void SeekToFirst() {
|
||||
Seek(prefix_);
|
||||
}
|
||||
void SeekToLast() {
|
||||
status_ = Status::NotSupported("SeekToLast is incompatible with prefixes");
|
||||
}
|
||||
bool Valid() const {
|
||||
return (status_.ok() && iter_->Valid() &&
|
||||
prefix_extractor_->Transform(iter_->key()) == prefix_);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -1,320 +0,0 @@
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/perf_context.h"
|
||||
#include "util/histogram.h"
|
||||
#include "util/stop_watch.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
DEFINE_bool(use_prefix_hash_memtable, true, "");
|
||||
DEFINE_bool(use_nolock_version, true, "");
|
||||
DEFINE_bool(trigger_deadlock, false,
|
||||
"issue delete in range scan to trigger PrefixHashMap deadlock");
|
||||
DEFINE_uint64(bucket_count, 100000, "number of buckets");
|
||||
DEFINE_uint64(num_locks, 10001, "number of locks");
|
||||
DEFINE_bool(random_prefix, false, "randomize prefix");
|
||||
DEFINE_uint64(total_prefixes, 1000, "total number of prefixes");
|
||||
DEFINE_uint64(items_per_prefix, 10, "total number of values per prefix");
|
||||
DEFINE_int64(write_buffer_size, 1000000000, "");
|
||||
DEFINE_int64(max_write_buffer_number, 8, "");
|
||||
DEFINE_int64(min_write_buffer_number_to_merge, 7, "");
|
||||
|
||||
// Path to the database on file system
|
||||
const std::string kDbName = rocksdb::test::TmpDir() + "/prefix_test";
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
struct TestKey {
|
||||
uint64_t prefix;
|
||||
uint64_t sorted;
|
||||
|
||||
TestKey(uint64_t prefix, uint64_t sorted) : prefix(prefix), sorted(sorted) {}
|
||||
};
|
||||
|
||||
// return a slice backed by test_key
|
||||
inline Slice TestKeyToSlice(const TestKey& test_key) {
|
||||
return Slice((const char*)&test_key, sizeof(test_key));
|
||||
}
|
||||
|
||||
inline const TestKey* SliceToTestKey(const Slice& slice) {
|
||||
assert(slice.size() == sizeof(TestKey));
|
||||
return (const TestKey*)slice.data();
|
||||
}
|
||||
|
||||
class TestKeyComparator : public Comparator {
|
||||
public:
|
||||
virtual int Compare(const Slice& a, const Slice& b) const {
|
||||
const TestKey* key_a = SliceToTestKey(a);
|
||||
const TestKey* key_b = SliceToTestKey(b);
|
||||
if (key_a->prefix != key_b->prefix) {
|
||||
if (key_a->prefix < key_b->prefix) return -1;
|
||||
if (key_a->prefix > key_b->prefix) return 1;
|
||||
assert(false);
|
||||
} else {
|
||||
if (key_a->sorted < key_b->sorted) return -1;
|
||||
if (key_a->sorted > key_b->sorted) return 1;
|
||||
if (key_a->sorted == key_b->sorted) return 0;
|
||||
assert(false);
|
||||
}
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
virtual const char* Name() const override {
|
||||
return "TestKeyComparator";
|
||||
}
|
||||
|
||||
virtual void FindShortestSeparator(
|
||||
std::string* start,
|
||||
const Slice& limit) const {
|
||||
}
|
||||
|
||||
virtual void FindShortSuccessor(std::string* key) const {}
|
||||
|
||||
};
|
||||
|
||||
class PrefixTest {
|
||||
public:
|
||||
std::shared_ptr<DB> OpenDb() {
|
||||
DB* db;
|
||||
|
||||
options.create_if_missing = true;
|
||||
options.write_buffer_size = FLAGS_write_buffer_size;
|
||||
options.max_write_buffer_number = FLAGS_max_write_buffer_number;
|
||||
options.min_write_buffer_number_to_merge =
|
||||
FLAGS_min_write_buffer_number_to_merge;
|
||||
|
||||
options.comparator = new TestKeyComparator();
|
||||
if (FLAGS_use_prefix_hash_memtable) {
|
||||
auto prefix_extractor = NewFixedPrefixTransform(8);
|
||||
options.prefix_extractor = prefix_extractor;
|
||||
if (FLAGS_use_nolock_version) {
|
||||
options.memtable_factory =
|
||||
std::make_shared<rocksdb::PrefixHashRepNoLockFactory>(
|
||||
prefix_extractor, FLAGS_bucket_count);
|
||||
} else {
|
||||
options.memtable_factory =
|
||||
std::make_shared<rocksdb::PrefixHashRepFactory>(
|
||||
prefix_extractor, FLAGS_bucket_count, FLAGS_num_locks);
|
||||
}
|
||||
}
|
||||
|
||||
Status s = DB::Open(options, kDbName, &db);
|
||||
ASSERT_OK(s);
|
||||
return std::shared_ptr<DB>(db);
|
||||
}
|
||||
~PrefixTest() {
|
||||
delete options.comparator;
|
||||
}
|
||||
protected:
|
||||
Options options;
|
||||
};
|
||||
|
||||
TEST(PrefixTest, DynamicPrefixIterator) {
|
||||
|
||||
DestroyDB(kDbName, Options());
|
||||
auto db = OpenDb();
|
||||
WriteOptions write_options;
|
||||
ReadOptions read_options;
|
||||
|
||||
std::vector<uint64_t> prefixes;
|
||||
for (uint64_t i = 0; i < FLAGS_total_prefixes; ++i) {
|
||||
prefixes.push_back(i);
|
||||
}
|
||||
|
||||
if (FLAGS_random_prefix) {
|
||||
std::random_shuffle(prefixes.begin(), prefixes.end());
|
||||
}
|
||||
|
||||
// insert x random prefix, each with y continuous element.
|
||||
for (auto prefix : prefixes) {
|
||||
for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
|
||||
TestKey test_key(prefix, sorted);
|
||||
|
||||
Slice key = TestKeyToSlice(test_key);
|
||||
std::string value = "v" + std::to_string(sorted);
|
||||
|
||||
ASSERT_OK(db->Put(write_options, key, value));
|
||||
}
|
||||
}
|
||||
|
||||
// test seek existing keys
|
||||
HistogramImpl hist_seek_time;
|
||||
HistogramImpl hist_seek_comparison;
|
||||
|
||||
if (FLAGS_use_prefix_hash_memtable) {
|
||||
read_options.prefix_seek = true;
|
||||
}
|
||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||
|
||||
for (auto prefix : prefixes) {
|
||||
TestKey test_key(prefix, FLAGS_items_per_prefix / 2);
|
||||
Slice key = TestKeyToSlice(test_key);
|
||||
std::string value = "v" + std::to_string(0);
|
||||
|
||||
perf_context.Reset();
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
uint64_t total_keys = 0;
|
||||
for (iter->Seek(key); iter->Valid(); iter->Next()) {
|
||||
if (FLAGS_trigger_deadlock) {
|
||||
std::cout << "Behold the deadlock!\n";
|
||||
db->Delete(write_options, iter->key());
|
||||
}
|
||||
auto test_key = SliceToTestKey(iter->key());
|
||||
if (test_key->prefix != prefix) break;
|
||||
total_keys++;
|
||||
}
|
||||
hist_seek_time.Add(timer.ElapsedNanos());
|
||||
hist_seek_comparison.Add(perf_context.user_key_comparison_count);
|
||||
ASSERT_EQ(total_keys, FLAGS_items_per_prefix - FLAGS_items_per_prefix/2);
|
||||
}
|
||||
|
||||
std::cout << "Seek key comparison: \n"
|
||||
<< hist_seek_comparison.ToString()
|
||||
<< "Seek time: \n"
|
||||
<< hist_seek_time.ToString();
|
||||
|
||||
// test non-existing keys
|
||||
HistogramImpl hist_no_seek_time;
|
||||
HistogramImpl hist_no_seek_comparison;
|
||||
|
||||
for (auto prefix = FLAGS_total_prefixes;
|
||||
prefix < FLAGS_total_prefixes + 100;
|
||||
prefix++) {
|
||||
TestKey test_key(prefix, 0);
|
||||
Slice key = TestKeyToSlice(test_key);
|
||||
|
||||
perf_context.Reset();
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
iter->Seek(key);
|
||||
hist_no_seek_time.Add(timer.ElapsedNanos());
|
||||
hist_no_seek_comparison.Add(perf_context.user_key_comparison_count);
|
||||
ASSERT_TRUE(!iter->Valid());
|
||||
}
|
||||
|
||||
std::cout << "non-existing Seek key comparison: \n"
|
||||
<< hist_no_seek_comparison.ToString()
|
||||
<< "non-existing Seek time: \n"
|
||||
<< hist_no_seek_time.ToString();
|
||||
}
|
||||
|
||||
TEST(PrefixTest, PrefixHash) {
|
||||
|
||||
DestroyDB(kDbName, Options());
|
||||
auto db = OpenDb();
|
||||
WriteOptions write_options;
|
||||
ReadOptions read_options;
|
||||
|
||||
std::vector<uint64_t> prefixes;
|
||||
for (uint64_t i = 0; i < FLAGS_total_prefixes; ++i) {
|
||||
prefixes.push_back(i);
|
||||
}
|
||||
|
||||
if (FLAGS_random_prefix) {
|
||||
std::random_shuffle(prefixes.begin(), prefixes.end());
|
||||
}
|
||||
|
||||
// insert x random prefix, each with y continuous element.
|
||||
HistogramImpl hist_put_time;
|
||||
HistogramImpl hist_put_comparison;
|
||||
|
||||
for (auto prefix : prefixes) {
|
||||
for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
|
||||
TestKey test_key(prefix, sorted);
|
||||
|
||||
Slice key = TestKeyToSlice(test_key);
|
||||
std::string value = "v" + std::to_string(sorted);
|
||||
|
||||
perf_context.Reset();
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
ASSERT_OK(db->Put(write_options, key, value));
|
||||
hist_put_time.Add(timer.ElapsedNanos());
|
||||
hist_put_comparison.Add(perf_context.user_key_comparison_count);
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "Put key comparison: \n" << hist_put_comparison.ToString()
|
||||
<< "Put time: \n" << hist_put_time.ToString();
|
||||
|
||||
|
||||
// test seek existing keys
|
||||
HistogramImpl hist_seek_time;
|
||||
HistogramImpl hist_seek_comparison;
|
||||
|
||||
for (auto prefix : prefixes) {
|
||||
TestKey test_key(prefix, 0);
|
||||
Slice key = TestKeyToSlice(test_key);
|
||||
std::string value = "v" + std::to_string(0);
|
||||
|
||||
Slice key_prefix;
|
||||
if (FLAGS_use_prefix_hash_memtable) {
|
||||
key_prefix = options.prefix_extractor->Transform(key);
|
||||
read_options.prefix = &key_prefix;
|
||||
}
|
||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||
|
||||
perf_context.Reset();
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
uint64_t total_keys = 0;
|
||||
for (iter->Seek(key); iter->Valid(); iter->Next()) {
|
||||
if (FLAGS_trigger_deadlock) {
|
||||
std::cout << "Behold the deadlock!\n";
|
||||
db->Delete(write_options, iter->key());
|
||||
}
|
||||
auto test_key = SliceToTestKey(iter->key());
|
||||
if (test_key->prefix != prefix) break;
|
||||
total_keys++;
|
||||
}
|
||||
hist_seek_time.Add(timer.ElapsedNanos());
|
||||
hist_seek_comparison.Add(perf_context.user_key_comparison_count);
|
||||
ASSERT_EQ(total_keys, FLAGS_items_per_prefix);
|
||||
}
|
||||
|
||||
std::cout << "Seek key comparison: \n"
|
||||
<< hist_seek_comparison.ToString()
|
||||
<< "Seek time: \n"
|
||||
<< hist_seek_time.ToString();
|
||||
|
||||
// test non-existing keys
|
||||
HistogramImpl hist_no_seek_time;
|
||||
HistogramImpl hist_no_seek_comparison;
|
||||
|
||||
for (auto prefix = FLAGS_total_prefixes;
|
||||
prefix < FLAGS_total_prefixes + 100;
|
||||
prefix++) {
|
||||
TestKey test_key(prefix, 0);
|
||||
Slice key = TestKeyToSlice(test_key);
|
||||
|
||||
if (FLAGS_use_prefix_hash_memtable) {
|
||||
Slice key_prefix = options.prefix_extractor->Transform(key);
|
||||
read_options.prefix = &key_prefix;
|
||||
}
|
||||
std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
|
||||
|
||||
perf_context.Reset();
|
||||
StopWatchNano timer(Env::Default(), true);
|
||||
iter->Seek(key);
|
||||
hist_no_seek_time.Add(timer.ElapsedNanos());
|
||||
hist_no_seek_comparison.Add(perf_context.user_key_comparison_count);
|
||||
ASSERT_TRUE(!iter->Valid());
|
||||
}
|
||||
|
||||
std::cout << "non-existing Seek key comparison: \n"
|
||||
<< hist_no_seek_comparison.ToString()
|
||||
<< "non-existing Seek time: \n"
|
||||
<< hist_no_seek_time.ToString();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
google::ParseCommandLineFlags(&argc, &argv, true);
|
||||
std::cout << kDbName << "\n";
|
||||
|
||||
rocksdb::test::RunAllTests();
|
||||
return 0;
|
||||
}
|
||||
+15
-29
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -39,11 +34,11 @@
|
||||
#include "db/table_cache.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "rocksdb/comparator.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/comparator.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -57,7 +52,7 @@ class Repairer {
|
||||
options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
|
||||
next_file_number_(1) {
|
||||
// TableCache can be small since we expect each table to be opened once.
|
||||
table_cache_ = new TableCache(dbname_, &options_, storage_options_, 10);
|
||||
table_cache_ = new TableCache(dbname_, &options_, 10);
|
||||
edit_ = new VersionEdit(options.num_levels);
|
||||
}
|
||||
|
||||
@@ -79,7 +74,7 @@ class Repairer {
|
||||
bytes += tables_[i].meta.file_size;
|
||||
}
|
||||
Log(options_.info_log,
|
||||
"**** Repaired rocksdb %s; "
|
||||
"**** Repaired leveldb %s; "
|
||||
"recovered %d files; %llu bytes. "
|
||||
"Some data may have been lost. "
|
||||
"****",
|
||||
@@ -93,7 +88,6 @@ class Repairer {
|
||||
private:
|
||||
struct TableInfo {
|
||||
FileMetaData meta;
|
||||
SequenceNumber min_sequence;
|
||||
SequenceNumber max_sequence;
|
||||
};
|
||||
|
||||
@@ -111,7 +105,6 @@ class Repairer {
|
||||
std::vector<uint64_t> logs_;
|
||||
std::vector<TableInfo> tables_;
|
||||
uint64_t next_file_number_;
|
||||
const EnvOptions storage_options_;
|
||||
|
||||
Status FindFiles() {
|
||||
std::vector<std::string> filenames;
|
||||
@@ -176,7 +169,7 @@ class Repairer {
|
||||
// Open the log file
|
||||
std::string logname = LogFileName(dbname_, log);
|
||||
unique_ptr<SequentialFile> lfile;
|
||||
Status status = env_->NewSequentialFile(logname, &lfile, storage_options_);
|
||||
Status status = env_->NewSequentialFile(logname, &lfile);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
@@ -197,8 +190,7 @@ class Repairer {
|
||||
std::string scratch;
|
||||
Slice record;
|
||||
WriteBatch batch;
|
||||
MemTable* mem = new MemTable(icmp_, options_.memtable_factory,
|
||||
options_.num_levels);
|
||||
MemTable* mem = new MemTable(icmp_, options_.num_levels);
|
||||
mem->Ref();
|
||||
int counter = 0;
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
@@ -208,7 +200,7 @@ class Repairer {
|
||||
continue;
|
||||
}
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
status = WriteBatchInternal::InsertInto(&batch, mem, &options_);
|
||||
status = WriteBatchInternal::InsertInto(&batch, mem);
|
||||
if (status.ok()) {
|
||||
counter += WriteBatchInternal::Count(&batch);
|
||||
} else {
|
||||
@@ -224,9 +216,8 @@ class Repairer {
|
||||
FileMetaData meta;
|
||||
meta.number = next_file_number_++;
|
||||
Iterator* iter = mem->NewIterator();
|
||||
status = BuildTable(dbname_, env_, options_, storage_options_,
|
||||
table_cache_, iter, &meta,
|
||||
icmp_.user_comparator(), 0, 0, true);
|
||||
status = BuildTable(dbname_, env_, options_, table_cache_, iter, &meta,
|
||||
icmp_.user_comparator(), 0, 0);
|
||||
delete iter;
|
||||
mem->Unref();
|
||||
mem = nullptr;
|
||||
@@ -267,10 +258,9 @@ class Repairer {
|
||||
Status status = env_->GetFileSize(fname, &t->meta.file_size);
|
||||
if (status.ok()) {
|
||||
Iterator* iter = table_cache_->NewIterator(
|
||||
ReadOptions(), storage_options_, t->meta.number, t->meta.file_size);
|
||||
ReadOptions(), t->meta.number, t->meta.file_size);
|
||||
bool empty = true;
|
||||
ParsedInternalKey parsed;
|
||||
t->min_sequence = 0;
|
||||
t->max_sequence = 0;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
Slice key = iter->key();
|
||||
@@ -287,9 +277,6 @@ class Repairer {
|
||||
t->meta.smallest.DecodeFrom(key);
|
||||
}
|
||||
t->meta.largest.DecodeFrom(key);
|
||||
if (parsed.sequence < t->min_sequence) {
|
||||
t->min_sequence = parsed.sequence;
|
||||
}
|
||||
if (parsed.sequence > t->max_sequence) {
|
||||
t->max_sequence = parsed.sequence;
|
||||
}
|
||||
@@ -309,7 +296,7 @@ class Repairer {
|
||||
Status WriteDescriptor() {
|
||||
std::string tmp = TempFileName(dbname_, 1);
|
||||
unique_ptr<WritableFile> file;
|
||||
Status status = env_->NewWritableFile(tmp, &file, storage_options_);
|
||||
Status status = env_->NewWritableFile(tmp, &file);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
@@ -330,8 +317,7 @@ class Repairer {
|
||||
// TODO(opt): separate out into multiple levels
|
||||
const TableInfo& t = tables_[i];
|
||||
edit_->AddFile(0, t.meta.number, t.meta.file_size,
|
||||
t.meta.smallest, t.meta.largest,
|
||||
t.min_sequence, t.max_sequence);
|
||||
t.meta.smallest, t.meta.largest);
|
||||
}
|
||||
|
||||
//fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
|
||||
@@ -388,4 +374,4 @@ Status RepairDB(const std::string& dbname, const Options& options) {
|
||||
return repairer.Run();
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
@@ -1,793 +0,0 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#include <algorithm>
|
||||
#include <set>
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/filter_policy.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/version_set.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "db/db_statistics.h"
|
||||
#include "rocksdb/cache.h"
|
||||
#include "rocksdb/compaction_filter.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "util/hash.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
#include "utilities/merge_operators.h"
|
||||
|
||||
using std::unique_ptr;
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
// SimpleTable is a simple table format for UNIT TEST ONLY. It is not built
|
||||
// as production quality.
|
||||
// SimpleTable requires the input key size to be fixed 16 bytes, value cannot
|
||||
// be longer than 150000 bytes and stored data on disk in this format:
|
||||
// +--------------------------------------------+ <= key1 offset
|
||||
// | key1 | value_size (4 bytes) | |
|
||||
// +----------------------------------------+ |
|
||||
// | value1 |
|
||||
// | |
|
||||
// +----------------------------------------+---+ <= key2 offset
|
||||
// | key2 | value_size (4 bytes) | |
|
||||
// +----------------------------------------+ |
|
||||
// | value2 |
|
||||
// | |
|
||||
// | ...... |
|
||||
// +-----------------+--------------------------+ <= index_block_offset
|
||||
// | key1 | key1 offset (8 bytes) |
|
||||
// +-----------------+--------------------------+
|
||||
// | key2 | key2 offset (8 bytes) |
|
||||
// +-----------------+--------------------------+
|
||||
// | key3 | key3 offset (8 bytes) |
|
||||
// +-----------------+--------------------------+
|
||||
// | ...... |
|
||||
// +-----------------+------------+-------------+
|
||||
// | index_block_offset (8 bytes) |
|
||||
// +------------------------------+
|
||||
|
||||
// SimpleTable is a simple table format for UNIT TEST ONLY. It is not built
|
||||
// as production quality.
|
||||
class SimpleTableReader: public TableReader {
|
||||
public:
|
||||
// Attempt to open the table that is stored in bytes [0..file_size)
|
||||
// of "file", and read the metadata entries necessary to allow
|
||||
// retrieving data from the table.
|
||||
//
|
||||
// If successful, returns ok and sets "*table" to the newly opened
|
||||
// table. The client should delete "*table" when no longer needed.
|
||||
// If there was an error while initializing the table, sets "*table"
|
||||
// to nullptr and returns a non-ok status. Does not take ownership of
|
||||
// "*source", but the client must ensure that "source" remains live
|
||||
// for the duration of the returned table's lifetime.
|
||||
//
|
||||
// *file must remain live while this Table is in use.
|
||||
static Status Open(const Options& options, const EnvOptions& soptions,
|
||||
unique_ptr<RandomAccessFile> && file, uint64_t file_size,
|
||||
unique_ptr<TableReader>* table_reader);
|
||||
|
||||
bool PrefixMayMatch(const Slice& internal_prefix) override;
|
||||
|
||||
Iterator* NewIterator(const ReadOptions&) override;
|
||||
|
||||
Status Get(
|
||||
const ReadOptions&, const Slice& key, void* arg,
|
||||
bool (*handle_result)(void* arg, const Slice& k, const Slice& v, bool),
|
||||
void (*mark_key_may_exist)(void*) = nullptr) override;
|
||||
|
||||
uint64_t ApproximateOffsetOf(const Slice& key) override;
|
||||
|
||||
bool TEST_KeyInCache(const ReadOptions& options, const Slice& key) override;
|
||||
|
||||
void SetupForCompaction() override;
|
||||
|
||||
TableStats& GetTableStats() override;
|
||||
|
||||
~SimpleTableReader();
|
||||
|
||||
private:
|
||||
struct Rep;
|
||||
Rep* rep_;
|
||||
|
||||
explicit SimpleTableReader(Rep* rep) {
|
||||
rep_ = rep;
|
||||
}
|
||||
friend class TableCache;
|
||||
friend class SimpleTableIterator;
|
||||
|
||||
Status GetOffset(const Slice& target, uint64_t* offset);
|
||||
|
||||
// No copying allowed
|
||||
explicit SimpleTableReader(const TableReader&) = delete;
|
||||
void operator=(const TableReader&) = delete;
|
||||
};
|
||||
|
||||
// Iterator to iterate SimpleTable
|
||||
class SimpleTableIterator: public Iterator {
|
||||
public:
|
||||
explicit SimpleTableIterator(SimpleTableReader* table);
|
||||
~SimpleTableIterator();
|
||||
|
||||
bool Valid() const;
|
||||
|
||||
void SeekToFirst();
|
||||
|
||||
void SeekToLast();
|
||||
|
||||
void Seek(const Slice& target);
|
||||
|
||||
void Next();
|
||||
|
||||
void Prev();
|
||||
|
||||
Slice key() const;
|
||||
|
||||
Slice value() const;
|
||||
|
||||
Status status() const;
|
||||
|
||||
private:
|
||||
SimpleTableReader* table_;
|
||||
uint64_t offset_;
|
||||
uint64_t next_offset_;
|
||||
Slice key_;
|
||||
Slice value_;
|
||||
char tmp_str_[4];
|
||||
char* key_str_;
|
||||
char* value_str_;
|
||||
int value_str_len_;
|
||||
Status status_;
|
||||
// No copying allowed
|
||||
SimpleTableIterator(const SimpleTableIterator&) = delete;
|
||||
void operator=(const Iterator&) = delete;
|
||||
};
|
||||
|
||||
struct SimpleTableReader::Rep {
|
||||
~Rep() {
|
||||
}
|
||||
Rep(const EnvOptions& storage_options, uint64_t index_start_offset,
|
||||
int num_entries) :
|
||||
soptions(storage_options), index_start_offset(index_start_offset),
|
||||
num_entries(num_entries) {
|
||||
}
|
||||
|
||||
Options options;
|
||||
const EnvOptions& soptions;
|
||||
Status status;
|
||||
unique_ptr<RandomAccessFile> file;
|
||||
uint64_t index_start_offset;
|
||||
int num_entries;
|
||||
TableStats table_stats;
|
||||
|
||||
const static int user_key_size = 16;
|
||||
const static int offset_length = 8;
|
||||
const static int key_footer_len = 8;
|
||||
|
||||
static int GetInternalKeyLength() {
|
||||
return user_key_size + key_footer_len;
|
||||
}
|
||||
};
|
||||
|
||||
SimpleTableReader::~SimpleTableReader() {
|
||||
delete rep_;
|
||||
}
|
||||
|
||||
Status SimpleTableReader::Open(const Options& options,
|
||||
const EnvOptions& soptions,
|
||||
unique_ptr<RandomAccessFile> && file,
|
||||
uint64_t size,
|
||||
unique_ptr<TableReader>* table_reader) {
|
||||
char footer_space[Rep::offset_length];
|
||||
Slice footer_input;
|
||||
Status s = file->Read(size - Rep::offset_length, Rep::offset_length,
|
||||
&footer_input, footer_space);
|
||||
if (s.ok()) {
|
||||
uint64_t index_start_offset = DecodeFixed64(footer_space);
|
||||
|
||||
int num_entries = (size - Rep::offset_length - index_start_offset)
|
||||
/ (Rep::GetInternalKeyLength() + Rep::offset_length);
|
||||
SimpleTableReader::Rep* rep = new SimpleTableReader::Rep(soptions,
|
||||
index_start_offset,
|
||||
num_entries);
|
||||
|
||||
rep->file = std::move(file);
|
||||
rep->options = options;
|
||||
table_reader->reset(new SimpleTableReader(rep));
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
void SimpleTableReader::SetupForCompaction() {
|
||||
}
|
||||
|
||||
TableStats& SimpleTableReader::GetTableStats() {
|
||||
return rep_->table_stats;
|
||||
}
|
||||
|
||||
bool SimpleTableReader::PrefixMayMatch(const Slice& internal_prefix) {
|
||||
return true;
|
||||
}
|
||||
|
||||
Iterator* SimpleTableReader::NewIterator(const ReadOptions& options) {
|
||||
return new SimpleTableIterator(this);
|
||||
}
|
||||
|
||||
Status SimpleTableReader::GetOffset(const Slice& target, uint64_t* offset) {
|
||||
uint32_t left = 0;
|
||||
uint32_t right = rep_->num_entries - 1;
|
||||
char key_chars[Rep::GetInternalKeyLength()];
|
||||
Slice tmp_slice;
|
||||
|
||||
uint32_t target_offset = 0;
|
||||
while (left <= right) {
|
||||
uint32_t mid = (left + right + 1) / 2;
|
||||
|
||||
uint64_t offset_to_read = rep_->index_start_offset
|
||||
+ (Rep::GetInternalKeyLength() + Rep::offset_length) * mid;
|
||||
Status s = rep_->file->Read(offset_to_read, Rep::GetInternalKeyLength(),
|
||||
&tmp_slice, key_chars);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
int compare_result = rep_->options.comparator->Compare(tmp_slice, target);
|
||||
|
||||
if (compare_result < 0) {
|
||||
if (left == right) {
|
||||
target_offset = right + 1;
|
||||
break;
|
||||
}
|
||||
left = mid;
|
||||
} else {
|
||||
if (left == right) {
|
||||
target_offset = left;
|
||||
break;
|
||||
}
|
||||
right = mid - 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (target_offset >= (uint32_t) rep_->num_entries) {
|
||||
*offset = rep_->index_start_offset;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
char value_offset_chars[Rep::offset_length];
|
||||
|
||||
int64_t offset_for_value_offset = rep_->index_start_offset
|
||||
+ (Rep::GetInternalKeyLength() + Rep::offset_length) * target_offset
|
||||
+ Rep::GetInternalKeyLength();
|
||||
Status s = rep_->file->Read(offset_for_value_offset, Rep::offset_length,
|
||||
&tmp_slice, value_offset_chars);
|
||||
if (s.ok()) {
|
||||
*offset = DecodeFixed64(value_offset_chars);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status SimpleTableReader::Get(
|
||||
const ReadOptions& options, const Slice& k, void* arg,
|
||||
bool (*saver)(void*, const Slice&, const Slice&, bool),
|
||||
void (*mark_key_may_exist)(void*)) {
|
||||
Status s;
|
||||
SimpleTableIterator* iter = new SimpleTableIterator(this);
|
||||
for (iter->Seek(k); iter->Valid(); iter->Next()) {
|
||||
if (!(*saver)(arg, iter->key(), iter->value(), true)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
s = iter->status();
|
||||
delete iter;
|
||||
return s;
|
||||
}
|
||||
|
||||
bool SimpleTableReader::TEST_KeyInCache(const ReadOptions& options,
|
||||
const Slice& key) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t SimpleTableReader::ApproximateOffsetOf(const Slice& key) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
SimpleTableIterator::SimpleTableIterator(SimpleTableReader* table) :
|
||||
table_(table) {
|
||||
key_str_ = new char[SimpleTableReader::Rep::GetInternalKeyLength()];
|
||||
value_str_len_ = -1;
|
||||
SeekToFirst();
|
||||
}
|
||||
|
||||
SimpleTableIterator::~SimpleTableIterator() {
|
||||
delete[] key_str_;
|
||||
if (value_str_len_ >= 0) {
|
||||
delete[] value_str_;
|
||||
}
|
||||
}
|
||||
|
||||
bool SimpleTableIterator::Valid() const {
|
||||
return offset_ < table_->rep_->index_start_offset && offset_ >= 0;
|
||||
}
|
||||
|
||||
void SimpleTableIterator::SeekToFirst() {
|
||||
next_offset_ = 0;
|
||||
Next();
|
||||
}
|
||||
|
||||
void SimpleTableIterator::SeekToLast() {
|
||||
assert(false);
|
||||
}
|
||||
|
||||
void SimpleTableIterator::Seek(const Slice& target) {
|
||||
Status s = table_->GetOffset(target, &next_offset_);
|
||||
if (!s.ok()) {
|
||||
status_ = s;
|
||||
}
|
||||
Next();
|
||||
}
|
||||
|
||||
void SimpleTableIterator::Next() {
|
||||
offset_ = next_offset_;
|
||||
if (offset_ >= table_->rep_->index_start_offset) {
|
||||
return;
|
||||
}
|
||||
Slice result;
|
||||
int internal_key_size = SimpleTableReader::Rep::GetInternalKeyLength();
|
||||
|
||||
Status s = table_->rep_->file->Read(next_offset_, internal_key_size, &result,
|
||||
key_str_);
|
||||
next_offset_ += internal_key_size;
|
||||
key_ = result;
|
||||
|
||||
Slice value_size_slice;
|
||||
s = table_->rep_->file->Read(next_offset_, 4, &value_size_slice, tmp_str_);
|
||||
next_offset_ += 4;
|
||||
uint32_t value_size = DecodeFixed32(tmp_str_);
|
||||
|
||||
Slice value_slice;
|
||||
if ((int) value_size > value_str_len_) {
|
||||
if (value_str_len_ >= 0) {
|
||||
delete[] value_str_;
|
||||
}
|
||||
value_str_ = new char[value_size];
|
||||
value_str_len_ = value_size;
|
||||
}
|
||||
s = table_->rep_->file->Read(next_offset_, value_size, &value_slice,
|
||||
value_str_);
|
||||
next_offset_ += value_size;
|
||||
value_ = value_slice;
|
||||
}
|
||||
|
||||
void SimpleTableIterator::Prev() {
|
||||
assert(false);
|
||||
}
|
||||
|
||||
Slice SimpleTableIterator::key() const {
|
||||
Log(table_->rep_->options.info_log, "key!!!!");
|
||||
return key_;
|
||||
}
|
||||
|
||||
Slice SimpleTableIterator::value() const {
|
||||
return value_;
|
||||
}
|
||||
|
||||
Status SimpleTableIterator::status() const {
|
||||
return status_;
|
||||
}
|
||||
|
||||
class SimpleTableBuilder: public TableBuilder {
|
||||
public:
|
||||
// Create a builder that will store the contents of the table it is
|
||||
// building in *file. Does not close the file. It is up to the
|
||||
// caller to close the file after calling Finish(). The output file
|
||||
// will be part of level specified by 'level'. A value of -1 means
|
||||
// that the caller does not know which level the output file will reside.
|
||||
SimpleTableBuilder(const Options& options, WritableFile* file,
|
||||
CompressionType compression_type);
|
||||
|
||||
// REQUIRES: Either Finish() or Abandon() has been called.
|
||||
~SimpleTableBuilder();
|
||||
|
||||
// Add key,value to the table being constructed.
|
||||
// REQUIRES: key is after any previously added key according to comparator.
|
||||
// REQUIRES: Finish(), Abandon() have not been called
|
||||
void Add(const Slice& key, const Slice& value) override;
|
||||
|
||||
// Return non-ok iff some error has been detected.
|
||||
Status status() const override;
|
||||
|
||||
// Finish building the table. Stops using the file passed to the
|
||||
// constructor after this function returns.
|
||||
// REQUIRES: Finish(), Abandon() have not been called
|
||||
Status Finish() override;
|
||||
|
||||
// Indicate that the contents of this builder should be abandoned. Stops
|
||||
// using the file passed to the constructor after this function returns.
|
||||
// If the caller is not going to call Finish(), it must call Abandon()
|
||||
// before destroying this builder.
|
||||
// REQUIRES: Finish(), Abandon() have not been called
|
||||
void Abandon() override;
|
||||
|
||||
// Number of calls to Add() so far.
|
||||
uint64_t NumEntries() const override;
|
||||
|
||||
// Size of the file generated so far. If invoked after a successful
|
||||
// Finish() call, returns the size of the final generated file.
|
||||
uint64_t FileSize() const override;
|
||||
|
||||
private:
|
||||
struct Rep;
|
||||
Rep* rep_;
|
||||
|
||||
// No copying allowed
|
||||
SimpleTableBuilder(const SimpleTableBuilder&) = delete;
|
||||
void operator=(const SimpleTableBuilder&) = delete;
|
||||
};
|
||||
|
||||
struct SimpleTableBuilder::Rep {
|
||||
Options options;
|
||||
WritableFile* file;
|
||||
uint64_t offset = 0;
|
||||
Status status;
|
||||
|
||||
uint64_t num_entries = 0;
|
||||
|
||||
bool closed = false; // Either Finish() or Abandon() has been called.
|
||||
|
||||
const static int user_key_size = 16;
|
||||
const static int offset_length = 8;
|
||||
const static int key_footer_len = 8;
|
||||
|
||||
static int GetInternalKeyLength() {
|
||||
return user_key_size + key_footer_len;
|
||||
}
|
||||
|
||||
std::string index;
|
||||
|
||||
Rep(const Options& opt, WritableFile* f) :
|
||||
options(opt), file(f) {
|
||||
}
|
||||
~Rep() {
|
||||
}
|
||||
};
|
||||
|
||||
SimpleTableBuilder::SimpleTableBuilder(const Options& options,
|
||||
WritableFile* file,
|
||||
CompressionType compression_type) :
|
||||
rep_(new SimpleTableBuilder::Rep(options, file)) {
|
||||
}
|
||||
|
||||
SimpleTableBuilder::~SimpleTableBuilder() {
|
||||
delete (rep_);
|
||||
}
|
||||
|
||||
void SimpleTableBuilder::Add(const Slice& key, const Slice& value) {
|
||||
assert((int ) key.size() == Rep::GetInternalKeyLength());
|
||||
|
||||
// Update index
|
||||
rep_->index.append(key.data(), key.size());
|
||||
PutFixed64(&(rep_->index), rep_->offset);
|
||||
|
||||
// Write key-value pair
|
||||
rep_->file->Append(key);
|
||||
rep_->offset += Rep::GetInternalKeyLength();
|
||||
|
||||
std::string size;
|
||||
int value_size = value.size();
|
||||
PutFixed32(&size, value_size);
|
||||
Slice sizeSlice(size);
|
||||
rep_->file->Append(sizeSlice);
|
||||
rep_->file->Append(value);
|
||||
rep_->offset += value_size + 4;
|
||||
|
||||
rep_->num_entries++;
|
||||
}
|
||||
|
||||
Status SimpleTableBuilder::status() const {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status SimpleTableBuilder::Finish() {
|
||||
Rep* r = rep_;
|
||||
assert(!r->closed);
|
||||
r->closed = true;
|
||||
|
||||
uint64_t index_offset = rep_->offset;
|
||||
Slice index_slice(rep_->index);
|
||||
rep_->file->Append(index_slice);
|
||||
rep_->offset += index_slice.size();
|
||||
|
||||
std::string index_offset_str;
|
||||
PutFixed64(&index_offset_str, index_offset);
|
||||
Slice foot_slice(index_offset_str);
|
||||
rep_->file->Append(foot_slice);
|
||||
rep_->offset += foot_slice.size();
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void SimpleTableBuilder::Abandon() {
|
||||
rep_->closed = true;
|
||||
}
|
||||
|
||||
uint64_t SimpleTableBuilder::NumEntries() const {
|
||||
return rep_->num_entries;
|
||||
}
|
||||
|
||||
uint64_t SimpleTableBuilder::FileSize() const {
|
||||
return rep_->offset;
|
||||
}
|
||||
|
||||
class SimpleTableFactory: public TableFactory {
|
||||
public:
|
||||
~SimpleTableFactory() {
|
||||
}
|
||||
SimpleTableFactory() {
|
||||
}
|
||||
const char* Name() const override {
|
||||
return "SimpleTable";
|
||||
}
|
||||
Status GetTableReader(const Options& options, const EnvOptions& soptions,
|
||||
unique_ptr<RandomAccessFile> && file,
|
||||
uint64_t file_size,
|
||||
unique_ptr<TableReader>* table_reader) const;
|
||||
|
||||
TableBuilder* GetTableBuilder(const Options& options, WritableFile* file,
|
||||
CompressionType compression_type) const;
|
||||
};
|
||||
|
||||
Status SimpleTableFactory::GetTableReader(
|
||||
const Options& options, const EnvOptions& soptions,
|
||||
unique_ptr<RandomAccessFile> && file, uint64_t file_size,
|
||||
unique_ptr<TableReader>* table_reader) const {
|
||||
|
||||
return SimpleTableReader::Open(options, soptions, std::move(file), file_size,
|
||||
table_reader);
|
||||
}
|
||||
|
||||
TableBuilder* SimpleTableFactory::GetTableBuilder(
|
||||
const Options& options, WritableFile* file,
|
||||
CompressionType compression_type) const {
|
||||
return new SimpleTableBuilder(options, file, compression_type);
|
||||
}
|
||||
|
||||
class SimpleTableDBTest {
|
||||
protected:
|
||||
public:
|
||||
std::string dbname_;
|
||||
Env* env_;
|
||||
DB* db_;
|
||||
|
||||
Options last_options_;
|
||||
|
||||
SimpleTableDBTest() :
|
||||
env_(Env::Default()) {
|
||||
dbname_ = test::TmpDir() + "/simple_table_db_test";
|
||||
ASSERT_OK(DestroyDB(dbname_, Options()));
|
||||
db_ = nullptr;
|
||||
Reopen();
|
||||
}
|
||||
|
||||
~SimpleTableDBTest() {
|
||||
delete db_;
|
||||
ASSERT_OK(DestroyDB(dbname_, Options()));
|
||||
}
|
||||
|
||||
// Return the current option configuration.
|
||||
Options CurrentOptions() {
|
||||
Options options;
|
||||
options.table_factory.reset(new SimpleTableFactory());
|
||||
return options;
|
||||
}
|
||||
|
||||
DBImpl* dbfull() {
|
||||
return reinterpret_cast<DBImpl*>(db_);
|
||||
}
|
||||
|
||||
void Reopen(Options* options = nullptr) {
|
||||
ASSERT_OK(TryReopen(options));
|
||||
}
|
||||
|
||||
void Close() {
|
||||
delete db_;
|
||||
db_ = nullptr;
|
||||
}
|
||||
|
||||
void DestroyAndReopen(Options* options = nullptr) {
|
||||
//Destroy using last options
|
||||
Destroy(&last_options_);
|
||||
ASSERT_OK(TryReopen(options));
|
||||
}
|
||||
|
||||
void Destroy(Options* options) {
|
||||
delete db_;
|
||||
db_ = nullptr;
|
||||
ASSERT_OK(DestroyDB(dbname_, *options));
|
||||
}
|
||||
|
||||
Status PureReopen(Options* options, DB** db) {
|
||||
return DB::Open(*options, dbname_, db);
|
||||
}
|
||||
|
||||
Status TryReopen(Options* options = nullptr) {
|
||||
delete db_;
|
||||
db_ = nullptr;
|
||||
Options opts;
|
||||
if (options != nullptr) {
|
||||
opts = *options;
|
||||
} else {
|
||||
opts = CurrentOptions();
|
||||
opts.create_if_missing = true;
|
||||
}
|
||||
last_options_ = opts;
|
||||
|
||||
return DB::Open(opts, dbname_, &db_);
|
||||
}
|
||||
|
||||
Status Put(const Slice& k, const Slice& v) {
|
||||
return db_->Put(WriteOptions(), k, v);
|
||||
}
|
||||
|
||||
Status Delete(const std::string& k) {
|
||||
return db_->Delete(WriteOptions(), k);
|
||||
}
|
||||
|
||||
std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
|
||||
ReadOptions options;
|
||||
options.snapshot = snapshot;
|
||||
std::string result;
|
||||
Status s = db_->Get(options, k, &result);
|
||||
if (s.IsNotFound()) {
|
||||
result = "NOT_FOUND";
|
||||
} else if (!s.ok()) {
|
||||
result = s.ToString();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
int NumTableFilesAtLevel(int level) {
|
||||
std::string property;
|
||||
ASSERT_TRUE(
|
||||
db_->GetProperty("rocksdb.num-files-at-level" + NumberToString(level),
|
||||
&property));
|
||||
return atoi(property.c_str());
|
||||
}
|
||||
|
||||
// Return spread of files per level
|
||||
std::string FilesPerLevel() {
|
||||
std::string result;
|
||||
int last_non_zero_offset = 0;
|
||||
for (int level = 0; level < db_->NumberLevels(); level++) {
|
||||
int f = NumTableFilesAtLevel(level);
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
|
||||
result += buf;
|
||||
if (f > 0) {
|
||||
last_non_zero_offset = result.size();
|
||||
}
|
||||
}
|
||||
result.resize(last_non_zero_offset);
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string IterStatus(Iterator* iter) {
|
||||
std::string result;
|
||||
if (iter->Valid()) {
|
||||
result = iter->key().ToString() + "->" + iter->value().ToString();
|
||||
} else {
|
||||
result = "(invalid)";
|
||||
}
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
TEST(SimpleTableDBTest, Empty) {
|
||||
ASSERT_TRUE(db_ != nullptr);
|
||||
ASSERT_EQ("NOT_FOUND", Get("0000000000000foo"));
|
||||
}
|
||||
|
||||
TEST(SimpleTableDBTest, ReadWrite) {
|
||||
ASSERT_OK(Put("0000000000000foo", "v1"));
|
||||
ASSERT_EQ("v1", Get("0000000000000foo"));
|
||||
ASSERT_OK(Put("0000000000000bar", "v2"));
|
||||
ASSERT_OK(Put("0000000000000foo", "v3"));
|
||||
ASSERT_EQ("v3", Get("0000000000000foo"));
|
||||
ASSERT_EQ("v2", Get("0000000000000bar"));
|
||||
}
|
||||
|
||||
TEST(SimpleTableDBTest, Flush) {
|
||||
ASSERT_OK(Put("0000000000000foo", "v1"));
|
||||
ASSERT_OK(Put("0000000000000bar", "v2"));
|
||||
ASSERT_OK(Put("0000000000000foo", "v3"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v3", Get("0000000000000foo"));
|
||||
ASSERT_EQ("v2", Get("0000000000000bar"));
|
||||
}
|
||||
|
||||
TEST(SimpleTableDBTest, Flush2) {
|
||||
ASSERT_OK(Put("0000000000000bar", "b"));
|
||||
ASSERT_OK(Put("0000000000000foo", "v1"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
|
||||
ASSERT_OK(Put("0000000000000foo", "v2"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v2", Get("0000000000000foo"));
|
||||
|
||||
ASSERT_OK(Put("0000000000000eee", "v3"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v3", Get("0000000000000eee"));
|
||||
|
||||
ASSERT_OK(Delete("0000000000000bar"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
|
||||
|
||||
ASSERT_OK(Put("0000000000000eee", "v5"));
|
||||
dbfull()->TEST_FlushMemTable();
|
||||
ASSERT_EQ("v5", Get("0000000000000eee"));
|
||||
}
|
||||
|
||||
static std::string Key(int i) {
|
||||
char buf[100];
|
||||
snprintf(buf, sizeof(buf), "key_______%06d", i);
|
||||
return std::string(buf);
|
||||
}
|
||||
|
||||
static std::string RandomString(Random* rnd, int len) {
|
||||
std::string r;
|
||||
test::RandomString(rnd, len, &r);
|
||||
return r;
|
||||
}
|
||||
|
||||
TEST(SimpleTableDBTest, CompactionTrigger) {
|
||||
Options options = CurrentOptions();
|
||||
options.write_buffer_size = 100 << 10; //100KB
|
||||
options.num_levels = 3;
|
||||
options.max_mem_compaction_level = 0;
|
||||
options.level0_file_num_compaction_trigger = 3;
|
||||
Reopen(&options);
|
||||
|
||||
Random rnd(301);
|
||||
|
||||
for (int num = 0; num < options.level0_file_num_compaction_trigger - 1;
|
||||
num++) {
|
||||
std::vector<std::string> values;
|
||||
// Write 120KB (12 values, each 10K)
|
||||
for (int i = 0; i < 12; i++) {
|
||||
values.push_back(RandomString(&rnd, 10000));
|
||||
ASSERT_OK(Put(Key(i), values[i]));
|
||||
}
|
||||
dbfull()->TEST_WaitForFlushMemTable();
|
||||
ASSERT_EQ(NumTableFilesAtLevel(0), num + 1);
|
||||
}
|
||||
|
||||
//generate one more file in level-0, and should trigger level-0 compaction
|
||||
std::vector<std::string> values;
|
||||
for (int i = 0; i < 12; i++) {
|
||||
values.push_back(RandomString(&rnd, 10000));
|
||||
ASSERT_OK(Put(Key(i), values[i]));
|
||||
}
|
||||
dbfull()->TEST_WaitForCompact();
|
||||
|
||||
ASSERT_EQ(NumTableFilesAtLevel(0), 0);
|
||||
ASSERT_EQ(NumTableFilesAtLevel(1), 1);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
}
|
||||
+10
-18
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -30,13 +25,18 @@
|
||||
// ... prev vs. next pointer ordering ...
|
||||
//
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
#define STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
#include "port/port.h"
|
||||
#include "util/arena.h"
|
||||
#include "util/random.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class Arena;
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
class SkipList {
|
||||
@@ -110,7 +110,6 @@ class SkipList {
|
||||
|
||||
// Used for optimizing sequential insert patterns
|
||||
Node* prev_[kMaxHeight];
|
||||
int prev_height_;
|
||||
|
||||
inline int GetMaxHeight() const {
|
||||
return static_cast<int>(
|
||||
@@ -271,10 +270,6 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
|
||||
Node* x = prev[0];
|
||||
Node* next = x->Next(0);
|
||||
if ((x == head_) || KeyIsAfterNode(key, x)) {
|
||||
// Adjust all relevant insertion points to the previous entry
|
||||
for (int i = 1; i < prev_height_; i++) {
|
||||
prev[i] = x;
|
||||
}
|
||||
return next;
|
||||
}
|
||||
}
|
||||
@@ -283,9 +278,6 @@ typename SkipList<Key,Comparator>::Node* SkipList<Key,Comparator>::FindGreaterOr
|
||||
int level = GetMaxHeight() - 1;
|
||||
while (true) {
|
||||
Node* next = x->Next(level);
|
||||
// Make sure the lists are sorted.
|
||||
// If x points to head_ or next points nullptr, it is trivially satisfied.
|
||||
assert((x == head_) || (next == nullptr) || KeyIsAfterNode(next->key, x));
|
||||
if (KeyIsAfterNode(key, next)) {
|
||||
// Keep searching in this list
|
||||
x = next;
|
||||
@@ -348,7 +340,6 @@ SkipList<Key,Comparator>::SkipList(Comparator cmp, Arena* arena)
|
||||
arena_(arena),
|
||||
head_(NewNode(0 /* any key will do */, kMaxHeight)),
|
||||
max_height_(reinterpret_cast<void*>(1)),
|
||||
prev_height_(1),
|
||||
rnd_(0xdeadbeef) {
|
||||
for (int i = 0; i < kMaxHeight; i++) {
|
||||
head_->SetNext(i, nullptr);
|
||||
@@ -390,7 +381,6 @@ void SkipList<Key,Comparator>::Insert(const Key& key) {
|
||||
prev_[i]->SetNext(i, x);
|
||||
}
|
||||
prev_[0] = x;
|
||||
prev_height_ = height;
|
||||
}
|
||||
|
||||
template<typename Key, class Comparator>
|
||||
@@ -403,4 +393,6 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
+11
-16
@@ -1,21 +1,16 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/skiplist.h"
|
||||
#include <set>
|
||||
#include "rocksdb/env.h"
|
||||
#include "util/arena_impl.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/arena.h"
|
||||
#include "util/hash.h"
|
||||
#include "util/random.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
typedef uint64_t Key;
|
||||
|
||||
@@ -34,9 +29,9 @@ struct TestComparator {
|
||||
class SkipTest { };
|
||||
|
||||
TEST(SkipTest, Empty) {
|
||||
ArenaImpl arena_impl;
|
||||
Arena arena;
|
||||
TestComparator cmp;
|
||||
SkipList<Key, TestComparator> list(cmp, &arena_impl);
|
||||
SkipList<Key, TestComparator> list(cmp, &arena);
|
||||
ASSERT_TRUE(!list.Contains(10));
|
||||
|
||||
SkipList<Key, TestComparator>::Iterator iter(&list);
|
||||
@@ -54,9 +49,9 @@ TEST(SkipTest, InsertAndLookup) {
|
||||
const int R = 5000;
|
||||
Random rnd(1000);
|
||||
std::set<Key> keys;
|
||||
ArenaImpl arena_impl;
|
||||
Arena arena;
|
||||
TestComparator cmp;
|
||||
SkipList<Key, TestComparator> list(cmp, &arena_impl);
|
||||
SkipList<Key, TestComparator> list(cmp, &arena);
|
||||
for (int i = 0; i < N; i++) {
|
||||
Key key = rnd.Next() % R;
|
||||
if (keys.insert(key).second) {
|
||||
@@ -209,14 +204,14 @@ class ConcurrentTest {
|
||||
// Current state of the test
|
||||
State current_;
|
||||
|
||||
ArenaImpl arena_impl_;
|
||||
Arena arena_;
|
||||
|
||||
// SkipList is not protected by mu_. We just use a single writer
|
||||
// thread to modify it.
|
||||
SkipList<Key, TestComparator> list_;
|
||||
|
||||
public:
|
||||
ConcurrentTest() : list_(TestComparator(), &arena_impl_) { }
|
||||
ConcurrentTest() : list_(TestComparator(), &arena_) { }
|
||||
|
||||
// REQUIRES: External synchronization
|
||||
void WriteStep(Random* rnd) {
|
||||
@@ -376,8 +371,8 @@ TEST(SkipTest, Concurrent3) { RunConcurrent(3); }
|
||||
TEST(SkipTest, Concurrent4) { RunConcurrent(4); }
|
||||
TEST(SkipTest, Concurrent5) { RunConcurrent(5); }
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
+12
-10
@@ -1,16 +1,13 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#include "rocksdb/db.h"
|
||||
#ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
|
||||
#define STORAGE_LEVELDB_DB_SNAPSHOT_H_
|
||||
|
||||
namespace rocksdb {
|
||||
#include "leveldb/db.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class SnapshotList;
|
||||
|
||||
@@ -62,9 +59,12 @@ class SnapshotList {
|
||||
|
||||
// retrieve all snapshot numbers. They are sorted in ascending order.
|
||||
void getAll(std::vector<SequenceNumber>& ret) {
|
||||
if (empty()) return;
|
||||
SnapshotImpl* s = &list_;
|
||||
if (empty()) return;
|
||||
SequenceNumber prev __attribute__((unused)) = 0;
|
||||
while (s->next_ != &list_) {
|
||||
assert(prev <= s->next_->number_);
|
||||
assert(prev = s->next_->number_); // assignment
|
||||
ret.push_back(s->next_->number_);
|
||||
s = s ->next_;
|
||||
}
|
||||
@@ -83,4 +83,6 @@ class SnapshotList {
|
||||
SnapshotImpl list_;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_SNAPSHOT_H_
|
||||
+42
-85
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -11,16 +6,23 @@
|
||||
|
||||
#include "db/filename.h"
|
||||
|
||||
#include "rocksdb/statistics.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/statistics.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/stop_watch.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
struct TableAndFile {
|
||||
unique_ptr<Table> table;
|
||||
};
|
||||
|
||||
static class Statistics* dbstatistics;
|
||||
|
||||
static void DeleteEntry(const Slice& key, void* value) {
|
||||
TableReader* table_reader = reinterpret_cast<TableReader*>(value);
|
||||
delete table_reader;
|
||||
TableAndFile* tf = reinterpret_cast<TableAndFile*>(value);
|
||||
RecordTick(dbstatistics, NO_FILE_CLOSES);
|
||||
delete tf;
|
||||
}
|
||||
|
||||
static void UnrefEntry(void* arg1, void* arg2) {
|
||||
@@ -31,93 +33,73 @@ static void UnrefEntry(void* arg1, void* arg2) {
|
||||
|
||||
TableCache::TableCache(const std::string& dbname,
|
||||
const Options* options,
|
||||
const EnvOptions& storage_options,
|
||||
int entries)
|
||||
: env_(options->env),
|
||||
dbname_(dbname),
|
||||
options_(options),
|
||||
storage_options_(storage_options),
|
||||
cache_(
|
||||
NewLRUCache(entries, options->table_cache_numshardbits,
|
||||
options->table_cache_remove_scan_count_limit)) {
|
||||
cache_(NewLRUCache(entries, options->table_cache_numshardbits)) {
|
||||
dbstatistics = options->statistics;
|
||||
}
|
||||
|
||||
TableCache::~TableCache() {
|
||||
}
|
||||
|
||||
Status TableCache::FindTable(const EnvOptions& toptions,
|
||||
uint64_t file_number, uint64_t file_size,
|
||||
Cache::Handle** handle, bool* table_io,
|
||||
const bool no_io) {
|
||||
Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
|
||||
Cache::Handle** handle, bool* tableIO) {
|
||||
Status s;
|
||||
char buf[sizeof(file_number)];
|
||||
EncodeFixed64(buf, file_number);
|
||||
Slice key(buf, sizeof(buf));
|
||||
*handle = cache_->Lookup(key);
|
||||
if (*handle == nullptr) {
|
||||
if (no_io) { // Dont do IO and return a not-found status
|
||||
return Status::Incomplete("Table not found in table_cache, no_io is set");
|
||||
}
|
||||
if (table_io != nullptr) {
|
||||
*table_io = true; // we had to do IO from storage
|
||||
if (tableIO != nullptr) {
|
||||
*tableIO = true; // we had to do IO from storage
|
||||
}
|
||||
std::string fname = TableFileName(dbname_, file_number);
|
||||
unique_ptr<RandomAccessFile> file;
|
||||
unique_ptr<TableReader> table_reader;
|
||||
s = env_->NewRandomAccessFile(fname, &file, toptions);
|
||||
unique_ptr<Table> table;
|
||||
s = env_->NewRandomAccessFile(fname, &file);
|
||||
RecordTick(options_->statistics, NO_FILE_OPENS);
|
||||
if (s.ok()) {
|
||||
if (options_->advise_random_on_open) {
|
||||
file->Hint(RandomAccessFile::RANDOM);
|
||||
}
|
||||
StopWatch sw(env_, options_->statistics, TABLE_OPEN_IO_MICROS);
|
||||
s = options_->table_factory->GetTableReader(*options_, toptions,
|
||||
std::move(file), file_size,
|
||||
&table_reader);
|
||||
s = Table::Open(*options_, std::move(file), file_size, &table);
|
||||
}
|
||||
|
||||
if (!s.ok()) {
|
||||
assert(table_reader == nullptr);
|
||||
assert(table == nullptr);
|
||||
RecordTick(options_->statistics, NO_FILE_ERRORS);
|
||||
// We do not cache error results so that if the error is transient,
|
||||
// or somebody repairs the file, we recover automatically.
|
||||
} else {
|
||||
TableAndFile* tf = new TableAndFile;
|
||||
tf->table = std::move(table);
|
||||
assert(file.get() == nullptr);
|
||||
*handle = cache_->Insert(key, table_reader.release(), 1, &DeleteEntry);
|
||||
*handle = cache_->Insert(key, tf, 1, &DeleteEntry);
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Iterator* TableCache::NewIterator(const ReadOptions& options,
|
||||
const EnvOptions& toptions,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
TableReader** table_reader_ptr,
|
||||
bool for_compaction) {
|
||||
if (table_reader_ptr != nullptr) {
|
||||
*table_reader_ptr = nullptr;
|
||||
Table** tableptr) {
|
||||
if (tableptr != nullptr) {
|
||||
*tableptr = nullptr;
|
||||
}
|
||||
|
||||
Cache::Handle* handle = nullptr;
|
||||
Status s = FindTable(toptions, file_number, file_size, &handle,
|
||||
nullptr, options.read_tier == kBlockCacheTier);
|
||||
Status s = FindTable(file_number, file_size, &handle);
|
||||
if (!s.ok()) {
|
||||
return NewErrorIterator(s);
|
||||
}
|
||||
|
||||
TableReader* table_reader =
|
||||
reinterpret_cast<TableReader*>(cache_->Value(handle));
|
||||
Iterator* result = table_reader->NewIterator(options);
|
||||
Table* table =
|
||||
reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table.get();
|
||||
Iterator* result = table->NewIterator(options);
|
||||
result->RegisterCleanup(&UnrefEntry, cache_.get(), handle);
|
||||
if (table_reader_ptr != nullptr) {
|
||||
*table_reader_ptr = table_reader;
|
||||
if (tableptr != nullptr) {
|
||||
*tableptr = table;
|
||||
}
|
||||
|
||||
if (for_compaction) {
|
||||
table_reader->SetupForCompaction();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -126,48 +108,23 @@ Status TableCache::Get(const ReadOptions& options,
|
||||
uint64_t file_size,
|
||||
const Slice& k,
|
||||
void* arg,
|
||||
bool (*saver)(void*, const Slice&, const Slice&, bool),
|
||||
bool* table_io,
|
||||
void (*mark_key_may_exist)(void*)) {
|
||||
void (*saver)(void*, const Slice&, const Slice&, bool),
|
||||
bool* tableIO) {
|
||||
Cache::Handle* handle = nullptr;
|
||||
Status s = FindTable(storage_options_, file_number, file_size,
|
||||
&handle, table_io,
|
||||
options.read_tier == kBlockCacheTier);
|
||||
Status s = FindTable(file_number, file_size, &handle, tableIO);
|
||||
if (s.ok()) {
|
||||
TableReader* t =
|
||||
reinterpret_cast<TableReader*>(cache_->Value(handle));
|
||||
s = t->Get(options, k, arg, saver, mark_key_may_exist);
|
||||
Table* t =
|
||||
reinterpret_cast<TableAndFile*>(cache_->Value(handle))->table.get();
|
||||
s = t->InternalGet(options, k, arg, saver);
|
||||
cache_->Release(handle);
|
||||
} else if (options.read_tier && s.IsIncomplete()) {
|
||||
// Couldnt find Table in cache but treat as kFound if no_io set
|
||||
(*mark_key_may_exist)(arg);
|
||||
return Status::OK();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
bool TableCache::PrefixMayMatch(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
const Slice& internal_prefix,
|
||||
bool* table_io) {
|
||||
Cache::Handle* handle = nullptr;
|
||||
Status s = FindTable(storage_options_, file_number,
|
||||
file_size, &handle, table_io);
|
||||
bool may_match = true;
|
||||
if (s.ok()) {
|
||||
TableReader* t =
|
||||
reinterpret_cast<TableReader*>(cache_->Value(handle));
|
||||
may_match = t->PrefixMayMatch(internal_prefix);
|
||||
cache_->Release(handle);
|
||||
}
|
||||
return may_match;
|
||||
}
|
||||
|
||||
void TableCache::Evict(uint64_t file_number) {
|
||||
char buf[sizeof(file_number)];
|
||||
EncodeFixed64(buf, file_number);
|
||||
cache_->Erase(Slice(buf, sizeof(buf)));
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+16
-31
@@ -1,31 +1,26 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// Thread-safe (provides internal synchronization)
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
#define STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
|
||||
#include <string>
|
||||
#include <stdint.h>
|
||||
#include "db/dbformat.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/cache.h"
|
||||
#include "leveldb/cache.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "port/port.h"
|
||||
#include "rocksdb/table.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class Env;
|
||||
|
||||
class TableCache {
|
||||
public:
|
||||
TableCache(const std::string& dbname, const Options* options,
|
||||
const EnvOptions& storage_options, int entries);
|
||||
TableCache(const std::string& dbname, const Options* options, int entries);
|
||||
~TableCache();
|
||||
|
||||
// Return an iterator for the specified file number (the corresponding
|
||||
@@ -36,29 +31,19 @@ class TableCache {
|
||||
// the cache and should not be deleted, and is valid for as long as the
|
||||
// returned iterator is live.
|
||||
Iterator* NewIterator(const ReadOptions& options,
|
||||
const EnvOptions& toptions,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
TableReader** table_reader_ptr = nullptr,
|
||||
bool for_compaction = false);
|
||||
Table** tableptr = nullptr);
|
||||
|
||||
// If a seek to internal key "k" in specified file finds an entry,
|
||||
// call (*handle_result)(arg, found_key, found_value) repeatedly until
|
||||
// it returns false.
|
||||
// call (*handle_result)(arg, found_key, found_value).
|
||||
Status Get(const ReadOptions& options,
|
||||
uint64_t file_number,
|
||||
uint64_t file_size,
|
||||
const Slice& k,
|
||||
void* arg,
|
||||
bool (*handle_result)(void*, const Slice&, const Slice&, bool),
|
||||
bool* table_io,
|
||||
void (*mark_key_may_exist)(void*) = nullptr);
|
||||
|
||||
// Determine whether the table may contain the specified prefix. If
|
||||
// the table index of blooms are not in memory, this may cause an I/O
|
||||
bool PrefixMayMatch(const ReadOptions& options, uint64_t file_number,
|
||||
uint64_t file_size, const Slice& internal_prefix,
|
||||
bool* table_io);
|
||||
void (*handle_result)(void*, const Slice&, const Slice&, bool),
|
||||
bool* tableIO);
|
||||
|
||||
// Evict any entry for the specified file number
|
||||
void Evict(uint64_t file_number);
|
||||
@@ -67,12 +52,12 @@ class TableCache {
|
||||
Env* const env_;
|
||||
const std::string dbname_;
|
||||
const Options* options_;
|
||||
const EnvOptions& storage_options_;
|
||||
std::shared_ptr<Cache> cache_;
|
||||
|
||||
Status FindTable(const EnvOptions& toptions, uint64_t file_number,
|
||||
uint64_t file_size, Cache::Handle**, bool* table_io=nullptr,
|
||||
const bool no_io = false);
|
||||
Status FindTable(uint64_t file_number, uint64_t file_size, Cache::Handle**,
|
||||
bool* tableIO = nullptr);
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_TABLE_CACHE_H_
|
||||
@@ -1,55 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include "db/table_stats_collector.h"
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
Status InternalKeyStatsCollector::Add(const Slice& key, const Slice& value) {
|
||||
ParsedInternalKey ikey;
|
||||
if (!ParseInternalKey(key, &ikey)) {
|
||||
return Status::InvalidArgument("Invalid internal key");
|
||||
}
|
||||
|
||||
if (ikey.type == ValueType::kTypeDeletion) {
|
||||
++deleted_keys_;
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status InternalKeyStatsCollector::Finish(
|
||||
TableStats::UserCollectedStats* stats) {
|
||||
assert(stats);
|
||||
assert(stats->find(InternalKeyTableStatsNames::kDeletedKeys) == stats->end());
|
||||
std::string val;
|
||||
|
||||
PutVarint64(&val, deleted_keys_);
|
||||
stats->insert(std::make_pair(InternalKeyTableStatsNames::kDeletedKeys, val));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status UserKeyTableStatsCollector::Add(const Slice& key, const Slice& value) {
|
||||
ParsedInternalKey ikey;
|
||||
if (!ParseInternalKey(key, &ikey)) {
|
||||
return Status::InvalidArgument("Invalid internal key");
|
||||
}
|
||||
|
||||
return collector_->Add(ikey.user_key, value);
|
||||
}
|
||||
|
||||
Status UserKeyTableStatsCollector::Finish(
|
||||
TableStats::UserCollectedStats* stats) {
|
||||
return collector_->Finish(stats);
|
||||
}
|
||||
|
||||
const std::string InternalKeyTableStatsNames::kDeletedKeys
|
||||
= "rocksdb.deleted.keys";
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -1,58 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// This file defines a collection of statistics collectors.
|
||||
#pragma once
|
||||
|
||||
#include "rocksdb/table_stats.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
struct InternalKeyTableStatsNames {
|
||||
static const std::string kDeletedKeys;
|
||||
};
|
||||
|
||||
// Collecting the statistics for internal keys. Visible only by internal
|
||||
// rocksdb modules.
|
||||
class InternalKeyStatsCollector : public TableStatsCollector {
|
||||
public:
|
||||
virtual Status Add(const Slice& key, const Slice& value);
|
||||
virtual Status Finish(TableStats::UserCollectedStats* stats);
|
||||
virtual const char* Name() const { return "InternalKeyStatsCollector"; }
|
||||
|
||||
private:
|
||||
uint64_t deleted_keys_ = 0;
|
||||
};
|
||||
|
||||
// When rocksdb creates a new table, it will encode all "user keys" into
|
||||
// "internal keys", which contains meta information of a given entry.
|
||||
//
|
||||
// This class extracts user key from the encoded internal key when Add() is
|
||||
// invoked.
|
||||
class UserKeyTableStatsCollector : public TableStatsCollector {
|
||||
public:
|
||||
explicit UserKeyTableStatsCollector(TableStatsCollector* collector):
|
||||
UserKeyTableStatsCollector(
|
||||
std::shared_ptr<TableStatsCollector>(collector)
|
||||
) {
|
||||
}
|
||||
|
||||
explicit UserKeyTableStatsCollector(
|
||||
std::shared_ptr<TableStatsCollector> collector) : collector_(collector) {
|
||||
}
|
||||
virtual ~UserKeyTableStatsCollector() { }
|
||||
virtual Status Add(const Slice& key, const Slice& value);
|
||||
virtual Status Finish(TableStats::UserCollectedStats* stats);
|
||||
virtual const char* Name() const { return collector_->Name(); }
|
||||
|
||||
protected:
|
||||
std::shared_ptr<TableStatsCollector> collector_;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -1,258 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "db/dbformat.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/table_stats_collector.h"
|
||||
#include "rocksdb/table_stats.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "util/coding.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class TableStatsTest {
|
||||
private:
|
||||
unique_ptr<TableReader> table_reader_;
|
||||
};
|
||||
|
||||
// TODO(kailiu) the following classes should be moved to some more general
|
||||
// places, so that other tests can also make use of them.
|
||||
// `FakeWritableFile` and `FakeRandomeAccessFile` bypass the real file system
|
||||
// and therefore enable us to quickly setup the tests.
|
||||
class FakeWritableFile : public WritableFile {
|
||||
public:
|
||||
~FakeWritableFile() { }
|
||||
|
||||
const std::string& contents() const { return contents_; }
|
||||
|
||||
virtual Status Close() { return Status::OK(); }
|
||||
virtual Status Flush() { return Status::OK(); }
|
||||
virtual Status Sync() { return Status::OK(); }
|
||||
|
||||
virtual Status Append(const Slice& data) {
|
||||
contents_.append(data.data(), data.size());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
std::string contents_;
|
||||
};
|
||||
|
||||
|
||||
class FakeRandomeAccessFile : public RandomAccessFile {
|
||||
public:
|
||||
explicit FakeRandomeAccessFile(const Slice& contents)
|
||||
: contents_(contents.data(), contents.size()) {
|
||||
}
|
||||
|
||||
virtual ~FakeRandomeAccessFile() { }
|
||||
|
||||
uint64_t Size() const { return contents_.size(); }
|
||||
|
||||
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
||||
char* scratch) const {
|
||||
if (offset > contents_.size()) {
|
||||
return Status::InvalidArgument("invalid Read offset");
|
||||
}
|
||||
if (offset + n > contents_.size()) {
|
||||
n = contents_.size() - offset;
|
||||
}
|
||||
memcpy(scratch, &contents_[offset], n);
|
||||
*result = Slice(scratch, n);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
std::string contents_;
|
||||
};
|
||||
|
||||
|
||||
class DumbLogger : public Logger {
|
||||
public:
|
||||
virtual void Logv(const char* format, va_list ap) { }
|
||||
virtual size_t GetLogFileSize() const { return 0; }
|
||||
};
|
||||
|
||||
// Utilities test functions
|
||||
void MakeBuilder(
|
||||
const Options& options,
|
||||
std::unique_ptr<FakeWritableFile>* writable,
|
||||
std::unique_ptr<TableBuilder>* builder) {
|
||||
writable->reset(new FakeWritableFile);
|
||||
builder->reset(
|
||||
options.table_factory->GetTableBuilder(options, writable->get(),
|
||||
options.compression));
|
||||
}
|
||||
|
||||
void OpenTable(
|
||||
const Options& options,
|
||||
const std::string& contents,
|
||||
std::unique_ptr<TableReader>* table_reader) {
|
||||
std::unique_ptr<RandomAccessFile> file(new FakeRandomeAccessFile(contents));
|
||||
auto s = options.table_factory->GetTableReader(
|
||||
options,
|
||||
EnvOptions(),
|
||||
std::move(file),
|
||||
contents.size(),
|
||||
table_reader
|
||||
);
|
||||
ASSERT_OK(s);
|
||||
}
|
||||
|
||||
// Collects keys that starts with "A" in a table.
|
||||
class RegularKeysStartWithA: public TableStatsCollector {
|
||||
public:
|
||||
const char* Name() const { return "RegularKeysStartWithA"; }
|
||||
|
||||
Status Finish(TableStats::UserCollectedStats* stats) {
|
||||
std::string encoded;
|
||||
PutVarint32(&encoded, count_);
|
||||
*stats = TableStats::UserCollectedStats {
|
||||
{ "TableStatsTest", "Rocksdb" },
|
||||
{ "Count", encoded }
|
||||
};
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status Add(const Slice& user_key, const Slice& value) {
|
||||
// simply asssume all user keys are not empty.
|
||||
if (user_key.data()[0] == 'A') {
|
||||
++count_;
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t count_ = 0;
|
||||
};
|
||||
|
||||
TEST(TableStatsTest, CustomizedTableStatsCollector) {
|
||||
Options options;
|
||||
|
||||
// make sure the entries will be inserted with order.
|
||||
std::map<std::string, std::string> kvs = {
|
||||
{"About", "val5"}, // starts with 'A'
|
||||
{"Abstract", "val2"}, // starts with 'A'
|
||||
{"Around", "val7"}, // starts with 'A'
|
||||
{"Beyond", "val3"},
|
||||
{"Builder", "val1"},
|
||||
{"Cancel", "val4"},
|
||||
{"Find", "val6"},
|
||||
};
|
||||
|
||||
// Test stats collectors with internal keys or regular keys
|
||||
for (bool encode_as_internal : { true, false }) {
|
||||
// -- Step 1: build table
|
||||
auto collector = new RegularKeysStartWithA();
|
||||
if (encode_as_internal) {
|
||||
options.table_stats_collectors = {
|
||||
std::make_shared<UserKeyTableStatsCollector>(collector)
|
||||
};
|
||||
} else {
|
||||
options.table_stats_collectors.resize(1);
|
||||
options.table_stats_collectors[0].reset(collector);
|
||||
}
|
||||
std::unique_ptr<TableBuilder> builder;
|
||||
std::unique_ptr<FakeWritableFile> writable;
|
||||
MakeBuilder(options, &writable, &builder);
|
||||
|
||||
for (const auto& kv : kvs) {
|
||||
if (encode_as_internal) {
|
||||
InternalKey ikey(kv.first, 0, ValueType::kTypeValue);
|
||||
builder->Add(ikey.Encode(), kv.second);
|
||||
} else {
|
||||
builder->Add(kv.first, kv.second);
|
||||
}
|
||||
}
|
||||
ASSERT_OK(builder->Finish());
|
||||
|
||||
// -- Step 2: Open table
|
||||
std::unique_ptr<TableReader> table_reader;
|
||||
OpenTable(options, writable->contents(), &table_reader);
|
||||
const auto& stats = table_reader->GetTableStats().user_collected_stats;
|
||||
|
||||
ASSERT_EQ("Rocksdb", stats.at("TableStatsTest"));
|
||||
|
||||
uint32_t starts_with_A = 0;
|
||||
Slice key(stats.at("Count"));
|
||||
ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
|
||||
ASSERT_EQ(3u, starts_with_A);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TableStatsTest, InternalKeyStatsCollector) {
|
||||
InternalKey keys[] = {
|
||||
InternalKey("A", 0, ValueType::kTypeValue),
|
||||
InternalKey("B", 0, ValueType::kTypeValue),
|
||||
InternalKey("C", 0, ValueType::kTypeValue),
|
||||
InternalKey("W", 0, ValueType::kTypeDeletion),
|
||||
InternalKey("X", 0, ValueType::kTypeDeletion),
|
||||
InternalKey("Y", 0, ValueType::kTypeDeletion),
|
||||
InternalKey("Z", 0, ValueType::kTypeDeletion),
|
||||
};
|
||||
|
||||
for (bool sanitized : { false, true }) {
|
||||
std::unique_ptr<TableBuilder> builder;
|
||||
std::unique_ptr<FakeWritableFile> writable;
|
||||
Options options;
|
||||
if (sanitized) {
|
||||
options.table_stats_collectors = {
|
||||
std::make_shared<RegularKeysStartWithA>()
|
||||
};
|
||||
// with sanitization, even regular stats collector will be able to
|
||||
// handle internal keys.
|
||||
auto comparator = options.comparator;
|
||||
// HACK: Set options.info_log to avoid writing log in
|
||||
// SanitizeOptions().
|
||||
options.info_log = std::make_shared<DumbLogger>();
|
||||
options = SanitizeOptions(
|
||||
"db", // just a place holder
|
||||
nullptr, // with skip internal key comparator
|
||||
nullptr, // don't care filter policy
|
||||
options
|
||||
);
|
||||
options.comparator = comparator;
|
||||
} else {
|
||||
options.table_stats_collectors = {
|
||||
std::make_shared<InternalKeyStatsCollector>()
|
||||
};
|
||||
}
|
||||
|
||||
MakeBuilder(options, &writable, &builder);
|
||||
for (const auto& k : keys) {
|
||||
builder->Add(k.Encode(), "val");
|
||||
}
|
||||
|
||||
ASSERT_OK(builder->Finish());
|
||||
|
||||
std::unique_ptr<TableReader> table_reader;
|
||||
OpenTable(options, writable->contents(), &table_reader);
|
||||
const auto& stats = table_reader->GetTableStats().user_collected_stats;
|
||||
|
||||
uint64_t deleted = 0;
|
||||
Slice key(stats.at(InternalKeyTableStatsNames::kDeletedKeys));
|
||||
ASSERT_TRUE(GetVarint64(&key, &deleted));
|
||||
ASSERT_EQ(4u, deleted);
|
||||
|
||||
if (sanitized) {
|
||||
uint32_t starts_with_A = 0;
|
||||
Slice key(stats.at("Count"));
|
||||
ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
|
||||
ASSERT_EQ(1u, starts_with_A);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
#include "db/transaction_log_impl.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
TransactionLogIteratorImpl::TransactionLogIteratorImpl(
|
||||
const std::string& dir,
|
||||
const Options* options,
|
||||
const EnvOptions& soptions,
|
||||
const SequenceNumber seq,
|
||||
std::unique_ptr<VectorLogPtr> files,
|
||||
DBImpl const * const dbimpl) :
|
||||
dir_(dir),
|
||||
options_(options),
|
||||
soptions_(soptions),
|
||||
startingSequenceNumber_(seq),
|
||||
files_(std::move(files)),
|
||||
started_(false),
|
||||
isValid_(false),
|
||||
currentFileIndex_(0),
|
||||
currentBatchSeq_(0),
|
||||
currentLastSeq_(0),
|
||||
dbimpl_(dbimpl) {
|
||||
assert(files_ != nullptr);
|
||||
assert(dbimpl_ != nullptr);
|
||||
|
||||
reporter_.env = options_->env;
|
||||
reporter_.info_log = options_->info_log.get();
|
||||
SeekToStartSequence(); // Seek till starting sequence
|
||||
}
|
||||
|
||||
Status TransactionLogIteratorImpl::OpenLogFile(
|
||||
const LogFile* logFile,
|
||||
unique_ptr<SequentialFile>* file) {
|
||||
Env* env = options_->env;
|
||||
if (logFile->Type() == kArchivedLogFile) {
|
||||
std::string fname = ArchivedLogFileName(dir_, logFile->LogNumber());
|
||||
return env->NewSequentialFile(fname, file, soptions_);
|
||||
} else {
|
||||
std::string fname = LogFileName(dir_, logFile->LogNumber());
|
||||
Status status = env->NewSequentialFile(fname, file, soptions_);
|
||||
if (!status.ok()) {
|
||||
// If cannot open file in DB directory.
|
||||
// Try the archive dir, as it could have moved in the meanwhile.
|
||||
fname = ArchivedLogFileName(dir_, logFile->LogNumber());
|
||||
status = env->NewSequentialFile(fname, file, soptions_);
|
||||
if (!status.ok()) {
|
||||
return Status::IOError("Requested file not present in the dir");
|
||||
}
|
||||
}
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
BatchResult TransactionLogIteratorImpl::GetBatch() {
|
||||
assert(isValid_); // cannot call in a non valid state.
|
||||
BatchResult result;
|
||||
result.sequence = currentBatchSeq_;
|
||||
result.writeBatchPtr = std::move(currentBatch_);
|
||||
return result;
|
||||
}
|
||||
|
||||
Status TransactionLogIteratorImpl::status() {
|
||||
return currentStatus_;
|
||||
}
|
||||
|
||||
bool TransactionLogIteratorImpl::Valid() {
|
||||
return started_ && isValid_;
|
||||
}
|
||||
|
||||
bool TransactionLogIteratorImpl::RestrictedRead(
|
||||
Slice* record,
|
||||
std::string* scratch) {
|
||||
// Don't read if no more complete entries to read from logs
|
||||
if (currentLastSeq_ >= dbimpl_->GetLatestSequenceNumber()) {
|
||||
return false;
|
||||
}
|
||||
return currentLogReader_->ReadRecord(record, scratch);
|
||||
}
|
||||
|
||||
void TransactionLogIteratorImpl::SeekToStartSequence(
|
||||
uint64_t startFileIndex,
|
||||
bool strict) {
|
||||
std::string scratch;
|
||||
Slice record;
|
||||
started_ = false;
|
||||
isValid_ = false;
|
||||
if (files_->size() <= startFileIndex) {
|
||||
return;
|
||||
}
|
||||
Status s = OpenLogReader(files_->at(startFileIndex).get());
|
||||
if (!s.ok()) {
|
||||
currentStatus_ = s;
|
||||
return;
|
||||
}
|
||||
while (RestrictedRead(&record, &scratch)) {
|
||||
if (record.size() < 12) {
|
||||
reporter_.Corruption(
|
||||
record.size(), Status::Corruption("very small log record"));
|
||||
continue;
|
||||
}
|
||||
UpdateCurrentWriteBatch(record);
|
||||
if (currentLastSeq_ >= startingSequenceNumber_) {
|
||||
if (strict && currentBatchSeq_ != startingSequenceNumber_) {
|
||||
currentStatus_ = Status::Corruption("Gap in sequence number. Could not "
|
||||
"seek to required sequence number");
|
||||
reporter_.Info(currentStatus_.ToString().c_str());
|
||||
return;
|
||||
} else if (strict) {
|
||||
reporter_.Info("Could seek required sequence number. Iterator will "
|
||||
"continue.");
|
||||
}
|
||||
isValid_ = true;
|
||||
started_ = true; // set started_ as we could seek till starting sequence
|
||||
return;
|
||||
} else {
|
||||
isValid_ = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Could not find start sequence in first file. Normally this must be the
|
||||
// only file. Otherwise log the error and let the iterator return next entry
|
||||
// If strict is set, we want to seek exactly till the start sequence and it
|
||||
// should have been present in the file we scanned above
|
||||
if (strict) {
|
||||
currentStatus_ = Status::Corruption("Gap in sequence number. Could not "
|
||||
"seek to required sequence number");
|
||||
reporter_.Info(currentStatus_.ToString().c_str());
|
||||
} else if (files_->size() != 1) {
|
||||
currentStatus_ = Status::Corruption("Start sequence was not found, "
|
||||
"skipping to the next available");
|
||||
reporter_.Info(currentStatus_.ToString().c_str());
|
||||
// Let NextImpl find the next available entry. started_ remains false
|
||||
// because we don't want to check for gaps while moving to start sequence
|
||||
NextImpl(true);
|
||||
}
|
||||
}
|
||||
|
||||
void TransactionLogIteratorImpl::Next() {
|
||||
return NextImpl(false);
|
||||
}
|
||||
|
||||
void TransactionLogIteratorImpl::NextImpl(bool internal) {
|
||||
std::string scratch;
|
||||
Slice record;
|
||||
isValid_ = false;
|
||||
if (!internal && !started_) {
|
||||
// Runs every time until we can seek to the start sequence
|
||||
return SeekToStartSequence();
|
||||
}
|
||||
while(true) {
|
||||
assert(currentLogReader_);
|
||||
if (currentLogReader_->IsEOF()) {
|
||||
currentLogReader_->UnmarkEOF();
|
||||
}
|
||||
while (RestrictedRead(&record, &scratch)) {
|
||||
if (record.size() < 12) {
|
||||
reporter_.Corruption(
|
||||
record.size(), Status::Corruption("very small log record"));
|
||||
continue;
|
||||
} else {
|
||||
// started_ should be true if called by application
|
||||
assert(internal || started_);
|
||||
// started_ should be false if called internally
|
||||
assert(!internal || !started_);
|
||||
UpdateCurrentWriteBatch(record);
|
||||
if (internal && !started_) {
|
||||
started_ = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Open the next file
|
||||
if (currentFileIndex_ < files_->size() - 1) {
|
||||
++currentFileIndex_;
|
||||
Status status =OpenLogReader(files_->at(currentFileIndex_).get());
|
||||
if (!status.ok()) {
|
||||
isValid_ = false;
|
||||
currentStatus_ = status;
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
isValid_ = false;
|
||||
if (currentLastSeq_ == dbimpl_->GetLatestSequenceNumber()) {
|
||||
currentStatus_ = Status::OK();
|
||||
} else {
|
||||
currentStatus_ = Status::IOError("NO MORE DATA LEFT");
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool TransactionLogIteratorImpl::IsBatchExpected(
|
||||
const WriteBatch* batch,
|
||||
const SequenceNumber expectedSeq) {
|
||||
assert(batch);
|
||||
SequenceNumber batchSeq = WriteBatchInternal::Sequence(batch);
|
||||
if (batchSeq != expectedSeq) {
|
||||
char buf[200];
|
||||
snprintf(buf, sizeof(buf),
|
||||
"Discontinuity in log records. Got seq=%lu, Expected seq=%lu, "
|
||||
"Last flushed seq=%lu.Log iterator will reseek the correct batch.",
|
||||
batchSeq, expectedSeq, dbimpl_->GetLatestSequenceNumber());
|
||||
reporter_.Info(buf);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
|
||||
std::unique_ptr<WriteBatch> batch(new WriteBatch());
|
||||
WriteBatchInternal::SetContents(batch.get(), record);
|
||||
|
||||
SequenceNumber expectedSeq = currentLastSeq_ + 1;
|
||||
// If the iterator has started, then confirm that we get continuous batches
|
||||
if (started_ && !IsBatchExpected(batch.get(), expectedSeq)) {
|
||||
// Seek to the batch having expected sequence number
|
||||
if (expectedSeq < files_->at(currentFileIndex_)->StartSequence()) {
|
||||
// Expected batch must lie in the previous log file
|
||||
currentFileIndex_--;
|
||||
currentFileIndex_ = (currentFileIndex_ >= 0) ? currentFileIndex_ : 0;
|
||||
}
|
||||
startingSequenceNumber_ = expectedSeq;
|
||||
// currentStatus_ will be set to Ok if reseek succeeds
|
||||
currentStatus_ = Status::NotFound("Gap in sequence numbers");
|
||||
return SeekToStartSequence(currentFileIndex_, true);
|
||||
}
|
||||
|
||||
currentBatchSeq_ = WriteBatchInternal::Sequence(batch.get());
|
||||
currentLastSeq_ = currentBatchSeq_ +
|
||||
WriteBatchInternal::Count(batch.get()) - 1;
|
||||
// currentBatchSeq_ can only change here
|
||||
assert(currentLastSeq_ <= dbimpl_->GetLatestSequenceNumber());
|
||||
|
||||
currentBatch_ = move(batch);
|
||||
isValid_ = true;
|
||||
currentStatus_ = Status::OK();
|
||||
}
|
||||
|
||||
Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) {
|
||||
unique_ptr<SequentialFile> file;
|
||||
Status status = OpenLogFile(logFile, &file);
|
||||
if (!status.ok()) {
|
||||
return status;
|
||||
}
|
||||
assert(file);
|
||||
currentLogReader_.reset(
|
||||
new log::Reader(std::move(file), &reporter_, true, 0)
|
||||
);
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace rocksdb
|
||||
@@ -1,118 +0,0 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
#pragma once
|
||||
#include <vector>
|
||||
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/options.h"
|
||||
#include "rocksdb/types.h"
|
||||
#include "rocksdb/transaction_log.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/filename.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
struct LogReporter : public log::Reader::Reporter {
|
||||
Env* env;
|
||||
Logger* info_log;
|
||||
virtual void Corruption(size_t bytes, const Status& s) {
|
||||
Log(info_log, "dropping %zu bytes; %s", bytes, s.ToString().c_str());
|
||||
}
|
||||
virtual void Info(const char* s) {
|
||||
Log(info_log, "%s", s);
|
||||
}
|
||||
};
|
||||
|
||||
class LogFileImpl : public LogFile {
|
||||
public:
|
||||
LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
|
||||
uint64_t sizeBytes) :
|
||||
logNumber_(logNum),
|
||||
type_(logType),
|
||||
startSequence_(startSeq),
|
||||
sizeFileBytes_(sizeBytes) {
|
||||
}
|
||||
|
||||
std::string PathName() const {
|
||||
if (type_ == kArchivedLogFile) {
|
||||
return ArchivedLogFileName("", logNumber_);
|
||||
}
|
||||
return LogFileName("", logNumber_);
|
||||
}
|
||||
|
||||
uint64_t LogNumber() const { return logNumber_; }
|
||||
|
||||
WalFileType Type() const { return type_; }
|
||||
|
||||
SequenceNumber StartSequence() const { return startSequence_; }
|
||||
|
||||
uint64_t SizeFileBytes() const { return sizeFileBytes_; }
|
||||
|
||||
bool operator < (const LogFile& that) const {
|
||||
return LogNumber() < that.LogNumber();
|
||||
}
|
||||
|
||||
private:
|
||||
uint64_t logNumber_;
|
||||
WalFileType type_;
|
||||
SequenceNumber startSequence_;
|
||||
uint64_t sizeFileBytes_;
|
||||
|
||||
};
|
||||
|
||||
class TransactionLogIteratorImpl : public TransactionLogIterator {
|
||||
public:
|
||||
TransactionLogIteratorImpl(const std::string& dir,
|
||||
const Options* options,
|
||||
const EnvOptions& soptions,
|
||||
const SequenceNumber seqNum,
|
||||
std::unique_ptr<VectorLogPtr> files,
|
||||
DBImpl const * const dbimpl);
|
||||
|
||||
virtual bool Valid();
|
||||
|
||||
virtual void Next();
|
||||
|
||||
virtual Status status();
|
||||
|
||||
virtual BatchResult GetBatch();
|
||||
|
||||
private:
|
||||
const std::string& dir_;
|
||||
const Options* options_;
|
||||
const EnvOptions& soptions_;
|
||||
SequenceNumber startingSequenceNumber_;
|
||||
std::unique_ptr<VectorLogPtr> files_;
|
||||
bool started_;
|
||||
bool isValid_; // not valid when it starts of.
|
||||
Status currentStatus_;
|
||||
size_t currentFileIndex_;
|
||||
std::unique_ptr<WriteBatch> currentBatch_;
|
||||
unique_ptr<log::Reader> currentLogReader_;
|
||||
Status OpenLogFile(const LogFile* logFile, unique_ptr<SequentialFile>* file);
|
||||
LogReporter reporter_;
|
||||
SequenceNumber currentBatchSeq_; // sequence number at start of current batch
|
||||
SequenceNumber currentLastSeq_; // last sequence in the current batch
|
||||
DBImpl const * const dbimpl_; // The db on whose log files this iterates
|
||||
|
||||
// Reads from transaction log only if the writebatch record has been written
|
||||
bool RestrictedRead(Slice* record, std::string* scratch);
|
||||
// Seeks to startingSequenceNumber reading from startFileIndex in files_.
|
||||
// If strict is set,then must get a batch starting with startingSequenceNumber
|
||||
void SeekToStartSequence(uint64_t startFileIndex = 0, bool strict = false);
|
||||
// Implementation of Next. SeekToStartSequence calls it internally with
|
||||
// internal=true to let it find next entry even if it has to jump gaps because
|
||||
// the iterator may start off from the first available entry but promises to
|
||||
// be continuous after that
|
||||
void NextImpl(bool internal = false);
|
||||
// Check if batch is expected, else return false
|
||||
bool IsBatchExpected(const WriteBatch* batch, SequenceNumber expectedSeq);
|
||||
// Update current batch if a continuous batch is found, else return false
|
||||
void UpdateCurrentWriteBatch(const Slice& record);
|
||||
Status OpenLogReader(const LogFile* file);
|
||||
};
|
||||
} // namespace rocksdb
|
||||
@@ -0,0 +1,172 @@
|
||||
#include "db/transaction_log_iterator_impl.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "db/filename.h"
|
||||
namespace leveldb {
|
||||
|
||||
TransactionLogIteratorImpl::TransactionLogIteratorImpl(
|
||||
const std::string& dbname,
|
||||
const Options* options,
|
||||
SequenceNumber& seq,
|
||||
std::vector<LogFile>* files,
|
||||
SequenceNumber const * const lastFlushedSequence) :
|
||||
dbname_(dbname),
|
||||
options_(options),
|
||||
sequenceNumber_(seq),
|
||||
files_(files),
|
||||
started_(false),
|
||||
isValid_(true),
|
||||
currentFileIndex_(0),
|
||||
lastFlushedSequence_(lastFlushedSequence) {
|
||||
assert(files_ != nullptr);
|
||||
assert(lastFlushedSequence_);
|
||||
}
|
||||
|
||||
LogReporter
|
||||
TransactionLogIteratorImpl::NewLogReporter(const uint64_t logNumber) {
|
||||
LogReporter reporter;
|
||||
reporter.env = options_->env;
|
||||
reporter.info_log = options_->info_log.get();
|
||||
reporter.log_number = logNumber;
|
||||
return reporter;
|
||||
}
|
||||
|
||||
Status TransactionLogIteratorImpl::OpenLogFile(
|
||||
const LogFile& logFile,
|
||||
unique_ptr<SequentialFile>* file) {
|
||||
Env* env = options_->env;
|
||||
if (logFile.type == kArchivedLogFile) {
|
||||
std::string fname = ArchivedLogFileName(dbname_, logFile.logNumber);
|
||||
return env->NewSequentialFile(fname, file);
|
||||
} else {
|
||||
std::string fname = LogFileName(dbname_, logFile.logNumber);
|
||||
Status status = env->NewSequentialFile(fname, file);
|
||||
if (!status.ok()) {
|
||||
// If cannot open file in DB directory.
|
||||
// Try the archive dir, as it could have moved in the meanwhile.
|
||||
fname = ArchivedLogFileName(dbname_, logFile.logNumber);
|
||||
status = env->NewSequentialFile(fname, file);
|
||||
if (!status.ok()) {
|
||||
return Status::IOError(" Requested file not present in the dir");
|
||||
}
|
||||
}
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
BatchResult TransactionLogIteratorImpl::GetBatch() {
|
||||
assert(isValid_); // cannot call in a non valid state.
|
||||
BatchResult result;
|
||||
result.sequence = currentSequence_;
|
||||
result.writeBatchPtr = std::move(currentBatch_);
|
||||
return result;
|
||||
}
|
||||
|
||||
Status TransactionLogIteratorImpl::status() {
|
||||
return currentStatus_;
|
||||
}
|
||||
|
||||
bool TransactionLogIteratorImpl::Valid() {
|
||||
return started_ && isValid_;
|
||||
}
|
||||
|
||||
void TransactionLogIteratorImpl::Next() {
|
||||
// First seek to the given seqNo. in the current file.
|
||||
LogFile currentLogFile = files_->at(currentFileIndex_);
|
||||
LogReporter reporter = NewLogReporter(currentLogFile.logNumber);
|
||||
std::string scratch;
|
||||
Slice record;
|
||||
|
||||
if (!started_) {
|
||||
isValid_ = false;
|
||||
if (sequenceNumber_ > *lastFlushedSequence_) {
|
||||
currentStatus_ = Status::IOError("Looking for a sequence, "
|
||||
"which is not flushed yet.");
|
||||
return;
|
||||
}
|
||||
unique_ptr<SequentialFile> file;
|
||||
Status status = OpenLogFile(currentLogFile, &file);
|
||||
if (!status.ok()) {
|
||||
currentStatus_ = status;
|
||||
return;
|
||||
}
|
||||
assert(file);
|
||||
unique_ptr<log::Reader> reader(
|
||||
new log::Reader(std::move(file), &reporter, true, 0));
|
||||
assert(reader);
|
||||
while (reader->ReadRecord(&record, &scratch)) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(
|
||||
record.size(), Status::Corruption("log record too small"));
|
||||
continue;
|
||||
}
|
||||
UpdateCurrentWriteBatch(record);
|
||||
if (currentSequence_ >= sequenceNumber_) {
|
||||
assert(currentSequence_ < *lastFlushedSequence_);
|
||||
isValid_ = true;
|
||||
currentLogReader_ = std::move(reader);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!isValid_) {
|
||||
// TODO read the entire first file. and did not find the seq number.
|
||||
// Error out.
|
||||
currentStatus_ =
|
||||
Status::NotFound("Did not find the Seq no. in first file");
|
||||
}
|
||||
started_ = true;
|
||||
} else {
|
||||
LOOK_NEXT_FILE:
|
||||
assert(currentLogReader_);
|
||||
bool openNextFile = true;
|
||||
|
||||
if (currentSequence_ == *lastFlushedSequence_) {
|
||||
// The last update has been read. and next is being called.
|
||||
isValid_ = false;
|
||||
currentStatus_ = Status::OK();
|
||||
}
|
||||
|
||||
while (currentLogReader_->ReadRecord(&record, &scratch) &&
|
||||
currentSequence_ < *lastFlushedSequence_) {
|
||||
if (record.size() < 12) {
|
||||
reporter.Corruption(
|
||||
record.size(), Status::Corruption("log record too small"));
|
||||
continue;
|
||||
} else {
|
||||
UpdateCurrentWriteBatch(record);
|
||||
openNextFile = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (openNextFile) {
|
||||
if (currentFileIndex_ < files_->size() - 1) {
|
||||
++currentFileIndex_;
|
||||
currentLogReader_.reset();
|
||||
unique_ptr<SequentialFile> file;
|
||||
Status status = OpenLogFile(files_->at(currentFileIndex_), &file);
|
||||
if (!status.ok()) {
|
||||
isValid_ = false;
|
||||
currentStatus_ = status;
|
||||
return;
|
||||
}
|
||||
currentLogReader_.reset(
|
||||
new log::Reader(std::move(file), &reporter, true, 0));
|
||||
goto LOOK_NEXT_FILE;
|
||||
} else {
|
||||
// LOOKED AT FILES. WE ARE DONE HERE.
|
||||
isValid_ = false;
|
||||
currentStatus_ = Status::IOError(" NO MORE DATA LEFT");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
|
||||
WriteBatch* batch = new WriteBatch();
|
||||
WriteBatchInternal::SetContents(batch, record);
|
||||
currentSequence_ = WriteBatchInternal::Sequence(batch);
|
||||
currentBatch_.reset(batch);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
@@ -0,0 +1,70 @@
|
||||
// Copyright 2008-present Facebook. All Rights Reserved.
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/types.h"
|
||||
#include "leveldb/transaction_log_iterator.h"
|
||||
#include "db/log_file.h"
|
||||
#include "db/log_reader.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
struct LogReporter : public log::Reader::Reporter {
|
||||
Env* env;
|
||||
Logger* info_log;
|
||||
uint64_t log_number;
|
||||
virtual void Corruption(size_t bytes, const Status& s) {
|
||||
Log(info_log, "%ld: dropping %d bytes; %s",
|
||||
log_number, static_cast<int>(bytes), s.ToString().c_str());
|
||||
}
|
||||
};
|
||||
|
||||
class TransactionLogIteratorImpl : public TransactionLogIterator {
|
||||
public:
|
||||
TransactionLogIteratorImpl(const std::string& dbname,
|
||||
const Options* options,
|
||||
SequenceNumber& seqNum,
|
||||
std::vector<LogFile>* files,
|
||||
SequenceNumber const * const lastFlushedSequence);
|
||||
|
||||
virtual ~TransactionLogIteratorImpl() {
|
||||
// TODO move to cc file.
|
||||
delete files_;
|
||||
}
|
||||
|
||||
virtual bool Valid();
|
||||
|
||||
virtual void Next();
|
||||
|
||||
virtual Status status();
|
||||
|
||||
virtual BatchResult GetBatch();
|
||||
|
||||
private:
|
||||
const std::string& dbname_;
|
||||
const Options* options_;
|
||||
const uint64_t sequenceNumber_;
|
||||
const std::vector<LogFile>* files_;
|
||||
bool started_;
|
||||
bool isValid_; // not valid when it starts of.
|
||||
Status currentStatus_;
|
||||
size_t currentFileIndex_;
|
||||
std::unique_ptr<WriteBatch> currentBatch_;
|
||||
unique_ptr<log::Reader> currentLogReader_;
|
||||
Status OpenLogFile(const LogFile& logFile, unique_ptr<SequentialFile>* file);
|
||||
LogReporter NewLogReporter(uint64_t logNumber);
|
||||
SequenceNumber const * const lastFlushedSequence_;
|
||||
// represents the sequence number being read currently.
|
||||
SequenceNumber currentSequence_;
|
||||
|
||||
void UpdateCurrentWriteBatch(const Slice& record);
|
||||
};
|
||||
|
||||
|
||||
|
||||
} // namespace leveldb
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_WRITES_ITERATOR_IMPL_H_
|
||||
+8
-34
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -12,7 +7,7 @@
|
||||
#include "db/version_set.h"
|
||||
#include "util/coding.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
// Tag numbers for serialized VersionEdit. These numbers are written to
|
||||
// disk and should not be changed.
|
||||
@@ -25,10 +20,7 @@ enum Tag {
|
||||
kDeletedFile = 6,
|
||||
kNewFile = 7,
|
||||
// 8 was used for large value refs
|
||||
kPrevLogNumber = 9,
|
||||
|
||||
// these are new formats divergent from open source leveldb
|
||||
kNewFile2 = 100 // store smallest & largest seqno
|
||||
kPrevLogNumber = 9
|
||||
};
|
||||
|
||||
void VersionEdit::Clear() {
|
||||
@@ -84,14 +76,12 @@ void VersionEdit::EncodeTo(std::string* dst) const {
|
||||
|
||||
for (size_t i = 0; i < new_files_.size(); i++) {
|
||||
const FileMetaData& f = new_files_[i].second;
|
||||
PutVarint32(dst, kNewFile2);
|
||||
PutVarint32(dst, kNewFile);
|
||||
PutVarint32(dst, new_files_[i].first); // level
|
||||
PutVarint64(dst, f.number);
|
||||
PutVarint64(dst, f.file_size);
|
||||
PutLengthPrefixedSlice(dst, f.smallest.Encode());
|
||||
PutLengthPrefixedSlice(dst, f.largest.Encode());
|
||||
PutVarint64(dst, f.smallest_seqno);
|
||||
PutVarint64(dst, f.largest_seqno);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,22 +201,6 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||
}
|
||||
break;
|
||||
|
||||
case kNewFile2:
|
||||
if (GetLevel(&input, &level, &msg) &&
|
||||
GetVarint64(&input, &f.number) &&
|
||||
GetVarint64(&input, &f.file_size) &&
|
||||
GetInternalKey(&input, &f.smallest) &&
|
||||
GetInternalKey(&input, &f.largest) &&
|
||||
GetVarint64(&input, &f.smallest_seqno) &&
|
||||
GetVarint64(&input, &f.largest_seqno) ) {
|
||||
new_files_.push_back(std::make_pair(level, f));
|
||||
} else {
|
||||
if (!msg) {
|
||||
msg = "new-file2 entry";
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
msg = "unknown tag";
|
||||
break;
|
||||
@@ -244,7 +218,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
|
||||
return result;
|
||||
}
|
||||
|
||||
std::string VersionEdit::DebugString(bool hex_key) const {
|
||||
std::string VersionEdit::DebugString() const {
|
||||
std::string r;
|
||||
r.append("VersionEdit {");
|
||||
if (has_comparator_) {
|
||||
@@ -271,7 +245,7 @@ std::string VersionEdit::DebugString(bool hex_key) const {
|
||||
r.append("\n CompactPointer: ");
|
||||
AppendNumberTo(&r, compact_pointers_[i].first);
|
||||
r.append(" ");
|
||||
r.append(compact_pointers_[i].second.DebugString(hex_key));
|
||||
r.append(compact_pointers_[i].second.DebugString());
|
||||
}
|
||||
for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
|
||||
iter != deleted_files_.end();
|
||||
@@ -290,12 +264,12 @@ std::string VersionEdit::DebugString(bool hex_key) const {
|
||||
r.append(" ");
|
||||
AppendNumberTo(&r, f.file_size);
|
||||
r.append(" ");
|
||||
r.append(f.smallest.DebugString(hex_key));
|
||||
r.append(f.smallest.DebugString());
|
||||
r.append(" .. ");
|
||||
r.append(f.largest.DebugString(hex_key));
|
||||
r.append(f.largest.DebugString());
|
||||
}
|
||||
r.append("\n}\n");
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+12
-25
@@ -1,19 +1,16 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|
||||
#define STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|
||||
|
||||
#include <set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "db/dbformat.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class VersionSet;
|
||||
|
||||
@@ -25,8 +22,6 @@ struct FileMetaData {
|
||||
InternalKey smallest; // Smallest internal key served by table
|
||||
InternalKey largest; // Largest internal key served by table
|
||||
bool being_compacted; // Is this file undergoing compaction?
|
||||
SequenceNumber smallest_seqno;// The smallest seqno in this file
|
||||
SequenceNumber largest_seqno; // The largest seqno in this file
|
||||
|
||||
FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0),
|
||||
being_compacted(false) { }
|
||||
@@ -34,9 +29,9 @@ struct FileMetaData {
|
||||
|
||||
class VersionEdit {
|
||||
public:
|
||||
explicit VersionEdit(int number_levels) :
|
||||
number_levels_(number_levels) {
|
||||
Clear();
|
||||
/* implicit */ VersionEdit(int number_levels) :
|
||||
number_levels_(number_levels) {
|
||||
Clear();
|
||||
}
|
||||
~VersionEdit() { }
|
||||
|
||||
@@ -72,17 +67,12 @@ class VersionEdit {
|
||||
void AddFile(int level, uint64_t file,
|
||||
uint64_t file_size,
|
||||
const InternalKey& smallest,
|
||||
const InternalKey& largest,
|
||||
const SequenceNumber& smallest_seqno,
|
||||
const SequenceNumber& largest_seqno) {
|
||||
const InternalKey& largest) {
|
||||
FileMetaData f;
|
||||
f.number = file;
|
||||
f.file_size = file_size;
|
||||
f.smallest = smallest;
|
||||
f.largest = largest;
|
||||
f.smallest_seqno = smallest_seqno;
|
||||
f.largest_seqno = largest_seqno;
|
||||
assert(smallest_seqno <= largest_seqno);
|
||||
new_files_.push_back(std::make_pair(level, f));
|
||||
}
|
||||
|
||||
@@ -91,15 +81,10 @@ class VersionEdit {
|
||||
deleted_files_.insert(std::make_pair(level, file));
|
||||
}
|
||||
|
||||
// Number of edits
|
||||
int NumEntries() {
|
||||
return new_files_.size() + deleted_files_.size();
|
||||
}
|
||||
|
||||
void EncodeTo(std::string* dst) const;
|
||||
Status DecodeFrom(const Slice& src);
|
||||
|
||||
std::string DebugString(bool hex_key = false) const;
|
||||
std::string DebugString() const;
|
||||
|
||||
private:
|
||||
friend class VersionSet;
|
||||
@@ -125,4 +110,6 @@ class VersionEdit {
|
||||
std::vector< std::pair<int, FileMetaData> > new_files_;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_VERSION_EDIT_H_
|
||||
+4
-11
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -10,7 +5,7 @@
|
||||
#include "db/version_edit.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
static void TestEncodeDecode(const VersionEdit& edit) {
|
||||
std::string encoded, encoded2;
|
||||
@@ -32,9 +27,7 @@ TEST(VersionEditTest, EncodeDecode) {
|
||||
TestEncodeDecode(edit);
|
||||
edit.AddFile(3, kBig + 300 + i, kBig + 400 + i,
|
||||
InternalKey("foo", kBig + 500 + i, kTypeValue),
|
||||
InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
|
||||
kBig + 500 + i,
|
||||
kBig + 600 + i);
|
||||
InternalKey("zoo", kBig + 600 + i, kTypeDeletion));
|
||||
edit.DeleteFile(4, kBig + 700 + i);
|
||||
edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue));
|
||||
}
|
||||
@@ -46,8 +39,8 @@ TEST(VersionEditTest, EncodeDecode) {
|
||||
TestEncodeDecode(edit);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
+159
-970
File diff suppressed because it is too large.
Load diff
+28
-106
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -17,7 +12,9 @@
|
||||
// Version,VersionSet are thread-compatible, but require external
|
||||
// synchronization on all accesses.
|
||||
|
||||
#pragma once
|
||||
#ifndef STORAGE_LEVELDB_DB_VERSION_SET_H_
|
||||
#define STORAGE_LEVELDB_DB_VERSION_SET_H_
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <set>
|
||||
@@ -28,16 +25,18 @@
|
||||
#include "port/port.h"
|
||||
#include "db/table_cache.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
namespace log { class Writer; }
|
||||
|
||||
class Compaction;
|
||||
class Iterator;
|
||||
class MemTable;
|
||||
class TableBuilder;
|
||||
class TableCache;
|
||||
class Version;
|
||||
class VersionSet;
|
||||
class WritableFile;
|
||||
|
||||
// Return the smallest index i such that files[i]->largest >= key.
|
||||
// Return files.size() if there is no such file.
|
||||
@@ -64,21 +63,17 @@ class Version {
|
||||
// Append to *iters a sequence of iterators that will
|
||||
// yield the contents of this Version when merged together.
|
||||
// REQUIRES: This version has been saved (see VersionSet::SaveTo)
|
||||
void AddIterators(const ReadOptions&, const EnvOptions& soptions,
|
||||
std::vector<Iterator*>* iters);
|
||||
void AddIterators(const ReadOptions&, std::vector<Iterator*>* iters);
|
||||
|
||||
// Lookup the value for key. If found, store it in *val and
|
||||
// return OK. Else return a non-OK status. Fills *stats.
|
||||
// Uses *operands to store merge_operator operations to apply later
|
||||
// REQUIRES: lock is not held
|
||||
struct GetStats {
|
||||
FileMetaData* seek_file;
|
||||
int seek_file_level;
|
||||
};
|
||||
void Get(const ReadOptions&, const LookupKey& key, std::string* val,
|
||||
Status* status, std::deque<std::string>* operands, GetStats* stats,
|
||||
const Options& db_option,
|
||||
bool* value_found = nullptr);
|
||||
Status Get(const ReadOptions&, const LookupKey& key, std::string* val,
|
||||
GetStats* stats);
|
||||
|
||||
// Adds "stats" into the current state. Returns true if a new
|
||||
// compaction may need to be triggered, false otherwise.
|
||||
@@ -111,7 +106,7 @@ class Version {
|
||||
const Slice& begin, // nullptr means before all keys
|
||||
const Slice& end, // nullptr means after all keys
|
||||
std::vector<FileMetaData*>* inputs,
|
||||
unsigned int index); // start extending from this index
|
||||
int index); // start extending from this index
|
||||
|
||||
// Returns true iff some file in the specified level overlaps
|
||||
// some part of [*smallest_user_key,*largest_user_key].
|
||||
@@ -121,14 +116,6 @@ class Version {
|
||||
const Slice* smallest_user_key,
|
||||
const Slice* largest_user_key);
|
||||
|
||||
// Returns true iff the first or last file in inputs contains
|
||||
// an overlapping user key to the file "just outside" of it (i.e.
|
||||
// just after the last file, or just before the first file)
|
||||
// REQUIRES: "*inputs" is a sorted list of non-overlapping files
|
||||
bool HasOverlappingUserKey(const std::vector<FileMetaData*>* inputs,
|
||||
int level);
|
||||
|
||||
|
||||
// Return the level at which we should place a new memtable compaction
|
||||
// result that covers the range [smallest_user_key,largest_user_key].
|
||||
int PickLevelForMemTableOutput(const Slice& smallest_user_key,
|
||||
@@ -147,14 +134,9 @@ class Version {
|
||||
private:
|
||||
friend class Compaction;
|
||||
friend class VersionSet;
|
||||
friend class DBImpl;
|
||||
|
||||
class LevelFileNumIterator;
|
||||
Iterator* NewConcatenatingIterator(const ReadOptions&,
|
||||
const EnvOptions& soptions,
|
||||
int level) const;
|
||||
bool PrefixMayMatch(const ReadOptions& options, const EnvOptions& soptions,
|
||||
const Slice& internal_prefix, Iterator* level_iter) const;
|
||||
Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const;
|
||||
|
||||
VersionSet* vset_; // VersionSet to which this Version belongs
|
||||
Version* next_; // Next version in linked list
|
||||
@@ -222,7 +204,6 @@ class VersionSet {
|
||||
public:
|
||||
VersionSet(const std::string& dbname,
|
||||
const Options* options,
|
||||
const EnvOptions& storage_options,
|
||||
TableCache* table_cache,
|
||||
const InternalKeyComparator*);
|
||||
~VersionSet();
|
||||
@@ -317,17 +298,7 @@ class VersionSet {
|
||||
// Returns true iff some level needs a compaction because it has
|
||||
// exceeded its target size.
|
||||
bool NeedsSizeCompaction() const {
|
||||
// In universal compaction case, this check doesn't really
|
||||
// check the compaction condition, but checks num of files threshold
|
||||
// only. We are not going to miss any compaction opportunity
|
||||
// but it's likely that more compactions are scheduled but
|
||||
// ending up with nothing to do. We can improve it later.
|
||||
// TODO: improve this function to be accurate for universal
|
||||
// compactions.
|
||||
int num_levels_to_check =
|
||||
(options_->compaction_style != kCompactionStyleUniversal) ?
|
||||
NumberLevels() - 1 : 1;
|
||||
for (int i = 0; i < num_levels_to_check; i++) {
|
||||
for (int i = 0; i < NumberLevels()-1; i++) {
|
||||
if (current_->compaction_score_[i] >= 1) {
|
||||
return true;
|
||||
}
|
||||
@@ -351,7 +322,8 @@ class VersionSet {
|
||||
}
|
||||
|
||||
// Add all files listed in any live version to *live.
|
||||
void AddLiveFiles(std::vector<uint64_t>* live_list);
|
||||
// May also mutate some internal state.
|
||||
void AddLiveFiles(std::set<uint64_t>* live);
|
||||
|
||||
// Add all files listed in the current version to *live.
|
||||
void AddLiveFilesCurrentVersion(std::set<uint64_t>* live);
|
||||
@@ -365,9 +337,6 @@ class VersionSet {
|
||||
struct LevelSummaryStorage {
|
||||
char buffer[100];
|
||||
};
|
||||
struct FileSummaryStorage {
|
||||
char buffer[1000];
|
||||
};
|
||||
const char* LevelSummary(LevelSummaryStorage* scratch) const;
|
||||
|
||||
// printf contents (for debugging)
|
||||
@@ -378,10 +347,6 @@ class VersionSet {
|
||||
// of files per level. Uses *scratch as backing store.
|
||||
const char* LevelDataSizeSummary(LevelSummaryStorage* scratch) const;
|
||||
|
||||
// Return a human-readable short (single-line) summary of files
|
||||
// in a specified level. Uses *scratch as backing store.
|
||||
const char* LevelFileSummary(FileSummaryStorage* scratch, int level) const;
|
||||
|
||||
// Return the size of the current manifest file
|
||||
const uint64_t ManifestFileSize() { return current_->offset_manifest_file_; }
|
||||
|
||||
@@ -391,16 +356,6 @@ class VersionSet {
|
||||
// function will return nullptr.
|
||||
Compaction* PickCompactionBySize(int level, double score);
|
||||
|
||||
// Pick files to compact in Universal mode
|
||||
Compaction* PickCompactionUniversal(int level, double score);
|
||||
|
||||
// Pick Universal compaction to limit read amplification
|
||||
Compaction* PickCompactionUniversalReadAmp(int level, double score,
|
||||
unsigned int ratio, unsigned int num_files);
|
||||
|
||||
// Pick Universal compaction to limit space amplification.
|
||||
Compaction* PickCompactionUniversalSizeAmp(int level, double score);
|
||||
|
||||
// Free up the files that were participated in a compaction
|
||||
void ReleaseCompactionFiles(Compaction* c, Status status);
|
||||
|
||||
@@ -423,14 +378,6 @@ class VersionSet {
|
||||
// Get the max file size in a given level.
|
||||
uint64_t MaxFileSizeForLevel(int level);
|
||||
|
||||
double MaxBytesForLevel(int level);
|
||||
|
||||
Status GetMetadataForFile(
|
||||
uint64_t number, int *filelevel, FileMetaData *metadata);
|
||||
|
||||
void GetLiveFilesMetaData(
|
||||
std::vector<LiveFileMetaData> *metadata);
|
||||
|
||||
private:
|
||||
class Builder;
|
||||
struct ManifestWriter;
|
||||
@@ -451,8 +398,6 @@ class VersionSet {
|
||||
InternalKey* smallest,
|
||||
InternalKey* largest);
|
||||
|
||||
void ExpandWhileOverlapping(Compaction* c);
|
||||
|
||||
void SetupOtherInputs(Compaction* c);
|
||||
|
||||
// Save current contents to *log
|
||||
@@ -462,9 +407,11 @@ class VersionSet {
|
||||
|
||||
bool ManifestContains(const std::string& record) const;
|
||||
|
||||
uint64_t ExpandedCompactionByteSizeLimit(int level);
|
||||
double MaxBytesForLevel(int level);
|
||||
|
||||
uint64_t MaxGrandParentOverlapBytes(int level);
|
||||
int64_t ExpandedCompactionByteSizeLimit(int level);
|
||||
|
||||
int64_t MaxGrandParentOverlapBytes(int level);
|
||||
|
||||
Env* const env_;
|
||||
const std::string dbname_;
|
||||
@@ -507,13 +454,6 @@ class VersionSet {
|
||||
// Save us the cost of checking file size twice in LogAndApply
|
||||
uint64_t last_observed_manifest_size_;
|
||||
|
||||
// storage options for all reads and writes except compactions
|
||||
const EnvOptions& storage_options_;
|
||||
|
||||
// storage options used for compactions. This is a copy of
|
||||
// storage_options_ but with readaheads set to readahead_compactions_.
|
||||
const EnvOptions storage_options_compactions_;
|
||||
|
||||
// No copying allowed
|
||||
VersionSet(const VersionSet&);
|
||||
void operator=(const VersionSet&);
|
||||
@@ -539,12 +479,9 @@ class Compaction {
|
||||
~Compaction();
|
||||
|
||||
// Return the level that is being compacted. Inputs from "level"
|
||||
// will be merged.
|
||||
// and "level+1" will be merged to produce a set of "level+1" files.
|
||||
int level() const { return level_; }
|
||||
|
||||
// Outputs will go to this level
|
||||
int output_level() const { return out_level_; }
|
||||
|
||||
// Return the object that holds the edits to the descriptor done
|
||||
// by this compaction.
|
||||
VersionEdit* edit() { return edit_; }
|
||||
@@ -558,9 +495,6 @@ class Compaction {
|
||||
// Maximum size of files to build during this compaction.
|
||||
uint64_t MaxOutputFileSize() const { return max_output_file_size_; }
|
||||
|
||||
// Whether compression will be enabled for compaction outputs
|
||||
bool enable_compression() const { return enable_compression_; }
|
||||
|
||||
// Is this a trivial compaction that can be implemented by just
|
||||
// moving a single input file to the next level (no merging or splitting)
|
||||
bool IsTrivialMove() const;
|
||||
@@ -586,30 +520,22 @@ class Compaction {
|
||||
// Return the score that was used to pick this compaction run.
|
||||
double score() const { return score_; }
|
||||
|
||||
// Is this compaction creating a file in the bottom most level?
|
||||
bool BottomMostLevel() { return bottommost_level_; }
|
||||
|
||||
// Does this compaction include all sst files?
|
||||
bool IsFullCompaction() { return is_full_compaction_; }
|
||||
|
||||
private:
|
||||
friend class Version;
|
||||
friend class VersionSet;
|
||||
|
||||
explicit Compaction(int level, int out_level, uint64_t target_file_size,
|
||||
explicit Compaction(int level, uint64_t target_file_size,
|
||||
uint64_t max_grandparent_overlap_bytes, int number_levels,
|
||||
bool seek_compaction = false, bool enable_compression = true);
|
||||
bool seek_compaction = false);
|
||||
|
||||
int level_;
|
||||
int out_level_; // levels to which output files are stored
|
||||
uint64_t max_output_file_size_;
|
||||
uint64_t maxGrandParentOverlapBytes_;
|
||||
int64_t maxGrandParentOverlapBytes_;
|
||||
Version* input_version_;
|
||||
VersionEdit* edit_;
|
||||
int number_levels_;
|
||||
|
||||
bool seek_compaction_;
|
||||
bool enable_compression_;
|
||||
|
||||
// Each compaction reads inputs from "level_" and "level_+1"
|
||||
std::vector<FileMetaData*> inputs_[2]; // The two sets of inputs
|
||||
@@ -619,32 +545,28 @@ class Compaction {
|
||||
std::vector<FileMetaData*> grandparents_;
|
||||
size_t grandparent_index_; // Index in grandparent_starts_
|
||||
bool seen_key_; // Some output key has been seen
|
||||
uint64_t overlapped_bytes_; // Bytes of overlap between current output
|
||||
int64_t overlapped_bytes_; // Bytes of overlap between current output
|
||||
// and grandparent files
|
||||
int base_index_; // index of the file in files_[level_]
|
||||
int parent_index_; // index of some file with same range in files_[level_+1]
|
||||
double score_; // score that was used to pick this compaction.
|
||||
|
||||
// Is this compaction creating a file in the bottom most level?
|
||||
bool bottommost_level_;
|
||||
// Does this compaction include all sst files?
|
||||
bool is_full_compaction_;
|
||||
// State for implementing IsBaseLevelForKey
|
||||
|
||||
// level_ptrs_ holds indices into input_version_->levels_: our state
|
||||
// is that we are positioned at one of the file ranges for each
|
||||
// higher level than the ones involved in this compaction (i.e. for
|
||||
// all L >= level_ + 2).
|
||||
std::vector<size_t> level_ptrs_;
|
||||
size_t* level_ptrs_;
|
||||
|
||||
// mark (or clear) all files that are being compacted
|
||||
void MarkFilesBeingCompacted(bool);
|
||||
|
||||
// Initialize whether compaction producing files at the bottommost level
|
||||
void SetupBottomMostLevel(bool isManual);
|
||||
|
||||
// In case of compaction error, reset the nextIndex that is used
|
||||
// to pick up the next file to be compacted from files_by_size_
|
||||
void ResetNextCompactionIndex();
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_VERSION_SET_H_
|
||||
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2012 Facebook. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
@@ -15,7 +10,7 @@
|
||||
#include "db/log_writer.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
Status VersionSet::ReduceNumberOfLevels(int new_levels, port::Mutex* mu) {
|
||||
|
||||
|
||||
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -12,7 +7,7 @@
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class FindFileTest {
|
||||
public:
|
||||
@@ -177,8 +172,8 @@ TEST(FindFileTest, OverlappingFiles) {
|
||||
ASSERT_TRUE(Overlaps("600", "700"));
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
+14
-107
@@ -1,8 +1,3 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@@ -12,26 +7,21 @@
|
||||
// count: fixed32
|
||||
// data: record[count]
|
||||
// record :=
|
||||
// kTypeValue varstring varstring
|
||||
// kTypeMerge varstring varstring
|
||||
// kTypeValue varstring varstring |
|
||||
// kTypeDeletion varstring
|
||||
// varstring :=
|
||||
// len: varint32
|
||||
// data: uint8[len]
|
||||
|
||||
#include "rocksdb/write_batch.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
|
||||
#include "rocksdb/options.h"
|
||||
#include "rocksdb/statistics.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "db/dbformat.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/memtable.h"
|
||||
#include "db/snapshot.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "util/coding.h"
|
||||
#include <stdexcept>
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
// WriteBatch header has an 8-byte sequence number followed by a 4-byte count.
|
||||
static const size_t kHeader = 12;
|
||||
@@ -44,28 +34,11 @@ WriteBatch::~WriteBatch() { }
|
||||
|
||||
WriteBatch::Handler::~Handler() { }
|
||||
|
||||
void WriteBatch::Handler::Merge(const Slice& key, const Slice& value) {
|
||||
throw std::runtime_error("Handler::Merge not implemented!");
|
||||
}
|
||||
|
||||
void WriteBatch::Handler::LogData(const Slice& blob) {
|
||||
// If the user has not specified something to do with blobs, then we ignore
|
||||
// them.
|
||||
}
|
||||
|
||||
bool WriteBatch::Handler::Continue() {
|
||||
return true;
|
||||
}
|
||||
|
||||
void WriteBatch::Clear() {
|
||||
rep_.clear();
|
||||
rep_.resize(kHeader);
|
||||
}
|
||||
|
||||
int WriteBatch::Count() const {
|
||||
return WriteBatchInternal::Count(this);
|
||||
}
|
||||
|
||||
Status WriteBatch::Iterate(Handler* handler) const {
|
||||
Slice input(rep_);
|
||||
if (input.size() < kHeader) {
|
||||
@@ -73,9 +46,10 @@ Status WriteBatch::Iterate(Handler* handler) const {
|
||||
}
|
||||
|
||||
input.remove_prefix(kHeader);
|
||||
Slice key, value, blob;
|
||||
Slice key, value;
|
||||
int found = 0;
|
||||
while (!input.empty() && handler->Continue()) {
|
||||
while (!input.empty()) {
|
||||
found++;
|
||||
char tag = input[0];
|
||||
input.remove_prefix(1);
|
||||
switch (tag) {
|
||||
@@ -83,7 +57,6 @@ Status WriteBatch::Iterate(Handler* handler) const {
|
||||
if (GetLengthPrefixedSlice(&input, &key) &&
|
||||
GetLengthPrefixedSlice(&input, &value)) {
|
||||
handler->Put(key, value);
|
||||
found++;
|
||||
} else {
|
||||
return Status::Corruption("bad WriteBatch Put");
|
||||
}
|
||||
@@ -91,27 +64,10 @@ Status WriteBatch::Iterate(Handler* handler) const {
|
||||
case kTypeDeletion:
|
||||
if (GetLengthPrefixedSlice(&input, &key)) {
|
||||
handler->Delete(key);
|
||||
found++;
|
||||
} else {
|
||||
return Status::Corruption("bad WriteBatch Delete");
|
||||
}
|
||||
break;
|
||||
case kTypeMerge:
|
||||
if (GetLengthPrefixedSlice(&input, &key) &&
|
||||
GetLengthPrefixedSlice(&input, &value)) {
|
||||
handler->Merge(key, value);
|
||||
found++;
|
||||
} else {
|
||||
return Status::Corruption("bad WriteBatch Merge");
|
||||
}
|
||||
break;
|
||||
case kTypeLogData:
|
||||
if (GetLengthPrefixedSlice(&input, &blob)) {
|
||||
handler->LogData(blob);
|
||||
} else {
|
||||
return Status::Corruption("bad WriteBatch Blob");
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return Status::Corruption("unknown WriteBatch tag");
|
||||
}
|
||||
@@ -152,77 +108,28 @@ void WriteBatch::Delete(const Slice& key) {
|
||||
PutLengthPrefixedSlice(&rep_, key);
|
||||
}
|
||||
|
||||
void WriteBatch::Merge(const Slice& key, const Slice& value) {
|
||||
WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
|
||||
rep_.push_back(static_cast<char>(kTypeMerge));
|
||||
PutLengthPrefixedSlice(&rep_, key);
|
||||
PutLengthPrefixedSlice(&rep_, value);
|
||||
}
|
||||
|
||||
void WriteBatch::PutLogData(const Slice& blob) {
|
||||
rep_.push_back(static_cast<char>(kTypeLogData));
|
||||
PutLengthPrefixedSlice(&rep_, blob);
|
||||
}
|
||||
|
||||
namespace {
|
||||
class MemTableInserter : public WriteBatch::Handler {
|
||||
public:
|
||||
SequenceNumber sequence_;
|
||||
MemTable* mem_;
|
||||
const Options* options_;
|
||||
DBImpl* db_;
|
||||
const bool filter_deletes_;
|
||||
|
||||
MemTableInserter(SequenceNumber sequence, MemTable* mem, const Options* opts,
|
||||
DB* db, const bool filter_deletes)
|
||||
: sequence_(sequence),
|
||||
mem_(mem),
|
||||
options_(opts),
|
||||
db_(reinterpret_cast<DBImpl*>(db)),
|
||||
filter_deletes_(filter_deletes) {
|
||||
assert(mem_);
|
||||
if (filter_deletes_) {
|
||||
assert(options_);
|
||||
assert(db_);
|
||||
}
|
||||
}
|
||||
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
if (options_->inplace_update_support
|
||||
&& mem_->Update(sequence_, kTypeValue, key, value)) {
|
||||
RecordTick(options_->statistics, NUMBER_KEYS_UPDATED);
|
||||
} else {
|
||||
mem_->Add(sequence_, kTypeValue, key, value);
|
||||
}
|
||||
sequence_++;
|
||||
}
|
||||
virtual void Merge(const Slice& key, const Slice& value) {
|
||||
mem_->Add(sequence_, kTypeMerge, key, value);
|
||||
mem_->Add(sequence_, kTypeValue, key, value);
|
||||
sequence_++;
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
if (filter_deletes_) {
|
||||
SnapshotImpl read_from_snapshot;
|
||||
read_from_snapshot.number_ = sequence_;
|
||||
ReadOptions ropts;
|
||||
ropts.snapshot = &read_from_snapshot;
|
||||
std::string value;
|
||||
if (!db_->KeyMayExist(ropts, key, &value)) {
|
||||
RecordTick(options_->statistics, NUMBER_FILTERED_DELETES);
|
||||
return;
|
||||
}
|
||||
}
|
||||
mem_->Add(sequence_, kTypeDeletion, key, Slice());
|
||||
sequence_++;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* mem,
|
||||
const Options* opts, DB* db,
|
||||
const bool filter_deletes) {
|
||||
MemTableInserter inserter(WriteBatchInternal::Sequence(b), mem, opts, db,
|
||||
filter_deletes);
|
||||
Status WriteBatchInternal::InsertInto(const WriteBatch* b,
|
||||
MemTable* memtable) {
|
||||
MemTableInserter inserter;
|
||||
inserter.sequence_ = WriteBatchInternal::Sequence(b);
|
||||
inserter.mem_ = memtable;
|
||||
return b->Iterate(&inserter);
|
||||
}
|
||||
|
||||
@@ -237,4 +144,4 @@ void WriteBatchInternal::Append(WriteBatch* dst, const WriteBatch* src) {
|
||||
dst->rep_.append(src->rep_.data() + kHeader, src->rep_.size() - kHeader);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
+11
-18
@@ -1,19 +1,14 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#include "rocksdb/types.h"
|
||||
#include "rocksdb/write_batch.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/options.h"
|
||||
#ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
|
||||
#define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
|
||||
|
||||
namespace rocksdb {
|
||||
#include "leveldb/types.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class MemTable;
|
||||
|
||||
@@ -44,14 +39,12 @@ class WriteBatchInternal {
|
||||
|
||||
static void SetContents(WriteBatch* batch, const Slice& contents);
|
||||
|
||||
// Inserts batch entries into memtable
|
||||
// Drops deletes in batch if filter_del is set to true and
|
||||
// db->KeyMayExist returns false
|
||||
static Status InsertInto(const WriteBatch* batch, MemTable* memtable,
|
||||
const Options* opts, DB* db = nullptr,
|
||||
const bool filter_del = false);
|
||||
static Status InsertInto(const WriteBatch* batch, MemTable* memtable);
|
||||
|
||||
static void Append(WriteBatch* dst, const WriteBatch* src);
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
|
||||
+9
-122
@@ -1,32 +1,23 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "rocksdb/db.h"
|
||||
#include "leveldb/db.h"
|
||||
|
||||
#include <memory>
|
||||
#include "db/memtable.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/memtablerep.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/logging.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
static std::string PrintContents(WriteBatch* b) {
|
||||
InternalKeyComparator cmp(BytewiseComparator());
|
||||
auto factory = std::make_shared<SkipListFactory>();
|
||||
MemTable* mem = new MemTable(cmp, factory);
|
||||
MemTable* mem = new MemTable(cmp);
|
||||
mem->Ref();
|
||||
std::string state;
|
||||
Options options;
|
||||
Status s = WriteBatchInternal::InsertInto(b, mem, &options);
|
||||
Status s = WriteBatchInternal::InsertInto(b, mem);
|
||||
int count = 0;
|
||||
Iterator* iter = mem->NewIterator();
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
@@ -42,30 +33,19 @@ static std::string PrintContents(WriteBatch* b) {
|
||||
state.append(")");
|
||||
count++;
|
||||
break;
|
||||
case kTypeMerge:
|
||||
state.append("Merge(");
|
||||
state.append(ikey.user_key.ToString());
|
||||
state.append(", ");
|
||||
state.append(iter->value().ToString());
|
||||
state.append(")");
|
||||
count++;
|
||||
break;
|
||||
case kTypeDeletion:
|
||||
state.append("Delete(");
|
||||
state.append(ikey.user_key.ToString());
|
||||
state.append(")");
|
||||
count++;
|
||||
break;
|
||||
case kTypeLogData:
|
||||
assert(false);
|
||||
break;
|
||||
}
|
||||
state.append("@");
|
||||
state.append(NumberToString(ikey.sequence));
|
||||
}
|
||||
delete iter;
|
||||
if (!s.ok()) {
|
||||
state.append(s.ToString());
|
||||
state.append("ParseError()");
|
||||
} else if (count != WriteBatchInternal::Count(b)) {
|
||||
state.append("CountMismatch()");
|
||||
}
|
||||
@@ -79,7 +59,6 @@ TEST(WriteBatchTest, Empty) {
|
||||
WriteBatch batch;
|
||||
ASSERT_EQ("", PrintContents(&batch));
|
||||
ASSERT_EQ(0, WriteBatchInternal::Count(&batch));
|
||||
ASSERT_EQ(0, batch.Count());
|
||||
}
|
||||
|
||||
TEST(WriteBatchTest, Multiple) {
|
||||
@@ -94,7 +73,6 @@ TEST(WriteBatchTest, Multiple) {
|
||||
"Delete(box)@101"
|
||||
"Put(foo, bar)@100",
|
||||
PrintContents(&batch));
|
||||
ASSERT_EQ(3, batch.Count());
|
||||
}
|
||||
|
||||
TEST(WriteBatchTest, Corruption) {
|
||||
@@ -106,7 +84,7 @@ TEST(WriteBatchTest, Corruption) {
|
||||
WriteBatchInternal::SetContents(&batch,
|
||||
Slice(contents.data(),contents.size()-1));
|
||||
ASSERT_EQ("Put(foo, bar)@200"
|
||||
"Corruption: bad WriteBatch Delete",
|
||||
"ParseError()",
|
||||
PrintContents(&batch));
|
||||
}
|
||||
|
||||
@@ -117,19 +95,16 @@ TEST(WriteBatchTest, Append) {
|
||||
WriteBatchInternal::Append(&b1, &b2);
|
||||
ASSERT_EQ("",
|
||||
PrintContents(&b1));
|
||||
ASSERT_EQ(0, b1.Count());
|
||||
b2.Put("a", "va");
|
||||
WriteBatchInternal::Append(&b1, &b2);
|
||||
ASSERT_EQ("Put(a, va)@200",
|
||||
PrintContents(&b1));
|
||||
ASSERT_EQ(1, b1.Count());
|
||||
b2.Clear();
|
||||
b2.Put("b", "vb");
|
||||
WriteBatchInternal::Append(&b1, &b2);
|
||||
ASSERT_EQ("Put(a, va)@200"
|
||||
"Put(b, vb)@201",
|
||||
PrintContents(&b1));
|
||||
ASSERT_EQ(2, b1.Count());
|
||||
b2.Delete("foo");
|
||||
WriteBatchInternal::Append(&b1, &b2);
|
||||
ASSERT_EQ("Put(a, va)@200"
|
||||
@@ -137,98 +112,10 @@ TEST(WriteBatchTest, Append) {
|
||||
"Put(b, vb)@201"
|
||||
"Delete(foo)@203",
|
||||
PrintContents(&b1));
|
||||
ASSERT_EQ(4, b1.Count());
|
||||
}
|
||||
|
||||
namespace {
|
||||
struct TestHandler : public WriteBatch::Handler {
|
||||
std::string seen;
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
|
||||
}
|
||||
virtual void Merge(const Slice& key, const Slice& value) {
|
||||
seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
|
||||
}
|
||||
virtual void LogData(const Slice& blob) {
|
||||
seen += "LogData(" + blob.ToString() + ")";
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
seen += "Delete(" + key.ToString() + ")";
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
TEST(WriteBatchTest, Blob) {
|
||||
WriteBatch batch;
|
||||
batch.Put(Slice("k1"), Slice("v1"));
|
||||
batch.Put(Slice("k2"), Slice("v2"));
|
||||
batch.Put(Slice("k3"), Slice("v3"));
|
||||
batch.PutLogData(Slice("blob1"));
|
||||
batch.Delete(Slice("k2"));
|
||||
batch.PutLogData(Slice("blob2"));
|
||||
batch.Merge(Slice("foo"), Slice("bar"));
|
||||
ASSERT_EQ(5, batch.Count());
|
||||
ASSERT_EQ("Merge(foo, bar)@4"
|
||||
"Put(k1, v1)@0"
|
||||
"Delete(k2)@3"
|
||||
"Put(k2, v2)@1"
|
||||
"Put(k3, v3)@2",
|
||||
PrintContents(&batch));
|
||||
|
||||
TestHandler handler;
|
||||
batch.Iterate(&handler);
|
||||
ASSERT_EQ(
|
||||
"Put(k1, v1)"
|
||||
"Put(k2, v2)"
|
||||
"Put(k3, v3)"
|
||||
"LogData(blob1)"
|
||||
"Delete(k2)"
|
||||
"LogData(blob2)"
|
||||
"Merge(foo, bar)",
|
||||
handler.seen);
|
||||
}
|
||||
|
||||
TEST(WriteBatchTest, Continue) {
|
||||
WriteBatch batch;
|
||||
|
||||
struct Handler : public TestHandler {
|
||||
int num_seen = 0;
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
++num_seen;
|
||||
TestHandler::Put(key, value);
|
||||
}
|
||||
virtual void Merge(const Slice& key, const Slice& value) {
|
||||
++num_seen;
|
||||
TestHandler::Merge(key, value);
|
||||
}
|
||||
virtual void LogData(const Slice& blob) {
|
||||
++num_seen;
|
||||
TestHandler::LogData(blob);
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
++num_seen;
|
||||
TestHandler::Delete(key);
|
||||
}
|
||||
virtual bool Continue() override {
|
||||
return num_seen < 3;
|
||||
}
|
||||
} handler;
|
||||
|
||||
batch.Put(Slice("k1"), Slice("v1"));
|
||||
batch.PutLogData(Slice("blob1"));
|
||||
batch.Delete(Slice("k1"));
|
||||
batch.PutLogData(Slice("blob2"));
|
||||
batch.Merge(Slice("foo"), Slice("bar"));
|
||||
batch.Iterate(&handler);
|
||||
ASSERT_EQ(
|
||||
"Put(k1, v1)"
|
||||
"LogData(blob1)"
|
||||
"Delete(k1)",
|
||||
handler.seen);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
@@ -111,7 +111,7 @@ static void WalCheckpoint(sqlite3* db_) {
|
||||
}
|
||||
}
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
// Helper for quickly generating random data.
|
||||
namespace {
|
||||
@@ -664,7 +664,7 @@ class Benchmark {
|
||||
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
std::string default_db_path;
|
||||
@@ -672,7 +672,7 @@ int main(int argc, char** argv) {
|
||||
double d;
|
||||
int n;
|
||||
char junk;
|
||||
if (rocksdb::Slice(argv[i]).starts_with("--benchmarks=")) {
|
||||
if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
|
||||
FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
|
||||
} else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
|
||||
(n == 0 || n == 1)) {
|
||||
@@ -688,7 +688,7 @@ int main(int argc, char** argv) {
|
||||
FLAGS_reads = n;
|
||||
} else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_value_size = n;
|
||||
} else if (rocksdb::Slice(argv[i]) == rocksdb::Slice("--no_transaction")) {
|
||||
} else if (leveldb::Slice(argv[i]) == leveldb::Slice("--no_transaction")) {
|
||||
FLAGS_transaction = false;
|
||||
} else if (sscanf(argv[i], "--page_size=%d%c", &n, &junk) == 1) {
|
||||
FLAGS_page_size = n;
|
||||
@@ -707,12 +707,12 @@ int main(int argc, char** argv) {
|
||||
|
||||
// Choose a location for the test database if none given with --db=<path>
|
||||
if (FLAGS_db == NULL) {
|
||||
rocksdb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
}
|
||||
|
||||
rocksdb::Benchmark benchmark;
|
||||
leveldb::Benchmark benchmark;
|
||||
benchmark.Run();
|
||||
return 0;
|
||||
}
|
||||
@@ -80,7 +80,7 @@ static void DBSynchronize(kyotocabinet::TreeDB* db_)
|
||||
}
|
||||
}
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
// Helper for quickly generating random data.
|
||||
namespace {
|
||||
@@ -479,7 +479,7 @@ class Benchmark {
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
std::string default_db_path;
|
||||
@@ -487,7 +487,7 @@ int main(int argc, char** argv) {
|
||||
double d;
|
||||
int n;
|
||||
char junk;
|
||||
if (rocksdb::Slice(argv[i]).starts_with("--benchmarks=")) {
|
||||
if (leveldb::Slice(argv[i]).starts_with("--benchmarks=")) {
|
||||
FLAGS_benchmarks = argv[i] + strlen("--benchmarks=");
|
||||
} else if (sscanf(argv[i], "--compression_ratio=%lf%c", &d, &junk) == 1) {
|
||||
FLAGS_compression_ratio = d;
|
||||
@@ -517,12 +517,12 @@ int main(int argc, char** argv) {
|
||||
|
||||
// Choose a location for the test database if none given with --db=<path>
|
||||
if (FLAGS_db == NULL) {
|
||||
rocksdb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
leveldb::Env::Default()->GetTestDirectory(&default_db_path);
|
||||
default_db_path += "/dbbench";
|
||||
FLAGS_db = default_db_path.c_str();
|
||||
}
|
||||
|
||||
rocksdb::Benchmark benchmark;
|
||||
leveldb::Benchmark benchmark;
|
||||
benchmark.Run();
|
||||
return 0;
|
||||
}
|
||||
+52
-52
@@ -25,27 +25,27 @@ creating it if necessary:
|
||||
#include <assert>
|
||||
#include "leveldb/db.h"
|
||||
|
||||
rocksdb::DB* db;
|
||||
rocksdb::Options options;
|
||||
leveldb::DB* db;
|
||||
leveldb::Options options;
|
||||
options.create_if_missing = true;
|
||||
rocksdb::Status status = rocksdb::DB::Open(options, "/tmp/testdb", &db);
|
||||
leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
|
||||
assert(status.ok());
|
||||
...
|
||||
</pre>
|
||||
If you want to raise an error if the database already exists, add
|
||||
the following line before the <code>rocksdb::DB::Open</code> call:
|
||||
the following line before the <code>leveldb::DB::Open</code> call:
|
||||
<pre>
|
||||
options.error_if_exists = true;
|
||||
</pre>
|
||||
<h1>Status</h1>
|
||||
<p>
|
||||
You may have noticed the <code>rocksdb::Status</code> type above. Values of this
|
||||
You may have noticed the <code>leveldb::Status</code> type above. Values of this
|
||||
type are returned by most functions in <code>leveldb</code> that may encounter an
|
||||
error. You can check if such a result is ok, and also print an
|
||||
associated error message:
|
||||
<p>
|
||||
<pre>
|
||||
rocksdb::Status s = ...;
|
||||
leveldb::Status s = ...;
|
||||
if (!s.ok()) cerr << s.ToString() << endl;
|
||||
</pre>
|
||||
<h1>Closing A Database</h1>
|
||||
@@ -65,9 +65,9 @@ modify/query the database. For example, the following code
|
||||
moves the value stored under key1 to key2.
|
||||
<pre>
|
||||
std::string value;
|
||||
rocksdb::Status s = db->Get(rocksdb::ReadOptions(), key1, &value);
|
||||
if (s.ok()) s = db->Put(rocksdb::WriteOptions(), key2, value);
|
||||
if (s.ok()) s = db->Delete(rocksdb::WriteOptions(), key1);
|
||||
leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
|
||||
if (s.ok()) s = db->Put(leveldb::WriteOptions(), key2, value);
|
||||
if (s.ok()) s = db->Delete(leveldb::WriteOptions(), key1);
|
||||
</pre>
|
||||
|
||||
<h1>Atomic Updates</h1>
|
||||
@@ -81,12 +81,12 @@ atomically apply a set of updates:
|
||||
#include "leveldb/write_batch.h"
|
||||
...
|
||||
std::string value;
|
||||
rocksdb::Status s = db->Get(rocksdb::ReadOptions(), key1, &value);
|
||||
leveldb::Status s = db->Get(leveldb::ReadOptions(), key1, &value);
|
||||
if (s.ok()) {
|
||||
rocksdb::WriteBatch batch;
|
||||
leveldb::WriteBatch batch;
|
||||
batch.Delete(key1);
|
||||
batch.Put(key2, value);
|
||||
s = db->Write(rocksdb::WriteOptions(), &batch);
|
||||
s = db->Write(leveldb::WriteOptions(), &batch);
|
||||
}
|
||||
</pre>
|
||||
The <code>WriteBatch</code> holds a sequence of edits to be made to the database,
|
||||
@@ -109,7 +109,7 @@ persistent storage. (On Posix systems, this is implemented by calling
|
||||
either <code>fsync(...)</code> or <code>fdatasync(...)</code> or
|
||||
<code>msync(..., MS_SYNC)</code> before the write operation returns.)
|
||||
<pre>
|
||||
rocksdb::WriteOptions write_options;
|
||||
leveldb::WriteOptions write_options;
|
||||
write_options.sync = true;
|
||||
db->Put(write_options, ...);
|
||||
</pre>
|
||||
@@ -144,7 +144,7 @@ the batch.
|
||||
A database may only be opened by one process at a time.
|
||||
The <code>leveldb</code> implementation acquires a lock from the
|
||||
operating system to prevent misuse. Within a single process, the
|
||||
same <code>rocksdb::DB</code> object may be safely shared by multiple
|
||||
same <code>leveldb::DB</code> object may be safely shared by multiple
|
||||
concurrent threads. I.e., different threads may write into or fetch
|
||||
iterators or call <code>Get</code> on the same database without any
|
||||
external synchronization (the leveldb implementation will
|
||||
@@ -160,7 +160,7 @@ The following example demonstrates how to print all key,value pairs
|
||||
in a database.
|
||||
<p>
|
||||
<pre>
|
||||
rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions());
|
||||
leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions());
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
cout << it->key().ToString() << ": " << it->value().ToString() << endl;
|
||||
}
|
||||
@@ -196,10 +196,10 @@ implicit snapshot of the current state.
|
||||
Snapshots are created by the DB::GetSnapshot() method:
|
||||
<p>
|
||||
<pre>
|
||||
rocksdb::ReadOptions options;
|
||||
leveldb::ReadOptions options;
|
||||
options.snapshot = db->GetSnapshot();
|
||||
... apply some updates to db ...
|
||||
rocksdb::Iterator* iter = db->NewIterator(options);
|
||||
leveldb::Iterator* iter = db->NewIterator(options);
|
||||
... read using iter to view the state when the snapshot was created ...
|
||||
delete iter;
|
||||
db->ReleaseSnapshot(options.snapshot);
|
||||
@@ -211,7 +211,7 @@ support reading as of that snapshot.
|
||||
<h1>Slice</h1>
|
||||
<p>
|
||||
The return value of the <code>it->key()</code> and <code>it->value()</code> calls above
|
||||
are instances of the <code>rocksdb::Slice</code> type. <code>Slice</code> is a simple
|
||||
are instances of the <code>leveldb::Slice</code> type. <code>Slice</code> is a simple
|
||||
structure that contains a length and a pointer to an external byte
|
||||
array. Returning a <code>Slice</code> is a cheaper alternative to returning a
|
||||
<code>std::string</code> since we do not need to copy potentially large keys and
|
||||
@@ -223,10 +223,10 @@ C++ strings and null-terminated C-style strings can be easily converted
|
||||
to a Slice:
|
||||
<p>
|
||||
<pre>
|
||||
rocksdb::Slice s1 = "hello";
|
||||
leveldb::Slice s1 = "hello";
|
||||
|
||||
std::string str("world");
|
||||
rocksdb::Slice s2 = str;
|
||||
leveldb::Slice s2 = str;
|
||||
</pre>
|
||||
A Slice can be easily converted back to a C++ string:
|
||||
<pre>
|
||||
@@ -238,7 +238,7 @@ the external byte array into which the Slice points remains live while
|
||||
the Slice is in use. For example, the following is buggy:
|
||||
<p>
|
||||
<pre>
|
||||
rocksdb::Slice slice;
|
||||
leveldb::Slice slice;
|
||||
if (...) {
|
||||
std::string str = ...;
|
||||
slice = str;
|
||||
@@ -255,16 +255,16 @@ which orders bytes lexicographically. You can however supply a custom
|
||||
comparator when opening a database. For example, suppose each
|
||||
database key consists of two numbers and we should sort by the first
|
||||
number, breaking ties by the second number. First, define a proper
|
||||
subclass of <code>rocksdb::Comparator</code> that expresses these rules:
|
||||
subclass of <code>leveldb::Comparator</code> that expresses these rules:
|
||||
<p>
|
||||
<pre>
|
||||
class TwoPartComparator : public rocksdb::Comparator {
|
||||
class TwoPartComparator : public leveldb::Comparator {
|
||||
public:
|
||||
// Three-way comparison function:
|
||||
// if a < b: negative result
|
||||
// if a > b: positive result
|
||||
// else: zero result
|
||||
int Compare(const rocksdb::Slice& a, const rocksdb::Slice& b) const {
|
||||
int Compare(const leveldb::Slice& a, const leveldb::Slice& b) const {
|
||||
int a1, a2, b1, b2;
|
||||
ParseKey(a, &a1, &a2);
|
||||
ParseKey(b, &b1, &b2);
|
||||
@@ -277,7 +277,7 @@ subclass of <code>rocksdb::Comparator</code> that expresses these rules:
|
||||
|
||||
// Ignore the following methods for now:
|
||||
const char* Name() const { return "TwoPartComparator"; }
|
||||
void FindShortestSeparator(std::string*, const rocksdb::Slice&) const { }
|
||||
void FindShortestSeparator(std::string*, const leveldb::Slice&) const { }
|
||||
void FindShortSuccessor(std::string*) const { }
|
||||
};
|
||||
</pre>
|
||||
@@ -285,18 +285,18 @@ Now create a database using this custom comparator:
|
||||
<p>
|
||||
<pre>
|
||||
TwoPartComparator cmp;
|
||||
rocksdb::DB* db;
|
||||
rocksdb::Options options;
|
||||
leveldb::DB* db;
|
||||
leveldb::Options options;
|
||||
options.create_if_missing = true;
|
||||
options.comparator = &cmp;
|
||||
rocksdb::Status status = rocksdb::DB::Open(options, "/tmp/testdb", &db);
|
||||
leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db);
|
||||
...
|
||||
</pre>
|
||||
<h2>Backwards compatibility</h2>
|
||||
<p>
|
||||
The result of the comparator's <code>Name</code> method is attached to the
|
||||
database when it is created, and is checked on every subsequent
|
||||
database open. If the name changes, the <code>rocksdb::DB::Open</code> call will
|
||||
database open. If the name changes, the <code>leveldb::DB::Open</code> call will
|
||||
fail. Therefore, change the name if and only if the new key format
|
||||
and comparison function are incompatible with existing databases, and
|
||||
it is ok to discard the contents of all existing databases.
|
||||
@@ -339,9 +339,9 @@ compression entirely, but should only do so if benchmarks show a
|
||||
performance improvement:
|
||||
<p>
|
||||
<pre>
|
||||
rocksdb::Options options;
|
||||
options.compression = rocksdb::kNoCompression;
|
||||
... rocksdb::DB::Open(options, name, ...) ....
|
||||
leveldb::Options options;
|
||||
options.compression = leveldb::kNoCompression;
|
||||
... leveldb::DB::Open(options, name, ...) ....
|
||||
</pre>
|
||||
<h2>Cache</h2>
|
||||
<p>
|
||||
@@ -353,10 +353,10 @@ uncompressed block contents.
|
||||
<pre>
|
||||
#include "leveldb/cache.h"
|
||||
|
||||
rocksdb::Options options;
|
||||
options.cache = rocksdb::NewLRUCache(100 * 1048576); // 100MB cache
|
||||
rocksdb::DB* db;
|
||||
rocksdb::DB::Open(options, name, &db);
|
||||
leveldb::Options options;
|
||||
options.cache = leveldb::NewLRUCache(100 * 1048576); // 100MB cache
|
||||
leveldb::DB* db;
|
||||
leveldb::DB::Open(options, name, &db);
|
||||
... use the db ...
|
||||
delete db
|
||||
delete options.cache;
|
||||
@@ -373,9 +373,9 @@ displacing most of the cached contents. A per-iterator option can be
|
||||
used to achieve this:
|
||||
<p>
|
||||
<pre>
|
||||
rocksdb::ReadOptions options;
|
||||
leveldb::ReadOptions options;
|
||||
options.fill_cache = false;
|
||||
rocksdb::Iterator* it = db->NewIterator(options);
|
||||
leveldb::Iterator* it = db->NewIterator(options);
|
||||
for (it->SeekToFirst(); it->Valid(); it->Next()) {
|
||||
...
|
||||
}
|
||||
@@ -407,10 +407,10 @@ a single <code>Get()</code> call may involve multiple reads from disk.
|
||||
The optional <code>FilterPolicy</code> mechanism can be used to reduce
|
||||
the number of disk reads substantially.
|
||||
<pre>
|
||||
rocksdb::Options options;
|
||||
leveldb::Options options;
|
||||
options.filter_policy = NewBloomFilter(10);
|
||||
rocksdb::DB* db;
|
||||
rocksdb::DB::Open(options, "/tmp/testdb", &db);
|
||||
leveldb::DB* db;
|
||||
leveldb::DB::Open(options, "/tmp/testdb", &db);
|
||||
... use the database ...
|
||||
delete db;
|
||||
delete options.filter_policy;
|
||||
@@ -434,7 +434,7 @@ consider a comparator that ignores trailing spaces when comparing keys.
|
||||
Instead, the application should provide a custom filter policy that
|
||||
also ignores trailing spaces. For example:
|
||||
<pre>
|
||||
class CustomFilterPolicy : public rocksdb::FilterPolicy {
|
||||
class CustomFilterPolicy : public leveldb::FilterPolicy {
|
||||
private:
|
||||
FilterPolicy* builtin_policy_;
|
||||
public:
|
||||
@@ -484,7 +484,7 @@ checksums are verified:
|
||||
parts of its persistent storage have been corrupted.
|
||||
<p>
|
||||
If a database is corrupted (perhaps it cannot be opened when
|
||||
paranoid checking is turned on), the <code>rocksdb::RepairDB</code> function
|
||||
paranoid checking is turned on), the <code>leveldb::RepairDB</code> function
|
||||
may be used to recover as much of the data as possible
|
||||
<p>
|
||||
</ul>
|
||||
@@ -494,11 +494,11 @@ The <code>GetApproximateSizes</code> method can used to get the approximate
|
||||
number of bytes of file system space used by one or more key ranges.
|
||||
<p>
|
||||
<pre>
|
||||
rocksdb::Range ranges[2];
|
||||
ranges[0] = rocksdb::Range("a", "c");
|
||||
ranges[1] = rocksdb::Range("x", "z");
|
||||
leveldb::Range ranges[2];
|
||||
ranges[0] = leveldb::Range("a", "c");
|
||||
ranges[1] = leveldb::Range("x", "z");
|
||||
uint64_t sizes[2];
|
||||
rocksdb::Status s = db->GetApproximateSizes(ranges, 2, sizes);
|
||||
leveldb::Status s = db->GetApproximateSizes(ranges, 2, sizes);
|
||||
</pre>
|
||||
The preceding call will set <code>sizes[0]</code> to the approximate number of
|
||||
bytes of file system space used by the key range <code>[a..c)</code> and
|
||||
@@ -508,21 +508,21 @@ bytes of file system space used by the key range <code>[a..c)</code> and
|
||||
<h1>Environment</h1>
|
||||
<p>
|
||||
All file operations (and other operating system calls) issued by the
|
||||
<code>leveldb</code> implementation are routed through a <code>rocksdb::Env</code> object.
|
||||
<code>leveldb</code> implementation are routed through a <code>leveldb::Env</code> object.
|
||||
Sophisticated clients may wish to provide their own <code>Env</code>
|
||||
implementation to get better control. For example, an application may
|
||||
introduce artificial delays in the file IO paths to limit the impact
|
||||
of <code>leveldb</code> on other activities in the system.
|
||||
<p>
|
||||
<pre>
|
||||
class SlowEnv : public rocksdb::Env {
|
||||
class SlowEnv : public leveldb::Env {
|
||||
.. implementation of the Env interface ...
|
||||
};
|
||||
|
||||
SlowEnv env;
|
||||
rocksdb::Options options;
|
||||
leveldb::Options options;
|
||||
options.env = &env;
|
||||
Status s = rocksdb::DB::Open(options, ...);
|
||||
Status s = leveldb::DB::Open(options, ...);
|
||||
</pre>
|
||||
<h1>Porting</h1>
|
||||
<p>
|
||||
@@ -531,7 +531,7 @@ specific implementations of the types/methods/functions exported by
|
||||
<code>leveldb/port/port.h</code>. See <code>leveldb/port/port_example.h</code> for more
|
||||
details.
|
||||
<p>
|
||||
In addition, the new platform may need a new default <code>rocksdb::Env</code>
|
||||
In addition, the new platform may need a new default <code>leveldb::Env</code>
|
||||
implementation. See <code>leveldb/util/env_posix.h</code> for an example.
|
||||
|
||||
<h1>Other Information</h1>
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 134 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 60 KiB |
@@ -4,29 +4,30 @@
|
||||
# fbcode settings. It uses the latest g++ compiler and also
|
||||
# uses jemalloc
|
||||
|
||||
TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
|
||||
TOOLCHAIN_REV=f365dbeae46a30414a2874a6f45e73e10f1caf7d
|
||||
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
|
||||
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
|
||||
TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
|
||||
TOOL_JEMALLOC=jemalloc-3.0.0/2f45f3a
|
||||
GLIBC_RUNTIME_PATH=/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1
|
||||
|
||||
# location of snappy headers and libraries
|
||||
SNAPPY_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/include"
|
||||
SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
|
||||
|
||||
# location of boost headers and libraries
|
||||
THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/include -std=gnu++0x"
|
||||
THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
|
||||
THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/bef9365/lib"
|
||||
|
||||
# location of libevent
|
||||
LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
|
||||
LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
|
||||
|
||||
# location of gflags headers and libraries
|
||||
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
|
||||
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
|
||||
|
||||
# use Intel SSE support for checksum calculations
|
||||
export USE_SSE=" -msse -msse4.2 "
|
||||
|
||||
CC="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang $CLANG_INCLUDES"
|
||||
CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $LIBEVENT_INCLUDE $GFLAGS_INCLUDE"
|
||||
CXX="$TOOLCHAIN_EXECUTABLES/clang/clang-3.1/6ca8a59/bin/clang++ $CLANG_INCLUDES $JINCLUDE $SNAPPY_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
|
||||
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
|
||||
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
|
||||
|
||||
@@ -36,8 +37,8 @@ CXXFLAGS="$CFLAGS -nostdinc++ -std=gnu++0x"
|
||||
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
|
||||
|
||||
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
|
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $LIBEVENT_LIBS $GFLAGS_LIBS"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
|
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,$GLIBC_RUNTIME_PATH/lib/ld-linux-x86-64.so.2"
|
||||
EXEC_LDFLAGS+=" -B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin"
|
||||
|
||||
@@ -4,10 +4,10 @@
|
||||
# fbcode settings. It uses the latest g++ compiler and also
|
||||
# uses jemalloc
|
||||
|
||||
TOOLCHAIN_REV=fbe3b095a4cc4a3713730050d182b7b4a80c342f
|
||||
TOOLCHAIN_REV=3748fabb2c5e033009597bae1f9ef8bf4b218581
|
||||
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
|
||||
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.7.1-glibc-2.14.1"
|
||||
TOOL_JEMALLOC=jemalloc-3.3.1/9202ce3
|
||||
TOOL_JEMALLOC=jemalloc-3.0.0/2f45f3a
|
||||
|
||||
# location of libhdfs libraries
|
||||
if test "$USE_HDFS"; then
|
||||
@@ -15,7 +15,7 @@ if test "$USE_HDFS"; then
|
||||
JINCLUDE="-I$JAVA_HOME/include -I$JAVA_HOME/include/linux"
|
||||
GLIBC_RUNTIME_PATH="/usr/local/fbcode/gcc-4.7.1-glibc-2.14.1"
|
||||
HDFSLIB=" -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64 "
|
||||
HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib "
|
||||
HDFSLIB+=" -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib "
|
||||
HDFSLIB+=" -ldl -lverify -ljava -ljvm "
|
||||
fi
|
||||
|
||||
@@ -27,19 +27,20 @@ SNAPPY_LIBS=" $TOOLCHAIN_LIB_BASE/snappy/snappy-1.0.3/7518bbe/lib/libsnappy.a"
|
||||
ZLIB_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/include"
|
||||
ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/91ddd43/lib/libz.a"
|
||||
|
||||
# location of boost headers and libraries
|
||||
THRIFT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/include"
|
||||
THRIFT_INCLUDE+=" -I./thrift -I./thrift/gen-cpp -I./thrift/lib/cpp"
|
||||
THRIFT_LIBS=" -L $TOOLCHAIN_LIB_BASE/boost/boost-1.48.0/2a0840d/lib"
|
||||
|
||||
# location of libevent
|
||||
LIBEVENT_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/include"
|
||||
LIBEVENT_LIBS=" -L $TOOLCHAIN_LIB_BASE/libevent/libevent-1.4.14b/91ddd43/lib"
|
||||
|
||||
# location of gflags headers and libraries
|
||||
GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/include"
|
||||
GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/91ddd43/lib/libgflags.a"
|
||||
|
||||
# use Intel SSE support for checksum calculations
|
||||
export USE_SSE=" -msse -msse4.2 "
|
||||
|
||||
CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/gcc"
|
||||
CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $LIBEVENT_INCLUDE $GFLAGS_INCLUDE"
|
||||
CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1-glibc-2.14.1/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $THRIFT_INCLUDE $LIBEVENT_INCLUDE"
|
||||
AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
|
||||
RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
|
||||
|
||||
@@ -47,8 +48,8 @@ CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/bin/gold -m64 -mtune=g
|
||||
CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/include -DHAVE_JEMALLOC"
|
||||
|
||||
EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/$TOOL_JEMALLOC/lib/libjemalloc.a"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/350336c/lib/libunwind.a"
|
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $LIBEVENT_LIBS $GFLAGS_LIBS"
|
||||
EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/91ddd43/lib/libunwind.a"
|
||||
EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $THRIFT_LIBS $LIBEVENT_LIBS"
|
||||
|
||||
PLATFORM_LDFLAGS="-L$TOOLCHAIN_LIB_BASE/libgcc/libgcc-4.7.1/afc21dc/lib -L$TOOLCHAIN_LIB_BASE/glibc/glibc-2.14.1/99df8fc/lib"
|
||||
|
||||
+4
-4
@@ -1,4 +1,4 @@
|
||||
This directory contains the hdfs extensions needed to make rocksdb store
|
||||
This directory contains the hdfs extensions needed to make leveldb store
|
||||
files in HDFS.
|
||||
|
||||
The hdfs.h file is copied from the Apache Hadoop 1.0 source code.
|
||||
@@ -7,12 +7,12 @@ It defines the libhdfs library
|
||||
data in HDFS. The libhdfs.a is copied from the Apache Hadoop 1.0 build.
|
||||
It implements the API defined in hdfs.h. If your hadoop cluster is running
|
||||
a different hadoop release, then install these two files manually from your
|
||||
hadoop distribution and then recompile rocksdb.
|
||||
hadoop distribution and then recompile leveldb.
|
||||
|
||||
The env_hdfs.h file defines the rocksdb objects that are needed to talk to an
|
||||
The env_hdfs.h file defines the leveldb objects that are needed to talk to an
|
||||
underlying filesystem.
|
||||
|
||||
If you want to compile rocksdb with hdfs support, please set the following
|
||||
If you want to compile leveldb with hdfs support, please set the following
|
||||
enviroment variables appropriately:
|
||||
USE_HDFS=1
|
||||
JAVA_HOME=/usr/local/jdk-6u22-64
|
||||
|
||||
+39
-37
@@ -1,22 +1,37 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
// Copyright (c) 2012 Facebook. All rights reserved.
|
||||
|
||||
#ifndef LEVELDB_HDFS_FILE_H
|
||||
#define LEVELDB_HDFS_FILE_H
|
||||
|
||||
#pragma once
|
||||
#include <algorithm>
|
||||
#include <stdio.h>
|
||||
#include <sys/time.h>
|
||||
#include <time.h>
|
||||
#include <iostream>
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
#ifdef USE_HDFS
|
||||
#include "hdfs/hdfs.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
static const std::string kProto = "hdfs://";
|
||||
static const std::string pathsep = "/";
|
||||
@@ -40,7 +55,7 @@ private:
|
||||
};
|
||||
|
||||
//
|
||||
// The HDFS environment for rocksdb. This class overrides all the
|
||||
// The HDFS environment for leveldb. This class overrides all the
|
||||
// file/dir access methods and delegates the thread-mgmt methods to the
|
||||
// default posix environment.
|
||||
//
|
||||
@@ -66,10 +81,6 @@ class HdfsEnv : public Env {
|
||||
virtual Status NewWritableFile(const std::string& fname,
|
||||
WritableFile** result);
|
||||
|
||||
virtual Status NewRandomRWFile(const std::string& fname,
|
||||
unique_ptr<RandomRWFile>* result,
|
||||
const EnvOptions& options);
|
||||
|
||||
virtual bool FileExists(const std::string& fname);
|
||||
|
||||
virtual Status GetChildren(const std::string& path,
|
||||
@@ -96,9 +107,8 @@ class HdfsEnv : public Env {
|
||||
|
||||
virtual Status NewLogger(const std::string& fname, Logger** result);
|
||||
|
||||
virtual void Schedule(void (*function)(void* arg), void* arg,
|
||||
Priority pri = LOW) {
|
||||
posixEnv->Schedule(function, arg, pri);
|
||||
virtual void Schedule( void (*function)(void* arg), void* arg) {
|
||||
posixEnv->Schedule(function, arg);
|
||||
}
|
||||
|
||||
virtual void StartThread(void (*function)(void* arg), void* arg) {
|
||||
@@ -130,8 +140,8 @@ class HdfsEnv : public Env {
|
||||
return posixEnv->GetAbsolutePath(db_path, output_path);
|
||||
}
|
||||
|
||||
virtual void SetBackgroundThreads(int number, Priority pri = LOW) {
|
||||
posixEnv->SetBackgroundThreads(number, pri);
|
||||
virtual void SetBackgroundThreads(int number) {
|
||||
posixEnv->SetBackgroundThreads(number);
|
||||
}
|
||||
|
||||
virtual std::string TimeToString(uint64_t number) {
|
||||
@@ -203,12 +213,12 @@ class HdfsEnv : public Env {
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#else // USE_HDFS
|
||||
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
static const Status notsup;
|
||||
|
||||
@@ -216,7 +226,7 @@ class HdfsEnv : public Env {
|
||||
|
||||
public:
|
||||
HdfsEnv(const std::string& fsname) {
|
||||
fprintf(stderr, "You have not build rocksdb with HDFS support\n");
|
||||
fprintf(stderr, "You have not build leveldb with HDFS support\n");
|
||||
fprintf(stderr, "Please see hdfs/README for details\n");
|
||||
throw new std::exception();
|
||||
}
|
||||
@@ -225,24 +235,15 @@ class HdfsEnv : public Env {
|
||||
}
|
||||
|
||||
virtual Status NewSequentialFile(const std::string& fname,
|
||||
unique_ptr<SequentialFile>* result,
|
||||
const EnvOptions& options);
|
||||
unique_ptr<SequentialFile>* result);
|
||||
|
||||
virtual Status NewRandomAccessFile(const std::string& fname,
|
||||
unique_ptr<RandomAccessFile>* result,
|
||||
const EnvOptions& options) {
|
||||
unique_ptr<RandomAccessFile>* result) {
|
||||
return notsup;
|
||||
}
|
||||
|
||||
virtual Status NewWritableFile(const std::string& fname,
|
||||
unique_ptr<WritableFile>* result,
|
||||
const EnvOptions& options) {
|
||||
return notsup;
|
||||
}
|
||||
|
||||
virtual Status NewRandomRWFile(const std::string& fname,
|
||||
unique_ptr<RandomRWFile>* result,
|
||||
const EnvOptions& options) {
|
||||
unique_ptr<WritableFile>* result) {
|
||||
return notsup;
|
||||
}
|
||||
|
||||
@@ -275,8 +276,7 @@ class HdfsEnv : public Env {
|
||||
virtual Status NewLogger(const std::string& fname,
|
||||
shared_ptr<Logger>* result){return notsup;}
|
||||
|
||||
virtual void Schedule(void (*function)(void* arg), void* arg,
|
||||
Priority pri = LOW) {}
|
||||
virtual void Schedule( void (*function)(void* arg), void* arg) {}
|
||||
|
||||
virtual void StartThread(void (*function)(void* arg), void* arg) {}
|
||||
|
||||
@@ -293,10 +293,12 @@ class HdfsEnv : public Env {
|
||||
virtual Status GetAbsolutePath(const std::string& db_path,
|
||||
std::string* outputpath) {return notsup;}
|
||||
|
||||
virtual void SetBackgroundThreads(int number, Priority pri = LOW) {}
|
||||
virtual void SetBackgroundThreads(int number) {}
|
||||
|
||||
virtual std::string TimeToString(uint64_t number) { return "";}
|
||||
};
|
||||
}
|
||||
|
||||
#endif // USE_HDFS
|
||||
|
||||
#endif // LEVELDB_HDFS_FILE_H
|
||||
+18
-5
@@ -1,8 +1,21 @@
|
||||
// Copyright (c) 2013, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
//
|
||||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef LIBHDFS_HDFS_H
|
||||
#define LIBHDFS_HDFS_H
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
|
||||
#include "helpers/memenv/memenv.h"
|
||||
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "port/port.h"
|
||||
#include "util/mutexlock.h"
|
||||
#include <map>
|
||||
@@ -13,7 +13,7 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -233,8 +233,7 @@ class InMemoryEnv : public EnvWrapper {
|
||||
|
||||
// Partial implementation of the Env interface.
|
||||
virtual Status NewSequentialFile(const std::string& fname,
|
||||
unique_ptr<SequentialFile>* result,
|
||||
const EnvOptions& soptions) {
|
||||
unique_ptr<SequentialFile>* result) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
*result = NULL;
|
||||
@@ -246,8 +245,7 @@ class InMemoryEnv : public EnvWrapper {
|
||||
}
|
||||
|
||||
virtual Status NewRandomAccessFile(const std::string& fname,
|
||||
unique_ptr<RandomAccessFile>* result,
|
||||
const EnvOptions& soptions) {
|
||||
unique_ptr<RandomAccessFile>* result) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) == file_map_.end()) {
|
||||
*result = NULL;
|
||||
@@ -259,8 +257,7 @@ class InMemoryEnv : public EnvWrapper {
|
||||
}
|
||||
|
||||
virtual Status NewWritableFile(const std::string& fname,
|
||||
unique_ptr<WritableFile>* result,
|
||||
const EnvOptions& soptions) {
|
||||
unique_ptr<WritableFile>* result) {
|
||||
MutexLock lock(&mutex_);
|
||||
if (file_map_.find(fname) != file_map_.end()) {
|
||||
DeleteFileInternal(fname);
|
||||
@@ -383,4 +380,4 @@ Env* NewMemEnv(Env* base_env) {
|
||||
return new InMemoryEnv(base_env);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
@@ -2,9 +2,10 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
|
||||
#define STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
|
||||
namespace rocksdb {
|
||||
#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
|
||||
#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class Env;
|
||||
|
||||
@@ -14,6 +15,6 @@ class Env;
|
||||
// *base_env must remain live while the result is in use.
|
||||
Env* NewMemEnv(Env* base_env);
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
|
||||
#endif // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
|
||||
@@ -5,19 +5,18 @@
|
||||
#include "helpers/memenv/memenv.h"
|
||||
|
||||
#include "db/db_impl.h"
|
||||
#include "rocksdb/db.h"
|
||||
#include "rocksdb/env.h"
|
||||
#include "leveldb/db.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "util/testharness.h"
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class MemEnvTest {
|
||||
public:
|
||||
Env* env_;
|
||||
const EnvOptions soptions_;
|
||||
|
||||
MemEnvTest()
|
||||
: env_(NewMemEnv(Env::Default())) {
|
||||
@@ -41,7 +40,7 @@ TEST(MemEnvTest, Basics) {
|
||||
ASSERT_EQ(0U, children.size());
|
||||
|
||||
// Create a file.
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_));
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
writable_file.reset();
|
||||
|
||||
// Check that the file exists.
|
||||
@@ -53,7 +52,7 @@ TEST(MemEnvTest, Basics) {
|
||||
ASSERT_EQ("f", children[0]);
|
||||
|
||||
// Write to the file.
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_));
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("abc"));
|
||||
writable_file.reset();
|
||||
|
||||
@@ -72,11 +71,9 @@ TEST(MemEnvTest, Basics) {
|
||||
// Check that opening non-existent file fails.
|
||||
unique_ptr<SequentialFile> seq_file;
|
||||
unique_ptr<RandomAccessFile> rand_file;
|
||||
ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file,
|
||||
soptions_).ok());
|
||||
ASSERT_TRUE(!env_->NewSequentialFile("/dir/non_existent", &seq_file).ok());
|
||||
ASSERT_TRUE(!seq_file);
|
||||
ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file,
|
||||
soptions_).ok());
|
||||
ASSERT_TRUE(!env_->NewRandomAccessFile("/dir/non_existent", &rand_file).ok());
|
||||
ASSERT_TRUE(!rand_file);
|
||||
|
||||
// Check that deleting works.
|
||||
@@ -97,13 +94,13 @@ TEST(MemEnvTest, ReadWrite) {
|
||||
|
||||
ASSERT_OK(env_->CreateDir("/dir"));
|
||||
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_));
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("hello "));
|
||||
ASSERT_OK(writable_file->Append("world"));
|
||||
writable_file.reset();
|
||||
|
||||
// Read sequentially.
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file, soptions_));
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
|
||||
ASSERT_OK(seq_file->Read(5, &result, scratch)); // Read "hello".
|
||||
ASSERT_EQ(0, result.compare("hello"));
|
||||
ASSERT_OK(seq_file->Skip(1));
|
||||
@@ -116,7 +113,7 @@ TEST(MemEnvTest, ReadWrite) {
|
||||
ASSERT_EQ(0U, result.size());
|
||||
|
||||
// Random reads.
|
||||
ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file, soptions_));
|
||||
ASSERT_OK(env_->NewRandomAccessFile("/dir/f", &rand_file));
|
||||
ASSERT_OK(rand_file->Read(6, 5, &result, scratch)); // Read "world".
|
||||
ASSERT_EQ(0, result.compare("world"));
|
||||
ASSERT_OK(rand_file->Read(0, 5, &result, scratch)); // Read "hello".
|
||||
@@ -142,7 +139,7 @@ TEST(MemEnvTest, Misc) {
|
||||
ASSERT_TRUE(!test_dir.empty());
|
||||
|
||||
unique_ptr<WritableFile> writable_file;
|
||||
ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file, soptions_));
|
||||
ASSERT_OK(env_->NewWritableFile("/a/b", &writable_file));
|
||||
|
||||
// These are no-ops, but we test they return success.
|
||||
ASSERT_OK(writable_file->Sync());
|
||||
@@ -161,14 +158,14 @@ TEST(MemEnvTest, LargeWrite) {
|
||||
}
|
||||
|
||||
unique_ptr<WritableFile> writable_file;
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file, soptions_));
|
||||
ASSERT_OK(env_->NewWritableFile("/dir/f", &writable_file));
|
||||
ASSERT_OK(writable_file->Append("foo"));
|
||||
ASSERT_OK(writable_file->Append(write_data));
|
||||
writable_file.reset();
|
||||
|
||||
unique_ptr<SequentialFile> seq_file;
|
||||
Slice result;
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file, soptions_));
|
||||
ASSERT_OK(env_->NewSequentialFile("/dir/f", &seq_file));
|
||||
ASSERT_OK(seq_file->Read(3, &result, scratch)); // Read "foo".
|
||||
ASSERT_EQ(0, result.compare("foo"));
|
||||
|
||||
@@ -215,7 +212,7 @@ TEST(MemEnvTest, DBTest) {
|
||||
delete iterator;
|
||||
|
||||
DBImpl* dbi = reinterpret_cast<DBImpl*>(db);
|
||||
ASSERT_OK(dbi->TEST_FlushMemTable());
|
||||
ASSERT_OK(dbi->TEST_CompactMemTable());
|
||||
|
||||
for (size_t i = 0; i < 3; ++i) {
|
||||
std::string res;
|
||||
@@ -226,8 +223,8 @@ TEST(MemEnvTest, DBTest) {
|
||||
delete db;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return rocksdb::test::RunAllTests();
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
||||
@@ -37,8 +37,8 @@
|
||||
(5) All of the pointer arguments must be non-NULL.
|
||||
*/
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_C_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_C_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_C_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_C_H_
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
@@ -278,4 +278,4 @@ extern void leveldb_env_destroy(leveldb_env_t*);
|
||||
} /* end extern "C" */
|
||||
#endif
|
||||
|
||||
#endif /* STORAGE_ROCKSDB_INCLUDE_C_H_ */
|
||||
#endif /* STORAGE_LEVELDB_INCLUDE_C_H_ */
|
||||
@@ -15,35 +15,23 @@
|
||||
// they want something more sophisticated (like scan-resistance, a
|
||||
// custom eviction policy, variable cache sizing, etc.)
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_CACHE_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_CACHE_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_CACHE_H_
|
||||
|
||||
#include <memory>
|
||||
#include <stdint.h>
|
||||
#include "rocksdb/slice.h"
|
||||
#include "leveldb/slice.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
using std::shared_ptr;
|
||||
|
||||
class Cache;
|
||||
|
||||
// Create a new cache with a fixed size capacity. The cache is sharded
|
||||
// to 2^numShardBits shards, by hash of the key. The total capacity
|
||||
// is divided and evenly assigned to each shard. Inside each shard,
|
||||
// the eviction is done in two passes: first try to free spaces by
|
||||
// evicting entries that are among the most least used removeScanCountLimit
|
||||
// entries and do not have reference other than by the cache itself, in
|
||||
// the least-used order. If not enough space is freed, further free the
|
||||
// entries in least used order.
|
||||
//
|
||||
// The functions without parameter numShardBits and/or removeScanCountLimit
|
||||
// use default values. removeScanCountLimit's default value is 0, which
|
||||
// means a strict LRU order inside each shard.
|
||||
// Create a new cache with a fixed size capacity. This implementation
|
||||
// of Cache uses a least-recently-used eviction policy.
|
||||
extern shared_ptr<Cache> NewLRUCache(size_t capacity);
|
||||
extern shared_ptr<Cache> NewLRUCache(size_t capacity, int numShardBits);
|
||||
extern shared_ptr<Cache> NewLRUCache(size_t capacity, int numShardBits,
|
||||
int removeScanCountLimit);
|
||||
|
||||
class Cache {
|
||||
public:
|
||||
@@ -113,6 +101,6 @@ class Cache {
|
||||
void operator=(const Cache&);
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_ROCKSDB_UTIL_CACHE_H_
|
||||
#endif // STORAGE_LEVELDB_UTIL_CACHE_H_
|
||||
@@ -2,18 +2,18 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class Slice;
|
||||
|
||||
// A Comparator object provides a total order across slices that are
|
||||
// used as keys in an sstable or a database. A Comparator implementation
|
||||
// must be thread-safe since rocksdb may invoke its methods concurrently
|
||||
// must be thread-safe since leveldb may invoke its methods concurrently
|
||||
// from multiple threads.
|
||||
class Comparator {
|
||||
public:
|
||||
@@ -33,7 +33,7 @@ class Comparator {
|
||||
// the comparator implementation changes in a way that will cause
|
||||
// the relative ordering of any two keys to change.
|
||||
//
|
||||
// Names starting with "rocksdb." are reserved and should not be used
|
||||
// Names starting with "leveldb." are reserved and should not be used
|
||||
// by any clients of this package.
|
||||
virtual const char* Name() const = 0;
|
||||
|
||||
@@ -58,6 +58,6 @@ class Comparator {
|
||||
// must not be deleted.
|
||||
extern const Comparator* BytewiseComparator();
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
|
||||
@@ -2,25 +2,25 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_DB_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_DB_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_DB_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_DB_H_
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include "rocksdb/iterator.h"
|
||||
#include "rocksdb/options.h"
|
||||
#include "rocksdb/types.h"
|
||||
#include "rocksdb/transaction_log.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/types.h"
|
||||
#include "leveldb/transaction_log_iterator.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
using std::unique_ptr;
|
||||
|
||||
// Update Makefile if you change these
|
||||
static const int kMajorVersion = 2;
|
||||
static const int kMinorVersion = 0;
|
||||
static const int kMajorVersion = 1;
|
||||
static const int kMinorVersion = 5;
|
||||
|
||||
struct Options;
|
||||
struct ReadOptions;
|
||||
@@ -28,17 +28,6 @@ struct WriteOptions;
|
||||
struct FlushOptions;
|
||||
class WriteBatch;
|
||||
|
||||
// Metadata associated with each SST file.
|
||||
struct LiveFileMetaData {
|
||||
std::string name; // Name of the file
|
||||
int level; // Level at which this file resides.
|
||||
size_t size; // File size in bytes.
|
||||
std::string smallestkey; // Smallest user defined key in the file.
|
||||
std::string largestkey; // Largest user defined key in the file.
|
||||
SequenceNumber smallest_seqno; // smallest seqno in file
|
||||
SequenceNumber largest_seqno; // largest seqno in file
|
||||
};
|
||||
|
||||
// Abstract handle to particular state of a DB.
|
||||
// A Snapshot is an immutable object and can therefore be safely
|
||||
// accessed from multiple threads without any external synchronization.
|
||||
@@ -81,8 +70,8 @@ class DB {
|
||||
DB() { }
|
||||
virtual ~DB();
|
||||
|
||||
// Set the database entry for "key" to "value".
|
||||
// Returns OK on success, and a non-OK status on error.
|
||||
// Set the database entry for "key" to "value". Returns OK on success,
|
||||
// and a non-OK status on error.
|
||||
// Note: consider setting options.sync = true.
|
||||
virtual Status Put(const WriteOptions& options,
|
||||
const Slice& key,
|
||||
@@ -94,14 +83,6 @@ class DB {
|
||||
// Note: consider setting options.sync = true.
|
||||
virtual Status Delete(const WriteOptions& options, const Slice& key) = 0;
|
||||
|
||||
// Merge the database entry for "key" with "value". Returns OK on success,
|
||||
// and a non-OK status on error. The semantics of this operation is
|
||||
// determined by the user provided merge_operator when opening DB.
|
||||
// Note: consider setting options.sync = true.
|
||||
virtual Status Merge(const WriteOptions& options,
|
||||
const Slice& key,
|
||||
const Slice& value) = 0;
|
||||
|
||||
// Apply the specified updates to the database.
|
||||
// Returns OK on success, non-OK on failure.
|
||||
// Note: consider setting options.sync = true.
|
||||
@@ -115,39 +96,7 @@ class DB {
|
||||
//
|
||||
// May return some other Status on an error.
|
||||
virtual Status Get(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value) = 0;
|
||||
|
||||
// If keys[i] does not exist in the database, then the i'th returned
|
||||
// status will be one for which Status::IsNotFound() is true, and
|
||||
// (*values)[i] will be set to some arbitrary value (often ""). Otherwise,
|
||||
// the i'th returned status will have Status::ok() true, and (*values)[i]
|
||||
// will store the value associated with keys[i].
|
||||
//
|
||||
// (*values) will always be resized to be the same size as (keys).
|
||||
// Similarly, the number of returned statuses will be the number of keys.
|
||||
// Note: keys will not be "de-duplicated". Duplicate keys will return
|
||||
// duplicate values in order.
|
||||
virtual std::vector<Status> MultiGet(const ReadOptions& options,
|
||||
const std::vector<Slice>& keys,
|
||||
std::vector<std::string>* values) = 0;
|
||||
|
||||
// If the key definitely does not exist in the database, then this method
|
||||
// returns false, else true. If the caller wants to obtain value when the key
|
||||
// is found in memory, a bool for 'value_found' must be passed. 'value_found'
|
||||
// will be true on return if value has been set properly.
|
||||
// This check is potentially lighter-weight than invoking DB::Get(). One way
|
||||
// to make this lighter weight is to avoid doing any IOs.
|
||||
// Default implementation here returns true and sets 'value_found' to false
|
||||
virtual bool KeyMayExist(const ReadOptions& options,
|
||||
const Slice& key,
|
||||
std::string* value,
|
||||
bool* value_found = nullptr) {
|
||||
if (value_found != nullptr) {
|
||||
*value_found = false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
const Slice& key, std::string* value) = 0;
|
||||
|
||||
// Return a heap-allocated iterator over the contents of the database.
|
||||
// The result of NewIterator() is initially invalid (caller must
|
||||
@@ -175,11 +124,11 @@ class DB {
|
||||
//
|
||||
// Valid property names include:
|
||||
//
|
||||
// "rocksdb.num-files-at-level<N>" - return the number of files at level <N>,
|
||||
// "leveldb.num-files-at-level<N>" - return the number of files at level <N>,
|
||||
// where <N> is an ASCII representation of a level number (e.g. "0").
|
||||
// "rocksdb.stats" - returns a multi-line string that describes statistics
|
||||
// "leveldb.stats" - returns a multi-line string that describes statistics
|
||||
// about the internal operation of the DB.
|
||||
// "rocksdb.sstables" - returns a multi-line string that describes all
|
||||
// "leveldb.sstables" - returns a multi-line string that describes all
|
||||
// of the sstables that make up the db contents.
|
||||
virtual bool GetProperty(const Slice& property, std::string* value) = 0;
|
||||
|
||||
@@ -204,15 +153,7 @@ class DB {
|
||||
// end==nullptr is treated as a key after all keys in the database.
|
||||
// Therefore the following call will compact the entire database:
|
||||
// db->CompactRange(nullptr, nullptr);
|
||||
// Note that after the entire database is compacted, all data are pushed
|
||||
// down to the last level containing any data. If the total data size
|
||||
// after compaction is reduced, that level might not be appropriate for
|
||||
// hosting all the files. In this case, client could set reduce_level
|
||||
// to true, to move the files back to the minimum level capable of holding
|
||||
// the data set or a given level (specified by non-negative target_level).
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end,
|
||||
bool reduce_level = false,
|
||||
int target_level = -1) = 0;
|
||||
virtual void CompactRange(const Slice* begin, const Slice* end) = 0;
|
||||
|
||||
// Number of levels used for this DB.
|
||||
virtual int NumberLevels() = 0;
|
||||
@@ -235,53 +176,31 @@ class DB {
|
||||
// Allow compactions to delete obselete files.
|
||||
virtual Status EnableFileDeletions() = 0;
|
||||
|
||||
// GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup
|
||||
|
||||
// THIS METHOD IS DEPRECATED. Use the GetTableMetaData to get more
|
||||
// detailed information on the live files.
|
||||
// Retrieve the list of all files in the database. The files are
|
||||
// relative to the dbname and are not absolute paths. The valid size of the
|
||||
// manifest file is returned in manifest_file_size. The manifest file is an
|
||||
// ever growing file, but only the portion specified by manifest_file_size is
|
||||
// valid for this snapshot.
|
||||
// Setting flush_memtable to true does Flush before recording the live files.
|
||||
// Setting flush_memtable to false is useful when we don't want to wait for
|
||||
// flush which may have to wait for compaction to complete taking an
|
||||
// indeterminate time. But this will have to use GetSortedWalFiles after
|
||||
// GetLiveFiles to compensate for memtables missed in this snapshot due to the
|
||||
// absence of Flush, by WAL files to recover the database consistently later
|
||||
// relative to the dbname and are not absolute paths. This list
|
||||
// can be used to generate a backup. The valid size of the manifest
|
||||
// file is returned in manifest_file_size. The manifest file is
|
||||
// an ever growing file, but only the portion specified
|
||||
// by manifest_file_size is valid for this snapshot.
|
||||
virtual Status GetLiveFiles(std::vector<std::string>&,
|
||||
uint64_t* manifest_file_size,
|
||||
bool flush_memtable = true) = 0;
|
||||
uint64_t* manifest_file_size) = 0;
|
||||
|
||||
// Retrieve the sorted list of all wal files with earliest file first
|
||||
virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0;
|
||||
// The sequence number of the most recent transaction.
|
||||
virtual SequenceNumber GetLatestSequenceNumber() = 0;
|
||||
|
||||
// The sequence number of the most recent transaction.
|
||||
virtual SequenceNumber GetLatestSequenceNumber() const = 0;
|
||||
|
||||
// Sets iter to an iterator that is positioned at a write-batch containing
|
||||
// seq_number. If the sequence number is non existent, it returns an iterator
|
||||
// at the first available seq_no after the requested seq_no
|
||||
// Returns Status::Ok if iterator is valid
|
||||
// Must set WAL_ttl_seconds or WAL_size_limit_MB to large values to
|
||||
// use this api, else the WAL files will get
|
||||
// Return's an iterator for all writes since the sequence number
|
||||
// Status::ok if iterator is valid.
|
||||
// The iterator internally holds references to the available log files.
|
||||
// It automatically takes care of closing a file with no-updates left, and
|
||||
// opening the next one.
|
||||
// If the sequence number is non existent. it returns an iterator at a seq_no
|
||||
// just greater than the requested seq_no.
|
||||
// Must set WAL_ttl_seconds to a large value to use this api.
|
||||
// else the WAL files will get
|
||||
// cleared aggressively and the iterator might keep getting invalid before
|
||||
// an update is read.
|
||||
virtual Status GetUpdatesSince(SequenceNumber seq_number,
|
||||
unique_ptr<TransactionLogIterator>* iter) = 0;
|
||||
|
||||
// Delete the file name from the db directory and update the internal state to
|
||||
// reflect that. Supports deletion of sst and log files only. 'name' must be
|
||||
// path relative to the db directory. eg. 000001.sst, /archive/000003.log
|
||||
virtual Status DeleteFile(std::string name) = 0;
|
||||
|
||||
// Returns a list of all table files with their level, start key
|
||||
// and end key
|
||||
virtual void GetLiveFilesMetaData(
|
||||
std::vector<LiveFileMetaData> *metadata) {
|
||||
}
|
||||
|
||||
private:
|
||||
// No copying allowed
|
||||
DB(const DB&);
|
||||
@@ -298,6 +217,6 @@ Status DestroyDB(const std::string& name, const Options& options);
|
||||
// on a database that contains important information.
|
||||
Status RepairDB(const std::string& dbname, const Options& options);
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_ROCKSDB_INCLUDE_DB_H_
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_DB_H_
|
||||
@@ -2,7 +2,7 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
//
|
||||
// An Env is an interface used by the rocksdb implementation to access
|
||||
// An Env is an interface used by the leveldb implementation to access
|
||||
// operating system functionality like the filesystem etc. Callers
|
||||
// may wish to provide a custom Env object when opening a database to
|
||||
// get fine gain control; e.g., to rate limit file system operations.
|
||||
@@ -10,17 +10,17 @@
|
||||
// All Env implementations are safe for concurrent access from
|
||||
// multiple threads without any external synchronization.
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_ENV_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_ENV_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_ENV_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_ENV_H_
|
||||
|
||||
#include <cstdarg>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <stdint.h>
|
||||
#include "rocksdb/status.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class FileLock;
|
||||
class Logger;
|
||||
@@ -28,42 +28,10 @@ class RandomAccessFile;
|
||||
class SequentialFile;
|
||||
class Slice;
|
||||
class WritableFile;
|
||||
class RandomRWFile;
|
||||
class Options;
|
||||
|
||||
using std::unique_ptr;
|
||||
using std::shared_ptr;
|
||||
|
||||
|
||||
// Options while opening a file to read/write
|
||||
struct EnvOptions {
|
||||
|
||||
// construct with default Options
|
||||
EnvOptions();
|
||||
|
||||
// construct from Options
|
||||
explicit EnvOptions(const Options& options);
|
||||
|
||||
// If true, then allow caching of data in environment buffers
|
||||
bool use_os_buffer;
|
||||
|
||||
// If true, then use mmap to read data
|
||||
bool use_mmap_reads;
|
||||
|
||||
// If true, then use mmap to write data
|
||||
bool use_mmap_writes;
|
||||
|
||||
// If true, set the FD_CLOEXEC on open fd.
|
||||
bool set_fd_cloexec;
|
||||
|
||||
// Allows OS to incrementally sync files to disk while they are being
|
||||
// written, in the background. Issue one request for every bytes_per_sync
|
||||
// written. 0 turns it off.
|
||||
// Default: 0
|
||||
uint64_t bytes_per_sync;
|
||||
|
||||
};
|
||||
|
||||
class Env {
|
||||
public:
|
||||
Env() { }
|
||||
@@ -73,7 +41,7 @@ class Env {
|
||||
// system. Sophisticated users may wish to provide their own Env
|
||||
// implementation instead of relying on this default environment.
|
||||
//
|
||||
// The result of Default() belongs to rocksdb and must never be deleted.
|
||||
// The result of Default() belongs to leveldb and must never be deleted.
|
||||
static Env* Default();
|
||||
|
||||
// Create a brand new sequentially-readable file with the specified name.
|
||||
@@ -83,9 +51,7 @@ class Env {
|
||||
//
|
||||
// The returned file will only be accessed by one thread at a time.
|
||||
virtual Status NewSequentialFile(const std::string& fname,
|
||||
unique_ptr<SequentialFile>* result,
|
||||
const EnvOptions& options)
|
||||
= 0;
|
||||
unique_ptr<SequentialFile>* result) = 0;
|
||||
|
||||
// Create a brand new random access read-only file with the
|
||||
// specified name. On success, stores a pointer to the new file in
|
||||
@@ -95,9 +61,7 @@ class Env {
|
||||
//
|
||||
// The returned file may be concurrently accessed by multiple threads.
|
||||
virtual Status NewRandomAccessFile(const std::string& fname,
|
||||
unique_ptr<RandomAccessFile>* result,
|
||||
const EnvOptions& options)
|
||||
= 0;
|
||||
unique_ptr<RandomAccessFile>* result) = 0;
|
||||
|
||||
// Create an object that writes to a new file with the specified
|
||||
// name. Deletes any existing file with the same name and creates a
|
||||
@@ -107,17 +71,7 @@ class Env {
|
||||
//
|
||||
// The returned file will only be accessed by one thread at a time.
|
||||
virtual Status NewWritableFile(const std::string& fname,
|
||||
unique_ptr<WritableFile>* result,
|
||||
const EnvOptions& options) = 0;
|
||||
|
||||
// Create an object that both reads and writes to a file on
|
||||
// specified offsets (random access). If file already exists,
|
||||
// does not overwrite it. On success, stores a pointer to the
|
||||
// new file in *result and returns OK. On failure stores nullptr
|
||||
// in *result and returns non-OK.
|
||||
virtual Status NewRandomRWFile(const std::string& fname,
|
||||
unique_ptr<RandomRWFile>* result,
|
||||
const EnvOptions& options) = 0;
|
||||
unique_ptr<WritableFile>* result) = 0;
|
||||
|
||||
// Returns true iff the named file exists.
|
||||
virtual bool FileExists(const std::string& fname) = 0;
|
||||
@@ -172,20 +126,15 @@ class Env {
|
||||
// REQUIRES: lock has not already been unlocked.
|
||||
virtual Status UnlockFile(FileLock* lock) = 0;
|
||||
|
||||
enum Priority { LOW, HIGH, TOTAL };
|
||||
|
||||
// Arrange to run "(*function)(arg)" once in a background thread, in
|
||||
// the thread pool specified by pri. By default, jobs go to the 'LOW'
|
||||
// priority thread pool.
|
||||
|
||||
// Arrange to run "(*function)(arg)" once in a background thread.
|
||||
//
|
||||
// "function" may run in an unspecified thread. Multiple functions
|
||||
// added to the same Env may run concurrently in different threads.
|
||||
// I.e., the caller may not assume that background work items are
|
||||
// serialized.
|
||||
virtual void Schedule(
|
||||
void (*function)(void* arg),
|
||||
void* arg,
|
||||
Priority pri = LOW) = 0;
|
||||
void* arg) = 0;
|
||||
|
||||
// Start a new thread, invoking "function(arg)" within the new thread.
|
||||
// When "function(arg)" returns, the thread will be destroyed.
|
||||
@@ -205,13 +154,6 @@ class Env {
|
||||
// useful for computing deltas of time.
|
||||
virtual uint64_t NowMicros() = 0;
|
||||
|
||||
// Returns the number of nano-seconds since some fixed point in time. Only
|
||||
// useful for computing deltas of time in one run.
|
||||
// Default implementation simply relies on NowMicros
|
||||
virtual uint64_t NowNanos() {
|
||||
return NowMicros() * 1000;
|
||||
}
|
||||
|
||||
// Sleep/delay the thread for the perscribed number of micro-seconds.
|
||||
virtual void SleepForMicroseconds(int micros) = 0;
|
||||
|
||||
@@ -225,17 +167,13 @@ class Env {
|
||||
virtual Status GetAbsolutePath(const std::string& db_path,
|
||||
std::string* output_path) = 0;
|
||||
|
||||
// The number of background worker threads of a specific thread pool
|
||||
// for this environment. 'LOW' is the default pool.
|
||||
// default number: 1
|
||||
virtual void SetBackgroundThreads(int number, Priority pri = LOW) = 0;
|
||||
// The number of background worker threads for this environment.
|
||||
// default: 1
|
||||
virtual void SetBackgroundThreads(int number) = 0;
|
||||
|
||||
// Converts seconds-since-Jan-01-1970 to a printable string
|
||||
virtual std::string TimeToString(uint64_t time) = 0;
|
||||
|
||||
// Generates a unique id that can be used to identify a db
|
||||
virtual std::string GenerateUniqueId();
|
||||
|
||||
private:
|
||||
// No copying allowed
|
||||
Env(const Env&);
|
||||
@@ -266,13 +204,6 @@ class SequentialFile {
|
||||
//
|
||||
// REQUIRES: External synchronization
|
||||
virtual Status Skip(uint64_t n) = 0;
|
||||
|
||||
// Remove any kind of caching of data from the offset to offset+length
|
||||
// of this file. If the length is 0, then it refers to the end of file.
|
||||
// If the system is not caching the file contents, then this is a noop.
|
||||
virtual Status InvalidateCache(size_t offset, size_t length) {
|
||||
return Status::NotSupported("InvalidateCache not supported.");
|
||||
}
|
||||
};
|
||||
|
||||
// A file abstraction for randomly reading the contents of a file.
|
||||
@@ -297,7 +228,7 @@ class RandomAccessFile {
|
||||
// the file is opened (and will stay the same while the file is open).
|
||||
// Furthermore, it tries to make this ID at most "max_size" bytes. If such an
|
||||
// ID can be created this function returns the length of the ID and places it
|
||||
// in "id"; otherwise, this function returns 0, in which case "id"
|
||||
// in "id"; otherwise, this function returns 0, in which case "id" may more
|
||||
// may not have been modified.
|
||||
//
|
||||
// This function guarantees, for IDs from a given environment, two unique ids
|
||||
@@ -312,18 +243,6 @@ class RandomAccessFile {
|
||||
return 0; // Default implementation to prevent issues with backwards
|
||||
// compatibility.
|
||||
};
|
||||
|
||||
|
||||
enum AccessPattern { NORMAL, RANDOM, SEQUENTIAL, WILLNEED, DONTNEED };
|
||||
|
||||
virtual void Hint(AccessPattern pattern) {}
|
||||
|
||||
// Remove any kind of caching of data from the offset to offset+length
|
||||
// of this file. If the length is 0, then it refers to the end of file.
|
||||
// If the system is not caching the file contents, then this is a noop.
|
||||
virtual Status InvalidateCache(size_t offset, size_t length) {
|
||||
return Status::NotSupported("InvalidateCache not supported.");
|
||||
}
|
||||
};
|
||||
|
||||
// A file abstraction for sequential writing. The implementation
|
||||
@@ -342,7 +261,7 @@ class WritableFile {
|
||||
|
||||
/*
|
||||
* Sync data and/or metadata as well.
|
||||
* By default, sync only data.
|
||||
* By default, sync only metadata.
|
||||
* Override this method for environments where we need to sync
|
||||
* metadata as well.
|
||||
*/
|
||||
@@ -373,19 +292,6 @@ class WritableFile {
|
||||
*block_size = preallocation_block_size_;
|
||||
}
|
||||
|
||||
// For documentation, refer to RandomAccessFile::GetUniqueId()
|
||||
virtual size_t GetUniqueId(char* id, size_t max_size) const {
|
||||
return 0; // Default implementation to prevent issues with backwards
|
||||
}
|
||||
|
||||
// Remove any kind of caching of data from the offset to offset+length
|
||||
// of this file. If the length is 0, then it refers to the end of file.
|
||||
// If the system is not caching the file contents, then this is a noop.
|
||||
// This call has no effect on dirty pages in the cache.
|
||||
virtual Status InvalidateCache(size_t offset, size_t length) {
|
||||
return Status::NotSupported("InvalidateCache not supported.");
|
||||
}
|
||||
|
||||
protected:
|
||||
// PrepareWrite performs any necessary preparation for a write
|
||||
// before the write actually occurs. This allows for pre-allocation
|
||||
@@ -418,16 +324,6 @@ class WritableFile {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Sync a file range with disk.
|
||||
// offset is the starting byte of the file range to be synchronized.
|
||||
// nbytes specifies the length of the range to be synchronized.
|
||||
// This asks the OS to initiate flushing the cached data to disk,
|
||||
// without waiting for completion.
|
||||
// Default implementation does nothing.
|
||||
virtual Status RangeSync(off64_t offset, off64_t nbytes) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
size_t last_preallocated_block_;
|
||||
size_t preallocation_block_size_;
|
||||
@@ -436,55 +332,6 @@ class WritableFile {
|
||||
void operator=(const WritableFile&);
|
||||
};
|
||||
|
||||
// A file abstraction for random reading and writing.
|
||||
class RandomRWFile {
|
||||
public:
|
||||
RandomRWFile() {}
|
||||
virtual ~RandomRWFile() {}
|
||||
|
||||
// Write data from Slice data to file starting from offset
|
||||
// Returns IOError on failure, but does not guarantee
|
||||
// atomicity of a write. Returns OK status on success.
|
||||
//
|
||||
// Safe for concurrent use.
|
||||
virtual Status Write(uint64_t offset, const Slice& data) = 0;
|
||||
// Read up to "n" bytes from the file starting at "offset".
|
||||
// "scratch[0..n-1]" may be written by this routine. Sets "*result"
|
||||
// to the data that was read (including if fewer than "n" bytes were
|
||||
// successfully read). May set "*result" to point at data in
|
||||
// "scratch[0..n-1]", so "scratch[0..n-1]" must be live when
|
||||
// "*result" is used. If an error was encountered, returns a non-OK
|
||||
// status.
|
||||
//
|
||||
// Safe for concurrent use by multiple threads.
|
||||
virtual Status Read(uint64_t offset, size_t n, Slice* result,
|
||||
char* scratch) const = 0;
|
||||
virtual Status Close() = 0; // closes the file
|
||||
virtual Status Sync() = 0; // sync data
|
||||
|
||||
/*
|
||||
* Sync data and/or metadata as well.
|
||||
* By default, sync only data.
|
||||
* Override this method for environments where we need to sync
|
||||
* metadata as well.
|
||||
*/
|
||||
virtual Status Fsync() {
|
||||
return Sync();
|
||||
}
|
||||
|
||||
/*
|
||||
* Pre-allocate space for a file.
|
||||
*/
|
||||
virtual Status Allocate(off_t offset, off_t len) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
// No copying allowed
|
||||
RandomRWFile(const RandomRWFile&);
|
||||
void operator=(const RandomRWFile&);
|
||||
};
|
||||
|
||||
// An interface for writing log messages.
|
||||
class Logger {
|
||||
public:
|
||||
@@ -497,8 +344,6 @@ class Logger {
|
||||
virtual size_t GetLogFileSize() const {
|
||||
return DO_NOT_SUPPORT_GET_LOG_FILE_SIZE;
|
||||
}
|
||||
// Flush to the OS buffers
|
||||
virtual void Flush() {}
|
||||
|
||||
private:
|
||||
// No copying allowed
|
||||
@@ -518,9 +363,6 @@ class FileLock {
|
||||
void operator=(const FileLock&);
|
||||
};
|
||||
|
||||
|
||||
extern void LogFlush(const shared_ptr<Logger>& info_log);
|
||||
|
||||
// Log the specified data to *info_log if info_log is non-nullptr.
|
||||
extern void Log(const shared_ptr<Logger>& info_log, const char* format, ...)
|
||||
# if defined(__GNUC__) || defined(__clang__)
|
||||
@@ -528,8 +370,6 @@ extern void Log(const shared_ptr<Logger>& info_log, const char* format, ...)
|
||||
# endif
|
||||
;
|
||||
|
||||
extern void LogFlush(Logger *info_log);
|
||||
|
||||
extern void Log(Logger* info_log, const char* format, ...)
|
||||
# if defined(__GNUC__) || defined(__clang__)
|
||||
__attribute__((__format__ (__printf__, 2, 3)))
|
||||
@@ -558,22 +398,15 @@ class EnvWrapper : public Env {
|
||||
|
||||
// The following text is boilerplate that forwards all methods to target()
|
||||
Status NewSequentialFile(const std::string& f,
|
||||
unique_ptr<SequentialFile>* r,
|
||||
const EnvOptions& options) {
|
||||
return target_->NewSequentialFile(f, r, options);
|
||||
unique_ptr<SequentialFile>* r) {
|
||||
return target_->NewSequentialFile(f, r);
|
||||
}
|
||||
Status NewRandomAccessFile(const std::string& f,
|
||||
unique_ptr<RandomAccessFile>* r,
|
||||
const EnvOptions& options) {
|
||||
return target_->NewRandomAccessFile(f, r, options);
|
||||
unique_ptr<RandomAccessFile>* r) {
|
||||
return target_->NewRandomAccessFile(f, r);
|
||||
}
|
||||
Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
|
||||
const EnvOptions& options) {
|
||||
return target_->NewWritableFile(f, r, options);
|
||||
}
|
||||
Status NewRandomRWFile(const std::string& f, unique_ptr<RandomRWFile>* r,
|
||||
const EnvOptions& options) {
|
||||
return target_->NewRandomRWFile(f, r, options);
|
||||
Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r) {
|
||||
return target_->NewWritableFile(f, r);
|
||||
}
|
||||
bool FileExists(const std::string& f) { return target_->FileExists(f); }
|
||||
Status GetChildren(const std::string& dir, std::vector<std::string>* r) {
|
||||
@@ -601,8 +434,8 @@ class EnvWrapper : public Env {
|
||||
return target_->LockFile(f, l);
|
||||
}
|
||||
Status UnlockFile(FileLock* l) { return target_->UnlockFile(l); }
|
||||
void Schedule(void (*f)(void*), void* a, Priority pri) {
|
||||
return target_->Schedule(f, a, pri);
|
||||
void Schedule(void (*f)(void*), void* a) {
|
||||
return target_->Schedule(f, a);
|
||||
}
|
||||
void StartThread(void (*f)(void*), void* a) {
|
||||
return target_->StartThread(f, a);
|
||||
@@ -630,8 +463,8 @@ class EnvWrapper : public Env {
|
||||
std::string* output_path) {
|
||||
return target_->GetAbsolutePath(db_path, output_path);
|
||||
}
|
||||
void SetBackgroundThreads(int num, Priority pri) {
|
||||
return target_->SetBackgroundThreads(num, pri);
|
||||
void SetBackgroundThreads(int num) {
|
||||
return target_->SetBackgroundThreads(num);
|
||||
}
|
||||
std::string TimeToString(uint64_t time) {
|
||||
return target_->TimeToString(time);
|
||||
@@ -641,6 +474,6 @@ class EnvWrapper : public Env {
|
||||
Env* target_;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_ROCKSDB_INCLUDE_ENV_H_
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_ENV_H_
|
||||
@@ -4,8 +4,8 @@
|
||||
//
|
||||
// A database can be configured with a custom FilterPolicy object.
|
||||
// This object is responsible for creating a small filter from a set
|
||||
// of keys. These filters are stored in rocksdb and are consulted
|
||||
// automatically by rocksdb to decide whether or not to read some
|
||||
// of keys. These filters are stored in leveldb and are consulted
|
||||
// automatically by leveldb to decide whether or not to read some
|
||||
// information from disk. In many cases, a filter can cut down the
|
||||
// number of disk seeks form a handful to a single disk seek per
|
||||
// DB::Get() call.
|
||||
@@ -13,12 +13,12 @@
|
||||
// Most people will want to use the builtin bloom filter support (see
|
||||
// NewBloomFilterPolicy() below).
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class Slice;
|
||||
|
||||
@@ -67,4 +67,4 @@ extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key);
|
||||
|
||||
}
|
||||
|
||||
#endif // STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
|
||||
@@ -12,13 +12,13 @@
|
||||
// non-const method, all threads accessing the same Iterator must use
|
||||
// external synchronization.
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
|
||||
|
||||
#include "rocksdb/slice.h"
|
||||
#include "rocksdb/status.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class Iterator {
|
||||
public:
|
||||
@@ -65,8 +65,6 @@ class Iterator {
|
||||
virtual Slice value() const = 0;
|
||||
|
||||
// If an error has occurred, return it. Else return an ok status.
|
||||
// If non-blocking IO is requested and this operation cannot be
|
||||
// satisfied without doing some IO, then this returns Status::Incomplete().
|
||||
virtual Status status() const = 0;
|
||||
|
||||
// Clients are allowed to register function/arg1/arg2 triples that
|
||||
@@ -97,6 +95,6 @@ extern Iterator* NewEmptyIterator();
|
||||
// Return an empty iterator with the specified status.
|
||||
extern Iterator* NewErrorIterator(const Status& status);
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
|
||||
@@ -2,36 +2,25 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
|
||||
|
||||
#include <stddef.h>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <stdint.h>
|
||||
#include "leveldb/slice.h"
|
||||
|
||||
#include "rocksdb/memtablerep.h"
|
||||
#include "rocksdb/slice.h"
|
||||
#include "rocksdb/slice_transform.h"
|
||||
#include "rocksdb/statistics.h"
|
||||
#include "rocksdb/table_stats.h"
|
||||
#include "rocksdb/universal_compaction.h"
|
||||
#include "rocksdb/memtablerep.h"
|
||||
#include "rocksdb/slice_transform.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class Cache;
|
||||
class Comparator;
|
||||
class Env;
|
||||
class FilterPolicy;
|
||||
class Logger;
|
||||
class MergeOperator;
|
||||
class Snapshot;
|
||||
class CompactionFilter;
|
||||
class CompactionFilterFactory;
|
||||
class TableFactory;
|
||||
class Statistics;
|
||||
|
||||
using std::shared_ptr;
|
||||
|
||||
@@ -39,7 +28,7 @@ using std::shared_ptr;
|
||||
// sequence of key,value pairs. Each block may be compressed before
|
||||
// being stored in a file. The following enum describes which
|
||||
// compression method (if any) is used to compress a block.
|
||||
enum CompressionType : char {
|
||||
enum CompressionType {
|
||||
// NOTE: do not change the values of existing entries, as these are
|
||||
// part of the persistent format on disk.
|
||||
kNoCompression = 0x0,
|
||||
@@ -48,11 +37,6 @@ enum CompressionType : char {
|
||||
kBZip2Compression = 0x3
|
||||
};
|
||||
|
||||
enum CompactionStyle : char {
|
||||
kCompactionStyleLevel = 0x0, // level based compaction style
|
||||
kCompactionStyleUniversal = 0x1 // Universal compaction style
|
||||
};
|
||||
|
||||
// Compression options for different compression algorithms like Zlib
|
||||
struct CompressionOptions {
|
||||
int window_bits;
|
||||
@@ -79,28 +63,6 @@ struct Options {
|
||||
// comparator provided to previous open calls on the same DB.
|
||||
const Comparator* comparator;
|
||||
|
||||
// REQUIRES: The client must provide a merge operator if Merge operation
|
||||
// needs to be accessed. Calling Merge on a DB without a merge operator
|
||||
// would result in Status::NotSupported. The client must ensure that the
|
||||
// merge operator supplied here has the same name and *exactly* the same
|
||||
// semantics as the merge operator provided to previous open calls on
|
||||
// the same DB. The only exception is reserved for upgrade, where a DB
|
||||
// previously without a merge operator is introduced to Merge operation
|
||||
// for the first time. It's necessary to specify a merge operator when
|
||||
// openning the DB in this case.
|
||||
// Default: nullptr
|
||||
shared_ptr<MergeOperator> merge_operator;
|
||||
|
||||
// The client must provide compaction_filter_factory if it requires a new
|
||||
// compaction filter to be used for different compaction processes
|
||||
// Allows an application to modify/delete a key-value during background
|
||||
// compaction.
|
||||
// Ideally, client should specify only one of filter or factory.
|
||||
// compaction_filter takes precedence over compaction_filter_factory if
|
||||
// client specifies both.
|
||||
// Default: nullptr
|
||||
const CompactionFilter* compaction_filter;
|
||||
|
||||
// If true, the database will be created if it is missing.
|
||||
// Default: false
|
||||
bool create_if_missing;
|
||||
@@ -114,9 +76,6 @@ struct Options {
|
||||
// errors. This may have unforeseen ramifications: for example, a
|
||||
// corruption of one DB entry may cause a large number of entries to
|
||||
// become unreadable or for the entire DB to become unopenable.
|
||||
// If any of the writes to the database fails (Put, Delete, Merge, Write),
|
||||
// the database will switch to read-only mode and fail all other
|
||||
// Write operations.
|
||||
// Default: false
|
||||
bool paranoid_checks;
|
||||
|
||||
@@ -153,15 +112,6 @@ struct Options {
|
||||
// Default: 2
|
||||
int max_write_buffer_number;
|
||||
|
||||
// The minimum number of write buffers that will be merged together
|
||||
// before writing to storage. If set to 1, then
|
||||
// all write buffers are fushed to L0 as individual files and this increases
|
||||
// read amplification because a get request has to check in all of these
|
||||
// files. Also, an in-memory merge may result in writing lesser
|
||||
// data to storage if there are duplicate records in each of these
|
||||
// individual write buffers. Default: 1
|
||||
int min_write_buffer_number_to_merge;
|
||||
|
||||
// Number of open files that can be used by the DB. You may need to
|
||||
// increase this if your database has a large working set (budget
|
||||
// one open file per 2MB of working set).
|
||||
@@ -173,15 +123,10 @@ struct Options {
|
||||
// a block is the unit of reading from disk).
|
||||
|
||||
// If non-NULL use the specified cache for blocks.
|
||||
// If NULL, rocksdb will automatically create and use an 8MB internal cache.
|
||||
// If NULL, leveldb will automatically create and use an 8MB internal cache.
|
||||
// Default: nullptr
|
||||
shared_ptr<Cache> block_cache;
|
||||
|
||||
// If non-NULL use the specified cache for compressed blocks.
|
||||
// If NULL, rocksdb will not use a compressed block cache.
|
||||
// Default: nullptr
|
||||
shared_ptr<Cache> block_cache_compressed;
|
||||
|
||||
// Approximate size of user data packed per block. Note that the
|
||||
// block size specified here corresponds to uncompressed data. The
|
||||
// actual size of the unit read from disk may be smaller if
|
||||
@@ -238,28 +183,6 @@ struct Options {
|
||||
// Default: nullptr
|
||||
const FilterPolicy* filter_policy;
|
||||
|
||||
// If non-nullptr, use the specified function to determine the
|
||||
// prefixes for keys. These prefixes will be placed in the filter.
|
||||
// Depending on the workload, this can reduce the number of read-IOP
|
||||
// cost for scans when a prefix is passed via ReadOptions to
|
||||
// db.NewIterator(). For prefix filtering to work properly,
|
||||
// "prefix_extractor" and "comparator" must be such that the following
|
||||
// properties hold:
|
||||
//
|
||||
// 1) key.starts_with(prefix(key))
|
||||
// 2) Compare(prefix(key), key) <= 0.
|
||||
// 3) If Compare(k1, k2) <= 0, then Compare(prefix(k1), prefix(k2)) <= 0
|
||||
// 4) prefix(prefix(key)) == prefix(key)
|
||||
//
|
||||
// Default: nullptr
|
||||
const SliceTransform* prefix_extractor;
|
||||
|
||||
// If true, place whole keys in the filter (not just prefixes).
|
||||
// This must generally be true for gets to be efficient.
|
||||
//
|
||||
// Default: true
|
||||
bool whole_key_filtering;
|
||||
|
||||
// Number of levels for this database
|
||||
int num_levels;
|
||||
|
||||
@@ -267,9 +190,9 @@ struct Options {
|
||||
// level-0 compaction will not be triggered by number of files at all.
|
||||
int level0_file_num_compaction_trigger;
|
||||
|
||||
// Soft limit on number of level-0 files. We start slowing down writes at this
|
||||
// point. A value <0 means that no writing slow down will be triggered by
|
||||
// number of files in level-0.
|
||||
// Soft limit on number of level-0 files. We slow down writes at this point.
|
||||
// A value <0 means that no writing slow down will be triggered by number
|
||||
// of files in level-0.
|
||||
int level0_slowdown_writes_trigger;
|
||||
|
||||
// Maximum number of level-0 files. We stop writes at this point.
|
||||
@@ -313,12 +236,6 @@ struct Options {
|
||||
// by default 'max_bytes_for_level_base' is 10.
|
||||
int max_bytes_for_level_multiplier;
|
||||
|
||||
// Different max-size multipliers for different levels.
|
||||
// These are multiplied by max_bytes_for_level_multiplier to arrive
|
||||
// at the max-size of each level.
|
||||
// Default: 1
|
||||
std::vector<int> max_bytes_for_level_multiplier_additional;
|
||||
|
||||
// Maximum number of bytes in all compacted files. We avoid expanding
|
||||
// the lower level file set of a compaction if it would make the
|
||||
// total compaction cover more than
|
||||
@@ -341,7 +258,7 @@ struct Options {
|
||||
// If non-null, then we should collect metrics about database operations
|
||||
// Statistics objects should not be shared between DB instances as
|
||||
// it does not use any locks to prevent concurrent updates.
|
||||
shared_ptr<Statistics> statistics;
|
||||
Statistics* statistics;
|
||||
|
||||
// If true, then the contents of data files are not synced
|
||||
// to stable storage. Their contents remain in the OS buffers till the
|
||||
@@ -364,21 +281,13 @@ struct Options {
|
||||
// Default value is 1800 (half an hour).
|
||||
int db_stats_log_interval;
|
||||
|
||||
// This specifies the info LOG dir.
|
||||
// This specifies the log dir.
|
||||
// If it is empty, the log files will be in the same dir as data.
|
||||
// If it is non empty, the log files will be in the specified dir,
|
||||
// and the db data dir's absolute path will be used as the log file
|
||||
// name's prefix.
|
||||
std::string db_log_dir;
|
||||
|
||||
// This specifies the absolute dir path for write-ahead logs (WAL).
|
||||
// If it is empty, the log files will be in the same dir as data,
|
||||
// dbname is used as the data dir by default
|
||||
// If it is non empty, the log files will be in kept the specified dir.
|
||||
// When destroying the db,
|
||||
// all log files in wal_dir and the dir itself is deleted
|
||||
std::string wal_dir;
|
||||
|
||||
// Disable compaction triggered by seek.
|
||||
// With bloomfilter and fast storage, a miss on one level
|
||||
// is very cheap if the file handle is cached in table cache
|
||||
@@ -390,23 +299,10 @@ struct Options {
|
||||
// every compaction run.
|
||||
uint64_t delete_obsolete_files_period_micros;
|
||||
|
||||
// Maximum number of concurrent background jobs, submitted to
|
||||
// the default LOW priority thread pool
|
||||
// Maximum number of concurrent background compactions.
|
||||
// Default: 1
|
||||
int max_background_compactions;
|
||||
|
||||
// Maximum number of concurrent background memtable flush jobs, submitted to
|
||||
// the HIGH priority thread pool.
|
||||
// By default, all background jobs (major compaction and memtable flush) go
|
||||
// to the LOW priority pool. If this option is set to a positive number,
|
||||
// memtable flush jobs will be submitted to the HIGH priority pool.
|
||||
// It is important when the same Env is shared by multiple db instances.
|
||||
// Without a separate pool, long running major compaction jobs could
|
||||
// potentially block memtable flush jobs of other db instances, leading to
|
||||
// unnecessary Put stalls.
|
||||
// Default: 0
|
||||
int max_background_flushes;
|
||||
|
||||
// Specify the maximal size of the info log file. If the log file
|
||||
// is larger than `max_log_file_size`, a new info log file will
|
||||
// be created.
|
||||
@@ -424,53 +320,26 @@ struct Options {
|
||||
// Default: 1000
|
||||
size_t keep_log_file_num;
|
||||
|
||||
// Puts are delayed 0-1 ms when any level has a compaction score that exceeds
|
||||
// soft_rate_limit. This is ignored when == 0.0.
|
||||
// CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not
|
||||
// hold, RocksDB will set soft_rate_limit = hard_rate_limit
|
||||
// Default: 0 (disabled)
|
||||
double soft_rate_limit;
|
||||
// Puts are delayed when any level has a compaction score that
|
||||
// exceeds rate_limit. This is ignored when <= 1.0.
|
||||
double rate_limit;
|
||||
|
||||
// Puts are delayed 1ms at a time when any level has a compaction score that
|
||||
// exceeds hard_rate_limit. This is ignored when <= 1.0.
|
||||
// Default: 0 (disabled)
|
||||
double hard_rate_limit;
|
||||
|
||||
// Max time a put will be stalled when hard_rate_limit is enforced. If 0, then
|
||||
// there is no limit.
|
||||
// Default: 1000
|
||||
unsigned int rate_limit_delay_max_milliseconds;
|
||||
// Max time a put will be stalled when rate_limit is enforced
|
||||
int rate_limit_delay_milliseconds;
|
||||
|
||||
// manifest file is rolled over on reaching this limit.
|
||||
// The older manifest file be deleted.
|
||||
// The default value is MAX_INT so that roll-over does not take place.
|
||||
uint64_t max_manifest_file_size;
|
||||
|
||||
// Disable block cache. If this is set to true,
|
||||
// Disable block cache. If this is set to false,
|
||||
// then no block cache should be used, and the block_cache should
|
||||
// point to a nullptr object.
|
||||
// Default: false
|
||||
bool no_block_cache;
|
||||
|
||||
// Number of shards used for table cache.
|
||||
int table_cache_numshardbits;
|
||||
|
||||
// During data eviction of table's LRU cache, it would be inefficient
|
||||
// to strictly follow LRU because this piece of memory will not really
|
||||
// be released unless its refcount falls to zero. Instead, make two
|
||||
// passes: the first pass will release items with refcount = 1,
|
||||
// and if not enough space releases after scanning the number of
|
||||
// elements specified by this parameter, we will remove items in LRU
|
||||
// order.
|
||||
int table_cache_remove_scan_count_limit;
|
||||
|
||||
// size of one block in arena memory allocation.
|
||||
// If <= 0, a proper value is automatically calculated (usually 1/10 of
|
||||
// writer_buffer_size).
|
||||
//
|
||||
// Default: 0
|
||||
size_t arena_block_size;
|
||||
|
||||
// Create an Options object with default values for all fields.
|
||||
Options();
|
||||
|
||||
@@ -485,24 +354,38 @@ struct Options {
|
||||
// from the database, because otherwise the read can be very slow.
|
||||
Options* PrepareForBulkLoad();
|
||||
|
||||
// This method allows an application to modify/delete a key-value at
|
||||
// the time of compaction. The compaction process invokes this
|
||||
// method for every kv that is being compacted. A return value
|
||||
// of false indicates that the kv should be preserved in the
|
||||
// output of this compaction run and a return value of true
|
||||
// indicates that this key-value should be removed from the
|
||||
// output of the compaction. The application can inspect
|
||||
// the existing value of the key, modify it if needed and
|
||||
// return back the new value for this key. The application
|
||||
// should allocate memory for the Slice object that is used to
|
||||
// return the new value and the leveldb framework will
|
||||
// free up that memory.
|
||||
// The compaction_filter_args, if specified here, are passed
|
||||
// back to the invocation of the CompactionFilter.
|
||||
void* compaction_filter_args;
|
||||
bool (*CompactionFilter)(void* compaction_filter_args,
|
||||
int level, const Slice& key,
|
||||
const Slice& existing_value, Slice** new_value);
|
||||
|
||||
// Disable automatic compactions. Manual compactions can still
|
||||
// be issued on this database.
|
||||
bool disable_auto_compactions;
|
||||
|
||||
// The following two fields affect how archived logs will be deleted.
|
||||
// 1. If both set to 0, logs will be deleted asap and will not get into
|
||||
// the archive.
|
||||
// 2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0,
|
||||
// WAL files will be checked every 10 min and if total size is greater
|
||||
// then WAL_size_limit_MB, they will be deleted starting with the
|
||||
// earliest until size_limit is met. All empty files will be deleted.
|
||||
// 3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then
|
||||
// WAL files will be checked every WAL_ttl_secondsi / 2 and those which
|
||||
// are older than WAL_ttl_seconds will be deleted.
|
||||
// 4. If both are not 0, WAL files will be checked every 10 min and both
|
||||
// checks will be performed with ttl being first.
|
||||
// The number of seconds a WAL(write ahead log) should be kept after it has
|
||||
// been marked as Not Live. If the value is set. The WAL files are moved to
|
||||
// the archive direcotory and deleted after the given TTL.
|
||||
// If set to 0, WAL files are deleted as soon as they are not required by
|
||||
// the database.
|
||||
// If set to std::numeric_limits<uint64_t>::max() the WAL files will never be
|
||||
// deleted.
|
||||
// Default : 0
|
||||
uint64_t WAL_ttl_seconds;
|
||||
uint64_t WAL_size_limit_MB;
|
||||
|
||||
// Number of bytes to preallocate (via fallocate) the manifest
|
||||
// files. Default is 4mb, which is reasonable to reduce random IO
|
||||
@@ -513,128 +396,6 @@ struct Options {
|
||||
// Purge duplicate/deleted keys when a memtable is flushed to storage.
|
||||
// Default: true
|
||||
bool purge_redundant_kvs_while_flush;
|
||||
|
||||
// Data being read from file storage may be buffered in the OS
|
||||
// Default: true
|
||||
bool allow_os_buffer;
|
||||
|
||||
// Allow the OS to mmap file for reading sst tables. Default: false
|
||||
bool allow_mmap_reads;
|
||||
|
||||
// Allow the OS to mmap file for writing. Default: true
|
||||
bool allow_mmap_writes;
|
||||
|
||||
// Disable child process inherit open files. Default: true
|
||||
bool is_fd_close_on_exec;
|
||||
|
||||
// Skip log corruption error on recovery (If client is ok with
|
||||
// losing most recent changes)
|
||||
// Default: false
|
||||
bool skip_log_error_on_recovery;
|
||||
|
||||
// if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec
|
||||
// Default: 3600 (1 hour)
|
||||
unsigned int stats_dump_period_sec;
|
||||
|
||||
// This is used to close a block before it reaches the configured
|
||||
// 'block_size'. If the percentage of free space in the current block is less
|
||||
// than this specified number and adding a new record to the block will
|
||||
// exceed the configured block size, then this block will be closed and the
|
||||
// new record will be written to the next block.
|
||||
// Default is 10.
|
||||
int block_size_deviation;
|
||||
|
||||
// If set true, will hint the underlying file system that the file
|
||||
// access pattern is random, when a sst file is opened.
|
||||
// Default: true
|
||||
bool advise_random_on_open;
|
||||
|
||||
// Specify the file access pattern once a compaction is started.
|
||||
// It will be applied to all input files of a compaction.
|
||||
// Default: NORMAL
|
||||
enum { NONE, NORMAL, SEQUENTIAL, WILLNEED } access_hint_on_compaction_start;
|
||||
|
||||
// Use adaptive mutex, which spins in the user space before resorting
|
||||
// to kernel. This could reduce context switch when the mutex is not
|
||||
// heavily contended. However, if the mutex is hot, we could end up
|
||||
// wasting spin time.
|
||||
// Default: false
|
||||
bool use_adaptive_mutex;
|
||||
|
||||
// Allows OS to incrementally sync files to disk while they are being
|
||||
// written, asynchronously, in the background.
|
||||
// Issue one request for every bytes_per_sync written. 0 turns it off.
|
||||
// Default: 0
|
||||
uint64_t bytes_per_sync;
|
||||
|
||||
// The compaction style. Default: kCompactionStyleLevel
|
||||
CompactionStyle compaction_style;
|
||||
|
||||
// The options needed to support Universal Style compactions
|
||||
CompactionOptionsUniversal compaction_options_universal;
|
||||
|
||||
// Use KeyMayExist API to filter deletes when this is true.
|
||||
// If KeyMayExist returns false, i.e. the key definitely does not exist, then
|
||||
// the delete is a noop. KeyMayExist only incurs in-memory look up.
|
||||
// This optimization avoids writing the delete to storage when appropriate.
|
||||
// Default: false
|
||||
bool filter_deletes;
|
||||
|
||||
// An iteration->Next() sequentially skips over keys with the same
|
||||
// user-key unless this option is set. This number specifies the number
|
||||
// of keys (with the same userkey) that will be sequentially
|
||||
// skipped before a reseek is issued.
|
||||
// Default: 8
|
||||
uint64_t max_sequential_skip_in_iterations;
|
||||
|
||||
// This is a factory that provides MemTableRep objects.
|
||||
// Default: a factory that provides a skip-list-based implementation of
|
||||
// MemTableRep.
|
||||
std::shared_ptr<MemTableRepFactory> memtable_factory;
|
||||
|
||||
// This is a factory that provides TableFactory objects.
|
||||
// Default: a factory that provides a default implementation of
|
||||
// Table and TableBuilder.
|
||||
std::shared_ptr<TableFactory> table_factory;
|
||||
|
||||
// This is a factory that provides compaction filter objects which allow
|
||||
// an application to modify/delete a key-value during background compaction.
|
||||
// Default: a factory that doesn't provide any object
|
||||
std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
|
||||
|
||||
// Remove the log file immediately after the corresponding memtable is flushed
|
||||
// to data file.
|
||||
// Default: true
|
||||
bool purge_log_after_memtable_flush;
|
||||
|
||||
// This option allows user to to collect their own interested statistics of
|
||||
// the tables.
|
||||
// Default: emtpy vector -- no user-defined statistics collection will be
|
||||
// performed.
|
||||
std::vector<std::shared_ptr<TableStatsCollector>> table_stats_collectors;
|
||||
|
||||
// Allows thread-safe inplace updates. Requires Updates iff
|
||||
// * key exists in current memtable
|
||||
// * new sizeof(new_value) <= sizeof(old_value)
|
||||
// * old_value for that key is a put i.e. kTypeValue
|
||||
// Default: false.
|
||||
bool inplace_update_support;
|
||||
|
||||
// Number of locks used for inplace update
|
||||
// Default: 10000, if inplace_update_support = true, else 0.
|
||||
size_t inplace_update_num_locks;
|
||||
};
|
||||
|
||||
//
|
||||
// An application can issue a read request (via Get/Iterators) and specify
|
||||
// if that read should process data that ALREADY resides on a specified cache
|
||||
// level. For example, if an application specifies kBlockCacheTier then the
|
||||
// Get call will process data that is already processed in the memtable or
|
||||
// the block cache. It will not page in data from the OS cache or data that
|
||||
// resides in storage.
|
||||
enum ReadTier {
|
||||
kReadAllTier = 0x0, // data in memtable, block cache, OS cache or storage
|
||||
kBlockCacheTier = 0x1 // data in memtable or block cache
|
||||
};
|
||||
|
||||
// Options that control read operations
|
||||
@@ -649,10 +410,6 @@ struct ReadOptions {
|
||||
// Default: true
|
||||
bool fill_cache;
|
||||
|
||||
// If this option is set and memtable implementation allows, Seek
|
||||
// might only return keys with the same prefix as the seek-key
|
||||
bool prefix_seek;
|
||||
|
||||
// If "snapshot" is non-nullptr, read as of the supplied snapshot
|
||||
// (which must belong to the DB that is being read and which must
|
||||
// not have been released). If "snapshot" is nullptr, use an impliicit
|
||||
@@ -660,37 +417,14 @@ struct ReadOptions {
|
||||
// Default: nullptr
|
||||
const Snapshot* snapshot;
|
||||
|
||||
// If "prefix" is non-nullptr, and ReadOptions is being passed to
|
||||
// db.NewIterator, only return results when the key begins with this
|
||||
// prefix. This field is ignored by other calls (e.g., Get).
|
||||
// Options.prefix_extractor must also be set, and
|
||||
// prefix_extractor.InRange(prefix) must be true. The iterator
|
||||
// returned by NewIterator when this option is set will behave just
|
||||
// as if the underlying store did not contain any non-matching keys,
|
||||
// with two exceptions. Seek() only accepts keys starting with the
|
||||
// prefix, and SeekToLast() is not supported. prefix filter with this
|
||||
// option will sometimes reduce the number of read IOPs.
|
||||
// Default: nullptr
|
||||
const Slice* prefix;
|
||||
|
||||
// Specify if this read request should process data that ALREADY
|
||||
// resides on a particular cache. If the required data is not
|
||||
// found at the specified cache, then Status::Incomplete is returned.
|
||||
// Default: kReadAllTier
|
||||
ReadTier read_tier;
|
||||
|
||||
ReadOptions()
|
||||
: verify_checksums(false),
|
||||
fill_cache(true),
|
||||
prefix_seek(false),
|
||||
snapshot(nullptr),
|
||||
prefix(nullptr),
|
||||
read_tier(kReadAllTier) {
|
||||
snapshot(nullptr) {
|
||||
}
|
||||
ReadOptions(bool cksum, bool cache) :
|
||||
verify_checksums(cksum), fill_cache(cache),
|
||||
prefix_seek(false), snapshot(nullptr), prefix(nullptr),
|
||||
read_tier(kReadAllTier) {
|
||||
snapshot(nullptr) {
|
||||
}
|
||||
};
|
||||
|
||||
@@ -735,6 +469,6 @@ struct FlushOptions {
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
|
||||
@@ -12,15 +12,15 @@
|
||||
// non-const method, all threads accessing the same Slice must use
|
||||
// external synchronization.
|
||||
|
||||
#ifndef STORAGE_ROCKSDB_INCLUDE_SLICE_H_
|
||||
#define STORAGE_ROCKSDB_INCLUDE_SLICE_H_
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_SLICE_H_
|
||||
|
||||
#include <assert.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
#include <string>
|
||||
|
||||
namespace rocksdb {
|
||||
namespace leveldb {
|
||||
|
||||
class Slice {
|
||||
public:
|
||||
@@ -31,11 +31,9 @@ class Slice {
|
||||
Slice(const char* d, size_t n) : data_(d), size_(n) { }
|
||||
|
||||
// Create a slice that refers to the contents of "s"
|
||||
/* implicit */
|
||||
Slice(const std::string& s) : data_(s.data()), size_(s.size()) { }
|
||||
|
||||
// Create a slice that refers to s[0,strlen(s)-1]
|
||||
/* implicit */
|
||||
Slice(const char* s) : data_(s), size_(strlen(s)) { }
|
||||
|
||||
// Return a pointer to the beginning of the referenced data
|
||||
@@ -91,7 +89,7 @@ class Slice {
|
||||
(memcmp(data_, x.data_, x.size_) == 0));
|
||||
}
|
||||
|
||||
// private: make these public for rocksdbjni access
|
||||
// private: make these public for leveldbjni access
|
||||
const char* data_;
|
||||
size_t size_;
|
||||
|
||||
@@ -117,6 +115,7 @@ inline int Slice::compare(const Slice& b) const {
|
||||
return r;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_ROCKSDB_INCLUDE_SLICE_H_
|
||||
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_SLICE_H_
|
||||
Loaded 100 of 571 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user