Compare commits

..
22 Commits
Author SHA1 Message Date
Peter Dillinger 91f175888a Remove slack CircleCI hook (#9982)
Summary:
Our Slack site is deprecated

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9982

Test Plan: CircleCI

Reviewed By: siying

Differential Revision: D36322050

Pulled By: pdillinger

fbshipit-source-id: 678202404d307e1547e4203d7e6bd467803ccd5e
2022-05-12 09:30:59 -07:00
Hui Xiao 00724f43bc Update version.h and HISTORY.md for 7.1.2 2022-04-19 02:55:12 -07:00
Hui Xiao 3d15d07740 Conditionally declare and define variable that is unused in LITE mode (#9854)
Summary:
Context:
As mentioned in https://github.com/facebook/rocksdb/issues/9701, we have the following in LITE=1 make static_lib for v7.0.2
```
  CC       file/sequence_file_reader.o
  CC       file/sst_file_manager_impl.o
  CC       file/writable_file_writer.o
In file included from file/writable_file_writer.cc:10:
./file/writable_file_writer.h:163:15: error: private field 'temperature_' is not used [-Werror,-Wunused-private-field]
  Temperature temperature_;
              ^
1 error generated.
make: *** [file/writable_file_writer.o] Error 1
```

 as titled

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9854

Test Plan:
- Local `LITE=1 make static_lib` reveals the same error and error is gone after this fix
- CI

Reviewed By: ajkr, jay-zhuang

Differential Revision: D35706585

Pulled By: hx235

fbshipit-source-id: 7743310298231ad6866304ffa2225c8abdc91d9a
2022-04-19 01:23:49 -07:00
Andrew Kryczka 3345d12a1d Remove explicit padding from CacheAlignedInstrumentedMutex (#9809)
Summary:
Fixes https://github.com/facebook/rocksdb/issues/9779.

The padding at the end of a struct is added implicitly according to the
sizeof spec: "When applied to a class, the result is the
number of bytes in an object of that class including any padding
required for placing objects of that type in an array"
(https://eel.is/c++draft/expr.sizeof#2.sentence-2). We should drop the
explicit padding since it assumed support for zero-length arrays, which
is non-standard.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9809

Test Plan: rely on CI

Reviewed By: riversand963

Differential Revision: D35413496

Pulled By: ajkr

fbshipit-source-id: 25d52ca45e648ad0d5657149f26f6adecbed1cb4
2022-04-18 10:45:02 -07:00
Andrew Kryczka 9ed2144ebd Fix GetMergeOperands() heap-use-after-free on flushed memtable (#9805)
Summary:
Fixes https://github.com/facebook/rocksdb/issues/9066.

Prior to the fix in this PR, this PR's unit test reported the following error under ASAN:

```
==2175705==ERROR: AddressSanitizer: heap-use-after-free on address 0x61f0000012a5 at pc 0x7f0fc36e76ce bp 0x7ffc103e9ca0 sp 0x7ffc103e9450
READ of size 5 at 0x61f0000012a5 thread T0
    #0 0x7f0fc36e76cd in __interceptor_memcpy /home/engshare/third-party2/gcc/9.x/src/gcc-10.x/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:790
    https://github.com/facebook/rocksdb/issues/1 0x7f0fc35a207e in std::char_traits<char>::copy(char*, char const*, unsigned long) /home/engshare/third-party2/libgcc/9.x/src/gcc-9.x/x86_64-facebook-linux/libstdc++-v3/include/bits/char_traits.h:365
    https://github.com/facebook/rocksdb/issues/2 0x7f0fc35a207e in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_S_copy(char*, char const*, unsigned long) /home/engshare/third-party2/libgcc/9.x/src/gcc-9.x/x86_64-facebook-linux/libstdc++-v3/include/bits/basic_string.h:351
    https://github.com/facebook/rocksdb/issues/3 0x7f0fc35a207e in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_replace(unsigned long, unsigned long, char const*, unsigned long) /home/engshare/third-party2/libgcc/9.x/src/gcc-9.x/x86_64-facebook-linux/libstdc++-v3/include/bits/basic_string.tcc:440
    https://github.com/facebook/rocksdb/issues/4 0x8679ca in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::assign(char const*, unsigned long) /mnt/gvfs/third-party2/libgcc/4959b39cfbe5965a37c861c4c327fa7c5c759b87/9.x/platform009/9202ce7/include/c++/9.3.0/bits/basic_string.h:1422
    https://github.com/facebook/rocksdb/issues/5 0x8679ca in rocksdb::PinnableSlice::PinSelf(rocksdb::Slice const&) include/rocksdb/slice.h:171
    https://github.com/facebook/rocksdb/issues/6 0x8679ca in rocksdb::DBImpl::GetImpl(rocksdb::ReadOptions const&, rocksdb::Slice const&, rocksdb::DBImpl::GetImplOptions&) db/db_impl/db_impl.cc:1930
    https://github.com/facebook/rocksdb/issues/7 0x547324 in rocksdb::DBImpl::GetMergeOperands(rocksdb::ReadOptions const&, rocksdb::ColumnFamilyHandle*, rocksdb::Slice const&, rocksdb::PinnableSlice*, rocksdb::GetMergeOperandsOptions*, int*) db/db_impl/db_impl.h:203
    https://github.com/facebook/rocksdb/issues/8 0x547324 in rocksdb::DBMergeOperandTest_FlushedMergeOperandReadAfterFreeBug_Test::TestBody() db/db_merge_operand_test.cc:117
    https://github.com/facebook/rocksdb/issues/9 0x7241da in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3899
    https://github.com/facebook/rocksdb/issues/10 0x7241da in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3935
    https://github.com/facebook/rocksdb/issues/11 0x701a47 in testing::Test::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3973
    https://github.com/facebook/rocksdb/issues/12 0x702040 in testing::Test::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3965
    https://github.com/facebook/rocksdb/issues/13 0x702040 in testing::TestInfo::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:4149
    https://github.com/facebook/rocksdb/issues/14 0x7025f7 in testing::TestInfo::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:4124
    https://github.com/facebook/rocksdb/issues/15 0x7025f7 in testing::TestCase::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:4267
    https://github.com/facebook/rocksdb/issues/16 0x704217 in testing::TestCase::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:4253
    https://github.com/facebook/rocksdb/issues/17 0x704217 in testing::internal::UnitTestImpl::RunAllTests() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:6633
    https://github.com/facebook/rocksdb/issues/18 0x72505a in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3899
    https://github.com/facebook/rocksdb/issues/19 0x72505a in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3935
    https://github.com/facebook/rocksdb/issues/20 0x704aa1 in testing::UnitTest::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:6242
    https://github.com/facebook/rocksdb/issues/21 0x4c4aff in RUN_ALL_TESTS() third-party/gtest-1.8.1/fused-src/gtest/gtest.h:22110
    https://github.com/facebook/rocksdb/issues/22 0x4c4aff in main db/db_merge_operand_test.cc:404
    https://github.com/facebook/rocksdb/issues/23 0x7f0fc3108dc4 in __libc_start_main ../csu/libc-start.c:308
    https://github.com/facebook/rocksdb/issues/24 0x5445fd in _start (/data/users/andrewkr/rocksdb/db_merge_operand_test+0x5445fd)

0x61f0000012a5 is located 1061 bytes inside of 3264-byte region [0x61f000000e80,0x61f000001b40)
freed by thread T0 here:
    #0 0x7f0fc375b6af in operator delete(void*, unsigned long) /home/engshare/third-party2/gcc/9.x/src/gcc-10.x/libsanitizer/asan/asan_new_delete.cc:177
    https://github.com/facebook/rocksdb/issues/1 0x743be8 in rocksdb::SuperVersion::~SuperVersion() db/column_family.cc:432
    https://github.com/facebook/rocksdb/issues/2 0x8052aa in rocksdb::DBImpl::CleanupSuperVersion(rocksdb::SuperVersion*) db/db_impl/db_impl.cc:3534
    https://github.com/facebook/rocksdb/issues/3 0x8676c2 in rocksdb::DBImpl::ReturnAndCleanupSuperVersion(rocksdb::ColumnFamilyData*, rocksdb::SuperVersion*) db/db_impl/db_impl.cc:3544
    https://github.com/facebook/rocksdb/issues/4 0x8676c2 in rocksdb::DBImpl::GetImpl(rocksdb::ReadOptions const&, rocksdb::Slice const&, rocksdb::DBImpl::GetImplOptions&) db/db_impl/db_impl.cc:1911
    https://github.com/facebook/rocksdb/issues/5 0x547324 in rocksdb::DBImpl::GetMergeOperands(rocksdb::ReadOptions const&, rocksdb::ColumnFamilyHandle*, rocksdb::Slice const&, rocksdb::PinnableSlice*, rocksdb::GetMergeOperandsOptions*, int*) db/db_impl/db_impl.h:203
    https://github.com/facebook/rocksdb/issues/6 0x547324 in rocksdb::DBMergeOperandTest_FlushedMergeOperandReadAfterFreeBug_Test::TestBody() db/db_merge_operand_test.cc:117
    https://github.com/facebook/rocksdb/issues/7 0x7241da in void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3899
    https://github.com/facebook/rocksdb/issues/8 0x7241da in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3935
    https://github.com/facebook/rocksdb/issues/9 0x701a47 in testing::Test::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3973
    https://github.com/facebook/rocksdb/issues/10 0x702040 in testing::Test::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3965
    https://github.com/facebook/rocksdb/issues/11 0x702040 in testing::TestInfo::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:4149
    https://github.com/facebook/rocksdb/issues/12 0x7025f7 in testing::TestInfo::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:4124
    https://github.com/facebook/rocksdb/issues/13 0x7025f7 in testing::TestCase::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:4267
    https://github.com/facebook/rocksdb/issues/14 0x704217 in testing::TestCase::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:4253
    https://github.com/facebook/rocksdb/issues/15 0x704217 in testing::internal::UnitTestImpl::RunAllTests() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:6633
    https://github.com/facebook/rocksdb/issues/16 0x72505a in bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3899
    https://github.com/facebook/rocksdb/issues/17 0x72505a in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:3935
    https://github.com/facebook/rocksdb/issues/18 0x704aa1 in testing::UnitTest::Run() third-party/gtest-1.8.1/fused-src/gtest/gtest-all.cc:6242
    https://github.com/facebook/rocksdb/issues/19 0x4c4aff in RUN_ALL_TESTS() third-party/gtest-1.8.1/fused-src/gtest/gtest.h:22110
    https://github.com/facebook/rocksdb/issues/20 0x4c4aff in main db/db_merge_operand_test.cc:404
    https://github.com/facebook/rocksdb/issues/21 0x7f0fc3108dc4 in __libc_start_main ../csu/libc-start.c:308
    https://github.com/facebook/rocksdb/issues/22 0x5445fd in _start (/data/users/andrewkr/rocksdb/db_merge_operand_test+0x5445fd)
...
```

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9805

Test Plan: following the fix in this PR, the new unit test passes

Reviewed By: jay-zhuang

Differential Revision: D35388415

Pulled By: ajkr

fbshipit-source-id: b39c5d002155906c8abc4a3429eca696dbf916d0
2022-04-18 10:44:33 -07:00
Yanqin Jin 7e42a5ba89 Encode min_log_number_to_keep and delete_wals_before in one version edit (#9766)
Summary:
min_log_number_to_keep denotes that the WALs whose numbers are below
this value **will** be deleted by RocksDB.
delete_wals_before will be used by RocksDB if
track_and_verify_wals_in_manifest is set to true. During recovery,
RocksDB uses the info encoded in delete_wals_before to reconstruct its
knowledge about what WALs to expect existing.
If these two tags are not encoded in the same VersionEdit, then it's
possible for min_log_number_to_keep=100 to exist, but
delete_wals_before=100 to be lost due to power failure. Subsequent
recovery will delete 99.log. If the db crashes again, the following
recovery will expect to see 99.log since there is no
delete_wals_before=100 in the MANIFEST, but the WAL is already deleted.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9766

Test Plan:
First of all, make check.
Second, format compatibility.
SHORT_TEST=1 ./tools/check_format_compatible.sh

Reviewed By: ltamasi

Differential Revision: D35203623

Pulled By: riversand963

fbshipit-source-id: 45623fc4b4b50d299d5e0f9559a3a4c5e9522c8f
2022-04-18 10:43:05 -07:00
sdong 8654ac5d9a Fix DB::Open() error logging (#9784)
Summary:
Right now we log a wrong error when DB::Open() fails. Fix it.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9784

Test Plan: CI runs should pass

Reviewed By: ajkr, riversand963

Differential Revision: D35290203

fbshipit-source-id: ffc640afa27f6b0a2382ee153dc43f28d9e242be
2022-04-18 10:42:37 -07:00
bbkot e07d836391 fixing issue #8345 RocksDB does not work when using UNC network paths (#9384)
Summary:
Fix https://github.com/facebook/rocksdb/issues/8345
RocksDB does not work with network filesystem paths on Windows, e.g. "\\hostname\folder\..."

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9384

Reviewed By: mrambacher

Differential Revision: D33830622

Pulled By: riversand963

fbshipit-source-id: 2a99dc3c94415eb1460e110784b97d71600218f1
2022-04-18 10:41:55 -07:00
Peter Dillinger e6bca1a54e Fix FileStorageInfo fields from GetLiveFilesMetaData (#9769)
Summary:
In making `SstFileMetaData` inherit from `FileStorageInfo`, I
overlooked setting some `FileStorageInfo` fields when then default
`SstFileMetaData()` ctor is used. This affected `GetLiveFilesMetaData()`.

Also removed some buggy `static_cast<size_t>`

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9769

Test Plan: Updated tests

Reviewed By: jay-zhuang

Differential Revision: D35220383

Pulled By: pdillinger

fbshipit-source-id: 05b4ee468258dbd3699517e1124838bf405fe7f8
2022-04-18 10:40:43 -07:00
Jack Robison c815f31ad3 Fix broken zlib dependency, update it from 1.2.11 to 1.2.12 (#9764)
Summary:
Zlib (https://www.zlib.net/) has been updated to 1.2.12 due to CVE-2018-25032

- https://nvd.nist.gov/vuln/detail/CVE-2018-25032
- https://github.com/madler/zlib/issues/605

The source .tar.gz is no longer available, and the Makefile for rocksdb now fails as a result. This PR updates the dependency to the newer (and available) version, 1.2.12

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9764

Reviewed By: ajkr

Differential Revision: D35220367

Pulled By: jay-zhuang

fbshipit-source-id: 1f68ff8f048a6dba42077f048ac143468f0e2478
2022-04-18 10:39:47 -07:00
myasuka 6c8620451c Enable READ_BLOCK_COMPACTION_MICROS to track stats (#9722)
Summary:
After commit [d642c60](https://github.com/facebook/rocksdb/commit/d642c60bdc100f7509ca77b383cd47b51d80d810), the stats `READ_BLOCK_COMPACTION_MICROS` cannot record any compaction read duration, and it always report zero.

This PR targets to distinguish `READ_BLOCK_COMPACTION_MICROS` with `READ_BLOCK_GET_MICROS` so that `READ_BLOCK_COMPACTION_MICROS` could record the correct stats.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9722

Reviewed By: ajkr

Differential Revision: D35021870

Pulled By: jay-zhuang

fbshipit-source-id: f1a804994265e51465de64c2a08f2e0eeb6fc5a3
2022-04-18 10:36:11 -07:00
Peter Dillinger fa99e8afbc Fix heap use-after-free race with DropColumnFamily (#9730)
Summary:
Although ColumnFamilySet comments say that DB mutex can be
freed during iteration, as long as you hold a ref while releasing DB
mutex, this is not quite true because UnrefAndTryDelete might delete cfd
right before it is needed to get ->next_ for the next iteration of the
loop.

This change solves the problem by making a wrapper class that makes such
iteration easier while handling the tricky details of UnrefAndTryDelete
on the previous cfd only after getting next_ in operator++.

FreeDeadColumnFamilies should already have been obsolete; this removes
it for good. Similarly, ColumnFamilySet::iterator doesn't need to check
for cfd with 0 refs, because those are immediately deleted.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9730

Test Plan:
was reported with ASAN on unit tests like
DBLogicalBlockSizeCacheTest.CreateColumnFamily (very rare); keep watching

Reviewed By: ltamasi

Differential Revision: D35038143

Pulled By: pdillinger

fbshipit-source-id: 0a5478d5be96c135343a00603711b7df43ae19c9
2022-04-18 10:35:14 -07:00
Yanqin Jin 43890774ed Fix a race condition in WAL tracking causing DB open failure (#9715)
Summary:
There is a race condition if WAL tracking in the MANIFEST is enabled in a database that disables 2PC.

The race condition is between two background flush threads trying to install flush results to the MANIFEST.

Consider an example database with two column families: "default" (cfd0) and "cf1" (cfd1). Initially,
both column families have one mutable (active) memtable whose data backed by 6.log.

1. Trigger a manual flush for "cf1", creating a 7.log
2. Insert another key to "default", and trigger flush for "default", creating 8.log
3. BgFlushThread1 finishes writing 9.sst
4. BgFlushThread2 finishes writing 10.sst

```
Time  BgFlushThread1                                    BgFlushThread2
 |    mutex_.Lock()
 |    precompute min_wal_to_keep as 6
 |    mutex_.Unlock()
 |                                                     mutex_.Lock()
 |                                                     precompute min_wal_to_keep as 6
 |                                                     join MANIFEST write queue and mutex_.Unlock()
 |    write to MANIFEST
 |    mutex_.Lock()
 |    cfd1->log_number = 7
 |    Signal bg_flush_2 and mutex_.Unlock()
 |                                                     wake up and mutex_.Lock()
 |                                                     cfd0->log_number = 8
 |                                                     FindObsoleteFiles() with job_context->log_number == 7
 |                                                     mutex_.Unlock()
 |                                                     PurgeObsoleteFiles() deletes 6.log
 V
```

As shown in the above, BgFlushThread2 thinks that the min wal to keep is 6.log because "cf1" has unflushed data in 6.log (cf1.log_number=6).
Similarly, BgThread1 thinks that min wal to keep is also 6.log because "default" has unflushed data (default.log_number=6).
No WAL deletion will be written to MANIFEST because 6 is equal to `versions_->wals_.min_wal_number_to_keep`,
due to https://github.com/facebook/rocksdb/blob/7.1.fb/db/memtable_list.cc#L513:L514.
The bg flush thread that finishes last will perform file purging. `job_context.log_number` will be evaluated as 7, i.e.
the min wal that contains unflushed data, causing 6.log to be deleted. However, MANIFEST thinks 6.log should still exist.
If you close the db at this point, you won't be able to re-open it if `track_and_verify_wal_in_manifest` is true.

We must handle the case of multiple bg flush threads, and it is difficult for one bg flush thread to know
the correct min wal number until the other bg flush threads have finished committing to the manifest and updated
the `cfd::log_number`.
To fix this issue, we rename an existing variable `min_log_number_to_keep_2pc` to `min_log_number_to_keep`,
and use it to track WAL file deletion in non-2pc mode as well.
This variable is updated only 1) during recovery with mutex held, or 2) in the MANIFEST write thread.
`min_log_number_to_keep` means RocksDB will delete WALs below it, although there may be WALs
above it which are also obsolete. Formally, we will have [min_wal_to_keep, max_obsolete_wal]. During recovery, we
make sure that only WALs above max_obsolete_wal are checked and added back to `alive_log_files_`.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9715

Test Plan:
```
make check
```
Also ran stress test below (with asan) to make sure it completes successfully.
```
TEST_TMPDIR=/dev/shm/rocksdb OPT=-g ASAN_OPTIONS=disable_coredump=0 \
CRASH_TEST_EXT_ARGS=--compression_type=zstd SKIP_FORMAT_BUCK_CHECKS=1 \
make J=52 -j52 blackbox_asan_crash_test
```

Reviewed By: ltamasi

Differential Revision: D34984412

Pulled By: riversand963

fbshipit-source-id: c7b21a8d84751bb55ea79c9f387103d21b231005
2022-04-18 10:34:13 -07:00
akankshamahajan ee6ee5ff08 Update version.h and HISTORY.md for 7.1.1 2022-04-07 11:38:53 -07:00
Akanksha Mahajan 80bea2ba8e Update stats for Read and ReadAsync in random_access_file_reader for async prefetching (#9810)
Summary:
Update stats in random_access_file_reader for Read and
ReadAsync API to take into account the read latency for async
prefetching.

It also fixes ERROR_HANDLER_AUTORESUME_RETRY_COUNT stat whose value was
incorrect in portal.h

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9810

Test Plan: Update unit test

Reviewed By: anand1976

Differential Revision: D35433081

Pulled By: akankshamahajan15

fbshipit-source-id: aeec3901270e58a003ce6b5214bd25ddcb3a12a9
2022-04-07 09:15:37 -07:00
Akanksha Mahajan 71158842aa Fix reseting of async_read_in_progress_ variable in FilePrefetchBuffer to call Poll API (#9815)
Summary:
Currently RocksDB reset async_read_in_progress_ in callback
due to which underlying filesystem relying on Poll API won't be called
leading to stale memory access.
In order to fix it, async_read_in_progress_ will be reset after Poll API
is called to make sure underlying file_system waiting on Poll can clear
its state or take appropriate action.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9815

Test Plan: CircleCI tests

Reviewed By: anand1976

Differential Revision: D35451534

Pulled By: akankshamahajan15

fbshipit-source-id: b70ef6251a7aa9ed4876ba5e5100baa33d7d474c
2022-04-07 09:03:53 -07:00
Akanksha Mahajan 78641c0930 Fix segfault in FilePrefetchBuffer with async_io enabled (#9777)
Summary:
If FilePrefetchBuffer object is destroyed and then later Poll() calls callback on object which has been destroyed, it gives segfault on accessing destroyed object. It was caught after adding unit tests that tests Posix implementation of ReadAsync and Poll APIs.
This PR also updates and fixes existing IOURing tests which were not running locally because RocksDbIOUringEnable function wasn't defined and IOUring was disabled for those tests

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9777

Test Plan: Added new unit test

Reviewed By: anand1976

Differential Revision: D35254002

Pulled By: akankshamahajan15

fbshipit-source-id: 68e80054ffb14ae25c255920ebc6548ca5f130a1
2022-04-06 21:02:55 -07:00
Adam Retter 9d3d5e72e0 Retrieve ZLib from the archive (#9790) 2022-04-04 18:07:30 -07:00
Akanksha Mahajan 45770b5d23 Fix some errors in async prefetching in FilePrefetchBuffer (#9734)
Summary:
In ReadOption `async_io` which prefetches the data asynchronously, db_bench and db_stress runs were failing  because wrong data was prefetched which resulted in Error: Checksum mismatched. Wrong data was copied because capacity was less than actual size needed. It has been fixed in this PR.

Since there are two separate methods for async and sync prefetching, these changes are in async prefetching methods and any changes would not effect normal prefetching. I ran the regressions to make sure normal prefetching is fine.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9734

Test Plan:
1. CircleCI jobs

2.  Ran db_bench
```
. /db_bench -use_existing_db=true
-db=/tmp/prefix_scan_prefetch_main -benchmarks="seekrandom" -key_size=32
-value_size=512 -num=5000000 -use_direct_reads=true -seek_nexts=327680
-duration=120 -ops_between_duration_checks=1 -async_io=1 -adaptive_readahead=1

```
3. Ran db_stress test
```
export CRASH_TEST_EXT_ARGS=" --async_io=1 --adaptive_readahead=1"
make crash_test -j
```

4. Run regressions for async_io disabled.

Old flow without any async changes:
```
./db_bench -use_existing_db=true -db=/tmp/prefix_scan_prefetch_main -benchmarks="seekrandom" -key_size=32 -value_size=512 -num=5000000 -use_direct_reads=true -seek_nexts=327680 -duration=120 -ops_between_duration_checks=1
Initializing RocksDB Options from the specified file
Initializing RocksDB Options from command-line flags
RocksDB:    version 7.0
Date:       Thu Mar 17 13:11:34 2022
CPU:        24 * Intel Core Processor (Broadwell)
CPUCache:   16384 KB
Keys:       32 bytes each (+ 0 bytes user-defined timestamp)
Values:     512 bytes each (256 bytes after compression)
Entries:    5000000
Prefix:    0 bytes
Keys per prefix:    0
RawSize:    2594.0 MB (estimated)
FileSize:   1373.3 MB (estimated)
Write rate: 0 bytes/second
Read rate: 0 ops/second
Compression: Snappy
Compression sampling rate: 0
Memtablerep: SkipListFactory
Perf Level: 1
------------------------------------------------
DB path: [/tmp/prefix_scan_prefetch_main]
seekrandom   :  483618.390 micros/op 2 ops/sec;  338.9 MB/s (249 of 249 found)
```

With async prefetching changes and async_io disabled to make sure in normal prefetching there is no regression.
 ```
 ./db_bench -use_existing_db=true -db=/tmp/prefix_scan_prefetch_main -benchmarks="seekrandom" -key_size=32 -value_size=512 -num=5000000 -use_direct_reads=true -seek_nexts=327680 -duration=120 -ops_between_duration_checks=1 --async_io=0
Initializing RocksDB Options from the specified file
Initializing RocksDB Options from command-line flags
RocksDB:    version 7.1
Date:       Wed Mar 23 15:56:37 2022
CPU:        24 * Intel Core Processor (Broadwell)
CPUCache:   16384 KB
Keys:       32 bytes each (+ 0 bytes user-defined timestamp)
Values:     512 bytes each (256 bytes after compression)
Entries:    5000000
Prefix:    0 bytes
Keys per prefix:    0
RawSize:    2594.0 MB (estimated)
FileSize:   1373.3 MB (estimated)
Write rate: 0 bytes/second
Read rate: 0 ops/second
Compression: Snappy
Compression sampling rate: 0
Memtablerep: SkipListFactory
Perf Level: 1
------------------------------------------------
DB path: [/tmp/prefix_scan_prefetch_main]
seekrandom   :  481819.816 micros/op 2 ops/sec;  340.2 MB/s (250 of 250 found)
```

Reviewed By: riversand963

Differential Revision: D35058471

Pulled By: akankshamahajan15

fbshipit-source-id: 9233a1e6d97cea0c7a8111bfb9e8ac3251c341ce
2022-03-25 18:36:49 -07:00
Peter Dillinger 4411488585 Tidy up HISTORY.md for 7.1.0 2022-03-23 16:38:06 -07:00
Peter Dillinger 8cc712c0eb Revise history of 7.1.0 for patch 2022-03-23 15:42:35 -07:00
Peter Dillinger 62b71b3094 Fix a major performance bug in 7.0 re: filter compatibility (#9736)
Summary:
Bloom filters generated by pre-7.0 releases are not read by
7.0.x releases (and vice-versa) due to changes to FilterPolicy::Name()
in https://github.com/facebook/rocksdb/issues/9590. This can severely impact read performance and read I/O on
upgrade or downgrade with existing DB, but not data correctness.

To fix, we go back using the old, unified name in SST metadata but (for
a while anyway) recognize the aliases that could be generated by early
7.0.x releases. This unfortunately requires a public API change to avoid
interfering with all the good changes from https://github.com/facebook/rocksdb/issues/9590, but the API change
only affects users with custom FilterPolicy, which should be very few.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9736

Test Plan:
manual

Generate DBs with
```
./db_bench.7.0 -db=/dev/shm/rocksdb.7.0 -bloom_bits=10 -cache_index_and_filter_blocks=1 -benchmarks=fillrandom -num=10000000 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0
```
and similar. Compare with
```
for IMPL in 6.29 7.0 fixed; do for DB in 6.29 7.0 fixed; do echo "Testing $IMPL on $DB:"; ./db_bench.$IMPL -db=/dev/shm/rocksdb.$DB -use_existing_db -readonly -bloom_bits=10 -benchmarks=readrandom -num=10000000 -compaction_style=2 -fifo_compaction_max_table_files_size_mb=10000 -fifo_compaction_allow_compaction=0 -duration=10 2>&1 | grep micros/op; done; done
```

Results:
```
Testing 6.29 on 6.29:
readrandom   :      34.381 micros/op 29085 ops/sec;    3.2 MB/s (291999 of 291999 found)
Testing 6.29 on 7.0:
readrandom   :     190.443 micros/op 5249 ops/sec;    0.6 MB/s (52999 of 52999 found)
Testing 6.29 on fixed:
readrandom   :      40.148 micros/op 24907 ops/sec;    2.8 MB/s (249999 of 249999 found)
Testing 7.0 on 6.29:
readrandom   :     229.430 micros/op 4357 ops/sec;    0.5 MB/s (43999 of 43999 found)
Testing 7.0 on 7.0:
readrandom   :      33.348 micros/op 29986 ops/sec;    3.3 MB/s (299999 of 299999 found)
Testing 7.0 on fixed:
readrandom   :     152.734 micros/op 6546 ops/sec;    0.7 MB/s (65999 of 65999 found)
Testing fixed on 6.29:
readrandom   :      32.024 micros/op 31224 ops/sec;    3.5 MB/s (312999 of 312999 found)
Testing fixed on 7.0:
readrandom   :      33.990 micros/op 29390 ops/sec;    3.3 MB/s (294999 of 294999 found)
Testing fixed on fixed:
readrandom   :      28.714 micros/op 34825 ops/sec;    3.9 MB/s (348999 of 348999 found)
```

Just paying attention to order of magnitude of ops/sec (short test
durations, lots of noise), it's clear that with the fix we can read <= 6.29
& >= 7.0 at full speed, where neither 6.29 nor 7.0 can on both. And 6.29
release can properly read fixed DB at full speed.

Reviewed By: siying, ajkr

Differential Revision: D35057844

Pulled By: pdillinger

fbshipit-source-id: a46893a6af4bf084375ebe4728066d00eb08f050
2022-03-23 15:00:14 -07:00
130 changed files with 4000 additions and 3612 deletions

No files matched your search

+8 -10
View File
@@ -2,7 +2,6 @@ version: 2.1
orbs:
win: circleci/windows@2.4.0
slack: circleci/slack@3.4.2
aliases:
- &notify-on-main-failure
@@ -57,7 +56,6 @@ commands:
post-steps:
steps:
- slack/status: *notify-on-main-failure
- store_test_results: # store test result if there's any
path: /tmp/test-results
- store_artifacts: # store LOG for debugging if there's any
@@ -102,16 +100,10 @@ commands:
install-benchmark:
steps:
- run:
name: Install ninja build
command: sudo apt-get update -y && sudo apt-get install -y ninja-build
- run:
name: Install benchmark
command: |
git clone --depth 1 --branch v1.6.1 https://github.com/google/benchmark.git ~/benchmark
cd ~/benchmark && mkdir build && cd build
cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0
ninja && sudo ninja install
sudo apt-get update -y && sudo apt-get install -y libbenchmark-dev
upgrade-cmake:
steps:
@@ -437,8 +429,8 @@ jobs:
resource_class: xlarge
steps:
- pre-steps
- run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-11 g++-11 libgflags-dev
- install-benchmark
- run: sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test && sudo apt-get update -y && sudo apt-get install gcc-11 g++-11 libgflags-dev
- run: CC=gcc-11 CXX=g++-11 V=1 make -j16 all microbench
- post-steps
@@ -559,6 +551,9 @@ jobs:
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run:
name: "Build RocksDBJava Shared Library"
command: make V=1 J=8 -j8 rocksdbjava
- run:
name: "Test RocksDBJava"
command: make V=1 J=8 -j8 jtest
@@ -604,6 +599,9 @@ jobs:
echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
which java && java -version
which javac && javac -version
- run:
name: "Build RocksDBJava Shared Library"
command: make V=1 J=16 -j16 rocksdbjava
- run:
name: "Test RocksDBJava"
command: make V=1 J=16 -j16 jtest
+2 -2
View File
@@ -239,10 +239,10 @@ script:
OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=only make -j$MK_PARALLEL all_but_some_tests check_some
;;
1)
OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_END=backup_engine_test make -j$MK_PARALLEL check_some
OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_END=backupable_db_test make -j$MK_PARALLEL check_some
;;
2)
OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 make -j$MK_PARALLEL tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=backup_engine_test ROCKSDBTESTS_END=db_universal_compaction_test make -j$MK_PARALLEL check_some
OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 make -j$MK_PARALLEL tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=backupable_db_test ROCKSDBTESTS_END=db_universal_compaction_test make -j$MK_PARALLEL check_some
;;
3)
OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=db_universal_compaction_test ROCKSDBTESTS_END=table_properties_collector_test make -j$MK_PARALLEL check_some
+3 -3
View File
@@ -2,7 +2,7 @@
# This cmake build is for Windows 64-bit only.
#
# Prerequisites:
# You must have at least Visual Studio 2019. Start the Developer Command Prompt window that is a part of Visual Studio installation.
# You must have at least Visual Studio 2015 Update 3. Start the Developer Command Prompt window that is a part of Visual Studio installation.
# Run the build commands from within the Developer Command Prompt window to have paths to the compiler and runtime libraries set.
# You must have git.exe in your %PATH% environment variable.
#
@@ -849,7 +849,7 @@ set(SOURCES
util/thread_local.cc
util/threadpool_imp.cc
util/xxhash.cc
utilities/backup/backup_engine.cc
utilities/backupable/backupable_db.cc
utilities/blob_db/blob_compaction_filter.cc
utilities/blob_db/blob_db.cc
utilities/blob_db/blob_db_impl.cc
@@ -1315,7 +1315,7 @@ if(WITH_TESTS)
util/thread_list_test.cc
util/thread_local_test.cc
util/work_queue_test.cc
utilities/backup/backup_engine_test.cc
utilities/backupable/backupable_db_test.cc
utilities/blob_db/blob_db_test.cc
utilities/cassandra/cassandra_functional_test.cc
utilities/cassandra/cassandra_format_test.cc
+5 -12
View File
@@ -1,5 +1,5 @@
# Rocksdb Change Log
## Unreleased
## 7.1.2 (04/19/2022)
### Bug Fixes
* Fixed bug which caused rocksdb failure in the situation when rocksdb was accessible using UNC path
* Fixed a race condition when 2PC is disabled and WAL tracking in the MANIFEST is enabled. The race condition is between two background flush threads trying to install flush results, causing a WAL deletion not tracked in the MANIFEST. A future DB open may fail.
@@ -7,17 +7,10 @@
* Fixed a bug that `rocksdb.read.block.compaction.micros` cannot track compaction stats (#9722).
* Fixed `file_type`, `relative_filename` and `directory` fields returned by `GetLiveFilesMetaData()`, which were added in inheriting from `FileStorageInfo`.
* Fixed a bug affecting `track_and_verify_wals_in_manifest`. Without the fix, application may see "open error: Corruption: Missing WAL with log number" while trying to open the db. The corruption is a false alarm but prevents DB open (#9766).
## 7.1.1 (04/07/2022)
### Bug Fixes
* Fix segfault in FilePrefetchBuffer with async_io as it doesn't wait for pending jobs to complete on destruction.
* Fix ERROR_HANDLER_AUTORESUME_RETRY_COUNT stat whose value was set wrong in portal.h
### New Features
* For db_bench when --seed=0 or --seed is not set then it uses the current time as the seed value. Previously it used the value 1000.
* For db_bench when --benchmark lists multiple tests and each test uses a seed for a RNG then the seeds across tests will no longer be repeated.
* Added an option to dynamically charge an updating estimated memory usage of block-based table reader to block cache if block cache available. To enable this feature, set `BlockBasedTableOptions::reserve_table_reader_memory = true`.
* Add new stat ASYNC_READ_BYTES that calculates number of bytes read during async read call and users can check if async code path is being called by RocksDB internal automatic prefetching for sequential reads.
### Behavior changes
* Disallow usage of commit-time-write-batch for write-prepared/write-unprepared transactions if TransactionOptions::use_only_the_last_commit_time_batch_for_recovery is false to prevent two (or more) uncommitted versions of the same key in the database. Otherwise, bottommost compaction may violate the internal key uniqueness invariant of SSTs if the sequence numbers of both internal keys are zeroed out (#9794).
## 7.1.0 (03/23/2022)
### New Features
@@ -1678,7 +1671,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
* Added a new way to report QPS from db_bench (check out --report_file and --report_interval_seconds)
* Added a cache for individual rows. See DBOptions::row_cache for more info.
* Several new features on EventListener (see include/rocksdb/listener.h):
- OnCompactionCompleted() now returns per-compaction job statistics, defined in include/rocksdb/compaction_job_stats.h.
- OnCompationCompleted() now returns per-compaction job statistics, defined in include/rocksdb/compaction_job_stats.h.
- Added OnTableFileCreated() and OnTableFileDeleted().
* Add compaction_options_universal.enable_trivial_move to true, to allow trivial move while performing universal compaction. Trivial move will happen only when all the input files are non overlapping.
+3 -2
View File
@@ -47,7 +47,7 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
* If you wish to build the RocksJava static target, then cmake is required for building Snappy.
* If you wish to run microbench (e.g, `make microbench`, `make ribbon_bench` or `cmake -DWITH_BENCHMARK=1`), Google benchmark >= 1.6.0 is needed.
* If you wish to run microbench (e.g, `make microbench`, `make ribbon_bench` or `cmake -DWITH_BENCHMARK=1`), Google benchmark < 1.6.0 is needed.
## Supported platforms
@@ -180,7 +180,8 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
* **iOS**:
* Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
* **Windows** (Visual Studio 2017 to up):
* **Windows**:
* For building with MS Visual Studio 13 you will need Update 4 installed.
* Read and follow the instructions at CMakeLists.txt
* Or install via [vcpkg](https://github.com/microsoft/vcpkg)
* run `vcpkg install rocksdb:x64-windows`
+9 -9
View File
@@ -826,8 +826,8 @@ release: clean
coverage: clean
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) J=1 all check
cd coverage && ./coverage_test.sh
# Delete intermediate files
$(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \;
# Delete intermediate files
$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm -f {} \;
ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
# Use /dev/shm if it has the sticky bit set (otherwise, /tmp),
@@ -1214,9 +1214,9 @@ clean-rocks:
rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(PARALLEL_TEST) $(ALL_STATIC_LIBS) $(ALL_SHARED_LIBS) $(MICROBENCHS)
rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
$(FIND) . -name "*.[oda]" -exec rm -f {} \;
$(FIND) . -type f \( -name "*.gcda" -o -name "*.gcno" \) -exec rm -f {} \;
$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm -f {} \;
clean-rocksjava: clean-rocks
clean-rocksjava:
rm -rf jl jls
cd java && $(MAKE) clean
@@ -1551,7 +1551,7 @@ perf_context_test: $(OBJ_DIR)/db/perf_context_test.o $(TEST_LIBRARY) $(LIBRARY)
prefix_test: $(OBJ_DIR)/db/prefix_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
backup_engine_test: $(OBJ_DIR)/utilities/backup/backup_engine_test.o $(TEST_LIBRARY) $(LIBRARY)
backupable_db_test: $(OBJ_DIR)/utilities/backupable/backupable_db_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
checkpoint_test: $(OBJ_DIR)/utilities/checkpoint/checkpoint_test.o $(TEST_LIBRARY) $(LIBRARY)
@@ -2372,10 +2372,10 @@ jtest: rocksdbjava
jdb_bench:
cd java;$(MAKE) db_bench;
commit_prereq:
echo "TODO: bring this back using parts of old precommit_checker.py and rocksdb-lego-determinator"
false # J=$(J) build_tools/precommit_checker.py unit clang_unit release clang_release tsan asan ubsan lite unit_non_shm
# $(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
commit_prereq: build_tools/rocksdb-lego-determinator \
build_tools/precommit_checker.py
J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit release release_481 clang_release tsan asan ubsan lite unit_non_shm
$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
# ---------------------------------------------------------------------------
# Platform-specific compilation
+1 -1
View File
@@ -4,7 +4,7 @@ RocksDBLite is a project focused on mobile use cases, which don't need a lot of
Some examples of the features disabled by ROCKSDB_LITE:
* compiled-in support for LDB tool
* No backup engine
* No backupable DB
* No support for replication (which we provide in form of TransactionalIterator)
* No advanced monitoring tools
* No special-purpose memtables that are highly optimized for specific use cases
+550 -292
View File
File diff suppressed because it is too large. Load diff
+220 -22
View File
File diff suppressed because it is too large. Load diff
+64 -16
View File
@@ -3,23 +3,27 @@
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 510.387506
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 497.077261
}
@@ -131,29 +135,33 @@
"expected_runtime_one_iter": 2438,
"name": "rocksdb_microbench_suite_0",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 510.88842
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 497.546918
}
@@ -258,29 +266,33 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_1",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 479.941992
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 513.865636
}
@@ -319,7 +331,7 @@
"expected_runtime_one_iter": 2446,
"name": "rocksdb_microbench_suite_2",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
@@ -345,23 +357,27 @@
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 471.459789
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 514.201628
}
@@ -445,7 +461,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_3",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
@@ -471,23 +487,27 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 694.628355
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 517.245591
}
@@ -570,7 +590,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_4",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
@@ -700,18 +720,20 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_5",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 658.890773
}
@@ -800,18 +822,20 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_6",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 505.270116
}
@@ -827,12 +851,14 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 532.702176
}
@@ -913,7 +939,7 @@
"expected_runtime_one_iter": 2438,
"name": "rocksdb_microbench_suite_7",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
@@ -1030,7 +1056,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_8",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
@@ -1056,12 +1082,14 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 497.710941
}
@@ -1127,40 +1155,46 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_9",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 624.132525
}
],
"DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 510.156004
}
],
"DBGet/comp_style:0/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 551.935026
}
@@ -1212,40 +1246,46 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_10",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
"db_basic_bench": {
"DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 507.805668
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 557.749335
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:1/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 623.018994
}
@@ -1280,7 +1320,7 @@
"expected_runtime_one_iter": 2446,
"name": "rocksdb_microbench_suite_11",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
@@ -1296,12 +1336,14 @@
}
],
"DBGet/comp_style:0/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 609.749844
}
@@ -1317,12 +1359,14 @@
}
],
"DBGet/comp_style:1/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:1/enable_filter:1/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 557.886298
}
@@ -1414,7 +1458,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_12",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
@@ -1440,12 +1484,14 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:256/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 578.538571
}
@@ -1513,7 +1559,7 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_13",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
},
{
"benchmarks": {
@@ -1539,12 +1585,14 @@
}
],
"DBGet/comp_style:2/max_data:134217728/per_key_size:1024/enable_statistics:1/negative_query:0/enable_filter:0/iterations:10240/threads:1": [
"get_p95",
"db_size",
"get_mean",
"threads",
"real_time",
"cpu_time",
"neg_qu_pct",
"get_p99",
{
"est_runtime": 501.900122
}
@@ -1613,6 +1661,6 @@
"expected_runtime_one_iter": 2437,
"name": "rocksdb_microbench_suite_14",
"regression_threshold": 10,
"sl_iterations": 3
"sl_iterations": 2
}
]
]
+13 -2
View File
@@ -63,7 +63,18 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
if [ "$LIB_MODE" == "shared" ]; then
PIC_BUILD=1
fi
source "$PWD/build_tools/fbcode_config_platform009.sh"
if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
# we need this to build with MySQL. Don't use for other purposes.
source "$PWD/build_tools/fbcode_config4.8.1.sh"
elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
source "$PWD/build_tools/fbcode_config.sh"
elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then
source "$PWD/build_tools/fbcode_config_platform007.sh"
elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM009" ]; then
source "$PWD/build_tools/fbcode_config_platform009.sh"
else
source "$PWD/build_tools/fbcode_config_platform009.sh"
fi
fi
# Delete existing output, if it exists
@@ -587,7 +598,7 @@ EOF
fi
if ! test $ROCKSDB_DISABLE_BENCHMARK; then
# Test whether google benchmark is available
$CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -lbenchmark -lpthread 2>/dev/null <<EOF
$CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -lbenchmark 2>/dev/null <<EOF
#include <benchmark/benchmark.h>
int main() {}
EOF
+19
View File
@@ -0,0 +1,19 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/5.x/centos7-native/c447969
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/1bd23f9917738974ad0ff305aa23eb5f93f18305/9.0.0/centos7-native/c9f9104
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/5.x/gcc-5-glibc-2.23/339d858
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.23/gcc-5-glibc-2.23/ca1d1c0
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/gcc-5-glibc-2.23/9bc6787
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/gcc-5-glibc-2.23/9bc6787
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/gcc-5-glibc-2.23/9bc6787
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/gcc-5-glibc-2.23/9bc6787
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/gcc-5-glibc-2.23/03859b5
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/gcc-5-glibc-2.23/9bc6787
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/gcc-5-glibc-2.23/0c8f76d
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/gcc-5-glibc-2.23/9bc6787
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/gcc-5-glibc-2.23/b443de1
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/gcc-5-glibc-2.23/9bc6787
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/2e3cb7d119b3cea5f1e738cc13a1ac69f49eb875/2.29.1/centos7-native/da39a3e
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/gcc-5-glibc-2.23/9bc6787
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.2.3/gcc-5-glibc-2.23/65372bd
+20
View File
@@ -0,0 +1,20 @@
# shellcheck disable=SC2148
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.17/gcc-4.8.1-glibc-2.17/99df8fc
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51
NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-4.8.1-glibc-2.17/675d945
TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.8.1-glibc-2.17/c3f970a
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e
+20
View File
@@ -0,0 +1,20 @@
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
LIBURING_BASE=/mnt/gvfs/third-party2/liburing/79427253fd0d42677255aacfe6d13bfe63f752eb/20190828/platform007/ca4da3d
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
+1 -1
View File
@@ -18,4 +18,4 @@ KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/32b8a2407b634df3f8f948
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/08634589372fa5f237bfd374e8c644a8364e78c1/2.32/platform009/ba86d1f/
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/6ae525939ad02e5e676855082fbbc7828dbafeac/3.15.0/platform009/7f3b187
LUA_BASE=/mnt/gvfs/third-party2/lua/162efd9561a3d21f6869f4814011e9cf1b3ff4dc/5.3.4/platform009/a6271c4
BENCHMARK_BASE=/mnt/gvfs/third-party2/benchmark/30bf49ad6414325e17f3425b0edcb64239427ae3/1.6.1/platform009/7f3b187
BENCHMARK_BASE=/mnt/gvfs/third-party2/benchmark/bce8d9564eaf161700aa3a20b1051564acf555fb/1.5.5/platform009/7f3b187
+120
View File
@@ -0,0 +1,120 @@
#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# Set environment variables so that we can compile rocksdb using
# fbcode settings. It uses the latest g++ compiler and also
# uses jemalloc
BASEDIR=`dirname $BASH_SOURCE`
source "$BASEDIR/dependencies_4.8.1.sh"
# location of libgcc
LIBGCC_INCLUDE="$LIBGCC_BASE/include"
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
# location of glibc
GLIBC_INCLUDE="$GLIBC_BASE/include"
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
# location of snappy headers and libraries
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include"
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $ZLIB_BASE/include"
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
# location of bzip headers and libraries
BZIP2_INCLUDE=" -I $BZIP2_BASE/include/"
BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a"
LZ4_INCLUDE=" -I $LZ4_BASE/include"
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
ZSTD_INCLUDE=" -I $ZSTD_BASE/include"
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
# location of jemalloc
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include"
JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a"
# location of numa
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
# location of libunwind
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
# location of tbb
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
test "$USE_SSE" || USE_SSE=1
export USE_SSE
test "$PORTABLE" || PORTABLE=1
export PORTABLE
BINUTILS="$BINUTILS_BASE/bin"
AR="$BINUTILS/ar"
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
STDLIBS="-L $GCC_BASE/lib64"
if [ -z "$USE_CLANG" ]; then
# gcc
CC="$GCC_BASE/bin/gcc"
CXX="$GCC_BASE/bin/g++"
CXX="$GCC_BASE/bin/gcc-ar"
CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
JEMALLOC=1
else
# clang
CLANG_BIN="$CLANG_BASE/bin"
CLANG_LIB="$CLANG_BASE/lib"
CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
CC="$CLANG_BIN/clang"
CXX="$CLANG_BIN/clang++"
AR="$CLANG_BIN/llvm-ar"
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib"
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux "
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
CFLAGS+=" -isystem $CLANG_INCLUDE"
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
CXXFLAGS="-nostdinc++"
fi
CFLAGS+=" $DEPS_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
CXXFLAGS+=" $CFLAGS"
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
EXEC_LDFLAGS+=" $LIBUNWIND"
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"
# required by libtbb
EXEC_LDFLAGS+=" -ldl"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
VALGRIND_VER="$VALGRIND_BASE/bin/"
LUA_PATH="$LUA_BASE"
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH
+170
View File
@@ -0,0 +1,170 @@
#!/bin/sh
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# Set environment variables so that we can compile rocksdb using
# fbcode settings. It uses the latest g++ and clang compilers and also
# uses jemalloc
# Environment variables that change the behavior of this script:
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
BASEDIR=`dirname $BASH_SOURCE`
source "$BASEDIR/dependencies_platform007.sh"
CFLAGS=""
# libgcc
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
# glibc
GLIBC_INCLUDE="$GLIBC_BASE/include"
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
# snappy
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
if test -z $PIC_BUILD; then
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
else
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
fi
CFLAGS+=" -DSNAPPY"
if test -z $PIC_BUILD; then
# location of zlib headers and libraries
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
CFLAGS+=" -DZLIB"
# location of bzip headers and libraries
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
CFLAGS+=" -DBZIP2"
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
CFLAGS+=" -DLZ4"
fi
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
if test -z $PIC_BUILD; then
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
else
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
fi
CFLAGS+=" -DZSTD"
# location of gflags headers and libraries
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
if test -z $PIC_BUILD; then
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
else
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
fi
CFLAGS+=" -DGFLAGS=gflags"
# location of jemalloc
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
if test -z $PIC_BUILD; then
# location of numa
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
CFLAGS+=" -DNUMA"
# location of libunwind
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
fi
# location of TBB
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
if test -z $PIC_BUILD; then
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
else
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
fi
CFLAGS+=" -DTBB"
# location of LIBURING
LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
if test -z $PIC_BUILD; then
LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
else
LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
fi
CFLAGS+=" -DLIBURING"
test "$USE_SSE" || USE_SSE=1
export USE_SSE
test "$PORTABLE" || PORTABLE=1
export PORTABLE
BINUTILS="$BINUTILS_BASE/bin"
AR="$BINUTILS/ar"
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE"
STDLIBS="-L $GCC_BASE/lib64"
CLANG_BIN="$CLANG_BASE/bin"
CLANG_LIB="$CLANG_BASE/lib"
CLANG_SRC="$CLANG_BASE/../../src"
CLANG_ANALYZER="$CLANG_BIN/clang++"
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
if [ -z "$USE_CLANG" ]; then
# gcc
CC="$GCC_BASE/bin/gcc"
CXX="$GCC_BASE/bin/g++"
AR="$GCC_BASE/bin/gcc-ar"
CFLAGS+=" -B$BINUTILS/gold"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
CFLAGS+=" -isystem $GLIBC_INCLUDE"
JEMALLOC=1
else
# clang
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
CC="$CLANG_BIN/clang"
CXX="$CLANG_BIN/clang++"
AR="$CLANG_BIN/llvm-ar"
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
CFLAGS+=" -isystem $GLIBC_INCLUDE"
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
CFLAGS+=" -isystem $CLANG_INCLUDE"
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
CFLAGS+=" -Wno-expansion-to-defined "
CXXFLAGS="-nostdinc++"
fi
CFLAGS+=" $DEPS_INCLUDE"
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
CXXFLAGS+=" $CFLAGS"
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
EXEC_LDFLAGS+=" $LIBUNWIND"
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
# required by libtbb
EXEC_LDFLAGS+=" -ldl"
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS"
VALGRIND_VER="$VALGRIND_BASE/bin/"
# lua not supported because it's on track for deprecation, I think
LUA_PATH=
LUA_LIB=
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
+209
View File
@@ -0,0 +1,209 @@
#!/usr/bin/env python2.7
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import argparse
import commands
import subprocess
import sys
import re
import os
import time
#
# Simple logger
#
class Log:
def __init__(self, filename):
self.filename = filename
self.f = open(self.filename, 'w+', 0)
def caption(self, str):
line = "\n##### %s #####\n" % str
if self.f:
self.f.write("%s \n" % line)
else:
print(line)
def error(self, str):
data = "\n\n##### ERROR ##### %s" % str
if self.f:
self.f.write("%s \n" % data)
else:
print(data)
def log(self, str):
if self.f:
self.f.write("%s \n" % str)
else:
print(str)
#
# Shell Environment
#
class Env(object):
def __init__(self, logfile, tests):
self.tests = tests
self.log = Log(logfile)
def shell(self, cmd, path=os.getcwd()):
if path:
os.chdir(path)
self.log.log("==== shell session ===========================")
self.log.log("%s> %s" % (path, cmd))
status = subprocess.call("cd %s; %s" % (path, cmd), shell=True,
stdout=self.log.f, stderr=self.log.f)
self.log.log("status = %s" % status)
self.log.log("============================================== \n\n")
return status
def GetOutput(self, cmd, path=os.getcwd()):
if path:
os.chdir(path)
self.log.log("==== shell session ===========================")
self.log.log("%s> %s" % (path, cmd))
status, out = commands.getstatusoutput(cmd)
self.log.log("status = %s" % status)
self.log.log("out = %s" % out)
self.log.log("============================================== \n\n")
return status, out
#
# Pre-commit checker
#
class PreCommitChecker(Env):
def __init__(self, args):
Env.__init__(self, args.logfile, args.tests)
self.ignore_failure = args.ignore_failure
#
# Get commands for a given job from the determinator file
#
def get_commands(self, test):
status, out = self.GetOutput(
"RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
return status, out
#
# Run a specific CI job
#
def run_test(self, test):
self.log.caption("Running test %s locally" % test)
# get commands for the CI job determinator
status, cmds = self.get_commands(test)
if status != 0:
self.log.error("Error getting commands for test %s" % test)
return False
# Parse the JSON to extract the commands to run
cmds = re.findall("'shell':'([^\']*)'", cmds)
if len(cmds) == 0:
self.log.log("No commands found")
return False
# Run commands
for cmd in cmds:
# Replace J=<..> with the local environment variable
if "J" in os.environ:
cmd = cmd.replace("J=1", "J=%s" % os.environ["J"])
cmd = cmd.replace("make ", "make -j%s " % os.environ["J"])
# Run the command
status = self.shell(cmd, ".")
if status != 0:
self.log.error("Error running command %s for test %s"
% (cmd, test))
return False
return True
#
# Run specified CI jobs
#
def run_tests(self):
if not self.tests:
self.log.error("Invalid args. Please provide tests")
return False
self.print_separator()
self.print_row("TEST", "RESULT")
self.print_separator()
result = True
for test in self.tests:
start_time = time.time()
self.print_test(test)
result = self.run_test(test)
elapsed_min = (time.time() - start_time) / 60
if not result:
self.log.error("Error running test %s" % test)
self.print_result("FAIL (%dm)" % elapsed_min)
if not self.ignore_failure:
return False
result = False
else:
self.print_result("PASS (%dm)" % elapsed_min)
self.print_separator()
return result
#
# Print a line
#
def print_separator(self):
print("".ljust(60, "-"))
#
# Print two colums
#
def print_row(self, c0, c1):
print("%s%s" % (c0.ljust(40), c1.ljust(20)))
def print_test(self, test):
print(test.ljust(40), end="")
sys.stdout.flush()
def print_result(self, result):
print(result.ljust(20))
#
# Main
#
parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')
# --log <logfile>
parser.add_argument('--logfile', default='/tmp/precommit-check.log',
help='Log file. Default is /tmp/precommit-check.log')
# --ignore_failure
parser.add_argument('--ignore_failure', action='store_true', default=False,
help='Stop when an error occurs')
# <test ....>
parser.add_argument('tests', nargs='+',
help='CI test(s) to run. e.g: unit punit asan tsan ubsan')
args = parser.parse_args()
checker = PreCommitChecker(args)
print("Please follow log %s" % checker.log.filename)
if not checker.run_tests():
print("Error running tests. Please check log file %s"
% checker.log.filename)
sys.exit(1)
sys.exit(0)
+1365
View File
File diff suppressed because it is too large. Load diff
+3 -1
View File
@@ -42,7 +42,7 @@ $RunOnly.Add("c_test") | Out-Null
$RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
$RunOnly.Add("merge_test") | Out-Null
$RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
$RunOnly.Add("backup_engine_test") | Out-Null # Disabled
$RunOnly.Add("backupable_db_test") | Out-Null # Disabled
$RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest
if($RunAll -and $SuiteRun -ne "") {
@@ -491,3 +491,5 @@ if(!$script:success) {
}
exit 0
-2
View File
@@ -21,7 +21,6 @@ std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
"WriteBuffer",
"CompressionDictionaryBuildingBuffer",
"FilterConstruction",
"BlockBasedTableReader",
"Misc",
}};
@@ -35,7 +34,6 @@ std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
"write-buffer",
"compression-dictionary-building-buffer",
"filter-construction",
"block-based-table-reader",
"misc",
}};
-3
View File
@@ -39,9 +39,6 @@ enum class CacheEntryRole {
// Filter reservations to account for
// (new) bloom and ribbon filter construction's memory usage
kFilterConstruction,
// BlockBasedTableReader reservations to account for
// its memory usage
kBlockBasedTableReader,
// Default bucket, for miscellaneous cache entries. Do not use for
// entries that could potentially add up to large usage.
kMisc,
+64 -59
View File
@@ -17,30 +17,12 @@
#include "rocksdb/cache.h"
#include "rocksdb/slice.h"
#include "rocksdb/status.h"
#include "table/block_based/reader_common.h"
#include "table/block_based/block_based_table_reader.h"
#include "util/coding.h"
namespace ROCKSDB_NAMESPACE {
template <CacheEntryRole R>
CacheReservationManagerImpl<R>::CacheReservationHandle::CacheReservationHandle(
std::size_t incremental_memory_used,
std::shared_ptr<CacheReservationManagerImpl> cache_res_mgr)
: incremental_memory_used_(incremental_memory_used) {
assert(cache_res_mgr);
cache_res_mgr_ = cache_res_mgr;
}
template <CacheEntryRole R>
CacheReservationManagerImpl<
R>::CacheReservationHandle::~CacheReservationHandle() {
Status s = cache_res_mgr_->ReleaseCacheReservation(incremental_memory_used_);
s.PermitUncheckedError();
}
template <CacheEntryRole R>
CacheReservationManagerImpl<R>::CacheReservationManagerImpl(
std::shared_ptr<Cache> cache, bool delayed_decrease)
CacheReservationManager::CacheReservationManager(std::shared_ptr<Cache> cache,
bool delayed_decrease)
: delayed_decrease_(delayed_decrease),
cache_allocated_size_(0),
memory_used_(0) {
@@ -48,15 +30,14 @@ CacheReservationManagerImpl<R>::CacheReservationManagerImpl(
cache_ = cache;
}
template <CacheEntryRole R>
CacheReservationManagerImpl<R>::~CacheReservationManagerImpl() {
CacheReservationManager::~CacheReservationManager() {
for (auto* handle : dummy_handles_) {
cache_->Release(handle, true);
}
}
template <CacheEntryRole R>
Status CacheReservationManagerImpl<R>::UpdateCacheReservation(
Status CacheReservationManager::UpdateCacheReservation(
std::size_t new_mem_used) {
memory_used_ = new_mem_used;
std::size_t cur_cache_allocated_size =
@@ -64,7 +45,7 @@ Status CacheReservationManagerImpl<R>::UpdateCacheReservation(
if (new_mem_used == cur_cache_allocated_size) {
return Status::OK();
} else if (new_mem_used > cur_cache_allocated_size) {
Status s = IncreaseCacheReservation(new_mem_used);
Status s = IncreaseCacheReservation<R>(new_mem_used);
return s;
} else {
// In delayed decrease mode, we don't decrease cache reservation
@@ -85,32 +66,41 @@ Status CacheReservationManagerImpl<R>::UpdateCacheReservation(
}
}
// Explicitly instantiate templates for "CacheEntryRole" values we use.
// This makes it possible to keep the template definitions in the .cc file.
template Status CacheReservationManager::UpdateCacheReservation<
CacheEntryRole::kWriteBuffer>(std::size_t new_mem_used);
template Status CacheReservationManager::UpdateCacheReservation<
CacheEntryRole::kCompressionDictionaryBuildingBuffer>(
std::size_t new_mem_used);
// For cache reservation manager unit tests
template Status CacheReservationManager::UpdateCacheReservation<
CacheEntryRole::kMisc>(std::size_t new_mem_used);
template <CacheEntryRole R>
Status CacheReservationManagerImpl<R>::MakeCacheReservation(
Status CacheReservationManager::MakeCacheReservation(
std::size_t incremental_memory_used,
std::unique_ptr<CacheReservationManager::CacheReservationHandle>* handle) {
assert(handle);
std::unique_ptr<CacheReservationHandle<R>>* handle) {
assert(handle != nullptr);
Status s =
UpdateCacheReservation(GetTotalMemoryUsed() + incremental_memory_used);
(*handle).reset(new CacheReservationManagerImpl::CacheReservationHandle(
incremental_memory_used,
std::enable_shared_from_this<
CacheReservationManagerImpl<R>>::shared_from_this()));
UpdateCacheReservation<R>(GetTotalMemoryUsed() + incremental_memory_used);
(*handle).reset(new CacheReservationHandle<R>(incremental_memory_used,
shared_from_this()));
return s;
}
template <CacheEntryRole R>
Status CacheReservationManagerImpl<R>::ReleaseCacheReservation(
std::size_t incremental_memory_used) {
assert(GetTotalMemoryUsed() >= incremental_memory_used);
std::size_t updated_total_mem_used =
GetTotalMemoryUsed() - incremental_memory_used;
Status s = UpdateCacheReservation(updated_total_mem_used);
return s;
}
template Status
CacheReservationManager::MakeCacheReservation<CacheEntryRole::kMisc>(
std::size_t incremental_memory_used,
std::unique_ptr<CacheReservationHandle<CacheEntryRole::kMisc>>* handle);
template Status CacheReservationManager::MakeCacheReservation<
CacheEntryRole::kFilterConstruction>(
std::size_t incremental_memory_used,
std::unique_ptr<
CacheReservationHandle<CacheEntryRole::kFilterConstruction>>* handle);
template <CacheEntryRole R>
Status CacheReservationManagerImpl<R>::IncreaseCacheReservation(
Status CacheReservationManager::IncreaseCacheReservation(
std::size_t new_mem_used) {
Status return_status = Status::OK();
while (new_mem_used > cache_allocated_size_.load(std::memory_order_relaxed)) {
@@ -128,8 +118,7 @@ Status CacheReservationManagerImpl<R>::IncreaseCacheReservation(
return return_status;
}
template <CacheEntryRole R>
Status CacheReservationManagerImpl<R>::DecreaseCacheReservation(
Status CacheReservationManager::DecreaseCacheReservation(
std::size_t new_mem_used) {
Status return_status = Status::OK();
@@ -148,18 +137,15 @@ Status CacheReservationManagerImpl<R>::DecreaseCacheReservation(
return return_status;
}
template <CacheEntryRole R>
std::size_t CacheReservationManagerImpl<R>::GetTotalReservedCacheSize() {
std::size_t CacheReservationManager::GetTotalReservedCacheSize() {
return cache_allocated_size_.load(std::memory_order_relaxed);
}
template <CacheEntryRole R>
std::size_t CacheReservationManagerImpl<R>::GetTotalMemoryUsed() {
std::size_t CacheReservationManager::GetTotalMemoryUsed() {
return memory_used_;
}
template <CacheEntryRole R>
Slice CacheReservationManagerImpl<R>::GetNextCacheKey() {
Slice CacheReservationManager::GetNextCacheKey() {
// Calling this function will have the side-effect of changing the
// underlying cache_key_ that is shared among other keys generated from this
// fucntion. Therefore please make sure the previous keys are saved/copied
@@ -169,15 +155,34 @@ Slice CacheReservationManagerImpl<R>::GetNextCacheKey() {
}
template <CacheEntryRole R>
Cache::DeleterFn CacheReservationManagerImpl<R>::TEST_GetNoopDeleterForRole() {
Cache::DeleterFn CacheReservationManager::TEST_GetNoopDeleterForRole() {
return GetNoopDeleterForRole<R>();
}
template class CacheReservationManagerImpl<
CacheEntryRole::kBlockBasedTableReader>;
template class CacheReservationManagerImpl<
CacheEntryRole::kCompressionDictionaryBuildingBuffer>;
template class CacheReservationManagerImpl<CacheEntryRole::kFilterConstruction>;
template class CacheReservationManagerImpl<CacheEntryRole::kMisc>;
template class CacheReservationManagerImpl<CacheEntryRole::kWriteBuffer>;
template Cache::DeleterFn CacheReservationManager::TEST_GetNoopDeleterForRole<
CacheEntryRole::kFilterConstruction>();
template <CacheEntryRole R>
CacheReservationHandle<R>::CacheReservationHandle(
std::size_t incremental_memory_used,
std::shared_ptr<CacheReservationManager> cache_res_mgr)
: incremental_memory_used_(incremental_memory_used) {
assert(cache_res_mgr != nullptr);
cache_res_mgr_ = cache_res_mgr;
}
template <CacheEntryRole R>
CacheReservationHandle<R>::~CacheReservationHandle() {
assert(cache_res_mgr_ != nullptr);
assert(cache_res_mgr_->GetTotalMemoryUsed() >= incremental_memory_used_);
Status s = cache_res_mgr_->UpdateCacheReservation<R>(
cache_res_mgr_->GetTotalMemoryUsed() - incremental_memory_used_);
s.PermitUncheckedError();
}
// Explicitly instantiate templates for "CacheEntryRole" values we use.
// This makes it possible to keep the template definitions in the .cc file.
template class CacheReservationHandle<CacheEntryRole::kMisc>;
template class CacheReservationHandle<CacheEntryRole::kFilterConstruction>;
} // namespace ROCKSDB_NAMESPACE
+49 -146
View File
@@ -13,90 +13,54 @@
#include <cstddef>
#include <cstdint>
#include <memory>
#include <mutex>
#include <vector>
#include "cache/cache_entry_roles.h"
#include "cache/cache_key.h"
#include "rocksdb/cache.h"
#include "rocksdb/slice.h"
#include "rocksdb/status.h"
#include "table/block_based/block_based_table_reader.h"
#include "util/coding.h"
namespace ROCKSDB_NAMESPACE {
// CacheReservationManager is an interface for reserving cache space for the
// memory used
class CacheReservationManager {
public:
// CacheReservationHandle is for managing the lifetime of a cache reservation
// for an incremental amount of memory used (i.e, incremental_memory_used)
class CacheReservationHandle {
public:
virtual ~CacheReservationHandle() {}
};
virtual ~CacheReservationManager() {}
virtual Status UpdateCacheReservation(std::size_t new_memory_used) = 0;
virtual Status MakeCacheReservation(
std::size_t incremental_memory_used,
std::unique_ptr<CacheReservationManager::CacheReservationHandle>
*handle) = 0;
virtual std::size_t GetTotalReservedCacheSize() = 0;
virtual std::size_t GetTotalMemoryUsed() = 0;
};
// CacheReservationManagerImpl implements interface CacheReservationManager
// for reserving cache space for the memory used by inserting/releasing dummy
// entries in the cache.
template <CacheEntryRole R>
class CacheReservationHandle;
// CacheReservationManager is for reserving cache space for the memory used
// through inserting/releasing dummy entries in the cache.
//
// This class is NOT thread-safe, except that GetTotalReservedCacheSize()
// can be called without external synchronization.
template <CacheEntryRole R>
class CacheReservationManagerImpl
: public CacheReservationManager,
public std::enable_shared_from_this<CacheReservationManagerImpl<R>> {
class CacheReservationManager
: public std::enable_shared_from_this<CacheReservationManager> {
public:
class CacheReservationHandle
: public CacheReservationManager::CacheReservationHandle {
public:
CacheReservationHandle(
std::size_t incremental_memory_used,
std::shared_ptr<CacheReservationManagerImpl> cache_res_mgr);
~CacheReservationHandle() override;
private:
std::size_t incremental_memory_used_;
std::shared_ptr<CacheReservationManagerImpl> cache_res_mgr_;
};
// Construct a CacheReservationManagerImpl
// Construct a CacheReservationManager
// @param cache The cache where dummy entries are inserted and released for
// reserving cache space
// @param delayed_decrease If set true, then dummy entries won't be released
// immediately when memory usage decreases.
// immediately when memory usage decreases.
// Instead, it will be released when the memory usage
// decreases to 3/4 of what we have reserved so far.
// This is for saving some future dummy entry
// insertion when memory usage increases are likely to
// happen in the near future.
//
// REQUIRED: cache is not nullptr
explicit CacheReservationManagerImpl(std::shared_ptr<Cache> cache,
bool delayed_decrease = false);
explicit CacheReservationManager(std::shared_ptr<Cache> cache,
bool delayed_decrease = false);
// no copy constructor, copy assignment, move constructor, move assignment
CacheReservationManagerImpl(const CacheReservationManagerImpl &) = delete;
CacheReservationManagerImpl &operator=(const CacheReservationManagerImpl &) =
delete;
CacheReservationManagerImpl(CacheReservationManagerImpl &&) = delete;
CacheReservationManagerImpl &operator=(CacheReservationManagerImpl &&) =
delete;
CacheReservationManager(const CacheReservationManager &) = delete;
CacheReservationManager &operator=(const CacheReservationManager &) = delete;
CacheReservationManager(CacheReservationManager &&) = delete;
CacheReservationManager &operator=(CacheReservationManager &&) = delete;
~CacheReservationManagerImpl() override;
~CacheReservationManager();
// One of the two ways of reserving/releasing cache space,
// see MakeCacheReservation() for the other.
//
// Use ONLY one of these two ways to prevent unexpected behavior.
template <CacheEntryRole R>
// One of the two ways of reserving/releasing cache,
// see CacheReservationManager::MakeCacheReservation() for the other.
// Use ONLY one of them to prevent unexpected behavior.
//
// Insert and release dummy entries in the cache to
// match the size of total dummy entries with the least multiple of
@@ -126,13 +90,11 @@ class CacheReservationManagerImpl
// Otherwise, it returns the first non-ok status;
// On releasing dummy entries, it always returns Status::OK().
// On keeping dummy entries the same, it always returns Status::OK().
Status UpdateCacheReservation(std::size_t new_memory_used) override;
Status UpdateCacheReservation(std::size_t new_memory_used);
// One of the two ways of reserving cache space and releasing is done through
// destruction of CacheReservationHandle.
// See UpdateCacheReservation() for the other way.
//
// Use ONLY one of these two ways to prevent unexpected behavior.
// One of the two ways of reserving/releasing cache,
// see CacheReservationManager::UpdateCacheReservation() for the other.
// Use ONLY one of them to prevent unexpected behavior.
//
// Insert dummy entries in the cache for the incremental memory usage
// to match the size of total dummy entries with the least multiple of
@@ -156,19 +118,21 @@ class CacheReservationManagerImpl
// calling MakeCacheReservation() is needed if you want
// GetTotalMemoryUsed() indeed returns the latest memory used.
//
// @param handle An pointer to std::unique_ptr<CacheReservationHandle> that
// manages the lifetime of the cache reservation represented by the
// handle.
// @param handle An pointer to std::unique_ptr<CacheReservationHandle<R>> that
// manages the lifetime of the handle and its cache reservation.
//
// @return It returns Status::OK() if all dummy
// entry insertions succeed.
// Otherwise, it returns the first non-ok status;
//
// REQUIRES: handle != nullptr
// REQUIRES: The CacheReservationManager object is NOT managed by
// std::unique_ptr as CacheReservationHandle needs to
// shares ownership to the CacheReservationManager object.
template <CacheEntryRole R>
Status MakeCacheReservation(
std::size_t incremental_memory_used,
std::unique_ptr<CacheReservationManager::CacheReservationHandle> *handle)
override;
std::unique_ptr<CacheReservationHandle<R>> *handle);
// Return the size of the cache (which is a multiple of kSizeDummyEntry)
// successfully reserved by calling UpdateCacheReservation().
@@ -178,25 +142,25 @@ class CacheReservationManagerImpl
// smaller number than the actual reserved cache size due to
// the returned number will always be a multiple of kSizeDummyEntry
// and cache full might happen in the middle of inserting a dummy entry.
std::size_t GetTotalReservedCacheSize() override;
std::size_t GetTotalReservedCacheSize();
// Return the latest total memory used indicated by the most recent call of
// UpdateCacheReservation(std::size_t new_memory_used);
std::size_t GetTotalMemoryUsed() override;
std::size_t GetTotalMemoryUsed();
static constexpr std::size_t GetDummyEntrySize() { return kSizeDummyEntry; }
// For testing only - it is to help ensure the NoopDeleterForRole<R>
// accessed from CacheReservationManagerImpl and the one accessed from the
// test are from the same translation units
// accessed from CacheReservationManager and the one accessed from the test
// are from the same translation units
template <CacheEntryRole R>
static Cache::DeleterFn TEST_GetNoopDeleterForRole();
private:
static constexpr std::size_t kSizeDummyEntry = 256 * 1024;
Slice GetNextCacheKey();
Status ReleaseCacheReservation(std::size_t incremental_memory_used);
template <CacheEntryRole R>
Status IncreaseCacheReservation(std::size_t new_mem_used);
Status DecreaseCacheReservation(std::size_t new_mem_used);
@@ -208,81 +172,20 @@ class CacheReservationManagerImpl
CacheKey cache_key_;
};
class ConcurrentCacheReservationManager
: public CacheReservationManager,
public std::enable_shared_from_this<ConcurrentCacheReservationManager> {
// CacheReservationHandle is for managing the lifetime of a cache reservation
// This class is NOT thread-safe
template <CacheEntryRole R>
class CacheReservationHandle {
public:
class CacheReservationHandle
: public CacheReservationManager::CacheReservationHandle {
public:
CacheReservationHandle(
std::shared_ptr<ConcurrentCacheReservationManager> cache_res_mgr,
std::unique_ptr<CacheReservationManager::CacheReservationHandle>
cache_res_handle) {
assert(cache_res_mgr && cache_res_handle);
cache_res_mgr_ = cache_res_mgr;
cache_res_handle_ = std::move(cache_res_handle);
}
~CacheReservationHandle() override {
std::lock_guard<std::mutex> lock(cache_res_mgr_->cache_res_mgr_mu_);
cache_res_handle_.reset();
}
private:
std::shared_ptr<ConcurrentCacheReservationManager> cache_res_mgr_;
std::unique_ptr<CacheReservationManager::CacheReservationHandle>
cache_res_handle_;
};
explicit ConcurrentCacheReservationManager(
std::shared_ptr<CacheReservationManager> cache_res_mgr) {
cache_res_mgr_ = std::move(cache_res_mgr);
}
ConcurrentCacheReservationManager(const ConcurrentCacheReservationManager &) =
delete;
ConcurrentCacheReservationManager &operator=(
const ConcurrentCacheReservationManager &) = delete;
ConcurrentCacheReservationManager(ConcurrentCacheReservationManager &&) =
delete;
ConcurrentCacheReservationManager &operator=(
ConcurrentCacheReservationManager &&) = delete;
~ConcurrentCacheReservationManager() override {}
inline Status UpdateCacheReservation(std::size_t new_memory_used) override {
std::lock_guard<std::mutex> lock(cache_res_mgr_mu_);
return cache_res_mgr_->UpdateCacheReservation(new_memory_used);
}
inline Status MakeCacheReservation(
// REQUIRES: cache_res_mgr != nullptr
explicit CacheReservationHandle(
std::size_t incremental_memory_used,
std::unique_ptr<CacheReservationManager::CacheReservationHandle> *handle)
override {
std::unique_ptr<CacheReservationManager::CacheReservationHandle>
wrapped_handle;
Status s;
{
std::lock_guard<std::mutex> lock(cache_res_mgr_mu_);
s = cache_res_mgr_->MakeCacheReservation(incremental_memory_used,
&wrapped_handle);
}
(*handle).reset(
new ConcurrentCacheReservationManager::CacheReservationHandle(
std::enable_shared_from_this<
ConcurrentCacheReservationManager>::shared_from_this(),
std::move(wrapped_handle)));
return s;
}
inline std::size_t GetTotalReservedCacheSize() override {
return cache_res_mgr_->GetTotalReservedCacheSize();
}
inline std::size_t GetTotalMemoryUsed() override {
std::lock_guard<std::mutex> lock(cache_res_mgr_mu_);
return cache_res_mgr_->GetTotalMemoryUsed();
}
std::shared_ptr<CacheReservationManager> cache_res_mgr);
~CacheReservationHandle();
private:
std::mutex cache_res_mgr_mu_;
std::size_t incremental_memory_used_;
std::shared_ptr<CacheReservationManager> cache_res_mgr_;
};
} // namespace ROCKSDB_NAMESPACE
+83 -44
View File
@@ -15,6 +15,7 @@
#include "cache/cache_entry_roles.h"
#include "rocksdb/cache.h"
#include "rocksdb/slice.h"
#include "table/block_based/block_based_table_reader.h"
#include "test_util/testharness.h"
#include "util/coding.h"
@@ -22,24 +23,25 @@ namespace ROCKSDB_NAMESPACE {
class CacheReservationManagerTest : public ::testing::Test {
protected:
static constexpr std::size_t kSizeDummyEntry =
CacheReservationManagerImpl<CacheEntryRole::kMisc>::GetDummyEntrySize();
CacheReservationManager::GetDummyEntrySize();
static constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry;
static constexpr int kNumShardBits = 0; // 2^0 shard
static constexpr std::size_t kMetaDataChargeOverhead = 10000;
std::shared_ptr<Cache> cache = NewLRUCache(kCacheCapacity, kNumShardBits);
std::shared_ptr<CacheReservationManager> test_cache_rev_mng;
std::unique_ptr<CacheReservationManager> test_cache_rev_mng;
CacheReservationManagerTest() {
test_cache_rev_mng =
std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
cache);
test_cache_rev_mng.reset(new CacheReservationManager(cache));
}
};
TEST_F(CacheReservationManagerTest, GenerateCacheKey) {
std::size_t new_mem_used = 1 * kSizeDummyEntry;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
ASSERT_EQ(s, Status::OK());
ASSERT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry);
ASSERT_LT(cache->GetPinnedUsage(),
@@ -64,7 +66,10 @@ TEST_F(CacheReservationManagerTest, GenerateCacheKey) {
TEST_F(CacheReservationManagerTest, KeepCacheReservationTheSame) {
std::size_t new_mem_used = 1 * kSizeDummyEntry;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
ASSERT_EQ(s, Status::OK());
ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
1 * kSizeDummyEntry);
@@ -74,7 +79,9 @@ TEST_F(CacheReservationManagerTest, KeepCacheReservationTheSame) {
ASSERT_LT(initial_pinned_usage,
1 * kSizeDummyEntry + kMetaDataChargeOverhead);
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK())
<< "Failed to keep cache reservation the same when new_mem_used equals "
"to current cache reservation";
@@ -93,7 +100,10 @@ TEST_F(CacheReservationManagerTest, KeepCacheReservationTheSame) {
TEST_F(CacheReservationManagerTest,
IncreaseCacheReservationByMultiplesOfDummyEntrySize) {
std::size_t new_mem_used = 2 * kSizeDummyEntry;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK())
<< "Failed to increase cache reservation correctly";
EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
@@ -111,7 +121,10 @@ TEST_F(CacheReservationManagerTest,
TEST_F(CacheReservationManagerTest,
IncreaseCacheReservationNotByMultiplesOfDummyEntrySize) {
std::size_t new_mem_used = 2 * kSizeDummyEntry + kSizeDummyEntry / 2;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK())
<< "Failed to increase cache reservation correctly";
EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
@@ -130,7 +143,7 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
IncreaseCacheReservationOnFullCache) {
;
constexpr std::size_t kSizeDummyEntry =
CacheReservationManagerImpl<CacheEntryRole::kMisc>::GetDummyEntrySize();
CacheReservationManager::GetDummyEntrySize();
constexpr std::size_t kSmallCacheCapacity = 4 * kSizeDummyEntry;
constexpr std::size_t kBigCacheCapacity = 4096 * kSizeDummyEntry;
constexpr std::size_t kMetaDataChargeOverhead = 10000;
@@ -140,12 +153,14 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
lo.num_shard_bits = 0; // 2^0 shard
lo.strict_capacity_limit = true;
std::shared_ptr<Cache> cache = NewLRUCache(lo);
std::shared_ptr<CacheReservationManager> test_cache_rev_mng =
std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
cache);
std::unique_ptr<CacheReservationManager> test_cache_rev_mng(
new CacheReservationManager(cache));
std::size_t new_mem_used = kSmallCacheCapacity + 1;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::Incomplete())
<< "Failed to return status to indicate failure of dummy entry insertion "
"during cache reservation on full cache";
@@ -168,7 +183,9 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
"encountering cache resevation failure due to full cache";
new_mem_used = kSmallCacheCapacity / 2; // 2 dummy entries
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK())
<< "Failed to decrease cache reservation after encountering cache "
"reservation failure due to full cache";
@@ -190,7 +207,9 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
// Create cache full again for subsequent tests
new_mem_used = kSmallCacheCapacity + 1;
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::Incomplete())
<< "Failed to return status to indicate failure of dummy entry insertion "
"during cache reservation on full cache";
@@ -216,7 +235,9 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
// succeed
cache->SetCapacity(kBigCacheCapacity);
new_mem_used = kSmallCacheCapacity + 1;
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK())
<< "Failed to increase cache reservation after increasing cache capacity "
"and mitigating cache full error";
@@ -238,7 +259,10 @@ TEST(CacheReservationManagerIncreaseReservcationOnFullCacheTest,
TEST_F(CacheReservationManagerTest,
DecreaseCacheReservationByMultiplesOfDummyEntrySize) {
std::size_t new_mem_used = 2 * kSizeDummyEntry;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
ASSERT_EQ(s, Status::OK());
ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
2 * kSizeDummyEntry);
@@ -248,7 +272,9 @@ TEST_F(CacheReservationManagerTest,
2 * kSizeDummyEntry + kMetaDataChargeOverhead);
new_mem_used = 1 * kSizeDummyEntry;
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK())
<< "Failed to decrease cache reservation correctly";
EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
@@ -266,7 +292,10 @@ TEST_F(CacheReservationManagerTest,
TEST_F(CacheReservationManagerTest,
DecreaseCacheReservationNotByMultiplesOfDummyEntrySize) {
std::size_t new_mem_used = 2 * kSizeDummyEntry;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
ASSERT_EQ(s, Status::OK());
ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
2 * kSizeDummyEntry);
@@ -276,7 +305,9 @@ TEST_F(CacheReservationManagerTest,
2 * kSizeDummyEntry + kMetaDataChargeOverhead);
new_mem_used = kSizeDummyEntry / 2;
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK())
<< "Failed to decrease cache reservation correctly";
EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
@@ -294,7 +325,7 @@ TEST_F(CacheReservationManagerTest,
TEST(CacheReservationManagerWithDelayedDecreaseTest,
DecreaseCacheReservationWithDelayedDecrease) {
constexpr std::size_t kSizeDummyEntry =
CacheReservationManagerImpl<CacheEntryRole::kMisc>::GetDummyEntrySize();
CacheReservationManager::GetDummyEntrySize();
constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry;
constexpr std::size_t kMetaDataChargeOverhead = 10000;
@@ -302,12 +333,14 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
lo.capacity = kCacheCapacity;
lo.num_shard_bits = 0;
std::shared_ptr<Cache> cache = NewLRUCache(lo);
std::shared_ptr<CacheReservationManager> test_cache_rev_mng =
std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
cache, true /* delayed_decrease */);
std::unique_ptr<CacheReservationManager> test_cache_rev_mng(
new CacheReservationManager(cache, true /* delayed_decrease */));
std::size_t new_mem_used = 8 * kSizeDummyEntry;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
ASSERT_EQ(s, Status::OK());
ASSERT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
8 * kSizeDummyEntry);
@@ -318,7 +351,9 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
8 * kSizeDummyEntry + kMetaDataChargeOverhead);
new_mem_used = 6 * kSizeDummyEntry;
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK()) << "Failed to delay decreasing cache reservation";
EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
8 * kSizeDummyEntry)
@@ -330,7 +365,9 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
<< "Failed to delay decreasing underlying dummy entries in cache";
new_mem_used = 7 * kSizeDummyEntry;
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK()) << "Failed to delay decreasing cache reservation";
EXPECT_EQ(test_cache_rev_mng->GetTotalReservedCacheSize(),
8 * kSizeDummyEntry)
@@ -342,7 +379,9 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
<< "Failed to delay decreasing underlying dummy entries in cache";
new_mem_used = 6 * kSizeDummyEntry - 1;
s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
s = test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
EXPECT_EQ(s, Status::OK())
<< "Failed to decrease cache reservation correctly when new_mem_used < "
"GetTotalReservedCacheSize() * 3 / 4 on delayed decrease mode";
@@ -366,7 +405,7 @@ TEST(CacheReservationManagerWithDelayedDecreaseTest,
TEST(CacheReservationManagerDestructorTest,
ReleaseRemainingDummyEntriesOnDestruction) {
constexpr std::size_t kSizeDummyEntry =
CacheReservationManagerImpl<CacheEntryRole::kMisc>::GetDummyEntrySize();
CacheReservationManager::GetDummyEntrySize();
constexpr std::size_t kCacheCapacity = 4096 * kSizeDummyEntry;
constexpr std::size_t kMetaDataChargeOverhead = 10000;
@@ -375,11 +414,13 @@ TEST(CacheReservationManagerDestructorTest,
lo.num_shard_bits = 0;
std::shared_ptr<Cache> cache = NewLRUCache(lo);
{
std::shared_ptr<CacheReservationManager> test_cache_rev_mng =
std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
cache);
std::unique_ptr<CacheReservationManager> test_cache_rev_mng(
new CacheReservationManager(cache));
std::size_t new_mem_used = 1 * kSizeDummyEntry;
Status s = test_cache_rev_mng->UpdateCacheReservation(new_mem_used);
Status s =
test_cache_rev_mng
->UpdateCacheReservation<ROCKSDB_NAMESPACE::CacheEntryRole::kMisc>(
new_mem_used);
ASSERT_EQ(s, Status::OK());
ASSERT_GE(cache->GetPinnedUsage(), 1 * kSizeDummyEntry);
ASSERT_LT(cache->GetPinnedUsage(),
@@ -401,19 +442,18 @@ TEST(CacheReservationHandleTest, HandleTest) {
std::shared_ptr<Cache> cache = NewLRUCache(lo);
std::shared_ptr<CacheReservationManager> test_cache_rev_mng(
std::make_shared<CacheReservationManagerImpl<CacheEntryRole::kMisc>>(
cache));
std::make_shared<CacheReservationManager>(cache));
std::size_t mem_used = 0;
const std::size_t incremental_mem_used_handle_1 = 1 * kSizeDummyEntry;
const std::size_t incremental_mem_used_handle_2 = 2 * kSizeDummyEntry;
std::unique_ptr<CacheReservationManager::CacheReservationHandle> handle_1,
std::unique_ptr<CacheReservationHandle<CacheEntryRole::kMisc>> handle_1,
handle_2;
// To test consecutive CacheReservationManager::MakeCacheReservation works
// correctly in terms of returning the handle as well as updating cache
// reservation and the latest total memory used
Status s = test_cache_rev_mng->MakeCacheReservation(
Status s = test_cache_rev_mng->MakeCacheReservation<CacheEntryRole::kMisc>(
incremental_mem_used_handle_1, &handle_1);
mem_used = mem_used + incremental_mem_used_handle_1;
ASSERT_EQ(s, Status::OK());
@@ -423,8 +463,8 @@ TEST(CacheReservationHandleTest, HandleTest) {
EXPECT_GE(cache->GetPinnedUsage(), mem_used);
EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead);
s = test_cache_rev_mng->MakeCacheReservation(incremental_mem_used_handle_2,
&handle_2);
s = test_cache_rev_mng->MakeCacheReservation<CacheEntryRole::kMisc>(
incremental_mem_used_handle_2, &handle_2);
mem_used = mem_used + incremental_mem_used_handle_2;
ASSERT_EQ(s, Status::OK());
EXPECT_TRUE(handle_2 != nullptr);
@@ -433,9 +473,8 @@ TEST(CacheReservationHandleTest, HandleTest) {
EXPECT_GE(cache->GetPinnedUsage(), mem_used);
EXPECT_LT(cache->GetPinnedUsage(), mem_used + kMetaDataChargeOverhead);
// To test
// CacheReservationManager::CacheReservationHandle::~CacheReservationHandle()
// works correctly in releasing the cache reserved for the handle
// To test CacheReservationHandle::~CacheReservationHandle() works correctly
// in releasing the cache reserved for the handle
handle_1.reset();
EXPECT_TRUE(handle_1 == nullptr);
mem_used = mem_used - incremental_mem_used_handle_1;
+5 -5
View File
@@ -286,8 +286,8 @@ class ClockCacheShard final : public CacheShard {
return Lookup(key, hash);
}
bool Release(Cache::Handle* handle, bool /*useful*/,
bool erase_if_last_ref) override {
return Release(handle, erase_if_last_ref);
bool force_erase) override {
return Release(handle, force_erase);
}
bool IsReady(Cache::Handle* /*handle*/) override { return true; }
void Wait(Cache::Handle* /*handle*/) override {}
@@ -297,7 +297,7 @@ class ClockCacheShard final : public CacheShard {
//
// Not necessary to hold mutex_ before being called.
bool Ref(Cache::Handle* handle) override;
bool Release(Cache::Handle* handle, bool erase_if_last_ref = false) override;
bool Release(Cache::Handle* handle, bool force_erase = false) override;
void Erase(const Slice& key, uint32_t hash) override;
bool EraseAndConfirm(const Slice& key, uint32_t hash,
CleanupContext* context);
@@ -725,11 +725,11 @@ Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
return reinterpret_cast<Cache::Handle*>(handle);
}
bool ClockCacheShard::Release(Cache::Handle* h, bool erase_if_last_ref) {
bool ClockCacheShard::Release(Cache::Handle* h, bool force_erase) {
CleanupContext context;
CacheHandle* handle = reinterpret_cast<CacheHandle*>(h);
bool erased = Unref(handle, true, &context);
if (erase_if_last_ref && !erased) {
if (force_erase && !erased) {
erased = EraseAndConfirm(handle->key, handle->hash, &context);
}
Cleanup(context);
+36 -35
View File
@@ -76,12 +76,13 @@ LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
void LRUHandleTable::Resize() {
if (length_bits_ >= max_length_bits_) {
// Due to reaching limit of hash information, if we made the table bigger,
// we would allocate more addresses but only the same number would be used.
// Due to reaching limit of hash information, if we made the table
// bigger, we would allocate more addresses but only the same
// number would be used.
return;
}
if (length_bits_ >= 31) {
// Avoid undefined behavior shifting uint32_t by 32.
// Avoid undefined behavior shifting uint32_t by 32
return;
}
@@ -124,7 +125,7 @@ LRUCacheShard::LRUCacheShard(
mutex_(use_adaptive_mutex),
secondary_cache_(secondary_cache) {
set_metadata_charge_policy(metadata_charge_policy);
// Make empty circular linked list.
// Make empty circular linked list
lru_.next = &lru_;
lru_.prev = &lru_;
lru_low_pri_ = &lru_;
@@ -137,7 +138,7 @@ void LRUCacheShard::EraseUnRefEntries() {
MutexLock l(&mutex_);
while (lru_.next != &lru_) {
LRUHandle* old = lru_.next;
// LRU list contains only elements which can be evicted.
// LRU list contains only elements which can be evicted
assert(old->InCache() && !old->HasRefs());
LRU_Remove(old);
table_.Remove(old->key(), old->hash);
@@ -167,7 +168,7 @@ void LRUCacheShard::ApplyToSomeEntries(
assert(average_entries_per_lock > 0);
// Assuming we are called with same average_entries_per_lock repeatedly,
// this simplifies some logic (index_end will not overflow).
// this simplifies some logic (index_end will not overflow)
assert(average_entries_per_lock < length || *state == 0);
uint32_t index_begin = *state >> (32 - length_bits);
@@ -273,7 +274,7 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
autovector<LRUHandle*>* deleted) {
while ((usage_ + charge) > capacity_ && lru_.next != &lru_) {
LRUHandle* old = lru_.next;
// LRU list contains only elements which can be evicted.
// LRU list contains only elements which can be evicted
assert(old->InCache() && !old->HasRefs());
LRU_Remove(old);
table_.Remove(old->key(), old->hash);
@@ -294,8 +295,8 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
EvictFromLRU(0, &last_reference_list);
}
// Try to insert the evicted entries into tiered cache.
// Free the entries outside of mutex for performance reasons.
// Try to insert the evicted entries into tiered cache
// Free the entries outside of mutex for performance reasons
for (auto entry : last_reference_list) {
if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
!entry->IsPromoted()) {
@@ -321,7 +322,7 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
MutexLock l(&mutex_);
// Free the space following strict LRU policy until enough space
// is freed or the lru list is empty.
// is freed or the lru list is empty
EvictFromLRU(total_charge, &last_reference_list);
if ((usage_ + total_charge) > capacity_ &&
@@ -348,7 +349,7 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
assert(old->InCache());
old->SetInCache(false);
if (!old->HasRefs()) {
// old is on LRU because it's in cache and its reference count is 0.
// old is on LRU because it's in cache and its reference count is 0
LRU_Remove(old);
size_t old_total_charge =
old->CalcTotalCharge(metadata_charge_policy_);
@@ -360,7 +361,7 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
if (handle == nullptr) {
LRU_Insert(e);
} else {
// If caller already holds a ref, no need to take one here.
// If caller already holds a ref, no need to take one here
if (!e->HasRefs()) {
e->Ref();
}
@@ -369,8 +370,8 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
}
}
// Try to insert the evicted entries into the secondary cache.
// Free the entries here outside of mutex for performance reasons.
// Try to insert the evicted entries into the secondary cache
// Free the entries here outside of mutex for performance reasons
for (auto entry : last_reference_list) {
if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
!entry->IsPromoted()) {
@@ -403,7 +404,7 @@ void LRUCacheShard::Promote(LRUHandle* e) {
Status s = InsertItem(e, &handle, /*free_handle_on_fail=*/false);
if (!s.ok()) {
// Item is in memory, but not accounted against the cache capacity.
// When the handle is released, the item should get deleted.
// When the handle is released, the item should get deleted
assert(!e->InCache());
}
} else {
@@ -436,8 +437,8 @@ Cache::Handle* LRUCacheShard::Lookup(
}
// If handle table lookup failed, then allocate a handle outside the
// mutex if we're going to lookup in the secondary cache.
// Only support synchronous for now.
// mutex if we're going to lookup in the secondary cache
// Only support synchronous for now
// TODO: Support asynchronous lookup in secondary cache
if (!e && secondary_cache_ && helper && helper->saveto_cb) {
// For objects from the secondary cache, we expect the caller to provide
@@ -468,7 +469,7 @@ Cache::Handle* LRUCacheShard::Lookup(
if (wait) {
Promote(e);
if (!e->value) {
// The secondary cache returned a handle, but the lookup failed.
// The secondary cache returned a handle, but the lookup failed
e->Unref();
e->Free();
e = nullptr;
@@ -478,7 +479,7 @@ Cache::Handle* LRUCacheShard::Lookup(
}
} else {
// If wait is false, we always return a handle and let the caller
// release the handle after checking for success or failure.
// release the handle after checking for success or failure
e->SetIncomplete(true);
// This may be slightly inaccurate, if the lookup eventually fails.
// But the probability is very low.
@@ -493,7 +494,7 @@ Cache::Handle* LRUCacheShard::Lookup(
bool LRUCacheShard::Ref(Cache::Handle* h) {
LRUHandle* e = reinterpret_cast<LRUHandle*>(h);
MutexLock l(&mutex_);
// To create another reference - entry must be already externally referenced.
// To create another reference - entry must be already externally referenced
assert(e->HasRefs());
e->Ref();
return true;
@@ -506,7 +507,7 @@ void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) {
MaintainPoolSize();
}
bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
if (handle == nullptr) {
return false;
}
@@ -516,15 +517,15 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
MutexLock l(&mutex_);
last_reference = e->Unref();
if (last_reference && e->InCache()) {
// The item is still in cache, and nobody else holds a reference to it.
if (usage_ > capacity_ || erase_if_last_ref) {
// The LRU list must be empty since the cache is full.
assert(lru_.next == &lru_ || erase_if_last_ref);
// Take this opportunity and remove the item.
// The item is still in cache, and nobody else holds a reference to it
if (usage_ > capacity_ || force_erase) {
// The LRU list must be empty since the cache is full
assert(lru_.next == &lru_ || force_erase);
// Take this opportunity and remove the item
table_.Remove(e->key(), e->hash);
e->SetInCache(false);
} else {
// Put the item back on the LRU list, and don't free it.
// Put the item back on the LRU list, and don't free it
LRU_Insert(e);
last_reference = false;
}
@@ -541,7 +542,7 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
}
}
// Free the entry here outside of mutex for performance reasons.
// Free the entry here outside of mutex for performance reasons
if (last_reference) {
e->Free();
}
@@ -553,8 +554,8 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
void (*deleter)(const Slice& key, void* value),
const Cache::CacheItemHelper* helper,
Cache::Handle** handle, Cache::Priority priority) {
// Allocate the memory here outside of the mutex.
// If the cache is full, we'll have to release it.
// Allocate the memory here outside of the mutex
// If the cache is full, we'll have to release it
// It shouldn't happen very often though.
LRUHandle* e = reinterpret_cast<LRUHandle*>(
new char[sizeof(LRUHandle) - 1 + key.size()]);
@@ -602,8 +603,8 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
}
}
// Free the entry here outside of mutex for performance reasons.
// last_reference will only be true if e != nullptr.
// Free the entry here outside of mutex for performance reasons
// last_reference will only be true if e != nullptr
if (last_reference) {
e->Free();
}
@@ -704,7 +705,7 @@ uint32_t LRUCache::GetHash(Handle* handle) const {
}
void LRUCache::DisownData() {
// Leak data only if that won't generate an ASAN/valgrind warning.
// Leak data only if that won't generate an ASAN/valgrind warning
if (!kMustFreeHeapAllocations) {
shards_ = nullptr;
num_shards_ = 0;
@@ -764,10 +765,10 @@ std::shared_ptr<Cache> NewLRUCache(
CacheMetadataChargePolicy metadata_charge_policy,
const std::shared_ptr<SecondaryCache>& secondary_cache) {
if (num_shard_bits >= 20) {
return nullptr; // The cache cannot be sharded into too many fine pieces.
return nullptr; // the cache cannot be sharded into too many fine pieces
}
if (high_pri_pool_ratio < 0.0 || high_pri_pool_ratio > 1.0) {
// Invalid high_pri_pool_ratio
// invalid high_pri_pool_ratio
return nullptr;
}
if (num_shard_bits < 0) {
+21 -21
View File
@@ -81,11 +81,11 @@ struct LRUHandle {
IN_HIGH_PRI_POOL = (1 << 2),
// Whether this entry has had any lookups (hits).
HAS_HIT = (1 << 3),
// Can this be inserted into the secondary cache.
// Can this be inserted into the secondary cache
IS_SECONDARY_CACHE_COMPATIBLE = (1 << 4),
// Is the handle still being read from a lower tier.
// Is the handle still being read from a lower tier
IS_PENDING = (1 << 5),
// Has the item been promoted from a lower tier.
// Has the item been promoted from a lower tier
IS_PROMOTED = (1 << 6),
};
@@ -208,7 +208,7 @@ struct LRUHandle {
delete[] reinterpret_cast<char*>(this);
}
// Calculate the memory usage by metadata.
// Calculate the memory usage by metadata
inline size_t CalcTotalCharge(
CacheMetadataChargePolicy metadata_charge_policy) {
size_t meta_charge = 0;
@@ -216,7 +216,7 @@ struct LRUHandle {
#ifdef ROCKSDB_MALLOC_USABLE_SIZE
meta_charge += malloc_usable_size(static_cast<void*>(this));
#else
// This is the size that is used when a new handle is created.
// This is the size that is used when a new handle is created
meta_charge += sizeof(LRUHandle) - 1 + key_length;
#endif
}
@@ -272,10 +272,10 @@ class LRUHandleTable {
// a linked list of cache entries that hash into the bucket.
std::unique_ptr<LRUHandle*[]> list_;
// Number of elements currently in the table.
// Number of elements currently in the table
uint32_t elems_;
// Set from max_upper_hash_bits (see constructor).
// Set from max_upper_hash_bits (see constructor)
const int max_length_bits_;
};
@@ -291,7 +291,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
// Separate from constructor so caller can easily make an array of LRUCache
// if current usage is more than new capacity, the function will attempt to
// free the needed space.
// free the needed space
virtual void SetCapacity(size_t capacity) override;
// Set the flag to reject insertion if cache if full.
@@ -314,7 +314,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
assert(helper);
return Insert(key, hash, value, charge, nullptr, helper, handle, priority);
}
// If helper_cb is null, the values of the following arguments don't matter.
// If helper_cb is null, the values of the following arguments don't
// matter
virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash,
const ShardedCache::CacheItemHelper* helper,
const ShardedCache::CreateCallback& create_cb,
@@ -325,14 +326,14 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
nullptr);
}
virtual bool Release(Cache::Handle* handle, bool /*useful*/,
bool erase_if_last_ref) override {
return Release(handle, erase_if_last_ref);
bool force_erase) override {
return Release(handle, force_erase);
}
virtual bool IsReady(Cache::Handle* /*handle*/) override;
virtual void Wait(Cache::Handle* /*handle*/) override {}
virtual bool Ref(Cache::Handle* handle) override;
virtual bool Release(Cache::Handle* handle,
bool erase_if_last_ref = false) override;
bool force_erase = false) override;
virtual void Erase(const Slice& key, uint32_t hash) override;
// Although in some platforms the update of size_t is atomic, to make sure
@@ -353,8 +354,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);
// Retrieves number of elements in LRU, for unit test purpose only.
// Not threadsafe.
// Retrieves number of elements in LRU, for unit test purpose only
// not threadsafe
size_t TEST_GetLRUSize();
// Retrieves high pri pool ratio
@@ -364,8 +365,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
friend class LRUCache;
// Insert an item into the hash table and, if handle is null, insert into
// the LRU list. Older items are evicted as necessary. If the cache is full
// and free_handle_on_fail is true, the item is deleted and handle is set to
// nullptr.
// and free_handle_on_fail is true, the item is deleted and handle is set to.
Status InsertItem(LRUHandle* item, Cache::Handle** handle,
bool free_handle_on_fail);
Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
@@ -389,7 +389,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
// Free some space following strict LRU policy until enough space
// to hold (usage_ + charge) is freed or the lru list is empty
// This function is not thread safe - it needs to be executed while
// holding the mutex_.
// holding the mutex_
void EvictFromLRU(size_t charge, autovector<LRUHandle*>* deleted);
// Initialized before use.
@@ -429,10 +429,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
// ------------vvvvvvvvvvvvv-----------
LRUHandleTable table_;
// Memory size for entries residing in the cache.
// Memory size for entries residing in the cache
size_t usage_;
// Memory size for entries residing only in the LRU list.
// Memory size for entries residing only in the LRU list
size_t lru_usage_;
// mutex_ protects the following state.
@@ -467,9 +467,9 @@ class LRUCache
virtual void DisownData() override;
virtual void WaitAll(std::vector<Handle*>& handles) override;
// Retrieves number of elements in LRU, for unit test purpose only.
// Retrieves number of elements in LRU, for unit test purpose only
size_t TEST_GetLRUSize();
// Retrieves high pri pool ratio.
// Retrieves high pri pool ratio
double GetHighPriPoolRatio();
private:
+4 -5
View File
@@ -104,15 +104,14 @@ bool ShardedCache::Ref(Handle* handle) {
return GetShard(Shard(hash))->Ref(handle);
}
bool ShardedCache::Release(Handle* handle, bool erase_if_last_ref) {
bool ShardedCache::Release(Handle* handle, bool force_erase) {
uint32_t hash = GetHash(handle);
return GetShard(Shard(hash))->Release(handle, erase_if_last_ref);
return GetShard(Shard(hash))->Release(handle, force_erase);
}
bool ShardedCache::Release(Handle* handle, bool useful,
bool erase_if_last_ref) {
bool ShardedCache::Release(Handle* handle, bool useful, bool force_erase) {
uint32_t hash = GetHash(handle);
return GetShard(Shard(hash))->Release(handle, useful, erase_if_last_ref);
return GetShard(Shard(hash))->Release(handle, useful, force_erase);
}
void ShardedCache::Erase(const Slice& key) {
+4 -4
View File
@@ -37,11 +37,11 @@ class CacheShard {
Cache::Priority priority, bool wait,
Statistics* stats) = 0;
virtual bool Release(Cache::Handle* handle, bool useful,
bool erase_if_last_ref) = 0;
bool force_erase) = 0;
virtual bool IsReady(Cache::Handle* handle) = 0;
virtual void Wait(Cache::Handle* handle) = 0;
virtual bool Ref(Cache::Handle* handle) = 0;
virtual bool Release(Cache::Handle* handle, bool erase_if_last_ref) = 0;
virtual bool Release(Cache::Handle* handle, bool force_erase) = 0;
virtual void Erase(const Slice& key, uint32_t hash) = 0;
virtual void SetCapacity(size_t capacity) = 0;
virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
@@ -94,11 +94,11 @@ class ShardedCache : public Cache {
const CreateCallback& create_cb, Priority priority,
bool wait, Statistics* stats = nullptr) override;
virtual bool Release(Handle* handle, bool useful,
bool erase_if_last_ref = false) override;
bool force_erase = false) override;
virtual bool IsReady(Handle* handle) override;
virtual void Wait(Handle* handle) override;
virtual bool Ref(Handle* handle) override;
virtual bool Release(Handle* handle, bool erase_if_last_ref = false) override;
virtual bool Release(Handle* handle, bool force_erase = false) override;
virtual void Erase(const Slice& key) override;
virtual uint64_t NewId() override;
virtual size_t GetCapacity() const override;
+1 -1
View File
@@ -12,7 +12,7 @@ fi
ROOT=".."
# Fetch right version of gcov
if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
source $ROOT/build_tools/fbcode_config_platform009.sh
source $ROOT/build_tools/fbcode_config_platform007.sh
GCOV=$GCC_BASE/bin/gcov
else
GCOV=$(which gcov)
-6
View File
@@ -42,12 +42,6 @@ crash_test_with_ts: $(DB_STRESS_CMD)
$(CRASHTEST_MAKE) whitebox_crash_test_with_ts
$(CRASHTEST_MAKE) blackbox_crash_test_with_ts
crash_test_with_multiops_wc_txn: $(DB_STRESS_CMD)
$(CRASHTEST_MAKE) blackbox_crash_test_with_multiops_wc_txn
crash_test_with_multiops_wp_txn: $(DB_STRESS_CMD)
$(CRASHTEST_MAKE) blackbox_crash_test_with_multiops_wp_txn
blackbox_crash_test: $(DB_STRESS_CMD)
$(CRASHTEST_PY) --simple blackbox $(CRASH_TEST_EXT_ARGS)
$(CRASHTEST_PY) blackbox $(CRASH_TEST_EXT_ARGS)
+1 -1
View File
@@ -2448,7 +2448,7 @@ int main(int argc, char** argv) {
rocksdb_fifo_compaction_options_destroy(fco);
}
StartPhase("backup_engine_option");
StartPhase("backupable_db_option");
{
rocksdb_backup_engine_options_t* bdo;
bdo = rocksdb_backup_engine_options_create("path");
+3 -9
View File
@@ -1341,8 +1341,6 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
CompactionRangeDelAggregator range_del_agg(&cfd->internal_comparator(),
existing_snapshots_);
// TODO: since we already use C++17, should use
// std::optional<const Slice> instead.
const Slice* const start = sub_compact->start;
const Slice* const end = sub_compact->end;
@@ -1364,13 +1362,9 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
// Although the v2 aggregator is what the level iterator(s) know about,
// the AddTombstones calls will be propagated down to the v1 aggregator.
std::unique_ptr<InternalIterator> raw_input(versions_->MakeInputIterator(
read_options, sub_compact->compaction, &range_del_agg,
file_options_for_read_,
(start == nullptr) ? std::optional<const Slice>{}
: std::optional<const Slice>{*start},
(end == nullptr) ? std::optional<const Slice>{}
: std::optional<const Slice>{*end}));
std::unique_ptr<InternalIterator> raw_input(
versions_->MakeInputIterator(read_options, sub_compact->compaction,
&range_del_agg, file_options_for_read_));
InternalIterator* input = raw_input.get();
IterKey start_ikey;
+80 -96
View File
@@ -17,11 +17,8 @@
#include "db/db_test_util.h"
#include "options/options_helper.h"
#include "port/stack_trace.h"
#include "rocksdb/advanced_options.h"
#include "rocksdb/convenience.h"
#include "rocksdb/filter_policy.h"
#include "rocksdb/perf_context.h"
#include "rocksdb/statistics.h"
#include "rocksdb/table.h"
#include "table/block_based/block_based_table_reader.h"
#include "table/block_based/filter_policy_internal.h"
@@ -801,87 +798,83 @@ TEST_F(DBBloomFilterTest, BloomFilterRate) {
}
}
namespace {
struct CompatibilityConfig {
std::shared_ptr<const FilterPolicy> policy;
bool partitioned;
uint32_t format_version;
void SetInTableOptions(BlockBasedTableOptions* table_options) {
table_options->filter_policy = policy;
table_options->partition_filters = partitioned;
if (partitioned) {
table_options->index_type =
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
} else {
table_options->index_type =
BlockBasedTableOptions::IndexType::kBinarySearch;
}
table_options->format_version = format_version;
}
};
// High bits per key -> almost no FPs
std::shared_ptr<const FilterPolicy> kCompatibilityBloomPolicy{
NewBloomFilterPolicy(20)};
// bloom_before_level=-1 -> always use Ribbon
std::shared_ptr<const FilterPolicy> kCompatibilityRibbonPolicy{
NewRibbonFilterPolicy(20, -1)};
std::vector<CompatibilityConfig> kCompatibilityConfigs = {
{Create(20, kDeprecatedBlock), false,
BlockBasedTableOptions().format_version},
{kCompatibilityBloomPolicy, false, BlockBasedTableOptions().format_version},
{kCompatibilityBloomPolicy, true, BlockBasedTableOptions().format_version},
{kCompatibilityBloomPolicy, false, /* legacy Bloom */ 4U},
{kCompatibilityRibbonPolicy, false,
BlockBasedTableOptions().format_version},
{kCompatibilityRibbonPolicy, true, BlockBasedTableOptions().format_version},
};
} // namespace
TEST_F(DBBloomFilterTest, BloomFilterCompatibility) {
Options options = CurrentOptions();
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
options.level0_file_num_compaction_trigger =
static_cast<int>(kCompatibilityConfigs.size()) + 1;
options.max_open_files = -1;
BlockBasedTableOptions table_options;
table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Close();
// Create with block based filter
CreateAndReopenWithCF({"pikachu"}, options);
// Create one file for each kind of filter. Each file covers a distinct key
// range.
for (size_t i = 0; i < kCompatibilityConfigs.size(); ++i) {
BlockBasedTableOptions table_options;
kCompatibilityConfigs[i].SetInTableOptions(&table_options);
ASSERT_TRUE(table_options.filter_policy != nullptr);
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options);
std::string prefix = ToString(i) + "_";
ASSERT_OK(Put(prefix + "A", "val"));
ASSERT_OK(Put(prefix + "Z", "val"));
ASSERT_OK(Flush());
const int maxKey = 10000;
for (int i = 0; i < maxKey; i++) {
ASSERT_OK(Put(1, Key(i), Key(i)));
}
ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
Flush(1);
// Test filter is used between each pair of {reader,writer} configurations,
// because any built-in FilterPolicy should be able to read filters from any
// other built-in FilterPolicy
for (size_t i = 0; i < kCompatibilityConfigs.size(); ++i) {
// Check db with full filter
table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
ReopenWithColumnFamilies({"default", "pikachu"}, options);
// Check if they can be found
for (int i = 0; i < maxKey; i++) {
ASSERT_EQ(Key(i), Get(1, Key(i)));
}
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
// Check db with partitioned full filter
table_options.partition_filters = true;
table_options.index_type =
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
ReopenWithColumnFamilies({"default", "pikachu"}, options);
// Check if they can be found
for (int i = 0; i < maxKey; i++) {
ASSERT_EQ(Key(i), Get(1, Key(i)));
}
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
}
TEST_F(DBBloomFilterTest, BloomFilterReverseCompatibility) {
for (bool partition_filters : {true, false}) {
Options options = CurrentOptions();
options.statistics = ROCKSDB_NAMESPACE::CreateDBStatistics();
BlockBasedTableOptions table_options;
kCompatibilityConfigs[i].SetInTableOptions(&table_options);
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
Reopen(options);
for (size_t j = 0; j < kCompatibilityConfigs.size(); ++j) {
std::string prefix = ToString(j) + "_";
ASSERT_EQ("val", Get(prefix + "A")); // Filter positive
ASSERT_EQ("val", Get(prefix + "Z")); // Filter positive
// Filter negative, with high probability
ASSERT_EQ("NOT_FOUND", Get(prefix + "Q"));
// FULL_POSITIVE does not include block-based filter case (j == 0)
EXPECT_EQ(TestGetAndResetTickerCount(options, BLOOM_FILTER_FULL_POSITIVE),
j == 0 ? 0 : 2);
EXPECT_EQ(TestGetAndResetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
if (partition_filters) {
table_options.partition_filters = true;
table_options.index_type =
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
}
table_options.filter_policy.reset(NewBloomFilterPolicy(10, false));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
DestroyAndReopen(options);
// Create with full filter
CreateAndReopenWithCF({"pikachu"}, options);
const int maxKey = 10000;
for (int i = 0; i < maxKey; i++) {
ASSERT_OK(Put(1, Key(i), Key(i)));
}
ASSERT_OK(Put(1, Key(maxKey + 55555), Key(maxKey + 55555)));
Flush(1);
// Check db with block_based filter
table_options.filter_policy.reset(NewBloomFilterPolicy(10, true));
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
ReopenWithColumnFamilies({"default", "pikachu"}, options);
// Check if they can be found
for (int i = 0; i < maxKey; i++) {
ASSERT_EQ(Key(i), Get(1, Key(i)));
}
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
}
}
@@ -926,7 +919,7 @@ class FilterConstructResPeakTrackingCache : public CacheWrapper {
}
using Cache::Release;
bool Release(Handle* handle, bool erase_if_last_ref = false) override {
bool Release(Handle* handle, bool force_erase = false) override {
auto deleter = GetDeleter(handle);
if (deleter == kNoopDeleterForFilterConstruction) {
if (!last_peak_tracked_) {
@@ -936,7 +929,7 @@ class FilterConstructResPeakTrackingCache : public CacheWrapper {
}
cur_cache_res_ -= GetCharge(handle);
}
bool is_successful = target_->Release(handle, erase_if_last_ref);
bool is_successful = target_->Release(handle, force_erase);
return is_successful;
}
@@ -959,8 +952,8 @@ class FilterConstructResPeakTrackingCache : public CacheWrapper {
const Cache::DeleterFn
FilterConstructResPeakTrackingCache::kNoopDeleterForFilterConstruction =
CacheReservationManagerImpl<
CacheEntryRole::kFilterConstruction>::TEST_GetNoopDeleterForRole();
CacheReservationManager::TEST_GetNoopDeleterForRole<
CacheEntryRole::kFilterConstruction>();
// To align with the type of hash entry being reserved in implementation.
using FilterConstructionReserveMemoryHash = uint64_t;
@@ -990,9 +983,7 @@ class DBFilterConstructionReserveMemoryTestWithParam
// trigger at least 1 dummy entry reservation each for hash entries and
// final filter, we need a large number of keys to ensure we have at least
// two partitions.
num_key_ = 18 *
CacheReservationManagerImpl<
CacheEntryRole::kFilterConstruction>::GetDummyEntrySize() /
num_key_ = 18 * CacheReservationManager::GetDummyEntrySize() /
sizeof(FilterConstructionReserveMemoryHash);
} else if (policy_ == kFastLocalBloom) {
// For Bloom Filter + FullFilter case, since we design the num_key_ to
@@ -1002,9 +993,7 @@ class DBFilterConstructionReserveMemoryTestWithParam
// behavior and we don't need a large number of keys to verify we
// indeed charge the final filter for cache reservation, even though final
// filter is a lot smaller than hash entries.
num_key_ = 1 *
CacheReservationManagerImpl<
CacheEntryRole::kFilterConstruction>::GetDummyEntrySize() /
num_key_ = 1 * CacheReservationManager::GetDummyEntrySize() /
sizeof(FilterConstructionReserveMemoryHash);
} else {
// For Ribbon Filter + FullFilter case, we need a large enough number of
@@ -1012,9 +1001,7 @@ class DBFilterConstructionReserveMemoryTestWithParam
// reservation will trigger at least another dummy entry (or equivalently
// to saying, causing another peak in cache reservation) as banding
// reservation might not be a multiple of dummy entry.
num_key_ = 12 *
CacheReservationManagerImpl<
CacheEntryRole::kFilterConstruction>::GetDummyEntrySize() /
num_key_ = 12 * CacheReservationManager::GetDummyEntrySize() /
sizeof(FilterConstructionReserveMemoryHash);
}
}
@@ -1169,8 +1156,8 @@ TEST_P(DBFilterConstructionReserveMemoryTestWithParam, ReserveMemory) {
return;
}
const std::size_t kDummyEntrySize = CacheReservationManagerImpl<
CacheEntryRole::kFilterConstruction>::GetDummyEntrySize();
const std::size_t kDummyEntrySize =
CacheReservationManager::GetDummyEntrySize();
const std::size_t predicted_hash_entries_cache_res =
num_key * sizeof(FilterConstructionReserveMemoryHash);
@@ -1358,12 +1345,9 @@ TEST_P(DBFilterConstructionReserveMemoryTestWithParam, ReserveMemory) {
*
*/
if (!partition_filters) {
ASSERT_GE(
std::floor(
1.0 * predicted_final_filter_cache_res /
CacheReservationManagerImpl<
CacheEntryRole::kFilterConstruction>::GetDummyEntrySize()),
1)
ASSERT_GE(std::floor(1.0 * predicted_final_filter_cache_res /
CacheReservationManager::GetDummyEntrySize()),
1)
<< "Final filter cache reservation too small for this test - please "
"increase the number of keys";
if (!detect_filter_construct_corruption) {
-68
View File
@@ -1354,74 +1354,6 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveTargetLevel) {
}
}
TEST_P(DBCompactionTestWithParam, PartialOverlappingL0) {
class SubCompactionEventListener : public EventListener {
public:
void OnSubcompactionCompleted(const SubcompactionJobInfo&) override {
sub_compaction_finished_++;
}
std::atomic<int> sub_compaction_finished_{0};
};
Options options = CurrentOptions();
options.disable_auto_compactions = true;
options.write_buffer_size = 10 * 1024 * 1024;
options.max_subcompactions = max_subcompactions_;
SubCompactionEventListener* listener = new SubCompactionEventListener();
options.listeners.emplace_back(listener);
DestroyAndReopen(options);
// For subcompactino to trigger, output level needs to be non-empty.
ASSERT_OK(Put("key", ""));
ASSERT_OK(Put("kez", ""));
ASSERT_OK(Flush());
ASSERT_OK(Put("key", ""));
ASSERT_OK(Put("kez", ""));
ASSERT_OK(Flush());
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
// Ranges that are only briefly overlapping so that they won't be trivially
// moved but subcompaction ranges would only contain a subset of files.
std::vector<std::pair<int32_t, int32_t>> ranges = {
{100, 199}, {198, 399}, {397, 600}, {598, 800}, {799, 900}, {895, 999},
};
int32_t value_size = 10 * 1024; // 10 KB
Random rnd(301);
std::map<int32_t, std::string> values;
for (size_t i = 0; i < ranges.size(); i++) {
for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) {
values[j] = rnd.RandomString(value_size);
ASSERT_OK(Put(Key(j), values[j]));
}
ASSERT_OK(Flush());
}
int32_t level0_files = NumTableFilesAtLevel(0, 0);
ASSERT_EQ(level0_files, ranges.size()); // Multiple files in L0
ASSERT_EQ(NumTableFilesAtLevel(1, 0), 1); // One file in L1
listener->sub_compaction_finished_ = 0;
ASSERT_OK(db_->EnableAutoCompaction({db_->DefaultColumnFamily()}));
ASSERT_OK(dbfull()->TEST_WaitForCompact());
if (max_subcompactions_ > 3) {
// RocksDB might not generate the exact number of sub compactions.
// Here we validate that at least subcompaction happened.
ASSERT_GT(listener->sub_compaction_finished_.load(), 2);
}
// We expect that all the files were compacted to L1
ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0);
ASSERT_GT(NumTableFilesAtLevel(1, 0), 1);
for (size_t i = 0; i < ranges.size(); i++) {
for (int32_t j = ranges[i].first; j <= ranges[i].second; j++) {
ASSERT_EQ(Get(Key(j)), values[j]);
}
}
}
TEST_P(DBCompactionTestWithParam, ManualCompactionPartial) {
int32_t trivial_move = 0;
int32_t non_trivial_move = 0;
+7 -6
View File
@@ -96,7 +96,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
3); // for CURRENT + MANIFEST + OPTIONS
// create names of the live files. The names are not absolute
// paths, instead they are relative to dbname_.
// paths, instead they are relative to dbname_;
for (const auto& table_file_number : live_table_files) {
ret.emplace_back(MakeTableFileName("", table_file_number));
}
@@ -166,7 +166,7 @@ Status DBImpl::GetCurrentWalFile(std::unique_ptr<LogFile>* current_log_file) {
Status DBImpl::GetLiveFilesStorageInfo(
const LiveFilesStorageInfoOptions& opts,
std::vector<LiveFileStorageInfo>* files) {
// To avoid returning partial results, only move results to files on success.
// To avoid returning partial results, only move to ouput on success
assert(files);
files->clear();
std::vector<LiveFileStorageInfo> results;
@@ -180,7 +180,7 @@ Status DBImpl::GetLiveFilesStorageInfo(
if (opts.wal_size_for_flush == port::kMaxUint64) {
flush_memtable = false;
} else if (opts.wal_size_for_flush > 0) {
// If the outstanding log files are small, we skip the flush.
// If out standing log files are small, we skip the flush.
s = GetSortedWalFiles(live_wal_files);
if (!s.ok()) {
@@ -313,7 +313,8 @@ Status DBImpl::GetLiveFilesStorageInfo(
info.relative_filename = kCurrentFileName;
info.directory = GetName();
info.file_type = kCurrentFile;
// CURRENT could be replaced so we have to record the contents as needed.
// CURRENT could be replaced so we have to record the contents we want
// for it
info.replacement_contents = manifest_fname + "\n";
info.size = manifest_fname.size() + 1;
if (opts.include_checksum_info) {
@@ -355,7 +356,7 @@ Status DBImpl::GetLiveFilesStorageInfo(
TEST_SYNC_POINT("CheckpointImpl::CreateCustomCheckpoint:AfterGetLive1");
TEST_SYNC_POINT("CheckpointImpl::CreateCustomCheckpoint:AfterGetLive2");
// If we have more than one column family, we also need to get WAL files.
// if we have more than one column family, we need to also get WAL files
if (s.ok()) {
s = GetSortedWalFiles(live_wal_files);
}
@@ -393,7 +394,7 @@ Status DBImpl::GetLiveFilesStorageInfo(
}
if (s.ok()) {
// Only move results to output on success.
// Only move output on success
*files = std::move(results);
}
return s;
+1 -2
View File
@@ -1081,6 +1081,7 @@ Status DBImpl::SetOptions(
MutableCFOptions new_options;
Status s;
Status persist_options_status;
persist_options_status.PermitUncheckedError(); // Allow uninitialized access
SuperVersionContext sv_context(/* create_superversion */ true);
{
auto db_options = GetDBOptions();
@@ -1116,11 +1117,9 @@ Status DBImpl::SetOptions(
"[%s] SetOptions() succeeded", cfd->GetName().c_str());
new_options.Dump(immutable_db_options_.info_log.get());
if (!persist_options_status.ok()) {
// NOTE: WriteOptionsFile already logs on failure
s = persist_options_status;
}
} else {
persist_options_status.PermitUncheckedError(); // less important
ROCKS_LOG_WARN(immutable_db_options_.info_log, "[%s] SetOptions() failed",
cfd->GetName().c_str());
}
+1 -1
View File
@@ -2144,7 +2144,7 @@ class DBImpl : public DB {
// only during recovery. Using this feature enables not writing the state to
// memtable on normal writes and hence improving the throughput. Each new
// write of the state will replace the previous state entirely even if the
// keys in the two consecutive states do not overlap.
// keys in the two consecuitive states do not overlap.
// It is protected by log_write_mutex_ when two_write_queues_ is enabled.
// Otherwise only the heaad of write_thread_ can access it.
WriteBatch cached_recoverable_state_;
+3 -3
View File
@@ -129,7 +129,7 @@ Status DBImpl::WriteImpl(const WriteOptions& write_options,
PreReleaseCallback* pre_release_callback) {
assert(!seq_per_batch_ || batch_cnt != 0);
if (my_batch == nullptr) {
return Status::InvalidArgument("Batch is nullptr!");
return Status::Corruption("Batch is nullptr!");
} else if (!disable_memtable &&
WriteBatchInternal::TimestampsUpdateNeeded(*my_batch)) {
// If writing to memtable, then we require the caller to set/update the
@@ -1882,11 +1882,9 @@ void DBImpl::NotifyOnMemTableSealed(ColumnFamilyData* /*cfd*/,
return;
}
mutex_.Unlock();
for (auto listener : immutable_db_options_.listeners) {
listener->OnMemTableSealed(mem_table_info);
}
mutex_.Lock();
}
#endif // ROCKSDB_LITE
@@ -2087,9 +2085,11 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
mutable_cf_options);
#ifndef ROCKSDB_LITE
mutex_.Unlock();
// Notify client that memtable is sealed, now that we have successfully
// installed a new memtable
NotifyOnMemTableSealed(cfd, memtable_info);
mutex_.Lock();
#endif // ROCKSDB_LITE
// It is possible that we got here without checking the value of i_os, but
// that is okay. If we did, it most likely means that s was already an error.
-89
View File
@@ -13,17 +13,12 @@
#include <string>
#include "db/db_test_util.h"
#include "options/cf_options.h"
#include "port/stack_trace.h"
#include "rocksdb/listener.h"
#include "rocksdb/options.h"
#include "rocksdb/perf_context.h"
#include "rocksdb/perf_level.h"
#include "rocksdb/table.h"
#include "table/block_based/block.h"
#include "table/format.h"
#include "table/meta_blocks.h"
#include "table/table_builder.h"
#include "test_util/mock_time_env.h"
#include "util/random.h"
#include "util/string_util.h"
@@ -1997,90 +1992,6 @@ TEST_F(DBPropertiesTest, GetMapPropertyDbStats) {
Close();
}
namespace {
std::string PopMetaIndexKey(InternalIterator* meta_iter) {
Status s = meta_iter->status();
if (!s.ok()) {
return s.ToString();
} else if (meta_iter->Valid()) {
std::string rv = meta_iter->key().ToString();
meta_iter->Next();
return rv;
} else {
return "NOT_FOUND";
}
}
} // namespace
TEST_F(DBPropertiesTest, TableMetaIndexKeys) {
// This is to detect unexpected churn in metaindex block keys. This is more
// of a "table test" but table_test.cc doesn't depend on db_test_util.h and
// we need ChangeOptions() for broad coverage.
constexpr int kKeyCount = 100;
do {
Options options;
options = CurrentOptions(options);
DestroyAndReopen(options);
// Create an SST file
for (int key = 0; key < kKeyCount; key++) {
ASSERT_OK(Put(Key(key), "val"));
}
ASSERT_OK(Flush());
// Find its file number
std::vector<LiveFileMetaData> files;
db_->GetLiveFilesMetaData(&files);
// 1 SST file
ASSERT_EQ(1, files.size());
// Open it for inspection
std::string sst_file =
files[0].directory + "/" + files[0].relative_filename;
std::unique_ptr<FSRandomAccessFile> f;
ASSERT_OK(env_->GetFileSystem()->NewRandomAccessFile(
sst_file, FileOptions(), &f, nullptr));
std::unique_ptr<RandomAccessFileReader> r;
r.reset(new RandomAccessFileReader(std::move(f), sst_file));
uint64_t file_size = 0;
ASSERT_OK(env_->GetFileSize(sst_file, &file_size));
// Read metaindex
BlockContents bc;
ASSERT_OK(ReadMetaIndexBlockInFile(r.get(), file_size, 0U,
ImmutableOptions(options), &bc));
Block metaindex_block(std::move(bc));
std::unique_ptr<InternalIterator> meta_iter;
meta_iter.reset(metaindex_block.NewMetaIterator());
meta_iter->SeekToFirst();
if (strcmp(options.table_factory->Name(),
TableFactory::kBlockBasedTableName()) == 0) {
auto bbto = options.table_factory->GetOptions<BlockBasedTableOptions>();
if (bbto->filter_policy) {
if (bbto->partition_filters) {
// The key names are intentionally hard-coded here to detect
// accidental regression on compatibility.
EXPECT_EQ("partitionedfilter.rocksdb.BuiltinBloomFilter",
PopMetaIndexKey(meta_iter.get()));
} else {
EXPECT_EQ("fullfilter.rocksdb.BuiltinBloomFilter",
PopMetaIndexKey(meta_iter.get()));
}
}
if (bbto->index_type == BlockBasedTableOptions::kHashSearch) {
EXPECT_EQ("rocksdb.hashindex.metadata",
PopMetaIndexKey(meta_iter.get()));
EXPECT_EQ("rocksdb.hashindex.prefixes",
PopMetaIndexKey(meta_iter.get()));
}
}
EXPECT_EQ("rocksdb.properties", PopMetaIndexKey(meta_iter.get()));
EXPECT_EQ("NOT_FOUND", PopMetaIndexKey(meta_iter.get()));
} while (ChangeOptions());
}
#endif // ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE
-65
View File
@@ -11,9 +11,7 @@
#include "file/sst_file_manager_impl.h"
#include "port/port.h"
#include "port/stack_trace.h"
#include "rocksdb/cache.h"
#include "rocksdb/sst_file_manager.h"
#include "rocksdb/table.h"
#include "util/random.h"
namespace ROCKSDB_NAMESPACE {
@@ -1395,69 +1393,6 @@ TEST_F(DBSSTTest, OpenDBWithInfiniteMaxOpenFiles) {
}
}
TEST_F(DBSSTTest, OpenDBWithInfiniteMaxOpenFilesSubjectToMemoryLimit) {
for (bool reserve_table_builder_memory : {true, false}) {
// Open DB with infinite max open files
// - First iteration use 1 thread to open files
// - Second iteration use 5 threads to open files
for (int iter = 0; iter < 2; iter++) {
Options options;
options.create_if_missing = true;
options.write_buffer_size = 100000;
options.disable_auto_compactions = true;
options.max_open_files = -1;
BlockBasedTableOptions table_options;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
if (iter == 0) {
options.max_file_opening_threads = 1;
} else {
options.max_file_opening_threads = 5;
}
DestroyAndReopen(options);
// Create 5 Files in L0 (then move then to L2)
for (int i = 0; i < 5; i++) {
std::string k = "L2_" + Key(i);
ASSERT_OK(Put(k, k + std::string(1000, 'a')));
ASSERT_OK(Flush()) << i;
}
CompactRangeOptions compact_options;
compact_options.change_level = true;
compact_options.target_level = 2;
ASSERT_OK(db_->CompactRange(compact_options, nullptr, nullptr));
// Create 5 Files in L0
for (int i = 0; i < 5; i++) {
std::string k = "L0_" + Key(i);
ASSERT_OK(Put(k, k + std::string(1000, 'a')));
ASSERT_OK(Flush());
}
Close();
table_options.reserve_table_reader_memory = reserve_table_builder_memory;
table_options.block_cache =
NewLRUCache(1024 /* capacity */, 0 /* num_shard_bits */,
true /* strict_capacity_limit */);
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
// Reopening the DB will try to load all existing files, conditionally
// subject to memory limit
Status s = TryReopen(options);
if (table_options.reserve_table_reader_memory) {
EXPECT_TRUE(s.IsMemoryLimit());
EXPECT_TRUE(s.ToString().find("memory limit based on cache capacity") !=
std::string::npos);
} else {
EXPECT_TRUE(s.ok());
ASSERT_EQ("5,0,5", FilesPerLevel(0));
}
}
}
}
TEST_F(DBSSTTest, GetTotalSstFilesSize) {
// We don't propagate oldest-key-time table property on compaction and
// just write 0 as default value. This affect the exact table size, since
-2
View File
@@ -7,7 +7,6 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file. See the AUTHORS file for names of contributors.
#include <memory>
#include <unordered_set>
#include <vector>
@@ -19,7 +18,6 @@
#include "rocksdb/utilities/table_properties_collectors.h"
#include "table/format.h"
#include "table/meta_blocks.h"
#include "table/table_properties_internal.h"
#include "test_util/testharness.h"
#include "test_util/testutil.h"
#include "util/random.h"
+2 -2
View File
@@ -894,8 +894,8 @@ class CacheWrapper : public Cache {
bool Ref(Handle* handle) override { return target_->Ref(handle); }
using Cache::Release;
bool Release(Handle* handle, bool erase_if_last_ref = false) override {
return target_->Release(handle, erase_if_last_ref);
bool Release(Handle* handle, bool force_erase = false) override {
return target_->Release(handle, force_erase);
}
void* Value(Handle* handle) override { return target_->Value(handle); }
+17 -49
View File
@@ -27,7 +27,6 @@
#include "db/blob/blob_log_format.h"
#include "db/compaction/compaction.h"
#include "db/compaction/file_pri.h"
#include "db/dbformat.h"
#include "db/internal_stats.h"
#include "db/log_reader.h"
#include "db/log_writer.h"
@@ -5126,6 +5125,9 @@ Status VersionSet::TryRecoverFromOneManifest(
Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
const std::string& dbname,
FileSystem* fs) {
// these are just for performance reasons, not correctness,
// so we're fine using the defaults
FileOptions soptions;
// Read "CURRENT" file, which contains a pointer to the current manifest file
std::string manifest_path;
uint64_t manifest_file_number;
@@ -5134,24 +5136,16 @@ Status VersionSet::ListColumnFamilies(std::vector<std::string>* column_families,
if (!s.ok()) {
return s;
}
return ListColumnFamiliesFromManifest(manifest_path, fs, column_families);
}
Status VersionSet::ListColumnFamiliesFromManifest(
const std::string& manifest_path, FileSystem* fs,
std::vector<std::string>* column_families) {
std::unique_ptr<SequentialFileReader> file_reader;
Status s;
{
std::unique_ptr<FSSequentialFile> file;
// these are just for performance reasons, not correctness,
// so we're fine using the defaults
s = fs->NewSequentialFile(manifest_path, FileOptions(), &file, nullptr);
s = fs->NewSequentialFile(manifest_path, soptions, &file, nullptr);
if (!s.ok()) {
return s;
}
file_reader = std::make_unique<SequentialFileReader>(
std::move(file), manifest_path, /*io_tracer=*/nullptr);
}
file_reader.reset(new SequentialFileReader(std::move(file), manifest_path,
nullptr /*IOTracer*/));
}
VersionSet::LogReporter reporter;
@@ -5341,16 +5335,9 @@ Status VersionSet::GetLiveFilesChecksumInfo(FileChecksumList* checksum_list) {
Status VersionSet::DumpManifest(Options& options, std::string& dscname,
bool verbose, bool hex, bool json) {
assert(options.env);
std::vector<std::string> column_families;
Status s = ListColumnFamiliesFromManifest(
dscname, options.env->GetFileSystem().get(), &column_families);
if (!s.ok()) {
return s;
}
// Open the specified manifest file.
std::unique_ptr<SequentialFileReader> file_reader;
Status s;
{
std::unique_ptr<FSSequentialFile> file;
const std::shared_ptr<FileSystem>& fs = options.env->GetFileSystem();
@@ -5361,16 +5348,14 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
if (!s.ok()) {
return s;
}
file_reader = std::make_unique<SequentialFileReader>(
std::move(file), dscname, db_options_->log_readahead_size, io_tracer_);
file_reader.reset(new SequentialFileReader(
std::move(file), dscname, db_options_->log_readahead_size, io_tracer_));
}
std::vector<ColumnFamilyDescriptor> cf_descs;
for (const auto& cf : column_families) {
cf_descs.emplace_back(cf, options);
}
DumpManifestHandler handler(cf_descs, this, io_tracer_, verbose, hex, json);
std::vector<ColumnFamilyDescriptor> column_families(
1, ColumnFamilyDescriptor(kDefaultColumnFamilyName, options));
DumpManifestHandler handler(column_families, this, io_tracer_, verbose, hex,
json);
{
VersionSet::LogReporter reporter;
reporter.status = &s;
@@ -5817,9 +5802,7 @@ void VersionSet::AddLiveFiles(std::vector<uint64_t>* live_table_files,
InternalIterator* VersionSet::MakeInputIterator(
const ReadOptions& read_options, const Compaction* c,
RangeDelAggregator* range_del_agg,
const FileOptions& file_options_compactions,
const std::optional<const Slice>& start,
const std::optional<const Slice>& end) {
const FileOptions& file_options_compactions) {
auto cfd = c->column_family_data();
// Level-0 files have to be merged together. For other levels,
// we will make a concatenating iterator per level.
@@ -5834,25 +5817,10 @@ InternalIterator* VersionSet::MakeInputIterator(
if (c->level(which) == 0) {
const LevelFilesBrief* flevel = c->input_levels(which);
for (size_t i = 0; i < flevel->num_files; i++) {
const FileMetaData& fmd = *flevel->files[i].file_metadata;
if (start.has_value() &&
cfd->user_comparator()->Compare(start.value(),
fmd.largest.user_key()) > 0) {
continue;
}
// We should be able to filter out the case where the end key
// equals to the end boundary, since the end key is exclusive.
// We try to be extra safe here.
if (end.has_value() &&
cfd->user_comparator()->Compare(end.value(),
fmd.smallest.user_key()) < 0) {
continue;
}
list[num++] = cfd->table_cache()->NewIterator(
read_options, file_options_compactions,
cfd->internal_comparator(), fmd, range_del_agg,
c->mutable_cf_options()->prefix_extractor,
cfd->internal_comparator(), *flevel->files[i].file_metadata,
range_del_agg, c->mutable_cf_options()->prefix_extractor,
/*table_reader_ptr=*/nullptr,
/*file_read_hist=*/nullptr, TableReaderCaller::kCompaction,
/*arena=*/nullptr,
+1 -8
View File
@@ -24,7 +24,6 @@
#include <limits>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <string>
#include <unordered_set>
@@ -1091,9 +1090,6 @@ class VersionSet {
// column_families.
static Status ListColumnFamilies(std::vector<std::string>* column_families,
const std::string& dbname, FileSystem* fs);
static Status ListColumnFamiliesFromManifest(
const std::string& manifest_path, FileSystem* fs,
std::vector<std::string>* column_families);
#ifndef ROCKSDB_LITE
// Try to reduce the number of levels. This call is valid when
@@ -1262,13 +1258,10 @@ class VersionSet {
// Create an iterator that reads over the compaction inputs for "*c".
// The caller should delete the iterator when no longer needed.
// @param read_options Must outlive the returned iterator.
// @param start, end indicates compaction range
InternalIterator* MakeInputIterator(
const ReadOptions& read_options, const Compaction* c,
RangeDelAggregator* range_del_agg,
const FileOptions& file_options_compactions,
const std::optional<const Slice>& start,
const std::optional<const Slice>& end);
const FileOptions& file_options_compactions);
// Add all files listed in any live version to *live_table_files and
// *live_blob_files. Note that these lists may contain duplicates.
+4 -5
View File
@@ -22,13 +22,12 @@
// kTypeColumnFamilySingleDeletion varint32 varstring
// kTypeColumnFamilyRangeDeletion varint32 varstring varstring
// kTypeColumnFamilyMerge varint32 varstring varstring
// kTypeBeginPrepareXID
// kTypeEndPrepareXID varstring
// kTypeBeginPrepareXID varstring
// kTypeEndPrepareXID
// kTypeCommitXID varstring
// kTypeCommitXIDAndTimestamp varstring varstring
// kTypeRollbackXID varstring
// kTypeBeginPersistedPrepareXID
// kTypeBeginUnprepareXID
// kTypeBeginPersistedPrepareXID varstring
// kTypeBeginUnprepareXID varstring
// kTypeNoop
// varstring :=
// len: varint32
-5
View File
@@ -134,7 +134,6 @@ DECLARE_int32(set_in_place_one_in);
DECLARE_int64(cache_size);
DECLARE_int32(cache_numshardbits);
DECLARE_bool(cache_index_and_filter_blocks);
DECLARE_bool(reserve_table_reader_memory);
DECLARE_int32(top_level_index_pinning);
DECLARE_int32(partition_pinning);
DECLARE_int32(unpartitioned_pinning);
@@ -283,10 +282,6 @@ DECLARE_uint64(wp_snapshot_cache_bits);
DECLARE_uint64(wp_commit_cache_bits);
#endif // !ROCKSDB_LITE
DECLARE_bool(adaptive_readahead);
DECLARE_bool(async_io);
DECLARE_string(wal_compression);
constexpr long KB = 1024;
constexpr int kRandomValueMaxFactor = 3;
constexpr int kValueMaxLen = 100;
-15
View File
@@ -304,11 +304,6 @@ DEFINE_int32(cache_numshardbits, 6,
DEFINE_bool(cache_index_and_filter_blocks, false,
"True if indexes/filters should be cached in block cache.");
DEFINE_bool(reserve_table_reader_memory, false,
"A dynamically updating charge to block cache, loosely based on "
"the actual memory usage of table reader, will occur to account "
"the memory, if block cache available.");
DEFINE_int32(
top_level_index_pinning,
static_cast<int32_t>(ROCKSDB_NAMESPACE::PinningTier::kFallback),
@@ -917,14 +912,4 @@ DEFINE_uint64(wp_commit_cache_bits, 23ull,
"commit cache. Default: 23 (8M entries)");
#endif // !ROCKSDB_LITE
DEFINE_bool(adaptive_readahead, false,
"Carry forward internal auto readahead size from one file to next "
"file at each level during iteration");
DEFINE_bool(
async_io, false,
"Does asynchronous prefetching when internal auto readahead is enabled");
DEFINE_string(wal_compression, "none",
"Algorithm to use for WAL compression. none to disable.");
#endif // GFLAGS
+1 -10
View File
@@ -21,7 +21,7 @@
#include "rocksdb/utilities/object_registry.h"
#include "test_util/testutil.h"
#include "util/cast_util.h"
#include "utilities/backup/backup_engine_impl.h"
#include "utilities/backupable/backupable_db_impl.h"
#include "utilities/fault_injection_fs.h"
#include "utilities/fault_injection_secondary_cache.h"
@@ -645,8 +645,6 @@ void StressTest::OperateDb(ThreadState* thread) {
ReadOptions read_opts(FLAGS_verify_checksum, true);
read_opts.rate_limiter_priority =
FLAGS_rate_limit_user_ops ? Env::IO_USER : Env::IO_TOTAL;
read_opts.async_io = FLAGS_async_io;
read_opts.adaptive_readahead = FLAGS_adaptive_readahead;
WriteOptions write_opts;
if (FLAGS_rate_limit_auto_wal_flush) {
write_opts.rate_limiter_priority = Env::IO_USER;
@@ -2296,8 +2294,6 @@ void StressTest::PrintEnv() const {
static_cast<int>(FLAGS_fail_if_options_file_error));
fprintf(stdout, "User timestamp size bytes : %d\n",
static_cast<int>(FLAGS_user_timestamp_size));
fprintf(stdout, "WAL compression : %s\n",
FLAGS_wal_compression.c_str());
fprintf(stdout, "------------------------------------------------\n");
}
@@ -2321,8 +2317,6 @@ void StressTest::Open() {
block_based_options.block_cache_compressed = compressed_cache_;
block_based_options.checksum = checksum_type_e;
block_based_options.block_size = FLAGS_block_size;
block_based_options.reserve_table_reader_memory =
FLAGS_reserve_table_reader_memory;
block_based_options.format_version =
static_cast<uint32_t>(FLAGS_format_version);
block_based_options.index_block_restart_interval =
@@ -2440,9 +2434,6 @@ void StressTest::Open() {
FLAGS_blob_garbage_collection_force_threshold;
options_.blob_compaction_readahead_size =
FLAGS_blob_compaction_readahead_size;
options_.wal_compression =
StringToCompressionType(FLAGS_wal_compression.c_str());
} else {
#ifdef ROCKSDB_LITE
fprintf(stderr, "--options_file not supported in lite mode\n");
+6 -47
View File
@@ -45,8 +45,8 @@ class NonBatchedOpsStressTest : public StressTest {
if (thread->shared->HasVerificationFailedYet()) {
break;
}
if (thread->rand.OneIn(4)) {
// 1/4 chance use iterator to verify this range
if (thread->rand.OneIn(3)) {
// 1/3 chance use iterator to verify this range
Slice prefix;
std::string seek_key = Key(start);
std::unique_ptr<Iterator> iter(
@@ -91,8 +91,8 @@ class NonBatchedOpsStressTest : public StressTest {
from_db.data(), from_db.length());
}
}
} else if (thread->rand.OneIn(3)) {
// 1/4 chance use Get to verify this range
} else if (thread->rand.OneIn(2)) {
// 1/3 chance use Get to verify this range
for (auto i = start; i < end; i++) {
if (thread->shared->HasVerificationFailedYet()) {
break;
@@ -108,8 +108,8 @@ class NonBatchedOpsStressTest : public StressTest {
from_db.data(), from_db.length());
}
}
} else if (thread->rand.OneIn(2)) {
// 1/4 chance use MultiGet to verify this range
} else {
// 1/3 chance use MultiGet to verify this range
for (auto i = start; i < end;) {
if (thread->shared->HasVerificationFailedYet()) {
break;
@@ -140,47 +140,6 @@ class NonBatchedOpsStressTest : public StressTest {
i += batch_size;
}
} else {
// 1/4 chance use GetMergeOperand to verify this range
// Start off with small size that will be increased later if necessary
std::vector<PinnableSlice> values(4);
GetMergeOperandsOptions merge_operands_info;
merge_operands_info.expected_max_number_of_operands =
static_cast<int>(values.size());
for (auto i = start; i < end; i++) {
if (thread->shared->HasVerificationFailedYet()) {
break;
}
std::string from_db;
std::string keystr = Key(i);
Slice k = keystr;
int number_of_operands = 0;
Status s = db_->GetMergeOperands(options, column_families_[cf], k,
values.data(), &merge_operands_info,
&number_of_operands);
if (s.IsIncomplete()) {
// Need to resize values as there are more than values.size() merge
// operands on this key. Should only happen a few times when we
// encounter a key that had more merge operands than any key seen so
// far
values.resize(number_of_operands);
merge_operands_info.expected_max_number_of_operands =
static_cast<int>(number_of_operands);
s = db_->GetMergeOperands(options, column_families_[cf], k,
values.data(), &merge_operands_info,
&number_of_operands);
}
// Assumed here that GetMergeOperands always sets number_of_operand
if (number_of_operands) {
from_db = values[number_of_operands - 1].ToString();
}
VerifyValue(static_cast<int>(cf), i, options, shared, from_db, s,
true);
if (from_db.length()) {
PrintKeyValue(static_cast<int>(cf), static_cast<uint32_t>(i),
from_db.data(), from_db.length());
}
}
}
}
}
+5 -11
View File
@@ -1099,12 +1099,8 @@ TEST_F(PrefetchTest2, ReadAsyncWithPosixFS) {
}
int buff_prefetch_count = 0;
bool read_async_called = false;
SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start",
[&](void*) { buff_prefetch_count++; });
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
"UpdateResults::io_uring_result",
[&](void* /*arg*/) { read_async_called = true; });
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
// Read the keys.
@@ -1127,13 +1123,11 @@ TEST_F(PrefetchTest2, ReadAsyncWithPosixFS) {
{
HistogramData async_read_bytes;
options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes);
// Not all platforms support iouring. In that case, ReadAsync in posix
// won't submit async requests.
if (read_async_called) {
ASSERT_GT(async_read_bytes.count, 0);
} else {
ASSERT_EQ(async_read_bytes.count, 0);
}
#if defined(ROCKSDB_IOURING_PRESENT)
ASSERT_GT(async_read_bytes.count, 0);
#else
ASSERT_EQ(async_read_bytes.count, 0);
#endif
}
}
+1 -1
View File
@@ -166,7 +166,7 @@ class SstFileManagerImpl : public SstFileManager {
std::unordered_map<std::string, uint64_t> tracked_files_;
// The maximum allowed space (in bytes) for sst and blob files.
uint64_t max_allowed_space_;
// DeleteScheduler used to throttle file deletion.
// DeleteScheduler used to throttle file deletition.
DeleteScheduler delete_scheduler_;
port::CondVar cv_;
// Flag to force error recovery thread to exit
+6 -2
View File
@@ -160,7 +160,9 @@ class WritableFileWriter {
bool perform_data_verification_;
uint32_t buffered_data_crc32c_checksum_;
bool buffered_data_with_checksum_;
#ifndef ROCKSDB_LITE
Temperature temperature_;
#endif // ROCKSDB_LITE
public:
WritableFileWriter(
@@ -191,8 +193,10 @@ class WritableFileWriter {
checksum_finalized_(false),
perform_data_verification_(perform_data_verification),
buffered_data_crc32c_checksum_(0),
buffered_data_with_checksum_(buffered_data_with_checksum),
temperature_(options.temperature) {
buffered_data_with_checksum_(buffered_data_with_checksum) {
#ifndef ROCKSDB_LITE
temperature_ = options.temperature;
#endif // ROCKSDB_LITE
assert(!use_direct_io() || max_buffer_size_ > 0);
TEST_SYNC_POINT_CALLBACK("WritableFileWriter::WritableFileWriter:0",
reinterpret_cast<void*>(max_buffer_size_));
+21 -19
View File
@@ -52,8 +52,9 @@ struct LRUCacheOptions {
// Capacity of the cache.
size_t capacity = 0;
// Cache is sharded into 2^num_shard_bits shards, by hash of key.
// Refer to NewLRUCache for further information.
// Cache is sharded into 2^num_shard_bits shards,
// by hash of key. Refer to NewLRUCache for further
// information.
int num_shard_bits = -1;
// If strict_capacity_limit is set,
@@ -90,7 +91,7 @@ struct LRUCacheOptions {
CacheMetadataChargePolicy metadata_charge_policy =
kDefaultCacheMetadataChargePolicy;
// A SecondaryCache instance to use a the non-volatile tier.
// A SecondaryCache instance to use a the non-volatile tier
std::shared_ptr<SecondaryCache> secondary_cache;
LRUCacheOptions() {}
@@ -327,16 +328,17 @@ class Cache {
/**
* Release a mapping returned by a previous Lookup(). A released entry might
* still remain in cache in case it is later looked up by others. If
* erase_if_last_ref is set then it also erases it from the cache if there is
* no other reference to it. Erasing it should call the deleter function that
* was provided when the entry was inserted.
* still remain in cache in case it is later looked up by others. If
* force_erase is set then it also erase it from the cache if there is no
* other reference to it. Erasing it should call the deleter function that
* was provided when the
* entry was inserted.
*
* Returns true if the entry was also erased.
*/
// REQUIRES: handle must not have been released yet.
// REQUIRES: handle must have been returned by a method on *this.
virtual bool Release(Handle* handle, bool erase_if_last_ref = false) = 0;
virtual bool Release(Handle* handle, bool force_erase = false) = 0;
// Return the value encapsulated in a handle returned by a
// successful Lookup().
@@ -344,7 +346,7 @@ class Cache {
// REQUIRES: handle must have been returned by a method on *this.
virtual void* Value(Handle* handle) = 0;
// If the cache contains the entry for the key, erase it. Note that the
// If the cache contains entry for key, erase it. Note that the
// underlying entry will be kept around until all existing handles
// to it have been released.
virtual void Erase(const Slice& key) = 0;
@@ -368,19 +370,19 @@ class Cache {
// full capacity.
virtual bool HasStrictCapacityLimit() const = 0;
// Returns the maximum configured capacity of the cache
// returns the maximum configured capacity of the cache
virtual size_t GetCapacity() const = 0;
// Returns the memory size for the entries residing in the cache.
// returns the memory size for the entries residing in the cache.
virtual size_t GetUsage() const = 0;
// Returns the memory size for a specific entry in the cache.
// returns the memory size for a specific entry in the cache.
virtual size_t GetUsage(Handle* handle) const = 0;
// Returns the memory size for the entries in use by the system
// returns the memory size for the entries in use by the system
virtual size_t GetPinnedUsage() const = 0;
// Returns the charge for the specific entry in the cache.
// returns the charge for the specific entry in the cache.
virtual size_t GetCharge(Handle* handle) const = 0;
// Returns the deleter for the specified entry. This might seem useless
@@ -439,8 +441,9 @@ class Cache {
// The Insert and Lookup APIs below are intended to allow cached objects
// to be demoted/promoted between the primary block cache and a secondary
// cache. The secondary cache could be a non-volatile cache, and will
// likely store the object in a different representation. They rely on a
// per object CacheItemHelper to do the conversions.
// likely store the object in a different representation more suitable
// for on disk storage. They rely on a per object CacheItemHelper to do
// the conversions.
// The secondary cache may persist across process and system restarts,
// and may even be moved between hosts. Therefore, the cache key must
// be repeatable across restarts/reboots, and globally unique if
@@ -514,9 +517,8 @@ class Cache {
// parameter specifies whether the data was actually used or not,
// which may be used by the cache implementation to decide whether
// to consider it as a hit for retention purposes.
virtual bool Release(Handle* handle, bool /*useful*/,
bool erase_if_last_ref) {
return Release(handle, erase_if_last_ref);
virtual bool Release(Handle* handle, bool /*useful*/, bool force_erase) {
return Release(handle, force_erase);
}
// Determines if the handle returned by Lookup() has a valid value yet. The
+106 -149
View File
@@ -161,13 +161,98 @@ class DB {
static Status Open(const Options& options, const std::string& name,
DB** dbptr);
// Open the database for read only. All DB interfaces
// that modify data, like put/delete, will return error.
// If the db is opened in read only mode, then no compactions
// will happen.
//
// While a given DB can be simultaneously open via OpenForReadOnly
// by any number of readers, if a DB is simultaneously open by Open
// and OpenForReadOnly, the read-only instance has undefined behavior
// (though can often succeed if quickly closed) and the read-write
// instance is unaffected. See also OpenAsSecondary.
//
// Not supported in ROCKSDB_LITE, in which case the function will
// return Status::NotSupported.
static Status OpenForReadOnly(const Options& options, const std::string& name,
DB** dbptr,
bool error_if_wal_file_exists = false);
// Open the database for read only with column families. When opening DB with
// read only, you can specify only a subset of column families in the
// database that should be opened. However, you always need to specify default
// column family. The default column family name is 'default' and it's stored
// in ROCKSDB_NAMESPACE::kDefaultColumnFamilyName
//
// While a given DB can be simultaneously open via OpenForReadOnly
// by any number of readers, if a DB is simultaneously open by Open
// and OpenForReadOnly, the read-only instance has undefined behavior
// (though can often succeed if quickly closed) and the read-write
// instance is unaffected. See also OpenAsSecondary.
//
// Not supported in ROCKSDB_LITE, in which case the function will
// return Status::NotSupported.
static Status OpenForReadOnly(
const DBOptions& db_options, const std::string& name,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
bool error_if_wal_file_exists = false);
// The following OpenAsSecondary functions create a secondary instance that
// can dynamically tail the MANIFEST of a primary that must have already been
// created. User can call TryCatchUpWithPrimary to make the secondary
// instance catch up with primary (WAL tailing is NOT supported now) whenever
// the user feels necessary. Column families created by the primary after the
// secondary instance starts are currently ignored by the secondary instance.
// Column families opened by secondary and dropped by the primary will be
// dropped by secondary as well. However the user of the secondary instance
// can still access the data of such dropped column family as long as they
// do not destroy the corresponding column family handle.
// WAL tailing is not supported at present, but will arrive soon.
//
// The options argument specifies the options to open the secondary instance.
// The name argument specifies the name of the primary db that you have used
// to open the primary instance.
// The secondary_path argument points to a directory where the secondary
// instance stores its info log.
// The dbptr is an out-arg corresponding to the opened secondary instance.
// The pointer points to a heap-allocated database, and the user should
// delete it after use.
// Open DB as secondary instance with only the default column family.
// Return OK on success, non-OK on failures.
static Status OpenAsSecondary(const Options& options, const std::string& name,
const std::string& secondary_path, DB** dbptr);
// Open DB as secondary instance with column families. You can open a subset
// of column families in secondary mode.
// The db_options specify the database specific options.
// The name argument specifies the name of the primary db that you have used
// to open the primary instance.
// The secondary_path argument points to a directory where the secondary
// instance stores its info log.
// The column_families argument specifies a list of column families to open.
// If any of the column families does not exist, the function returns non-OK
// status.
// The handles is an out-arg corresponding to the opened database column
// family handles.
// The dbptr is an out-arg corresponding to the opened secondary instance.
// The pointer points to a heap-allocated database, and the caller should
// delete it after use. Before deleting the dbptr, the user should also
// delete the pointers stored in handles vector.
// Return OK on success, on-OK on failures.
static Status OpenAsSecondary(
const DBOptions& db_options, const std::string& name,
const std::string& secondary_path,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr);
// Open DB with column families.
// db_options specify database specific options
// column_families is the vector of all column families in the database,
// containing column family name and options. You need to open ALL column
// families in the database. To get the list of column families, you can use
// ListColumnFamilies().
//
// ListColumnFamilies(). Also, you can open only a subset of column families
// for read-only access.
// The default column family name is 'default' and it's stored
// in ROCKSDB_NAMESPACE::kDefaultColumnFamilyName.
// If everything is OK, handles will on return be the same size
@@ -179,107 +264,6 @@ class DB {
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr);
// OpenForReadOnly() creates a Read-only instance that supports reads alone.
//
// All DB interfaces that modify data, like put/delete, will return error.
// Automatic Flush and Compactions are disabled and any manual calls
// to Flush/Compaction will return error.
//
// While a given DB can be simultaneously opened via OpenForReadOnly
// by any number of readers, if a DB is simultaneously opened by Open
// and OpenForReadOnly, the read-only instance has undefined behavior
// (though can often succeed if quickly closed) and the read-write
// instance is unaffected. See also OpenAsSecondary.
// Open the database for read only.
//
// Not supported in ROCKSDB_LITE, in which case the function will
// return Status::NotSupported.
static Status OpenForReadOnly(const Options& options, const std::string& name,
DB** dbptr,
bool error_if_wal_file_exists = false);
// Open the database for read only with column families.
//
// When opening DB with read only, you can specify only a subset of column
// families in the database that should be opened. However, you always need
// to specify default column family. The default column family name is
// 'default' and it's stored in ROCKSDB_NAMESPACE::kDefaultColumnFamilyName
//
// Not supported in ROCKSDB_LITE, in which case the function will
// return Status::NotSupported.
static Status OpenForReadOnly(
const DBOptions& db_options, const std::string& name,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
bool error_if_wal_file_exists = false);
// OpenAsSecondary() creates a secondary instance that supports read-only
// operations and supports dynamic catch up with the primary (through a
// call to TryCatchUpWithPrimary()).
//
// All DB interfaces that modify data, like put/delete, will return error.
// Automatic Flush and Compactions are disabled and any manual calls
// to Flush/Compaction will return error.
//
// Multiple secondary instances can co-exist at the same time.
//
// Open DB as secondary instance
//
// The options argument specifies the options to open the secondary instance.
// Options.max_open_files should be set to -1.
// The name argument specifies the name of the primary db that you have used
// to open the primary instance.
// The secondary_path argument points to a directory where the secondary
// instance stores its info log.
// The dbptr is an out-arg corresponding to the opened secondary instance.
// The pointer points to a heap-allocated database, and the caller should
// delete it after use.
//
// Return OK on success, non-OK on failures.
static Status OpenAsSecondary(const Options& options, const std::string& name,
const std::string& secondary_path, DB** dbptr);
// Open DB as secondary instance with specified column families
//
// When opening DB in secondary mode, you can specify only a subset of column
// families in the database that should be opened. However, you always need
// to specify default column family. The default column family name is
// 'default' and it's stored in ROCKSDB_NAMESPACE::kDefaultColumnFamilyName
//
// Column families created by the primary after the secondary instance starts
// are currently ignored by the secondary instance. Column families opened
// by secondary and dropped by the primary will be dropped by secondary as
// well (on next invocation of TryCatchUpWithPrimary()). However the user
// of the secondary instance can still access the data of such dropped column
// family as long as they do not destroy the corresponding column family
// handle.
//
// The options argument specifies the options to open the secondary instance.
// Options.max_open_files should be set to -1.
// The name argument specifies the name of the primary db that you have used
// to open the primary instance.
// The secondary_path argument points to a directory where the secondary
// instance stores its info log.
// The column_families argument specifies a list of column families to open.
// If default column family is not specified or if any specified column
// families does not exist, the function returns non-OK status.
// The handles is an out-arg corresponding to the opened database column
// family handles.
// The dbptr is an out-arg corresponding to the opened secondary instance.
// The pointer points to a heap-allocated database, and the caller should
// delete it after use. Before deleting the dbptr, the user should also
// delete the pointers stored in handles vector.
//
// Return OK on success, non-OK on failures.
static Status OpenAsSecondary(
const DBOptions& db_options, const std::string& name,
const std::string& secondary_path,
const std::vector<ColumnFamilyDescriptor>& column_families,
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr);
// Open DB and run the compaction.
// It's a read-only operation, the result won't be installed to the DB, it
// will be output to the `output_directory`. The API should only be used with
@@ -454,6 +438,10 @@ class DB {
// If "end_key" comes before "start_key" according to the user's comparator,
// a `Status::InvalidArgument` is returned.
//
// WARNING: Do not use `Iterator::Refresh()` API on DBs where `DeleteRange()`
// has been used or will be used. This feature combination is neither
// supported nor programmatically prevented.
//
// This feature is now usable in production, with the following caveats:
// 1) Accumulating many range tombstones in the memtable will degrade read
// performance; this can be avoided by manually flushing occasionally.
@@ -553,20 +541,16 @@ class DB {
return Get(options, DefaultColumnFamily(), key, value, timestamp);
}
// Populates the `merge_operands` array with all the merge operands in the DB
// for `key`. The `merge_operands` array will be populated in the order of
// insertion. The number of entries populated in `merge_operands` will be
// assigned to `*number_of_operands`.
//
// If the number of merge operands in DB for `key` is greater than
// `merge_operands_options.expected_max_number_of_operands`,
// `merge_operands` is not populated and the return value is
// `Status::Incomplete`. In that case, `*number_of_operands` will be assigned
// the number of merge operands found in the DB for `key`.
//
// `merge_operands`- Points to an array of at-least
// Returns all the merge operands corresponding to the key. If the
// number of merge operands in DB is greater than
// merge_operands_options.expected_max_number_of_operands
// no merge operands are returned and status is Incomplete. Merge operands
// returned are in the order of insertion.
// merge_operands- Points to an array of at-least
// merge_operands_options.expected_max_number_of_operands and the
// caller is responsible for allocating it.
// caller is responsible for allocating it. If the status
// returned is Incomplete then number_of_operands will contain
// the total number of merge operands found in DB for key.
virtual Status GetMergeOperands(
const ReadOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* merge_operands,
@@ -940,8 +924,6 @@ class DB {
// "rocksdb.is-file-deletions-enabled" - returns 0 if deletion of obsolete
// files is enabled; otherwise, returns a non-zero number.
// This name may be misleading because true(non-zero) means disable,
// but we keep the name for backward compatibility.
static const std::string kIsFileDeletionsEnabled;
// "rocksdb.num-snapshots" - returns number of unreleased snapshots of the
@@ -1219,44 +1201,20 @@ class DB {
return CompactRange(options, DefaultColumnFamily(), begin, end);
}
// Dynamically change column family options or table factory options in a
// running DB, for the specified column family. Only options internally
// marked as "mutable" can be changed. Options not listed in `opts_map` will
// keep their current values. See GetColumnFamilyOptionsFromMap() in
// convenience.h for the details of `opts_map`. Not supported in LITE mode.
//
// USABILITY NOTE: SetOptions is intended only for expert users, and does
// not apply the same sanitization to options as the standard DB::Open code
// path does. Use with caution.
//
// RELIABILITY & PERFORMANCE NOTE: SetOptions is not fully stress-tested for
// reliability, and this is a slow call because a new OPTIONS file is
// serialized and persisted for each call. Use only infrequently.
//
// EXAMPLES:
// s = db->SetOptions(cfh, {{"ttl", "36000"}});
// s = db->SetOptions(cfh, {{"block_based_table_factory",
// "{prepopulate_block_cache=kDisable;}"}});
// TODO: documentation needed
// NOTE: SetOptions is intended only for expert users, and does not apply the
// same sanitization to options as the standard DB::Open code path does. Use
// with caution.
virtual Status SetOptions(
ColumnFamilyHandle* /*column_family*/,
const std::unordered_map<std::string, std::string>& /*opts_map*/) {
const std::unordered_map<std::string, std::string>& /*new_options*/) {
return Status::NotSupported("Not implemented");
}
// Shortcut for SetOptions on the default column family handle.
virtual Status SetOptions(
const std::unordered_map<std::string, std::string>& new_options) {
return SetOptions(DefaultColumnFamily(), new_options);
}
// Like SetOptions but for DBOptions, including the same caveats for
// usability, reliability, and performance. See GetDBOptionsFromMap() (and
// GetColumnFamilyOptionsFromMap()) in convenience.h for details on
// `opts_map`. Note supported in LITE mode.
//
// EXAMPLES:
// s = db->SetDBOptions({{"max_subcompactions", "2"}});
// s = db->SetDBOptions({{"stats_dump_period_sec", "0"},
// {"stats_persist_period_sec", "0"}});
virtual Status SetDBOptions(
const std::unordered_map<std::string, std::string>& new_options) = 0;
@@ -1341,8 +1299,6 @@ class DB {
// Get Env object from the DB
virtual Env* GetEnv() const = 0;
// A shortcut for GetEnv()->->GetFileSystem().get(), possibly cached for
// efficiency.
virtual FileSystem* GetFileSystem() const;
// Get DB Options that we use. During the process of opening the
@@ -1717,6 +1673,7 @@ class DB {
// secondary instance does not delete the corresponding column family
// handles, the data of the column family is still accessible to the
// secondary.
// TODO: we will support WAL tailing soon.
virtual Status TryCatchUpWithPrimary() {
return Status::NotSupported("Supported only by secondary instance");
}
-7
View File
@@ -89,12 +89,6 @@ struct IOOptions {
// Priority - high or low
IOPriority prio;
// Priority used to charge rate limiter configured in file system level (if
// any)
// Limitation: right now RocksDB internal does not consider this
// rate_limiter_priority
Env::IOPriority rate_limiter_priority;
// Type of data being read/written
IOType type;
@@ -115,7 +109,6 @@ struct IOOptions {
explicit IOOptions(bool force_dir_fsync_)
: timeout(std::chrono::microseconds::zero()),
prio(IOPriority::kIOLow),
rate_limiter_priority(Env::IO_TOTAL),
type(IOType::kUnknown),
force_dir_fsync(force_dir_fsync_) {}
};
+4
View File
@@ -92,6 +92,10 @@ class Iterator : public Cleanable {
// If supported, renew the iterator to represent the latest state. The
// iterator will be invalidated after the call. Not supported if
// ReadOptions.snapshot is given when creating the iterator.
//
// WARNING: Do not use `Iterator::Refresh()` API on DBs where `DeleteRange()`
// has been used or will be used. This feature combination is neither
// supported nor programmatically prevented.
virtual Status Refresh() {
return Status::NotSupported("Refresh() is not supported");
}
-7
View File
@@ -507,9 +507,6 @@ struct DBOptions {
// - Reads associated with a `ReadOptions` can be charged at
// `ReadOptions::rate_limiter_priority` (see that option's API doc for usage
// and limitations).
// - Writes associated with a `WriteOptions` can be charged at
// `WriteOptions::rate_limiter_priority` (see that option's API doc for
// usage and limitations).
//
// Rate limiting is disabled if nullptr. If rate limiter is enabled,
// bytes_per_sync is set to 1MB by default.
@@ -551,10 +548,6 @@ struct DBOptions {
// on target_file_size_base and target_file_size_multiplier for level-based
// compaction. For universal-style compaction, you can usually set it to -1.
//
// A high value or -1 for this option can cause high memory usage.
// See BlockBasedTableOptions::reserve_table_reader_memory to constrain
// memory usage in case of block based table format.
//
// Default: -1
//
// Dynamically changeable through SetDBOptions() API.
-2
View File
@@ -431,8 +431,6 @@ enum Tickers : uint32_t {
NON_LAST_LEVEL_READ_BYTES,
NON_LAST_LEVEL_READ_COUNT,
BLOCK_CHECKSUM_COMPUTE_COUNT,
TICKER_ENUM_MAX
};
+2 -25
View File
@@ -300,35 +300,12 @@ struct BlockBasedTableOptions {
//
// If additional temporary memory of Ribbon Filter uses up too much memory
// relative to the avaible space left in the block cache
// at some point (i.e, causing a cache full under
// LRUCacheOptions::strict_capacity_limit = true), construction will fall back
// to Bloom Filter.
// at some point (i.e, causing a cache full when strict_capacity_limit =
// true), construction will fall back to Bloom Filter.
//
// Default: false
bool reserve_table_builder_memory = false;
// If true, a dynamically updating charge to block cache, loosely based
// on the actual memory usage of table reader, will occur to account
// the memory, if block cache available.
//
// Charged memory usage includes:
// 1. Table properties
// 2. Index block/Filter block/Uncompression dictionary if stored in table
// reader (i.e, BlockBasedTableOptions::cache_index_and_filter_blocks ==
// false)
// 3. Some internal data structures
// 4. More to come...
//
// Note:
// If creation of a table reader uses up too much memory
// relative to the avaible space left in the block cache
// at some point (i.e, causing a cache full under
// LRUCacheOptions::strict_capacity_limit = true), such creation will fail
// with Status::MemoryLimit().
//
// Default: false
bool reserve_table_reader_memory = false;
// Note: currently this option requires kTwoLevelIndexSearch to be set as
// well.
// TODO(myabandeh): remove the note above once the limitation is lifted
-4
View File
@@ -301,10 +301,6 @@ struct TableProperties {
// between tables. Keys match field names in this class instead
// of using full property names.
std::map<std::string, uint64_t> GetAggregatablePropertiesAsMap() const;
// Return the approximated memory usage of this TableProperties object,
// including memory used by the string properties and UserCollectedProperties
std::size_t ApproximateMemoryUsage() const;
};
// Extra properties
-12
View File
@@ -557,18 +557,6 @@ class Transaction {
virtual Status RebuildFromWriteBatch(WriteBatch* src_batch) = 0;
// Note: data in the commit-time-write-batch bypasses concurrency control,
// thus should be used with great caution.
// For write-prepared/write-unprepared transactions,
// GetCommitTimeWriteBatch() can be used only if the transaction is started
// with
// `TransactionOptions::use_only_the_last_commit_time_batch_for_recovery` set
// to true. Otherwise, it is possible that two uncommitted versions of the
// same key exist in the database due to the current implementation (see the
// explanation in WritePreparedTxn::CommitInternal).
// During bottommost compaction, RocksDB may
// set the sequence numbers of both to zero once becoming committed, causing
// output SST file to have two identical internal keys.
virtual WriteBatch* GetCommitTimeWriteBatch() = 0;
virtual void SetLogNumber(uint64_t log) { log_number_ = log; }
+4 -4
View File
@@ -160,7 +160,8 @@ struct TransactionDBOptions {
// Increasing this value will increase the concurrency by dividing the lock
// table (per column family) into more sub-tables, each with their own
// separate mutex.
// separate
// mutex.
size_t num_stripes = 16;
// If positive, specifies the default wait timeout in milliseconds when
@@ -170,7 +171,8 @@ struct TransactionDBOptions {
// If 0, no waiting is done if a lock cannot instantly be acquired.
// If negative, there is no timeout. Not using a timeout is not recommended
// as it can lead to deadlocks. Currently, there is no deadlock-detection to
// recover from a deadlock.
// recover
// from a deadlock.
int64_t transaction_lock_timeout = 1000; // 1 second
// If positive, specifies the wait timeout in milliseconds when writing a key
@@ -259,8 +261,6 @@ struct TransactionOptions {
// meant to be used later during recovery. It enables an optimization to
// postpone updating the memtable with CommitTimeWriteBatch to only
// SwitchMemtable or recovery.
// This option does not affect write-committed. Only
// write-prepared/write-unprepared transactions will be affected.
bool use_only_the_last_commit_time_batch_for_recovery = false;
// TODO(agiardullo): TransactionDB does not yet support comparators that allow
@@ -18,6 +18,7 @@ namespace ROCKSDB_NAMESPACE {
//
// To open a TransactionDB with a custom TransactionDBMutexFactory, set
// TransactionDBOptions.custom_mutex_factory.
class TransactionDBMutex {
public:
virtual ~TransactionDBMutex() {}
+1 -1
View File
@@ -11,7 +11,7 @@
#define ROCKSDB_MAJOR 7
#define ROCKSDB_MINOR 1
#define ROCKSDB_PATCH 0
#define ROCKSDB_PATCH 2
// Do not use these. We made the mistake of declaring macros starting with
// double underscore. Now we have to live with our choice. We'll deprecate these
+1 -1
View File
@@ -158,7 +158,7 @@ class WriteBufferManager final {
std::atomic<size_t> memory_used_;
// Memory that hasn't been scheduled to free.
std::atomic<size_t> memory_active_;
std::shared_ptr<CacheReservationManager> cache_res_mgr_;
std::unique_ptr<CacheReservationManager> cache_res_mgr_;
// Protects cache_res_mgr_
std::mutex cache_res_mgr_mu_;
+1 -1
View File
@@ -7,7 +7,7 @@ endif()
set(CMAKE_JAVA_COMPILE_FLAGS -source 7)
set(JNI_NATIVE_SOURCES
rocksjni/backup_engine_options.cc
rocksjni/backupablejni.cc
rocksjni/backupenginejni.cc
rocksjni/cassandra_compactionfilterjni.cc
rocksjni/cassandra_value_operator.cc
-1
View File
@@ -187,7 +187,6 @@ JAVA_TESTS = \
org.rocksdb.TtlDBTest\
org.rocksdb.StatisticsTest\
org.rocksdb.StatisticsCollectorTest\
org.rocksdb.VerifyChecksumsTest\
org.rocksdb.WalFilterTest\
org.rocksdb.WALRecoveryModeTest\
org.rocksdb.WriteBatchHandlerTest\
File renamed without changes.
+24 -95
View File
@@ -6,15 +6,13 @@
// This file implements the "bridge" between Java and C++ and enables
// calling c++ ROCKSDB_NAMESPACE::Iterator methods from Java side.
#include "rocksdb/iterator.h"
#include <jni.h>
#include <stdio.h>
#include <stdlib.h>
#include <algorithm>
#include "include/org_rocksdb_RocksIterator.h"
#include "rocksdb/iterator.h"
#include "rocksjni/portal.h"
/*
@@ -89,7 +87,7 @@ void Java_org_rocksdb_RocksIterator_prev0(JNIEnv* /*env*/, jobject /*jobj*/,
* Signature: (J)V
*/
void Java_org_rocksdb_RocksIterator_refresh0(JNIEnv* env, jobject /*jobj*/,
jlong handle) {
jlong handle) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
ROCKSDB_NAMESPACE::Status s = it->Refresh();
@@ -108,31 +106,19 @@ void Java_org_rocksdb_RocksIterator_refresh0(JNIEnv* env, jobject /*jobj*/,
void Java_org_rocksdb_RocksIterator_seek0(JNIEnv* env, jobject /*jobj*/,
jlong handle, jbyteArray jtarget,
jint jtarget_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) {
it->Seek(target_slice);
};
ROCKSDB_NAMESPACE::JniUtil::k_op_region(seek, env, jtarget, 0, jtarget_len);
}
jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
if (target == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
ROCKSDB_NAMESPACE::Slice target_slice(reinterpret_cast<char*>(target),
jtarget_len);
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
* In this case, the buffer offset of the key may be non-zero.
*
* Class: org_rocksdb_RocksIterator
* Method: seek0
* Signature: (J[BII)V
*/
void Java_org_rocksdb_RocksIterator_seekByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget,
jint jtarget_off, jint jtarget_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) {
it->Seek(target_slice);
};
ROCKSDB_NAMESPACE::JniUtil::k_op_region(seek, env, jtarget, jtarget_off,
jtarget_len);
it->Seek(target_slice);
env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
}
/*
@@ -177,31 +163,19 @@ void Java_org_rocksdb_RocksIterator_seekForPrev0(JNIEnv* env, jobject /*jobj*/,
jlong handle,
jbyteArray jtarget,
jint jtarget_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) {
it->SeekForPrev(target_slice);
};
ROCKSDB_NAMESPACE::JniUtil::k_op_region(seek, env, jtarget, 0, jtarget_len);
}
jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
if (target == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
ROCKSDB_NAMESPACE::Slice target_slice(reinterpret_cast<char*>(target),
jtarget_len);
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
* In this case, the buffer offset of the key may be non-zero.
*
* Class: org_rocksdb_RocksIterator
* Method: seek0
* Signature: (J[BII)V
*/
void Java_org_rocksdb_RocksIterator_seekForPrevByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget,
jint jtarget_off, jint jtarget_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
auto seek = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) {
it->SeekForPrev(target_slice);
};
ROCKSDB_NAMESPACE::JniUtil::k_op_region(seek, env, jtarget, jtarget_off,
jtarget_len);
it->SeekForPrev(target_slice);
env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
}
/*
@@ -257,29 +231,6 @@ jint Java_org_rocksdb_RocksIterator_keyDirect0(JNIEnv* env, jobject /*jobj*/,
jtarget_off, jtarget_len);
}
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
*
* Class: org_rocksdb_RocksIterator
* Method: keyByteArray0
* Signature: (J[BII)I
*/
jint Java_org_rocksdb_RocksIterator_keyByteArray0(JNIEnv* env, jobject /*jobj*/,
jlong handle, jbyteArray jkey,
jint jkey_off,
jint jkey_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
ROCKSDB_NAMESPACE::Slice key_slice = it->key();
jsize copy_size = std::min(static_cast<uint32_t>(key_slice.size()),
static_cast<uint32_t>(jkey_len));
env->SetByteArrayRegion(
jkey, jkey_off, copy_size,
const_cast<jbyte*>(reinterpret_cast<const jbyte*>(key_slice.data())));
return static_cast<jsize>(key_slice.size());
}
/*
* Class: org_rocksdb_RocksIterator
* Method: value0
@@ -316,25 +267,3 @@ jint Java_org_rocksdb_RocksIterator_valueDirect0(JNIEnv* env, jobject /*jobj*/,
return ROCKSDB_NAMESPACE::JniUtil::copyToDirect(env, value_slice, jtarget,
jtarget_off, jtarget_len);
}
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
*
* Class: org_rocksdb_RocksIterator
* Method: valueByteArray0
* Signature: (J[BII)I
*/
jint Java_org_rocksdb_RocksIterator_valueByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jvalue_target,
jint jvalue_off, jint jvalue_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
ROCKSDB_NAMESPACE::Slice value_slice = it->value();
jsize copy_size = std::min(static_cast<uint32_t>(value_slice.size()),
static_cast<uint32_t>(jvalue_len));
env->SetByteArrayRegion(
jvalue_target, jvalue_off, copy_size,
const_cast<jbyte*>(reinterpret_cast<const jbyte*>(value_slice.data())));
return static_cast<jsize>(value_slice.size());
}
+3 -40
View File
@@ -2127,7 +2127,7 @@ class JniUtil {
std::function<ROCKSDB_NAMESPACE::Status(ROCKSDB_NAMESPACE::Slice)> op,
JNIEnv* env, jobject /*jobj*/, jbyteArray jkey, jint jkey_len) {
jbyte* key = env->GetByteArrayElements(jkey, nullptr);
if (env->ExceptionCheck()) {
if(env->ExceptionCheck()) {
// exception thrown: OutOfMemoryError
return nullptr;
}
@@ -2137,7 +2137,7 @@ class JniUtil {
auto status = op(key_slice);
if (key != nullptr) {
if(key != nullptr) {
env->ReleaseByteArrayElements(jkey, key, JNI_ABORT);
}
@@ -2145,37 +2145,6 @@ class JniUtil {
new ROCKSDB_NAMESPACE::Status(status));
}
/*
* Helper for operations on a key which is a region of an array
* Used to extract the common code from seek/seekForPrev.
* Possible that it can be generalised from that.
*
* We use GetByteArrayRegion to copy the key region of the whole array into
* a char[] We suspect this is not much slower than GetByteArrayElements,
* which probably copies anyway.
*/
static void k_op_region(std::function<void(ROCKSDB_NAMESPACE::Slice&)> op,
JNIEnv* env, jbyteArray jkey, jint jkey_off,
jint jkey_len) {
const std::unique_ptr<char[]> key(new char[jkey_len]);
if (key == nullptr) {
jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError");
env->ThrowNew(oom_class,
"Memory allocation failed in RocksDB JNI function");
return;
}
env->GetByteArrayRegion(jkey, jkey_off, jkey_len,
reinterpret_cast<jbyte*>(key.get()));
if (env->ExceptionCheck()) {
// exception thrown: OutOfMemoryError
return;
}
ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast<char*>(key.get()),
jkey_len);
op(key_slice);
}
/*
* Helper for operations on a value
* for example WriteBatchWithIndex->GetFromBatch
@@ -3468,7 +3437,7 @@ class BackupEngineJni
BackupEngineJni> {
public:
/**
* Get the Java Class org.rocksdb.BackupEngine
* Get the Java Class org.rocksdb.BackupableEngine
*
* @param env A pointer to the Java environment
*
@@ -5086,8 +5055,6 @@ class TickerTypeJni {
return -0x2C;
case ROCKSDB_NAMESPACE::Tickers::NON_LAST_LEVEL_READ_COUNT:
return -0x2D;
case ROCKSDB_NAMESPACE::Tickers::BLOCK_CHECKSUM_COMPUTE_COUNT:
return -0x2E;
case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX:
// 0x5F was the max value in the initial copy of tickers to Java.
// Since these values are exposed directly to Java clients, we keep
@@ -5457,8 +5424,6 @@ class TickerTypeJni {
return ROCKSDB_NAMESPACE::Tickers::NON_LAST_LEVEL_READ_BYTES;
case -0x2D:
return ROCKSDB_NAMESPACE::Tickers::NON_LAST_LEVEL_READ_COUNT;
case -0x2E:
return ROCKSDB_NAMESPACE::Tickers::BLOCK_CHECKSUM_COMPUTE_COUNT;
case 0x5F:
// 0x5F was the max value in the initial copy of tickers to Java.
// Since these values are exposed directly to Java clients, we keep
@@ -6578,8 +6543,6 @@ class ChecksumTypeJni {
return 0x2;
case ROCKSDB_NAMESPACE::ChecksumType::kxxHash64:
return 0x3;
case ROCKSDB_NAMESPACE::ChecksumType::kXXH3:
return 0x4;
default:
return 0x7F; // undefined
}
+7 -6
View File
@@ -829,11 +829,9 @@ bool rocksdb_single_delete_helper(
const ROCKSDB_NAMESPACE::WriteOptions& write_options,
ROCKSDB_NAMESPACE::ColumnFamilyHandle* cf_handle, jbyteArray jkey,
jint jkey_len) {
jbyte* key = new jbyte[jkey_len];
env->GetByteArrayRegion(jkey, 0, jkey_len, key);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
delete[] key;
jbyte* key = env->GetByteArrayElements(jkey, nullptr);
if (key == nullptr) {
// exception thrown: OutOfMemoryError
return false;
}
ROCKSDB_NAMESPACE::Slice key_slice(reinterpret_cast<char*>(key), jkey_len);
@@ -846,7 +844,10 @@ bool rocksdb_single_delete_helper(
s = db->SingleDelete(write_options, key_slice);
}
delete[] key;
// trigger java unref on key and value.
// by passing JNI_ABORT, it will simply release the reference without
// copying the result back to the java byte array.
env->ReleaseByteArrayElements(jkey, key, JNI_ABORT);
if (s.ok()) {
return true;
-100
View File
@@ -206,29 +206,6 @@ jint Java_org_rocksdb_SstFileReaderIterator_keyDirect0(
jtarget_off, jtarget_len);
}
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
*
* Class: org_rocksdb_SstFileReaderIterator
* Method: keyByteArray0
* Signature: (J[BII)I
*/
jint Java_org_rocksdb_SstFileReaderIterator_keyByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jkey, jint jkey_off,
jint jkey_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
ROCKSDB_NAMESPACE::Slice key_slice = it->key();
auto slice_size = key_slice.size();
jsize copy_size = std::min(static_cast<uint32_t>(slice_size),
static_cast<uint32_t>(jkey_len));
env->SetByteArrayRegion(
jkey, jkey_off, copy_size,
const_cast<jbyte*>(reinterpret_cast<const jbyte*>(key_slice.data())));
return static_cast<jsize>(slice_size);
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: valueDirect0
@@ -243,29 +220,6 @@ jint Java_org_rocksdb_SstFileReaderIterator_valueDirect0(
jtarget_off, jtarget_len);
}
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
*
* Class: org_rocksdb_SstFileReaderIterator
* Method: valueByteArray0
* Signature: (J[BII)I
*/
jint Java_org_rocksdb_SstFileReaderIterator_valueByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jvalue_target,
jint jvalue_off, jint jvalue_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
ROCKSDB_NAMESPACE::Slice value_slice = it->value();
auto slice_size = value_slice.size();
jsize copy_size = std::min(static_cast<uint32_t>(slice_size),
static_cast<uint32_t>(jvalue_len));
env->SetByteArrayRegion(
jvalue_target, jvalue_off, copy_size,
const_cast<jbyte*>(reinterpret_cast<const jbyte*>(value_slice.data())));
return static_cast<jsize>(slice_size);
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: seekDirect0
@@ -298,60 +252,6 @@ void Java_org_rocksdb_SstFileReaderIterator_seekForPrevDirect0(
jtarget_len);
}
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
*
* Class: org_rocksdb_SstFileReaderIterator
* Method: seekByteArray0
* Signature: (J[BII)V
*/
void Java_org_rocksdb_SstFileReaderIterator_seekByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget,
jint jtarget_off, jint jtarget_len) {
const std::unique_ptr<char[]> target(new char[jtarget_len]);
if (target == nullptr) {
jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError");
env->ThrowNew(oom_class,
"Memory allocation failed in RocksDB JNI function");
return;
}
env->GetByteArrayRegion(jtarget, jtarget_off, jtarget_len,
reinterpret_cast<jbyte*>(target.get()));
ROCKSDB_NAMESPACE::Slice target_slice(target.get(), jtarget_len);
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
it->Seek(target_slice);
}
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
*
* Class: org_rocksdb_SstFileReaderIterator
* Method: seekForPrevByteArray0
* Signature: (J[BII)V
*/
void Java_org_rocksdb_SstFileReaderIterator_seekForPrevByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget,
jint jtarget_off, jint jtarget_len) {
const std::unique_ptr<char[]> target(new char[jtarget_len]);
if (target == nullptr) {
jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError");
env->ThrowNew(oom_class,
"Memory allocation failed in RocksDB JNI function");
return;
}
env->GetByteArrayRegion(jtarget, jtarget_off, jtarget_len,
reinterpret_cast<jbyte*>(target.get()));
ROCKSDB_NAMESPACE::Slice target_slice(target.get(), jtarget_len);
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::Iterator*>(handle);
it->SeekForPrev(target_slice);
}
/*
* Class: org_rocksdb_SstFileReaderIterator
* Method: refresh0
+8 -82
View File
@@ -735,11 +735,9 @@ void Java_org_rocksdb_WBWIRocksIterator_seek0(JNIEnv* env, jobject /*jobj*/,
jlong handle, jbyteArray jtarget,
jint jtarget_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::WBWIIterator*>(handle);
jbyte* target = new jbyte[jtarget_len];
env->GetByteArrayRegion(jtarget, 0, jtarget_len, target);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
delete[] target;
jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
if (target == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
@@ -748,7 +746,7 @@ void Java_org_rocksdb_WBWIRocksIterator_seek0(JNIEnv* env, jobject /*jobj*/,
it->Seek(target_slice);
delete[] target;
env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
}
/*
@@ -767,33 +765,6 @@ void Java_org_rocksdb_WBWIRocksIterator_seekDirect0(
jtarget_len);
}
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
*
* Class: org_rocksdb_WBWIRocksIterator
* Method: seekByteArray0
* Signature: (J[BII)V
*/
void Java_org_rocksdb_WBWIRocksIterator_seekByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget,
jint jtarget_off, jint jtarget_len) {
const std::unique_ptr<char[]> target(new char[jtarget_len]);
if (target == nullptr) {
jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError");
env->ThrowNew(oom_class,
"Memory allocation failed in RocksDB JNI function");
return;
}
env->GetByteArrayRegion(jtarget, jtarget_off, jtarget_len,
reinterpret_cast<jbyte*>(target.get()));
ROCKSDB_NAMESPACE::Slice target_slice(target.get(), jtarget_len);
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::WBWIIterator*>(handle);
it->Seek(target_slice);
}
/*
* Class: org_rocksdb_WBWIRocksIterator
* Method: seekForPrev0
@@ -805,11 +776,9 @@ void Java_org_rocksdb_WBWIRocksIterator_seekForPrev0(JNIEnv* env,
jbyteArray jtarget,
jint jtarget_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::WBWIIterator*>(handle);
jbyte* target = new jbyte[jtarget_len];
env->GetByteArrayRegion(jtarget, 0, jtarget_len, target);
if (env->ExceptionCheck()) {
// exception thrown: ArrayIndexOutOfBoundsException
delete[] target;
jbyte* target = env->GetByteArrayElements(jtarget, nullptr);
if (target == nullptr) {
// exception thrown: OutOfMemoryError
return;
}
@@ -818,50 +787,7 @@ void Java_org_rocksdb_WBWIRocksIterator_seekForPrev0(JNIEnv* env,
it->SeekForPrev(target_slice);
delete[] target;
}
/*
* Class: org_rocksdb_WBWIRocksIterator
* Method: seekForPrevDirect0
* Signature: (JLjava/nio/ByteBuffer;II)V
*/
void Java_org_rocksdb_WBWIRocksIterator_seekForPrevDirect0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jobject jtarget,
jint jtarget_off, jint jtarget_len) {
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::WBWIIterator*>(handle);
auto seek_for_prev = [&it](ROCKSDB_NAMESPACE::Slice& target_slice) {
it->SeekForPrev(target_slice);
};
ROCKSDB_NAMESPACE::JniUtil::k_op_direct(seek_for_prev, env, jtarget,
jtarget_off, jtarget_len);
}
/*
* This method supports fetching into indirect byte buffers;
* the Java wrapper extracts the byte[] and passes it here.
*
* Class: org_rocksdb_WBWIRocksIterator
* Method: seekForPrevByteArray0
* Signature: (J[BII)V
*/
void Java_org_rocksdb_WBWIRocksIterator_seekForPrevByteArray0(
JNIEnv* env, jobject /*jobj*/, jlong handle, jbyteArray jtarget,
jint jtarget_off, jint jtarget_len) {
const std::unique_ptr<char[]> target(new char[jtarget_len]);
if (target == nullptr) {
jclass oom_class = env->FindClass("/lang/java/OutOfMemoryError");
env->ThrowNew(oom_class,
"Memory allocation failed in RocksDB JNI function");
return;
}
env->GetByteArrayRegion(jtarget, jtarget_off, jtarget_len,
reinterpret_cast<jbyte*>(target.get()));
ROCKSDB_NAMESPACE::Slice target_slice(target.get(), jtarget_len);
auto* it = reinterpret_cast<ROCKSDB_NAMESPACE::WBWIIterator*>(handle);
it->SeekForPrev(target_slice);
env->ReleaseByteArrayElements(jtarget, target, JNI_ABORT);
}
/*
@@ -55,40 +55,30 @@ public abstract class AbstractRocksIterator<P extends RocksObject>
}
@Override
public void seek(final byte[] target) {
public void seek(byte[] target) {
assert (isOwningHandle());
seek0(nativeHandle_, target, target.length);
}
@Override
public void seekForPrev(final byte[] target) {
assert (isOwningHandle());
seekForPrev0(nativeHandle_, target, target.length);
}
@Override
public void seekForPrev(byte[] target) {
assert (isOwningHandle());
seekForPrev0(nativeHandle_, target, target.length);
}
@Override
public void seek(final ByteBuffer target) {
assert (isOwningHandle());
if (target.isDirect()) {
seekDirect0(nativeHandle_, target, target.position(), target.remaining());
} else {
seekByteArray0(nativeHandle_, target.array(), target.arrayOffset() + target.position(),
target.remaining());
}
target.position(target.limit());
}
@Override
public void seek(ByteBuffer target) {
assert (isOwningHandle() && target.isDirect());
seekDirect0(nativeHandle_, target, target.position(), target.remaining());
target.position(target.limit());
}
@Override
public void seekForPrev(final ByteBuffer target) {
assert (isOwningHandle());
if (target.isDirect()) {
seekForPrevDirect0(nativeHandle_, target, target.position(), target.remaining());
} else {
seekForPrevByteArray0(nativeHandle_, target.array(), target.arrayOffset() + target.position(),
target.remaining());
}
target.position(target.limit());
}
@Override
public void seekForPrev(ByteBuffer target) {
assert (isOwningHandle() && target.isDirect());
seekForPrevDirect0(nativeHandle_, target, target.position(), target.remaining());
target.position(target.limit());
}
@Override
public void next() {
@@ -139,8 +129,5 @@ public abstract class AbstractRocksIterator<P extends RocksObject>
abstract void seekForPrev0(long handle, byte[] target, int targetLen);
abstract void seekDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen);
abstract void seekForPrevDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen);
abstract void seekByteArray0(long handle, byte[] target, int targetOffset, int targetLen);
abstract void seekForPrevByteArray0(long handle, byte[] target, int targetOffset, int targetLen);
abstract void status0(long handle) throws RocksDBException;
}
@@ -226,8 +226,8 @@ public class BackupEngine extends RocksObject implements AutoCloseable {
restoreOptions.nativeHandle_);
}
private native static long open(final long env, final long backupEngineOptions)
throws RocksDBException;
private native static long open(final long env,
final long backupableDbOptions) throws RocksDBException;
private native void createNewBackup(final long handle, final long dbHandle,
final boolean flushBeforeBackup) throws RocksDBException;
@@ -8,8 +8,8 @@ package org.rocksdb;
import java.io.File;
/**
* <p>BackupEngineOptions controls the behavior of a
* {@link org.rocksdb.BackupEngine}.
* <p>BackupEngineOptions to control the behavior of a backupable database.
* It will be used during the creation of a {@link org.rocksdb.BackupEngine}.
* </p>
* <p>Note that dispose() must be called before an Options instance
* become out-of-scope to release the allocated memory in c++.</p>
@@ -24,9 +24,7 @@ public enum ChecksumType {
/**
* XX Hash 64
*/
kxxHash64((byte) 3),
kXXH3((byte) 4);
kxxHash64((byte) 3);
/**
* Returns the byte value of the enumerations value
@@ -37,7 +35,7 @@ public enum ChecksumType {
return value_;
}
private ChecksumType(final byte value) {
private ChecksumType(byte value) {
value_ = value;
}
@@ -21,7 +21,7 @@ import java.nio.ByteBuffer;
* @see org.rocksdb.RocksObject
*/
public class RocksIterator extends AbstractRocksIterator<RocksDB> {
protected RocksIterator(final RocksDB rocksDB, final long nativeHandle) {
protected RocksIterator(RocksDB rocksDB, long nativeHandle) {
super(rocksDB, nativeHandle);
}
@@ -54,16 +54,9 @@ public class RocksIterator extends AbstractRocksIterator<RocksDB> {
* input buffer {@code key} is insufficient and partial result will
* be returned.
*/
public int key(final ByteBuffer key) {
assert isOwningHandle();
final int result;
if (key.isDirect()) {
result = keyDirect0(nativeHandle_, key, key.position(), key.remaining());
} else {
assert key.hasArray();
result = keyByteArray0(
nativeHandle_, key.array(), key.arrayOffset() + key.position(), key.remaining());
}
public int key(ByteBuffer key) {
assert (isOwningHandle() && key.isDirect());
int result = keyDirect0(nativeHandle_, key, key.position(), key.remaining());
key.limit(Math.min(key.position() + result, key.limit()));
return result;
}
@@ -96,16 +89,9 @@ public class RocksIterator extends AbstractRocksIterator<RocksDB> {
* input buffer {@code value} is insufficient and partial result will
* be returned.
*/
public int value(final ByteBuffer value) {
assert isOwningHandle();
final int result;
if (value.isDirect()) {
result = valueDirect0(nativeHandle_, value, value.position(), value.remaining());
} else {
assert value.hasArray();
result = valueByteArray0(
nativeHandle_, value.array(), value.arrayOffset() + value.position(), value.remaining());
}
public int value(ByteBuffer value) {
assert (isOwningHandle() && value.isDirect());
int result = valueDirect0(nativeHandle_, value, value.position(), value.remaining());
value.limit(Math.min(value.position() + result, value.limit()));
return result;
}
@@ -122,19 +108,12 @@ public class RocksIterator extends AbstractRocksIterator<RocksDB> {
@Override
final native void seekDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen);
@Override
final native void seekByteArray0(long handle, byte[] target, int targetOffset, int targetLen);
@Override
final native void seekForPrevDirect0(
long handle, ByteBuffer target, int targetOffset, int targetLen);
@Override
final native void seekForPrevByteArray0(
long handle, byte[] target, int targetOffset, int targetLen);
@Override final native void status0(long handle) throws RocksDBException;
private native byte[] key0(long handle);
private native byte[] value0(long handle);
private native int keyDirect0(long handle, ByteBuffer buffer, int bufferOffset, int bufferLen);
private native int keyByteArray0(long handle, byte[] array, int arrayOffset, int arrayLen);
private native int valueDirect0(long handle, ByteBuffer buffer, int bufferOffset, int bufferLen);
private native int valueByteArray0(long handle, byte[] array, int arrayOffset, int arrayLen);
}
@@ -21,7 +21,7 @@ import java.nio.ByteBuffer;
* @see RocksObject
*/
public class SstFileReaderIterator extends AbstractRocksIterator<SstFileReader> {
protected SstFileReaderIterator(final SstFileReader reader, final long nativeHandle) {
protected SstFileReaderIterator(SstFileReader reader, long nativeHandle) {
super(reader, nativeHandle);
}
@@ -54,15 +54,9 @@ public class SstFileReaderIterator extends AbstractRocksIterator<SstFileReader>
* input buffer {@code key} is insufficient and partial result will
* be returned.
*/
public int key(final ByteBuffer key) {
assert (isOwningHandle());
final int result;
if (key.isDirect()) {
result = keyDirect0(nativeHandle_, key, key.position(), key.remaining());
} else {
result = keyByteArray0(
nativeHandle_, key.array(), key.arrayOffset() + key.position(), key.remaining());
}
public int key(ByteBuffer key) {
assert (isOwningHandle() && key.isDirect());
int result = keyDirect0(nativeHandle_, key, key.position(), key.remaining());
key.limit(Math.min(key.position() + result, key.limit()));
return result;
}
@@ -95,15 +89,9 @@ public class SstFileReaderIterator extends AbstractRocksIterator<SstFileReader>
* input buffer {@code value} is insufficient and partial result will
* be returned.
*/
public int value(final ByteBuffer value) {
assert (isOwningHandle());
final int result;
if (value.isDirect()) {
result = valueDirect0(nativeHandle_, value, value.position(), value.remaining());
} else {
result = valueByteArray0(
nativeHandle_, value.array(), value.arrayOffset() + value.position(), value.remaining());
}
public int value(ByteBuffer value) {
assert (isOwningHandle() && value.isDirect());
int result = valueDirect0(nativeHandle_, value, value.position(), value.remaining());
value.limit(Math.min(value.position() + result, value.limit()));
return result;
}
@@ -118,23 +106,16 @@ public class SstFileReaderIterator extends AbstractRocksIterator<SstFileReader>
@Override final native void seek0(long handle, byte[] target, int targetLen);
@Override final native void seekForPrev0(long handle, byte[] target, int targetLen);
@Override final native void status0(long handle) throws RocksDBException;
@Override
final native void seekDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen);
@Override
final native void seekForPrevDirect0(
long handle, ByteBuffer target, int targetOffset, int targetLen);
@Override
final native void seekByteArray0(
final long handle, final byte[] target, final int targetOffset, final int targetLen);
@Override
final native void seekForPrevByteArray0(
final long handle, final byte[] target, final int targetOffset, final int targetLen);
private native byte[] key0(long handle);
private native byte[] value0(long handle);
private native int keyDirect0(long handle, ByteBuffer buffer, int bufferOffset, int bufferLen);
private native int keyByteArray0(long handle, byte[] buffer, int bufferOffset, int bufferLen);
private native int valueDirect0(long handle, ByteBuffer buffer, int bufferOffset, int bufferLen);
private native int valueByteArray0(long handle, byte[] buffer, int bufferOffset, int bufferLen);
@Override
final native void seekDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen);
@Override
final native void seekForPrevDirect0(
long handle, ByteBuffer target, int targetOffset, int targetLen);
}
@@ -804,8 +804,6 @@ public enum TickerType {
NON_LAST_LEVEL_READ_BYTES((byte) -0x2C),
NON_LAST_LEVEL_READ_COUNT((byte) -0x2D),
BLOCK_CHECKSUM_COMPUTE_COUNT((byte) -0x2E),
TICKER_ENUM_MAX((byte) 0x5F);
private final byte value;
@@ -31,7 +31,7 @@ public class WBWIRocksIterator
*/
public WriteEntry entry() {
assert(isOwningHandle());
final long[] ptrs = entry1(nativeHandle_);
final long ptrs[] = entry1(nativeHandle_);
entry.type = WriteType.fromId((byte)ptrs[0]);
entry.key.resetNativeHandle(ptrs[1], ptrs[1] != 0);
@@ -51,17 +51,7 @@ public class WBWIRocksIterator
@Override final native void seekForPrev0(long handle, byte[] target, int targetLen);
@Override final native void status0(long handle) throws RocksDBException;
@Override
final native void seekDirect0(
final long handle, final ByteBuffer target, final int targetOffset, final int targetLen);
@Override
final native void seekForPrevDirect0(
final long handle, final ByteBuffer target, final int targetOffset, final int targetLen);
@Override
final native void seekByteArray0(
final long handle, final byte[] target, final int targetOffset, final int targetLen);
@Override
final native void seekForPrevByteArray0(
final long handle, final byte[] target, final int targetOffset, final int targetLen);
final native void seekDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen);
private native long[] entry1(final long handle);
@@ -200,4 +190,9 @@ public class WBWIRocksIterator
key.close();
}
}
@Override
void seekForPrevDirect0(long handle, ByteBuffer target, int targetOffset, int targetLen) {
throw new IllegalAccessError("Not implemented");
}
}
@@ -30,36 +30,41 @@ public class BackupEngineOptionsTest {
@Test
public void backupDir() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupEngineOptions.backupDir()).isEqualTo(ARBITRARY_PATH);
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupableDBOptions.backupDir()).
isEqualTo(ARBITRARY_PATH);
}
}
@Test
public void env() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupEngineOptions.backupEnv()).isNull();
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupableDBOptions.backupEnv()).
isNull();
try(final Env env = new RocksMemEnv(Env.getDefault())) {
backupEngineOptions.setBackupEnv(env);
assertThat(backupEngineOptions.backupEnv()).isEqualTo(env);
backupableDBOptions.setBackupEnv(env);
assertThat(backupableDBOptions.backupEnv())
.isEqualTo(env);
}
}
}
@Test
public void shareTableFiles() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
final boolean value = rand.nextBoolean();
backupEngineOptions.setShareTableFiles(value);
assertThat(backupEngineOptions.shareTableFiles()).isEqualTo(value);
backupableDBOptions.setShareTableFiles(value);
assertThat(backupableDBOptions.shareTableFiles()).
isEqualTo(value);
}
}
@Test
public void infoLog() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupEngineOptions.infoLog()).isNull();
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupableDBOptions.infoLog()).
isNull();
try(final Options options = new Options();
final Logger logger = new Logger(options){
@@ -68,113 +73,127 @@ public class BackupEngineOptionsTest {
}
}) {
backupEngineOptions.setInfoLog(logger);
assertThat(backupEngineOptions.infoLog()).isEqualTo(logger);
backupableDBOptions.setInfoLog(logger);
assertThat(backupableDBOptions.infoLog())
.isEqualTo(logger);
}
}
}
@Test
public void sync() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
final boolean value = rand.nextBoolean();
backupEngineOptions.setSync(value);
assertThat(backupEngineOptions.sync()).isEqualTo(value);
backupableDBOptions.setSync(value);
assertThat(backupableDBOptions.sync()).isEqualTo(value);
}
}
@Test
public void destroyOldData() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH);) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH);) {
final boolean value = rand.nextBoolean();
backupEngineOptions.setDestroyOldData(value);
assertThat(backupEngineOptions.destroyOldData()).isEqualTo(value);
backupableDBOptions.setDestroyOldData(value);
assertThat(backupableDBOptions.destroyOldData()).
isEqualTo(value);
}
}
@Test
public void backupLogFiles() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
final boolean value = rand.nextBoolean();
backupEngineOptions.setBackupLogFiles(value);
assertThat(backupEngineOptions.backupLogFiles()).isEqualTo(value);
backupableDBOptions.setBackupLogFiles(value);
assertThat(backupableDBOptions.backupLogFiles()).
isEqualTo(value);
}
}
@Test
public void backupRateLimit() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
final long value = Math.abs(rand.nextLong());
backupEngineOptions.setBackupRateLimit(value);
assertThat(backupEngineOptions.backupRateLimit()).isEqualTo(value);
backupableDBOptions.setBackupRateLimit(value);
assertThat(backupableDBOptions.backupRateLimit()).
isEqualTo(value);
// negative will be mapped to 0
backupEngineOptions.setBackupRateLimit(-1);
assertThat(backupEngineOptions.backupRateLimit()).isEqualTo(0);
backupableDBOptions.setBackupRateLimit(-1);
assertThat(backupableDBOptions.backupRateLimit()).
isEqualTo(0);
}
}
@Test
public void backupRateLimiter() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupEngineOptions.backupEnv()).isNull();
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupableDBOptions.backupEnv()).
isNull();
try(final RateLimiter backupRateLimiter =
new RateLimiter(999)) {
backupEngineOptions.setBackupRateLimiter(backupRateLimiter);
assertThat(backupEngineOptions.backupRateLimiter()).isEqualTo(backupRateLimiter);
backupableDBOptions.setBackupRateLimiter(backupRateLimiter);
assertThat(backupableDBOptions.backupRateLimiter())
.isEqualTo(backupRateLimiter);
}
}
}
@Test
public void restoreRateLimit() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
final long value = Math.abs(rand.nextLong());
backupEngineOptions.setRestoreRateLimit(value);
assertThat(backupEngineOptions.restoreRateLimit()).isEqualTo(value);
backupableDBOptions.setRestoreRateLimit(value);
assertThat(backupableDBOptions.restoreRateLimit()).
isEqualTo(value);
// negative will be mapped to 0
backupEngineOptions.setRestoreRateLimit(-1);
assertThat(backupEngineOptions.restoreRateLimit()).isEqualTo(0);
backupableDBOptions.setRestoreRateLimit(-1);
assertThat(backupableDBOptions.restoreRateLimit()).
isEqualTo(0);
}
}
@Test
public void restoreRateLimiter() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupEngineOptions.backupEnv()).isNull();
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
assertThat(backupableDBOptions.backupEnv()).
isNull();
try(final RateLimiter restoreRateLimiter =
new RateLimiter(911)) {
backupEngineOptions.setRestoreRateLimiter(restoreRateLimiter);
assertThat(backupEngineOptions.restoreRateLimiter()).isEqualTo(restoreRateLimiter);
backupableDBOptions.setRestoreRateLimiter(restoreRateLimiter);
assertThat(backupableDBOptions.restoreRateLimiter())
.isEqualTo(restoreRateLimiter);
}
}
}
@Test
public void shareFilesWithChecksum() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
boolean value = rand.nextBoolean();
backupEngineOptions.setShareFilesWithChecksum(value);
assertThat(backupEngineOptions.shareFilesWithChecksum()).isEqualTo(value);
backupableDBOptions.setShareFilesWithChecksum(value);
assertThat(backupableDBOptions.shareFilesWithChecksum()).
isEqualTo(value);
}
}
@Test
public void maxBackgroundOperations() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
final int value = rand.nextInt();
backupEngineOptions.setMaxBackgroundOperations(value);
assertThat(backupEngineOptions.maxBackgroundOperations()).isEqualTo(value);
backupableDBOptions.setMaxBackgroundOperations(value);
assertThat(backupableDBOptions.maxBackgroundOperations()).
isEqualTo(value);
}
}
@Test
public void callbackTriggerIntervalSize() {
try (final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
try (final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH)) {
final long value = rand.nextLong();
backupEngineOptions.setCallbackTriggerIntervalSize(value);
assertThat(backupEngineOptions.callbackTriggerIntervalSize()).isEqualTo(value);
backupableDBOptions.setCallbackTriggerIntervalSize(value);
assertThat(backupableDBOptions.callbackTriggerIntervalSize()).
isEqualTo(value);
}
}
@@ -292,9 +311,9 @@ public class BackupEngineOptionsTest {
}
private BackupEngineOptions setupUninitializedBackupEngineOptions(ExpectedException exception) {
final BackupEngineOptions backupEngineOptions = new BackupEngineOptions(ARBITRARY_PATH);
backupEngineOptions.close();
final BackupEngineOptions backupableDBOptions = new BackupEngineOptions(ARBITRARY_PATH);
backupableDBOptions.close();
exception.expect(AssertionError.class);
return backupEngineOptions;
return backupableDBOptions;
}
}
@@ -62,37 +62,35 @@ public class BlockBasedTableConfigTest {
final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assertThat(IndexType.values().length).isEqualTo(4);
blockBasedTableConfig.setIndexType(IndexType.kHashSearch);
assertThat(blockBasedTableConfig.indexType()).isEqualTo(IndexType.kHashSearch);
assertThat(blockBasedTableConfig.indexType().equals(
IndexType.kHashSearch));
assertThat(IndexType.valueOf("kBinarySearch")).isNotNull();
blockBasedTableConfig.setIndexType(IndexType.valueOf("kBinarySearch"));
assertThat(blockBasedTableConfig.indexType()).isEqualTo(IndexType.kBinarySearch);
assertThat(blockBasedTableConfig.indexType().equals(
IndexType.kBinarySearch));
}
@Test
public void dataBlockIndexType() {
final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
blockBasedTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash);
assertThat(blockBasedTableConfig.dataBlockIndexType())
.isEqualTo(DataBlockIndexType.kDataBlockBinaryAndHash);
assertThat(blockBasedTableConfig.dataBlockIndexType().equals(
DataBlockIndexType.kDataBlockBinaryAndHash));
blockBasedTableConfig.setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch);
assertThat(blockBasedTableConfig.dataBlockIndexType())
.isEqualTo(DataBlockIndexType.kDataBlockBinarySearch);
assertThat(blockBasedTableConfig.dataBlockIndexType().equals(
DataBlockIndexType.kDataBlockBinarySearch));
}
@Test
public void checksumType() {
final BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig();
assertThat(ChecksumType.values().length).isEqualTo(5);
assertThat(ChecksumType.values().length).isEqualTo(4);
assertThat(ChecksumType.valueOf("kxxHash")).
isEqualTo(ChecksumType.kxxHash);
blockBasedTableConfig.setChecksumType(ChecksumType.kNoChecksum);
assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kNoChecksum);
blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash);
assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kxxHash);
blockBasedTableConfig.setChecksumType(ChecksumType.kxxHash64);
assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kxxHash64);
blockBasedTableConfig.setChecksumType(ChecksumType.kXXH3);
assertThat(blockBasedTableConfig.checksumType()).isEqualTo(ChecksumType.kXXH3);
assertThat(blockBasedTableConfig.checksumType().equals(
ChecksumType.kxxHash));
}
@Test
@@ -7,7 +7,6 @@ package org.rocksdb;
import static org.assertj.core.api.Assertions.assertThat;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
@@ -22,33 +21,13 @@ public class RocksIteratorTest {
@Rule
public TemporaryFolder dbFolder = new TemporaryFolder();
private void validateByteBufferResult(
final int fetched, final ByteBuffer byteBuffer, final String expected) {
assertThat(fetched).isEqualTo(expected.length());
assertThat(byteBuffer.position()).isEqualTo(0);
assertThat(byteBuffer.limit()).isEqualTo(Math.min(byteBuffer.remaining(), expected.length()));
final int bufferSpace = byteBuffer.remaining();
final byte[] contents = new byte[bufferSpace];
byteBuffer.get(contents, 0, bufferSpace);
assertThat(contents).isEqualTo(
expected.substring(0, bufferSpace).getBytes(StandardCharsets.UTF_8));
}
private void validateKey(
final RocksIterator iterator, final ByteBuffer byteBuffer, final String key) {
validateByteBufferResult(iterator.key(byteBuffer), byteBuffer, key);
}
private void validateValue(
final RocksIterator iterator, final ByteBuffer byteBuffer, final String value) {
validateByteBufferResult(iterator.value(byteBuffer), byteBuffer, value);
}
@Test
public void rocksIterator() throws RocksDBException {
try (final Options options =
new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) {
try (final Options options = new Options()
.setCreateIfMissing(true)
.setCreateMissingColumnFamilies(true);
final RocksDB db = RocksDB.open(options,
dbFolder.getRoot().getAbsolutePath())) {
db.put("key1".getBytes(), "value1".getBytes());
db.put("key2".getBytes(), "value2".getBytes());
@@ -58,20 +37,37 @@ public class RocksIteratorTest {
assertThat(iterator.key()).isEqualTo("key1".getBytes());
assertThat(iterator.value()).isEqualTo("value1".getBytes());
validateKey(iterator, ByteBuffer.allocateDirect(2), "key1");
validateKey(iterator, ByteBuffer.allocateDirect(2), "key0");
validateKey(iterator, ByteBuffer.allocateDirect(4), "key1");
validateKey(iterator, ByteBuffer.allocateDirect(5), "key1");
validateValue(iterator, ByteBuffer.allocateDirect(2), "value2");
validateValue(iterator, ByteBuffer.allocateDirect(2), "vasicu");
validateValue(iterator, ByteBuffer.allocateDirect(8), "value1");
ByteBuffer key = ByteBuffer.allocateDirect(2);
ByteBuffer value = ByteBuffer.allocateDirect(2);
assertThat(iterator.key(key)).isEqualTo(4);
assertThat(iterator.value(value)).isEqualTo(6);
validateKey(iterator, ByteBuffer.allocate(2), "key1");
validateKey(iterator, ByteBuffer.allocate(2), "key0");
validateKey(iterator, ByteBuffer.allocate(4), "key1");
validateKey(iterator, ByteBuffer.allocate(5), "key1");
validateValue(iterator, ByteBuffer.allocate(2), "value1");
validateValue(iterator, ByteBuffer.allocate(8), "value1");
assertThat(key.position()).isEqualTo(0);
assertThat(key.limit()).isEqualTo(2);
assertThat(value.position()).isEqualTo(0);
assertThat(value.limit()).isEqualTo(2);
byte[] tmp = new byte[2];
key.get(tmp);
assertThat(tmp).isEqualTo("ke".getBytes());
value.get(tmp);
assertThat(tmp).isEqualTo("va".getBytes());
key = ByteBuffer.allocateDirect(12);
value = ByteBuffer.allocateDirect(12);
assertThat(iterator.key(key)).isEqualTo(4);
assertThat(iterator.value(value)).isEqualTo(6);
assertThat(key.position()).isEqualTo(0);
assertThat(key.limit()).isEqualTo(4);
assertThat(value.position()).isEqualTo(0);
assertThat(value.limit()).isEqualTo(6);
tmp = new byte[4];
key.get(tmp);
assertThat(tmp).isEqualTo("key1".getBytes());
tmp = new byte[6];
value.get(tmp);
assertThat(tmp).isEqualTo("value1".getBytes());
iterator.next();
assertThat(iterator.isValid()).isTrue();
@@ -91,85 +87,24 @@ public class RocksIteratorTest {
assertThat(iterator.value()).isEqualTo("value2".getBytes());
iterator.status();
{
final ByteBuffer key = ByteBuffer.allocate(12);
key.put("key1".getBytes()).flip();
iterator.seek(key);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.value()).isEqualTo("value1".getBytes());
assertThat(key.position()).isEqualTo(4);
assertThat(key.limit()).isEqualTo(4);
key.clear();
key.put("key1".getBytes());
key.flip();
iterator.seek(key);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.value()).isEqualTo("value1".getBytes());
assertThat(key.position()).isEqualTo(4);
assertThat(key.limit()).isEqualTo(4);
validateValue(iterator, ByteBuffer.allocateDirect(12), "value1");
validateValue(iterator, ByteBuffer.allocateDirect(4), "valu56");
}
{
final ByteBuffer key = ByteBuffer.allocate(12);
key.put("key2".getBytes()).flip();
iterator.seekForPrev(key);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.value()).isEqualTo("value2".getBytes());
assertThat(key.position()).isEqualTo(4);
assertThat(key.limit()).isEqualTo(4);
}
{
final ByteBuffer key = ByteBuffer.allocate(12);
key.put("key1".getBytes()).flip();
iterator.seek(key);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.value()).isEqualTo("value1".getBytes());
assertThat(key.position()).isEqualTo(4);
assertThat(key.limit()).isEqualTo(4);
}
{
// Check offsets of slice byte buffers
final ByteBuffer key0 = ByteBuffer.allocate(24);
key0.put("key2key2".getBytes());
final ByteBuffer key = key0.slice();
key.put("key1".getBytes()).flip();
iterator.seek(key);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.value()).isEqualTo("value1".getBytes());
assertThat(key.position()).isEqualTo(4);
assertThat(key.limit()).isEqualTo(4);
}
{
// Check offsets of slice byte buffers
final ByteBuffer key0 = ByteBuffer.allocateDirect(24);
key0.put("key2key2".getBytes());
final ByteBuffer key = key0.slice();
key.put("key1".getBytes()).flip();
iterator.seek(key);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.value()).isEqualTo("value1".getBytes());
assertThat(key.position()).isEqualTo(4);
assertThat(key.limit()).isEqualTo(4);
}
{
final ByteBuffer key = ByteBuffer.allocate(12);
key.put("key2".getBytes()).flip();
iterator.seekForPrev(key);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.value()).isEqualTo("value2".getBytes());
assertThat(key.position()).isEqualTo(4);
assertThat(key.limit()).isEqualTo(4);
}
key.clear();
key.put("key2".getBytes());
key.flip();
iterator.seekForPrev(key);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.value()).isEqualTo("value2".getBytes());
assertThat(key.position()).isEqualTo(4);
assertThat(key.limit()).isEqualTo(4);
}
}
}
@Test
public void rocksIteratorSeekAndInsert() throws RocksDBException {
try (final Options options =
new Options().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) {
db.put("key1".getBytes(), "value1".getBytes());
db.put("key2".getBytes(), "value2".getBytes());
try (final RocksIterator iterator = db.newIterator()) {
iterator.seek("key0".getBytes());
@@ -257,8 +192,8 @@ public class RocksIteratorTest {
}
// Test case: release iterator after custom CF close
final ColumnFamilyDescriptor cfd1 = new ColumnFamilyDescriptor("cf1".getBytes());
final ColumnFamilyHandle cfHandle1 = db.createColumnFamily(cfd1);
ColumnFamilyDescriptor cfd1 = new ColumnFamilyDescriptor("cf1".getBytes());
ColumnFamilyHandle cfHandle1 = db.createColumnFamily(cfd1);
db.put(cfHandle1, "key1".getBytes(), "value1".getBytes());
try (final RocksIterator iterator = db.newIterator(cfHandle1)) {
@@ -271,8 +206,8 @@ public class RocksIteratorTest {
}
// Test case: release iterator after custom CF drop & close
final ColumnFamilyDescriptor cfd2 = new ColumnFamilyDescriptor("cf2".getBytes());
final ColumnFamilyHandle cfHandle2 = db.createColumnFamily(cfd2);
ColumnFamilyDescriptor cfd2 = new ColumnFamilyDescriptor("cf2".getBytes());
ColumnFamilyHandle cfHandle2 = db.createColumnFamily(cfd2);
db.put(cfHandle2, "key2".getBytes(), "value2".getBytes());
try (final RocksIterator iterator = db.newIterator(cfHandle2)) {
@@ -13,21 +13,17 @@ import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.rocksdb.util.ByteBufferAllocator;
import org.rocksdb.util.BytewiseComparator;
@RunWith(Parameterized.class)
public class SstFileReaderTest {
private static final String SST_FILE_NAME = "test.sst";
static class KeyValueWithOp {
KeyValueWithOp(final String key, final String value, final OpType opType) {
class KeyValueWithOp {
KeyValueWithOp(String key, String value, OpType opType) {
this.key = key;
this.value = value;
this.opType = opType;
@@ -45,23 +41,13 @@ public class SstFileReaderTest {
return opType;
}
private final String key;
private final String value;
private final OpType opType;
private String key;
private String value;
private OpType opType;
}
@Rule public TemporaryFolder parentFolder = new TemporaryFolder();
@Parameterized.Parameters(name = "{0}")
public static Iterable<Object[]> parameters() {
return Arrays.asList(new Object[][] {
{"direct", ByteBufferAllocator.DIRECT}, {"indirect", ByteBufferAllocator.HEAP}});
}
@Parameterized.Parameter(0) public String name;
@Parameterized.Parameter(1) public ByteBufferAllocator byteBufferAllocator;
enum OpType { PUT, PUT_BYTES, MERGE, MERGE_BYTES, DELETE, DELETE_BYTES }
private File newSstFile(final List<KeyValueWithOp> keyValues)
@@ -69,17 +55,17 @@ public class SstFileReaderTest {
final EnvOptions envOptions = new EnvOptions();
final StringAppendOperator stringAppendOperator = new StringAppendOperator();
final Options options = new Options().setMergeOperator(stringAppendOperator);
final SstFileWriter sstFileWriter;
SstFileWriter sstFileWriter;
sstFileWriter = new SstFileWriter(envOptions, options);
final File sstFile = parentFolder.newFile(SST_FILE_NAME);
try {
sstFileWriter.open(sstFile.getAbsolutePath());
for (final KeyValueWithOp keyValue : keyValues) {
final Slice keySlice = new Slice(keyValue.getKey());
final Slice valueSlice = new Slice(keyValue.getValue());
final byte[] keyBytes = keyValue.getKey().getBytes();
final byte[] valueBytes = keyValue.getValue().getBytes();
for (KeyValueWithOp keyValue : keyValues) {
Slice keySlice = new Slice(keyValue.getKey());
Slice valueSlice = new Slice(keyValue.getValue());
byte[] keyBytes = keyValue.getKey().getBytes();
byte[] valueBytes = keyValue.getValue().getBytes();
switch (keyValue.getOpType()) {
case PUT:
sstFileWriter.put(keySlice, valueSlice);
@@ -119,8 +105,6 @@ public class SstFileReaderTest {
public void readSstFile() throws RocksDBException, IOException {
final List<KeyValueWithOp> keyValues = new ArrayList<>();
keyValues.add(new KeyValueWithOp("key1", "value1", OpType.PUT));
keyValues.add(new KeyValueWithOp("key2", "value2", OpType.PUT));
keyValues.add(new KeyValueWithOp("key3", "value3", OpType.PUT));
final File sstFile = newSstFile(keyValues);
try (final StringAppendOperator stringAppendOperator = new StringAppendOperator();
@@ -139,84 +123,33 @@ public class SstFileReaderTest {
reader.verifyChecksum();
// Verify Table Properties
assertEquals(reader.getTableProperties().getNumEntries(), 3);
assertEquals(reader.getTableProperties().getNumEntries(), 1);
// Check key and value
assertThat(iterator.key()).isEqualTo("key1".getBytes());
assertThat(iterator.value()).isEqualTo("value1".getBytes());
final ByteBuffer byteBuffer = byteBufferAllocator.allocate(128);
byteBuffer.put("key1".getBytes()).flip();
iterator.seek(byteBuffer);
assertThat(byteBuffer.position()).isEqualTo(4);
assertThat(byteBuffer.limit()).isEqualTo(4);
ByteBuffer direct = ByteBuffer.allocateDirect(128);
direct.put("key1".getBytes()).flip();
iterator.seek(direct);
assertThat(direct.position()).isEqualTo(4);
assertThat(direct.limit()).isEqualTo(4);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.key()).isEqualTo("key1".getBytes());
assertThat(iterator.value()).isEqualTo("value1".getBytes());
{
byteBuffer.clear();
assertThat(iterator.key(byteBuffer)).isEqualTo("key1".getBytes().length);
final byte[] dst = new byte["key1".getBytes().length];
byteBuffer.get(dst);
assertThat(new String(dst)).isEqualTo("key1");
}
direct.clear();
assertThat(iterator.key(direct)).isEqualTo("key1".getBytes().length);
byte[] dst = new byte["key1".getBytes().length];
direct.get(dst);
assertThat(new String(dst)).isEqualTo("key1");
{
byteBuffer.clear();
byteBuffer.put("PREFIX".getBytes());
final ByteBuffer slice = byteBuffer.slice();
assertThat(iterator.key(byteBuffer)).isEqualTo("key1".getBytes().length);
final byte[] dst = new byte["key1".getBytes().length];
slice.get(dst);
assertThat(new String(dst)).isEqualTo("key1");
}
{
byteBuffer.clear();
assertThat(iterator.value(byteBuffer)).isEqualTo("value1".getBytes().length);
final byte[] dst = new byte["value1".getBytes().length];
byteBuffer.get(dst);
assertThat(new String(dst)).isEqualTo("value1");
}
byteBuffer.clear();
byteBuffer.put("key1point5".getBytes()).flip();
iterator.seek(byteBuffer);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.key()).isEqualTo("key2".getBytes());
assertThat(iterator.value()).isEqualTo("value2".getBytes());
byteBuffer.clear();
byteBuffer.put("key1point5".getBytes()).flip();
iterator.seekForPrev(byteBuffer);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.key()).isEqualTo("key1".getBytes());
assertThat(iterator.value()).isEqualTo("value1".getBytes());
byteBuffer.clear();
byteBuffer.put("key2point5".getBytes()).flip();
iterator.seek(byteBuffer);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.key()).isEqualTo("key3".getBytes());
assertThat(iterator.value()).isEqualTo("value3".getBytes());
byteBuffer.clear();
byteBuffer.put("key2point5".getBytes()).flip();
iterator.seekForPrev(byteBuffer);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.key()).isEqualTo("key2".getBytes());
assertThat(iterator.value()).isEqualTo("value2".getBytes());
byteBuffer.clear();
byteBuffer.put("PREFIX".getBytes());
final ByteBuffer slice = byteBuffer.slice();
slice.put("key1point5".getBytes()).flip();
iterator.seekForPrev(slice);
assertThat(iterator.isValid()).isTrue();
assertThat(iterator.key()).isEqualTo("key1".getBytes());
assertThat(iterator.value()).isEqualTo("value1".getBytes());
direct.clear();
assertThat(iterator.value(direct)).isEqualTo("value1".getBytes().length);
dst = new byte["value1".getBytes().length];
direct.get(dst);
assertThat(new String(dst)).isEqualTo("value1");
}
}
}
@@ -1,213 +0,0 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
package org.rocksdb;
import static org.assertj.core.api.Assertions.assertThat;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
public class VerifyChecksumsTest {
@ClassRule
public static final RocksNativeLibraryResource ROCKS_NATIVE_LIBRARY_RESOURCE =
new RocksNativeLibraryResource();
@Rule public TemporaryFolder dbFolder = new TemporaryFolder();
/**
* Class to factor out the specific DB operations within the test
*/
abstract static class Operations {
final int kv_count;
final List<String> elements = new ArrayList<>();
final List<String> sortedElements = new ArrayList<>();
Operations(final int kv_count) {
this.kv_count = kv_count;
for (int i = 0; i < kv_count; i++) elements.add(MessageFormat.format("{0,number,#}", i));
sortedElements.addAll(elements);
Collections.sort(sortedElements);
}
void fill(final RocksDB db) throws RocksDBException {
for (int i = 0; i < kv_count; i++) {
final String key = MessageFormat.format("key{0}", elements.get(i));
final String value = MessageFormat.format("value{0}", elements.get(i));
// noinspection ObjectAllocationInLoop
db.put(key.getBytes(), value.getBytes());
}
db.flush(new FlushOptions());
}
@SuppressWarnings("ObjectAllocationInLoop")
void get(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
try (final ReadOptions readOptions = new ReadOptions()) {
readOptions.setReadaheadSize(32 * 1024);
readOptions.setFillCache(false);
readOptions.setVerifyChecksums(verifyFlag);
for (int i = 0; i < kv_count / 10; i++) {
@SuppressWarnings("UnsecureRandomNumberGeneration")
final int index = Double.valueOf(Math.random() * kv_count).intValue();
final String key = MessageFormat.format("key{0}", sortedElements.get(index));
final String expectedValue = MessageFormat.format("value{0}", sortedElements.get(index));
final byte[] value = db.get(readOptions, key.getBytes());
assertThat(value).isEqualTo(expectedValue.getBytes());
}
}
}
@SuppressWarnings("ObjectAllocationInLoop")
void multiGet(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
try (final ReadOptions readOptions = new ReadOptions()) {
readOptions.setReadaheadSize(32 * 1024);
readOptions.setFillCache(false);
readOptions.setVerifyChecksums(verifyFlag);
final List<byte[]> keys = new ArrayList<>();
final List<String> expectedValues = new ArrayList<>();
for (int i = 0; i < kv_count / 10; i++) {
@SuppressWarnings("UnsecureRandomNumberGeneration")
final int index = Double.valueOf(Math.random() * kv_count).intValue();
keys.add(MessageFormat.format("key{0}", sortedElements.get(index)).getBytes());
expectedValues.add(MessageFormat.format("value{0}", sortedElements.get(index)));
}
final List<byte[]> values = db.multiGetAsList(readOptions, keys);
for (int i = 0; i < keys.size(); i++) {
assertThat(values.get(i)).isEqualTo(expectedValues.get(i).getBytes());
}
}
}
void iterate(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
final ReadOptions readOptions = new ReadOptions();
readOptions.setReadaheadSize(32 * 1024);
readOptions.setFillCache(false);
readOptions.setVerifyChecksums(verifyFlag);
int i = 0;
try (final RocksIterator rocksIterator = db.newIterator(readOptions)) {
rocksIterator.seekToFirst();
rocksIterator.status();
while (rocksIterator.isValid()) {
final byte[] key = rocksIterator.key();
final byte[] value = rocksIterator.value();
// noinspection ObjectAllocationInLoop
assertThat(key).isEqualTo(
(MessageFormat.format("key{0}", sortedElements.get(i))).getBytes());
// noinspection ObjectAllocationInLoop
assertThat(value).isEqualTo(
(MessageFormat.format("value{0}", sortedElements.get(i))).getBytes());
rocksIterator.next();
rocksIterator.status();
i++;
}
}
assertThat(i).isEqualTo(kv_count);
}
abstract void performOperations(final RocksDB db, final boolean verifyFlag)
throws RocksDBException;
}
private static final int KV_COUNT = 10000;
/**
* Run some operations and count the TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT before and after
* It should GO UP when the read options have checksum verification turned on.
* It shoulld REMAIN UNCHANGED when the read options have checksum verification turned off.
* As the read options refer only to the read operations, there are still a few checksums
* performed outside this (blocks are getting loaded for lots of reasons, not aways directly due
* to reads) but this test provides a good enough proxy for whether the flag is being noticed.
*
* @param operations the DB reading operations to perform which affect the checksum stats
*
* @throws RocksDBException
*/
private void verifyChecksums(final Operations operations) throws RocksDBException {
final String dbPath = dbFolder.getRoot().getAbsolutePath();
// noinspection SingleStatementInBlock
try (final Statistics statistics = new Statistics();
final Options options = new Options().setCreateIfMissing(true).setStatistics(statistics)) {
try (final RocksDB db = RocksDB.open(options, dbPath)) {
// 0
System.out.println(MessageFormat.format(
"newly open {0}", statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT)));
operations.fill(db);
//
System.out.println(MessageFormat.format(
"flushed {0}", statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT)));
}
// 2
System.out.println(MessageFormat.format("closed-after-write {0}",
statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT)));
for (final boolean verifyFlag : new boolean[] {false, true, false, true}) {
try (final RocksDB db = RocksDB.open(options, dbPath)) {
final long beforeOperationsCount =
statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT);
System.out.println(MessageFormat.format("re-opened {0}", beforeOperationsCount));
operations.performOperations(db, verifyFlag);
final long afterOperationsCount =
statistics.getTickerCount(TickerType.BLOCK_CHECKSUM_COMPUTE_COUNT);
if (verifyFlag) {
// We don't need to be exact - we are checking that the checksums happen
// exactly how many depends on block size etc etc, so may not be entirely stable
System.out.println(MessageFormat.format("verify=true {0}", afterOperationsCount));
assertThat(afterOperationsCount).isGreaterThan(beforeOperationsCount + 20);
} else {
System.out.println(MessageFormat.format("verify=false {0}", afterOperationsCount));
assertThat(afterOperationsCount).isEqualTo(beforeOperationsCount);
}
}
}
}
}
@Test
public void verifyChecksumsInIteration() throws RocksDBException {
// noinspection AnonymousInnerClassMayBeStatic
verifyChecksums(new Operations(KV_COUNT) {
@Override
void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
iterate(db, verifyFlag);
}
});
}
@Test
public void verifyChecksumsGet() throws RocksDBException {
// noinspection AnonymousInnerClassMayBeStatic
verifyChecksums(new Operations(KV_COUNT) {
@Override
void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
get(db, verifyFlag);
}
});
}
@Test
public void verifyChecksumsMultiGet() throws RocksDBException {
// noinspection AnonymousInnerClassMayBeStatic
verifyChecksums(new Operations(KV_COUNT) {
@Override
void performOperations(final RocksDB db, final boolean verifyFlag) throws RocksDBException {
multiGet(db, verifyFlag);
}
});
}
}
@@ -20,7 +20,6 @@ import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.rocksdb.util.ByteBufferAllocator;
public class WriteBatchWithIndexTest {
@@ -193,236 +192,6 @@ public class WriteBatchWithIndexTest {
}
}
@Test
public void readYourOwnWritesCfIterDirectBB() throws RocksDBException {
readYourOwnWritesCfIterDirect(ByteBufferAllocator.DIRECT);
}
@Test
public void readYourOwnWritesCfIterIndirectBB() throws RocksDBException {
readYourOwnWritesCfIterDirect(ByteBufferAllocator.HEAP);
}
public void readYourOwnWritesCfIterDirect(final ByteBufferAllocator byteBufferAllocator)
throws RocksDBException {
final List<ColumnFamilyDescriptor> cfNames =
Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY),
new ColumnFamilyDescriptor("new_cf".getBytes()));
final List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<>();
// Test open database with column family names
try (final DBOptions options =
new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
final RocksDB db = RocksDB.open(
options, dbFolder.getRoot().getAbsolutePath(), cfNames, columnFamilyHandleList)) {
final ColumnFamilyHandle newCf = columnFamilyHandleList.get(1);
try {
final byte[] kv1 = "key1".getBytes();
final byte[] vv1 = "value1".getBytes();
final ByteBuffer k1 = byteBufferAllocator.allocate(12);
k1.put(kv1);
final byte[] kv2 = "key2".getBytes();
final byte[] vv2 = "value2".getBytes();
final ByteBuffer k2 = byteBufferAllocator.allocate(12);
k2.put(kv2);
db.put(newCf, kv1, vv1);
db.put(newCf, kv2, vv2);
try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true);
final ReadOptions readOptions = new ReadOptions();
final RocksIterator base = db.newIterator(newCf, readOptions);
final RocksIterator it = wbwi.newIteratorWithBase(newCf, base, readOptions)) {
k1.flip();
it.seek(k1);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv1);
assertThat(it.value()).isEqualTo(vv1);
k2.flip();
it.seek(k2);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv2);
assertThat(it.value()).isEqualTo(vv2);
final byte[] kv1point5 = "key1point5".getBytes();
final ByteBuffer k1point5 = byteBufferAllocator.allocate(12);
k1point5.put(kv1point5);
k1point5.flip();
it.seek(k1point5);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv2);
assertThat(it.value()).isEqualTo(vv2);
k1point5.flip();
it.seekForPrev(k1point5);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv1);
assertThat(it.value()).isEqualTo(vv1);
// put data to the write batch and make sure we can read it.
final byte[] kv3 = "key3".getBytes();
final ByteBuffer k3 = byteBufferAllocator.allocate(12);
k3.put(kv3);
final byte[] vv3 = "value3".getBytes();
wbwi.put(newCf, kv3, vv3);
k3.flip();
it.seek(k3);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv3);
assertThat(it.value()).isEqualTo(vv3);
// update k2 in the write batch and check the value
final byte[] v2Other = "otherValue2".getBytes();
wbwi.put(newCf, kv2, v2Other);
k2.flip();
it.seek(k2);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv2);
assertThat(it.value()).isEqualTo(v2Other);
// delete k1 and make sure we can read back the write
wbwi.delete(newCf, kv1);
k1.flip();
it.seek(k1);
assertThat(it.key()).isNotEqualTo(kv1);
// reinsert k1 and make sure we see the new value
final byte[] v1Other = "otherValue1".getBytes();
wbwi.put(newCf, kv1, v1Other);
k1.flip();
it.seek(k1);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv1);
assertThat(it.value()).isEqualTo(v1Other);
// single remove k3 and make sure we can read back the write
wbwi.singleDelete(newCf, kv3);
k3.flip();
it.seek(k3);
assertThat(it.isValid()).isEqualTo(false);
// reinsert k3 and make sure we see the new value
final byte[] v3Other = "otherValue3".getBytes();
wbwi.put(newCf, kv3, v3Other);
k3.flip();
it.seek(k3);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv3);
assertThat(it.value()).isEqualTo(v3Other);
}
} finally {
for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) {
columnFamilyHandle.close();
}
}
}
}
@Test
public void readYourOwnWritesCfIterIndirect() throws RocksDBException {
final List<ColumnFamilyDescriptor> cfNames =
Arrays.asList(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY),
new ColumnFamilyDescriptor("new_cf".getBytes()));
final List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<>();
// Test open database with column family names
try (final DBOptions options =
new DBOptions().setCreateIfMissing(true).setCreateMissingColumnFamilies(true);
final RocksDB db = RocksDB.open(
options, dbFolder.getRoot().getAbsolutePath(), cfNames, columnFamilyHandleList)) {
final ColumnFamilyHandle newCf = columnFamilyHandleList.get(1);
try {
final byte[] kv1 = "key1".getBytes();
final byte[] vv1 = "value1".getBytes();
final ByteBuffer k1 = ByteBuffer.allocate(12);
k1.put(kv1).flip();
final byte[] kv2 = "key2".getBytes();
final byte[] vv2 = "value2".getBytes();
final ByteBuffer k2 = ByteBuffer.allocate(12);
k2.put(kv2).flip();
db.put(newCf, kv1, vv1);
db.put(newCf, kv2, vv2);
try (final WriteBatchWithIndex wbwi = new WriteBatchWithIndex(true);
final ReadOptions readOptions = new ReadOptions();
final RocksIterator base = db.newIterator(newCf, readOptions);
final RocksIterator it = wbwi.newIteratorWithBase(newCf, base, readOptions)) {
it.seek(k1);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv1);
assertThat(it.value()).isEqualTo(vv1);
it.seek(k2);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv2);
assertThat(it.value()).isEqualTo(vv2);
// put data to the write batch and make sure we can read it.
final byte[] kv3 = "key3".getBytes();
final ByteBuffer k3 = ByteBuffer.allocate(12);
k3.put(kv3);
final byte[] vv3 = "value3".getBytes();
wbwi.put(newCf, kv3, vv3);
k3.flip();
it.seek(k3);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv3);
assertThat(it.value()).isEqualTo(vv3);
// update k2 in the write batch and check the value
final byte[] v2Other = "otherValue2".getBytes();
wbwi.put(newCf, kv2, v2Other);
k2.flip();
it.seek(k2);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv2);
assertThat(it.value()).isEqualTo(v2Other);
// delete k1 and make sure we can read back the write
wbwi.delete(newCf, kv1);
k1.flip();
it.seek(k1);
assertThat(it.key()).isNotEqualTo(kv1);
// reinsert k1 and make sure we see the new value
final byte[] v1Other = "otherValue1".getBytes();
wbwi.put(newCf, kv1, v1Other);
k1.flip();
it.seek(k1);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv1);
assertThat(it.value()).isEqualTo(v1Other);
// single remove k3 and make sure we can read back the write
wbwi.singleDelete(newCf, kv3);
k3.flip();
it.seek(k3);
assertThat(it.isValid()).isEqualTo(false);
// reinsert k3 and make sure we see the new value
final byte[] v3Other = "otherValue3".getBytes();
wbwi.put(newCf, kv3, v3Other);
k3.flip();
it.seek(k3);
assertThat(it.isValid()).isTrue();
assertThat(it.key()).isEqualTo(kv3);
assertThat(it.value()).isEqualTo(v3Other);
}
} finally {
for (final ColumnFamilyHandle columnFamilyHandle : columnFamilyHandleList) {
columnFamilyHandle.close();
}
}
}
}
@Test
public void writeBatchWithIndex() throws RocksDBException {
try (final Options options = new Options().setCreateIfMissing(true);
@@ -451,10 +220,10 @@ public class WriteBatchWithIndexTest {
public void write_writeBatchWithIndexDirect() throws RocksDBException {
try (final Options options = new Options().setCreateIfMissing(true);
final RocksDB db = RocksDB.open(options, dbFolder.getRoot().getAbsolutePath())) {
final ByteBuffer k1 = ByteBuffer.allocateDirect(16);
final ByteBuffer v1 = ByteBuffer.allocateDirect(16);
final ByteBuffer k2 = ByteBuffer.allocateDirect(16);
final ByteBuffer v2 = ByteBuffer.allocateDirect(16);
ByteBuffer k1 = ByteBuffer.allocateDirect(16);
ByteBuffer v1 = ByteBuffer.allocateDirect(16);
ByteBuffer k2 = ByteBuffer.allocateDirect(16);
ByteBuffer v2 = ByteBuffer.allocateDirect(16);
k1.put("key1".getBytes()).flip();
v1.put("value1".getBytes()).flip();
k2.put("key2".getBytes()).flip();
@@ -489,6 +258,8 @@ public class WriteBatchWithIndexTest {
final String v3 = "value3";
final String k4 = "key4";
final String k5 = "key5";
final String k6 = "key6";
final String k7 = "key7";
final String v8 = "value8";
final byte[] k1b = k1.getBytes(UTF_8);
final byte[] v1b = v1.getBytes(UTF_8);
@@ -498,11 +269,10 @@ public class WriteBatchWithIndexTest {
final byte[] v3b = v3.getBytes(UTF_8);
final byte[] k4b = k4.getBytes(UTF_8);
final byte[] k5b = k5.getBytes(UTF_8);
final byte[] k6b = k6.getBytes(UTF_8);
final byte[] k7b = k7.getBytes(UTF_8);
final byte[] v8b = v8.getBytes(UTF_8);
final String k1point5 = "key1point5";
final String k2point5 = "key2point5";
// add put records
wbwi.put(k1b, v1b);
wbwi.put(k2b, v2b);
@@ -533,7 +303,9 @@ public class WriteBatchWithIndexTest {
try (final WBWIRocksIterator it = wbwi.newIterator()) {
//direct access - seek to key offsets
final int[] testOffsets = {2, 0, 3, 4, 1};
for (final int testOffset : testOffsets) {
for (int i = 0; i < testOffsets.length; i++) {
final int testOffset = testOffsets[i];
final byte[] key = toArray(expected[testOffset].getKey().data());
it.seek(key);
@@ -541,94 +313,13 @@ public class WriteBatchWithIndexTest {
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[testOffset]);
}
for (final int testOffset : testOffsets) {
final byte[] key = toArray(expected[testOffset].getKey().data());
// Direct buffer seek
final ByteBuffer db = expected[testOffset].getKey().data();
expected[testOffset].getKey().data().mark();
ByteBuffer db = expected[testOffset].getKey().data();
it.seek(db);
assertThat(db.position()).isEqualTo(key.length);
assertThat(it.isValid()).isTrue();
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[testOffset]);
}
for (final int testOffset : testOffsets) {
final byte[] key = toArray(expected[testOffset].getKey().data());
// Direct buffer seek
final ByteBuffer db = expected[testOffset].getKey().data();
it.seekForPrev(db);
assertThat(db.position()).isEqualTo(key.length);
assertThat(it.isValid()).isTrue();
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[testOffset]);
}
for (final int testOffset : testOffsets) {
final byte[] key = toArray(expected[testOffset].getKey().data());
// Indirect buffer seek
final ByteBuffer db = ByteBuffer.allocate(key.length);
System.arraycopy(key, 0, db.array(), 0, key.length);
it.seek(db);
assertThat(db.position()).isEqualTo(key.length);
assertThat(it.isValid()).isTrue();
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[testOffset]);
}
for (final int testOffset : testOffsets) {
final byte[] key = toArray(expected[testOffset].getKey().data());
// Indirect buffer seek for prev
final ByteBuffer db = ByteBuffer.allocate(key.length);
System.arraycopy(key, 0, db.array(), 0, key.length);
it.seekForPrev(db);
assertThat(db.position()).isEqualTo(key.length);
assertThat(it.isValid()).isTrue();
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[testOffset]);
}
{
it.seekForPrev(k2point5.getBytes());
assertThat(it.isValid()).isTrue();
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[1]);
}
{
it.seekForPrev(k1point5.getBytes());
assertThat(it.isValid()).isTrue();
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[0]);
}
{
final ByteBuffer db = ByteBuffer.allocate(k2point5.length());
db.put(k2point5.getBytes());
db.flip();
it.seekForPrev(db);
assertThat(it.isValid()).isTrue();
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[1]);
}
{
final ByteBuffer db = ByteBuffer.allocate(k1point5.length());
db.put(k1point5.getBytes());
db.flip();
it.seekForPrev(db);
assertThat(it.isValid()).isTrue();
final WBWIRocksIterator.WriteEntry entry = it.entry();
assertThat(entry).isEqualTo(expected[0]);
}
//forward iterative access
@@ -1,10 +0,0 @@
package org.rocksdb.util;
import java.nio.ByteBuffer;
public interface ByteBufferAllocator {
ByteBuffer allocate(int capacity);
ByteBufferAllocator DIRECT = new DirectByteBufferAllocator();
ByteBufferAllocator HEAP = new HeapByteBufferAllocator();
}
@@ -1,12 +0,0 @@
package org.rocksdb.util;
import java.nio.ByteBuffer;
public final class DirectByteBufferAllocator implements ByteBufferAllocator {
DirectByteBufferAllocator(){};
@Override
public ByteBuffer allocate(final int capacity) {
return ByteBuffer.allocateDirect(capacity);
}
}
@@ -1,12 +0,0 @@
package org.rocksdb.util;
import java.nio.ByteBuffer;
public final class HeapByteBufferAllocator implements ByteBufferAllocator {
HeapByteBufferAllocator(){};
@Override
public ByteBuffer allocate(final int capacity) {
return ByteBuffer.allocate(capacity);
}
}
+8 -7
View File
@@ -9,8 +9,6 @@
#include "rocksdb/write_buffer_manager.h"
#include <memory>
#include "cache/cache_entry_roles.h"
#include "cache/cache_reservation_manager.h"
#include "db/db_impl/db_impl.h"
@@ -33,9 +31,8 @@ WriteBufferManager::WriteBufferManager(size_t _buffer_size,
// Memtable's memory usage tends to fluctuate frequently
// therefore we set delayed_decrease = true to save some dummy entry
// insertion on memory increase right after memory decrease
cache_res_mgr_ = std::make_shared<
CacheReservationManagerImpl<CacheEntryRole::kWriteBuffer>>(
cache, true /* delayed_decrease */);
cache_res_mgr_.reset(
new CacheReservationManager(cache, true /* delayed_decrease */));
}
#else
(void)cache;
@@ -78,7 +75,9 @@ void WriteBufferManager::ReserveMemWithCache(size_t mem) {
size_t new_mem_used = memory_used_.load(std::memory_order_relaxed) + mem;
memory_used_.store(new_mem_used, std::memory_order_relaxed);
Status s = cache_res_mgr_->UpdateCacheReservation(new_mem_used);
Status s =
cache_res_mgr_->UpdateCacheReservation<CacheEntryRole::kWriteBuffer>(
new_mem_used);
// We absorb the error since WriteBufferManager is not able to handle
// this failure properly. Ideallly we should prevent this allocation
@@ -115,7 +114,9 @@ void WriteBufferManager::FreeMemWithCache(size_t mem) {
std::lock_guard<std::mutex> lock(cache_res_mgr_mu_);
size_t new_mem_used = memory_used_.load(std::memory_order_relaxed) - mem;
memory_used_.store(new_mem_used, std::memory_order_relaxed);
Status s = cache_res_mgr_->UpdateCacheReservation(new_mem_used);
Status s =
cache_res_mgr_->UpdateCacheReservation<CacheEntryRole::kWriteBuffer>(
new_mem_used);
// We absorb the error since WriteBufferManager is not able to handle
// this failure properly.
+1 -2
View File
@@ -11,7 +11,6 @@ foreach(ONE_BENCH_CPP ${ALL_BENCH_CPP})
# add_test(Bench_${TARGET_NAME} ${TARGET_NAME})
list(APPEND ALL_BENCH_TARGETS ${TARGET_NAME})
endforeach()
add_custom_target(microbench DEPENDS ${ALL_BENCH_TARGETS})
add_custom_target(run_microbench
add_custom_target(microbench
COMMAND for t in ${ALL_BENCH_TARGETS}\; do \.\/$$t \|\| exit 1\; done
DEPENDS ${ALL_BENCH_TARGETS})
-60
View File
@@ -1,60 +0,0 @@
# RocksDB Micro-Benchmark
## Overview
RocksDB micro-benchmark is a set of tests for benchmarking a single component or simple DB operations. The test artificially generates input data and executes the same operation with it to collect and report performance metrics. As it's focusing on testing a single, well-defined operation, the result is more precise and reproducible, which also has its limitation of not representing a real production use case. The test author needs to carefully design the microbench to represent its true purpose.
The tests are based on [Google Benchmark](https://github.com/google/benchmark) library, which provides a standard framework for writing benchmarks.
## How to Run
### Prerequisite
Install the [Google Benchmark](https://github.com/google/benchmark) version `1.6.0` or above.
*Note: Google Benchmark `1.6.x` is incompatible with previous versions like `1.5.x`, please make sure you're using the newer version.*
### Build and Run
With `Makefile`:
```bash
$ DEBUG_LEVEL=0 make run_microbench
```
Or with cmake:
```bash
$ mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DWITH_BENCHMARK
$ make run_microbench
```
*Note: Please run the benchmark code in release build.*
### Run Single Test
Example:
```bash
$ make db_basic_bench
$ ./db_basic_bench --benchmark_filter=<TEST_NAME>
```
## Best Practices
#### * Use the Same Test Directory Setting as Unittest
Most of the Micro-benchmark tests use the same test directory setup as unittest, so it could be overridden by:
```bash
$ TEST_TMPDIR=/mydata/tmp/ ./db_basic_bench --benchmark_filter=<TEST_NAME>
```
Please also follow that when designing new tests.
#### * Avoid Using Debug API
Even though micro-benchmark is a test, avoid using internal Debug API like TEST_WaitForRun() which is designed for unittest. As benchmark tests are designed for release build, don't use any of that.
#### * Pay Attention to Local Optimization
As a micro-benchmark is focusing on a single component or area, make sure it is a key part for impacting the overall application performance.
The compiler might be able to optimize the code that not the same way as the whole application, and if the test data input is simple and small, it may be able to all cached in CPU memory, which is leading to a wrong metric. Take these into consideration when designing the tests.
#### * Names of user-defined counters/metrics has to be `[A-Za-z0-9_]`
It's a restriction of the metrics collecting and reporting system RocksDB is using internally. It will also help integrate with more systems.
#### * Minimize the Metrics Variation
Try reducing the test result variation, one way to check that is running the test multiple times and check the CV (Coefficient of Variation) reported by gbenchmark.
```bash
$ ./db_basic_bench --benchmark_filter=<TEST_NAME> --benchmark_repetitions=10
...
<TEST_NAME>_cv 3.2%
```
RocksDB has background compaction jobs which may cause the test result to vary a lot. If the micro-benchmark is not purposely testing the operation while compaction is in progress, it should wait for the compaction to finish (`db_impl->WaitForCompact()`) or disable auto-compaction.
+38 -38
View File
@@ -262,10 +262,10 @@ static void DBPut(benchmark::State& state) {
}
options.compaction_style = compaction_style;
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, key_num);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "DBPut");
}
@@ -283,7 +283,7 @@ static void DBPut(benchmark::State& state) {
}
}
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
auto db_full = static_cast_with_check<DBImpl>(db);
Status s = db_full->WaitForCompact(true);
if (!s.ok()) {
@@ -293,9 +293,9 @@ static void DBPut(benchmark::State& state) {
if (enable_statistics) {
HistogramData histogram_data;
options.statistics->histogramData(DB_WRITE, &histogram_data);
state.counters["put_mean"] = histogram_data.average * std::milli::den;
state.counters["put_p95"] = histogram_data.percentile95 * std::milli::den;
state.counters["put_p99"] = histogram_data.percentile99 * std::milli::den;
state.counters["put_mean"] = histogram_data.average;
state.counters["put_p95"] = histogram_data.percentile95;
state.counters["put_p99"] = histogram_data.percentile99;
}
TeardownDB(state, db, options, kg);
@@ -346,10 +346,10 @@ static void ManualCompaction(benchmark::State& state) {
options.soft_pending_compaction_bytes_limit = 0;
options.hard_pending_compaction_bytes_limit = 0;
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, key_num);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "ManualCompaction");
}
@@ -390,7 +390,7 @@ static void ManualCompaction(benchmark::State& state) {
}
}
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
auto db_full = static_cast_with_check<DBImpl>(db);
s = db_full->WaitForCompact(true);
if (!s.ok()) {
@@ -445,7 +445,7 @@ static void ManualCompactionArguments(benchmark::internal::Benchmark* b) {
b->ArgNames({"comp_style", "max_data", "per_key_size", "enable_statistics"});
}
BENCHMARK(ManualCompaction)->Iterations(1)->Apply(ManualCompactionArguments);
BENCHMARK(ManualCompaction)->Iterations(10)->Apply(ManualCompactionArguments);
static void ManualFlush(benchmark::State& state) {
uint64_t key_num = state.range(0);
@@ -466,10 +466,10 @@ static void ManualFlush(benchmark::State& state) {
options.hard_pending_compaction_bytes_limit = 0;
options.write_buffer_size = 2l << 30; // 2G to avoid auto flush
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, key_num);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "ManualFlush");
}
@@ -488,7 +488,7 @@ static void ManualFlush(benchmark::State& state) {
}
}
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
auto db_full = static_cast_with_check<DBImpl>(db);
Status s = db_full->WaitForCompact(true);
if (!s.ok()) {
@@ -516,7 +516,7 @@ static void ManualFlushArguments(benchmark::internal::Benchmark* b) {
b->ArgNames({"key_num", "per_key_size"});
}
BENCHMARK(ManualFlush)->Iterations(1)->Apply(ManualFlushArguments);
BENCHMARK(ManualFlush)->Iterations(100)->Apply(ManualFlushArguments);
static void DBGet(benchmark::State& state) {
auto compaction_style = static_cast<CompactionStyle>(state.range(0));
@@ -541,10 +541,10 @@ static void DBGet(benchmark::State& state) {
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
}
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, key_num);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "DBGet");
// load db
@@ -595,13 +595,13 @@ static void DBGet(benchmark::State& state) {
state.counters["neg_qu_pct"] = benchmark::Counter(
static_cast<double>(not_found * 100), benchmark::Counter::kAvgIterations);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
if (enable_statistics) {
HistogramData histogram_data;
options.statistics->histogramData(DB_GET, &histogram_data);
state.counters["get_mean"] = histogram_data.average * std::milli::den;
state.counters["get_p95"] = histogram_data.percentile95 * std::milli::den;
state.counters["get_p99"] = histogram_data.percentile99 * std::milli::den;
state.counters["get_mean"] = histogram_data.average;
state.counters["get_p95"] = histogram_data.percentile95;
state.counters["get_p99"] = histogram_data.percentile99;
}
TeardownDB(state, db, options, kg);
@@ -640,10 +640,10 @@ static void SimpleGetWithPerfContext(benchmark::State& state) {
options.create_if_missing = true;
options.arena_block_size = 8 << 20;
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, 1024);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
auto env = Env::Default();
std::string db_path;
Status s = env->GetTestDirectory(&db_path);
@@ -748,7 +748,7 @@ static void SimpleGetWithPerfContext(benchmark::State& state) {
benchmark::Counter(static_cast<double>(get_from_table_nanos),
benchmark::Counter::kAvgIterations);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
TeardownDB(state, db, options, kg);
}
}
@@ -857,10 +857,10 @@ static void IteratorSeek(benchmark::State& state) {
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
}
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, key_num);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "IteratorSeek");
// load db
@@ -905,7 +905,7 @@ static void IteratorSeek(benchmark::State& state) {
}
}
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
TeardownDB(state, db, options, kg);
}
}
@@ -951,10 +951,10 @@ static void IteratorNext(benchmark::State& state) {
Options options;
options.compaction_style = compaction_style;
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, key_num);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "IteratorNext");
// load db
auto wo = WriteOptions();
@@ -999,7 +999,7 @@ static void IteratorNext(benchmark::State& state) {
}
}
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
TeardownDB(state, db, options, kg);
}
}
@@ -1025,10 +1025,10 @@ static void IteratorNextWithPerfContext(benchmark::State& state) {
static DB* db;
Options options;
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, 1024);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "IteratorNextWithPerfContext");
// load db
auto wo = WriteOptions();
@@ -1100,7 +1100,7 @@ static void IteratorNextWithPerfContext(benchmark::State& state) {
benchmark::Counter(static_cast<double>(iter_next_cpu_nanos),
benchmark::Counter::kAvgIterations);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
TeardownDB(state, db, options, kg);
}
}
@@ -1119,10 +1119,10 @@ static void IteratorPrev(benchmark::State& state) {
Options options;
options.compaction_style = compaction_style;
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, key_num);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "IteratorPrev");
// load db
auto wo = WriteOptions();
@@ -1167,7 +1167,7 @@ static void IteratorPrev(benchmark::State& state) {
}
}
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
TeardownDB(state, db, options, kg);
}
}
@@ -1212,10 +1212,10 @@ static void PrefixSeek(benchmark::State& state) {
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
}
auto rnd = Random(301 + state.thread_index());
auto rnd = Random(301 + state.thread_index);
KeyGenerator kg(&rnd, key_num, key_num / 100);
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
SetupDB(state, options, &db, "PrefixSeek");
// load db
@@ -1259,7 +1259,7 @@ static void PrefixSeek(benchmark::State& state) {
}
}
if (state.thread_index() == 0) {
if (state.thread_index == 0) {
TeardownDB(state, db, options, kg);
}
}
+1 -1
View File
@@ -226,7 +226,7 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
{LAST_LEVEL_READ_COUNT, "rocksdb.last.level.read.count"},
{NON_LAST_LEVEL_READ_BYTES, "rocksdb.non.last.level.read.bytes"},
{NON_LAST_LEVEL_READ_COUNT, "rocksdb.non.last.level.read.count"},
{BLOCK_CHECKSUM_COMPUTE_COUNT, "rocksdb.block.checksum.compute.count"}};
};
const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
{DB_GET, "rocksdb.db.get.micros"},
+5 -1
View File
@@ -738,7 +738,11 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
lowest_used_cache_tier(options.lowest_used_cache_tier),
compaction_service(options.compaction_service) {
fs = env->GetFileSystem();
clock = env->GetSystemClock().get();
if (env != nullptr) {
clock = env->GetSystemClock().get();
} else {
clock = SystemClock::Default().get();
}
logger = info_log.get();
stats = statistics.get();
}
-1
View File
@@ -189,7 +189,6 @@ TEST_F(OptionsSettableTest, BlockBasedTableOptionsAllFieldsSettable) {
"filter_policy=bloomfilter:4:true;whole_key_filtering=1;detect_filter_"
"construct_corruption=false;"
"reserve_table_builder_memory=false;"
"reserve_table_reader_memory=false;"
"format_version=1;"
"verify_compression=true;read_amp_bytes_per_bit=0;"
"enable_index_compression=false;"
-2
View File
@@ -855,7 +855,6 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
"block_size_deviation=8;block_restart_interval=4;"
"format_version=5;whole_key_filtering=1;"
"reserve_table_builder_memory=true;"
"reserve_table_reader_memory=true;"
"filter_policy=bloomfilter:4.567:false;detect_filter_construct_"
"corruption=true;"
// A bug caused read_amp_bytes_per_bit to be a large integer in OPTIONS
@@ -878,7 +877,6 @@ TEST_F(OptionsTest, GetBlockBasedTableOptionsFromString) {
ASSERT_EQ(new_opt.whole_key_filtering, true);
ASSERT_EQ(new_opt.detect_filter_construct_corruption, true);
ASSERT_EQ(new_opt.reserve_table_builder_memory, true);
ASSERT_EQ(new_opt.reserve_table_reader_memory, true);
ASSERT_TRUE(new_opt.filter_policy != nullptr);
auto bfp = new_opt.filter_policy->CheckedCast<BloomFilterPolicy>();
ASSERT_NE(bfp, nullptr);
Loaded 100 of 130 files, more files were not shown because too many files have changed in this diff. Show more