Exercise the remote compaction pool in stress tools

Summary:
The stress and benchmark tools need a way to select the separate remote compaction pool when exercising a compaction service.

Expose `use_separate_remote_compaction_pool` as a default-false flag in `db_stress` and `db_bench`, and pass it into their DB options. Randomize the flag in `db_crashtest.py` to cover both scheduling modes. The option takes effect when a compaction service is configured.

X-link: https://github.com/facebook/rocksdb/pull/15406

Reviewed By: xingbowang

Differential Revision: D125484658

fbshipit-source-id: 1fb3f0b92ff156f86d035f11b14e2c5316c60d19

db_stress_tool/db_stress_common.h

@@ -488,6 +488,7 @@ DECLARE_bool(track_and_verify_wals);
 DECLARE_int32(remote_compaction_worker_threads);
 DECLARE_int32(remote_compaction_worker_interval);
 DECLARE_bool(remote_compaction_failure_fall_back_to_local);
+DECLARE_bool(use_separate_remote_compaction_pool);
 DECLARE_uint32(openandcompact_max_secondary_open_retries);
 DECLARE_int32(allow_resumption_one_in);
 DECLARE_bool(auto_refresh_iterator_with_snapshot);

db_stress_tool/db_stress_gflags.cc

@@ -1127,6 +1127,9 @@ DEFINE_bool(remote_compaction_failure_fall_back_to_local, true,
             "If true, remote compaction failures will be ignored and "
             "compactions will fall back to local and retried");
 
+DEFINE_bool(use_separate_remote_compaction_pool, false,
+            "Sets DBOptions.use_separate_remote_compaction_pool");
+
 DEFINE_uint32(
     openandcompact_max_secondary_open_retries,
     ROCKSDB_NAMESPACE::OpenAndCompactOptions().max_secondary_open_retries,

db_stress_tool/db_stress_test_base.cc

@@ -5896,6 +5896,8 @@ void StressTest::Open(SharedState* shared, bool reopen) {
         shared, FLAGS_remote_compaction_failure_fall_back_to_local);
 
     options_.compaction_service = compaction_service;
+    options_.use_separate_remote_compaction_pool =
+        FLAGS_use_separate_remote_compaction_pool;
   }
 
   if (FLAGS_allow_resumption_one_in > 0) {

tools/db_bench_tool.cc

@@ -996,6 +996,9 @@ DEFINE_bool(sync, false, "Sync all writes to disk");
 
 DEFINE_bool(use_fsync, false, "If true, issue fsync instead of fdatasync");
 
+DEFINE_bool(use_separate_remote_compaction_pool, false,
+            "Sets DBOptions.use_separate_remote_compaction_pool");
+
 DEFINE_bool(disable_wal, false, "If true, do not write WAL for write.");
 
 DEFINE_bool(manual_wal_flush, false,
@@ -5870,6 +5873,8 @@ class Benchmark {
     options.log_readahead_size = FLAGS_log_readahead_size;
     options.writable_file_max_buffer_size = FLAGS_writable_file_max_buffer_size;
     options.use_fsync = FLAGS_use_fsync;
+    options.use_separate_remote_compaction_pool =
+        FLAGS_use_separate_remote_compaction_pool;
     options.num_levels = FLAGS_num_levels;
     options.target_file_size_base = FLAGS_target_file_size_base;
     options.target_file_size_multiplier = FLAGS_target_file_size_multiplier;

tools/db_crashtest.py

@@ -652,6 +652,7 @@ default_params = {
     "remote_compaction_worker_threads": lambda: random.choice([0, 8]),
     "allow_resumption_one_in": lambda: random.choice([0, 1, 2, 20]),
     "remote_compaction_failure_fall_back_to_local": lambda: random.choice([0, 1]),
+    "use_separate_remote_compaction_pool": lambda: random.choice([0, 1]),
     "auto_refresh_iterator_with_snapshot": lambda: random.choice([0, 1]),
     "memtable_op_scan_flush_trigger": lambda: random.choice([0, 10, 100, 1000]),
     "memtable_avg_op_scan_flush_trigger": lambda: random.choice([0, 2, 20, 200]),