db_bench_tool: fix buffer size

Summary:
Found by gcc warning:

x86_64-pc-linux-gnu-g++ --version
x86_64-pc-linux-gnu-g++ (GCC) 7.1.1 20170710

tools/db_bench_tool.cc: In member function 'void rocksdb::Benchmark::RandomWithVerify(rocksdb::ThreadState*)':
tools/db_bench_tool.cc:4430:8: error: '%lu' directive output may be truncated writing between 1 and 19 bytes into a region of size between 0 and 66 [-Werror=format-truncation=]
   void RandomWithVerify(ThreadState* thread) {
        ^~~~~~~~~~~~~~~~
tools/db_bench_tool.cc:4430:8: note: directive argument in the range [0, 9223372036854775807]
tools/db_bench_tool.cc:4492:13: note: 'snprintf' output between 37 and 128 bytes into a destination of size 100
     snprintf(msg, sizeof(msg),
     ~~~~~~~~^~~~~~~~~~~~~~~~~~
              "( get:%" PRIu64 " put:%" PRIu64 " del:%" PRIu64 " total:%" \
              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
              PRIu64 " found:%" PRIu64 ")",
              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
              gets_done, puts_done, deletes_done, readwrites_, found);
              ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
cc1plus: all warnings being treated as errors
Makefile:1707: recipe for target 'tools/db_bench_tool.o' failed
Closes https://github.com/facebook/rocksdb/pull/2558

Differential Revision: D5398703

Pulled By: siying

fbshipit-source-id: 6ffa552bbd8b59cfc2c36289f86ff9b9acca8ca6

.circleci/cat_ignore_eagain

@@ -1,54 +0,0 @@
-#! /bin/bash
-
-# Work around issue with parallel make output causing random error, as in
-# make[1]: write error: stdout
-# Probably due to a kernel bug:
-#   https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
-# Seems to affect image ubuntu-1604:201903-01 and ubuntu-1604:202004-01
-
-cd "$(dirname $0)"
-
-if [ ! -x cat_ignore_eagain.out ]; then
-    cc -x c -o cat_ignore_eagain.out - << EOF
-#include <unistd.h>
-#include <errno.h>
-#include <stdio.h>
-int main() {
-    int n, m, p;
-    char buf[1024];
-    for (;;) {
-        n = read(STDIN_FILENO, buf, 1024);
-        if (n > 0 && n <= 1024) {
-            for (m = 0; m < n;) {
-                p = write(STDOUT_FILENO, buf + m, n - m);
-                if (p < 0) {
-                    if (errno == EAGAIN) {
-                        // ignore but pause a bit
-                        usleep(100);
-                    } else {
-                        perror("write failed");
-                        return 42;
-                    }
-                } else {
-                    m += p;
-                }
-            }
-        } else if (n < 0) {
-            if (errno == EAGAIN) {
-                // ignore but pause a bit
-                usleep(100);
-            } else {
-                // Some non-ignorable error
-                perror("read failed");
-                return 43;
-            }
-        } else {
-            // EOF
-            return 0;
-        }
-    }
-}
-EOF
-fi
-
-exec ./cat_ignore_eagain.out

.circleci/config.yml

@@ -1,807 +0,0 @@
-version: 2.1
-
-orbs:
-  win: circleci/windows@2.4.0
-  slack: circleci/slack@3.4.2
-
-aliases:
-  - &notify-on-master-failure
-    fail_only: true
-    only_for_branches: master
-
-commands:
-  install-pyenv-on-macos:
-    steps:
-      - run:
-          name: Install pyenv on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv
-
-  install-cmake-on-macos:
-    steps:
-      - run:
-          name: Install cmake on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
-
-  increase-max-open-files-on-macos:
-    steps:
-      - run:
-          name: Increase max open files
-          command: |
-            sudo sysctl -w kern.maxfiles=1048576
-            sudo sysctl -w kern.maxfilesperproc=1048576
-            sudo launchctl limit maxfiles 1048576
-
-  pre-steps:
-    parameters:
-      python-version:
-        default: "3.5.9"
-        type: string
-    steps:
-      - checkout
-      - run: pyenv install --skip-existing <<parameters.python-version>>
-      - run: pyenv global <<parameters.python-version>>
-      - run:
-          name: Setup Environment Variables
-          command: |
-            echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
-            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
-            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
-            echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
-            echo "export GTEST_COLOR=1" >> $BASH_ENV
-            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
-            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
-
-  pre-steps-macos:
-      steps:
-        - pre-steps:
-            python-version: "3.6.0"
-
-  post-steps:
-    steps:
-      - slack/status: *notify-on-master-failure
-      - store_test_results: # store test result if there's any
-          path: /tmp/test-results
-      - store_artifacts: # store LOG for debugging if there's any
-          path: LOG
-      - run: # on fail, compress Test Logs for diagnosing the issue
-         name: Compress Test Logs
-         command: tar -cvzf t.tar.gz t
-         when: on_fail
-      - store_artifacts: # on fail, store Test Logs for diagnosing the issue
-          path: t.tar.gz
-          destination: test_logs
-          when: on_fail
-
-  install-clang-10:
-    steps:
-      - run:
-          name: Install Clang 10
-          command: |
-            echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
-            echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
-            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
-            sudo apt-get update -y && sudo apt-get install -y clang-10
-
-  install-gflags:
-    steps:
-      - run:
-          name: Install gflags
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-
-  install-benchmark:
-    steps:
-      - run: # currently doesn't support ubuntu-1604 which doesn't have libbenchmark package, user can still install by building it youself
-          name: Install benchmark
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libbenchmark-dev
-
-  upgrade-cmake:
-    steps:
-      - run:
-          name: Upgrade cmake
-          command: |
-            sudo apt remove --purge cmake
-            sudo snap install cmake --classic
-
-  install-gflags-on-macos:
-    steps:
-      - run:
-          name: Install gflags on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
-
-  install-gtest-parallel:
-    steps:
-      - run:
-          name: Install gtest-parallel
-          command: |
-            git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
-            echo 'export PATH=$HOME/gtest-parallel:$PATH' >> $BASH_ENV
-
-  install-compression-libs:
-    steps:
-      - run:
-          name: Install compression libs
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
-
-executors:
-  windows-2xlarge:
-    machine:
-      image: 'windows-server-2019-vs2019:stable'
-      resource_class: windows.2xlarge
-      shell: bash.exe
-
-jobs:
-  build-macos:
-    macos:
-      xcode: 11.3.0
-    resource_class: large
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-cmake:
-    macos:
-      xcode: 11.3.0
-    resource_class: large
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-cmake-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-mem-env:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: MEM_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-encrypted-env:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-shared_lib-alt_namespace-status_checked:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-release:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - install-gflags
-      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-      - post-steps
-
-  build-linux-release-rtti:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-
-  build-linux-lite:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-lite-release:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - install-gflags
-      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-      - post-steps
-
-  build-linux-clang-no_test_run:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev libtbb-dev
-      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-clang10-asan:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - post-steps
-
-  build-linux-clang10-mini-tsan:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
-      - post-steps
-
-  build-linux-clang10-ubsan:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - post-steps
-
-  build-linux-clang10-clang-analyze:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
-      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
-      - post-steps
-
-  build-linux-cmake:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - upgrade-cmake
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-cmake-ubuntu-20:
-    machine:
-      image: ubuntu-2004:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-benchmark
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 && make microbench) | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-unity:
-    docker: # executor type
-      - image: gcc:latest
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: apt-get update -y && apt-get install -y libgflags-dev
-      - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-gcc-4_8-no_test_run:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
-      - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-8-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
-      - run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-9-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-9 g++-9 libgflags-dev
-      - run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-10-cxx20-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: xlarge
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
-      - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
-  build-linux-microbench:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: xlarge
-    steps:
-      - pre-steps
-      - install-benchmark
-      - run: DEBUG_LEVEL=0 make microbench | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-windows:
-    executor: windows-2xlarge
-    parameters:
-      extra_cmake_opt:
-        default: ""
-        type: string
-      vs_year:
-        default: "2019"
-        type: string
-      cmake_generator:
-        default: "Visual Studio 16 2019"
-        type: string
-    environment:
-      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
-      CMAKE_HOME: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
-      CMAKE_BIN: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
-      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.7
-      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
-      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
-      VS_YEAR: <<parameters.vs_year>>
-      CMAKE_GENERATOR: <<parameters.cmake_generator>>
-    steps:
-      - checkout
-      - run:
-          name: "Setup VS"
-          command: |
-            if [[ "${VS_YEAR}" == "2019" ]]; then
-              echo "VS2019 already present."
-            elif [[ "${VS_YEAR}" == "2017" ]]; then
-              echo "Installing VS2017..."
-              powershell .circleci/vs2017_install.ps1
-            elif [[ "${VS_YEAR}" == "2015" ]]; then
-              echo "Installing VS2015..."
-              powershell .circleci/vs2015_install.ps1
-            fi
-      - store_artifacts:
-          path: \Users\circleci\AppData\Local\Temp\vslogs.zip
-      - run:
-          name: "Install thirdparty dependencies"
-          command: |
-            mkdir ${THIRDPARTY_HOME}
-            cd ${THIRDPARTY_HOME}
-            echo "Installing CMake..."
-            curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
-            unzip -q cmake-3.16.4-win64-x64.zip
-            echo "Building Snappy dependency..."
-            curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
-            unzip -q snappy-1.1.7.zip
-            cd snappy-1.1.7
-            mkdir build
-            cd build
-            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
-            msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
-      - run:
-          name: "Build RocksDB"
-          command: |
-            mkdir build
-            cd build
-            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
-            cd ..
-            echo "Building with VS version: ${CMAKE_GENERATOR}"
-            msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
-      - run:
-          name: "Test RocksDB"
-          shell: powershell.exe
-          command: |
-            build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
-
-  build-linux-java:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
-      - run:
-          name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-java-static:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-java:
-    macos:
-      xcode: 11.3.0
-    resource_class: medium
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
-      - run:
-          name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-java-static:
-    macos:
-      xcode: 11.3.0
-    resource_class: medium
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - install-cmake-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-examples:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Build examples"
-          command: |
-            OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain
-      - post-steps
-
-  build-cmake-mingw:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y mingw-w64
-      - run: sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
-      - run:
-          name: "Build cmake-mingw"
-          command: |
-            sudo apt-get install snapd && sudo snap install cmake --beta --classic
-            export PATH=/snap/bin:$PATH
-            sudo apt-get install -y openjdk-8-jdk
-            export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-            export PATH=$JAVA_HOME/bin:$PATH
-            echo "JAVA_HOME=${JAVA_HOME}"
-            which java && java -version
-            which javac && javac -version
-            mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
-      - post-steps
-
-  build-linux-non-shm:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    parameters:
-      start_test:
-        default: ""
-        type: string
-      end_test:
-        default: ""
-        type: string
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-gtest-parallel
-      - run:
-          name: "Build unit tests"
-          command: |
-            echo "env: $(env)"
-            echo "** done env"
-            ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests
-      - run:
-          name: "Run unit tests in parallel"
-          command: |
-            sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
-            cat /tmp/test_list
-            export TEST_TMPDIR=/tmp/rocksdb_test_tmp
-            gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat  # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
-      - post-steps
-
-  build-linux-arm-test-full:
-    machine:
-      image: ubuntu-2004:202101-01
-    resource_class: arm.large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: make V=1 J=4 -j4 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-arm:
-    machine:
-      image: ubuntu-2004:202101-01
-    resource_class: arm.large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-arm-cmake-no_test_run:
-    machine:
-      image: ubuntu-2004:202101-01
-    resource_class: arm.large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build with cmake"
-          command: |
-            mkdir build
-            cd build
-            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
-            make -j4
-      - run:
-          name: "Build Java with cmake"
-          command: |
-            rm -rf build
-            mkdir build
-            cd build
-            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
-            make -j4 rocksdb rocksdbjni
-      - post-steps
-
-  build-format-compatible:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-compression-libs
-      - run:
-          name: "test"
-          command: |
-            export TEST_TMPDIR=/dev/shm/rocksdb
-            rm -rf /dev/shm/rocksdb
-            mkdir /dev/shm/rocksdb
-            tools/check_format_compatible.sh
-      - post-steps
-
-workflows:
-  version: 2
-  build-linux:
-    jobs:
-      - build-linux
-  build-linux-cmake:
-    jobs:
-      - build-linux-cmake
-      - build-linux-cmake-ubuntu-20
-  build-linux-mem-env:
-    jobs:
-      - build-linux-mem-env
-  build-linux-encrypted-env:
-    jobs:
-      - build-linux-encrypted-env
-  build-linux-shared_lib-alt_namespace-status_checked:
-    jobs:
-      - build-linux-shared_lib-alt_namespace-status_checked
-  build-linux-lite:
-    jobs:
-      - build-linux-lite
-  build-linux-release:
-    jobs:
-      - build-linux-release
-  build-linux-release-rtti:
-    jobs:
-      - build-linux-release-rtti
-  build-linux-lite-release:
-    jobs:
-      - build-linux-lite-release
-  build-linux-clang10-asan:
-    jobs:
-      - build-linux-clang10-asan
-  build-linux-clang10-mini-tsan:
-    jobs:
-      - build-linux-clang10-mini-tsan
-  build-linux-clang10-ubsan:
-    jobs:
-      - build-linux-clang10-ubsan
-  build-linux-clang10-clang-analyze:
-    jobs:
-      - build-linux-clang10-clang-analyze
-  build-linux-unity:
-    jobs:
-      - build-linux-unity
-  build-windows-vs2019:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2019"
-  build-windows-vs2019-cxx20:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2019-cxx20"
-          extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
-  build-windows-vs2017:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2017"
-          vs_year: "2017"
-          cmake_generator: "Visual Studio 15 Win64"
-  build-java:
-    jobs:
-      - build-linux-java
-      - build-linux-java-static
-      - build-macos-java
-      - build-macos-java-static
-  build-examples:
-    jobs:
-      - build-examples
-  build-linux-non-shm:
-    jobs:
-      - build-linux-non-shm:
-          start_test: ""
-          end_test: "db_options_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "db_options_test" # make sure unique in src.mk
-          end_test: "filename_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "filename_test" # make sure unique in src.mk
-          end_test: "statistics_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "statistics_test" # make sure unique in src.mk
-          end_test: ""
-  build-linux-compilers-no_test_run:
-    jobs:
-      - build-linux-clang-no_test_run
-      - build-linux-gcc-4_8-no_test_run
-      - build-linux-gcc-8-no_test_run
-      - build-linux-gcc-9-no_test_run
-      - build-linux-gcc-10-cxx20-no_test_run
-      - build-linux-arm-cmake-no_test_run
-  build-macos:
-    jobs:
-      - build-macos
-  build-macos-cmake:
-    jobs:
-      - build-macos-cmake
-  build-cmake-mingw:
-    jobs:
-      - build-cmake-mingw
-  build-linux-arm:
-    jobs:
-      - build-linux-arm
-  build-microbench:
-    jobs:
-      - build-linux-microbench
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - build-format-compatible
-      - build-linux-arm-test-full

.circleci/ubsan_suppression_list.txt

@@ -1,6 +0,0 @@
-# Supress UBSAN warnings related to stl_tree.h, e.g.
-# UndefinedBehaviorSanitizer: undefined-behavior /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43 in 
-# /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43:
-# runtime error: upcast of address 0x000001fa8820 with insufficient space for an object of type
-# 'std::_Rb_tree_node<std::pair<const std::__cxx11::basic_string<char>, rocksdb::(anonymous namespace)::LockHoldingInfo> >'
-src:*bits/stl_tree.h

.circleci/vs2015_install.ps1

@@ -1,24 +0,0 @@
-$VS_DOWNLOAD_LINK = "https://go.microsoft.com/fwlink/?LinkId=691126"
-$COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
-curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
-if ($LASTEXITCODE -ne 0) {
-    echo "Download of the VS 2015 installer failed"
-    exit 1
-}
-$VS_INSTALL_ARGS = @("/Quiet", "/NoRestart")
-$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
-Remove-Item -Path vs_installer.exe -Force
-$exitCode = $process.ExitCode
-if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-    echo "VS 2015 installer exited with code $exitCode, which should be one of [0, 3010]."
-    curl.exe --retry 3 -kL $COLLECT_DOWNLOAD_LINK --output Collect.exe
-    if ($LASTEXITCODE -ne 0) {
-        echo "Download of the VS Collect tool failed."
-        exit 1
-    }
-    Start-Process "${PWD}\Collect.exe" -NoNewWindow -Wait -PassThru
-    New-Item -Path "C:\w\build-results" -ItemType "directory" -Force
-    Copy-Item -Path "C:\Users\circleci\AppData\Local\Temp\vslogs.zip" -Destination "C:\w\build-results\"
-    exit 1
-}
-echo "VS 2015 installed."

.circleci/vs2017_install.ps1

@@ -1,35 +0,0 @@
-$VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
-$COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
-$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.14.13",
-                                                     "--add Microsoft.Component.MSBuild",
-                                                     "--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
-                                                     "--add Microsoft.VisualStudio.Component.TextTemplating",
-                                                     "--add Microsoft.VisualStudio.Component.VC.CoreIde",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
-                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
-                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81")
-
-curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
-if ($LASTEXITCODE -ne 0) {
-    echo "Download of the VS 2017 installer failed"
-    exit 1
-}
-
-$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
-Remove-Item -Path vs_installer.exe -Force
-$exitCode = $process.ExitCode
-if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-    echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
-    curl.exe --retry 3 -kL $COLLECT_DOWNLOAD_LINK --output Collect.exe
-    if ($LASTEXITCODE -ne 0) {
-        echo "Download of the VS Collect tool failed."
-        exit 1
-    }
-    Start-Process "${PWD}\Collect.exe" -NoNewWindow -Wait -PassThru
-    New-Item -Path "C:\w\build-results" -ItemType "directory" -Force
-    Copy-Item -Path "C:\Users\circleci\AppData\Local\Temp\vslogs.zip" -Destination "C:\w\build-results\"
-    exit 1
-}
-echo "VS 2017 installed."

.deprecated_arcconfig

@@ -0,0 +1,17 @@
+{
+  "project_id" : "rocksdb",
+  "conduit_uri" : "https://phabricator.fb.com/api/",
+  "copyright_holder" : "Facebook",
+  "load" : [
+    "arcanist_util"
+  ],
+  "lint.engine" : "FacebookFbcodeLintEngine",
+  "lint.engine.single.linter" : "FbcodeCppLinter",
+  "unit.engine" : "FacebookFbcodeUnitTestEngine",
+  "arcanist_configuration" : "FacebookArcanistConfiguration",
+  "base" : "git:HEAD^, hg:.^",
+  "git.default-relative-commit" : "HEAD^",
+  "git:arc.feature.start.default" : "origin/master",
+  "arc.feature.start.default" : "master",
+  "history.immutable" : false
+}

.github/workflows/sanity_check.yml

@@ -1,41 +0,0 @@
-name: Check buck targets and code format
-on: [push, pull_request]
-jobs:
-  check:
-    name: Check TARGETS file and code format
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout feature branch
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: Fetch from upstream
-      run: |
-        git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream
-
-    - name: Where am I
-      run: |
-        echo git status && git status
-        echo "git remote -v" && git remote -v
-        echo git branch && git branch
-
-    - name: Setup Python
-      uses: actions/setup-python@v1
-
-    - name: Install Dependencies
-      run: python -m pip install --upgrade pip
-
-    - name: Install argparse
-      run: pip install argparse
-
-    - name: Download clang-format-diff.py
-      uses: wei/wget@v1
-      with:
-        args: https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py
-
-    - name: Check format
-      run: VERBOSE_CHECK=1 make check-format
-
-    - name: Compare buckify output
-      run: make check-buck-targets

.gitignore

@@ -1,5 +1,4 @@
 make_config.mk
-rocksdb.pc
 
 *.a
 *.arc
@@ -8,7 +7,6 @@ rocksdb.pc
 *.gcda
 *.gcno
 *.o
-*.o.tmp
 *.so
 *.so.*
 *_test
@@ -26,7 +24,6 @@ rocksdb.pc
 *.vcxproj.filters
 *.sln
 *.cmake
-.watchmanconfig
 CMakeCache.txt
 CMakeFiles/
 build/
@@ -35,9 +32,6 @@ ldb
 manifest_dump
 sst_dump
 blob_dump
-block_cache_trace_analyzer
-db_with_timestamp_basic_test
-tools/block_cache_analyzer/*.pyc
 column_aware_encoding_exp
 util/build_version.cc
 build_tools/VALGRIND_LOGS/
@@ -45,19 +39,13 @@ coverage/COVERAGE_REPORT
 .gdbhistory
 .gdb_history
 package/
+.phutil_module_cache
 unity.a
 tags
 etags
 rocksdb_dump
 rocksdb_undump
 db_test2
-trace_analyzer
-trace_analyzer_test
-block_cache_trace_analyzer
-io_tracer_parser
-.DS_Store
-.vs
-.vscode
 
 java/out
 java/target
@@ -85,13 +73,3 @@ tp2/
 fbcode/
 fbcode
 buckifier/*.pyc
-buckifier/__pycache__
-
-compile_commands.json
-clang-format-diff.py
-.py3/
-
-fuzz/proto/gen/
-fuzz/crash-*
-
-cmake-build-*

.lgtm.yml

@@ -1,4 +0,0 @@
-extraction:
-  cpp:
-    index:
-      build_command: make static_lib

.travis.yml

@@ -1,36 +1,26 @@
-dist: xenial
+sudo: false
+dist: trusty
 language: cpp
 os:
   - linux
-arch:
-  - arm64
-  - ppc64le
+  - osx
 compiler:
   - clang
   - gcc
+osx_image: xcode8.3
+jdk:
+  - oraclejdk7
 cache:
   - ccache
+  - apt
 
 addons:
-  apt:
-    update: true
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - libgflags-dev
-      - libbz2-dev
-      - liblz4-dev
-      - libsnappy-dev
-      - liblzma-dev  # xv
-      - libzstd-dev
-      - zlib1g-dev
-
+   apt:
+      packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
 env:
   - TEST_GROUP=platform_dependent # 16-18 minutes
   - TEST_GROUP=1 # 33-35 minutes
-  - TEST_GROUP=2 # 18-20 minutes
-  - TEST_GROUP=3 # 20-22 minutes
-  - TEST_GROUP=4 # 12-14 minutes
+  - TEST_GROUP=2 # 30-32 minutes
   # Run java tests
   - JOB_NAME=java_test # 4-11 minutes
   # Build ROCKSDB_LITE
@@ -38,160 +28,26 @@ env:
   # Build examples
   - JOB_NAME=examples # 5-7 minutes
   - JOB_NAME=cmake # 3-5 minutes
-  - JOB_NAME=cmake-gcc8 # 3-5 minutes
-  - JOB_NAME=cmake-gcc9 # 3-5 minutes
-  - JOB_NAME=cmake-gcc9-c++20 # 3-5 minutes
   - JOB_NAME=cmake-mingw # 3 minutes
-  - JOB_NAME=make-gcc4.8
-  - JOB_NAME=status_checked
 
 matrix:
   exclude:
-  - os : linux
-    arch: arm64
+  - os: osx
+    env: TEST_GROUP=1
+  - os: osx
+    env: TEST_GROUP=2
+  - os : osx
     env: JOB_NAME=cmake-mingw
   - os : linux
-    arch: arm64
-    env: JOB_NAME=make-gcc4.8
-  - os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-mingw
-  - os: linux
-    arch: ppc64le
-    env: JOB_NAME=make-gcc4.8
-  - os: linux
     compiler: clang
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: arm64
-    env: TEST_GROUP=platform_dependent
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=status_checked
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=status_checked
+  - os : osx
+    compiler: gcc
 
+# https://docs.travis-ci.com/user/caching/#ccache-cache
 install:
-  - if [ "${JOB_NAME}" == cmake-gcc8 ]; then
-      sudo apt-get install -y g++-8 || exit $?;
-      CC=gcc-8 && CXX=g++-8;
-    fi
-  - if [ "${JOB_NAME}" == cmake-gcc9 ] || [ "${JOB_NAME}" == cmake-gcc9-c++20 ]; then
-      sudo apt-get install -y g++-9 || exit $?;
-      CC=gcc-9 && CXX=g++-9;
-    fi
-  - if [ "${JOB_NAME}" == cmake-mingw ]; then
-      sudo apt-get install -y mingw-w64 || exit $?;
-    fi
-  - if [ "${JOB_NAME}" == make-gcc4.8 ]; then
-      sudo apt-get install -y g++-4.8 || exit $?;
-      CC=gcc-4.8 && CXX=g++-4.8;
-    fi
-  - |
-    if [[ "${JOB_NAME}" == cmake* ]]; then
-      sudo apt-get remove -y cmake cmake-data
-      export CMAKE_DEB="cmake-3.14.5-Linux-$(uname -m).deb"
-      export CMAKE_DEB_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/${CMAKE_DEB}"
-      curl --silent --fail --show-error --location --output "${CMAKE_DEB}" "${CMAKE_DEB_URL}" || exit $?
-      sudo dpkg -i "${CMAKE_DEB}" || exit $?
-      which cmake && cmake --version
-    fi
-  - |
-    if [[ "${JOB_NAME}" == java_test || "${JOB_NAME}" == cmake* ]]; then
-      # Ensure JDK 8
-      sudo apt-get install -y openjdk-8-jdk || exit $?
-      export PATH=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin:$PATH
-      export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)
-      echo "JAVA_HOME=${JAVA_HOME}"
-      which java && java -version
-      which javac && javac -version
+  - if [ "${TRAVIS_OS_NAME}" == osx ]; then
+      brew install ccache;
+      PATH=$PATH:/usr/local/opt/ccache/libexec;
     fi
 
 before_script:
@@ -200,55 +56,17 @@ before_script:
   - ulimit -n 8192
 
 script:
-  - date; ${CXX} --version
-  - if [ `command -v ccache` ]; then ccache -C; fi
-  - case $TEST_GROUP in
-    platform_dependent)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=only make -j4 all_but_some_tests check_some
-      ;;
-    1)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_END=backupable_db_test make -j4 check_some
-      ;;
-    2)
-      OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 make -j4 tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=backupable_db_test ROCKSDBTESTS_END=db_universal_compaction_test make -j4 check_some
-      ;;
-    3)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=db_universal_compaction_test ROCKSDBTESTS_END=table_properties_collector_test make -j4 check_some
-      ;;
-    4)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=table_properties_collector_test make -j4 check_some
-      ;;
-    esac
-  - case $JOB_NAME in
-    java_test)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make rocksdbjava jtest
-      ;;
-    lite_build)
-      OPT='-DTRAVIS -DROCKSDB_LITE' LIB_MODE=shared V=1 make -j4 all
-      ;;
-    examples)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make -j4 static_lib && cd examples && make -j4
-      ;;
-    cmake-mingw)
-      sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
-      mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
-      ;;
-    cmake*)
-      case $JOB_NAME in
-        *-c++20)
-          OPT=-DCMAKE_CXX_STANDARD=20
-          ;;
-      esac
-
-      mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. && make -j4 && cd  .. && rm -rf build && mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j4 rocksdb rocksdbjni
-      ;;
-    make-gcc4.8)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 SKIP_LINK=1 make -j4 all && [ "Linking broken because libgflags compiled with newer ABI" ]
-      ;;
-    status_checked)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ASSERT_STATUS_CHECKED=1 make -j4 check_some
-      ;;
-    esac
+  - ${CXX} --version
+  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 make -j4 all; OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
+  - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
+  - if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
+  - if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
+  - if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake .. && make -j4 rocksdb; fi
+  - if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb; fi
 notifications:
     email:
       - leveldb@fb.com
+    webhooks:
+      - https://buildtimetrend.herokuapp.com/travis

.watchmanconfig

@@ -1,6 +0,0 @@
-{
-  "content_hash_warming": true,
-  "content_hash_max_items": 333333,
-  "hint_num_files_per_dir": 8,
-  "fsevents_latency": 0.05
-}

AUTHORS

@@ -9,4 +9,3 @@ Sanjay Ghemawat <sanjay@google.com>
 # Partial list of contributors:
 Kevin Regan <kevin.d.regan@gmail.com>
 Johan Bilien <jobi@litl.com>
-Matthew Von-Maszewski <https://github.com/matthewvon> (Basho Technologies)

CODE_OF_CONDUCT.md

@@ -1,77 +0,0 @@
-# Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies within all project spaces, and it also applies when
-an individual is representing the project or its community in public spaces.
-Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting
-as an appointed representative at an online or offline event. Representation of
-a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at <opensource-conduct@fb.com>. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
-

CONTRIBUTING.md

@@ -1,8 +1,5 @@
 # Contributing to RocksDB
 
-## Code of Conduct
-The code of conduct is described in [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md)
-
 ## Contributor License Agreement ("CLA")
 
 In order to accept your pull request, we need you to submit a CLA. You

INSTALL.md

@@ -32,7 +32,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 * You can link RocksDB with following compression libraries:
   - [zlib](http://www.zlib.net/) - a library for data compression.
   - [bzip2](http://www.bzip.org/) - a library for data compression.
-  - [lz4](https://github.com/lz4/lz4) - a library for extremely fast data compression.
   - [snappy](http://google.github.io/snappy/) - a library for fast
       data compression.
   - [zstandard](http://www.zstd.net) - Fast real-time compression
@@ -43,10 +42,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
       command line flags processing. You can compile rocksdb library even
       if you don't have gflags installed.
 
-* `make check` will also check code formatting, which requires [clang-format](https://clang.llvm.org/docs/ClangFormat.html)
-
-* If you wish to build the RocksJava static target, then cmake is required for building Snappy.
-
 ## Supported platforms
 
 * **Linux - Ubuntu**
@@ -58,7 +53,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
       `sudo apt-get install libsnappy-dev`.
     * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
     * Install bzip2: `sudo apt-get install libbz2-dev`.
-    * Install lz4: `sudo apt-get install liblz4-dev`.
     * Install zstandard: `sudo apt-get install libzstd-dev`.
 
 * **Linux - CentOS / RHEL**
@@ -87,30 +81,17 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
 
               sudo yum install bzip2 bzip2-devel
 
-    * Install lz4:
-
-              sudo yum install lz4-devel
-
     * Install ASAN (optional for debugging):
 
               sudo yum install libasan
 
     * Install zstandard:
-        * With [EPEL](https://fedoraproject.org/wiki/EPEL):
 
-              sudo yum install libzstd-devel
-
-        * With CentOS 8:
-
-              sudo dnf install libzstd-devel
-
-        * From source:
-
-              wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
-              mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
-              tar zxvf zstd-1.1.3.tar.gz
-              cd zstd-1.1.3
-              make && sudo make install
+             wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
+             mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
+             tar zxvf zstd-1.1.3.tar.gz
+             cd zstd-1.1.3
+             make && sudo make install
 
 * **OS X**:
     * Install latest C++ compiler that supports C++ 11:
@@ -120,70 +101,12 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
             * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
     * run `brew install rocksdb`
 
-* **FreeBSD** (11.01):
-
-    * You can either install RocksDB from the Ports system using `cd /usr/ports/databases/rocksdb && make install`, or you can follow the details below to install dependencies and compile from source code:
-
-    * Install the dependencies for RocksDB:
-
-        export BATCH=YES
-        cd /usr/ports/devel/gmake && make install
-        cd /usr/ports/devel/gflags && make install
-
-        cd /usr/ports/archivers/snappy && make install
-        cd /usr/ports/archivers/bzip2 && make install
-        cd /usr/ports/archivers/liblz4 && make install
-        cd /usr/ports/archivesrs/zstd && make install
-
-        cd /usr/ports/devel/git && make install
-
-
-    * Install the dependencies for RocksJava (optional):
-
-        export BATCH=yes
-        cd /usr/ports/java/openjdk7 && make install
-
-    * Build RocksDB from source:
-        cd ~
-        git clone https://github.com/facebook/rocksdb.git
-        cd rocksdb
-        gmake static_lib
-
-    * Build RocksJava from source (optional):
-        cd rocksdb
-        export JAVA_HOME=/usr/local/openjdk7
-        gmake rocksdbjava
-
-* **OpenBSD** (6.3/-current):
-
-    * As RocksDB is not available in the ports yet you have to build it on your own:
-
-    * Install the dependencies for RocksDB:
-
-        pkg_add gmake gflags snappy bzip2 lz4 zstd git jdk bash findutils gnuwatch 
-
-    * Build RocksDB from source:
-
-        cd ~
-        git clone https://github.com/facebook/rocksdb.git
-        cd rocksdb
-        gmake static_lib
-
-    * Build RocksJava from source (optional):
-
-        cd rocksdb
-        export JAVA_HOME=/usr/local/jdk-1.8.0
-        export PATH=$PATH:/usr/local/jdk-1.8.0/bin
-        gmake rocksdbjava
-
 * **iOS**:
   * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
 
 * **Windows**:
   * For building with MS Visual Studio 13 you will need Update 4 installed.
   * Read and follow the instructions at CMakeLists.txt
-  * Or install via [vcpkg](https://github.com/microsoft/vcpkg) 
-       * run `vcpkg install rocksdb:x64-windows`
 
 * **AIX 6.1**
     * Install AIX Toolbox rpms with gcc

LANGUAGE-BINDINGS.md

@@ -1,24 +1,13 @@
 This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
 
 * Java - https://github.com/facebook/rocksdb/tree/master/java
-* Python
-    * http://python-rocksdb.readthedocs.io/en/latest/
-    * http://pyrocksdb.readthedocs.org/en/latest/ (unmaintained)
+* Python - http://pyrocksdb.readthedocs.org/en/latest/
 * Perl - https://metacpan.org/pod/RocksDB
 * Node.js - https://npmjs.org/package/rocksdb
 * Go - https://github.com/tecbot/gorocksdb
 * Ruby - http://rubygems.org/gems/rocksdb-ruby
 * Haskell - https://hackage.haskell.org/package/rocksdb-haskell
 * PHP - https://github.com/Photonios/rocksdb-php
-* C#
-    * https://github.com/warrenfalk/rocksdb-sharp
-    * https://github.com/curiosity-ai/rocksdb-sharp
-* Rust
-    * https://github.com/pingcap/rust-rocksdb (used in production fork of https://github.com/spacejam/rust-rocksdb)
-    * https://github.com/spacejam/rust-rocksdb
-    * https://github.com/bh1xuw/rust-rocks
+* C# - https://github.com/warrenfalk/rocksdb-sharp
+* Rust - https://github.com/spacejam/rust-rocksdb
 * D programming language - https://github.com/b1naryth1ef/rocksdb
-* Erlang - https://gitlab.com/barrel-db/erlang-rocksdb
-* Elixir - https://github.com/urbint/rox
-* Nim - https://github.com/status-im/nim-rocksdb
-* Swift and Objective-C (iOS/OSX) - https://github.com/iabudiab/ObjectiveRocks 

PLUGINS.md

@@ -1,4 +0,0 @@
-This is the list of all known third-party plugins for RocksDB. If something is missing, please open a pull request to add it.
-
-* [Dedupfs](https://github.com/ajkr/dedupfs): an example for plugin developers to reference
-* [ZenFS](https://github.com/westerndigitalcorporation/zenfs): a file system for zoned block devices

README.md

@@ -1,20 +1,19 @@
 ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage
 
-[![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb)
-[![TravisCI Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
-[![Appveyor Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
-[![PPC64le Build Status](http://140-211-168-68-openstack.osuosl.org:8080/buildStatus/icon?job=rocksdb&style=plastic)](http://140-211-168-68-openstack.osuosl.org:8080/job/rocksdb)
+[![Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
+[![Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
+
 
 RocksDB is developed and maintained by Facebook Database Engineering Team.
-It is built on earlier work on [LevelDB](https://github.com/google/leveldb) by Sanjay Ghemawat (sanjay@google.com)
+It is built on earlier work on LevelDB by Sanjay Ghemawat (sanjay@google.com)
 and Jeff Dean (jeff@google.com)
 
 This code is a library that forms the core building block for a fast
-key-value server, especially suited for storing data on flash drives.
+key value server, especially suited for storing data on flash drives.
 It has a Log-Structured-Merge-Database (LSM) design with flexible tradeoffs
 between Write-Amplification-Factor (WAF), Read-Amplification-Factor (RAF)
 and Space-Amplification-Factor (SAF). It has multi-threaded compactions,
-making it especially suitable for storing multiple terabytes of data in a
+making it specially suitable for storing multiple terabytes of data in a
 single database.
 
 Start with example usage here: https://github.com/facebook/rocksdb/tree/master/examples
@@ -25,8 +24,4 @@ The public interface is in `include/`.  Callers should not include or
 rely on the details of any other header files in this package.  Those
 internal APIs may be changed without warning.
 
-Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/ and https://rocksdb.slack.com/
-
-## License
-
-RocksDB is dual-licensed under both the GPLv2 (found in the COPYING file in the root directory) and Apache 2.0 License (found in the LICENSE.Apache file in the root directory).  You may select, at your option, one of the above-listed licenses.
+Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/

ROCKSDB_LITE.md

@@ -5,7 +5,7 @@ RocksDBLite is a project focused on mobile use cases, which don't need a lot of
 Some examples of the features disabled by ROCKSDB_LITE:
 * compiled-in support for LDB tool
 * No backupable DB
-* No support for replication (which we provide in form of TransactionalIterator)
+* No support for replication (which we provide in form of TrasactionalIterator)
 * No advanced monitoring tools
 * No special-purpose memtables that are highly optimized for specific use cases
 * No Transactions

USERS.md

@@ -5,15 +5,12 @@ At Facebook, we use RocksDB as storage engines in multiple data management servi
 
 1. MyRocks -- https://github.com/MySQLOnRocksDB/mysql-5.6
 2. MongoRocks -- https://github.com/mongodb-partners/mongo-rocks
-3. ZippyDB --  Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[1] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
-4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[1]
+3. ZippyDB --  Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[*] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
+4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[*]
 4. Dragon -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/
-5. Stylus -- a low-level stream processing framework writtenin C++.[1]
-6. LogDevice -- a distributed data store for logs [2]
+5. Stylus -- a low-level stream processing framework writtenin C++.[*]
 
-[1] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
-
-[2] https://code.facebook.com/posts/357056558062811/logdevice-a-distributed-data-store-for-logs/
+[*] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
 
 ## LinkedIn
 Two different use cases at Linkedin are using RocksDB as a storage engine:
@@ -26,11 +23,8 @@ Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasu
 ## Yahoo
 Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights
 
-## Baidu
-[Apache Doris](http://doris.apache.org/master/en/) is a MPP analytical database engine released by Baidu. It [uses RocksDB](http://doris.apache.org/master/en/administrator-guide/operation/tablet-meta-tool.html) to manage its tablet's metadata.
-
 ## CockroachDB
-CockroachDB is an open-source geo-replicated transactional database. They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
+CockroachDB is an open-source geo-replicated transactional database (still in development). They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
 
 ## DNANexus
 DNANexus is using RocksDB to speed up processing of genomics data.
@@ -47,16 +41,12 @@ Tango is using RocksDB as a graph storage to store all users' connection data an
 Turn is using RocksDB as a storage layer for their key/value store, serving at peak 2.4MM QPS out of different datacenters.
 Check out our RocksDB Protobuf merge operator at: https://github.com/vladb38/rocksdb_protobuf
 
-## Santander UK/Cloudera Profession Services
+## Santanader UK/Cloudera Profession Services
 Check out their blog post: http://blog.cloudera.com/blog/2015/08/inside-santanders-near-real-time-data-ingest-architecture/
 
 ## Airbnb
 Airbnb is using RocksDB as a storage engine for their personalized search service. You can learn more about it here: https://www.youtube.com/watch?v=ASQ6XMtogMs
 
-## Alluxio
-[Alluxio](https://www.alluxio.io) uses RocksDB to serve and scale file system metadata to beyond 1 Billion files. The detailed design and implementation is described in this engineering blog:
-https://www.alluxio.io/blog/scalable-metadata-service-in-alluxio-storing-billions-of-files/
-
 ## Pinterest
 Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtube.com/watch?v=MtFEVEs_2Vo
 
@@ -70,7 +60,7 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub
 [VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
 
 ## quasardb
-[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark.
+[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
 quasardb uses a heavily tuned RocksDB as its persistence layer.
 
 ## Netflix
@@ -86,32 +76,7 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 [Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine.
 
 ## Uber
-[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue.
+[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and and scalable task queue.
 
 ## 360 Pika
-[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been used in many companies.
-
-## LzLabs
-LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
-
-## ProfaneDB
-[ProfaneDB](https://profanedb.gitlab.io/) is a database for Protocol Buffers, and uses RocksDB for storage. It is accessible via gRPC, and the schema is defined using directly `.proto` files.
-
-## IOTA Foundation
- [IOTA Foundation](https://www.iota.org/) is using RocksDB in the [IOTA Reference Implementation (IRI)](https://github.com/iotaledger/iri) to store the local state of the Tangle. The Tangle is the first open-source distributed ledger powering the future of the Internet of Things.
-
-## Avrio Project
- [Avrio Project](http://avrio-project.github.io/avrio.network/) is using RocksDB in [Avrio ](https://github.com/avrio-project/avrio) to store blocks, account balances and data and other blockchain-releated data. Avrio is a multiblockchain decentralized cryptocurrency empowering monetary transactions.
-
-## Crux
-[Crux](https://github.com/juxt/crux) is a document database that uses RocksDB for local [EAV](https://en.wikipedia.org/wiki/Entity%E2%80%93attribute%E2%80%93value_model) index storage to enable point-in-time bitemporal Datalog queries. The "unbundled" architecture uses Kafka to provide horizontal scalability.
-
-## Nebula Graph
-[Nebula Graph](https://github.com/vesoft-inc/nebula) is a distributed, scalable, lightning-fast, open source graph database capable of hosting super large scale graphs with dozens of billions of vertices (nodes) and trillions of edges, with milliseconds of latency.
-
-## YugabyteDB
-[YugabyteDB](https://www.yugabyte.com/) is an open source, high performance, distributed SQL database that uses RocksDB as its storage layer. For more information, please see https://github.com/yugabyte/yugabyte-db/.
-
-## ArangoDB
-[ArangoDB](https://www.arangodb.com/) is a native multi-model database with flexible data models for documents, graphs, and key-values, for building high performance applications using a convenient SQL-like query language or JavaScript extensions. It uses RocksDB as its sotrage engine.
-
+[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company

Vagrantfile

@@ -14,11 +14,6 @@ Vagrant.configure("2") do |config|
     box.vm.box = "chef/centos-6.5"
   end
 
-  config.vm.define "centos7" do |box|
-    box.vm.box = "centos/7"
-    box.vm.provision "shell", path: "build_tools/setup_centos7.sh"
-  end
-
   config.vm.define "FreeBSD10" do |box|
     box.vm.guest = :freebsd
     box.vm.box = "robin/freebsd-10"

WINDOWS_PORT.md

@@ -43,9 +43,9 @@ We plan to use this port for our business purposes here at Bing and this provide
 
 * Certain headers that are not present and not necessary on Windows were simply `#ifndef OS_WIN` in a few places (`unistd.h`)
 * All posix specific headers were replaced to port/port.h which worked well
-* Replaced `dirent.h` for `port/port_dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
+* Replaced `dirent.h` for `port/dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
 * Replaced `sys/time.h` to `port/sys_time.h` (few places) implemented equivalents within `rocksdb::port`
-* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `zu` on posix systems and to `Iu` on windows.
+* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `%z` on posix systems and to Iu on windows.
 * in class member initialization were moved to a __ctors in some cases
 * `constexpr` is not supported. We had to replace `std::numeric_limits<>::max/min()` to its C macros for constants. Sometimes we had to make class members `static const` and place a definition within a .cc file.
 * `constexpr` for functions was replaced to a template specialization (1 place)

appveyor.yml

@@ -1,77 +1,15 @@
 version: 1.0.{build}
-
-image: Visual Studio 2019
-
-environment:
-  JAVA_HOME: C:\Program Files\Java\jdk1.8.0
-  THIRDPARTY_HOME: $(APPVEYOR_BUILD_FOLDER)\thirdparty
-  SNAPPY_HOME: $(THIRDPARTY_HOME)\snappy-1.1.7
-  SNAPPY_INCLUDE: $(SNAPPY_HOME);$(SNAPPY_HOME)\build
-  SNAPPY_LIB_DEBUG: $(SNAPPY_HOME)\build\Debug\snappy.lib
-  SNAPPY_LIB_RELEASE: $(SNAPPY_HOME)\build\Release\snappy.lib
-  LZ4_HOME: $(THIRDPARTY_HOME)\lz4-1.8.3
-  LZ4_INCLUDE: $(LZ4_HOME)\lib
-  LZ4_LIB_DEBUG: $(LZ4_HOME)\visual\VS2010\bin\x64_Debug\liblz4_static.lib
-  LZ4_LIB_RELEASE: $(LZ4_HOME)\visual\VS2010\bin\x64_Release\liblz4_static.lib
-  ZSTD_HOME: $(THIRDPARTY_HOME)\zstd-1.4.0
-  ZSTD_INCLUDE: $(ZSTD_HOME)\lib;$(ZSTD_HOME)\lib\dictBuilder
-  ZSTD_LIB_DEBUG: $(ZSTD_HOME)\build\VS2010\bin\x64_Debug\libzstd_static.lib
-  ZSTD_LIB_RELEASE: $(ZSTD_HOME)\build\VS2010\bin\x64_Release\libzstd_static.lib
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-      CMAKE_GENERATOR: Visual Studio 14 Win64
-      DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE\devenv.com
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-      CMAKE_GENERATOR: Visual Studio 15 Win64
-      DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com
-
-install:
-  - md %THIRDPARTY_HOME%
-  - echo "Building Snappy dependency..."
-  - cd %THIRDPARTY_HOME%
-  - curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
-  - unzip snappy-1.1.7.zip
-  - cd snappy-1.1.7
-  - mkdir build
-  - cd build
-  - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
-  - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT%
-  - msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
-  - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
-  - echo "Building LZ4 dependency..."
-  - cd %THIRDPARTY_HOME%
-  - curl --fail --silent --show-error --output lz4-1.8.3.zip --location https://github.com/lz4/lz4/archive/v1.8.3.zip
-  - unzip lz4-1.8.3.zip
-  - cd lz4-1.8.3\visual\VS2010
-  - ps: $CMD="$Env:DEV_ENV"; & $CMD lz4.sln /upgrade
-  - msbuild lz4.sln /p:Configuration=Debug /p:Platform=x64
-  - msbuild lz4.sln /p:Configuration=Release /p:Platform=x64
-  - echo "Building ZStd dependency..."
-  - cd %THIRDPARTY_HOME%
-  - curl --fail --silent --show-error --output zstd-1.4.0.zip --location https://github.com/facebook/zstd/archive/v1.4.0.zip
-  - unzip zstd-1.4.0.zip
-  - cd zstd-1.4.0\build\VS2010
-  - ps: $CMD="$Env:DEV_ENV"; & $CMD zstd.sln /upgrade
-  - msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64
-  - msbuild zstd.sln /p:Configuration=Release /p:Platform=x64
-
+image: Visual Studio 2015
 before_build:
-  - md %APPVEYOR_BUILD_FOLDER%\build
-  - cd %APPVEYOR_BUILD_FOLDER%\build
-  - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
-  - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT% %CMAKE_OPT% -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 -DWITH_ALL_TESTS=0
-  - cd ..
-
+- md %APPVEYOR_BUILD_FOLDER%\build
+- cd %APPVEYOR_BUILD_FOLDER%\build
+- cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DXPRESS=1 ..
+- cd ..
 build:
   project: build\rocksdb.sln
   parallel: true
-  verbosity: normal
-
+  verbosity: minimal
 test:
-
 test_script:
-  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,env_basic_test -Concurrency 8
-
-on_failure:
-  - cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip
+- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8
 

arcanist_util/INTERNAL_ONLY_DIR

@@ -0,0 +1,2 @@
+arcanist_util are only used internaly, If you want to change it please check
+<internal_rocksdb_repo>/arcanist_util

buckifier/buckify_rocksdb.py

@@ -1,37 +1,16 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-try:
-    from builtins import str
-except ImportError:
-    from __builtin__ import str
 from targets_builder import TARGETSBuilder
-import json
+from optparse import OptionParser
 import os
 import fnmatch
 import sys
+import tempfile
 
 from util import ColorString
-
-# This script generates TARGETS file for Buck.
-# Buck is a build tool specifying dependencies among different build targets.
-# User can pass extra dependencies as a JSON object via command line, and this
-# script can include these dependencies in the generate TARGETS file.
-# Usage:
-# $python3 buckifier/buckify_rocksdb.py
-# (This generates a TARGET file without user-specified dependency for unit
-# tests.)
-# $python3 buckifier/buckify_rocksdb.py \
-#        '{"fake": {
-#                      "extra_deps": [":test_dep", "//fakes/module:mock1"],
-#                      "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]
-#                  }
-#         }'
-# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
-# unit tests, and will use the extra_compiler_flags to compile the unit test
-# source.)
+import util
 
 # tests to export as libraries for inclusion in other projects
 _EXPORTED_TEST_LIBS = ["env_basic_test"]
@@ -48,8 +27,8 @@ def parse_src_mk(repo_path):
         if '=' in line:
             current_src = line.split('=')[0].strip()
             src_files[current_src] = []
-        elif '.c' in line:
-            src_path = line.split('\\')[0].strip()
+        elif '.cc' in line:
+            src_path = line.split('.cc')[0].strip() + '.cc'
             src_files[current_src].append(src_path)
     return src_files
 
@@ -57,7 +36,7 @@ def parse_src_mk(repo_path):
 # get all .cc / .c files
 def get_cc_files(repo_path):
     cc_files = []
-    for root, dirnames, filenames in os.walk(repo_path):  # noqa: B007 T25377293 Grandfathered in
+    for root, dirnames, filenames in os.walk(repo_path):
         root = root[(len(repo_path) + 1):]
         if "java" in root:
             # Skip java
@@ -69,160 +48,97 @@ def get_cc_files(repo_path):
     return cc_files
 
 
-# Get non_parallel tests from Makefile
-def get_non_parallel_tests(repo_path):
+# Get tests from Makefile
+def get_tests(repo_path):
     Makefile = repo_path + "/Makefile"
 
-    s = set({})
+    # Dictionary TEST_NAME => IS_PARALLEL
+    tests = {}
 
-    found_non_parallel_tests = False
+    found_tests = False
     for line in open(Makefile):
         line = line.strip()
-        if line.startswith("NON_PARALLEL_TEST ="):
-            found_non_parallel_tests = True
-        elif found_non_parallel_tests:
+        if line.startswith("TESTS ="):
+            found_tests = True
+        elif found_tests:
             if line.endswith("\\"):
                 # remove the trailing \
                 line = line[:-1]
                 line = line.strip()
-                s.add(line)
+                tests[line] = False
             else:
-                # we consumed all the non_parallel tests
+                # we consumed all the tests
                 break
 
-    return s
-
-# Parse extra dependencies passed by user from command line
-def get_dependencies():
-    deps_map = {
-        '': {
-            'extra_deps': [],
-            'extra_compiler_flags': []
-        }
-    }
-    if len(sys.argv) < 2:
-        return deps_map
-
-    def encode_dict(data):
-        rv = {}
-        for k, v in data.items():
-            if isinstance(v, dict):
-                v = encode_dict(v)
-            rv[k] = v
-        return rv
-    extra_deps = json.loads(sys.argv[1], object_hook=encode_dict)
-    for target_alias, deps in extra_deps.items():
-        deps_map[target_alias] = deps
-    return deps_map
+    found_parallel_tests = False
+    for line in open(Makefile):
+        line = line.strip()
+        if line.startswith("PARALLEL_TEST ="):
+            found_parallel_tests = True
+        elif found_parallel_tests:
+            if line.endswith("\\"):
+                # remove the trailing \
+                line = line[:-1]
+                line = line.strip()
+                tests[line] = True
+            else:
+                # we consumed all the parallel tests
+                break
+    
+    return tests
 
 
 # Prepare TARGETS file for buck
-def generate_targets(repo_path, deps_map):
+def generate_targets(repo_path):
     print(ColorString.info("Generating TARGETS"))
     # parsed src.mk file
     src_mk = parse_src_mk(repo_path)
     # get all .cc files
     cc_files = get_cc_files(repo_path)
-    # get non_parallel tests from Makefile
-    non_parallel_tests = get_non_parallel_tests(repo_path)
+    # get tests from Makefile
+    tests = get_tests(repo_path)
 
-    if src_mk is None or cc_files is None or non_parallel_tests is None:
+    if src_mk is None or cc_files is None or tests is None:
         return False
 
-    extra_argv = ""
-    if len(sys.argv) >= 2:
-        # Heuristically quote and canonicalize whitespace for inclusion
-        # in how the file was generated.
-        extra_argv = " '{0}'".format(" ".join(sys.argv[1].split()))
-
-    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path, extra_argv)
-
+    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
     # rocksdb_lib
     TARGETS.add_library(
         "rocksdb_lib",
         src_mk["LIB_SOURCES"] +
-        # always add range_tree, it's only excluded on ppc64, which we don't use internally
-        src_mk["RANGE_TREE_SOURCES"] +
         src_mk["TOOL_LIB_SOURCES"])
-    # rocksdb_whole_archive_lib
-    TARGETS.add_library(
-        "rocksdb_whole_archive_lib",
-        src_mk["LIB_SOURCES"] +
-        # always add range_tree, it's only excluded on ppc64, which we don't use internally
-        src_mk["RANGE_TREE_SOURCES"] +
-        src_mk["TOOL_LIB_SOURCES"],
-        deps=None,
-        headers=None,
-        extra_external_deps="",
-        link_whole=True)
     # rocksdb_test_lib
     TARGETS.add_library(
         "rocksdb_test_lib",
         src_mk.get("MOCK_LIB_SOURCES", []) +
         src_mk.get("TEST_LIB_SOURCES", []) +
-        src_mk.get("EXP_LIB_SOURCES", []) +
-        src_mk.get("ANALYZER_LIB_SOURCES", []),
-        [":rocksdb_lib"],
-        extra_external_deps=""" + [
-        ("googletest", None, "gtest"),
-    ]""")
+        src_mk.get("EXP_LIB_SOURCES", []),
+        [":rocksdb_lib"])
     # rocksdb_tools_lib
     TARGETS.add_library(
         "rocksdb_tools_lib",
         src_mk.get("BENCH_LIB_SOURCES", []) +
-        src_mk.get("ANALYZER_LIB_SOURCES", []) +
-        ["test_util/testutil.cc"],
+        ["util/testutil.cc"],
         [":rocksdb_lib"])
-    # rocksdb_cache_bench_tools_lib
-    TARGETS.add_library(
-        "rocksdb_cache_bench_tools_lib",
-        src_mk.get("CACHE_BENCH_LIB_SOURCES", []),
-        [":rocksdb_lib"])
-    # rocksdb_stress_lib
-    TARGETS.add_rocksdb_library(
-        "rocksdb_stress_lib",
-        src_mk.get("ANALYZER_LIB_SOURCES", [])
-        + src_mk.get('STRESS_LIB_SOURCES', [])
-        + ["test_util/testutil.cc"])
 
-    print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
+    # test for every test we found in the Makefile
+    for test in sorted(tests):
+        match_src = [src for src in cc_files if ("/%s.c" % test) in src]
+        if len(match_src) == 0:
+            print(ColorString.warning("Cannot find .cc file for %s" % test))
+            continue
+        elif len(match_src) > 1:
+            print(ColorString.warning("Found more than one .cc for %s" % test))
+            print(match_src)
+            continue
 
-    # Dictionary test executable name -> relative source file path
-    test_source_map = {}
-    print(src_mk)
+        assert(len(match_src) == 1)
+        is_parallel = tests[test]
+        TARGETS.register_test(test, match_src[0], is_parallel)
 
-    # c_test.c is added through TARGETS.add_c_test(). If there
-    # are more than one .c test file, we need to extend
-    # TARGETS.add_c_test() to include other C tests too.
-    for test_src in src_mk.get("TEST_MAIN_SOURCES_C", []):
-        if test_src != 'db/c_test.c':
-            print("Don't know how to deal with " + test_src)
-            return False
-    TARGETS.add_c_test()
-
-    for test_src in src_mk.get("TEST_MAIN_SOURCES", []):
-        test = test_src.split('.c')[0].strip().split('/')[-1].strip()
-        test_source_map[test] = test_src
-        print("" + test + " " + test_src)
-
-    for target_alias, deps in deps_map.items():
-        for test, test_src in sorted(test_source_map.items()):
-            if len(test) == 0:
-                print(ColorString.warning("Failed to get test name for %s" % test_src))
-                continue
-
-            test_target_name = \
-                test if not target_alias else test + "_" + target_alias
-            TARGETS.register_test(
-                test_target_name,
-                test_src,
-                test not in non_parallel_tests,
-                json.dumps(deps['extra_deps']),
-                json.dumps(deps['extra_compiler_flags']))
-
-            if test in _EXPORTED_TEST_LIBS:
-                test_library = "%s_lib" % test_target_name
-                TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"])
+        if test in _EXPORTED_TEST_LIBS:
+            test_library = "%s_lib" % test
+            TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
     TARGETS.flush_tests()
 
     print(ColorString.info("Generated TARGETS Summary:"))
@@ -241,16 +157,14 @@ def get_rocksdb_path():
 
     return rocksdb_path
 
-
 def exit_with_error(msg):
     print(ColorString.error(msg))
     sys.exit(1)
 
 
 def main():
-    deps_map = get_dependencies()
     # Generate TARGETS file for buck
-    ok = generate_targets(get_rocksdb_path(), deps_map)
+    ok = generate_targets(get_rocksdb_path())
     if not ok:
         exit_with_error("Failed to generate TARGETS files")
 

buckifier/check_buck_targets.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# If clang_format_diff.py command is not specfied, we assume we are able to
-# access directly without any path.
-
-TGT_DIFF=`git diff TARGETS | head -n 1`
-
-if [ ! -z "$TGT_DIFF" ]
-then
-  echo "TARGETS file has uncommitted changes. Skip this check."
-  exit 0
-fi
-
-echo Backup original TARGETS file.
-
-cp TARGETS TARGETS.bkp
-
-${PYTHON:-python3} buckifier/buckify_rocksdb.py
-
-TGT_DIFF=`git diff TARGETS | head -n 1`
-
-if [ -z "$TGT_DIFF" ]
-then
-  mv TARGETS.bkp TARGETS
-  exit 0
-else
-  echo "Please run '${PYTHON:-python3} buckifier/buckify_rocksdb.py' to update TARGETS file."
-  echo "Do not manually update TARGETS file."
-  ${PYTHON:-python3} --version
-  mv TARGETS.bkp TARGETS
-  exit 1
-fi

buckifier/rocks_test_runner.sh

@@ -1,6 +1,4 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
-# shellcheck disable=SC2068
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"

buckifier/targets_builder.py

@@ -1,17 +1,12 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-try:
-    from builtins import object
-    from builtins import str
-except ImportError:
-    from __builtin__ import object
-    from __builtin__ import str
 import targets_cfg
+import pprint
 
-def pretty_list(lst, indent=8):
+# TODO(tec): replace this with PrettyPrinter
+def pretty_list(lst, indent=6):
     if lst is None or len(lst) == 0:
         return ""
 
@@ -19,106 +14,52 @@ def pretty_list(lst, indent=8):
         return "\"%s\"" % lst[0]
 
     separator = "\",\n%s\"" % (" " * indent)
-    res = separator.join(sorted(lst))
-    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 4))
+    res = separator.join(lst)
+    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2))
     return res
 
 
-class TARGETSBuilder(object):
-    def __init__(self, path, extra_argv):
+class TARGETSBuilder:
+    def __init__(self, path):
         self.path = path
-        self.targets_file = open(path, 'wb')
-        header = targets_cfg.rocksdb_target_header_template.format(
-            extra_argv=extra_argv)
-        self.targets_file.write(header.encode("utf-8"))
+        self.targets_file = open(path, 'w')
+        self.targets_file.write(targets_cfg.rocksdb_target_header)
         self.total_lib = 0
         self.total_bin = 0
         self.total_test = 0
-        self.tests_cfg = ""
+        self.tests_cfg = []
 
     def __del__(self):
         self.targets_file.close()
 
-    def add_library(self, name, srcs, deps=None, headers=None,
-                    extra_external_deps="", link_whole=False):
-        headers_attr_prefix = ""
+    def add_library(self, name, srcs, deps=None, headers=None):
         if headers is None:
-            headers_attr_prefix = "auto_"
             headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
-            headers = "[" + pretty_list(headers) + "]"
-        self.targets_file.write(targets_cfg.library_template.format(
-            name=name,
-            srcs=pretty_list(srcs),
-            headers_attr_prefix=headers_attr_prefix,
-            headers=headers,
-            deps=pretty_list(deps),
-            extra_external_deps=extra_external_deps,
-            link_whole=link_whole).encode("utf-8"))
-        self.total_lib = self.total_lib + 1
-
-    def add_rocksdb_library(self, name, srcs, headers=None):
-        headers_attr_prefix = ""
-        if headers is None:
-            headers_attr_prefix = "auto_"
-            headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
-            headers = "[" + pretty_list(headers) + "]"
-        self.targets_file.write(targets_cfg.rocksdb_library_template.format(
-            name=name,
-            srcs=pretty_list(srcs),
-            headers_attr_prefix=headers_attr_prefix,
-            headers=headers).encode("utf-8"))
+        self.targets_file.write(targets_cfg.library_template % (
+            name,
+            headers,
+            pretty_list(srcs),
+            pretty_list(deps)))
         self.total_lib = self.total_lib + 1
 
     def add_binary(self, name, srcs, deps=None):
-        self.targets_file.write(targets_cfg.binary_template.format(
-            name=name,
-            srcs=pretty_list(srcs),
-            deps=pretty_list(deps)).encode("utf-8"))
+        self.targets_file.write(targets_cfg.binary_template % (
+            name,
+            pretty_list(srcs),
+            pretty_list(deps)))
         self.total_bin = self.total_bin + 1
 
-    def add_c_test(self):
-        self.targets_file.write(b"""
-cpp_binary(
-    name = "c_test_bin",
-    srcs = ["db/c_test.c"],
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [":rocksdb_test_lib"],
-) if not is_opt_mode else None
-
-custom_unittest(
-    name = "c_test",
-    command = [
-        native.package_name() + "/buckifier/rocks_test_runner.sh",
-        "$(location :{})".format("c_test_bin"),
-    ],
-    type = "simple",
-) if not is_opt_mode else None
-""")
-
-    def register_test(self,
-                      test_name,
-                      src,
-                      is_parallel,
-                      extra_deps,
-                      extra_compiler_flags):
+    def register_test(self, test_name, src, is_parallel):
         exec_mode = "serial"
         if is_parallel:
             exec_mode = "parallel"
-        self.tests_cfg += targets_cfg.test_cfg_template % (
-            test_name,
-            str(src),
-            str(exec_mode),
-            extra_deps,
-            extra_compiler_flags)
+        self.tests_cfg.append([test_name, str(src), str(exec_mode)])
 
         self.total_test = self.total_test + 1
 
     def flush_tests(self):
-        self.targets_file.write(targets_cfg.unittests_template.format(tests=self.tests_cfg).encode("utf-8"))
-        self.tests_cfg = ""
+        self.targets_file.write(targets_cfg.unittests_template % (
+            pprint.PrettyPrinter().pformat(self.tests_cfg)
+        ))
+
+        self.tests_cfg = []

buckifier/targets_cfg.py

@@ -1,220 +1,120 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
+rocksdb_target_header = """
+import os
 
-rocksdb_target_header_template = \
-    """# This file \100generated by:
-#$ python3 buckifier/buckify_rocksdb.py{extra_argv}
-# --> DO NOT EDIT MANUALLY <--
-# This file is a Facebook-specific integration for buck builds, so can
-# only be validated by Facebook employees.
-#
-load("@fbcode_macros//build_defs:auto_headers.bzl", "AutoHeaders")
-load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
-load(":defs.bzl", "test_binary")
-
-REPO_PATH = package_name() + "/"
-
-ROCKSDB_COMPILER_FLAGS_0 = [
-    "-fno-builtin-memcmp",
-    # Needed to compile in fbcode
-    "-Wno-expansion-to-defined",
-    # Added missing flags from output of build_detect_platform
-    "-Wnarrowing",
-    "-DROCKSDB_NO_DYNAMIC_EXTENSION",
+TARGETS_PATH = os.path.dirname(__file__)
+REPO_PATH = TARGETS_PATH[(TARGETS_PATH.find('fbcode/') + len('fbcode/')):] + "/"
+BUCK_BINS = "buck-out/gen/" + REPO_PATH
+TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
+rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
+  "-fno-builtin-memcmp",
+  "-DROCKSDB_PLATFORM_POSIX",
+  "-DROCKSDB_LIB_IO_POSIX",
+  "-DROCKSDB_FALLOCATE_PRESENT",
+  "-DROCKSDB_MALLOC_USABLE_SIZE",
+  "-DROCKSDB_RANGESYNC_PRESENT",
+  "-DROCKSDB_SCHED_GETCPU_PRESENT",
+  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+  "-DHAVE_SSE42",
+  "-DOS_LINUX",
+  # Flags to enable libs we include
+  "-DSNAPPY",
+  "-DZLIB",
+  "-DBZIP2",
+  "-DLZ4",
+  "-DZSTD",
+  "-DGFLAGS=gflags",
+  "-DNUMA",
+  "-DTBB",
+  # Needed to compile in fbcode
+  "-Wno-expansion-to-defined",
 ]
 
-ROCKSDB_EXTERNAL_DEPS = [
-    ("bzip2", None, "bz2"),
-    ("snappy", None, "snappy"),
-    ("zlib", None, "z"),
-    ("gflags", None, "gflags"),
-    ("lz4", None, "lz4"),
-    ("zstd", None, "zstd"),
+rocksdb_external_deps = [
+  ('bzip2', None, 'bz2'),
+  ('snappy', None, "snappy"),
+  ('zlib', None, 'z'),
+  ('gflags', None, 'gflags'),
+  ('lz4', None, 'lz4'),
+  ('zstd', None),
+  ('tbb', None),
+  ("numa", None, "numa"),
+  ("googletest", None, "gtest"),
 ]
 
-ROCKSDB_OS_DEPS_0 = [
-    (
-        "linux",
-        ["third-party//numa:numa", "third-party//liburing:uring", "third-party//tbb:tbb"],
-    ),
-    (
-        "macos",
-        ["third-party//tbb:tbb"],
-    ),
+rocksdb_preprocessor_flags = [
+  # Directories with files for #include
+  "-I" + REPO_PATH + "include/",
+  "-I" + REPO_PATH,
 ]
-
-ROCKSDB_OS_PREPROCESSOR_FLAGS_0 = [
-    (
-        "linux",
-        [
-            "-DOS_LINUX",
-            "-DROCKSDB_FALLOCATE_PRESENT",
-            "-DROCKSDB_MALLOC_USABLE_SIZE",
-            "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
-            "-DROCKSDB_RANGESYNC_PRESENT",
-            "-DROCKSDB_SCHED_GETCPU_PRESENT",
-            "-DROCKSDB_IOURING_PRESENT",
-            "-DHAVE_SSE42",
-            "-DLIBURING",
-            "-DNUMA",
-            "-DROCKSDB_PLATFORM_POSIX",
-            "-DROCKSDB_LIB_IO_POSIX",
-            "-DTBB",
-        ],
-    ),
-    (
-        "macos",
-        [
-            "-DOS_MACOSX",
-            "-DROCKSDB_PLATFORM_POSIX",
-            "-DROCKSDB_LIB_IO_POSIX",
-            "-DTBB",
-        ],
-    ),
-    (
-        "windows",
-        [ "-DOS_WIN", "-DWIN32", "-D_MBCS", "-DWIN64", "-DNOMINMAX" ]
-    ),
-]
-
-ROCKSDB_PREPROCESSOR_FLAGS = [
-    "-DROCKSDB_SUPPORT_THREAD_LOCAL",
-
-    # Flags to enable libs we include
-    "-DSNAPPY",
-    "-DZLIB",
-    "-DBZIP2",
-    "-DLZ4",
-    "-DZSTD",
-    "-DZSTD_STATIC_LINKING_ONLY",
-    "-DGFLAGS=gflags",
-
-    # Added missing flags from output of build_detect_platform
-    "-DROCKSDB_BACKTRACE",
-]
-
-# Directories with files for #include
-ROCKSDB_INCLUDE_PATHS = [
-    "",
-    "include",
-]
-
-ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{
-    "x86_64": [
-        "-DHAVE_PCLMUL",
-    ],
-}}
-
-build_mode = read_config("fbcode", "build_mode")
-
-is_opt_mode = build_mode.startswith("opt")
-
-# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
-# doesn't harm and avoid forgetting to add it.
-ROCKSDB_COMPILER_FLAGS = ROCKSDB_COMPILER_FLAGS_0 + (["-DNDEBUG"] if is_opt_mode else [])
-
-sanitizer = read_config("fbcode", "sanitizer")
-
-# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
-# whether the binary is linked with jemalloc at runtime.
-ROCKSDB_OS_PREPROCESSOR_FLAGS = ROCKSDB_OS_PREPROCESSOR_FLAGS_0 + ([(
-    "linux",
-    ["-DROCKSDB_JEMALLOC"],
-)] if sanitizer == "" else [])
-
-ROCKSDB_OS_DEPS = ROCKSDB_OS_DEPS_0 + ([(
-    "linux",
-    ["third-party//jemalloc:headers"],
-)] if sanitizer == "" else [])
-
-ROCKSDB_LIB_DEPS = [
-    ":rocksdb_lib",
-    ":rocksdb_test_lib",
-] if not is_opt_mode else [":rocksdb_lib"]
 """
 
 
 library_template = """
 cpp_library(
-    name = "{name}",
-    srcs = [{srcs}],
-    {headers_attr_prefix}headers = {headers},
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    os_deps = ROCKSDB_OS_DEPS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [{deps}],
-    external_deps = ROCKSDB_EXTERNAL_DEPS{extra_external_deps},
-    link_whole = {link_whole},
-)
-"""
-
-rocksdb_library_template = """
-cpp_library(
-    name = "{name}",
-    srcs = [{srcs}],
-    {headers_attr_prefix}headers = {headers},
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    os_deps = ROCKSDB_OS_DEPS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = ROCKSDB_LIB_DEPS,
-    external_deps = ROCKSDB_EXTERNAL_DEPS,
+    name = "%s",
+    headers = %s,
+    srcs = [%s],
+    deps = [%s],
+    preprocessor_flags = rocksdb_preprocessor_flags,
+    compiler_flags = rocksdb_compiler_flags,
+    external_deps = rocksdb_external_deps,
 )
 """
 
 binary_template = """
 cpp_binary(
-    name = "{name}",
-    srcs = [{srcs}],
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [{deps}],
-    external_deps = ROCKSDB_EXTERNAL_DEPS,
+  name = "%s",
+  srcs = [%s],
+  deps = [%s],
+  preprocessor_flags = rocksdb_preprocessor_flags,
+  compiler_flags = rocksdb_compiler_flags,
+  external_deps = rocksdb_external_deps,
 )
 """
 
-test_cfg_template = """    [
-        "%s",
-        "%s",
-        "%s",
-        %s,
-        %s,
-    ],
-"""
-
 unittests_template = """
-# [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
-ROCKS_TESTS = [
-{tests}]
+# [test_name, test_src, test_type]
+ROCKS_TESTS = %s
+
 
 # Generate a test rule for each entry in ROCKS_TESTS
-# Do not build the tests in opt mode, since SyncPoint and other test code
-# will not be included.
-[
-    cpp_unittest(
-        name = test_name,
-        srcs = [test_cc],
-        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        compiler_flags = ROCKSDB_COMPILER_FLAGS + extra_compiler_flags,
-        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        include_paths = ROCKSDB_INCLUDE_PATHS,
-        deps = [":rocksdb_test_lib"] + extra_deps,
-        external_deps = ROCKSDB_EXTERNAL_DEPS + [
-            ("googletest", None, "gtest"),
-        ],
+for test_cfg in ROCKS_TESTS:
+    test_name = test_cfg[0]
+    test_cc = test_cfg[1]
+    ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
+    test_bin = test_name + "_bin"
+
+    cpp_binary (
+      name = test_bin,
+      srcs = [test_cc],
+      deps = [":rocksdb_test_lib"],
+      preprocessor_flags = rocksdb_preprocessor_flags,
+      compiler_flags = rocksdb_compiler_flags,
+      external_deps = rocksdb_external_deps,
     )
-    for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
-    if not is_opt_mode
-]
+
+    custom_unittest(
+      name = test_name,
+      type = ttype,
+      deps = [":" + test_bin],
+      command = [TEST_RUNNER, BUCK_BINS + test_bin]
+    )
+
+custom_unittest(
+    name = "make_rocksdbjavastatic",
+    type = "simple",
+    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
+)
+
+custom_unittest(
+    name = "make_rocksdb_lite_release",
+    type = "simple",
+    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
+)
 """

buckifier/util.py

@@ -1,4 +1,3 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 """
 This module keeps commonly used components.
 """
@@ -6,16 +5,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-try:
-    from builtins import object
-except ImportError:
-    from __builtin__ import object
 import subprocess
-import sys
 import os
 import time
 
-class ColorString(object):
+class ColorString:
     """ Generate colorful strings on terminal """
     HEADER = '\033[95m'
     BLUE = '\033[94m'
@@ -26,13 +20,7 @@ class ColorString(object):
 
     @staticmethod
     def _make_color_str(text, color):
-        # In Python2, default encoding for unicode string is ASCII
-        if sys.version_info.major <= 2:
-            return "".join(
-                [color, text.encode('utf-8'), ColorString.ENDC])
-        # From Python3, default encoding for unicode string is UTF-8
-        return "".join(
-            [color, text, ColorString.ENDC])
+        return "".join([color, text.encode('utf-8'), ColorString.ENDC])
 
     @staticmethod
     def ok(text):

build_tools/amalgamate.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
 # amalgamate.py creates an amalgamation from a unity build.
 # It can be run with either Python 2 or 3.

build_tools/build_detect_platform

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Detects OS we're compiling on and outputs a file specified by the first
 # argument, which in turn gets read while processing Makefile.
@@ -9,7 +9,6 @@
 #   PLATFORM_LDFLAGS            Linker flags
 #   JAVA_LDFLAGS                Linker flags for RocksDBJava
 #   JAVA_STATIC_LDFLAGS         Linker flags for RocksDBJava static build
-#   JAVAC_ARGS                  Arguments for javac
 #   PLATFORM_SHARED_EXT         Extension for shared libraries
 #   PLATFORM_SHARED_LDFLAGS     Flags for building shared library
 #   PLATFORM_SHARED_CFLAGS      Flags for compiling objects for shared library
@@ -17,8 +16,6 @@
 #   PLATFORM_CXXFLAGS           C++ compiler flags.  Will contain:
 #   PLATFORM_SHARED_VERSIONED   Set to 'true' if platform supports versioned
 #                               shared libraries, empty otherwise.
-#   FIND			Command for the find utility
-#   WATCH			Command for the watch utility
 #
 # The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
 #
@@ -28,7 +25,6 @@
 #       -DZSTD                      if the ZSTD library is present
 #       -DNUMA                      if the NUMA library is present
 #       -DTBB                       if the TBB library is present
-#       -DMEMKIND                   if the memkind library is present
 #
 # Using gflags in rocksdb:
 # Our project depends on gflags, which requires users to take some extra steps
@@ -45,13 +41,8 @@ if test -z "$OUTPUT"; then
   exit 1
 fi
 
-# we depend on C++11, but should be compatible with newer standards
-if [ "$ROCKSDB_CXX_STANDARD" ]; then
-  PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD"
-else
-  PLATFORM_CXXFLAGS="-std=c++11"
-fi
-
+# we depend on C++11
+PLATFORM_CXXFLAGS="-std=c++11"
 # we currently depend on POSIX platform
 COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
 
@@ -65,12 +56,8 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
       source "$PWD/build_tools/fbcode_config4.8.1.sh"
     elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
       source "$PWD/build_tools/fbcode_config.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM009" ]; then
-      source "$PWD/build_tools/fbcode_config_platform009.sh"
     else
-      source "$PWD/build_tools/fbcode_config_platform009.sh"
+      source "$PWD/build_tools/fbcode_config_platform007.sh"
     fi
 fi
 
@@ -79,33 +66,11 @@ rm -f "$OUTPUT"
 touch "$OUTPUT"
 
 if test -z "$CC"; then
-    if [ -x "$(command -v cc)" ]; then
-        CC=cc
-    elif [ -x "$(command -v clang)" ]; then
-        CC=clang
-    else
-        CC=cc
-    fi
+   CC=cc
 fi
 
 if test -z "$CXX"; then
-    if [ -x "$(command -v g++)" ]; then
-        CXX=g++
-    elif [ -x "$(command -v clang++)" ]; then
-        CXX=clang++
-    else
-        CXX=g++
-    fi
-fi
-
-if test -z "$AR"; then
-    if [ -x "$(command -v gcc-ar)" ]; then
-        AR=gcc-ar
-    elif [ -x "$(command -v llvm-ar)" ]; then
-        AR=llvm-ar
-    else
-        AR=ar
-    fi
+    CXX=g++
 fi
 
 # Detect OS
@@ -122,15 +87,7 @@ if test -z "$CLANG_SCAN_BUILD"; then
 fi
 
 if test -z "$CLANG_ANALYZER"; then
-    CLANG_ANALYZER=$(command -v clang++ 2> /dev/null)
-fi
-
-if test -z "$FIND"; then
-    FIND=find
-fi
-
-if test -z "$WATCH"; then
-    WATCH=watch
+    CLANG_ANALYZER=$(which clang++ 2> /dev/null)
 fi
 
 COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}"
@@ -167,28 +124,8 @@ case "$TARGET_OS" in
         COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
         if [ -z "$USE_CLANG" ]; then
             COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
-        fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
-        if test -z "$ROCKSDB_USE_IO_URING"; then
-            ROCKSDB_USE_IO_URING=1
-        fi
-        if test "$ROCKSDB_USE_IO_URING" -ne 0; then
-            # check for liburing
-            $CXX $PLATFORM_CXXFLAGS -x c++ - -luring -o /dev/null 2>/dev/null  <<EOF
-              #include <liburing.h>
-              int main() {
-                struct io_uring ring;
-                io_uring_queue_init(1, &ring, 0);
-                return 0;
-              }
-EOF
-            if [ "$?" = 0 ]; then
-                PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -luring"
-                COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_IOURING_PRESENT"
-            fi
         fi
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/linux/linux_specific.cc
         ;;
     SunOS)
@@ -206,22 +143,10 @@ EOF
         ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
-        CXX=clang++
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
         # PORT_FILES=port/freebsd/freebsd_specific.cc
         ;;
-    GNU/kFreeBSD)
-        PLATFORM=OS_GNU_KFREEBSD
-        COMMON_FLAGS="$COMMON_FLAGS -DOS_GNU_KFREEBSD"
-        if [ -z "$USE_CLANG" ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
-        fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
-        # PORT_FILES=port/gnu_kfreebsd/gnu_kfreebsd_specific.cc
-        ;;
     NetBSD)
         PLATFORM=OS_NETBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
@@ -230,12 +155,9 @@ EOF
         ;;
     OpenBSD)
         PLATFORM=OS_OPENBSD
-	CXX=clang++
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
         # PORT_FILES=port/openbsd/openbsd_specific.cc
-	FIND=gfind
-	WATCH=gnuwatch
         ;;
     DragonFly)
         PLATFORM=OS_DRAGONFLYBSD
@@ -250,8 +172,6 @@ EOF
         COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN"
         if [ -z "$USE_CLANG" ]; then
             COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
         fi
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/linux/linux_specific.cc
@@ -271,25 +191,20 @@ esac
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
 JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
 JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"
-JAVAC_ARGS="-source 7"
 
 if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
     # Cross-compiling; do not try any compilation tests.
     # Also don't need any compilation tests if compiling on fbcode
-    if [ "$FBCODE_BUILD" = "true" ]; then
-      # Enable backtrace on fbcode since the necessary libraries are present
-      COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-    fi
     true
 else
     if ! test $ROCKSDB_DISABLE_FALLOCATE; then
         # Test whether fallocate is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <fcntl.h>
           #include <linux/falloc.h>
           int main() {
       int fd = open("/dev/null", 0);
-      fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
+      fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
           }
 EOF
         if [ "$?" = 0 ]; then
@@ -297,311 +212,207 @@ EOF
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_SNAPPY; then
-        # Test whether Snappy library is installed
-        # http://code.google.com/p/snappy/
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <snappy.h>
-          int main() {}
+    # Test whether Snappy library is installed
+    # http://code.google.com/p/snappy/
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <snappy.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
     fi
 
-    if ! test $ROCKSDB_DISABLE_GFLAGS; then
-        # Test whether gflags library is installed
-        # http://gflags.github.io/gflags/
-        # check if the namespace is gflags
-        if $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-          #include <gflags/gflags.h>
-          using namespace GFLAGS_NAMESPACE;
-          int main() {}
+    # Test whether gflags library is installed
+    # http://gflags.github.io/gflags/
+    # check if the namespace is gflags
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+      #include <gflags/gflags.h>
+      using namespace gflags;
+      int main() {}
 EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+    else
+      # check if namespace is google
+      $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+        #include <gflags/gflags.h>
+        using namespace google;
+        int main() {}
+EOF
+      if [ "$?" = 0 ]; then
+          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
           PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        # check if namespace is gflags
-        elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-            #include <gflags/gflags.h>
-            using namespace gflags;
-            int main() {}
-EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=gflags"
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        # check if namespace is google
-        elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-            #include <gflags/gflags.h>
-            using namespace google;
-            int main() {}
-EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google"
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        fi
+      fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_ZLIB; then
-        # Test whether zlib library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <zlib.h>
-          int main() {}
+    # Test whether zlib library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <zlib.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DZLIB"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DZLIB"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
     fi
 
-    if ! test $ROCKSDB_DISABLE_BZIP; then
-        # Test whether bzip library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <bzlib.h>
-          int main() {}
+    # Test whether bzip library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <bzlib.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
     fi
 
-    if ! test $ROCKSDB_DISABLE_LZ4; then
-        # Test whether lz4 library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <lz4.h>
-          #include <lz4hc.h>
-          int main() {}
+    # Test whether lz4 library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <lz4.h>
+      #include <lz4hc.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
     fi
 
-    if ! test $ROCKSDB_DISABLE_ZSTD; then
-        # Test whether zstd library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <zstd.h>
-          int main() {}
+    # Test whether zstd library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <zstd.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
     fi
 
-    if ! test $ROCKSDB_DISABLE_NUMA; then
-        # Test whether numa is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
-          #include <numa.h>
-          #include <numaif.h>
-          int main() {}
+    # Test whether numa is available
+    $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
+      #include <numa.h>
+      #include <numaif.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
     fi
 
-    if ! test $ROCKSDB_DISABLE_TBB; then
-        # Test whether tbb is available
-        $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
-          #include <tbb/tbb.h>
-          int main() {}
+    # Test whether tbb is available
+    $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
+      #include <tbb/tbb.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DTBB"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DTBB"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
     fi
 
-    if ! test $ROCKSDB_DISABLE_JEMALLOC; then
-        # Test whether jemalloc is available
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -ljemalloc \
-          2>/dev/null; then
-            # This will enable some preprocessor identifiers in the Makefile
-            JEMALLOC=1
-            # JEMALLOC can be enabled either using the flag (like here) or by
-            # providing direct link to the jemalloc library
-            WITH_JEMALLOC_FLAG=1
-            # check for JEMALLOC installed with HomeBrew
-            if [ "$PLATFORM" == "OS_MACOSX" ]; then
-                if hash brew 2>/dev/null && brew ls --versions jemalloc > /dev/null; then
-                    JEMALLOC_VER=$(brew ls --versions jemalloc | tail -n 1 | cut -f 2 -d ' ')
-                    JEMALLOC_INCLUDE="-I/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/include"
-                    JEMALLOC_LIB="/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/lib/libjemalloc_pic.a"
-                    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $JEMALLOC_LIB"
-                    JAVA_STATIC_LDFLAGS="$JAVA_STATIC_LDFLAGS $JEMALLOC_LIB"
-                fi
-            fi
-        fi
-    fi
-    if ! test $JEMALLOC && ! test $ROCKSDB_DISABLE_TCMALLOC; then
+    # Test whether jemalloc is available
+    if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
+      2>/dev/null; then
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"
+        JEMALLOC=1
+    else
         # jemalloc is not available. Let's try tcmalloc
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null \
+        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
           -ltcmalloc 2>/dev/null; then
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
             JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_MALLOC_USABLE_SIZE; then
-        # Test whether malloc_usable_size is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <malloc.h>
-          int main() {
-            size_t res = malloc_usable_size(0);
-            (void)res;
-            return 0;
-          }
+    # Test whether malloc_usable_size is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <malloc.h>
+      int main() {
+        size_t res = malloc_usable_size(0);
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
     fi
 
-    if ! test $ROCKSDB_DISABLE_MEMKIND; then
-        # Test whether memkind library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -lmemkind -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <memkind.h>
-          int main() {
-            memkind_malloc(MEMKIND_DAX_KMEM, 1024);
-            return 0;
-          }
+    # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <pthread.h>
+      int main() {
+        int x = PTHREAD_MUTEX_ADAPTIVE_NP;
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DMEMKIND"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lmemkind"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lmemkind"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
     fi
 
-    if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
-        # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <pthread.h>
-          int main() {
-            int x = PTHREAD_MUTEX_ADAPTIVE_NP;
-            (void)x;
-            return 0;
-          }
+    # Test whether backtrace is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <execinfo.h>>
+      int main() {
+        void* frames[1];
+        backtrace_symbols(frames, backtrace(frames, 1));
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_BACKTRACE; then
-        # Test whether backtrace is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
+    else
+        # Test whether execinfo library is installed
+        $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <execinfo.h>
           int main() {
             void* frames[1];
             backtrace_symbols(frames, backtrace(frames, 1));
-            return 0;
           }
 EOF
         if [ "$?" = 0 ]; then
             COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-        else
-            # Test whether execinfo library is installed
-            $CXX $PLATFORM_CXXFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
-              #include <execinfo.h>
-              int main() {
-                void* frames[1];
-                backtrace_symbols(frames, backtrace(frames, 1));
-              }
-EOF
-            if [ "$?" = 0 ]; then
-                COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-                PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
-                JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
-            fi
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
+            JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_PG; then
-        # Test if -pg is supported
-        $CXX $PLATFORM_CXXFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          int main() {
-            return 0;
-          }
+    # Test if -pg is supported
+    $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      int main() {
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            PROFILING_FLAGS=-pg
-        fi
+    if [ "$?" = 0 ]; then
+        PROFILING_FLAGS=-pg
     fi
 
-    if ! test $ROCKSDB_DISABLE_SYNC_FILE_RANGE; then
-        # Test whether sync_file_range is supported for compatibility with an old glibc
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <fcntl.h>
-          int main() {
-            int fd = open("/dev/null", 0);
-            sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
-          }
+    # Test whether sync_file_range is supported for compatibility with an old glibc
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <fcntl.h>
+      int main() {
+        int fd = open("/dev/null", 0);
+        sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
     fi
 
-    if ! test $ROCKSDB_DISABLE_SCHED_GETCPU; then
-        # Test whether sched_getcpu is supported
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <sched.h>
-          int main() {
-            int cpuid = sched_getcpu();
-            (void)cpuid;
-          }
+    # Test whether sched_getcpu is supported
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <sched.h>
+      int main() {
+        int cpuid = sched_getcpu();
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_AUXV_GETAUXVAL; then
-        # Test whether getauxval is supported
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <sys/auxv.h>
-          int main() {
-            uint64_t auxv = getauxval(AT_HWCAP);
-            (void)auxv;
-          }
-EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_AUXV_GETAUXVAL_PRESENT"
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_ALIGNED_NEW; then
-        # Test whether c++17 aligned-new is supported
-        $CXX $PLATFORM_CXXFLAGS -faligned-new -x c++ - -o /dev/null 2>/dev/null <<EOF
-            struct alignas(1024) t {int a;};
-            int main() {}
-EOF
-        if [ "$?" = 0 ]; then
-            PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -faligned-new -DHAVE_ALIGNED_NEW"
-        fi
-    fi
-    if ! test $ROCKSDB_DISABLE_BENCHMARK; then
-        # Test whether google benchmark is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -lbenchmark 2>/dev/null  <<EOF
-          #include <benchmark/benchmark.h>
-          int main() {}
-EOF
-        if [ "$?" = 0 ]; then
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
     fi
 fi
 
@@ -609,7 +420,7 @@ fi
 # -Wshorten-64-to-32 breaks compilation on FreeBSD i386
 if ! [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
   # Test whether -Wshorten-64-to-32 is available
-  $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
+  $CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
     int main() {}
 EOF
   if [ "$?" = 0 ]; then
@@ -621,11 +432,11 @@ fi
 
 if test "$USE_HDFS"; then
   if test -z "$JAVA_HOME"; then
-    echo "JAVA_HOME has to be set for HDFS usage." >&2
+    echo "JAVA_HOME has to be set for HDFS usage."
     exit 1
   fi
-  HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS -I$HADOOP_HOME/include"
-  HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64 -L$HADOOP_HOME/lib/native"
+  HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS"
+  HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64"
   HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib"
   HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm"
   COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS"
@@ -633,170 +444,31 @@ if test "$USE_HDFS"; then
   JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
 fi
 
-if test "0$PORTABLE" -eq 0; then
+if test "$USE_SSE"; then
+  COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
+elif test -z "$PORTABLE"; then
   if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
     # Tune for this POWER processor, treating '+' models as base models
     POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
   elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
     COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep -e^arm -e^aarch64`"; then
-    # TODO: Handle this with approprite options.
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^aarch64`"; then
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif [ "$TARGET_OS" == "IOS" ]; then
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif [ "$TARGET_OS" == "AIX" ] || [ "$TARGET_OS" == "SunOS" ]; then
-    # TODO: Not sure why we don't use -march=native on these OSes
-    if test "$USE_SSE"; then
-      TRY_SSE_ETC="1"
-    fi
-  else
+  elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
   fi
-else
-  # PORTABLE=1
-  if test "$USE_SSE"; then
-    TRY_SSE_ETC="1"
-  fi
-
-  if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
-    # For portability compile for macOS 10.12 (2016) or newer
-    COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12"
-    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.12"
-    PLATFORM_SHARED_LDFLAGS="$PLATFORM_SHARED_LDFLAGS -mmacosx-version-min=10.12"
-    PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.12"
-    JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.12"
-    JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-    JAVA_STATIC_DEPS_CCFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-    JAVA_STATIC_DEPS_CXXFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-  fi
 fi
 
-if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
-  # check for GNU libc on ppc64
-  $CXX -x c++ - -o /dev/null 2>/dev/null <<EOF
-    #include <stdio.h>
-    #include <stdlib.h>
-    #include <gnu/libc-version.h>
-
-    int main(int argc, char *argv[]) {
-      printf("GNU libc version: %s\n", gnu_get_libc_version());
-      return 0;
-    }
-EOF
-  if [ "$?" != 0 ]; then
-      PPC_LIBC_IS_GNU=0
-  fi
-fi
-
-if test "$TRY_SSE_ETC"; then
-  # The USE_SSE flag now means "attempt to compile with widely-available
-  # Intel architecture extensions utilized by specific optimizations in the
-  # source code." It's a qualifier on PORTABLE=1 that means "mostly portable."
-  # It doesn't even really check that your current CPU is compatible.
-  #
-  # SSE4.2 available since nehalem, ca. 2008-2010
-  # Includes POPCNT for BitsSetToOne, BitParity
-  TRY_SSE42="-msse4.2"
-  # PCLMUL available since westmere, ca. 2010-2011
-  TRY_PCLMUL="-mpclmul"
-  # AVX2 available since haswell, ca. 2013-2015
-  TRY_AVX2="-mavx2"
-  # BMI available since haswell, ca. 2013-2015
-  # Primarily for TZCNT for CountTrailingZeroBits
-  TRY_BMI="-mbmi"
-  # LZCNT available since haswell, ca. 2013-2015
-  # For FloorLog2
-  TRY_LZCNT="-mlzcnt"
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_SSE42 -x c++ - -o /dev/null 2>/dev/null <<EOF
+$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
   #include <cstdint>
   #include <nmmintrin.h>
   int main() {
     volatile uint32_t x = _mm_crc32_u32(0, 0);
-    (void)x;
   }
 EOF
 if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_SSE42 -DHAVE_SSE42"
+  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
 elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_PCLMUL -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <wmmintrin.h>
-  int main() {
-    const auto a = _mm_set_epi64x(0, 0);
-    const auto b = _mm_set_epi64x(0, 0);
-    const auto c = _mm_clmulepi64_si128(a, b, 0x00);
-    auto d = _mm_cvtsi128_si64(c);
-    (void)d;
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_PCLMUL -DHAVE_PCLMUL"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use PCLMUL intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_AVX2 -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <immintrin.h>
-  int main() {
-    const auto a = _mm256_setr_epi32(0, 1, 2, 3, 4, 7, 6, 5);
-    const auto b = _mm256_permutevar8x32_epi32(a, a);
-    (void)b;
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_AVX2 -DHAVE_AVX2"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use AVX2 intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_BMI -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <immintrin.h>
-  int main(int argc, char *argv[]) {
-    (void)argv;
-    return (int)_tzcnt_u64((uint64_t)argc);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_BMI -DHAVE_BMI"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use BMI intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_LZCNT -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <immintrin.h>
-  int main(int argc, char *argv[]) {
-    (void)argv;
-    return (int)_lzcnt_u64((uint64_t)argc);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_LZCNT -DHAVE_LZCNT"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use LZCNT intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  int main() {
-    uint64_t a = 0xffffFFFFffffFFFF;
-    __uint128_t b = __uint128_t(a) * a;
-    a = static_cast<uint64_t>(b >> 64);
-    (void)a;
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_UINT128_EXTENSION"
+  echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
 fi
 
 # iOS doesn't support thread-local storage, but this check would erroneously
@@ -809,7 +481,6 @@ if [ "$PLATFORM" != IOS ]; then
   #endif
   int main() {
     static __thread int tls;
-    (void)tls;
   }
 EOF
   if [ "$?" = 0 ]; then
@@ -817,20 +488,6 @@ EOF
   fi
 fi
 
-
-if [ "$FBCODE_BUILD" != "true" -a "$PLATFORM" = OS_LINUX ]; then
-  $CXX $COMMON_FLAGS $PLATFORM_SHARED_CFLAGS -x c++ -c - -o test_dl.o 2>/dev/null <<EOF
-  void dummy_func() {}
-EOF
-  if [ "$?" = 0 ]; then
-    $CXX $COMMON_FLAGS $PLATFORM_SHARED_LDFLAGS test_dl.o -o /dev/null 2>/dev/null
-    if [ "$?" = 0 ]; then
-      EXEC_LDFLAGS+="-ldl"
-      rm -f test_dl.o
-    fi
-  fi
-fi
-
 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
 
@@ -842,16 +499,10 @@ ROCKSDB_PATCH=`build_tools/version.sh patch`
 
 echo "CC=$CC" >> "$OUTPUT"
 echo "CXX=$CXX" >> "$OUTPUT"
-echo "AR=$AR" >> "$OUTPUT"
 echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
 echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
-echo "PLATFORM_CMAKE_FLAGS=$PLATFORM_CMAKE_FLAGS" >> "$OUTPUT"
 echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
 echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_CCFLAGS=$JAVA_STATIC_DEPS_CCFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_CXXFLAGS=$JAVA_STATIC_DEPS_CXXFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_LDFLAGS=$JAVA_STATIC_DEPS_LDFLAGS" >> "$OUTPUT"
-echo "JAVAC_ARGS=$JAVAC_ARGS" >> "$OUTPUT"
 echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
 echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
 echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"
@@ -868,22 +519,7 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
 echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
 echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
-echo "FIND=$FIND" >> "$OUTPUT"
-echo "WATCH=$WATCH" >> "$OUTPUT"
-# This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
   echo "JEMALLOC=1" >> "$OUTPUT"
 fi
-# Indicates that jemalloc should be enabled using -ljemalloc flag
-# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
-# and JEMALLOC_INCLUDE
-if test -n "$WITH_JEMALLOC_FLAG"; then
-  echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
-fi
 echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
-if test -n "$USE_FOLLY_DISTRIBUTED_MUTEX"; then
-  echo "USE_FOLLY_DISTRIBUTED_MUTEX=$USE_FOLLY_DISTRIBUTED_MUTEX" >> "$OUTPUT"
-fi
-if test -n "$PPC_LIBC_IS_GNU"; then
-  echo "PPC_LIBC_IS_GNU=$PPC_LIBC_IS_GNU" >> "$OUTPUT"
-fi

build_tools/cont_integration.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+#
+# Copyright (c) 2016, Facebook. All rights reserved.
+#
+# Overall wrapper script for RocksDB continuous builds. The implementation is a
+# trivial pulling scheme. We loop infinitely, check if any new changes have been
+# committed, if yes then trigger a Sandcastle run, and finally go to sleep again
+# for a certain interval.
+#
+
+SRC_GIT_REPO=/data/git/rocksdb-public
+error=0
+
+function log {
+  DATE=`date +%Y-%m-%d:%H:%M:%S`
+  echo $DATE $@
+}
+
+function log_err {
+  log "ERROR: $@ Error code: $error."
+}
+
+function update_repo_status {
+  # Update the parent first.
+  pushd $SRC_GIT_REPO
+
+  # This is a fatal error. Something in the environment isn't right and we will
+  # terminate the execution.
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "Where is $SRC_GIT_REPO?"
+    exit $error
+  fi
+
+  HTTPS_PROXY=fwdproxy:8080 git fetch -f
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git fetch -f failed."
+    popd
+    return $error
+  fi
+
+  git update-ref refs/heads/master refs/remotes/origin/master
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git update-ref failed."
+    popd
+    return $error
+  fi
+
+  popd
+
+  # We're back in an instance-specific directory. Get the latest changes.
+  git pull --rebase
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git pull --rebase failed."
+    return $error
+  fi
+}
+
+#
+# Execution starts here.
+#
+
+# Path to the determinator from the root of the RocksDB repo.
+CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonHelper.php
+
+# Value of the previous commit.
+PREV_COMMIT=
+
+log "Starting to monitor for new RocksDB changes ..."
+log "Running under `pwd` as `whoami`."
+
+# Paranoia. Make sure that we're using the right branch.
+git checkout master
+
+error=$?
+if [ ! $error -eq 0 ]; then
+  log_err "This is not good. Can't checkout master. Bye-bye!"
+  exit 1
+fi
+
+# We'll run forever and let the execution environment terminate us if we'll
+# exceed whatever timeout is set for the job.
+while true;
+do
+  # Get the latest changes committed.
+  update_repo_status
+
+  error=$?
+  if [  $error -eq 0 ]; then
+    LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'`
+
+    log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'."
+
+    if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then
+      log "There were no changes since the last time I checked. Going to sleep."
+    else
+      if [ ! -z "$LAST_COMMIT" ]; then
+        log "New code has been committed or previous commit not known. " \
+            "Will trigger the tests."
+
+        PREV_COMMIT=$LAST_COMMIT
+        log "Updated previous commit to '$PREV_COMMIT'."
+
+        #
+        # This is where we'll trigger the Sandcastle run. The values for
+        # HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're
+        # running in.
+        #
+        POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR
+
+        error=$?
+        if [ $error -eq 0 ]; then
+          log "Sandcastle run successfully triggered."
+        else
+          log_err "Failed to trigger Sandcastle run."
+        fi
+      else
+        log_err "Previous commit not updated. Don't know what the last one is."
+      fi
+    fi
+  else
+    log_err "Getting latest changes failed. Will skip running tests for now."
+  fi
+
+  # Always sleep, even if errors happens while trying to determine the latest
+  # commit. This will prevent us terminating in case of transient errors.
+  log "Will go to sleep for 5 minutes."
+  sleep 5m
+done

build_tools/dependencies.sh

@@ -1,19 +1,18 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/5.x/centos7-native/c447969
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/1bd23f9917738974ad0ff305aa23eb5f93f18305/9.0.0/centos7-native/c9f9104
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/5.x/gcc-5-glibc-2.23/339d858
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.23/gcc-5-glibc-2.23/ca1d1c0
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/gcc-5-glibc-2.23/9bc6787
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/gcc-5-glibc-2.23/9bc6787
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/gcc-5-glibc-2.23/9bc6787
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/gcc-5-glibc-2.23/9bc6787
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/gcc-5-glibc-2.23/03859b5
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/gcc-5-glibc-2.23/9bc6787
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/gcc-5-glibc-2.23/0c8f76d
-NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/gcc-5-glibc-2.23/9bc6787
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/gcc-5-glibc-2.23/b443de1
-TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/gcc-5-glibc-2.23/9bc6787
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/2e3cb7d119b3cea5f1e738cc13a1ac69f49eb875/2.29.1/centos7-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/gcc-5-glibc-2.23/9bc6787
-LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.2.3/gcc-5-glibc-2.23/65372bd
+GCC_BASE=/mnt/gvfs/third-party2/gcc/2928bb3ed95bf64f5b388ee88c30dc74710c3b35/5.x/centos6-native/f4950a1
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/a5fea028cb7ba43498976e1f8054b0b2e790c295/stable/centos6-native/6aaf4de
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/7a9099f6587ee4378c0b1fa32bb8934019d30ca4/5.x/gcc-5-glibc-2.23/339d858
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/3b7c6469854dfc7832a1c3cc5b86919a84e5f865/2.23/gcc-5-glibc-2.23/ca1d1c0
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-5-glibc-2.23/9bc6787
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/d7861abe6f0e27ab98c9303b95a662f0e4cdedb5/1.2.8/gcc-5-glibc-2.23/9bc6787
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-5-glibc-2.23/9bc6787
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0815d59804160c96caac5f27ca004f51af893dc6/r131/gcc-5-glibc-2.23/9bc6787
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/c15a4f5f619a2930478d01e2e34dc1e0652b0873/1.1.4/gcc-5-glibc-2.23/03859b5
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f905a5e1032fb30c05db3d3752319857388c0c49/2.2.0/gcc-5-glibc-2.23/9bc6787
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/8d60633d822a2a55849c73db24e74a25e52b71db/master/gcc-5-glibc-2.23/1c32b4b
+NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-5-glibc-2.23/9bc6787
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/8db74270cd6d0212ac92d69e7fc7beefe617d772/trunk/gcc-5-glibc-2.23/b1847cb
+TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-5-glibc-2.23/9bc6787
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/90c9734afc5579c9d1db529fa788d09f97763b85/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/9e829389ef61b92c62de8748c80169aaf25ce1f0/2.26.1/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.11.0/gcc-5-glibc-2.23/9bc6787
+LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/gcc-5-glibc-2.23/65372bd

build_tools/dependencies_4.8.1.sh

@@ -1,5 +1,3 @@
-# shellcheck disable=SC2148
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc

build_tools/dependencies_platform007.sh

@@ -1,20 +1,18 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
+GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
 SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
 BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
 GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
-NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
-TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
-LIBURING_BASE=/mnt/gvfs/third-party2/liburing/79427253fd0d42677255aacfe6d13bfe63f752eb/20190828/platform007/ca4da3d
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
+NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
 LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832

build_tools/dependencies_platform009.sh

@@ -1,20 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/1795efe5f06778c15a92c8f9a2aba5dc496d9d4d/9.x/centos7-native/3bed279
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/7318eaac22659b6ff2fe43918e4b69fd0772a8a7/9.0.0/platform009/651ee30
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/4959b39cfbe5965a37c861c4c327fa7c5c759b87/9.x/platform009/9202ce7
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/45ce3375cdc77ecb2520bbf8f0ecddd3f98efd7a/2.30/platform009/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/be4de3205e029101b18aa8103daa696c2bef3b19/1.1.3/platform009/7f3b187
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/3c160ac5c67e257501e24c6c1d00ad5e01d73db6/1.2.8/platform009/7f3b187
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/73a237ac5bc0a5f5d67b39b8d253cfebaab88684/1.0.6/platform009/7f3b187
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/ec6573523b0ce55ef6373a4801189027cf07bb2c/1.9.1/platform009/7f3b187
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/64c58a207d2495e83abc57a500a956df09b79a7c/1.4.x/platform009/ba86d1f
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/824d0a8a5abb5b121afd1b35fc3896407ea50092/2.2.0/platform009/7f3b187
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9aef9feb850b168a68736420f217b01cce11a89/master/platform009/c305944
-NUMA_BASE=/mnt/gvfs/third-party2/numa/0af65f71e23a67bf65dc91b11f95caa39325c432/2.0.11/platform009/7f3b187
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/02486dac347645d31dce116f44e1de3177315be2/1.4/platform009/5191652
-TBB_BASE=/mnt/gvfs/third-party2/tbb/2e0ec671e550bfca347300bf3f789d9c0fff24ad/2018_U5/platform009/7f3b187
-LIBURING_BASE=/mnt/gvfs/third-party2/liburing/70dbd9cfee63a25611417d09433a86d7711b3990/20200729/platform009/7f3b187
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/32b8a2407b634df3f8f948ba373fc4acc6a18296/fb/platform009/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/08634589372fa5f237bfd374e8c644a8364e78c1/2.32/platform009/ba86d1f/
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/6ae525939ad02e5e676855082fbbc7828dbafeac/3.15.0/platform009/7f3b187
-LUA_BASE=/mnt/gvfs/third-party2/lua/162efd9561a3d21f6869f4814011e9cf1b3ff4dc/5.3.4/platform009/a6271c4

build_tools/dockerbuild.sh

@@ -1,3 +1,2 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 docker run -v $PWD:/rocks -w /rocks buildpack-deps make

build_tools/error_filter.py

@@ -64,12 +64,8 @@ class MatchErrorParser(ErrorParserBase):
 
 class CompilerErrorParser(MatchErrorParser):
     def __init__(self):
-        # format (compile error):
-        #   '<filename>:<line #>:<column #>: error: <error msg>'
-        # format (link error):
-        #   '<filename>:<line #>: error: <error msg>'
-        # The below regex catches both
-        super(CompilerErrorParser, self).__init__(r'\S+:\d+: error:')
+        # format: '<filename>:<line #>:<column #>: error: <error msg>'
+        super(CompilerErrorParser, self).__init__(r'\S+:\d+:\d+: error:')
 
 
 class ScanBuildErrorParser(MatchErrorParser):
@@ -132,17 +128,11 @@ _TEST_NAME_TO_PARSERS = {
     'lite': [CompilerErrorParser],
     'lite_test': [CompilerErrorParser, GTestErrorParser],
     'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
-    'stress_crash_with_atomic_flush': [CompilerErrorParser, DbCrashErrorParser],
-    'stress_crash_with_txn': [CompilerErrorParser, DbCrashErrorParser],
     'write_stress': [CompilerErrorParser, WriteStressErrorParser],
     'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
     'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
-    'asan_crash_with_atomic_flush': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
-    'asan_crash_with_txn': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
     'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
     'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
-    'ubsan_crash_with_atomic_flush': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
-    'ubsan_crash_with_txn': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
     'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
     'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
     'format_compatible': [CompilerErrorParser, CompatErrorParser],

build_tools/fb_compile_mongo.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+
 # fail early
 set -e
 

build_tools/fbcode_config.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 #
 # Set environment variables so that we can compile rocksdb using
 # fbcode settings.  It uses the latest g++ and clang compilers and also
@@ -44,15 +43,11 @@ if test -z $PIC_BUILD; then
   LZ4_INCLUDE=" -I $LZ4_BASE/include/"
   LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
   CFLAGS+=" -DLZ4"
-fi
 
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
+  ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
   ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+  CFLAGS+=" -DZSTD"
 fi
-CFLAGS+=" -DZSTD -DZSTD_STATIC_LINKING_ONLY"
 
 # location of gflags headers and libraries
 GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
@@ -86,10 +81,8 @@ else
 fi
 CFLAGS+=" -DTBB"
 
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
 
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
@@ -109,7 +102,6 @@ if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS+=" -B$BINUTILS/gold"
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
@@ -120,7 +112,6 @@ else
   CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
@@ -162,6 +153,4 @@ else
   LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
 fi
 
-USE_FOLLY_DISTRIBUTED_MUTEX=1
-
 export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/fbcode_config4.8.1.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 #
 # Set environment variables so that we can compile rocksdb using
 # fbcode settings.  It uses the latest g++ compiler and also
@@ -53,10 +52,8 @@ LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
 TBB_INCLUDE=" -isystem $TBB_BASE/include/"
 TBB_LIBS="$TBB_BASE/lib/libtbb.a"
 
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
 
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
@@ -69,7 +66,6 @@ if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  CXX="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
@@ -82,7 +78,6 @@ else
   CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
 

build_tools/fbcode_config_platform007.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 #
 # Set environment variables so that we can compile rocksdb using
 # fbcode settings.  It uses the latest g++ and clang compilers and also
@@ -86,24 +85,14 @@ else
 fi
 CFLAGS+=" -DTBB"
 
-# location of LIBURING
-LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
-if test -z $PIC_BUILD; then
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
-else
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
-fi
-CFLAGS+=" -DLIBURING"
-
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
+export PORTABLE=1
 
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
 
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
 
 STDLIBS="-L $GCC_BASE/lib64"
 
@@ -118,7 +107,6 @@ if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS+=" -B$BINUTILS/gold"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
@@ -129,7 +117,6 @@ else
   CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
@@ -146,10 +133,10 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"
 
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
 EXEC_LDFLAGS+=" -B$BINUTILS/gold"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
@@ -159,7 +146,7 @@ EXEC_LDFLAGS+=" -ldl"
 
 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
 
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
 
 VALGRIND_VER="$VALGRIND_BASE/bin/"
 

build_tools/fbcode_config_platform009.sh

@@ -1,171 +0,0 @@
-#!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform009.sh"
-
-CFLAGS=""
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/9.3.0"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-
-# snappy
-SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-else
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-fi
-CFLAGS+=" -DSNAPPY"
-
-if test -z $PIC_BUILD; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-  CFLAGS+=" -DZLIB"
-
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-fi
-CFLAGS+=" -DZSTD"
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
-else
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
-fi
-CFLAGS+=" -DGFLAGS=gflags"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
-
-if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
-  # location of libunwind
-  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
-fi
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-if test -z $PIC_BUILD; then
-  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-else
-  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
-fi
-CFLAGS+=" -DTBB"
-
-# location of LIBURING
-LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
-if test -z $PIC_BUILD; then
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
-else
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
-fi
-CFLAGS+=" -DLIBURING"
-
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/llvm/clang/tools/scan-build/bin/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
-
-  CFLAGS+=" -B$BINUTILS"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
-
-  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-
-  CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform009/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform009/lib"
-EXEC_LDFLAGS+=" -Wl,-rpath=$GCC_BASE/lib64"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-# lua not supported because it's on track for deprecation, I think
-LUA_PATH=
-LUA_LIB=
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/format-diff.sh

@@ -1,100 +1,36 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 # If clang_format_diff.py command is not specfied, we assume we are able to
 # access directly without any path.
+if [ -z $CLANG_FORMAT_DIFF ]
+then
+CLANG_FORMAT_DIFF="clang-format-diff.py"
+fi
 
-print_usage () {
-  echo "Usage:"
-  echo "format-diff.sh [OPTIONS]"
-  echo "-c: check only."
-  echo "-h: print this message."
-}
+# Check clang-format-diff.py
+if ! which $CLANG_FORMAT_DIFF &> /dev/null
+then
+  echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
+  echo "You can download clang-format-diff.py by running: "
+  echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
+  echo "You can download clang-format by running: "
+  echo "    brew install clang-format"
+  echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+  exit 128
+fi
 
-while getopts ':ch' OPTION; do
-  case "$OPTION" in
-    c)
-      CHECK_ONLY=1
-      ;;
-    h)
-      print_usage
-      exit 1
-      ;;
-    ?)
-      print_usage
-      exit 1
-      ;;
-  esac
-done
+# Check argparse, a library that clang-format-diff.py requires.
+python 2>/dev/null << EOF
+import argparse
+EOF
 
-REPO_ROOT="$(git rev-parse --show-toplevel)"
-
-if [ "$CLANG_FORMAT_DIFF" ]; then
-  echo "Note: CLANG_FORMAT_DIFF='$CLANG_FORMAT_DIFF'"
-  # Dry run to confirm dependencies like argparse
-  if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-    true #Good
-  else
-    exit 128
-  fi
-else
-  # First try directly executing the possibilities
-  if clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=clang-format-diff.py
-  elif $REPO_ROOT/clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=$REPO_ROOT/clang-format-diff.py
-  else
-    # This probably means we need to directly invoke the interpreter.
-    # But first find clang-format-diff.py
-    if [ -f "$REPO_ROOT/clang-format-diff.py" ]; then
-      CFD_PATH="$REPO_ROOT/clang-format-diff.py"
-    elif which clang-format-diff.py &> /dev/null; then
-      CFD_PATH="$(which clang-format-diff.py)"
-    else
-      echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
-      echo "You can download clang-format-diff.py by running: "
-      echo "    curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py"
-      echo "You should make sure the downloaded script is not compromised."
-      echo "You can download clang-format by running:"
-      echo "    brew install clang-format"
-      echo "  Or"
-      echo "    apt install clang-format"
-      echo "  This might work too:"
-      echo "    yum install git-clang-format"
-      echo "Then make sure clang-format is available and executable from \$PATH:"
-      echo "    clang-format --version"
-      exit 128
-    fi
-    # Check argparse pre-req on interpreter, or it will fail
-    if echo import argparse | ${PYTHON:-python3}; then
-      true # Good
-    else
-      echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
-      echo "installed. You can try either of the follow ways to install it:"
-      echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
-      echo "  2. easy_install argparse (if you have easy_install)"
-      echo "  3. pip install argparse (if you have pip)"
-      exit 129
-    fi
-    # Unfortunately, some machines have a Python2 clang-format-diff.py
-    # installed but only a Python3 interpreter installed. Unfortunately,
-    # automatic 2to3 migration is insufficient, so suggest downloading latest.
-    if grep -q "print '" "$CFD_PATH" && \
-       ${PYTHON:-python3} --version | grep -q 'ython 3'; then
-      echo "You have clang-format-diff.py for Python 2 but are using a Python 3"
-      echo "interpreter (${PYTHON:-python3})."
-      echo "You can download clang-format-diff.py for Python 3 by running: "
-      echo "    curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py"
-      echo "You should make sure the downloaded script is not compromised."
-      exit 130
-    fi
-    CLANG_FORMAT_DIFF="${PYTHON:-python3} $CFD_PATH"
-    # This had better work after all those checks
-    if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-      true #Good
-    else
-      exit 128
-    fi
-  fi
+if [ "$?" != 0 ]
+then
+  echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
+  echo "installed. You can try either of the follow ways to install it:"
+  echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
+  echo "  2. easy_install argparse (if you have easy_install)"
+  echo "  3. pip install argparse (if you have pip)"
+  exit 129
 fi
 
 # TODO(kailiu) following work is not complete since we still need to figure
@@ -117,54 +53,34 @@ fi
 set -e
 
 uncommitted_code=`git diff HEAD`
+LAST_MASTER=`git merge-base master HEAD`
 
 # If there's no uncommitted changes, we assume user are doing post-commit
-# format check, in which case we'll try to check the modified lines vs. the
-# facebook/rocksdb.git master branch. Otherwise, we'll check format of the
-# uncommitted code only.
+# format check, in which case we'll check the modified lines since last commit
+# from master. Otherwise, we'll check format of the uncommitted code only.
 if [ -z "$uncommitted_code" ]
 then
-  # Attempt to get name of facebook/rocksdb.git remote.
-  [ "$FORMAT_REMOTE" ] || FORMAT_REMOTE="$(git remote -v | grep 'facebook/rocksdb.git' | head -n 1 | cut -f 1)"
-  # Fall back on 'origin' if that fails
-  [ "$FORMAT_REMOTE" ] || FORMAT_REMOTE=origin
-  # Use master branch from that remote
-  [ "$FORMAT_UPSTREAM" ] || FORMAT_UPSTREAM="$FORMAT_REMOTE/master"
-  # Get the common ancestor with that remote branch. Everything after that
-  # common ancestor would be considered the contents of a pull request, so
-  # should be relevant for formatting fixes.
-  FORMAT_UPSTREAM_MERGE_BASE="$(git merge-base "$FORMAT_UPSTREAM" HEAD)"
-  # Get the differences
-  diffs=$(git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -p 1)
-  echo "Checking format of changes not yet in $FORMAT_UPSTREAM..."
+  # Check the format of last commit
+  diffs=$(git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -p 1)
 else
   # Check the format of uncommitted lines,
   diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
-  echo "Checking format of uncommitted changes..."
 fi
 
 if [ -z "$diffs" ]
 then
   echo "Nothing needs to be reformatted!"
   exit 0
-elif [ $CHECK_ONLY ]
-then
-  echo "Your change has unformatted code. Please run make format!"
-  if [ $VERBOSE_CHECK ]; then
-    clang-format --version
-    echo "$diffs"
-  fi
-  exit 1
 fi
 
 # Highlight the insertion/deletion from the clang-format-diff.py's output
 COLOR_END="\033[0m"
-COLOR_RED="\033[0;31m"
-COLOR_GREEN="\033[0;32m"
+COLOR_RED="\033[0;31m" 
+COLOR_GREEN="\033[0;32m" 
 
 echo -e "Detect lines that doesn't follow the format rules:\r"
 # Add the color to the diff. lines added will be green; lines removed will be red.
-echo "$diffs" |
+echo "$diffs" | 
   sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" |
   sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/"
 
@@ -187,9 +103,9 @@ fi
 # Do in-place format adjustment.
 if [ -z "$uncommitted_code" ]
 then
-  git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -i -p 1
+  git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -i -p 1
 else
-  git diff -U0 HEAD | $CLANG_FORMAT_DIFF -i -p 1
+  git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
 fi
 echo "Files reformatted!"
 

build_tools/gnu_parallel

@@ -5082,8 +5082,8 @@ sub openoutputfiles {
 	# Set reading FD if using --group (--ungroup does not need)
 	for my $fdno (1,2) {
 	    # Re-open the file for reading
-	    # so fdw can be closed separately
-	    # and fdr can be seeked separately (for --line-buffer)
+	    # so fdw can be closed seperately
+	    # and fdr can be seeked seperately (for --line-buffer)
 	    open(my $fdr,"<", $self->fh($fdno,'name')) ||
 		::die_bug("fdr: Cannot open ".$self->fh($fdno,'name'));
 	    $self->set_fh($fdno,'r',$fdr);
@@ -5801,7 +5801,7 @@ sub workdir {
 		    . "-" . $self->seq();
 	    } else {
 		$workdir = $opt::workdir;
-		# Rsync treats /./ special. We don't want that
+		# Rsync treats /./ special. We dont want that
 		$workdir =~ s:/\./:/:g; # Remove /./
 		$workdir =~ s:/+$::; # Remove ending / if any
 		$workdir =~ s:^\./::g; # Remove starting ./ if any

build_tools/make_package.sh

@@ -1,6 +1,4 @@
-# shellcheck disable=SC1113
 #/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
 set -e
 
@@ -30,14 +28,12 @@ function package() {
     if dpkg --get-selections | grep --quiet $1; then
       log "$1 is already installed. skipping."
     else
-      # shellcheck disable=SC2068
       apt-get install $@ -y
     fi
   elif [[ $OS = "centos" ]]; then
     if rpm -qa | grep --quiet $1; then
       log "$1 is already installed. skipping."
     else
-      # shellcheck disable=SC2068
       yum install $@ -y
     fi
   fi
@@ -56,7 +52,6 @@ function gem_install() {
   if gem list | grep --quiet $1; then
     log "$1 is already installed. skipping."
   else
-    # shellcheck disable=SC2068
     gem install $@
   fi
 }
@@ -103,27 +98,31 @@ function main() {
   gem_install fpm
 
   make static_lib
-  LIBDIR=/usr/lib
-  if [[ $FPM_OUTPUT = "rpm" ]]; then
-      LIBDIR=$(rpm --eval '%_libdir')
-  fi
+  make install INSTALL_PATH=package
 
-  rm -rf package
-  make install DESTDIR=package PREFIX=/usr LIBDIR=$LIBDIR
+  cd package
+
+  LIB_DIR=lib
+  if [[ -z "$ARCH" ]]; then
+      ARCH=$(getconf LONG_BIT)
+  fi
+  if [[ ("$FPM_OUTPUT" = "rpm") && ($ARCH -eq 64) ]]; then
+      mv lib lib64
+      LIB_DIR=lib64
+  fi
 
   fpm \
     -s dir \
     -t $FPM_OUTPUT \
-    -C package \
     -n rocksdb \
     -v $1 \
+    --prefix /usr \
     --url http://rocksdb.org/ \
     -m rocksdb@fb.com \
     --license BSD \
     --vendor Facebook \
     --description "RocksDB is an embeddable persistent key-value store for fast storage." \
-    usr
+    include $LIB_DIR
 }
 
-# shellcheck disable=SC2068
 main $@

build_tools/precommit_checker.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2.7
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/usr/local/fbcode/gcc-4.9-glibc-2.20-fb/bin/python2.7
 
 from __future__ import absolute_import
 from __future__ import division

build_tools/regression_build_test.sh

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 
 set -e
 
@@ -378,7 +377,7 @@ function send_to_ods {
     echo >&2 "ERROR: Key $key doesn't have a value."
     return
   fi
-  curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \
+  curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \
     --connect-timeout 60
 }
 

build_tools/run_ci_db_test.ps1

@@ -1,17 +1,13 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 # This script enables you running RocksDB tests by running
 # All the tests concurrently and utilizing all the cores
 Param(
-  [switch]$EnableJE = $false,  # Look for and use test executable, append _je to listed exclusions
+  [switch]$EnableJE = $false,  # Look for and use _je executable, append _je to listed exclusions
   [switch]$RunAll = $false,    # Will attempt discover all *_test[_je].exe binaries and run all
                                # of them as Google suites. I.e. It will run test cases concurrently
                                # except those mentioned as $Run, those will run as individual test cases
                                # And any execlued with $ExcludeExes or $ExcludeCases
                                # It will also not run any individual test cases
                                # excluded but $ExcludeCasese
-  [switch]$RunAllExe = $false, # Look for and use test exdcutables, append _je to exclusions automatically
-                               # It will attempt to run them in parallel w/o breaking them up on individual
-                               # test cases. Those listed with $ExcludeExes will be excluded
   [string]$SuiteRun = "",      # Split test suites in test cases and run in parallel, not compatible with $RunAll
   [string]$Run = "",           # Run specified executables in parallel but do not split to test cases
   [string]$ExcludeCases = "",  # Exclude test cases, expects a comma separated list, no spaces
@@ -43,18 +39,13 @@ $RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
 $RunOnly.Add("merge_test") | Out-Null
 $RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
 $RunOnly.Add("backupable_db_test") | Out-Null # Disabled
-$RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest
+
 
 if($RunAll -and $SuiteRun -ne "") {
     Write-Error "$RunAll and $SuiteRun are not compatible"
     exit 1
 }
 
-if($RunAllExe -and $Run -ne "") {
-    Write-Error "$RunAllExe and $Run are not compatible"
-    exit 1
-}
-
 # If running under Appveyor assume that root
 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
 if($Appveyor -ne "") {
@@ -68,7 +59,7 @@ $BinariesFolder = -Join($RootFolder, "\build\Debug\")
 
 if($WorkFolder -eq "") {
 
-    # If TEST_TMPDIR is set use it
+    # If TEST_TMPDIR is set use it    
     [string]$var = $Env:TEST_TMPDIR
     if($var -eq "") {
         $WorkFolder = -Join($RootFolder, "\db_tests\")
@@ -93,7 +84,7 @@ $ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
 if($ExcludeCases -ne "") {
     Write-Host "ExcludeCases: $ExcludeCases"
     $l = $ExcludeCases -split ' '
-    ForEach($t in $l) {
+    ForEach($t in $l) { 
       $ExcludeCasesSet.Add($t) | Out-Null
     }
 }
@@ -102,7 +93,7 @@ $ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
 if($ExcludeExes -ne "") {
     Write-Host "ExcludeExe: $ExcludeExes"
     $l = $ExcludeExes -split ' '
-    ForEach($t in $l) {
+    ForEach($t in $l) { 
       $ExcludeExesSet.Add($t) | Out-Null
     }
 }
@@ -118,10 +109,6 @@ if($ExcludeExes -ne "") {
 #   MultiThreaded/MultiThreadedDBTest.
 #     MultiThreaded/0  # GetParam() = 0
 #     MultiThreaded/1  # GetParam() = 1
-#   RibbonTypeParamTest/0.  # TypeParam = struct DefaultTypesAndSettings
-#     CompactnessAndBacktrackAndFpRate
-#     Extremes
-#     FindOccupancyForSuccessRate
 #
 # into this:
 #
@@ -129,9 +116,6 @@ if($ExcludeExes -ne "") {
 #   DBTest.WriteEmptyBatch
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
-#   RibbonTypeParamTest/0.CompactnessAndBacktrackAndFpRate
-#   RibbonTypeParamTest/0.Extremes
-#   RibbonTypeParamTest/0.FindOccupancyForSuccessRate
 #
 # Output into the parameter in a form TestName -> Log File Name
 function ExtractTestCases([string]$GTestExe, $HashTable) {
@@ -145,17 +129,20 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
 
     ForEach( $l in $Tests) {
 
-      # remove trailing comment if any
-      $l = $l -replace '\s+\#.*',''
       # Leading whitespace is fine
       $l = $l -replace '^\s+',''
+      # but no whitespace any other place
+      if($l -match "\s+") {
+        continue
+      }
       # Trailing dot is a test group but no whitespace
-      if ($l -match "\.$" -and $l -notmatch "\s+") {
+      elseif ( $l -match "\.$" ) {
         $Group = $l
       }  else {
         # Otherwise it is a test name, remove leading space
         $test = $l
-        # create a log name
+        # remove trailing comment if any and create a log name
+        $test = $test -replace '\s+\#.*',''
         $test = "$Group$test"
 
         if($ExcludeCasesSet.Contains($test)) {
@@ -236,11 +223,13 @@ $TestExes = [ordered]@{}
 if($Run -ne "") {
 
   $test_list = $Run -split ' '
+
   ForEach($t in $test_list) {
 
     if($EnableJE) {
       $t += "_je"
     }
+
     MakeAndAdd -token $t -HashTable $TestExes
   }
 
@@ -248,38 +237,6 @@ if($Run -ne "") {
      Write-Error "Failed to extract tests from $Run"
      exit 1
   }
-} elseif($RunAllExe) {
-  # Discover all the test binaries
-  if($EnableJE) {
-    $pattern = "*_test_je.exe"
-  } else {
-    $pattern = "*_test.exe"
-  }
-
-  $search_path = -join ($BinariesFolder, $pattern)
-  Write-Host "Binaries Search Path: $search_path"
-
-  $DiscoveredExe = @()
-  dir -Path $search_path | ForEach-Object {
-     $DiscoveredExe += ($_.Name)
-  }
-
-  # Remove exclusions
-  ForEach($e in $DiscoveredExe) {
-    $e = $e -replace '.exe$', ''
-    $bare_name = $e -replace '_je$', ''
-
-    if($ExcludeExesSet.Contains($bare_name)) {
-      Write-Warning "Test $e is excluded"
-      continue
-    }
-    MakeAndAdd -token $e -HashTable $TestExes
-  }
-
-  if($TestExes.Count -lt 1) {
-     Write-Error "Failed to discover test executables"
-     exit 1
-  }
 }
 
 # Ordered by exe @{ Exe = @{ TestCase = LogName }}
@@ -288,7 +245,9 @@ $CasesToRun = [ordered]@{}
 if($SuiteRun -ne "") {
   $suite_list = $SuiteRun -split ' '
   ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
-} elseif ($RunAll) {
+}
+
+if($RunAll) {
 # Discover all the test binaries
   if($EnableJE) {
     $pattern = "*_test_je.exe"
@@ -296,12 +255,13 @@ if($SuiteRun -ne "") {
     $pattern = "*_test.exe"
   }
 
+
   $search_path = -join ($BinariesFolder, $pattern)
   Write-Host "Binaries Search Path: $search_path"
 
   $ListOfExe = @()
   dir -Path $search_path | ForEach-Object {
-     $ListOfExe += ($_.Name)
+     $ListOfExe += ($_.Name)     
   }
 
   # Exclude those in RunOnly from running as suites
@@ -327,6 +287,8 @@ if($SuiteRun -ne "") {
 }
 
 
+Write-Host "Attempting to start: $NumTestsToStart tests"
+
 # Invoke a test with a filter and redirect all output
 $InvokeTestCase = {
     param($exe, $test, $log);
@@ -345,7 +307,7 @@ $InvokeTestAsync = {
 # Test limiting factor here
 [int]$count = 0
 # Overall status
-[bool]$script:success = $true;
+[bool]$success = $true;
 
 function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 {
@@ -356,7 +318,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 
     # Wait for all to finish and get the results
     while(($JobToLog.Count -gt 0) -or
-          ($TestCmds.Count -gt 0) -or
+          ($TestCmds.Count -gt 0) -or 
            ($Suites.Count -gt 0)) {
 
         # Make sure we have maximum concurrent jobs running if anything
@@ -403,7 +365,6 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
                  break
                }
 
-              Write-Host "Starting $exe_name"
               [string]$Exe =  -Join ($BinariesFolder, $exe_name)
               $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
               $JobToLog.Add($job, $log_path)
@@ -434,7 +395,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
         $log_content = @(Get-Content $log)
 
         if($completed.State -ne "Completed") {
-            $script:success = $false
+            $success = $false
             Write-Warning $message
             $log_content | Write-Warning
         } else {
@@ -458,7 +419,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
             }
 
             if(!$pass_found) {
-                $script:success = $false;
+                $success = $false;
                 Write-Warning $message
                 $log_content | Write-Warning
             } else {
@@ -476,13 +437,13 @@ RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
 
 $EndDate = (Get-Date)
 
-New-TimeSpan -Start $StartDate -End $EndDate |
-  ForEach-Object {
+New-TimeSpan -Start $StartDate -End $EndDate | 
+  ForEach-Object { 
     "Elapsed time: {0:g}" -f $_
   }
 
 
-if(!$script:success) {
+if(!$success) {
 # This does not succeed killing off jobs quick
 # So we simply exit
 #    Remove-Job -Job $jobs -Force
@@ -492,4 +453,4 @@ if(!$script:success) {
 
  exit 0
 
-
+ 

build_tools/setup_centos7.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-set -ex
-
-ROCKSDB_VERSION="6.7.3"
-ZSTD_VERSION="1.4.4"
-
-echo "This script configures CentOS with everything needed to build and run RocksDB"
-
-yum update -y && yum install epel-release -y
-
-yum install -y \
-  wget \
-  gcc-c++ \
-  snappy snappy-devel \
-  zlib zlib-devel \
-  bzip2 bzip2-devel \
-  lz4-devel \
-  libasan \
-  gflags
-
-mkdir -pv /usr/local/rocksdb-${ROCKSDB_VERSION}
-ln -sfT /usr/local/rocksdb-${ROCKSDB_VERSION} /usr/local/rocksdb
-
-wget -qO /tmp/zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz
-wget -qO /tmp/rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz
-
-cd /tmp
-
-tar xzvf zstd-${ZSTD_VERSION}.tar.gz
-tar xzvf rocksdb-${ROCKSDB_VERSION}.tar.gz -C /usr/local/
-
-echo "Installing ZSTD..."
-pushd zstd-${ZSTD_VERSION}
-make && make install
-popd
-
-echo "Compiling RocksDB..."
-cd /usr/local/rocksdb
-chown -R vagrant:vagrant /usr/local/rocksdb/
-sudo -u vagrant make static_lib
-cd examples/
-sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ make all
-sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ ./c_simple_example
-

build_tools/update_dependencies.sh

@@ -1,17 +1,10 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 #
 # Update dependencies.sh file with the latest avaliable versions
 
 BASEDIR=$(dirname $0)
 OUTPUT=""
 
-function log_header()
-{
-  echo "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved." >> "$OUTPUT"
-}
-
-
 function log_variable()
 {
   echo "$1=${!1}" >> "$OUTPUT"
@@ -45,7 +38,6 @@ function get_lib_base()
     # platform is not provided, use latest gcc
     result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1`
   else
-    echo $lib_platform
     result="$result/$lib_platform/"
   fi
   
@@ -75,7 +67,6 @@ echo "Writing dependencies to $OUTPUT"
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
 CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
 
-log_header
 log_variable GCC_BASE
 log_variable CLANG_BASE
 
@@ -92,7 +83,6 @@ get_lib_base jemalloc   LATEST  platform007
 get_lib_base numa       LATEST  platform007
 get_lib_base libunwind  LATEST  platform007
 get_lib_base tbb        LATEST  platform007
-get_lib_base liburing   LATEST  platform007
 
 get_lib_base kernel-headers fb platform007
 get_lib_base binutils   LATEST centos7-native
@@ -113,30 +103,29 @@ touch "$OUTPUT"
 echo "Writing dependencies to $OUTPUT"
 
 # Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos7-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos6-native/*/`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
-log_header
 log_variable GCC_BASE
 log_variable CLANG_BASE
 
 # Libraries locations
-get_lib_base libgcc     5.x     gcc-5-glibc-2.23
-get_lib_base glibc      2.23    gcc-5-glibc-2.23
-get_lib_base snappy     LATEST  gcc-5-glibc-2.23
-get_lib_base zlib       LATEST  gcc-5-glibc-2.23
-get_lib_base bzip2      LATEST  gcc-5-glibc-2.23
-get_lib_base lz4        LATEST  gcc-5-glibc-2.23
-get_lib_base zstd       LATEST  gcc-5-glibc-2.23
-get_lib_base gflags     LATEST  gcc-5-glibc-2.23
-get_lib_base jemalloc   LATEST  gcc-5-glibc-2.23
-get_lib_base numa       LATEST  gcc-5-glibc-2.23
-get_lib_base libunwind  LATEST  gcc-5-glibc-2.23
-get_lib_base tbb        LATEST  gcc-5-glibc-2.23
+get_lib_base libgcc     5.x
+get_lib_base glibc      2.23
+get_lib_base snappy     LATEST gcc-5-glibc-2.23
+get_lib_base zlib       LATEST
+get_lib_base bzip2      LATEST
+get_lib_base lz4        LATEST
+get_lib_base zstd       LATEST
+get_lib_base gflags     LATEST
+get_lib_base jemalloc   LATEST
+get_lib_base numa       LATEST
+get_lib_base libunwind  LATEST
+get_lib_base tbb        4.0_update2 gcc-5-glibc-2.23
 
-get_lib_base kernel-headers 4.0.9-36_fbk5_2933_gd092e3f gcc-5-glibc-2.23
-get_lib_base binutils   LATEST centos7-native
-get_lib_base valgrind   LATEST gcc-5-glibc-2.23
+get_lib_base kernel-headers LATEST 
+get_lib_base binutils   LATEST centos6-native 
+get_lib_base valgrind   3.10.0 gcc-5-glibc-2.23
 get_lib_base lua        5.2.3 gcc-5-glibc-2.23
 
 git diff $OUTPUT
@@ -156,7 +145,6 @@ echo "Writing 4.8.1 dependencies to $OUTPUT"
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
 CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
-log_header
 log_variable GCC_BASE
 log_variable CLANG_BASE
 

build_tools/version.sh

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/sh
 if [ "$#" = "0" ]; then
   echo "Usage: $0 major|minor|patch|full"
   exit 1

cache/cache.cc

@@ -1,72 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "rocksdb/cache.h"
-
-#include "cache/lru_cache.h"
-#include "rocksdb/secondary_cache.h"
-#include "rocksdb/utilities/customizable_util.h"
-#include "rocksdb/utilities/options_type.h"
-#include "util/string_util.h"
-
-namespace ROCKSDB_NAMESPACE {
-#ifndef ROCKSDB_LITE
-static std::unordered_map<std::string, OptionTypeInfo>
-    lru_cache_options_type_info = {
-        {"capacity",
-         {offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT,
-          OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
-        {"num_shard_bits",
-         {offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt,
-          OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
-        {"strict_capacity_limit",
-         {offsetof(struct LRUCacheOptions, strict_capacity_limit),
-          OptionType::kBoolean, OptionVerificationType::kNormal,
-          OptionTypeFlags::kMutable}},
-        {"high_pri_pool_ratio",
-         {offsetof(struct LRUCacheOptions, high_pri_pool_ratio),
-          OptionType::kDouble, OptionVerificationType::kNormal,
-          OptionTypeFlags::kMutable}},
-};
-#endif  // ROCKSDB_LITE
-
-Status SecondaryCache::CreateFromString(
-    const ConfigOptions& config_options, const std::string& value,
-    std::shared_ptr<SecondaryCache>* result) {
-  return LoadSharedObject<SecondaryCache>(config_options, value, nullptr,
-                                          result);
-}
-
-Status Cache::CreateFromString(const ConfigOptions& config_options,
-                               const std::string& value,
-                               std::shared_ptr<Cache>* result) {
-  Status status;
-  std::shared_ptr<Cache> cache;
-  if (value.find('=') == std::string::npos) {
-    cache = NewLRUCache(ParseSizeT(value));
-  } else {
-#ifndef ROCKSDB_LITE
-    LRUCacheOptions cache_opts;
-    status = OptionTypeInfo::ParseStruct(config_options, "",
-                                         &lru_cache_options_type_info, "",
-                                         value, &cache_opts);
-    if (status.ok()) {
-      cache = NewLRUCache(cache_opts);
-    }
-#else
-    (void)config_options;
-    status = Status::NotSupported("Cannot load cache in LITE mode ", value);
-#endif  //! ROCKSDB_LITE
-  }
-  if (status.ok()) {
-    result->swap(cache);
-  }
-  return status;
-}
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_bench.cc

@@ -1,11 +1,11 @@
-//  Copyright (c) 2013-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
 #ifndef GFLAGS
 #include <cstdio>
 int main() {
@@ -13,8 +13,272 @@ int main() {
   return 1;
 }
 #else
-#include <rocksdb/cache_bench_tool.h>
-int main(int argc, char** argv) {
-  return ROCKSDB_NAMESPACE::cache_bench_tool(argc, argv);
+
+#include <inttypes.h>
+#include <sys/types.h>
+#include <stdio.h>
+#include <gflags/gflags.h>
+
+#include "rocksdb/db.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/env.h"
+#include "port/port.h"
+#include "util/mutexlock.h"
+#include "util/random.h"
+
+using GFLAGS::ParseCommandLineFlags;
+
+static const uint32_t KB = 1024;
+
+DEFINE_int32(threads, 16, "Number of concurrent threads to run.");
+DEFINE_int64(cache_size, 8 * KB * KB,
+             "Number of bytes to use as a cache of uncompressed data.");
+DEFINE_int32(num_shard_bits, 4, "shard_bits.");
+
+DEFINE_int64(max_key, 1 * KB * KB * KB, "Max number of key to place in cache");
+DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread.");
+
+DEFINE_bool(populate_cache, false, "Populate cache before operations");
+DEFINE_int32(insert_percent, 40,
+             "Ratio of insert to total workload (expressed as a percentage)");
+DEFINE_int32(lookup_percent, 50,
+             "Ratio of lookup to total workload (expressed as a percentage)");
+DEFINE_int32(erase_percent, 10,
+             "Ratio of erase to total workload (expressed as a percentage)");
+
+DEFINE_bool(use_clock_cache, false, "");
+
+namespace rocksdb {
+
+class CacheBench;
+namespace {
+void deleter(const Slice& key, void* value) {
+    delete reinterpret_cast<char *>(value);
 }
+
+// State shared by all concurrent executions of the same benchmark.
+class SharedState {
+ public:
+  explicit SharedState(CacheBench* cache_bench)
+      : cv_(&mu_),
+        num_threads_(FLAGS_threads),
+        num_initialized_(0),
+        start_(false),
+        num_done_(0),
+        cache_bench_(cache_bench) {
+  }
+
+  ~SharedState() {}
+
+  port::Mutex* GetMutex() {
+    return &mu_;
+  }
+
+  port::CondVar* GetCondVar() {
+    return &cv_;
+  }
+
+  CacheBench* GetCacheBench() const {
+    return cache_bench_;
+  }
+
+  void IncInitialized() {
+    num_initialized_++;
+  }
+
+  void IncDone() {
+    num_done_++;
+  }
+
+  bool AllInitialized() const {
+    return num_initialized_ >= num_threads_;
+  }
+
+  bool AllDone() const {
+    return num_done_ >= num_threads_;
+  }
+
+  void SetStart() {
+    start_ = true;
+  }
+
+  bool Started() const {
+    return start_;
+  }
+
+ private:
+  port::Mutex mu_;
+  port::CondVar cv_;
+
+  const uint64_t num_threads_;
+  uint64_t num_initialized_;
+  bool start_;
+  uint64_t num_done_;
+
+  CacheBench* cache_bench_;
+};
+
+// Per-thread state for concurrent executions of the same benchmark.
+struct ThreadState {
+  uint32_t tid;
+  Random rnd;
+  SharedState* shared;
+
+  ThreadState(uint32_t index, SharedState* _shared)
+      : tid(index), rnd(1000 + index), shared(_shared) {}
+};
+}  // namespace
+
+class CacheBench {
+ public:
+  CacheBench() : num_threads_(FLAGS_threads) {
+    if (FLAGS_use_clock_cache) {
+      cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
+      if (!cache_) {
+        fprintf(stderr, "Clock cache not supported.\n");
+        exit(1);
+      }
+    } else {
+      cache_ = NewLRUCache(FLAGS_cache_size, FLAGS_num_shard_bits);
+    }
+  }
+
+  ~CacheBench() {}
+
+  void PopulateCache() {
+    Random rnd(1);
+    for (int64_t i = 0; i < FLAGS_cache_size; i++) {
+      uint64_t rand_key = rnd.Next() % FLAGS_max_key;
+      // Cast uint64* to be char*, data would be copied to cache
+      Slice key(reinterpret_cast<char*>(&rand_key), 8);
+      // do insert
+      cache_->Insert(key, new char[10], 1, &deleter);
+    }
+  }
+
+  bool Run() {
+    rocksdb::Env* env = rocksdb::Env::Default();
+
+    PrintEnv();
+    SharedState shared(this);
+    std::vector<ThreadState*> threads(num_threads_);
+    for (uint32_t i = 0; i < num_threads_; i++) {
+      threads[i] = new ThreadState(i, &shared);
+      env->StartThread(ThreadBody, threads[i]);
+    }
+    {
+      MutexLock l(shared.GetMutex());
+      while (!shared.AllInitialized()) {
+        shared.GetCondVar()->Wait();
+      }
+      // Record start time
+      uint64_t start_time = env->NowMicros();
+
+      // Start all threads
+      shared.SetStart();
+      shared.GetCondVar()->SignalAll();
+
+      // Wait threads to complete
+      while (!shared.AllDone()) {
+        shared.GetCondVar()->Wait();
+      }
+
+      // Record end time
+      uint64_t end_time = env->NowMicros();
+      double elapsed = static_cast<double>(end_time - start_time) * 1e-6;
+      uint32_t qps = static_cast<uint32_t>(
+          static_cast<double>(FLAGS_threads * FLAGS_ops_per_thread) / elapsed);
+      fprintf(stdout, "Complete in %.3f s; QPS = %u\n", elapsed, qps);
+    }
+    return true;
+  }
+
+ private:
+  std::shared_ptr<Cache> cache_;
+  uint32_t num_threads_;
+
+  static void ThreadBody(void* v) {
+    ThreadState* thread = reinterpret_cast<ThreadState*>(v);
+    SharedState* shared = thread->shared;
+
+    {
+      MutexLock l(shared->GetMutex());
+      shared->IncInitialized();
+      if (shared->AllInitialized()) {
+        shared->GetCondVar()->SignalAll();
+      }
+      while (!shared->Started()) {
+        shared->GetCondVar()->Wait();
+      }
+    }
+    thread->shared->GetCacheBench()->OperateCache(thread);
+
+    {
+      MutexLock l(shared->GetMutex());
+      shared->IncDone();
+      if (shared->AllDone()) {
+        shared->GetCondVar()->SignalAll();
+      }
+    }
+  }
+
+  void OperateCache(ThreadState* thread) {
+    for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
+      uint64_t rand_key = thread->rnd.Next() % FLAGS_max_key;
+      // Cast uint64* to be char*, data would be copied to cache
+      Slice key(reinterpret_cast<char*>(&rand_key), 8);
+      int32_t prob_op = thread->rnd.Uniform(100);
+      if (prob_op >= 0 && prob_op < FLAGS_insert_percent) {
+        // do insert
+        cache_->Insert(key, new char[10], 1, &deleter);
+      } else if (prob_op -= FLAGS_insert_percent &&
+                 prob_op < FLAGS_lookup_percent) {
+        // do lookup
+        auto handle = cache_->Lookup(key);
+        if (handle) {
+          cache_->Release(handle);
+        }
+      } else if (prob_op -= FLAGS_lookup_percent &&
+                 prob_op < FLAGS_erase_percent) {
+        // do erase
+        cache_->Erase(key);
+      }
+    }
+  }
+
+  void PrintEnv() const {
+    printf("RocksDB version     : %d.%d\n", kMajorVersion, kMinorVersion);
+    printf("Number of threads   : %d\n", FLAGS_threads);
+    printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
+    printf("Cache size          : %" PRIu64 "\n", FLAGS_cache_size);
+    printf("Num shard bits      : %d\n", FLAGS_num_shard_bits);
+    printf("Max key             : %" PRIu64 "\n", FLAGS_max_key);
+    printf("Populate cache      : %d\n", FLAGS_populate_cache);
+    printf("Insert percentage   : %d%%\n", FLAGS_insert_percent);
+    printf("Lookup percentage   : %d%%\n", FLAGS_lookup_percent);
+    printf("Erase percentage    : %d%%\n", FLAGS_erase_percent);
+    printf("----------------------------\n");
+  }
+};
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  ParseCommandLineFlags(&argc, &argv, true);
+
+  if (FLAGS_threads <= 0) {
+    fprintf(stderr, "threads number <= 0\n");
+    exit(1);
+  }
+
+  rocksdb::CacheBench bench;
+  if (FLAGS_populate_cache) {
+    bench.PopulateCache();
+  }
+  if (bench.Run()) {
+    return 0;
+  } else {
+    return 1;
+  }
+}
+
 #endif  // GFLAGS

cache/cache_bench_tool.cc

@@ -1,572 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifdef GFLAGS
-#include <cinttypes>
-#include <cstdio>
-#include <limits>
-#include <set>
-#include <sstream>
-
-#include "monitoring/histogram.h"
-#include "port/port.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/convenience.h"
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/secondary_cache.h"
-#include "rocksdb/system_clock.h"
-#include "table/block_based/cachable_entry.h"
-#include "util/coding.h"
-#include "util/gflags_compat.h"
-#include "util/hash.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
-#include "util/stop_watch.h"
-#include "util/string_util.h"
-
-using GFLAGS_NAMESPACE::ParseCommandLineFlags;
-
-static constexpr uint32_t KiB = uint32_t{1} << 10;
-static constexpr uint32_t MiB = KiB << 10;
-static constexpr uint64_t GiB = MiB << 10;
-
-DEFINE_uint32(threads, 16, "Number of concurrent threads to run.");
-DEFINE_uint64(cache_size, 1 * GiB,
-              "Number of bytes to use as a cache of uncompressed data.");
-DEFINE_uint32(num_shard_bits, 6, "shard_bits.");
-
-DEFINE_double(resident_ratio, 0.25,
-              "Ratio of keys fitting in cache to keyspace.");
-DEFINE_uint64(ops_per_thread, 2000000U, "Number of operations per thread.");
-DEFINE_uint32(value_bytes, 8 * KiB, "Size of each value added.");
-
-DEFINE_uint32(skew, 5, "Degree of skew in key selection");
-DEFINE_bool(populate_cache, true, "Populate cache before operations");
-
-DEFINE_uint32(lookup_insert_percent, 87,
-              "Ratio of lookup (+ insert on not found) to total workload "
-              "(expressed as a percentage)");
-DEFINE_uint32(insert_percent, 2,
-              "Ratio of insert to total workload (expressed as a percentage)");
-DEFINE_uint32(lookup_percent, 10,
-              "Ratio of lookup to total workload (expressed as a percentage)");
-DEFINE_uint32(erase_percent, 1,
-              "Ratio of erase to total workload (expressed as a percentage)");
-DEFINE_bool(gather_stats, false,
-            "Whether to periodically simulate gathering block cache stats, "
-            "using one more thread.");
-DEFINE_uint32(
-    gather_stats_sleep_ms, 1000,
-    "How many milliseconds to sleep between each gathering of stats.");
-
-DEFINE_uint32(gather_stats_entries_per_lock, 256,
-              "For Cache::ApplyToAllEntries");
-DEFINE_bool(skewed, false, "If true, skew the key access distribution");
-#ifndef ROCKSDB_LITE
-DEFINE_string(secondary_cache_uri, "",
-              "Full URI for creating a custom secondary cache object");
-static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;
-#endif  // ROCKSDB_LITE
-
-DEFINE_bool(use_clock_cache, false, "");
-
-namespace ROCKSDB_NAMESPACE {
-
-class CacheBench;
-namespace {
-// State shared by all concurrent executions of the same benchmark.
-class SharedState {
- public:
-  explicit SharedState(CacheBench* cache_bench)
-      : cv_(&mu_),
-        num_initialized_(0),
-        start_(false),
-        num_done_(0),
-        cache_bench_(cache_bench) {}
-
-  ~SharedState() {}
-
-  port::Mutex* GetMutex() { return &mu_; }
-
-  port::CondVar* GetCondVar() { return &cv_; }
-
-  CacheBench* GetCacheBench() const { return cache_bench_; }
-
-  void IncInitialized() { num_initialized_++; }
-
-  void IncDone() { num_done_++; }
-
-  bool AllInitialized() const { return num_initialized_ >= FLAGS_threads; }
-
-  bool AllDone() const { return num_done_ >= FLAGS_threads; }
-
-  void SetStart() { start_ = true; }
-
-  bool Started() const { return start_; }
-
- private:
-  port::Mutex mu_;
-  port::CondVar cv_;
-
-  uint64_t num_initialized_;
-  bool start_;
-  uint64_t num_done_;
-
-  CacheBench* cache_bench_;
-};
-
-// Per-thread state for concurrent executions of the same benchmark.
-struct ThreadState {
-  uint32_t tid;
-  Random64 rnd;
-  SharedState* shared;
-  HistogramImpl latency_ns_hist;
-  uint64_t duration_us = 0;
-
-  ThreadState(uint32_t index, SharedState* _shared)
-      : tid(index), rnd(1000 + index), shared(_shared) {}
-};
-
-struct KeyGen {
-  char key_data[27];
-
-  Slice GetRand(Random64& rnd, uint64_t max_key, int max_log) {
-    uint64_t key = 0;
-    if (!FLAGS_skewed) {
-      uint64_t raw = rnd.Next();
-      // Skew according to setting
-      for (uint32_t i = 0; i < FLAGS_skew; ++i) {
-        raw = std::min(raw, rnd.Next());
-      }
-      key = FastRange64(raw, max_key);
-    } else {
-      key = rnd.Skewed(max_log);
-      if (key > max_key) {
-        key -= max_key;
-      }
-    }
-    // Variable size and alignment
-    size_t off = key % 8;
-    key_data[0] = char{42};
-    EncodeFixed64(key_data + 1, key);
-    key_data[9] = char{11};
-    EncodeFixed64(key_data + 10, key);
-    key_data[18] = char{4};
-    EncodeFixed64(key_data + 19, key);
-    return Slice(&key_data[off], sizeof(key_data) - off);
-  }
-};
-
-char* createValue(Random64& rnd) {
-  char* rv = new char[FLAGS_value_bytes];
-  // Fill with some filler data, and take some CPU time
-  for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) {
-    EncodeFixed64(rv + i, rnd.Next());
-  }
-  return rv;
-}
-
-// Callbacks for secondary cache
-size_t SizeFn(void* /*obj*/) { return FLAGS_value_bytes; }
-
-Status SaveToFn(void* obj, size_t /*offset*/, size_t size, void* out) {
-  memcpy(out, obj, size);
-  return Status::OK();
-}
-
-// Different deleters to simulate using deleter to gather
-// stats on the code origin and kind of cache entries.
-void deleter1(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
-void deleter2(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
-void deleter3(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
-
-Cache::CacheItemHelper helper1(SizeFn, SaveToFn, deleter1);
-Cache::CacheItemHelper helper2(SizeFn, SaveToFn, deleter2);
-Cache::CacheItemHelper helper3(SizeFn, SaveToFn, deleter3);
-}  // namespace
-
-class CacheBench {
-  static constexpr uint64_t kHundredthUint64 =
-      std::numeric_limits<uint64_t>::max() / 100U;
-
- public:
-  CacheBench()
-      : max_key_(static_cast<uint64_t>(FLAGS_cache_size / FLAGS_resident_ratio /
-                                       FLAGS_value_bytes)),
-        lookup_insert_threshold_(kHundredthUint64 *
-                                 FLAGS_lookup_insert_percent),
-        insert_threshold_(lookup_insert_threshold_ +
-                          kHundredthUint64 * FLAGS_insert_percent),
-        lookup_threshold_(insert_threshold_ +
-                          kHundredthUint64 * FLAGS_lookup_percent),
-        erase_threshold_(lookup_threshold_ +
-                         kHundredthUint64 * FLAGS_erase_percent),
-        skewed_(FLAGS_skewed) {
-    if (erase_threshold_ != 100U * kHundredthUint64) {
-      fprintf(stderr, "Percentages must add to 100.\n");
-      exit(1);
-    }
-
-    max_log_ = 0;
-    if (skewed_) {
-      uint64_t max_key = max_key_;
-      while (max_key >>= 1) max_log_++;
-      if (max_key > (1u << max_log_)) max_log_++;
-    }
-
-    if (FLAGS_use_clock_cache) {
-      cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
-      if (!cache_) {
-        fprintf(stderr, "Clock cache not supported.\n");
-        exit(1);
-      }
-    } else {
-      LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false, 0.5);
-#ifndef ROCKSDB_LITE
-      if (!FLAGS_secondary_cache_uri.empty()) {
-        Status s = SecondaryCache::CreateFromString(
-            ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache);
-        if (secondary_cache == nullptr) {
-          fprintf(
-              stderr,
-              "No secondary cache registered matching string: %s status=%s\n",
-              FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str());
-          exit(1);
-        }
-        opts.secondary_cache = secondary_cache;
-      }
-#endif  // ROCKSDB_LITE
-
-      cache_ = NewLRUCache(opts);
-    }
-  }
-
-  ~CacheBench() {}
-
-  void PopulateCache() {
-    Random64 rnd(1);
-    KeyGen keygen;
-    for (uint64_t i = 0; i < 2 * FLAGS_cache_size; i += FLAGS_value_bytes) {
-      cache_->Insert(keygen.GetRand(rnd, max_key_, max_log_), createValue(rnd),
-                     &helper1, FLAGS_value_bytes);
-    }
-  }
-
-  bool Run() {
-    const auto clock = SystemClock::Default().get();
-
-    PrintEnv();
-    SharedState shared(this);
-    std::vector<std::unique_ptr<ThreadState> > threads(FLAGS_threads);
-    for (uint32_t i = 0; i < FLAGS_threads; i++) {
-      threads[i].reset(new ThreadState(i, &shared));
-      std::thread(ThreadBody, threads[i].get()).detach();
-    }
-
-    HistogramImpl stats_hist;
-    std::string stats_report;
-    std::thread stats_thread(StatsBody, &shared, &stats_hist, &stats_report);
-
-    uint64_t start_time;
-    {
-      MutexLock l(shared.GetMutex());
-      while (!shared.AllInitialized()) {
-        shared.GetCondVar()->Wait();
-      }
-      // Record start time
-      start_time = clock->NowMicros();
-
-      // Start all threads
-      shared.SetStart();
-      shared.GetCondVar()->SignalAll();
-
-      // Wait threads to complete
-      while (!shared.AllDone()) {
-        shared.GetCondVar()->Wait();
-      }
-    }
-
-    // Stats gathering is considered background work. This time measurement
-    // is for foreground work, and not really ideal for that. See below.
-    uint64_t end_time = clock->NowMicros();
-    stats_thread.join();
-
-    // Wall clock time - includes idle time if threads
-    // finish at different times (not ideal).
-    double elapsed_secs = static_cast<double>(end_time - start_time) * 1e-6;
-    uint32_t ops_per_sec = static_cast<uint32_t>(
-        1.0 * FLAGS_threads * FLAGS_ops_per_thread / elapsed_secs);
-    printf("Complete in %.3f s; Rough parallel ops/sec = %u\n", elapsed_secs,
-           ops_per_sec);
-
-    // Total time in each thread (more accurate throughput measure)
-    elapsed_secs = 0;
-    for (uint32_t i = 0; i < FLAGS_threads; i++) {
-      elapsed_secs += threads[i]->duration_us * 1e-6;
-    }
-    ops_per_sec = static_cast<uint32_t>(1.0 * FLAGS_threads *
-                                        FLAGS_ops_per_thread / elapsed_secs);
-    printf("Thread ops/sec = %u\n", ops_per_sec);
-
-    printf("\nOperation latency (ns):\n");
-    HistogramImpl combined;
-    for (uint32_t i = 0; i < FLAGS_threads; i++) {
-      combined.Merge(threads[i]->latency_ns_hist);
-    }
-    printf("%s", combined.ToString().c_str());
-
-    if (FLAGS_gather_stats) {
-      printf("\nGather stats latency (us):\n");
-      printf("%s", stats_hist.ToString().c_str());
-    }
-
-    printf("\n%s", stats_report.c_str());
-
-    return true;
-  }
-
- private:
-  std::shared_ptr<Cache> cache_;
-  const uint64_t max_key_;
-  // Cumulative thresholds in the space of a random uint64_t
-  const uint64_t lookup_insert_threshold_;
-  const uint64_t insert_threshold_;
-  const uint64_t lookup_threshold_;
-  const uint64_t erase_threshold_;
-  const bool skewed_;
-  int max_log_;
-
-  // A benchmark version of gathering stats on an active block cache by
-  // iterating over it. The primary purpose is to measure the impact of
-  // gathering stats with ApplyToAllEntries on throughput- and
-  // latency-sensitive Cache users. Performance of stats gathering is
-  // also reported. The last set of gathered stats is also reported, for
-  // manual sanity checking for logical errors or other unexpected
-  // behavior of cache_bench or the underlying Cache.
-  static void StatsBody(SharedState* shared, HistogramImpl* stats_hist,
-                        std::string* stats_report) {
-    if (!FLAGS_gather_stats) {
-      return;
-    }
-    const auto clock = SystemClock::Default().get();
-    uint64_t total_key_size = 0;
-    uint64_t total_charge = 0;
-    uint64_t total_entry_count = 0;
-    std::set<Cache::DeleterFn> deleters;
-    StopWatchNano timer(clock);
-
-    for (;;) {
-      uint64_t time;
-      time = clock->NowMicros();
-      uint64_t deadline = time + uint64_t{FLAGS_gather_stats_sleep_ms} * 1000;
-
-      {
-        MutexLock l(shared->GetMutex());
-        for (;;) {
-          if (shared->AllDone()) {
-            std::ostringstream ostr;
-            ostr << "Most recent cache entry stats:\n"
-                 << "Number of entries: " << total_entry_count << "\n"
-                 << "Total charge: " << BytesToHumanString(total_charge) << "\n"
-                 << "Average key size: "
-                 << (1.0 * total_key_size / total_entry_count) << "\n"
-                 << "Average charge: "
-                 << BytesToHumanString(1.0 * total_charge / total_entry_count)
-                 << "\n"
-                 << "Unique deleters: " << deleters.size() << "\n";
-            *stats_report = ostr.str();
-            return;
-          }
-          if (clock->NowMicros() >= deadline) {
-            break;
-          }
-          uint64_t diff = deadline - std::min(clock->NowMicros(), deadline);
-          shared->GetCondVar()->TimedWait(diff + 1);
-        }
-      }
-
-      // Now gather stats, outside of mutex
-      total_key_size = 0;
-      total_charge = 0;
-      total_entry_count = 0;
-      deleters.clear();
-      auto fn = [&](const Slice& key, void* /*value*/, size_t charge,
-                    Cache::DeleterFn deleter) {
-        total_key_size += key.size();
-        total_charge += charge;
-        ++total_entry_count;
-        // Something slightly more expensive as in (future) stats by category
-        deleters.insert(deleter);
-      };
-      timer.Start();
-      Cache::ApplyToAllEntriesOptions opts;
-      opts.average_entries_per_lock = FLAGS_gather_stats_entries_per_lock;
-      shared->GetCacheBench()->cache_->ApplyToAllEntries(fn, opts);
-      stats_hist->Add(timer.ElapsedNanos() / 1000);
-    }
-  }
-
-  static void ThreadBody(ThreadState* thread) {
-    SharedState* shared = thread->shared;
-
-    {
-      MutexLock l(shared->GetMutex());
-      shared->IncInitialized();
-      if (shared->AllInitialized()) {
-        shared->GetCondVar()->SignalAll();
-      }
-      while (!shared->Started()) {
-        shared->GetCondVar()->Wait();
-      }
-    }
-    thread->shared->GetCacheBench()->OperateCache(thread);
-
-    {
-      MutexLock l(shared->GetMutex());
-      shared->IncDone();
-      if (shared->AllDone()) {
-        shared->GetCondVar()->SignalAll();
-      }
-    }
-  }
-
-  void OperateCache(ThreadState* thread) {
-    // To use looked-up values
-    uint64_t result = 0;
-    // To hold handles for a non-trivial amount of time
-    Cache::Handle* handle = nullptr;
-    KeyGen gen;
-    const auto clock = SystemClock::Default().get();
-    uint64_t start_time = clock->NowMicros();
-    StopWatchNano timer(clock);
-
-    for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
-      timer.Start();
-      Slice key = gen.GetRand(thread->rnd, max_key_, max_log_);
-      uint64_t random_op = thread->rnd.Next();
-      Cache::CreateCallback create_cb =
-          [](void* buf, size_t size, void** out_obj, size_t* charge) -> Status {
-        *out_obj = reinterpret_cast<void*>(new char[size]);
-        memcpy(*out_obj, buf, size);
-        *charge = size;
-        return Status::OK();
-      };
-
-      if (random_op < lookup_insert_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
-        // do lookup
-        handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
-                                true);
-        if (handle) {
-          // do something with the data
-          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
-                             FLAGS_value_bytes);
-        } else {
-          // do insert
-          cache_->Insert(key, createValue(thread->rnd), &helper2,
-                         FLAGS_value_bytes, &handle);
-        }
-      } else if (random_op < insert_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
-        // do insert
-        cache_->Insert(key, createValue(thread->rnd), &helper3,
-                       FLAGS_value_bytes, &handle);
-      } else if (random_op < lookup_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
-        // do lookup
-        handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
-                                true);
-        if (handle) {
-          // do something with the data
-          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
-                             FLAGS_value_bytes);
-        }
-      } else if (random_op < erase_threshold_) {
-        // do erase
-        cache_->Erase(key);
-      } else {
-        // Should be extremely unlikely (noop)
-        assert(random_op >= kHundredthUint64 * 100U);
-      }
-      thread->latency_ns_hist.Add(timer.ElapsedNanos());
-    }
-    if (handle) {
-      cache_->Release(handle);
-      handle = nullptr;
-    }
-    // Ensure computations on `result` are not optimized away.
-    if (result == 1) {
-      printf("You are extremely unlucky(2). Try again.\n");
-      exit(1);
-    }
-    thread->duration_us = clock->NowMicros() - start_time;
-  }
-
-  void PrintEnv() const {
-    printf("RocksDB version     : %d.%d\n", kMajorVersion, kMinorVersion);
-    printf("Number of threads   : %u\n", FLAGS_threads);
-    printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
-    printf("Cache size          : %s\n",
-           BytesToHumanString(FLAGS_cache_size).c_str());
-    printf("Num shard bits      : %u\n", FLAGS_num_shard_bits);
-    printf("Max key             : %" PRIu64 "\n", max_key_);
-    printf("Resident ratio      : %g\n", FLAGS_resident_ratio);
-    printf("Skew degree         : %u\n", FLAGS_skew);
-    printf("Populate cache      : %d\n", int{FLAGS_populate_cache});
-    printf("Lookup+Insert pct   : %u%%\n", FLAGS_lookup_insert_percent);
-    printf("Insert percentage   : %u%%\n", FLAGS_insert_percent);
-    printf("Lookup percentage   : %u%%\n", FLAGS_lookup_percent);
-    printf("Erase percentage    : %u%%\n", FLAGS_erase_percent);
-    std::ostringstream stats;
-    if (FLAGS_gather_stats) {
-      stats << "enabled (" << FLAGS_gather_stats_sleep_ms << "ms, "
-            << FLAGS_gather_stats_entries_per_lock << "/lock)";
-    } else {
-      stats << "disabled";
-    }
-    printf("Gather stats        : %s\n", stats.str().c_str());
-    printf("----------------------------\n");
-  }
-};
-
-int cache_bench_tool(int argc, char** argv) {
-  ParseCommandLineFlags(&argc, &argv, true);
-
-  if (FLAGS_threads <= 0) {
-    fprintf(stderr, "threads number <= 0\n");
-    exit(1);
-  }
-
-  ROCKSDB_NAMESPACE::CacheBench bench;
-  if (FLAGS_populate_cache) {
-    bench.PopulateCache();
-    printf("Population complete\n");
-    printf("----------------------------\n");
-  }
-  if (bench.Run()) {
-    return 0;
-  } else {
-    return 1;
-  }
-}  // namespace ROCKSDB_NAMESPACE
-}  // namespace ROCKSDB_NAMESPACE
-
-#endif  // GFLAGS

cache/cache_entry_roles.cc

@@ -1,66 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "cache/cache_entry_roles.h"
-
-#include <mutex>
-
-#include "port/lang.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
-    "DataBlock",
-    "FilterBlock",
-    "FilterMetaBlock",
-    "DeprecatedFilterBlock",
-    "IndexBlock",
-    "OtherBlock",
-    "WriteBuffer",
-    "Misc",
-}};
-
-std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
-    "data-block",
-    "filter-block",
-    "filter-meta-block",
-    "deprecated-filter-block",
-    "index-block",
-    "other-block",
-    "write-buffer",
-    "misc",
-}};
-
-namespace {
-
-struct Registry {
-  std::mutex mutex;
-  std::unordered_map<Cache::DeleterFn, CacheEntryRole> role_map;
-  void Register(Cache::DeleterFn fn, CacheEntryRole role) {
-    std::lock_guard<std::mutex> lock(mutex);
-    role_map[fn] = role;
-  }
-  std::unordered_map<Cache::DeleterFn, CacheEntryRole> Copy() {
-    std::lock_guard<std::mutex> lock(mutex);
-    return role_map;
-  }
-};
-
-Registry& GetRegistry() {
-  STATIC_AVOID_DESTRUCTION(Registry, registry);
-  return registry;
-}
-
-}  // namespace
-
-void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role) {
-  GetRegistry().Register(fn, role);
-}
-
-std::unordered_map<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap() {
-  return GetRegistry().Copy();
-}
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_entry_roles.h

@@ -1,122 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <array>
-#include <cstdint>
-#include <unordered_map>
-
-#include "rocksdb/cache.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Classifications of block cache entries, for reporting statistics
-enum class CacheEntryRole {
-  // Block-based table data block
-  kDataBlock,
-  // Block-based table filter block (full or partitioned)
-  kFilterBlock,
-  // Block-based table metadata block for partitioned filter
-  kFilterMetaBlock,
-  // Block-based table deprecated filter block (old "block-based" filter)
-  kDeprecatedFilterBlock,
-  // Block-based table index block
-  kIndexBlock,
-  // Other kinds of block-based table block
-  kOtherBlock,
-  // WriteBufferManager reservations to account for memtable usage
-  kWriteBuffer,
-  // Default bucket, for miscellaneous cache entries. Do not use for
-  // entries that could potentially add up to large usage.
-  kMisc,
-};
-constexpr uint32_t kNumCacheEntryRoles =
-    static_cast<uint32_t>(CacheEntryRole::kMisc) + 1;
-
-extern std::array<const char*, kNumCacheEntryRoles>
-    kCacheEntryRoleToCamelString;
-extern std::array<const char*, kNumCacheEntryRoles>
-    kCacheEntryRoleToHyphenString;
-
-// To associate cache entries with their role, we use a hack on the
-// existing Cache interface. Because the deleter of an entry can authenticate
-// the code origin of an entry, we can elaborate the choice of deleter to
-// also encode role information, without inferring false role information
-// from entries not choosing to encode a role.
-//
-// The rest of this file is for handling mappings between deleters and
-// roles.
-
-// To infer a role from a deleter, the deleter must be registered. This
-// can be done "manually" with this function. This function is thread-safe,
-// and the registration mappings go into private but static storage. (Note
-// that DeleterFn is a function pointer, not std::function. Registrations
-// should not be too many.)
-void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role);
-
-// Gets a copy of the registered deleter -> role mappings. This is the only
-// function for reading the mappings made with RegisterCacheDeleterRole.
-// Why only this interface for reading?
-// * This function has to be thread safe, which could incur substantial
-// overhead. We should not pay this overhead for every deleter look-up.
-// * This is suitable for preparing for batch operations, like with
-// CacheEntryStatsCollector.
-// * The number of mappings should be sufficiently small (dozens).
-std::unordered_map<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap();
-
-// ************************************************************** //
-// An automatic registration infrastructure. This enables code
-// to simply ask for a deleter associated with a particular type
-// and role, and registration is automatic. In a sense, this is
-// a small dependency injection infrastructure, because linking
-// in new deleter instantiations is essentially sufficient for
-// making stats collection (using CopyCacheDeleterRoleMap) aware
-// of them.
-
-namespace cache_entry_roles_detail {
-
-template <typename T, CacheEntryRole R>
-struct RegisteredDeleter {
-  RegisteredDeleter() { RegisterCacheDeleterRole(Delete, R); }
-
-  // These have global linkage to help ensure compiler optimizations do not
-  // break uniqueness for each <T,R>
-  static void Delete(const Slice& /* key */, void* value) {
-    delete static_cast<T*>(value);
-  }
-};
-
-template <CacheEntryRole R>
-struct RegisteredNoopDeleter {
-  RegisteredNoopDeleter() { RegisterCacheDeleterRole(Delete, R); }
-
-  static void Delete(const Slice& /* key */, void* value) {
-    (void)value;
-    assert(value == nullptr);
-  }
-};
-
-}  // namespace cache_entry_roles_detail
-
-// Get an automatically registered deleter for value type T and role R.
-// Based on C++ semantics, registration is invoked exactly once in a
-// thread-safe way on first call to this function, for each <T, R>.
-template <typename T, CacheEntryRole R>
-Cache::DeleterFn GetCacheEntryDeleterForRole() {
-  static cache_entry_roles_detail::RegisteredDeleter<T, R> reg;
-  return reg.Delete;
-}
-
-// Get an automatically registered no-op deleter (value should be nullptr)
-// and associated with role R. This is used for Cache "reservation" entries
-// such as for WriteBufferManager.
-template <CacheEntryRole R>
-Cache::DeleterFn GetNoopDeleterForRole() {
-  static cache_entry_roles_detail::RegisteredNoopDeleter<R> reg;
-  return reg.Delete;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_entry_stats.h

@@ -1,180 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <array>
-#include <cstdint>
-#include <memory>
-#include <mutex>
-
-#include "cache/cache_helpers.h"
-#include "port/lang.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/status.h"
-#include "rocksdb/system_clock.h"
-#include "util/coding_lean.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// A generic helper object for gathering stats about cache entries by
-// iterating over them with ApplyToAllEntries. This class essentially
-// solves the problem of slowing down a Cache with too many stats
-// collectors that could be sharing stat results, such as from multiple
-// column families or multiple DBs sharing a Cache. We employ a few
-// mitigations:
-// * Only one collector for a particular kind of Stats is alive
-// for each Cache. This is guaranteed using the Cache itself to hold
-// the collector.
-// * A mutex ensures only one thread is gathering stats for this
-// collector.
-// * The most recent gathered stats are saved and simply copied to
-// satisfy requests within a time window (default: 3 minutes) of
-// completion of the most recent stat gathering.
-//
-// Template parameter Stats must be copyable and trivially constructable,
-// as well as...
-// concept Stats {
-//   // Notification before applying callback to all entries
-//   void BeginCollection(Cache*, SystemClock*, uint64_t start_time_micros);
-//   // Get the callback to apply to all entries. `callback`
-//   // type must be compatible with Cache::ApplyToAllEntries
-//   callback GetEntryCallback();
-//   // Notification after applying callback to all entries
-//   void EndCollection(Cache*, SystemClock*, uint64_t end_time_micros);
-//   // Notification that a collection was skipped because of
-//   // sufficiently recent saved results.
-//   void SkippedCollection();
-// }
-template <class Stats>
-class CacheEntryStatsCollector {
- public:
-  // Gather and save stats if saved stats are too old. (Use GetStats() to
-  // read saved stats.)
-  //
-  // Maximum allowed age for a "hit" on saved results is determined by the
-  // two interval parameters. Both set to 0 forces a re-scan. For example
-  // with min_interval_seconds=300 and min_interval_factor=100, if the last
-  // scan took 10s, we would only rescan ("miss") if the age in seconds of
-  // the saved results is > max(300, 100*10).
-  // Justification: scans can vary wildly in duration, e.g. from 0.02 sec
-  // to as much as 20 seconds, so we want to be able to cap the absolute
-  // and relative frequency of scans.
-  void CollectStats(int min_interval_seconds, int min_interval_factor) {
-    // Waits for any pending reader or writer (collector)
-    std::lock_guard<std::mutex> lock(working_mutex_);
-
-    uint64_t max_age_micros =
-        static_cast<uint64_t>(std::max(min_interval_seconds, 0)) * 1000000U;
-
-    if (last_end_time_micros_ > last_start_time_micros_ &&
-        min_interval_factor > 0) {
-      max_age_micros = std::max(
-          max_age_micros, min_interval_factor * (last_end_time_micros_ -
-                                                 last_start_time_micros_));
-    }
-
-    uint64_t start_time_micros = clock_->NowMicros();
-    if ((start_time_micros - last_end_time_micros_) > max_age_micros) {
-      last_start_time_micros_ = start_time_micros;
-      working_stats_.BeginCollection(cache_, clock_, start_time_micros);
-
-      cache_->ApplyToAllEntries(working_stats_.GetEntryCallback(), {});
-      TEST_SYNC_POINT_CALLBACK(
-          "CacheEntryStatsCollector::GetStats:AfterApplyToAllEntries", nullptr);
-
-      uint64_t end_time_micros = clock_->NowMicros();
-      last_end_time_micros_ = end_time_micros;
-      working_stats_.EndCollection(cache_, clock_, end_time_micros);
-    } else {
-      working_stats_.SkippedCollection();
-    }
-
-    // Save so that we don't need to wait for an outstanding collection in
-    // order to make of copy of the last saved stats
-    std::lock_guard<std::mutex> lock2(saved_mutex_);
-    saved_stats_ = working_stats_;
-  }
-
-  // Gets saved stats, regardless of age
-  void GetStats(Stats *stats) {
-    std::lock_guard<std::mutex> lock(saved_mutex_);
-    *stats = saved_stats_;
-  }
-
-  Cache *GetCache() const { return cache_; }
-
-  // Gets or creates a shared instance of CacheEntryStatsCollector in the
-  // cache itself, and saves into `ptr`. This shared_ptr will hold the
-  // entry in cache until all refs are destroyed.
-  static Status GetShared(Cache *cache, SystemClock *clock,
-                          std::shared_ptr<CacheEntryStatsCollector> *ptr) {
-    std::array<uint64_t, 3> cache_key_data{
-        {// First 16 bytes == md5 of class name
-         0x7eba5a8fb5437c90U, 0x8ca68c9b11655855U,
-         // Last 8 bytes based on a function pointer to make unique for each
-         // template instantiation
-         reinterpret_cast<uint64_t>(&CacheEntryStatsCollector::GetShared)}};
-    Slice cache_key = GetSlice(&cache_key_data);
-
-    Cache::Handle *h = cache->Lookup(cache_key);
-    if (h == nullptr) {
-      // Not yet in cache, but Cache doesn't provide a built-in way to
-      // avoid racing insert. So we double-check under a shared mutex,
-      // inspired by TableCache.
-      STATIC_AVOID_DESTRUCTION(std::mutex, static_mutex);
-      std::lock_guard<std::mutex> lock(static_mutex);
-
-      h = cache->Lookup(cache_key);
-      if (h == nullptr) {
-        auto new_ptr = new CacheEntryStatsCollector(cache, clock);
-        // TODO: non-zero charge causes some tests that count block cache
-        // usage to go flaky. Fix the problem somehow so we can use an
-        // accurate charge.
-        size_t charge = 0;
-        Status s = cache->Insert(cache_key, new_ptr, charge, Deleter, &h,
-                                 Cache::Priority::HIGH);
-        if (!s.ok()) {
-          assert(h == nullptr);
-          delete new_ptr;
-          return s;
-        }
-      }
-    }
-    // If we reach here, shared entry is in cache with handle `h`.
-    assert(cache->GetDeleter(h) == Deleter);
-
-    // Build an aliasing shared_ptr that keeps `ptr` in cache while there
-    // are references.
-    *ptr = MakeSharedCacheHandleGuard<CacheEntryStatsCollector>(cache, h);
-    return Status::OK();
-  }
-
- private:
-  explicit CacheEntryStatsCollector(Cache *cache, SystemClock *clock)
-      : saved_stats_(),
-        working_stats_(),
-        last_start_time_micros_(0),
-        last_end_time_micros_(/*pessimistic*/ 10000000),
-        cache_(cache),
-        clock_(clock) {}
-
-  static void Deleter(const Slice &, void *value) {
-    delete static_cast<CacheEntryStatsCollector *>(value);
-  }
-
-  std::mutex saved_mutex_;
-  Stats saved_stats_;
-
-  std::mutex working_mutex_;
-  Stats working_stats_;
-  uint64_t last_start_time_micros_;
-  uint64_t last_end_time_micros_;
-
-  Cache *const cache_;
-  SystemClock *const clock_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_helpers.h

@@ -1,125 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-
-#include "rocksdb/cache.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Returns the cached value given a cache handle.
-template <typename T>
-T* GetFromCacheHandle(Cache* cache, Cache::Handle* handle) {
-  assert(cache);
-  assert(handle);
-
-  return static_cast<T*>(cache->Value(handle));
-}
-
-// Simple generic deleter for Cache (to be used with Cache::Insert).
-template <typename T>
-void DeleteCacheEntry(const Slice& /* key */, void* value) {
-  delete static_cast<T*>(value);
-}
-
-// Turns a T* into a Slice so it can be used as a key with Cache.
-template <typename T>
-Slice GetSlice(const T* t) {
-  return Slice(reinterpret_cast<const char*>(t), sizeof(T));
-}
-
-// Generic resource management object for cache handles that releases the handle
-// when destroyed. Has unique ownership of the handle, so copying it is not
-// allowed, while moving it transfers ownership.
-template <typename T>
-class CacheHandleGuard {
- public:
-  CacheHandleGuard() = default;
-
-  CacheHandleGuard(Cache* cache, Cache::Handle* handle)
-      : cache_(cache),
-        handle_(handle),
-        value_(GetFromCacheHandle<T>(cache, handle)) {
-    assert(cache_ && handle_ && value_);
-  }
-
-  CacheHandleGuard(const CacheHandleGuard&) = delete;
-  CacheHandleGuard& operator=(const CacheHandleGuard&) = delete;
-
-  CacheHandleGuard(CacheHandleGuard&& rhs) noexcept
-      : cache_(rhs.cache_), handle_(rhs.handle_), value_(rhs.value_) {
-    assert((!cache_ && !handle_ && !value_) || (cache_ && handle_ && value_));
-
-    rhs.ResetFields();
-  }
-
-  CacheHandleGuard& operator=(CacheHandleGuard&& rhs) noexcept {
-    if (this == &rhs) {
-      return *this;
-    }
-
-    ReleaseHandle();
-
-    cache_ = rhs.cache_;
-    handle_ = rhs.handle_;
-    value_ = rhs.value_;
-
-    assert((!cache_ && !handle_ && !value_) || (cache_ && handle_ && value_));
-
-    rhs.ResetFields();
-
-    return *this;
-  }
-
-  ~CacheHandleGuard() { ReleaseHandle(); }
-
-  bool IsEmpty() const { return !handle_; }
-
-  Cache* GetCache() const { return cache_; }
-  Cache::Handle* GetCacheHandle() const { return handle_; }
-  T* GetValue() const { return value_; }
-
-  void Reset() {
-    ReleaseHandle();
-    ResetFields();
-  }
-
- private:
-  void ReleaseHandle() {
-    if (IsEmpty()) {
-      return;
-    }
-
-    assert(cache_);
-    cache_->Release(handle_);
-  }
-
-  void ResetFields() {
-    cache_ = nullptr;
-    handle_ = nullptr;
-    value_ = nullptr;
-  }
-
- private:
-  Cache* cache_ = nullptr;
-  Cache::Handle* handle_ = nullptr;
-  T* value_ = nullptr;
-};
-
-// Build an aliasing shared_ptr that keeps `handle` in cache while there
-// are references, but the pointer is to the value for that cache entry,
-// which must be of type T. This is copyable, unlike CacheHandleGuard, but
-// does not provide access to caching details.
-template <typename T>
-std::shared_ptr<T> MakeSharedCacheHandleGuard(Cache* cache,
-                                              Cache::Handle* handle) {
-  auto wrapper = std::make_shared<CacheHandleGuard<T>>(cache, handle);
-  return std::shared_ptr<T>(wrapper, static_cast<T*>(cache->Value(handle)));
-}
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_test.cc

@@ -16,11 +16,11 @@
 #include <vector>
 #include "cache/clock_cache.h"
 #include "cache/lru_cache.h"
-#include "test_util/testharness.h"
 #include "util/coding.h"
 #include "util/string_util.h"
+#include "util/testharness.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 // Conversions between numeric keys/values and the types expected by Cache.
 static std::string EncodeKey(int k) {
@@ -40,9 +40,9 @@ static int DecodeValue(void* v) {
 const std::string kLRU = "lru";
 const std::string kClock = "clock";
 
-void dumbDeleter(const Slice& /*key*/, void* /*value*/) {}
+void dumbDeleter(const Slice& key, void* value) {}
 
-void eraseDeleter(const Slice& /*key*/, void* value) {
+void eraseDeleter(const Slice& key, void* value) {
   Cache* cache = reinterpret_cast<Cache*>(value);
   cache->Erase("foo");
 }
@@ -64,8 +64,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
 
   std::vector<int> deleted_keys_;
   std::vector<int> deleted_values_;
-  std::shared_ptr<Cache> cache_;
-  std::shared_ptr<Cache> cache2_;
+  shared_ptr<Cache> cache_;
+  shared_ptr<Cache> cache2_;
 
   CacheTest()
       : cache_(NewCache(kCacheSize, kNumShardBits, false)),
@@ -73,7 +73,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
     current_ = this;
   }
 
-  ~CacheTest() override {}
+  ~CacheTest() {
+  }
 
   std::shared_ptr<Cache> NewCache(size_t capacity) {
     auto type = GetParam();
@@ -86,27 +87,19 @@ class CacheTest : public testing::TestWithParam<std::string> {
     return nullptr;
   }
 
-  std::shared_ptr<Cache> NewCache(
-      size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-      CacheMetadataChargePolicy charge_policy = kDontChargeCacheMetadata) {
+  std::shared_ptr<Cache> NewCache(size_t capacity, int num_shard_bits,
+                                  bool strict_capacity_limit) {
     auto type = GetParam();
     if (type == kLRU) {
-      LRUCacheOptions co;
-      co.capacity = capacity;
-      co.num_shard_bits = num_shard_bits;
-      co.strict_capacity_limit = strict_capacity_limit;
-      co.high_pri_pool_ratio = 0;
-      co.metadata_charge_policy = charge_policy;
-      return NewLRUCache(co);
+      return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit);
     }
     if (type == kClock) {
-      return NewClockCache(capacity, num_shard_bits, strict_capacity_limit,
-                           charge_policy);
+      return NewClockCache(capacity, num_shard_bits, strict_capacity_limit);
     }
     return nullptr;
   }
 
-  int Lookup(std::shared_ptr<Cache> cache, int key) {
+  int Lookup(shared_ptr<Cache> cache, int key) {
     Cache::Handle* handle = cache->Lookup(EncodeKey(key));
     const int r = (handle == nullptr) ? -1 : DecodeValue(cache->Value(handle));
     if (handle != nullptr) {
@@ -115,16 +108,16 @@ class CacheTest : public testing::TestWithParam<std::string> {
     return r;
   }
 
-  void Insert(std::shared_ptr<Cache> cache, int key, int value,
-              int charge = 1) {
-    EXPECT_OK(cache->Insert(EncodeKey(key), EncodeValue(value), charge,
-                            &CacheTest::Deleter));
+  void Insert(shared_ptr<Cache> cache, int key, int value, int charge = 1) {
+    cache->Insert(EncodeKey(key), EncodeValue(value), charge,
+                  &CacheTest::Deleter);
   }
 
-  void Erase(std::shared_ptr<Cache> cache, int key) {
+  void Erase(shared_ptr<Cache> cache, int key) {
     cache->Erase(EncodeKey(key));
   }
 
+
   int Lookup(int key) {
     return Lookup(cache_, key);
   }
@@ -151,15 +144,10 @@ class CacheTest : public testing::TestWithParam<std::string> {
 };
 CacheTest* CacheTest::current_;
 
-class LRUCacheTest : public CacheTest {};
-
 TEST_P(CacheTest, UsageTest) {
-  // cache is std::shared_ptr and will be automatically cleaned up.
+  // cache is shared_ptr and will be automatically cleaned up.
   const uint64_t kCapacity = 100000;
-  auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
-  auto precise_cache = NewCache(kCapacity, 0, false, kFullChargeCacheMetadata);
-  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(0, precise_cache->GetUsage());
+  auto cache = NewCache(kCapacity, 8, false);
 
   size_t usage = 0;
   char value[10] = "abcdef";
@@ -167,47 +155,32 @@ TEST_P(CacheTest, UsageTest) {
   for (int i = 1; i < 100; ++i) {
     std::string key(i, 'a');
     auto kv_size = key.size() + 5;
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), kv_size, dumbDeleter);
     usage += kv_size;
     ASSERT_EQ(usage, cache->GetUsage());
-    ASSERT_LT(usage, precise_cache->GetUsage());
   }
 
-  cache->EraseUnRefEntries();
-  precise_cache->EraseUnRefEntries();
-  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(0, precise_cache->GetUsage());
-
   // make sure the cache will be overloaded
   for (uint64_t i = 1; i < kCapacity; ++i) {
     auto key = ToString(i);
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
+                  dumbDeleter);
   }
 
   // the usage should be close to the capacity
   ASSERT_GT(kCapacity, cache->GetUsage());
-  ASSERT_GT(kCapacity, precise_cache->GetUsage());
   ASSERT_LT(kCapacity * 0.95, cache->GetUsage());
-  ASSERT_LT(kCapacity * 0.95, precise_cache->GetUsage());
 }
 
 TEST_P(CacheTest, PinnedUsageTest) {
-  // cache is std::shared_ptr and will be automatically cleaned up.
-  const uint64_t kCapacity = 200000;
-  auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
-  auto precise_cache = NewCache(kCapacity, 8, false, kFullChargeCacheMetadata);
+  // cache is shared_ptr and will be automatically cleaned up.
+  const uint64_t kCapacity = 100000;
+  auto cache = NewCache(kCapacity, 8, false);
 
   size_t pinned_usage = 0;
   char value[10] = "abcdef";
 
   std::forward_list<Cache::Handle*> unreleased_handles;
-  std::forward_list<Cache::Handle*> unreleased_handles_in_precise_cache;
 
   // Add entries. Unpin some of them after insertion. Then, pin some of them
   // again. Check GetPinnedUsage().
@@ -215,73 +188,40 @@ TEST_P(CacheTest, PinnedUsageTest) {
     std::string key(i, 'a');
     auto kv_size = key.size() + 5;
     Cache::Handle* handle;
-    Cache::Handle* handle_in_precise_cache;
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            dumbDeleter, &handle));
-    assert(handle);
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, dumbDeleter,
-                                    &handle_in_precise_cache));
-    assert(handle_in_precise_cache);
+    cache->Insert(key, reinterpret_cast<void*>(value), kv_size, dumbDeleter,
+                  &handle);
     pinned_usage += kv_size;
     ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-    ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage());
     if (i % 2 == 0) {
       cache->Release(handle);
-      precise_cache->Release(handle_in_precise_cache);
       pinned_usage -= kv_size;
       ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-      ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage());
     } else {
       unreleased_handles.push_front(handle);
-      unreleased_handles_in_precise_cache.push_front(handle_in_precise_cache);
     }
     if (i % 3 == 0) {
       unreleased_handles.push_front(cache->Lookup(key));
-      auto x = precise_cache->Lookup(key);
-      assert(x);
-      unreleased_handles_in_precise_cache.push_front(x);
       // If i % 2 == 0, then the entry was unpinned before Lookup, so pinned
       // usage increased
       if (i % 2 == 0) {
         pinned_usage += kv_size;
       }
       ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-      ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage());
     }
   }
-  auto precise_cache_pinned_usage = precise_cache->GetPinnedUsage();
-  ASSERT_LT(pinned_usage, precise_cache_pinned_usage);
 
   // check that overloading the cache does not change the pinned usage
   for (uint64_t i = 1; i < 2 * kCapacity; ++i) {
     auto key = ToString(i);
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
+                  dumbDeleter);
   }
   ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-  ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage());
-
-  cache->EraseUnRefEntries();
-  precise_cache->EraseUnRefEntries();
-  ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-  ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage());
 
   // release handles for pinned entries to prevent memory leaks
   for (auto handle : unreleased_handles) {
     cache->Release(handle);
   }
-  for (auto handle : unreleased_handles_in_precise_cache) {
-    precise_cache->Release(handle);
-  }
-  ASSERT_EQ(0, cache->GetPinnedUsage());
-  ASSERT_EQ(0, precise_cache->GetPinnedUsage());
-  cache->EraseUnRefEntries();
-  precise_cache->EraseUnRefEntries();
-  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(0, precise_cache->GetUsage());
 }
 
 TEST_P(CacheTest, HitAndMiss) {
@@ -367,7 +307,7 @@ TEST_P(CacheTest, EvictionPolicy) {
   Insert(200, 201);
 
   // Frequently used entry must be kept around
-  for (int i = 0; i < kCacheSize * 2; i++) {
+  for (int i = 0; i < kCacheSize + 100; i++) {
     Insert(1000+i, 2000+i);
     ASSERT_EQ(101, Lookup(100));
   }
@@ -420,7 +360,7 @@ TEST_P(CacheTest, EvictionPolicyRef) {
   Insert(303, 104);
 
   // Insert entries much more than Cache capacity
-  for (int i = 0; i < kCacheSize * 2; i++) {
+  for (int i = 0; i < kCacheSize + 100; i++) {
     Insert(1000 + i, 2000 + i);
   }
 
@@ -530,7 +470,7 @@ class Value {
 };
 
 namespace {
-void deleter(const Slice& /*key*/, void* value) {
+void deleter(const Slice& key, void* value) {
   delete static_cast<Value *>(value);
 }
 }  // namespace
@@ -611,10 +551,10 @@ TEST_P(CacheTest, SetCapacity) {
   }
 }
 
-TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
+TEST_P(CacheTest, SetStrictCapacityLimit) {
   // test1: set the flag to false. Insert more keys than capacity. See if they
   // all go through.
-  std::shared_ptr<Cache> cache = NewCache(5, 0, false);
+  std::shared_ptr<Cache> cache = NewLRUCache(5, 0, false);
   std::vector<Cache::Handle*> handles(10);
   Status s;
   for (size_t i = 0; i < 10; i++) {
@@ -623,7 +563,6 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
     ASSERT_OK(s);
     ASSERT_NE(nullptr, handles[i]);
   }
-  ASSERT_EQ(10, cache->GetUsage());
 
   // test2: set the flag to true. Insert and check if it fails.
   std::string extra_key = "extra";
@@ -633,14 +572,13 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
   s = cache->Insert(extra_key, extra_value, 1, &deleter, &handle);
   ASSERT_TRUE(s.IsIncomplete());
   ASSERT_EQ(nullptr, handle);
-  ASSERT_EQ(10, cache->GetUsage());
 
   for (size_t i = 0; i < 10; i++) {
     cache->Release(handles[i]);
   }
 
   // test3: init with flag being true.
-  std::shared_ptr<Cache> cache2 = NewCache(5, 0, true);
+  std::shared_ptr<Cache> cache2 = NewLRUCache(5, 0, true);
   for (size_t i = 0; i < 5; i++) {
     std::string key = ToString(i + 1);
     s = cache2->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
@@ -654,7 +592,7 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
   s = cache2->Insert(extra_key, extra_value, 1, &deleter);
   // AS if the key have been inserted into cache but get evicted immediately.
   ASSERT_OK(s);
-  ASSERT_EQ(5, cache2->GetUsage());
+  ASSERT_EQ(5, cache->GetUsage());
   ASSERT_EQ(nullptr, cache2->Lookup(extra_key));
 
   for (size_t i = 0; i < 5; i++) {
@@ -712,98 +650,25 @@ TEST_P(CacheTest, OverCapacity) {
 }
 
 namespace {
-std::vector<std::pair<int, int>> legacy_callback_state;
-void legacy_callback(void* value, size_t charge) {
-  legacy_callback_state.push_back(
-      {DecodeValue(value), static_cast<int>(charge)});
+std::vector<std::pair<int, int>> callback_state;
+void callback(void* entry, size_t charge) {
+  callback_state.push_back({DecodeValue(entry), static_cast<int>(charge)});
 }
 };
 
-TEST_P(CacheTest, ApplyToAllCacheEntriesTest) {
+TEST_P(CacheTest, ApplyToAllCacheEntiresTest) {
   std::vector<std::pair<int, int>> inserted;
-  legacy_callback_state.clear();
+  callback_state.clear();
 
   for (int i = 0; i < 10; ++i) {
     Insert(i, i * 2, i + 1);
     inserted.push_back({i * 2, i + 1});
   }
-  cache_->ApplyToAllCacheEntries(legacy_callback, true);
-
-  std::sort(inserted.begin(), inserted.end());
-  std::sort(legacy_callback_state.begin(), legacy_callback_state.end());
-  ASSERT_EQ(inserted.size(), legacy_callback_state.size());
-  for (size_t i = 0; i < inserted.size(); ++i) {
-    EXPECT_EQ(inserted[i], legacy_callback_state[i]);
-  }
-}
-
-TEST_P(CacheTest, ApplyToAllEntriesTest) {
-  std::vector<std::string> callback_state;
-  const auto callback = [&](const Slice& key, void* value, size_t charge,
-                            Cache::DeleterFn deleter) {
-    callback_state.push_back(ToString(DecodeKey(key)) + "," +
-                             ToString(DecodeValue(value)) + "," +
-                             ToString(charge));
-    assert(deleter == &CacheTest::Deleter);
-  };
-
-  std::vector<std::string> inserted;
-  callback_state.clear();
-
-  for (int i = 0; i < 10; ++i) {
-    Insert(i, i * 2, i + 1);
-    inserted.push_back(ToString(i) + "," + ToString(i * 2) + "," +
-                       ToString(i + 1));
-  }
-  cache_->ApplyToAllEntries(callback, /*opts*/ {});
+  cache_->ApplyToAllCacheEntries(callback, true);
 
   std::sort(inserted.begin(), inserted.end());
   std::sort(callback_state.begin(), callback_state.end());
-  ASSERT_EQ(inserted.size(), callback_state.size());
-  for (size_t i = 0; i < inserted.size(); ++i) {
-    EXPECT_EQ(inserted[i], callback_state[i]);
-  }
-}
-
-TEST_P(CacheTest, ApplyToAllEntriesDuringResize) {
-  // This is a mini-stress test of ApplyToAllEntries, to ensure
-  // items in the cache that are neither added nor removed
-  // during ApplyToAllEntries are counted exactly once.
-
-  // Insert some entries that we expect to be seen exactly once
-  // during iteration.
-  constexpr int kSpecialCharge = 2;
-  constexpr int kNotSpecialCharge = 1;
-  constexpr int kSpecialCount = 100;
-  for (int i = 0; i < kSpecialCount; ++i) {
-    Insert(i, i * 2, kSpecialCharge);
-  }
-
-  // For callback
-  int special_count = 0;
-  const auto callback = [&](const Slice&, void*, size_t charge,
-                            Cache::DeleterFn) {
-    if (charge == static_cast<size_t>(kSpecialCharge)) {
-      ++special_count;
-    }
-  };
-
-  // Start counting
-  std::thread apply_thread([&]() {
-    // Use small average_entries_per_lock to make the problem difficult
-    Cache::ApplyToAllEntriesOptions opts;
-    opts.average_entries_per_lock = 2;
-    cache_->ApplyToAllEntries(callback, opts);
-  });
-
-  // In parallel, add more entries, enough to cause resize but not enough
-  // to cause ejections
-  for (int i = kSpecialCount * 1; i < kSpecialCount * 6; ++i) {
-    Insert(i, i * 2, kNotSpecialCharge);
-  }
-
-  apply_thread.join();
-  ASSERT_EQ(special_count, kSpecialCount);
+  ASSERT_TRUE(inserted == callback_state);
 }
 
 TEST_P(CacheTest, DefaultShardBits) {
@@ -822,26 +687,15 @@ TEST_P(CacheTest, DefaultShardBits) {
   ASSERT_EQ(6, sc->GetNumShardBits());
 }
 
-TEST_P(CacheTest, GetChargeAndDeleter) {
-  Insert(1, 2);
-  Cache::Handle* h1 = cache_->Lookup(EncodeKey(1));
-  ASSERT_EQ(2, DecodeValue(cache_->Value(h1)));
-  ASSERT_EQ(1, cache_->GetCharge(h1));
-  ASSERT_EQ(&CacheTest::Deleter, cache_->GetDeleter(h1));
-  cache_->Release(h1);
-}
-
 #ifdef SUPPORT_CLOCK_CACHE
-std::shared_ptr<Cache> (*new_clock_cache_func)(
-    size_t, int, bool, CacheMetadataChargePolicy) = NewClockCache;
+shared_ptr<Cache> (*new_clock_cache_func)(size_t, int, bool) = NewClockCache;
 INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
                         testing::Values(kLRU, kClock));
 #else
 INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest, testing::Values(kLRU));
 #endif  // SUPPORT_CLOCK_CACHE
-INSTANTIATE_TEST_CASE_P(CacheTestInstance, LRUCacheTest, testing::Values(kLRU));
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);

cache/clock_cache.cc

@@ -11,16 +11,15 @@
 
 #ifndef SUPPORT_CLOCK_CACHE
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
-std::shared_ptr<Cache> NewClockCache(
-    size_t /*capacity*/, int /*num_shard_bits*/, bool /*strict_capacity_limit*/,
-    CacheMetadataChargePolicy /*metadata_charge_policy*/) {
+std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
+                                     bool strict_capacity_limit) {
   // Clock cache not supported.
   return nullptr;
 }
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb
 
 #else
 
@@ -28,20 +27,14 @@ std::shared_ptr<Cache> NewClockCache(
 #include <atomic>
 #include <deque>
 
-// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
-// Disable it so users can chooose to disable RTTI.
-#ifndef ROCKSDB_USE_RTTI
-#define TBB_USE_EXCEPTIONS 0
-#endif
 #include "tbb/concurrent_hash_map.h"
 
 #include "cache/sharded_cache.h"
-#include "port/malloc.h"
 #include "port/port.h"
 #include "util/autovector.h"
 #include "util/mutexlock.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 namespace {
 
@@ -176,16 +169,13 @@ namespace {
 // Cache entry meta data.
 struct CacheHandle {
   Slice key;
+  uint32_t hash;
   void* value;
   size_t charge;
-  Cache::DeleterFn deleter;
-  uint32_t hash;
-
-  // Addition to "charge" to get "total charge" under metadata policy.
-  uint32_t meta_charge;
+  void (*deleter)(const Slice&, void* value);
 
   // Flags and counters associated with the cache handle:
-  //   lowest bit: in-cache bit
+  //   lowest bit: n-cache bit
   //   second lowest bit: usage bit
   //   the rest bits: reference count
   // The handle is unused when flags equals to 0. The thread decreases the count
@@ -207,24 +197,6 @@ struct CacheHandle {
     deleter = a.deleter;
     return *this;
   }
-
-  inline static uint32_t CalcMetadataCharge(
-      Slice key, CacheMetadataChargePolicy metadata_charge_policy) {
-    size_t meta_charge = 0;
-    if (metadata_charge_policy == kFullChargeCacheMetadata) {
-      meta_charge += sizeof(CacheHandle);
-#ifdef ROCKSDB_MALLOC_USABLE_SIZE
-      meta_charge +=
-          malloc_usable_size(static_cast<void*>(const_cast<char*>(key.data())));
-#else
-      meta_charge += key.size();
-#endif
-    }
-    assert(meta_charge <= UINT32_MAX);
-    return static_cast<uint32_t>(meta_charge);
-  }
-
-  inline size_t GetTotalCharge() { return charge + meta_charge; }
 };
 
 // Key of hash map. We store hash value with the key for convenience.
@@ -257,56 +229,38 @@ struct CleanupContext {
 };
 
 // A cache shard which maintains its own CLOCK cache.
-class ClockCacheShard final : public CacheShard {
+class ClockCacheShard : public CacheShard {
  public:
   // Hash map type.
   typedef tbb::concurrent_hash_map<CacheKey, CacheHandle*, CacheKey> HashTable;
 
   ClockCacheShard();
-  ~ClockCacheShard() override;
+  ~ClockCacheShard();
 
   // Interfaces
-  void SetCapacity(size_t capacity) override;
-  void SetStrictCapacityLimit(bool strict_capacity_limit) override;
-  Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
-                void (*deleter)(const Slice& key, void* value),
-                Cache::Handle** handle, Cache::Priority priority) override;
-  Status Insert(const Slice& key, uint32_t hash, void* value,
-                const Cache::CacheItemHelper* helper, size_t charge,
-                Cache::Handle** handle, Cache::Priority priority) override {
-    return Insert(key, hash, value, charge, helper->del_cb, handle, priority);
-  }
-  Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
-  Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                        const Cache::CacheItemHelper* /*helper*/,
-                        const Cache::CreateCallback& /*create_cb*/,
-                        Cache::Priority /*priority*/, bool /*wait*/,
-                        Statistics* /*stats*/) override {
-    return Lookup(key, hash);
-  }
-  bool Release(Cache::Handle* handle, bool /*useful*/,
-               bool force_erase) override {
-    return Release(handle, force_erase);
-  }
-  bool IsReady(Cache::Handle* /*handle*/) override { return true; }
-  void Wait(Cache::Handle* /*handle*/) override {}
-
+  virtual void SetCapacity(size_t capacity) override;
+  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
+  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
+                        size_t charge,
+                        void (*deleter)(const Slice& key, void* value),
+                        Cache::Handle** handle,
+                        Cache::Priority priority) override;
+  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
   // If the entry in in cache, increase reference count and return true.
   // Return false otherwise.
   //
   // Not necessary to hold mutex_ before being called.
-  bool Ref(Cache::Handle* handle) override;
-  bool Release(Cache::Handle* handle, bool force_erase = false) override;
-  void Erase(const Slice& key, uint32_t hash) override;
+  virtual bool Ref(Cache::Handle* handle) override;
+  virtual bool Release(Cache::Handle* handle,
+                       bool force_erase = false) override;
+  virtual void Erase(const Slice& key, uint32_t hash) override;
   bool EraseAndConfirm(const Slice& key, uint32_t hash,
                        CleanupContext* context);
-  size_t GetUsage() const override;
-  size_t GetPinnedUsage() const override;
-  void EraseUnRefEntries() override;
-  void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
-      uint32_t average_entries_per_lock, uint32_t* state) override;
+  virtual size_t GetUsage() const override;
+  virtual size_t GetPinnedUsage() const override;
+  virtual void EraseUnRefEntries() override;
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) override;
 
  private:
   static const uint32_t kInCacheBit = 1;
@@ -362,8 +316,7 @@ class ClockCacheShard final : public CacheShard {
   CacheHandle* Insert(const Slice& key, uint32_t hash, void* value,
                       size_t change,
                       void (*deleter)(const Slice& key, void* value),
-                      bool hold_reference, CleanupContext* context,
-                      bool* overwritten);
+                      bool hold_reference, CleanupContext* context);
 
   // Guards list_, head_, and recycle_. In addition, updating table_ also has
   // to hold the mutex, to avoid the cache being in inconsistent state.
@@ -409,9 +362,7 @@ ClockCacheShard::~ClockCacheShard() {
   for (auto& handle : list_) {
     uint32_t flags = handle.flags.load(std::memory_order_relaxed);
     if (InCache(flags) || CountRefs(flags) > 0) {
-      if (handle.deleter != nullptr) {
-        (*handle.deleter)(handle.key, handle.value);
-      }
+      (*handle.deleter)(handle.key, handle.value);
       delete[] handle.key.data();
     }
   }
@@ -425,46 +376,22 @@ size_t ClockCacheShard::GetPinnedUsage() const {
   return pinned_usage_.load(std::memory_order_relaxed);
 }
 
-void ClockCacheShard::ApplyToSomeEntries(
-    const std::function<void(const Slice& key, void* value, size_t charge,
-                             DeleterFn deleter)>& callback,
-    uint32_t average_entries_per_lock, uint32_t* state) {
-  assert(average_entries_per_lock > 0);
-  MutexLock lock(&mutex_);
-
-  // Figure out the range to iterate, update `state`
-  size_t list_size = list_.size();
-  size_t start_idx = *state;
-  size_t end_idx = start_idx + average_entries_per_lock;
-  if (start_idx > list_size) {
-    // Shouldn't reach here, but recoverable
-    assert(false);
-    // Mark finished with all
-    *state = UINT32_MAX;
-    return;
+void ClockCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                             bool thread_safe) {
+  if (thread_safe) {
+    mutex_.Lock();
   }
-  if (end_idx >= list_size || end_idx >= UINT32_MAX) {
-    // This also includes the hypothetical case of >4 billion
-    // cache handles.
-    end_idx = list_size;
-    // Mark finished with all
-    *state = UINT32_MAX;
-  } else {
-    *state = static_cast<uint32_t>(end_idx);
-  }
-
-  // Do the iteration
-  auto cur = list_.begin() + start_idx;
-  auto end = list_.begin() + end_idx;
-  for (; cur != end; ++cur) {
-    const CacheHandle& handle = *cur;
-    // Use relaxed semantics instead of acquire semantics since we are
-    // holding mutex
+  for (auto& handle : list_) {
+    // Use relaxed semantics instead of acquire semantics since we are either
+    // holding mutex, or don't have thread safe requirement.
     uint32_t flags = handle.flags.load(std::memory_order_relaxed);
     if (InCache(flags)) {
-      callback(handle.key, handle.value, handle.charge, handle.deleter);
+      callback(handle.value, handle.charge);
     }
   }
+  if (thread_safe) {
+    mutex_.Unlock();
+  }
 }
 
 void ClockCacheShard::RecycleHandle(CacheHandle* handle,
@@ -473,10 +400,11 @@ void ClockCacheShard::RecycleHandle(CacheHandle* handle,
   assert(!InCache(handle->flags) && CountRefs(handle->flags) == 0);
   context->to_delete_key.push_back(handle->key.data());
   context->to_delete_value.emplace_back(*handle);
-  size_t total_charge = handle->GetTotalCharge();
-  // clearing `handle` fields would go here but not strictly required
+  handle->key.clear();
+  handle->value = nullptr;
+  handle->deleter = nullptr;
   recycle_.push_back(handle);
-  usage_.fetch_sub(total_charge, std::memory_order_relaxed);
+  usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
 }
 
 void ClockCacheShard::Cleanup(const CleanupContext& context) {
@@ -502,8 +430,7 @@ bool ClockCacheShard::Ref(Cache::Handle* h) {
                                             std::memory_order_relaxed)) {
       if (CountRefs(flags) == 0) {
         // No reference count before the operation.
-        size_t total_charge = handle->GetTotalCharge();
-        pinned_usage_.fetch_add(total_charge, std::memory_order_relaxed);
+        pinned_usage_.fetch_add(handle->charge, std::memory_order_relaxed);
       }
       return true;
     }
@@ -516,11 +443,6 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
   if (set_usage) {
     handle->flags.fetch_or(kUsageBit, std::memory_order_relaxed);
   }
-  // If the handle reaches state refs=0 and InCache=true after this
-  // atomic operation then we cannot access `handle` afterward, because
-  // it could be evicted before we access the `handle`.
-  size_t total_charge = handle->GetTotalCharge();
-
   // Use acquire-release semantics as previous operations on the cache entry
   // has to be order before reference count is decreased, and potential cleanup
   // of the entry has to be order after.
@@ -528,7 +450,7 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
   assert(CountRefs(flags) > 0);
   if (CountRefs(flags) == 1) {
     // this is the last reference.
-    pinned_usage_.fetch_sub(total_charge, std::memory_order_relaxed);
+    pinned_usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
     // Cleanup if it is the last reference.
     if (!InCache(flags)) {
       MutexLock l(&mutex_);
@@ -612,13 +534,9 @@ void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
 CacheHandle* ClockCacheShard::Insert(
     const Slice& key, uint32_t hash, void* value, size_t charge,
     void (*deleter)(const Slice& key, void* value), bool hold_reference,
-    CleanupContext* context, bool* overwritten) {
-  assert(overwritten != nullptr && *overwritten == false);
-  uint32_t meta_charge =
-      CacheHandle::CalcMetadataCharge(key, metadata_charge_policy_);
-  size_t total_charge = charge + meta_charge;
+    CleanupContext* context) {
   MutexLock l(&mutex_);
-  bool success = EvictFromCache(total_charge, context);
+  bool success = EvictFromCache(charge, context);
   bool strict = strict_capacity_limit_.load(std::memory_order_relaxed);
   if (!success && (strict || !hold_reference)) {
     context->to_delete_key.push_back(key.data());
@@ -642,31 +560,20 @@ CacheHandle* ClockCacheShard::Insert(
   handle->hash = hash;
   handle->value = value;
   handle->charge = charge;
-  handle->meta_charge = meta_charge;
   handle->deleter = deleter;
   uint32_t flags = hold_reference ? kInCacheBit + kOneRef : kInCacheBit;
-
-  // TODO investigate+fix suspected race condition:
-  // [thread 1] Lookup starts, up to Ref()
-  // [thread 2] Erase/evict the entry just looked up
-  // [thread 1] Ref() the handle, even though it's in the recycle bin
-  // [thread 2] Insert with recycling that handle
-  // Here we obliterate the other thread's Ref
-  // Possible fix: never blindly overwrite the flags, but only make
-  // relative updates (fetch_add, etc).
   handle->flags.store(flags, std::memory_order_relaxed);
   HashTable::accessor accessor;
   if (table_.find(accessor, CacheKey(key, hash))) {
-    *overwritten = true;
     CacheHandle* existing_handle = accessor->second;
     table_.erase(accessor);
     UnsetInCache(existing_handle, context);
   }
   table_.insert(HashTable::value_type(CacheKey(key, hash), handle));
   if (hold_reference) {
-    pinned_usage_.fetch_add(total_charge, std::memory_order_relaxed);
+    pinned_usage_.fetch_add(charge, std::memory_order_relaxed);
   }
-  usage_.fetch_add(total_charge, std::memory_order_relaxed);
+  usage_.fetch_add(charge, std::memory_order_relaxed);
   return handle;
 }
 
@@ -674,15 +581,14 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
                                size_t charge,
                                void (*deleter)(const Slice& key, void* value),
                                Cache::Handle** out_handle,
-                               Cache::Priority /*priority*/) {
+                               Cache::Priority priority) {
   CleanupContext context;
   HashTable::accessor accessor;
   char* key_data = new char[key.size()];
   memcpy(key_data, key.data(), key.size());
   Slice key_copy(key_data, key.size());
-  bool overwritten = false;
   CacheHandle* handle = Insert(key_copy, hash, value, charge, deleter,
-                               out_handle != nullptr, &context, &overwritten);
+                               out_handle != nullptr, &context);
   Status s;
   if (out_handle != nullptr) {
     if (handle == nullptr) {
@@ -691,10 +597,6 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
       *out_handle = reinterpret_cast<Cache::Handle*>(handle);
     }
   }
-  if (overwritten) {
-    assert(s.ok());
-    s = Status::OkOverwritten();
-  }
   Cleanup(context);
   return s;
 }
@@ -766,55 +668,41 @@ void ClockCacheShard::EraseUnRefEntries() {
   Cleanup(context);
 }
 
-class ClockCache final : public ShardedCache {
+class ClockCache : public ShardedCache {
  public:
-  ClockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-             CacheMetadataChargePolicy metadata_charge_policy)
+  ClockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit)
       : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
     int num_shards = 1 << num_shard_bits;
     shards_ = new ClockCacheShard[num_shards];
-    for (int i = 0; i < num_shards; i++) {
-      shards_[i].set_metadata_charge_policy(metadata_charge_policy);
-    }
     SetCapacity(capacity);
     SetStrictCapacityLimit(strict_capacity_limit);
   }
 
-  ~ClockCache() override { delete[] shards_; }
+  virtual ~ClockCache() { delete[] shards_; }
 
-  const char* Name() const override { return "ClockCache"; }
+  virtual const char* Name() const override { return "ClockCache"; }
 
-  CacheShard* GetShard(uint32_t shard) override {
+  virtual CacheShard* GetShard(int shard) override {
     return reinterpret_cast<CacheShard*>(&shards_[shard]);
   }
 
-  const CacheShard* GetShard(uint32_t shard) const override {
+  virtual const CacheShard* GetShard(int shard) const override {
     return reinterpret_cast<CacheShard*>(&shards_[shard]);
   }
 
-  void* Value(Handle* handle) override {
+  virtual void* Value(Handle* handle) override {
     return reinterpret_cast<const CacheHandle*>(handle)->value;
   }
 
-  size_t GetCharge(Handle* handle) const override {
+  virtual size_t GetCharge(Handle* handle) const override {
     return reinterpret_cast<const CacheHandle*>(handle)->charge;
   }
 
-  uint32_t GetHash(Handle* handle) const override {
+  virtual uint32_t GetHash(Handle* handle) const override {
     return reinterpret_cast<const CacheHandle*>(handle)->hash;
   }
 
-  DeleterFn GetDeleter(Handle* handle) const override {
-    return reinterpret_cast<const CacheHandle*>(handle)->deleter;
-  }
-
-  void DisownData() override {
-#ifndef MUST_FREE_HEAP_ALLOCATIONS
-    shards_ = nullptr;
-#endif
-  }
-
-  void WaitAll(std::vector<Handle*>& /*handles*/) override {}
+  virtual void DisownData() override { shards_ = nullptr; }
 
  private:
   ClockCacheShard* shards_;
@@ -822,16 +710,15 @@ class ClockCache final : public ShardedCache {
 
 }  // end anonymous namespace
 
-std::shared_ptr<Cache> NewClockCache(
-    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-    CacheMetadataChargePolicy metadata_charge_policy) {
+std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
+                                     bool strict_capacity_limit) {
   if (num_shard_bits < 0) {
     num_shard_bits = GetDefaultCacheShardBits(capacity);
   }
-  return std::make_shared<ClockCache>(
-      capacity, num_shard_bits, strict_capacity_limit, metadata_charge_policy);
+  return std::make_shared<ClockCache>(capacity, num_shard_bits,
+                                      strict_capacity_limit);
 }
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb
 
 #endif  // SUPPORT_CLOCK_CACHE

cache/lru_cache.cc

@@ -7,32 +7,32 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
 #include "cache/lru_cache.h"
 
-#include <cassert>
-#include <cstdint>
-#include <cstdio>
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
 
-#include "monitoring/perf_context_imp.h"
-#include "monitoring/statistics.h"
 #include "util/mutexlock.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
-LRUHandleTable::LRUHandleTable(int max_upper_hash_bits)
-    : length_bits_(/* historical starting size*/ 4),
-      list_(new LRUHandle* [size_t{1} << length_bits_] {}),
-      elems_(0),
-      max_length_bits_(max_upper_hash_bits) {}
+LRUHandleTable::LRUHandleTable() : length_(0), elems_(0), list_(nullptr) {
+  Resize();
+}
 
 LRUHandleTable::~LRUHandleTable() {
-  ApplyToEntriesRange(
-      [](LRUHandle* h) {
-        if (!h->HasRefs()) {
-          h->Free();
-        }
-      },
-      0, uint32_t{1} << length_bits_);
+  ApplyToAllCacheEntries([](LRUHandle* h) {
+    if (h->refs == 1) {
+      h->Free();
+    }
+  });
+  delete[] list_;
 }
 
 LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) {
@@ -46,7 +46,7 @@ LRUHandle* LRUHandleTable::Insert(LRUHandle* h) {
   *ptr = h;
   if (old == nullptr) {
     ++elems_;
-    if ((elems_ >> length_bits_) > 0) {  // elems_ >= length
+    if (elems_ > length_) {
       // Since each cache entry is fairly large, we aim for a small
       // average linked list length (<= 1).
       Resize();
@@ -66,7 +66,7 @@ LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) {
 }
 
 LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
-  LRUHandle** ptr = &list_[hash >> (32 - length_bits_)];
+  LRUHandle** ptr = &list_[hash & (length_ - 1)];
   while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
     ptr = &(*ptr)->next_hash;
   }
@@ -74,29 +74,19 @@ LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
 }
 
 void LRUHandleTable::Resize() {
-  if (length_bits_ >= max_length_bits_) {
-    // Due to reaching limit of hash information, if we made the table
-    // bigger, we would allocate more addresses but only the same
-    // number would be used.
-    return;
+  uint32_t new_length = 16;
+  while (new_length < elems_ * 1.5) {
+    new_length *= 2;
   }
-  if (length_bits_ >= 31) {
-    // Avoid undefined behavior shifting uint32_t by 32
-    return;
-  }
-
-  uint32_t old_length = uint32_t{1} << length_bits_;
-  int new_length_bits = length_bits_ + 1;
-  std::unique_ptr<LRUHandle* []> new_list {
-    new LRUHandle* [size_t{1} << new_length_bits] {}
-  };
+  LRUHandle** new_list = new LRUHandle*[new_length];
+  memset(new_list, 0, sizeof(new_list[0]) * new_length);
   uint32_t count = 0;
-  for (uint32_t i = 0; i < old_length; i++) {
+  for (uint32_t i = 0; i < length_; i++) {
     LRUHandle* h = list_[i];
     while (h != nullptr) {
       LRUHandle* next = h->next_hash;
       uint32_t hash = h->hash;
-      LRUHandle** ptr = &new_list[hash >> (32 - new_length_bits)];
+      LRUHandle** ptr = &new_list[hash & (new_length - 1)];
       h->next_hash = *ptr;
       *ptr = h;
       h = next;
@@ -104,47 +94,43 @@ void LRUHandleTable::Resize() {
     }
   }
   assert(elems_ == count);
-  list_ = std::move(new_list);
-  length_bits_ = new_length_bits;
+  delete[] list_;
+  list_ = new_list;
+  length_ = new_length;
 }
 
-LRUCacheShard::LRUCacheShard(
-    size_t capacity, bool strict_capacity_limit, double high_pri_pool_ratio,
-    bool use_adaptive_mutex, CacheMetadataChargePolicy metadata_charge_policy,
-    int max_upper_hash_bits,
-    const std::shared_ptr<SecondaryCache>& secondary_cache)
-    : capacity_(0),
-      high_pri_pool_usage_(0),
-      strict_capacity_limit_(strict_capacity_limit),
-      high_pri_pool_ratio_(high_pri_pool_ratio),
-      high_pri_pool_capacity_(0),
-      table_(max_upper_hash_bits),
-      usage_(0),
-      lru_usage_(0),
-      mutex_(use_adaptive_mutex),
-      secondary_cache_(secondary_cache) {
-  set_metadata_charge_policy(metadata_charge_policy);
+LRUCacheShard::LRUCacheShard()
+    : usage_(0), lru_usage_(0), high_pri_pool_usage_(0) {
   // Make empty circular linked list
   lru_.next = &lru_;
   lru_.prev = &lru_;
   lru_low_pri_ = &lru_;
-  SetCapacity(capacity);
 }
 
+LRUCacheShard::~LRUCacheShard() {}
+
+bool LRUCacheShard::Unref(LRUHandle* e) {
+  assert(e->refs > 0);
+  e->refs--;
+  return e->refs == 0;
+}
+
+// Call deleter and free
+
 void LRUCacheShard::EraseUnRefEntries() {
   autovector<LRUHandle*> last_reference_list;
   {
     MutexLock l(&mutex_);
     while (lru_.next != &lru_) {
       LRUHandle* old = lru_.next;
-      // LRU list contains only elements which can be evicted
-      assert(old->InCache() && !old->HasRefs());
+      assert(old->InCache());
+      assert(old->refs ==
+             1);  // LRU list contains elements which may be evicted
       LRU_Remove(old);
       table_.Remove(old->key(), old->hash);
       old->SetInCache(false);
-      size_t total_charge = old->CalcTotalCharge(metadata_charge_policy_);
-      assert(usage_ >= total_charge);
-      usage_ -= total_charge;
+      Unref(old);
+      usage_ -= old->charge;
       last_reference_list.push_back(old);
     }
   }
@@ -154,64 +140,23 @@ void LRUCacheShard::EraseUnRefEntries() {
   }
 }
 
-void LRUCacheShard::ApplyToSomeEntries(
-    const std::function<void(const Slice& key, void* value, size_t charge,
-                             DeleterFn deleter)>& callback,
-    uint32_t average_entries_per_lock, uint32_t* state) {
-  // The state is essentially going to be the starting hash, which works
-  // nicely even if we resize between calls because we use upper-most
-  // hash bits for table indexes.
-  MutexLock l(&mutex_);
-  uint32_t length_bits = table_.GetLengthBits();
-  uint32_t length = uint32_t{1} << length_bits;
-
-  assert(average_entries_per_lock > 0);
-  // Assuming we are called with same average_entries_per_lock repeatedly,
-  // this simplifies some logic (index_end will not overflow)
-  assert(average_entries_per_lock < length || *state == 0);
-
-  uint32_t index_begin = *state >> (32 - length_bits);
-  uint32_t index_end = index_begin + average_entries_per_lock;
-  if (index_end >= length) {
-    // Going to end
-    index_end = length;
-    *state = UINT32_MAX;
-  } else {
-    *state = index_end << (32 - length_bits);
+void LRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                           bool thread_safe) {
+  if (thread_safe) {
+    mutex_.Lock();
+  }
+  table_.ApplyToAllCacheEntries(
+      [callback](LRUHandle* h) { callback(h->value, h->charge); });
+  if (thread_safe) {
+    mutex_.Unlock();
   }
-
-  table_.ApplyToEntriesRange(
-      [callback](LRUHandle* h) {
-        DeleterFn deleter = h->IsSecondaryCacheCompatible()
-                                ? h->info_.helper->del_cb
-                                : h->info_.deleter;
-        callback(h->key(), h->value, h->charge, deleter);
-      },
-      index_begin, index_end);
 }
 
 void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
-  MutexLock l(&mutex_);
   *lru = &lru_;
   *lru_low_pri = lru_low_pri_;
 }
 
-size_t LRUCacheShard::TEST_GetLRUSize() {
-  MutexLock l(&mutex_);
-  LRUHandle* lru_handle = lru_.next;
-  size_t lru_size = 0;
-  while (lru_handle != &lru_) {
-    lru_size++;
-    lru_handle = lru_handle->next;
-  }
-  return lru_size;
-}
-
-double LRUCacheShard::GetHighPriPoolRatio() {
-  MutexLock l(&mutex_);
-  return high_pri_pool_ratio_;
-}
-
 void LRUCacheShard::LRU_Remove(LRUHandle* e) {
   assert(e->next != nullptr);
   assert(e->prev != nullptr);
@@ -221,27 +166,24 @@ void LRUCacheShard::LRU_Remove(LRUHandle* e) {
   e->next->prev = e->prev;
   e->prev->next = e->next;
   e->prev = e->next = nullptr;
-  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-  assert(lru_usage_ >= total_charge);
-  lru_usage_ -= total_charge;
+  lru_usage_ -= e->charge;
   if (e->InHighPriPool()) {
-    assert(high_pri_pool_usage_ >= total_charge);
-    high_pri_pool_usage_ -= total_charge;
+    assert(high_pri_pool_usage_ >= e->charge);
+    high_pri_pool_usage_ -= e->charge;
   }
 }
 
 void LRUCacheShard::LRU_Insert(LRUHandle* e) {
   assert(e->next == nullptr);
   assert(e->prev == nullptr);
-  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-  if (high_pri_pool_ratio_ > 0 && (e->IsHighPri() || e->HasHit())) {
+  if (high_pri_pool_ratio_ > 0 && e->IsHighPri()) {
     // Inset "e" to head of LRU list.
     e->next = &lru_;
     e->prev = lru_.prev;
     e->prev->next = e;
     e->next->prev = e;
     e->SetInHighPriPool(true);
-    high_pri_pool_usage_ += total_charge;
+    high_pri_pool_usage_ += e->charge;
     MaintainPoolSize();
   } else {
     // Insert "e" to the head of low-pri pool. Note that when
@@ -253,7 +195,7 @@ void LRUCacheShard::LRU_Insert(LRUHandle* e) {
     e->SetInHighPriPool(false);
     lru_low_pri_ = e;
   }
-  lru_usage_ += total_charge;
+  lru_usage_ += e->charge;
 }
 
 void LRUCacheShard::MaintainPoolSize() {
@@ -262,25 +204,21 @@ void LRUCacheShard::MaintainPoolSize() {
     lru_low_pri_ = lru_low_pri_->next;
     assert(lru_low_pri_ != &lru_);
     lru_low_pri_->SetInHighPriPool(false);
-    size_t total_charge =
-        lru_low_pri_->CalcTotalCharge(metadata_charge_policy_);
-    assert(high_pri_pool_usage_ >= total_charge);
-    high_pri_pool_usage_ -= total_charge;
+    high_pri_pool_usage_ -= lru_low_pri_->charge;
   }
 }
 
 void LRUCacheShard::EvictFromLRU(size_t charge,
                                  autovector<LRUHandle*>* deleted) {
-  while ((usage_ + charge) > capacity_ && lru_.next != &lru_) {
+  while (usage_ + charge > capacity_ && lru_.next != &lru_) {
     LRUHandle* old = lru_.next;
-    // LRU list contains only elements which can be evicted
-    assert(old->InCache() && !old->HasRefs());
+    assert(old->InCache());
+    assert(old->refs == 1);  // LRU list contains elements which may be evicted
     LRU_Remove(old);
     table_.Remove(old->key(), old->hash);
     old->SetInCache(false);
-    size_t old_total_charge = old->CalcTotalCharge(metadata_charge_policy_);
-    assert(usage_ >= old_total_charge);
-    usage_ -= old_total_charge;
+    Unref(old);
+    usage_ -= old->charge;
     deleted->push_back(old);
   }
 }
@@ -293,15 +231,9 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
     high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
     EvictFromLRU(0, &last_reference_list);
   }
-
-  // Try to insert the evicted entries into tiered cache
-  // Free the entries outside of mutex for performance reasons
+  // we free the entries here outside of mutex for
+  // performance reasons
   for (auto entry : last_reference_list) {
-    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
-        !entry->IsPromoted()) {
-      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
-          .PermitUncheckedError();
-    }
     entry->Free();
   }
 }
@@ -311,191 +243,26 @@ void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
   strict_capacity_limit_ = strict_capacity_limit;
 }
 
-Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
-                                 bool free_handle_on_fail) {
-  Status s = Status::OK();
-  autovector<LRUHandle*> last_reference_list;
-  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-
-  {
-    MutexLock l(&mutex_);
-
-    // Free the space following strict LRU policy until enough space
-    // is freed or the lru list is empty
-    EvictFromLRU(total_charge, &last_reference_list);
-
-    if ((usage_ + total_charge) > capacity_ &&
-        (strict_capacity_limit_ || handle == nullptr)) {
-      e->SetInCache(false);
-      if (handle == nullptr) {
-        // Don't insert the entry but still return ok, as if the entry inserted
-        // into cache and get evicted immediately.
-        last_reference_list.push_back(e);
-      } else {
-        if (free_handle_on_fail) {
-          delete[] reinterpret_cast<char*>(e);
-          *handle = nullptr;
-        }
-        s = Status::Incomplete("Insert failed due to LRU cache being full.");
-      }
-    } else {
-      // Insert into the cache. Note that the cache might get larger than its
-      // capacity if not enough space was freed up.
-      LRUHandle* old = table_.Insert(e);
-      usage_ += total_charge;
-      if (old != nullptr) {
-        s = Status::OkOverwritten();
-        assert(old->InCache());
-        old->SetInCache(false);
-        if (!old->HasRefs()) {
-          // old is on LRU because it's in cache and its reference count is 0
-          LRU_Remove(old);
-          size_t old_total_charge =
-              old->CalcTotalCharge(metadata_charge_policy_);
-          assert(usage_ >= old_total_charge);
-          usage_ -= old_total_charge;
-          last_reference_list.push_back(old);
-        }
-      }
-      if (handle == nullptr) {
-        LRU_Insert(e);
-      } else {
-        // If caller already holds a ref, no need to take one here
-        if (!e->HasRefs()) {
-          e->Ref();
-        }
-        *handle = reinterpret_cast<Cache::Handle*>(e);
-      }
-    }
-  }
-
-  // Try to insert the evicted entries into the secondary cache
-  // Free the entries here outside of mutex for performance reasons
-  for (auto entry : last_reference_list) {
-    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
-        !entry->IsPromoted()) {
-      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
-          .PermitUncheckedError();
-    }
-    entry->Free();
-  }
-
-  return s;
-}
-
-void LRUCacheShard::Promote(LRUHandle* e) {
-  SecondaryCacheResultHandle* secondary_handle = e->sec_handle;
-
-  assert(secondary_handle->IsReady());
-  e->SetIncomplete(false);
-  e->SetInCache(true);
-  e->SetPromoted(true);
-  e->value = secondary_handle->Value();
-  e->charge = secondary_handle->Size();
-  delete secondary_handle;
-
-  // This call could fail if the cache is over capacity and
-  // strict_capacity_limit_ is true. In such a case, we don't want
-  // InsertItem() to free the handle, since the item is already in memory
-  // and the caller will most likely just read from disk if we erase it here.
-  if (e->value) {
-    Cache::Handle* handle = reinterpret_cast<Cache::Handle*>(e);
-    Status s = InsertItem(e, &handle, /*free_handle_on_fail=*/false);
-    if (!s.ok()) {
-      // Item is in memory, but not accounted against the cache capacity.
-      // When the handle is released, the item should get deleted
-      assert(!e->InCache());
-    }
-  } else {
-    // Since the secondary cache lookup failed, mark the item as not in cache
-    // Don't charge the cache as its only metadata that'll shortly be released
-    MutexLock l(&mutex_);
-    e->charge = 0;
-    e->SetInCache(false);
-  }
-}
-
-Cache::Handle* LRUCacheShard::Lookup(
-    const Slice& key, uint32_t hash,
-    const ShardedCache::CacheItemHelper* helper,
-    const ShardedCache::CreateCallback& create_cb, Cache::Priority priority,
-    bool wait, Statistics* stats) {
-  LRUHandle* e = nullptr;
-  {
-    MutexLock l(&mutex_);
-    e = table_.Lookup(key, hash);
-    if (e != nullptr) {
-      assert(e->InCache());
-      if (!e->HasRefs()) {
-        // The entry is in LRU since it's in hash and has no external references
-        LRU_Remove(e);
-      }
-      e->Ref();
-      e->SetHit();
-    }
-  }
-
-  // If handle table lookup failed, then allocate a handle outside the
-  // mutex if we're going to lookup in the secondary cache
-  // Only support synchronous for now
-  // TODO: Support asynchronous lookup in secondary cache
-  if (!e && secondary_cache_ && helper && helper->saveto_cb) {
-    // For objects from the secondary cache, we expect the caller to provide
-    // a way to create/delete the primary cache object. The only case where
-    // a deleter would not be required is for dummy entries inserted for
-    // accounting purposes, which we won't demote to the secondary cache
-    // anyway.
-    assert(create_cb && helper->del_cb);
-    std::unique_ptr<SecondaryCacheResultHandle> secondary_handle =
-        secondary_cache_->Lookup(key, create_cb, wait);
-    if (secondary_handle != nullptr) {
-      e = reinterpret_cast<LRUHandle*>(
-          new char[sizeof(LRUHandle) - 1 + key.size()]);
-
-      e->flags = 0;
-      e->SetSecondaryCacheCompatible(true);
-      e->info_.helper = helper;
-      e->key_length = key.size();
-      e->hash = hash;
-      e->refs = 0;
-      e->next = e->prev = nullptr;
-      e->SetPriority(priority);
-      memcpy(e->key_data, key.data(), key.size());
-      e->value = nullptr;
-      e->sec_handle = secondary_handle.release();
-      e->Ref();
-
-      if (wait) {
-        Promote(e);
-        if (!e->value) {
-          // The secondary cache returned a handle, but the lookup failed
-          e->Unref();
-          e->Free();
-          e = nullptr;
-        } else {
-          PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
-          RecordTick(stats, SECONDARY_CACHE_HITS);
-        }
-      } else {
-        // If wait is false, we always return a handle and let the caller
-        // release the handle after checking for success or failure
-        e->SetIncomplete(true);
-        // This may be slightly inaccurate, if the lookup eventually fails.
-        // But the probability is very low.
-        PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
-        RecordTick(stats, SECONDARY_CACHE_HITS);
-      }
+Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
+  MutexLock l(&mutex_);
+  LRUHandle* e = table_.Lookup(key, hash);
+  if (e != nullptr) {
+    assert(e->InCache());
+    if (e->refs == 1) {
+      LRU_Remove(e);
     }
+    e->refs++;
   }
   return reinterpret_cast<Cache::Handle*>(e);
 }
 
 bool LRUCacheShard::Ref(Cache::Handle* h) {
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(h);
+  LRUHandle* handle = reinterpret_cast<LRUHandle*>(h);
   MutexLock l(&mutex_);
-  // To create another reference - entry must be already externally referenced
-  assert(e->HasRefs());
-  e->Ref();
+  if (handle->InCache() && handle->refs == 1) {
+    LRU_Remove(handle);
+  }
+  handle->refs++;
   return true;
 }
 
@@ -514,34 +281,30 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
   bool last_reference = false;
   {
     MutexLock l(&mutex_);
-    last_reference = e->Unref();
-    if (last_reference && e->InCache()) {
+    last_reference = Unref(e);
+    if (last_reference) {
+      usage_ -= e->charge;
+    }
+    if (e->refs == 1 && e->InCache()) {
       // The item is still in cache, and nobody else holds a reference to it
       if (usage_ > capacity_ || force_erase) {
+        // the cache is full
         // The LRU list must be empty since the cache is full
-        assert(lru_.next == &lru_ || force_erase);
-        // Take this opportunity and remove the item
+        assert(!(usage_ > capacity_) || lru_.next == &lru_);
+        // take this opportunity and remove the item
         table_.Remove(e->key(), e->hash);
         e->SetInCache(false);
+        Unref(e);
+        usage_ -= e->charge;
+        last_reference = true;
       } else {
-        // Put the item back on the LRU list, and don't free it
+        // put the item on the list to be potentially freed
         LRU_Insert(e);
-        last_reference = false;
       }
     }
-    // If it was the last reference, and the entry is either not secondary
-    // cache compatible (i.e a dummy entry for accounting), or is secondary
-    // cache compatible and has a non-null value, then decrement the cache
-    // usage. If value is null in the latter case, taht means the lookup
-    // failed and we didn't charge the cache.
-    if (last_reference && (!e->IsSecondaryCacheCompatible() || e->value)) {
-      size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-      assert(usage_ >= total_charge);
-      usage_ -= total_charge;
-    }
   }
 
-  // Free the entry here outside of mutex for performance reasons
+  // free outside of mutex
   if (last_reference) {
     e->Free();
   }
@@ -551,35 +314,78 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
 Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
                              size_t charge,
                              void (*deleter)(const Slice& key, void* value),
-                             const Cache::CacheItemHelper* helper,
                              Cache::Handle** handle, Cache::Priority priority) {
   // Allocate the memory here outside of the mutex
   // If the cache is full, we'll have to release it
   // It shouldn't happen very often though.
   LRUHandle* e = reinterpret_cast<LRUHandle*>(
       new char[sizeof(LRUHandle) - 1 + key.size()]);
+  Status s;
+  autovector<LRUHandle*> last_reference_list;
 
   e->value = value;
-  e->flags = 0;
-  if (helper) {
-    e->SetSecondaryCacheCompatible(true);
-    e->info_.helper = helper;
-  } else {
-#ifdef __SANITIZE_THREAD__
-    e->is_secondary_cache_compatible_for_tsan = false;
-#endif  // __SANITIZE_THREAD__
-    e->info_.deleter = deleter;
-  }
+  e->deleter = deleter;
   e->charge = charge;
   e->key_length = key.size();
   e->hash = hash;
-  e->refs = 0;
+  e->refs = (handle == nullptr
+                 ? 1
+                 : 2);  // One from LRUCache, one for the returned handle
   e->next = e->prev = nullptr;
   e->SetInCache(true);
   e->SetPriority(priority);
   memcpy(e->key_data, key.data(), key.size());
 
-  return InsertItem(e, handle, /* free_handle_on_fail */ true);
+  {
+    MutexLock l(&mutex_);
+
+    // Free the space following strict LRU policy until enough space
+    // is freed or the lru list is empty
+    EvictFromLRU(charge, &last_reference_list);
+
+    if (usage_ - lru_usage_ + charge > capacity_ &&
+        (strict_capacity_limit_ || handle == nullptr)) {
+      if (handle == nullptr) {
+        // Don't insert the entry but still return ok, as if the entry inserted
+        // into cache and get evicted immediately.
+        last_reference_list.push_back(e);
+      } else {
+        delete[] reinterpret_cast<char*>(e);
+        *handle = nullptr;
+        s = Status::Incomplete("Insert failed due to LRU cache being full.");
+      }
+    } else {
+      // insert into the cache
+      // note that the cache might get larger than its capacity if not enough
+      // space was freed
+      LRUHandle* old = table_.Insert(e);
+      usage_ += e->charge;
+      if (old != nullptr) {
+        old->SetInCache(false);
+        if (Unref(old)) {
+          usage_ -= old->charge;
+          // old is on LRU because it's in cache and its reference count
+          // was just 1 (Unref returned 0)
+          LRU_Remove(old);
+          last_reference_list.push_back(old);
+        }
+      }
+      if (handle == nullptr) {
+        LRU_Insert(e);
+      } else {
+        *handle = reinterpret_cast<Cache::Handle*>(e);
+      }
+      s = Status::OK();
+    }
+  }
+
+  // we free the entries here outside of mutex for
+  // performance reasons
+  for (auto entry : last_reference_list) {
+    entry->Free();
+  }
+
+  return s;
 }
 
 void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
@@ -589,38 +395,24 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
     MutexLock l(&mutex_);
     e = table_.Remove(key, hash);
     if (e != nullptr) {
-      assert(e->InCache());
-      e->SetInCache(false);
-      if (!e->HasRefs()) {
-        // The entry is in LRU since it's in hash and has no external references
-        LRU_Remove(e);
-        size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-        assert(usage_ >= total_charge);
-        usage_ -= total_charge;
-        last_reference = true;
+      last_reference = Unref(e);
+      if (last_reference) {
+        usage_ -= e->charge;
       }
+      if (last_reference && e->InCache()) {
+        LRU_Remove(e);
+      }
+      e->SetInCache(false);
     }
   }
 
-  // Free the entry here outside of mutex for performance reasons
+  // mutex not held here
   // last_reference will only be true if e != nullptr
   if (last_reference) {
     e->Free();
   }
 }
 
-bool LRUCacheShard::IsReady(Cache::Handle* handle) {
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
-  MutexLock l(&mutex_);
-  bool ready = true;
-  if (e->IsPending()) {
-    assert(secondary_cache_);
-    assert(e->sec_handle);
-    ready = e->sec_handle->IsReady();
-  }
-  return ready;
-}
-
 size_t LRUCacheShard::GetUsage() const {
   MutexLock l(&mutex_);
   return usage_;
@@ -644,41 +436,24 @@ std::string LRUCacheShard::GetPrintableOptions() const {
 }
 
 LRUCache::LRUCache(size_t capacity, int num_shard_bits,
-                   bool strict_capacity_limit, double high_pri_pool_ratio,
-                   std::shared_ptr<MemoryAllocator> allocator,
-                   bool use_adaptive_mutex,
-                   CacheMetadataChargePolicy metadata_charge_policy,
-                   const std::shared_ptr<SecondaryCache>& secondary_cache)
-    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit,
-                   std::move(allocator)) {
-  num_shards_ = 1 << num_shard_bits;
-  shards_ = reinterpret_cast<LRUCacheShard*>(
-      port::cacheline_aligned_alloc(sizeof(LRUCacheShard) * num_shards_));
-  size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
-  for (int i = 0; i < num_shards_; i++) {
-    new (&shards_[i]) LRUCacheShard(
-        per_shard, strict_capacity_limit, high_pri_pool_ratio,
-        use_adaptive_mutex, metadata_charge_policy,
-        /* max_upper_hash_bits */ 32 - num_shard_bits, secondary_cache);
-  }
-  secondary_cache_ = secondary_cache;
-}
-
-LRUCache::~LRUCache() {
-  if (shards_ != nullptr) {
-    assert(num_shards_ > 0);
-    for (int i = 0; i < num_shards_; i++) {
-      shards_[i].~LRUCacheShard();
-    }
-    port::cacheline_aligned_free(shards_);
+                   bool strict_capacity_limit, double high_pri_pool_ratio)
+    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
+  int num_shards = 1 << num_shard_bits;
+  shards_ = new LRUCacheShard[num_shards];
+  SetCapacity(capacity);
+  SetStrictCapacityLimit(strict_capacity_limit);
+  for (int i = 0; i < num_shards; i++) {
+    shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
   }
 }
 
-CacheShard* LRUCache::GetShard(uint32_t shard) {
+LRUCache::~LRUCache() { delete[] shards_; }
+
+CacheShard* LRUCache::GetShard(int shard) {
   return reinterpret_cast<CacheShard*>(&shards_[shard]);
 }
 
-const CacheShard* LRUCache::GetShard(uint32_t shard) const {
+const CacheShard* LRUCache::GetShard(int shard) const {
   return reinterpret_cast<CacheShard*>(&shards_[shard]);
 }
 
@@ -690,79 +465,15 @@ size_t LRUCache::GetCharge(Handle* handle) const {
   return reinterpret_cast<const LRUHandle*>(handle)->charge;
 }
 
-Cache::DeleterFn LRUCache::GetDeleter(Handle* handle) const {
-  auto h = reinterpret_cast<const LRUHandle*>(handle);
-  if (h->IsSecondaryCacheCompatible()) {
-    return h->info_.helper->del_cb;
-  } else {
-    return h->info_.deleter;
-  }
-}
-
 uint32_t LRUCache::GetHash(Handle* handle) const {
   return reinterpret_cast<const LRUHandle*>(handle)->hash;
 }
 
-void LRUCache::DisownData() {
-// Do not drop data if compile with ASAN to suppress leak warning.
-#ifndef MUST_FREE_HEAP_ALLOCATIONS
-  shards_ = nullptr;
-  num_shards_ = 0;
-#endif
-}
+void LRUCache::DisownData() { shards_ = nullptr; }
 
-size_t LRUCache::TEST_GetLRUSize() {
-  size_t lru_size_of_all_shards = 0;
-  for (int i = 0; i < num_shards_; i++) {
-    lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
-  }
-  return lru_size_of_all_shards;
-}
-
-double LRUCache::GetHighPriPoolRatio() {
-  double result = 0.0;
-  if (num_shards_ > 0) {
-    result = shards_[0].GetHighPriPoolRatio();
-  }
-  return result;
-}
-
-void LRUCache::WaitAll(std::vector<Handle*>& handles) {
-  if (secondary_cache_) {
-    std::vector<SecondaryCacheResultHandle*> sec_handles;
-    sec_handles.reserve(handles.size());
-    for (Handle* handle : handles) {
-      if (!handle) {
-        continue;
-      }
-      LRUHandle* lru_handle = reinterpret_cast<LRUHandle*>(handle);
-      if (!lru_handle->IsPending()) {
-        continue;
-      }
-      sec_handles.emplace_back(lru_handle->sec_handle);
-    }
-    secondary_cache_->WaitAll(sec_handles);
-    for (Handle* handle : handles) {
-      if (!handle) {
-        continue;
-      }
-      LRUHandle* lru_handle = reinterpret_cast<LRUHandle*>(handle);
-      if (!lru_handle->IsPending()) {
-        continue;
-      }
-      uint32_t hash = GetHash(handle);
-      LRUCacheShard* shard = static_cast<LRUCacheShard*>(GetShard(Shard(hash)));
-      shard->Promote(lru_handle);
-    }
-  }
-}
-
-std::shared_ptr<Cache> NewLRUCache(
-    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-    double high_pri_pool_ratio,
-    std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
-    CacheMetadataChargePolicy metadata_charge_policy,
-    const std::shared_ptr<SecondaryCache>& secondary_cache) {
+std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
+                                   bool strict_capacity_limit,
+                                   double high_pri_pool_ratio) {
   if (num_shard_bits >= 20) {
     return nullptr;  // the cache cannot be sharded into too many fine pieces
   }
@@ -773,27 +484,8 @@ std::shared_ptr<Cache> NewLRUCache(
   if (num_shard_bits < 0) {
     num_shard_bits = GetDefaultCacheShardBits(capacity);
   }
-  return std::make_shared<LRUCache>(
-      capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio,
-      std::move(memory_allocator), use_adaptive_mutex, metadata_charge_policy,
-      secondary_cache);
+  return std::make_shared<LRUCache>(capacity, num_shard_bits,
+                                    strict_capacity_limit, high_pri_pool_ratio);
 }
 
-std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
-  return NewLRUCache(
-      cache_opts.capacity, cache_opts.num_shard_bits,
-      cache_opts.strict_capacity_limit, cache_opts.high_pri_pool_ratio,
-      cache_opts.memory_allocator, cache_opts.use_adaptive_mutex,
-      cache_opts.metadata_charge_policy, cache_opts.secondary_cache);
-}
-
-std::shared_ptr<Cache> NewLRUCache(
-    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-    double high_pri_pool_ratio,
-    std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
-    CacheMetadataChargePolicy metadata_charge_policy) {
-  return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
-                     high_pri_pool_ratio, memory_allocator, use_adaptive_mutex,
-                     metadata_charge_policy, nullptr);
-}
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb

cache/lru_cache.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
@@ -8,220 +8,107 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #pragma once
 
-#include <memory>
 #include <string>
 
 #include "cache/sharded_cache.h"
-#include "port/lang.h"
-#include "port/malloc.h"
+
 #include "port/port.h"
-#include "rocksdb/secondary_cache.h"
 #include "util/autovector.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
-// LRU cache implementation. This class is not thread-safe.
+// LRU cache implementation
 
 // An entry is a variable length heap-allocated structure.
 // Entries are referenced by cache and/or by any external entity.
-// The cache keeps all its entries in a hash table. Some elements
+// The cache keeps all its entries in table. Some elements
 // are also stored on LRU list.
 //
 // LRUHandle can be in these states:
 // 1. Referenced externally AND in hash table.
-//    In that case the entry is *not* in the LRU list
-//    (refs >= 1 && in_cache == true)
-// 2. Not referenced externally AND in hash table.
-//    In that case the entry is in the LRU list and can be freed.
-//    (refs == 0 && in_cache == true)
-// 3. Referenced externally AND not in hash table.
-//    In that case the entry is not in the LRU list and not in hash table.
-//    The entry can be freed when refs becomes 0.
-//    (refs >= 1 && in_cache == false)
+//  In that case the entry is *not* in the LRU. (refs > 1 && in_cache == true)
+// 2. Not referenced externally and in hash table. In that case the entry is
+// in the LRU and can be freed. (refs == 1 && in_cache == true)
+// 3. Referenced externally and not in hash table. In that case the entry is
+// in not on LRU and not in table. (refs >= 1 && in_cache == false)
 //
 // All newly created LRUHandles are in state 1. If you call
-// LRUCacheShard::Release on entry in state 1, it will go into state 2.
-// To move from state 1 to state 3, either call LRUCacheShard::Erase or
-// LRUCacheShard::Insert with the same key (but possibly different value).
+// LRUCacheShard::Release
+// on entry in state 1, it will go into state 2. To move from state 1 to
+// state 3, either call LRUCacheShard::Erase or LRUCacheShard::Insert with the
+// same key.
 // To move from state 2 to state 1, use LRUCacheShard::Lookup.
 // Before destruction, make sure that no handles are in state 1. This means
 // that any successful LRUCacheShard::Lookup/LRUCacheShard::Insert have a
-// matching LRUCache::Release (to move into state 2) or LRUCacheShard::Erase
-// (to move into state 3).
+// matching
+// RUCache::Release (to move into state 2) or LRUCacheShard::Erase (for state 3)
 
 struct LRUHandle {
   void* value;
-  union Info {
-    Info() {}
-    ~Info() {}
-    Cache::DeleterFn deleter;
-    const ShardedCache::CacheItemHelper* helper;
-  } info_;
-  // An entry is not added to the LRUHandleTable until the secondary cache
-  // lookup is complete, so its safe to have this union.
-  union {
-    LRUHandle* next_hash;
-    SecondaryCacheResultHandle* sec_handle;
-  };
+  void (*deleter)(const Slice&, void* value);
+  LRUHandle* next_hash;
   LRUHandle* next;
   LRUHandle* prev;
   size_t charge;  // TODO(opt): Only allow uint32_t?
   size_t key_length;
-  // The hash of key(). Used for fast sharding and comparisons.
-  uint32_t hash;
-  // The number of external refs to this entry. The cache itself is not counted.
-  uint32_t refs;
+  uint32_t refs;     // a number of refs to this entry
+                     // cache itself is counted as 1
 
-  enum Flags : uint8_t {
-    // Whether this entry is referenced by the hash table.
-    IN_CACHE = (1 << 0),
-    // Whether this entry is high priority entry.
-    IS_HIGH_PRI = (1 << 1),
-    // Whether this entry is in high-pri pool.
-    IN_HIGH_PRI_POOL = (1 << 2),
-    // Whether this entry has had any lookups (hits).
-    HAS_HIT = (1 << 3),
-    // Can this be inserted into the secondary cache
-    IS_SECONDARY_CACHE_COMPATIBLE = (1 << 4),
-    // Is the handle still being read from a lower tier
-    IS_PENDING = (1 << 5),
-    // Has the item been promoted from a lower tier
-    IS_PROMOTED = (1 << 6),
-  };
+  // Include the following flags:
+  //   in_cache:    whether this entry is referenced by the hash table.
+  //   is_high_pri: whether this entry is high priority entry.
+  //   in_high_pro_pool: whether this entry is in high-pri pool.
+  char flags;
 
-  uint8_t flags;
+  uint32_t hash;     // Hash of key(); used for fast sharding and comparisons
 
-#ifdef __SANITIZE_THREAD__
-  // TSAN can report a false data race on flags, where one thread is writing
-  // to one of the mutable bits and another thread is reading this immutable
-  // bit. So precisely suppress that TSAN warning, we separate out this bit
-  // during TSAN runs.
-  bool is_secondary_cache_compatible_for_tsan;
-#endif  // __SANITIZE_THREAD__
+  char key_data[1];  // Beginning of key
 
-  // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!)
-  char key_data[1];
-
-  Slice key() const { return Slice(key_data, key_length); }
-
-  // Increase the reference count by 1.
-  void Ref() { refs++; }
-
-  // Just reduce the reference count by 1. Return true if it was last reference.
-  bool Unref() {
-    assert(refs > 0);
-    refs--;
-    return refs == 0;
+  Slice key() const {
+    // For cheaper lookups, we allow a temporary Handle object
+    // to store a pointer to a key in "value".
+    if (next == this) {
+      return *(reinterpret_cast<Slice*>(value));
+    } else {
+      return Slice(key_data, key_length);
+    }
   }
 
-  // Return true if there are external refs, false otherwise.
-  bool HasRefs() const { return refs > 0; }
-
-  bool InCache() const { return flags & IN_CACHE; }
-  bool IsHighPri() const { return flags & IS_HIGH_PRI; }
-  bool InHighPriPool() const { return flags & IN_HIGH_PRI_POOL; }
-  bool HasHit() const { return flags & HAS_HIT; }
-  bool IsSecondaryCacheCompatible() const {
-#ifdef __SANITIZE_THREAD__
-    return is_secondary_cache_compatible_for_tsan;
-#else
-    return flags & IS_SECONDARY_CACHE_COMPATIBLE;
-#endif  // __SANITIZE_THREAD__
-  }
-  bool IsPending() const { return flags & IS_PENDING; }
-  bool IsPromoted() const { return flags & IS_PROMOTED; }
+  bool InCache() { return flags & 1; }
+  bool IsHighPri() { return flags & 2; }
+  bool InHighPriPool() { return flags & 4; }
 
   void SetInCache(bool in_cache) {
     if (in_cache) {
-      flags |= IN_CACHE;
+      flags |= 1;
     } else {
-      flags &= ~IN_CACHE;
+      flags &= ~1;
     }
   }
 
   void SetPriority(Cache::Priority priority) {
     if (priority == Cache::Priority::HIGH) {
-      flags |= IS_HIGH_PRI;
+      flags |= 2;
     } else {
-      flags &= ~IS_HIGH_PRI;
+      flags &= ~2;
     }
   }
 
   void SetInHighPriPool(bool in_high_pri_pool) {
     if (in_high_pri_pool) {
-      flags |= IN_HIGH_PRI_POOL;
+      flags |= 4;
     } else {
-      flags &= ~IN_HIGH_PRI_POOL;
-    }
-  }
-
-  void SetHit() { flags |= HAS_HIT; }
-
-  void SetSecondaryCacheCompatible(bool compat) {
-    if (compat) {
-      flags |= IS_SECONDARY_CACHE_COMPATIBLE;
-    } else {
-      flags &= ~IS_SECONDARY_CACHE_COMPATIBLE;
-    }
-#ifdef __SANITIZE_THREAD__
-    is_secondary_cache_compatible_for_tsan = compat;
-#endif  // __SANITIZE_THREAD__
-  }
-
-  void SetIncomplete(bool incomp) {
-    if (incomp) {
-      flags |= IS_PENDING;
-    } else {
-      flags &= ~IS_PENDING;
-    }
-  }
-
-  void SetPromoted(bool promoted) {
-    if (promoted) {
-      flags |= IS_PROMOTED;
-    } else {
-      flags &= ~IS_PROMOTED;
+      flags &= ~4;
     }
   }
 
   void Free() {
-    assert(refs == 0);
-#ifdef __SANITIZE_THREAD__
-    // Here we can safely assert they are the same without a data race reported
-    assert(((flags & IS_SECONDARY_CACHE_COMPATIBLE) != 0) ==
-           is_secondary_cache_compatible_for_tsan);
-#endif  // __SANITIZE_THREAD__
-    if (!IsSecondaryCacheCompatible() && info_.deleter) {
-      (*info_.deleter)(key(), value);
-    } else if (IsSecondaryCacheCompatible()) {
-      if (IsPending()) {
-        assert(sec_handle != nullptr);
-        SecondaryCacheResultHandle* tmp_sec_handle = sec_handle;
-        tmp_sec_handle->Wait();
-        value = tmp_sec_handle->Value();
-        delete tmp_sec_handle;
-      }
-      if (value) {
-        (*info_.helper->del_cb)(key(), value);
-      }
+    assert((refs == 1 && InCache()) || (refs == 0 && !InCache()));
+    if (deleter) {
+      (*deleter)(key(), value);
     }
     delete[] reinterpret_cast<char*>(this);
   }
-
-  // Calculate the memory usage by metadata
-  inline size_t CalcTotalCharge(
-      CacheMetadataChargePolicy metadata_charge_policy) {
-    size_t meta_charge = 0;
-    if (metadata_charge_policy == kFullChargeCacheMetadata) {
-#ifdef ROCKSDB_MALLOC_USABLE_SIZE
-      meta_charge += malloc_usable_size(static_cast<void*>(this));
-#else
-      // This is the size that is used when a new handle is created
-      meta_charge += sizeof(LRUHandle) - 1 + key_length;
-#endif
-    }
-    return charge + meta_charge;
-  }
 };
 
 // We provide our own simple hash table since it removes a whole bunch
@@ -231,10 +118,7 @@ struct LRUHandle {
 // 4.4.3's builtin hashtable.
 class LRUHandleTable {
  public:
-  // If the table uses more hash bits than `max_upper_hash_bits`,
-  // it will eat into the bits used for sharding, which are constant
-  // for a given LRUHandleTable.
-  explicit LRUHandleTable(int max_upper_hash_bits);
+  LRUHandleTable();
   ~LRUHandleTable();
 
   LRUHandle* Lookup(const Slice& key, uint32_t hash);
@@ -242,8 +126,8 @@ class LRUHandleTable {
   LRUHandle* Remove(const Slice& key, uint32_t hash);
 
   template <typename T>
-  void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end) {
-    for (uint32_t i = index_begin; i < index_end; i++) {
+  void ApplyToAllCacheEntries(T func) {
+    for (uint32_t i = 0; i < length_; i++) {
       LRUHandle* h = list_[i];
       while (h != nullptr) {
         auto n = h->next_hash;
@@ -254,8 +138,6 @@ class LRUHandleTable {
     }
   }
 
-  int GetLengthBits() const { return length_bits_; }
-
  private:
   // Return a pointer to slot that points to a cache entry that
   // matches key/hash.  If there is no such cache entry, return a
@@ -264,30 +146,18 @@ class LRUHandleTable {
 
   void Resize();
 
-  // Number of hash bits (upper because lower bits used for sharding)
-  // used for table index. Length == 1 << length_bits_
-  int length_bits_;
-
   // The table consists of an array of buckets where each bucket is
   // a linked list of cache entries that hash into the bucket.
-  std::unique_ptr<LRUHandle*[]> list_;
-
-  // Number of elements currently in the table
+  uint32_t length_;
   uint32_t elems_;
-
-  // Set from max_upper_hash_bits (see constructor)
-  const int max_length_bits_;
+  LRUHandle** list_;
 };
 
 // A single shard of sharded cache.
-class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
+class LRUCacheShard : public CacheShard {
  public:
-  LRUCacheShard(size_t capacity, bool strict_capacity_limit,
-                double high_pri_pool_ratio, bool use_adaptive_mutex,
-                CacheMetadataChargePolicy metadata_charge_policy,
-                int max_upper_hash_bits,
-                const std::shared_ptr<SecondaryCache>& secondary_cache);
-  virtual ~LRUCacheShard() override = default;
+  LRUCacheShard();
+  virtual ~LRUCacheShard();
 
   // Separate from constructor so caller can easily make an array of LRUCache
   // if current usage is more than new capacity, the function will attempt to
@@ -302,35 +172,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
 
   // Like Cache methods, but with an extra "hash" parameter.
   virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge, Cache::DeleterFn deleter,
+                        size_t charge,
+                        void (*deleter)(const Slice& key, void* value),
                         Cache::Handle** handle,
-                        Cache::Priority priority) override {
-    return Insert(key, hash, value, charge, deleter, nullptr, handle, priority);
-  }
-  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        const Cache::CacheItemHelper* helper, size_t charge,
-                        Cache::Handle** handle,
-                        Cache::Priority priority) override {
-    assert(helper);
-    return Insert(key, hash, value, charge, nullptr, helper, handle, priority);
-  }
-  // If helper_cb is null, the values of the following arguments don't
-  // matter
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                                const ShardedCache::CacheItemHelper* helper,
-                                const ShardedCache::CreateCallback& create_cb,
-                                ShardedCache::Priority priority, bool wait,
-                                Statistics* stats) override;
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override {
-    return Lookup(key, hash, nullptr, nullptr, Cache::Priority::LOW, true,
-                  nullptr);
-  }
-  virtual bool Release(Cache::Handle* handle, bool /*useful*/,
-                       bool force_erase) override {
-    return Release(handle, force_erase);
-  }
-  virtual bool IsReady(Cache::Handle* /*handle*/) override;
-  virtual void Wait(Cache::Handle* /*handle*/) override {}
+                        Cache::Priority priority) override;
+  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
   virtual bool Ref(Cache::Handle* handle) override;
   virtual bool Release(Cache::Handle* handle,
                        bool force_erase = false) override;
@@ -343,10 +189,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   virtual size_t GetUsage() const override;
   virtual size_t GetPinnedUsage() const override;
 
-  virtual void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
-      uint32_t average_entries_per_lock, uint32_t* state) override;
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) override;
 
   virtual void EraseUnRefEntries() override;
 
@@ -354,31 +198,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
 
   void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);
 
-  //  Retrieves number of elements in LRU, for unit test purpose only
-  //  not threadsafe
-  size_t TEST_GetLRUSize();
-
-  //  Retrieves high pri pool ratio
-  double GetHighPriPoolRatio();
-
  private:
-  friend class LRUCache;
-  // Insert an item into the hash table and, if handle is null, insert into
-  // the LRU list. Older items are evicted as necessary. If the cache is full
-  // and free_handle_on_fail is true, the item is deleted and handle is set to.
-  Status InsertItem(LRUHandle* item, Cache::Handle** handle,
-                    bool free_handle_on_fail);
-  Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
-                DeleterFn deleter, const Cache::CacheItemHelper* helper,
-                Cache::Handle** handle, Cache::Priority priority);
-  // Promote an item looked up from the secondary cache to the LRU cache. The
-  // item is only inserted into the hash table and not the LRU list, and only
-  // if the cache is not at full capacity, as is the case during Insert.  The
-  // caller should hold a reference on the LRUHandle. When the caller releases
-  // the last reference, the item is added to the LRU list.
-  // The item is promoted to the high pri or low pri pool as specified by the
-  // caller in Lookup.
-  void Promote(LRUHandle* e);
   void LRU_Remove(LRUHandle* e);
   void LRU_Insert(LRUHandle* e);
 
@@ -386,6 +206,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // high-pri pool is no larger than the size specify by high_pri_pool_pct.
   void MaintainPoolSize();
 
+  // Just reduce the reference count by 1.
+  // Return true if last reference
+  bool Unref(LRUHandle* e);
+
   // Free some space following strict LRU policy until enough space
   // to hold (usage_ + charge) is freed or the lru list is empty
   // This function is not thread safe - it needs to be executed while
@@ -395,6 +219,12 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Initialized before use.
   size_t capacity_;
 
+  // Memory size for entries residing in the cache
+  size_t usage_;
+
+  // Memory size for entries residing only in the LRU list
+  size_t lru_usage_;
+
   // Memory size for entries in high-pri pool.
   size_t high_pri_pool_usage_;
 
@@ -408,6 +238,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Remember the value to avoid recomputing each time.
   double high_pri_pool_capacity_;
 
+  // mutex_ protects the following state.
+  // We don't count mutex_ as the cache's internal state so semantically we
+  // don't mind mutex_ invoking the non-const actions.
+  mutable port::Mutex mutex_;
+
   // Dummy head of LRU list.
   // lru.prev is newest entry, lru.next is oldest entry.
   // LRU contains items which can be evicted, ie reference only by cache
@@ -416,66 +251,24 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Pointer to head of low-pri pool in LRU list.
   LRUHandle* lru_low_pri_;
 
-  // ------------^^^^^^^^^^^^^-----------
-  // Not frequently modified data members
-  // ------------------------------------
-  //
-  // We separate data members that are updated frequently from the ones that
-  // are not frequently updated so that they don't share the same cache line
-  // which will lead into false cache sharing
-  //
-  // ------------------------------------
-  // Frequently modified data members
-  // ------------vvvvvvvvvvvvv-----------
   LRUHandleTable table_;
-
-  // Memory size for entries residing in the cache
-  size_t usage_;
-
-  // Memory size for entries residing only in the LRU list
-  size_t lru_usage_;
-
-  // mutex_ protects the following state.
-  // We don't count mutex_ as the cache's internal state so semantically we
-  // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
-
-  std::shared_ptr<SecondaryCache> secondary_cache_;
 };
 
-class LRUCache
-#ifdef NDEBUG
-    final
-#endif
-    : public ShardedCache {
+class LRUCache : public ShardedCache {
  public:
   LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-           double high_pri_pool_ratio,
-           std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
-           bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
-           CacheMetadataChargePolicy metadata_charge_policy =
-               kDontChargeCacheMetadata,
-           const std::shared_ptr<SecondaryCache>& secondary_cache = nullptr);
+           double high_pri_pool_ratio);
   virtual ~LRUCache();
   virtual const char* Name() const override { return "LRUCache"; }
-  virtual CacheShard* GetShard(uint32_t shard) override;
-  virtual const CacheShard* GetShard(uint32_t shard) const override;
+  virtual CacheShard* GetShard(int shard) override;
+  virtual const CacheShard* GetShard(int shard) const override;
   virtual void* Value(Handle* handle) override;
   virtual size_t GetCharge(Handle* handle) const override;
   virtual uint32_t GetHash(Handle* handle) const override;
-  virtual DeleterFn GetDeleter(Handle* handle) const override;
   virtual void DisownData() override;
-  virtual void WaitAll(std::vector<Handle*>& handles) override;
-
-  //  Retrieves number of elements in LRU, for unit test purpose only
-  size_t TEST_GetLRUSize();
-  //  Retrieves high pri pool ratio
-  double GetHighPriPoolRatio();
 
  private:
-  LRUCacheShard* shards_ = nullptr;
-  int num_shards_ = 0;
-  std::shared_ptr<SecondaryCache> secondary_cache_;
+  LRUCacheShard* shards_;
 };
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb

cache/sharded_cache.cc

@@ -7,98 +7,57 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
 #include "cache/sharded_cache.h"
 
-#include <algorithm>
-#include <cstdint>
-#include <memory>
+#include <string>
 
-#include "util/hash.h"
-#include "util/math.h"
 #include "util/mutexlock.h"
 
-namespace ROCKSDB_NAMESPACE {
-
-namespace {
-
-inline uint32_t HashSlice(const Slice& s) {
-  return Lower32of64(GetSliceNPHash64(s));
-}
-
-}  // namespace
+namespace rocksdb {
 
 ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
-                           bool strict_capacity_limit,
-                           std::shared_ptr<MemoryAllocator> allocator)
-    : Cache(std::move(allocator)),
-      shard_mask_((uint32_t{1} << num_shard_bits) - 1),
+                           bool strict_capacity_limit)
+    : num_shard_bits_(num_shard_bits),
       capacity_(capacity),
       strict_capacity_limit_(strict_capacity_limit),
       last_id_(1) {}
 
 void ShardedCache::SetCapacity(size_t capacity) {
-  uint32_t num_shards = GetNumShards();
+  int num_shards = 1 << num_shard_bits_;
   const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
   MutexLock l(&capacity_mutex_);
-  for (uint32_t s = 0; s < num_shards; s++) {
+  for (int s = 0; s < num_shards; s++) {
     GetShard(s)->SetCapacity(per_shard);
   }
   capacity_ = capacity;
 }
 
 void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  uint32_t num_shards = GetNumShards();
+  int num_shards = 1 << num_shard_bits_;
   MutexLock l(&capacity_mutex_);
-  for (uint32_t s = 0; s < num_shards; s++) {
+  for (int s = 0; s < num_shards; s++) {
     GetShard(s)->SetStrictCapacityLimit(strict_capacity_limit);
   }
   strict_capacity_limit_ = strict_capacity_limit;
 }
 
 Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
-                            DeleterFn deleter, Handle** handle,
-                            Priority priority) {
+                            void (*deleter)(const Slice& key, void* value),
+                            Handle** handle, Priority priority) {
   uint32_t hash = HashSlice(key);
   return GetShard(Shard(hash))
       ->Insert(key, hash, value, charge, deleter, handle, priority);
 }
 
-Status ShardedCache::Insert(const Slice& key, void* value,
-                            const CacheItemHelper* helper, size_t charge,
-                            Handle** handle, Priority priority) {
-  uint32_t hash = HashSlice(key);
-  if (!helper) {
-    return Status::InvalidArgument();
-  }
-  return GetShard(Shard(hash))
-      ->Insert(key, hash, value, helper, charge, handle, priority);
-}
-
-Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* /*stats*/) {
+Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* stats) {
   uint32_t hash = HashSlice(key);
   return GetShard(Shard(hash))->Lookup(key, hash);
 }
 
-Cache::Handle* ShardedCache::Lookup(const Slice& key,
-                                    const CacheItemHelper* helper,
-                                    const CreateCallback& create_cb,
-                                    Priority priority, bool wait,
-                                    Statistics* stats) {
-  uint32_t hash = HashSlice(key);
-  return GetShard(Shard(hash))
-      ->Lookup(key, hash, helper, create_cb, priority, wait, stats);
-}
-
-bool ShardedCache::IsReady(Handle* handle) {
-  uint32_t hash = GetHash(handle);
-  return GetShard(Shard(hash))->IsReady(handle);
-}
-
-void ShardedCache::Wait(Handle* handle) {
-  uint32_t hash = GetHash(handle);
-  GetShard(Shard(hash))->Wait(handle);
-}
-
 bool ShardedCache::Ref(Handle* handle) {
   uint32_t hash = GetHash(handle);
   return GetShard(Shard(hash))->Ref(handle);
@@ -109,11 +68,6 @@ bool ShardedCache::Release(Handle* handle, bool force_erase) {
   return GetShard(Shard(hash))->Release(handle, force_erase);
 }
 
-bool ShardedCache::Release(Handle* handle, bool useful, bool force_erase) {
-  uint32_t hash = GetHash(handle);
-  return GetShard(Shard(hash))->Release(handle, useful, force_erase);
-}
-
 void ShardedCache::Erase(const Slice& key) {
   uint32_t hash = HashSlice(key);
   GetShard(Shard(hash))->Erase(key, hash);
@@ -135,9 +89,9 @@ bool ShardedCache::HasStrictCapacityLimit() const {
 
 size_t ShardedCache::GetUsage() const {
   // We will not lock the cache when getting the usage from shards.
-  uint32_t num_shards = GetNumShards();
+  int num_shards = 1 << num_shard_bits_;
   size_t usage = 0;
-  for (uint32_t s = 0; s < num_shards; s++) {
+  for (int s = 0; s < num_shards; s++) {
     usage += GetShard(s)->GetUsage();
   }
   return usage;
@@ -149,42 +103,25 @@ size_t ShardedCache::GetUsage(Handle* handle) const {
 
 size_t ShardedCache::GetPinnedUsage() const {
   // We will not lock the cache when getting the usage from shards.
-  uint32_t num_shards = GetNumShards();
+  int num_shards = 1 << num_shard_bits_;
   size_t usage = 0;
-  for (uint32_t s = 0; s < num_shards; s++) {
+  for (int s = 0; s < num_shards; s++) {
     usage += GetShard(s)->GetPinnedUsage();
   }
   return usage;
 }
 
-void ShardedCache::ApplyToAllEntries(
-    const std::function<void(const Slice& key, void* value, size_t charge,
-                             DeleterFn deleter)>& callback,
-    const ApplyToAllEntriesOptions& opts) {
-  uint32_t num_shards = GetNumShards();
-  // Iterate over part of each shard, rotating between shards, to
-  // minimize impact on latency of concurrent operations.
-  std::unique_ptr<uint32_t[]> states(new uint32_t[num_shards]{});
-
-  uint32_t aepl_in_32 = static_cast<uint32_t>(
-      std::min(size_t{UINT32_MAX}, opts.average_entries_per_lock));
-  aepl_in_32 = std::min(aepl_in_32, uint32_t{1});
-
-  bool remaining_work;
-  do {
-    remaining_work = false;
-    for (uint32_t s = 0; s < num_shards; s++) {
-      if (states[s] != UINT32_MAX) {
-        GetShard(s)->ApplyToSomeEntries(callback, aepl_in_32, &states[s]);
-        remaining_work |= states[s] != UINT32_MAX;
-      }
-    }
-  } while (remaining_work);
+void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                          bool thread_safe) {
+  int num_shards = 1 << num_shard_bits_;
+  for (int s = 0; s < num_shards; s++) {
+    GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
+  }
 }
 
 void ShardedCache::EraseUnRefEntries() {
-  uint32_t num_shards = GetNumShards();
-  for (uint32_t s = 0; s < num_shards; s++) {
+  int num_shards = 1 << num_shard_bits_;
+  for (int s = 0; s < num_shards; s++) {
     GetShard(s)->EraseUnRefEntries();
   }
 }
@@ -199,16 +136,12 @@ std::string ShardedCache::GetPrintableOptions() const {
     snprintf(buffer, kBufferSize, "    capacity : %" ROCKSDB_PRIszt "\n",
              capacity_);
     ret.append(buffer);
-    snprintf(buffer, kBufferSize, "    num_shard_bits : %d\n",
-             GetNumShardBits());
+    snprintf(buffer, kBufferSize, "    num_shard_bits : %d\n", num_shard_bits_);
     ret.append(buffer);
     snprintf(buffer, kBufferSize, "    strict_capacity_limit : %d\n",
              strict_capacity_limit_);
     ret.append(buffer);
   }
-  snprintf(buffer, kBufferSize, "    memory_allocator : %s\n",
-           memory_allocator() ? memory_allocator()->Name() : "None");
-  ret.append(buffer);
   ret.append(GetShard(0)->GetPrintableOptions());
   return ret;
 }
@@ -225,8 +158,4 @@ int GetDefaultCacheShardBits(size_t capacity) {
   return num_shard_bits;
 }
 
-int ShardedCache::GetNumShardBits() const { return BitsSetToOne(shard_mask_); }
-
-uint32_t ShardedCache::GetNumShards() const { return shard_mask_ + 1; }
-
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb

cache/sharded_cache.h

@@ -14,8 +14,9 @@
 
 #include "port/port.h"
 #include "rocksdb/cache.h"
+#include "util/hash.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 // Single cache shard interface.
 class CacheShard {
@@ -23,47 +24,22 @@ class CacheShard {
   CacheShard() = default;
   virtual ~CacheShard() = default;
 
-  using DeleterFn = Cache::DeleterFn;
   virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge, DeleterFn deleter,
-                        Cache::Handle** handle, Cache::Priority priority) = 0;
-  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        const Cache::CacheItemHelper* helper, size_t charge,
+                        size_t charge,
+                        void (*deleter)(const Slice& key, void* value),
                         Cache::Handle** handle, Cache::Priority priority) = 0;
   virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0;
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                                const Cache::CacheItemHelper* helper,
-                                const Cache::CreateCallback& create_cb,
-                                Cache::Priority priority, bool wait,
-                                Statistics* stats) = 0;
-  virtual bool Release(Cache::Handle* handle, bool useful,
-                       bool force_erase) = 0;
-  virtual bool IsReady(Cache::Handle* handle) = 0;
-  virtual void Wait(Cache::Handle* handle) = 0;
   virtual bool Ref(Cache::Handle* handle) = 0;
-  virtual bool Release(Cache::Handle* handle, bool force_erase) = 0;
+  virtual bool Release(Cache::Handle* handle, bool force_erase = false) = 0;
   virtual void Erase(const Slice& key, uint32_t hash) = 0;
   virtual void SetCapacity(size_t capacity) = 0;
   virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
   virtual size_t GetUsage() const = 0;
   virtual size_t GetPinnedUsage() const = 0;
-  // Handles iterating over roughly `average_entries_per_lock` entries, using
-  // `state` to somehow record where it last ended up. Caller initially uses
-  // *state == 0 and implementation sets *state = UINT32_MAX to indicate
-  // completion.
-  virtual void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
-      uint32_t average_entries_per_lock, uint32_t* state) = 0;
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) = 0;
   virtual void EraseUnRefEntries() = 0;
   virtual std::string GetPrintableOptions() const { return ""; }
-  void set_metadata_charge_policy(
-      CacheMetadataChargePolicy metadata_charge_policy) {
-    metadata_charge_policy_ = metadata_charge_policy;
-  }
-
- protected:
-  CacheMetadataChargePolicy metadata_charge_policy_ = kDontChargeCacheMetadata;
 };
 
 // Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
@@ -71,32 +47,23 @@ class CacheShard {
 // Keys are sharded by the highest num_shard_bits bits of hash value.
 class ShardedCache : public Cache {
  public:
-  ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-               std::shared_ptr<MemoryAllocator> memory_allocator = nullptr);
+  ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
   virtual ~ShardedCache() = default;
-  virtual CacheShard* GetShard(uint32_t shard) = 0;
-  virtual const CacheShard* GetShard(uint32_t shard) const = 0;
-
+  virtual const char* Name() const override = 0;
+  virtual CacheShard* GetShard(int shard) = 0;
+  virtual const CacheShard* GetShard(int shard) const = 0;
+  virtual void* Value(Handle* handle) override = 0;
+  virtual size_t GetCharge(Handle* handle) const = 0;
   virtual uint32_t GetHash(Handle* handle) const = 0;
+  virtual void DisownData() override = 0;
 
   virtual void SetCapacity(size_t capacity) override;
   virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
 
   virtual Status Insert(const Slice& key, void* value, size_t charge,
-                        DeleterFn deleter, Handle** handle,
-                        Priority priority) override;
-  virtual Status Insert(const Slice& key, void* value,
-                        const CacheItemHelper* helper, size_t chargge,
-                        Handle** handle = nullptr,
-                        Priority priority = Priority::LOW) override;
+                        void (*deleter)(const Slice& key, void* value),
+                        Handle** handle, Priority priority) override;
   virtual Handle* Lookup(const Slice& key, Statistics* stats) override;
-  virtual Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
-                         const CreateCallback& create_cb, Priority priority,
-                         bool wait, Statistics* stats = nullptr) override;
-  virtual bool Release(Handle* handle, bool useful,
-                       bool force_erase = false) override;
-  virtual bool IsReady(Handle* handle) override;
-  virtual void Wait(Handle* handle) override;
   virtual bool Ref(Handle* handle) override;
   virtual bool Release(Handle* handle, bool force_erase = false) override;
   virtual void Erase(const Slice& key) override;
@@ -106,21 +73,24 @@ class ShardedCache : public Cache {
   virtual size_t GetUsage() const override;
   virtual size_t GetUsage(Handle* handle) const override;
   virtual size_t GetPinnedUsage() const override;
-  virtual void ApplyToAllEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
-      const ApplyToAllEntriesOptions& opts) override;
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) override;
   virtual void EraseUnRefEntries() override;
   virtual std::string GetPrintableOptions() const override;
 
-  int GetNumShardBits() const;
-  uint32_t GetNumShards() const;
-
- protected:
-  inline uint32_t Shard(uint32_t hash) { return hash & shard_mask_; }
+  int GetNumShardBits() const { return num_shard_bits_; }
 
  private:
-  const uint32_t shard_mask_;
+  static inline uint32_t HashSlice(const Slice& s) {
+    return Hash(s.data(), s.size(), 0);
+  }
+
+  uint32_t Shard(uint32_t hash) {
+    // Note, hash >> 32 yields hash in gcc, not the zero we expect!
+    return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
+  }
+
+  int num_shard_bits_;
   mutable port::Mutex capacity_mutex_;
   size_t capacity_;
   bool strict_capacity_limit_;
@@ -129,4 +99,4 @@ class ShardedCache : public Cache {
 
 extern int GetDefaultCacheShardBits(size_t capacity);
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb

cmake/RocksDBConfig.cmake.in

@@ -1,54 +0,0 @@
-@PACKAGE_INIT@
-
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
-
-include(CMakeFindDependencyMacro)
-
-set(GFLAGS_USE_TARGET_NAMESPACE @GFLAGS_USE_TARGET_NAMESPACE@)
-
-if(@WITH_JEMALLOC@)
-  find_dependency(JeMalloc)
-endif()
-
-if(@WITH_GFLAGS@)
-  find_dependency(gflags CONFIG)
-  if(NOT gflags_FOUND)
-    find_dependency(gflags)
-  endif()
-endif()
-
-if(@WITH_SNAPPY@)
-  find_dependency(Snappy CONFIG)
-  if(NOT Snappy_FOUND)
-    find_dependency(Snappy)
-  endif()
-endif()
-
-if(@WITH_ZLIB@)
-  find_dependency(ZLIB)
-endif()
-
-if(@WITH_BZ2@)
-  find_dependency(BZip2)
-endif()
-
-if(@WITH_LZ4@)
-  find_dependency(lz4)
-endif()
-
-if(@WITH_ZSTD@)
-  find_dependency(zstd)
-endif()
-
-if(@WITH_NUMA@)
-  find_dependency(NUMA)
-endif()
-
-if(@WITH_TBB@)
-  find_dependency(TBB)
-endif()
-
-find_dependency(Threads)
-
-include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake")
-check_required_components(RocksDB)

cmake/modules/CxxFlags.cmake

@@ -1,7 +0,0 @@
-macro(get_cxx_std_flags FLAGS_VARIABLE)
-  if( CMAKE_CXX_STANDARD_REQUIRED )
-    set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION})
-  else()
-    set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_EXTENSION_COMPILE_OPTION})
-  endif()
-endmacro()

cmake/modules/FindJeMalloc.cmake

@@ -1,29 +1,21 @@
 # - Find JeMalloc library
 # Find the native JeMalloc includes and library
 #
-# JeMalloc_INCLUDE_DIRS - where to find jemalloc.h, etc.
-# JeMalloc_LIBRARIES - List of libraries when using jemalloc.
-# JeMalloc_FOUND - True if jemalloc found.
+# JEMALLOC_INCLUDE_DIR - where to find jemalloc.h, etc.
+# JEMALLOC_LIBRARIES - List of libraries when using jemalloc.
+# JEMALLOC_FOUND - True if jemalloc found.
 
-find_path(JeMalloc_INCLUDE_DIRS
+find_path(JEMALLOC_INCLUDE_DIR
   NAMES jemalloc/jemalloc.h
   HINTS ${JEMALLOC_ROOT_DIR}/include)
 
-find_library(JeMalloc_LIBRARIES
+find_library(JEMALLOC_LIBRARIES
   NAMES jemalloc
   HINTS ${JEMALLOC_ROOT_DIR}/lib)
 
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(JeMalloc DEFAULT_MSG JeMalloc_LIBRARIES JeMalloc_INCLUDE_DIRS)
+find_package_handle_standard_args(jemalloc DEFAULT_MSG JEMALLOC_LIBRARIES JEMALLOC_INCLUDE_DIR)
 
 mark_as_advanced(
-  JeMalloc_LIBRARIES
-  JeMalloc_INCLUDE_DIRS)
-
-if(JeMalloc_FOUND AND NOT (TARGET JeMalloc::JeMalloc))
-  add_library (JeMalloc::JeMalloc UNKNOWN IMPORTED)
-  set_target_properties(JeMalloc::JeMalloc
-    PROPERTIES
-      IMPORTED_LOCATION ${JeMalloc_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${JeMalloc_INCLUDE_DIRS})
-endif()
+  JEMALLOC_LIBRARIES
+  JEMALLOC_INCLUDE_DIR)

cmake/modules/FindNUMA.cmake

@@ -1,29 +0,0 @@
-# - Find NUMA
-# Find the NUMA library and includes
-#
-# NUMA_INCLUDE_DIRS - where to find numa.h, etc.
-# NUMA_LIBRARIES - List of libraries when using NUMA.
-# NUMA_FOUND - True if NUMA found.
-
-find_path(NUMA_INCLUDE_DIRS
-  NAMES numa.h numaif.h
-  HINTS ${NUMA_ROOT_DIR}/include)
-
-find_library(NUMA_LIBRARIES
-  NAMES numa
-  HINTS ${NUMA_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(NUMA DEFAULT_MSG NUMA_LIBRARIES NUMA_INCLUDE_DIRS)
-
-mark_as_advanced(
-  NUMA_LIBRARIES
-  NUMA_INCLUDE_DIRS)
-
-if(NUMA_FOUND AND NOT (TARGET NUMA::NUMA))
-  add_library (NUMA::NUMA UNKNOWN IMPORTED)
-  set_target_properties(NUMA::NUMA
-    PROPERTIES
-      IMPORTED_LOCATION ${NUMA_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${NUMA_INCLUDE_DIRS})
-endif()

cmake/modules/FindSnappy.cmake

@@ -1,29 +0,0 @@
-# - Find Snappy
-# Find the snappy compression library and includes
-#
-# Snappy_INCLUDE_DIRS - where to find snappy.h, etc.
-# Snappy_LIBRARIES - List of libraries when using snappy.
-# Snappy_FOUND - True if snappy found.
-
-find_path(Snappy_INCLUDE_DIRS
-  NAMES snappy.h
-  HINTS ${snappy_ROOT_DIR}/include)
-
-find_library(Snappy_LIBRARIES
-  NAMES snappy
-  HINTS ${snappy_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_LIBRARIES Snappy_INCLUDE_DIRS)
-
-mark_as_advanced(
-  Snappy_LIBRARIES
-  Snappy_INCLUDE_DIRS)
-
-if(Snappy_FOUND AND NOT (TARGET Snappy::snappy))
-  add_library (Snappy::snappy UNKNOWN IMPORTED)
-  set_target_properties(Snappy::snappy
-    PROPERTIES
-      IMPORTED_LOCATION ${Snappy_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${Snappy_INCLUDE_DIRS})
-endif()

cmake/modules/FindTBB.cmake

@@ -1,33 +0,0 @@
-# - Find TBB
-# Find the Thread Building Blocks library and includes
-#
-# TBB_INCLUDE_DIRS - where to find tbb.h, etc.
-# TBB_LIBRARIES - List of libraries when using TBB.
-# TBB_FOUND - True if TBB found.
-
-if(NOT DEFINED TBB_ROOT_DIR)
-  set(TBB_ROOT_DIR "$ENV{TBBROOT}")
-endif()
-
-find_path(TBB_INCLUDE_DIRS
-  NAMES tbb/tbb.h
-  HINTS ${TBB_ROOT_DIR}/include)
-
-find_library(TBB_LIBRARIES
-  NAMES tbb
-  HINTS ${TBB_ROOT_DIR}/lib ENV LIBRARY_PATH)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARIES TBB_INCLUDE_DIRS)
-
-mark_as_advanced(
-  TBB_LIBRARIES
-  TBB_INCLUDE_DIRS)
-
-if(TBB_FOUND AND NOT (TARGET TBB::TBB))
-  add_library (TBB::TBB UNKNOWN IMPORTED)
-  set_target_properties(TBB::TBB
-    PROPERTIES
-      IMPORTED_LOCATION ${TBB_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${TBB_INCLUDE_DIRS})
-endif()

cmake/modules/Findgflags.cmake

@@ -1,29 +0,0 @@
-# - Find gflags library
-# Find the gflags includes and library
-#
-# GFLAGS_INCLUDE_DIR - where to find gflags.h.
-# GFLAGS_LIBRARIES - List of libraries when using gflags.
-# gflags_FOUND - True if gflags found.
-
-find_path(GFLAGS_INCLUDE_DIR
-  NAMES gflags/gflags.h)
-
-find_library(GFLAGS_LIBRARIES
-  NAMES gflags)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(gflags
-  DEFAULT_MSG GFLAGS_LIBRARIES GFLAGS_INCLUDE_DIR)
-
-mark_as_advanced(
-  GFLAGS_LIBRARIES
-  GFLAGS_INCLUDE_DIR)
-
-if(gflags_FOUND AND NOT (TARGET gflags::gflags))
-  add_library(gflags::gflags UNKNOWN IMPORTED)
-  set_target_properties(gflags::gflags
-    PROPERTIES
-      IMPORTED_LOCATION ${GFLAGS_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${GFLAGS_INCLUDE_DIR}
-      IMPORTED_LINK_INTERFACE_LANGUAGES "CXX")
-endif()

cmake/modules/Findlz4.cmake

@@ -1,29 +0,0 @@
-# - Find Lz4
-# Find the lz4 compression library and includes
-#
-# lz4_INCLUDE_DIRS - where to find lz4.h, etc.
-# lz4_LIBRARIES - List of libraries when using lz4.
-# lz4_FOUND - True if lz4 found.
-
-find_path(lz4_INCLUDE_DIRS
-  NAMES lz4.h
-  HINTS ${lz4_ROOT_DIR}/include)
-
-find_library(lz4_LIBRARIES
-  NAMES lz4
-  HINTS ${lz4_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(lz4 DEFAULT_MSG lz4_LIBRARIES lz4_INCLUDE_DIRS)
-
-mark_as_advanced(
-  lz4_LIBRARIES
-  lz4_INCLUDE_DIRS)
-
-if(lz4_FOUND AND NOT (TARGET lz4::lz4))
-  add_library(lz4::lz4 UNKNOWN IMPORTED)
-  set_target_properties(lz4::lz4
-    PROPERTIES
-      IMPORTED_LOCATION ${lz4_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${lz4_INCLUDE_DIRS})
-endif()

cmake/modules/Findsnappy.cmake

@@ -0,0 +1,21 @@
+# - Find Snappy
+# Find the snappy compression library and includes
+#
+# SNAPPY_INCLUDE_DIR - where to find snappy.h, etc.
+# SNAPPY_LIBRARIES - List of libraries when using snappy.
+# SNAPPY_FOUND - True if snappy found.
+
+find_path(SNAPPY_INCLUDE_DIR
+  NAMES snappy.h
+  HINTS ${SNAPPY_ROOT_DIR}/include)
+
+find_library(SNAPPY_LIBRARIES
+  NAMES snappy
+  HINTS ${SNAPPY_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(snappy DEFAULT_MSG SNAPPY_LIBRARIES SNAPPY_INCLUDE_DIR)
+
+mark_as_advanced(
+  SNAPPY_LIBRARIES
+  SNAPPY_INCLUDE_DIR)

cmake/modules/Findzstd.cmake

@@ -1,29 +0,0 @@
-# - Find zstd
-# Find the zstd compression library and includes
-#
-# zstd_INCLUDE_DIRS - where to find zstd.h, etc.
-# zstd_LIBRARIES - List of libraries when using zstd.
-# zstd_FOUND - True if zstd found.
-
-find_path(zstd_INCLUDE_DIRS
-  NAMES zstd.h
-  HINTS ${zstd_ROOT_DIR}/include)
-
-find_library(zstd_LIBRARIES
-  NAMES zstd
-  HINTS ${zstd_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARIES zstd_INCLUDE_DIRS)
-
-mark_as_advanced(
-  zstd_LIBRARIES
-  zstd_INCLUDE_DIRS)
-
-if(zstd_FOUND AND NOT (TARGET zstd::zstd))
-  add_library (zstd::zstd UNKNOWN IMPORTED)
-  set_target_properties(zstd::zstd
-    PROPERTIES
-      IMPORTED_LOCATION ${zstd_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${zstd_INCLUDE_DIRS})
-endif()

cmake/modules/ReadVersion.cmake

@@ -1,10 +0,0 @@
-# Read rocksdb version from version.h header file.
-
-function(get_rocksdb_version version_var)
-  file(READ "${CMAKE_CURRENT_SOURCE_DIR}/include/rocksdb/version.h" version_header_file)
-  foreach(component MAJOR MINOR PATCH)
-    string(REGEX MATCH "#define ROCKSDB_${component} ([0-9]+)" _ ${version_header_file})
-    set(ROCKSDB_VERSION_${component} ${CMAKE_MATCH_1})
-  endforeach()
-  set(${version_var} "${ROCKSDB_VERSION_MAJOR}.${ROCKSDB_VERSION_MINOR}.${ROCKSDB_VERSION_PATCH}" PARENT_SCOPE)
-endfunction()

coverage/coverage_test.sh

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 
 # Exit on error.
 set -e
@@ -12,24 +11,21 @@ fi
 ROOT=".."
 # Fetch right version of gcov
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
-  source $ROOT/build_tools/fbcode_config_platform007.sh
+  source $ROOT/build_tools/fbcode_config.sh
   GCOV=$GCC_BASE/bin/gcov
 else
   GCOV=$(which gcov)
 fi
-echo -e "Using $GCOV"
 
 COVERAGE_DIR="$PWD/COVERAGE_REPORT"
 mkdir -p $COVERAGE_DIR
 
 # Find all gcno files to generate the coverage report
 
-PYTHON=${1:-`which python`}
-echo -e "Using $PYTHON"
 GCNO_FILES=`find $ROOT -name "*.gcno"`
 $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
   # Parse the raw gcov report to more human readable form.
-  $PYTHON $ROOT/coverage/parse_gcov_output.py |
+  python $ROOT/coverage/parse_gcov_output.py |
   # Write the output to both stdout and report file.
   tee $COVERAGE_DIR/coverage_report_all.txt &&
 echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n"
@@ -44,7 +40,7 @@ RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt
 
 echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT
 $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
-  $PYTHON $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
+  python $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
   tee -a $RECENT_REPORT &&
 echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
 

coverage/parse_gcov_output.py

@@ -1,12 +1,9 @@
-#!/usr/bin/env python
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-
-from __future__ import print_function
-
 import optparse
 import re
 import sys
 
+from optparse import OptionParser
+
 # the gcov report follows certain pattern. Each file will have two lines
 # of report, from which we can extract the file name, total lines and coverage
 # percentage.
@@ -50,7 +47,7 @@ def parse_gcov_report(gcov_input):
 def get_option_parser():
     usage = "Parse the gcov output and generate more human-readable code " +\
             "coverage report."
-    parser = optparse.OptionParser(usage)
+    parser = OptionParser(usage)
 
     parser.add_option(
         "--interested-files", "-i",
@@ -75,8 +72,8 @@ def display_file_coverage(per_file_coverage, total_coverage):
     header_template = \
         "%" + str(max_file_name_length) + "s\t%s\t%s"
     separator = "-" * (max_file_name_length + 10 + 20)
-    print(header_template % ("Filename", "Coverage", "Lines"))  # noqa: E999 T25377293 Grandfathered in
-    print(separator)
+    print header_template % ("Filename", "Coverage", "Lines")
+    print separator
 
     # -- Print body
     # template for printing coverage report for each file.
@@ -84,12 +81,12 @@ def display_file_coverage(per_file_coverage, total_coverage):
 
     for fname, coverage_info in per_file_coverage.items():
         coverage, lines = coverage_info
-        print(record_template % (fname, coverage, lines))
+        print record_template % (fname, coverage, lines)
 
     # -- Print footer
     if total_coverage:
-        print(separator)
-        print(record_template % ("Total", total_coverage[0], total_coverage[1]))
+        print separator
+        print record_template % ("Total", total_coverage[0], total_coverage[1])
 
 def report_coverage():
     parser = get_option_parser()
@@ -113,7 +110,7 @@ def report_coverage():
         total_coverage = None
 
     if not len(per_file_coverage):
-        print("Cannot find coverage info for the given files.", file=sys.stderr)
+        print >> sys.stderr, "Cannot find coverage info for the given files."
         return
     display_file_coverage(per_file_coverage, total_coverage)
 

db/arena_wrapped_db_iter.cc

@@ -1,106 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/arena_wrapped_db_iter.h"
-#include "memory/arena.h"
-#include "rocksdb/env.h"
-#include "rocksdb/iterator.h"
-#include "rocksdb/options.h"
-#include "table/internal_iterator.h"
-#include "table/iterator_wrapper.h"
-#include "util/user_comparator_wrapper.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
-                                       std::string* prop) {
-  if (prop_name == "rocksdb.iterator.super-version-number") {
-    // First try to pass the value returned from inner iterator.
-    if (!db_iter_->GetProperty(prop_name, prop).ok()) {
-      *prop = ToString(sv_number_);
-    }
-    return Status::OK();
-  }
-  return db_iter_->GetProperty(prop_name, prop);
-}
-
-void ArenaWrappedDBIter::Init(
-    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iteration,
-    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
-    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
-  auto mem = arena_.AllocateAligned(sizeof(DBIter));
-  db_iter_ =
-      new (mem) DBIter(env, read_options, ioptions, mutable_cf_options,
-                       ioptions.user_comparator, /* iter */ nullptr, version,
-                       sequence, true, max_sequential_skip_in_iteration,
-                       read_callback, db_impl, cfd, expose_blob_index);
-  sv_number_ = version_number;
-  read_options_ = read_options;
-  allow_refresh_ = allow_refresh;
-}
-
-Status ArenaWrappedDBIter::Refresh() {
-  if (cfd_ == nullptr || db_impl_ == nullptr || !allow_refresh_) {
-    return Status::NotSupported("Creating renew iterator is not allowed.");
-  }
-  assert(db_iter_ != nullptr);
-  // TODO(yiwu): For last_seq_same_as_publish_seq_==false, this is not the
-  // correct behavior. Will be corrected automatically when we take a snapshot
-  // here for the case of WritePreparedTxnDB.
-  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
-  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1");
-  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2");
-  if (sv_number_ != cur_sv_number) {
-    Env* env = db_iter_->env();
-    db_iter_->~DBIter();
-    arena_.~Arena();
-    new (&arena_) Arena();
-
-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
-    SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
-    if (read_callback_) {
-      read_callback_->Refresh(latest_seq);
-    }
-    Init(env, read_options_, *(cfd_->ioptions()), sv->mutable_cf_options,
-         sv->current, latest_seq,
-         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number, read_callback_, db_impl_, cfd_, expose_blob_index_,
-         allow_refresh_);
-
-    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, db_iter_->GetRangeDelAggregator(),
-        latest_seq, /* allow_unprepared_value */ true);
-    SetIterUnderDBIter(internal_iter);
-  } else {
-    db_iter_->set_sequence(db_impl_->GetLatestSequenceNumber());
-    db_iter_->set_valid(false);
-  }
-  return Status::OK();
-}
-
-ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
-    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
-  ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
-  iter->Init(env, read_options, ioptions, mutable_cf_options, version, sequence,
-             max_sequential_skip_in_iterations, version_number, read_callback,
-             db_impl, cfd, expose_blob_index, allow_refresh);
-  if (db_impl != nullptr && cfd != nullptr && allow_refresh) {
-    iter->StoreRefreshInfo(db_impl, cfd, read_callback, expose_blob_index);
-  }
-
-  return iter;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/arena_wrapped_db_iter.h

@@ -1,114 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-#include <stdint.h>
-#include <string>
-#include "db/db_impl/db_impl.h"
-#include "db/db_iter.h"
-#include "db/dbformat.h"
-#include "db/range_del_aggregator.h"
-#include "memory/arena.h"
-#include "options/cf_options.h"
-#include "rocksdb/db.h"
-#include "rocksdb/iterator.h"
-#include "util/autovector.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Arena;
-class Version;
-
-// A wrapper iterator which wraps DB Iterator and the arena, with which the DB
-// iterator is supposed to be allocated. This class is used as an entry point of
-// a iterator hierarchy whose memory can be allocated inline. In that way,
-// accessing the iterator tree can be more cache friendly. It is also faster
-// to allocate.
-// When using the class's Iterator interface, the behavior is exactly
-// the same as the inner DBIter.
-class ArenaWrappedDBIter : public Iterator {
- public:
-  virtual ~ArenaWrappedDBIter() { db_iter_->~DBIter(); }
-
-  // Get the arena to be used to allocate memory for DBIter to be wrapped,
-  // as well as child iterators in it.
-  virtual Arena* GetArena() { return &arena_; }
-  virtual ReadRangeDelAggregator* GetRangeDelAggregator() {
-    return db_iter_->GetRangeDelAggregator();
-  }
-  const ReadOptions& GetReadOptions() { return read_options_; }
-
-  // Set the internal iterator wrapped inside the DB Iterator. Usually it is
-  // a merging iterator.
-  virtual void SetIterUnderDBIter(InternalIterator* iter) {
-    db_iter_->SetIter(iter);
-  }
-
-  bool Valid() const override { return db_iter_->Valid(); }
-  void SeekToFirst() override { db_iter_->SeekToFirst(); }
-  void SeekToLast() override { db_iter_->SeekToLast(); }
-  // 'target' does not contain timestamp, even if user timestamp feature is
-  // enabled.
-  void Seek(const Slice& target) override { db_iter_->Seek(target); }
-  void SeekForPrev(const Slice& target) override {
-    db_iter_->SeekForPrev(target);
-  }
-  void Next() override { db_iter_->Next(); }
-  void Prev() override { db_iter_->Prev(); }
-  Slice key() const override { return db_iter_->key(); }
-  Slice value() const override { return db_iter_->value(); }
-  Status status() const override { return db_iter_->status(); }
-  Slice timestamp() const override { return db_iter_->timestamp(); }
-  bool IsBlob() const { return db_iter_->IsBlob(); }
-
-  Status GetProperty(std::string prop_name, std::string* prop) override;
-
-  Status Refresh() override;
-
-  void Init(Env* env, const ReadOptions& read_options,
-            const ImmutableOptions& ioptions,
-            const MutableCFOptions& mutable_cf_options, const Version* version,
-            const SequenceNumber& sequence,
-            uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-            ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd,
-            bool expose_blob_index, bool allow_refresh);
-
-  // Store some parameters so we can refresh the iterator at a later point
-  // with these same params
-  void StoreRefreshInfo(DBImpl* db_impl, ColumnFamilyData* cfd,
-                        ReadCallback* read_callback, bool expose_blob_index) {
-    db_impl_ = db_impl;
-    cfd_ = cfd;
-    read_callback_ = read_callback;
-    expose_blob_index_ = expose_blob_index;
-  }
-
- private:
-  DBIter* db_iter_;
-  Arena arena_;
-  uint64_t sv_number_;
-  ColumnFamilyData* cfd_ = nullptr;
-  DBImpl* db_impl_ = nullptr;
-  ReadOptions read_options_;
-  ReadCallback* read_callback_;
-  bool expose_blob_index_ = false;
-  bool allow_refresh_ = true;
-};
-
-// Generate the arena wrapped iterator class.
-// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
-// be supported.
-extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, ReadCallback* read_callback,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr,
-    bool expose_blob_index = false, bool allow_refresh = true);
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_constants.h

@@ -1,16 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cstdint>
-
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-constexpr uint64_t kInvalidBlobFileNumber = 0;
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_counting_iterator.h

@@ -1,146 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-
-#include "db/blob/blob_garbage_meter.h"
-#include "rocksdb/rocksdb_namespace.h"
-#include "rocksdb/status.h"
-#include "table/internal_iterator.h"
-#include "test_util/sync_point.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// An internal iterator that passes each key-value encountered to
-// BlobGarbageMeter as inflow in order to measure the total number and size of
-// blobs in the compaction input on a per-blob file basis.
-class BlobCountingIterator : public InternalIterator {
- public:
-  BlobCountingIterator(InternalIterator* iter,
-                       BlobGarbageMeter* blob_garbage_meter)
-      : iter_(iter), blob_garbage_meter_(blob_garbage_meter) {
-    assert(iter_);
-    assert(blob_garbage_meter_);
-
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  bool Valid() const override { return iter_->Valid() && status_.ok(); }
-
-  void SeekToFirst() override {
-    iter_->SeekToFirst();
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  void SeekToLast() override {
-    iter_->SeekToLast();
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  void Seek(const Slice& target) override {
-    iter_->Seek(target);
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  void SeekForPrev(const Slice& target) override {
-    iter_->SeekForPrev(target);
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  void Next() override {
-    assert(Valid());
-
-    iter_->Next();
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  bool NextAndGetResult(IterateResult* result) override {
-    assert(Valid());
-
-    const bool res = iter_->NextAndGetResult(result);
-    UpdateAndCountBlobIfNeeded();
-    return res;
-  }
-
-  void Prev() override {
-    assert(Valid());
-
-    iter_->Prev();
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  Slice key() const override {
-    assert(Valid());
-    return iter_->key();
-  }
-
-  Slice user_key() const override {
-    assert(Valid());
-    return iter_->user_key();
-  }
-
-  Slice value() const override {
-    assert(Valid());
-    return iter_->value();
-  }
-
-  Status status() const override { return status_; }
-
-  bool PrepareValue() override {
-    assert(Valid());
-    return iter_->PrepareValue();
-  }
-
-  bool MayBeOutOfLowerBound() override {
-    assert(Valid());
-    return iter_->MayBeOutOfLowerBound();
-  }
-
-  IterBoundCheck UpperBoundCheckResult() override {
-    assert(Valid());
-    return iter_->UpperBoundCheckResult();
-  }
-
-  void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override {
-    iter_->SetPinnedItersMgr(pinned_iters_mgr);
-  }
-
-  bool IsKeyPinned() const override {
-    assert(Valid());
-    return iter_->IsKeyPinned();
-  }
-
-  bool IsValuePinned() const override {
-    assert(Valid());
-    return iter_->IsValuePinned();
-  }
-
-  Status GetProperty(std::string prop_name, std::string* prop) override {
-    return iter_->GetProperty(prop_name, prop);
-  }
-
- private:
-  void UpdateAndCountBlobIfNeeded() {
-    assert(!iter_->Valid() || iter_->status().ok());
-
-    if (!iter_->Valid()) {
-      status_ = iter_->status();
-      return;
-    }
-
-    TEST_SYNC_POINT(
-        "BlobCountingIterator::UpdateAndCountBlobIfNeeded:ProcessInFlow");
-
-    status_ = blob_garbage_meter_->ProcessInFlow(key(), value());
-  }
-
-  InternalIterator* iter_;
-  BlobGarbageMeter* blob_garbage_meter_;
-  Status status_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_counting_iterator_test.cc

@@ -1,325 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_counting_iterator.h"
-
-#include <string>
-#include <vector>
-
-#include "db/blob/blob_garbage_meter.h"
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/dbformat.h"
-#include "test_util/testharness.h"
-#include "test_util/testutil.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-void CheckInFlow(const BlobGarbageMeter& blob_garbage_meter,
-                 uint64_t blob_file_number, uint64_t count, uint64_t bytes) {
-  const auto& flows = blob_garbage_meter.flows();
-
-  const auto it = flows.find(blob_file_number);
-  if (it == flows.end()) {
-    ASSERT_EQ(count, 0);
-    ASSERT_EQ(bytes, 0);
-    return;
-  }
-
-  const auto& in = it->second.GetInFlow();
-
-  ASSERT_EQ(in.GetCount(), count);
-  ASSERT_EQ(in.GetBytes(), bytes);
-}
-
-TEST(BlobCountingIteratorTest, CountBlobs) {
-  // Note: the input consists of three key-values: two are blob references to
-  // different blob files, while the third one is a plain value.
-  constexpr char user_key0[] = "key0";
-  constexpr char user_key1[] = "key1";
-  constexpr char user_key2[] = "key2";
-
-  const std::vector<std::string> keys{
-      test::KeyStr(user_key0, 1, kTypeBlobIndex),
-      test::KeyStr(user_key1, 2, kTypeBlobIndex),
-      test::KeyStr(user_key2, 3, kTypeValue)};
-
-  constexpr uint64_t first_blob_file_number = 4;
-  constexpr uint64_t first_offset = 1000;
-  constexpr uint64_t first_size = 2000;
-
-  std::string first_blob_index;
-  BlobIndex::EncodeBlob(&first_blob_index, first_blob_file_number, first_offset,
-                        first_size, kNoCompression);
-
-  constexpr uint64_t second_blob_file_number = 6;
-  constexpr uint64_t second_offset = 2000;
-  constexpr uint64_t second_size = 4000;
-
-  std::string second_blob_index;
-  BlobIndex::EncodeBlob(&second_blob_index, second_blob_file_number,
-                        second_offset, second_size, kNoCompression);
-
-  const std::vector<std::string> values{first_blob_index, second_blob_index,
-                                        "raw_value"};
-
-  assert(keys.size() == values.size());
-
-  test::VectorIterator input(keys, values);
-  BlobGarbageMeter blob_garbage_meter;
-
-  BlobCountingIterator blob_counter(&input, &blob_garbage_meter);
-
-  constexpr uint64_t first_expected_bytes =
-      first_size +
-      BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(user_key0) - 1);
-  constexpr uint64_t second_expected_bytes =
-      second_size +
-      BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(user_key1) - 1);
-
-  // Call SeekToFirst and iterate forward
-  blob_counter.SeekToFirst();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 1,
-              first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 0, 0);
-
-  blob_counter.Next();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[1]);
-  ASSERT_EQ(blob_counter.user_key(), user_key1);
-  ASSERT_EQ(blob_counter.value(), values[1]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 1,
-              first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 1,
-              second_expected_bytes);
-
-  blob_counter.Next();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[2]);
-  ASSERT_EQ(blob_counter.user_key(), user_key2);
-  ASSERT_EQ(blob_counter.value(), values[2]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 1,
-              first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 1,
-              second_expected_bytes);
-
-  blob_counter.Next();
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 1,
-              first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 1,
-              second_expected_bytes);
-
-  // Do it again using NextAndGetResult
-  blob_counter.SeekToFirst();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-              2 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 1,
-              second_expected_bytes);
-
-  {
-    IterateResult result;
-    ASSERT_TRUE(blob_counter.NextAndGetResult(&result));
-    ASSERT_EQ(result.key, keys[1]);
-    ASSERT_EQ(blob_counter.user_key(), user_key1);
-    ASSERT_TRUE(blob_counter.Valid());
-    ASSERT_OK(blob_counter.status());
-    ASSERT_EQ(blob_counter.key(), keys[1]);
-    ASSERT_EQ(blob_counter.value(), values[1]);
-    CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-                2 * first_expected_bytes);
-    CheckInFlow(blob_garbage_meter, second_blob_file_number, 2,
-                2 * second_expected_bytes);
-  }
-
-  {
-    IterateResult result;
-    ASSERT_TRUE(blob_counter.NextAndGetResult(&result));
-    ASSERT_EQ(result.key, keys[2]);
-    ASSERT_EQ(blob_counter.user_key(), user_key2);
-    ASSERT_TRUE(blob_counter.Valid());
-    ASSERT_OK(blob_counter.status());
-    ASSERT_EQ(blob_counter.key(), keys[2]);
-    ASSERT_EQ(blob_counter.value(), values[2]);
-    CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-                2 * first_expected_bytes);
-    CheckInFlow(blob_garbage_meter, second_blob_file_number, 2,
-                2 * second_expected_bytes);
-  }
-
-  {
-    IterateResult result;
-    ASSERT_FALSE(blob_counter.NextAndGetResult(&result));
-    ASSERT_FALSE(blob_counter.Valid());
-    ASSERT_OK(blob_counter.status());
-    CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-                2 * first_expected_bytes);
-    CheckInFlow(blob_garbage_meter, second_blob_file_number, 2,
-                2 * second_expected_bytes);
-  }
-
-  // Call SeekToLast and iterate backward
-  blob_counter.SeekToLast();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[2]);
-  ASSERT_EQ(blob_counter.user_key(), user_key2);
-  ASSERT_EQ(blob_counter.value(), values[2]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-              2 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 2,
-              2 * second_expected_bytes);
-
-  blob_counter.Prev();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[1]);
-  ASSERT_EQ(blob_counter.user_key(), user_key1);
-  ASSERT_EQ(blob_counter.value(), values[1]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-              2 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 3,
-              3 * second_expected_bytes);
-
-  blob_counter.Prev();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 3,
-              3 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 3,
-              3 * second_expected_bytes);
-
-  blob_counter.Prev();
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 3,
-              3 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 3,
-              3 * second_expected_bytes);
-
-  // Call Seek for all keys (plus one that's greater than all of them)
-  blob_counter.Seek(keys[0]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 3,
-              3 * second_expected_bytes);
-
-  blob_counter.Seek(keys[1]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[1]);
-  ASSERT_EQ(blob_counter.user_key(), user_key1);
-  ASSERT_EQ(blob_counter.value(), values[1]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  blob_counter.Seek(keys[2]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[2]);
-  ASSERT_EQ(blob_counter.user_key(), user_key2);
-  ASSERT_EQ(blob_counter.value(), values[2]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  blob_counter.Seek("zzz");
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  // Call SeekForPrev for all keys (plus one that's less than all of them)
-  blob_counter.SeekForPrev("aaa");
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  blob_counter.SeekForPrev(keys[0]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 5,
-              5 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  blob_counter.SeekForPrev(keys[1]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[1]);
-  ASSERT_EQ(blob_counter.user_key(), user_key1);
-  ASSERT_EQ(blob_counter.value(), values[1]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 5,
-              5 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 5,
-              5 * second_expected_bytes);
-
-  blob_counter.SeekForPrev(keys[2]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[2]);
-  ASSERT_EQ(blob_counter.user_key(), user_key2);
-  ASSERT_EQ(blob_counter.value(), values[2]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 5,
-              5 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 5,
-              5 * second_expected_bytes);
-}
-
-TEST(BlobCountingIteratorTest, CorruptBlobIndex) {
-  const std::vector<std::string> keys{
-      test::KeyStr("user_key", 1, kTypeBlobIndex)};
-  const std::vector<std::string> values{"i_am_not_a_blob_index"};
-
-  assert(keys.size() == values.size());
-
-  test::VectorIterator input(keys, values);
-  BlobGarbageMeter blob_garbage_meter;
-
-  BlobCountingIterator blob_counter(&input, &blob_garbage_meter);
-
-  blob_counter.SeekToFirst();
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_NOK(blob_counter.status());
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_fetcher.cc

@@ -1,22 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_fetcher.h"
-
-#include "db/version_set.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Status BlobFetcher::FetchBlob(const Slice& user_key, const Slice& blob_index,
-                              PinnableSlice* blob_value) {
-  Status s;
-  assert(version_);
-  constexpr uint64_t* bytes_read = nullptr;
-  s = version_->GetBlob(read_options_, user_key, blob_index, blob_value,
-                        bytes_read);
-  return s;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_fetcher.h

@@ -1,26 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include "rocksdb/options.h"
-#include "rocksdb/status.h"
-
-namespace ROCKSDB_NAMESPACE {
-class Version;
-
-class BlobFetcher {
- public:
-  BlobFetcher(Version* version, const ReadOptions& read_options)
-      : version_(version), read_options_(read_options) {}
-
-  Status FetchBlob(const Slice& user_key, const Slice& blob_index,
-                   PinnableSlice* blob_value);
-
- private:
-  Version* version_;
-  ReadOptions read_options_;
-};
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_addition.cc

@@ -1,156 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_addition.h"
-
-#include <ostream>
-#include <sstream>
-
-#include "logging/event_logger.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "test_util/sync_point.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Tags for custom fields. Note that these get persisted in the manifest,
-// so existing tags should not be modified.
-enum BlobFileAddition::CustomFieldTags : uint32_t {
-  kEndMarker,
-
-  // Add forward compatible fields here
-
-  /////////////////////////////////////////////////////////////////////
-
-  kForwardIncompatibleMask = 1 << 6,
-
-  // Add forward incompatible fields here
-};
-
-void BlobFileAddition::EncodeTo(std::string* output) const {
-  PutVarint64(output, blob_file_number_);
-  PutVarint64(output, total_blob_count_);
-  PutVarint64(output, total_blob_bytes_);
-  PutLengthPrefixedSlice(output, checksum_method_);
-  PutLengthPrefixedSlice(output, checksum_value_);
-
-  // Encode any custom fields here. The format to use is a Varint32 tag (see
-  // CustomFieldTags above) followed by a length prefixed slice. Unknown custom
-  // fields will be ignored during decoding unless they're in the forward
-  // incompatible range.
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileAddition::EncodeTo::CustomFields", output);
-
-  PutVarint32(output, kEndMarker);
-}
-
-Status BlobFileAddition::DecodeFrom(Slice* input) {
-  constexpr char class_name[] = "BlobFileAddition";
-
-  if (!GetVarint64(input, &blob_file_number_)) {
-    return Status::Corruption(class_name, "Error decoding blob file number");
-  }
-
-  if (!GetVarint64(input, &total_blob_count_)) {
-    return Status::Corruption(class_name, "Error decoding total blob count");
-  }
-
-  if (!GetVarint64(input, &total_blob_bytes_)) {
-    return Status::Corruption(class_name, "Error decoding total blob bytes");
-  }
-
-  Slice checksum_method;
-  if (!GetLengthPrefixedSlice(input, &checksum_method)) {
-    return Status::Corruption(class_name, "Error decoding checksum method");
-  }
-  checksum_method_ = checksum_method.ToString();
-
-  Slice checksum_value;
-  if (!GetLengthPrefixedSlice(input, &checksum_value)) {
-    return Status::Corruption(class_name, "Error decoding checksum value");
-  }
-  checksum_value_ = checksum_value.ToString();
-
-  while (true) {
-    uint32_t custom_field_tag = 0;
-    if (!GetVarint32(input, &custom_field_tag)) {
-      return Status::Corruption(class_name, "Error decoding custom field tag");
-    }
-
-    if (custom_field_tag == kEndMarker) {
-      break;
-    }
-
-    if (custom_field_tag & kForwardIncompatibleMask) {
-      return Status::Corruption(
-          class_name, "Forward incompatible custom field encountered");
-    }
-
-    Slice custom_field_value;
-    if (!GetLengthPrefixedSlice(input, &custom_field_value)) {
-      return Status::Corruption(class_name,
-                                "Error decoding custom field value");
-    }
-  }
-
-  return Status::OK();
-}
-
-std::string BlobFileAddition::DebugString() const {
-  std::ostringstream oss;
-
-  oss << *this;
-
-  return oss.str();
-}
-
-std::string BlobFileAddition::DebugJSON() const {
-  JSONWriter jw;
-
-  jw << *this;
-
-  jw.EndObject();
-
-  return jw.Get();
-}
-
-bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs) {
-  return lhs.GetBlobFileNumber() == rhs.GetBlobFileNumber() &&
-         lhs.GetTotalBlobCount() == rhs.GetTotalBlobCount() &&
-         lhs.GetTotalBlobBytes() == rhs.GetTotalBlobBytes() &&
-         lhs.GetChecksumMethod() == rhs.GetChecksumMethod() &&
-         lhs.GetChecksumValue() == rhs.GetChecksumValue();
-}
-
-bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs) {
-  return !(lhs == rhs);
-}
-
-std::ostream& operator<<(std::ostream& os,
-                         const BlobFileAddition& blob_file_addition) {
-  os << "blob_file_number: " << blob_file_addition.GetBlobFileNumber()
-     << " total_blob_count: " << blob_file_addition.GetTotalBlobCount()
-     << " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes()
-     << " checksum_method: " << blob_file_addition.GetChecksumMethod()
-     << " checksum_value: "
-     << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true);
-
-  return os;
-}
-
-JSONWriter& operator<<(JSONWriter& jw,
-                       const BlobFileAddition& blob_file_addition) {
-  jw << "BlobFileNumber" << blob_file_addition.GetBlobFileNumber()
-     << "TotalBlobCount" << blob_file_addition.GetTotalBlobCount()
-     << "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes()
-     << "ChecksumMethod" << blob_file_addition.GetChecksumMethod()
-     << "ChecksumValue"
-     << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true);
-
-  return jw;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_addition.h

@@ -1,67 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-#include <cstdint>
-#include <iosfwd>
-#include <string>
-
-#include "db/blob/blob_constants.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class JSONWriter;
-class Slice;
-class Status;
-
-class BlobFileAddition {
- public:
-  BlobFileAddition() = default;
-
-  BlobFileAddition(uint64_t blob_file_number, uint64_t total_blob_count,
-                   uint64_t total_blob_bytes, std::string checksum_method,
-                   std::string checksum_value)
-      : blob_file_number_(blob_file_number),
-        total_blob_count_(total_blob_count),
-        total_blob_bytes_(total_blob_bytes),
-        checksum_method_(std::move(checksum_method)),
-        checksum_value_(std::move(checksum_value)) {
-    assert(checksum_method_.empty() == checksum_value_.empty());
-  }
-
-  uint64_t GetBlobFileNumber() const { return blob_file_number_; }
-  uint64_t GetTotalBlobCount() const { return total_blob_count_; }
-  uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; }
-  const std::string& GetChecksumMethod() const { return checksum_method_; }
-  const std::string& GetChecksumValue() const { return checksum_value_; }
-
-  void EncodeTo(std::string* output) const;
-  Status DecodeFrom(Slice* input);
-
-  std::string DebugString() const;
-  std::string DebugJSON() const;
-
- private:
-  enum CustomFieldTags : uint32_t;
-
-  uint64_t blob_file_number_ = kInvalidBlobFileNumber;
-  uint64_t total_blob_count_ = 0;
-  uint64_t total_blob_bytes_ = 0;
-  std::string checksum_method_;
-  std::string checksum_value_;
-};
-
-bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs);
-bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs);
-
-std::ostream& operator<<(std::ostream& os,
-                         const BlobFileAddition& blob_file_addition);
-JSONWriter& operator<<(JSONWriter& jw,
-                       const BlobFileAddition& blob_file_addition);
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_addition_test.cc

@@ -1,210 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_addition.h"
-
-#include <cstdint>
-#include <cstring>
-#include <string>
-
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class BlobFileAdditionTest : public testing::Test {
- public:
-  static void TestEncodeDecode(const BlobFileAddition& blob_file_addition) {
-    std::string encoded;
-    blob_file_addition.EncodeTo(&encoded);
-
-    BlobFileAddition decoded;
-    Slice input(encoded);
-    ASSERT_OK(decoded.DecodeFrom(&input));
-
-    ASSERT_EQ(blob_file_addition, decoded);
-  }
-};
-
-TEST_F(BlobFileAdditionTest, Empty) {
-  BlobFileAddition blob_file_addition;
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), kInvalidBlobFileNumber);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 0);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), 0);
-  ASSERT_TRUE(blob_file_addition.GetChecksumMethod().empty());
-  ASSERT_TRUE(blob_file_addition.GetChecksumValue().empty());
-
-  TestEncodeDecode(blob_file_addition);
-}
-
-TEST_F(BlobFileAdditionTest, NonEmpty) {
-  constexpr uint64_t blob_file_number = 123;
-  constexpr uint64_t total_blob_count = 2;
-  constexpr uint64_t total_blob_bytes = 123456;
-  const std::string checksum_method("SHA1");
-  const std::string checksum_value(
-      "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
-      "\x5c\xbd");
-
-  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
-                                      total_blob_bytes, checksum_method,
-                                      checksum_value);
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), total_blob_count);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), total_blob_bytes);
-  ASSERT_EQ(blob_file_addition.GetChecksumMethod(), checksum_method);
-  ASSERT_EQ(blob_file_addition.GetChecksumValue(), checksum_value);
-
-  TestEncodeDecode(blob_file_addition);
-}
-
-TEST_F(BlobFileAdditionTest, DecodeErrors) {
-  std::string str;
-  Slice slice(str);
-
-  BlobFileAddition blob_file_addition;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "blob file number"));
-  }
-
-  constexpr uint64_t blob_file_number = 123;
-  PutVarint64(&str, blob_file_number);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "total blob count"));
-  }
-
-  constexpr uint64_t total_blob_count = 4567;
-  PutVarint64(&str, total_blob_count);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "total blob bytes"));
-  }
-
-  constexpr uint64_t total_blob_bytes = 12345678;
-  PutVarint64(&str, total_blob_bytes);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "checksum method"));
-  }
-
-  constexpr char checksum_method[] = "SHA1";
-  PutLengthPrefixedSlice(&str, checksum_method);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "checksum value"));
-  }
-
-  constexpr char checksum_value[] =
-      "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
-      "\x5c\xbd";
-  PutLengthPrefixedSlice(&str, checksum_value);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "custom field tag"));
-  }
-
-  constexpr uint32_t custom_tag = 2;
-  PutVarint32(&str, custom_tag);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "custom field value"));
-  }
-}
-
-TEST_F(BlobFileAdditionTest, ForwardCompatibleCustomField) {
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) {
-        std::string* output = static_cast<std::string*>(arg);
-
-        constexpr uint32_t forward_compatible_tag = 2;
-        PutVarint32(output, forward_compatible_tag);
-
-        PutLengthPrefixedSlice(output, "deadbeef");
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr uint64_t blob_file_number = 678;
-  constexpr uint64_t total_blob_count = 9999;
-  constexpr uint64_t total_blob_bytes = 100000000;
-  const std::string checksum_method("CRC32");
-  const std::string checksum_value("\x3d\x87\xff\x57");
-
-  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
-                                      total_blob_bytes, checksum_method,
-                                      checksum_value);
-
-  TestEncodeDecode(blob_file_addition);
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) {
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) {
-        std::string* output = static_cast<std::string*>(arg);
-
-        constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1;
-        PutVarint32(output, forward_incompatible_tag);
-
-        PutLengthPrefixedSlice(output, "foobar");
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr uint64_t blob_file_number = 456;
-  constexpr uint64_t total_blob_count = 100;
-  constexpr uint64_t total_blob_bytes = 2000000;
-  const std::string checksum_method("CRC32B");
-  const std::string checksum_value("\x6d\xbd\xf2\x3a");
-
-  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
-                                      total_blob_bytes, checksum_method,
-                                      checksum_value);
-
-  std::string encoded;
-  blob_file_addition.EncodeTo(&encoded);
-
-  BlobFileAddition decoded_blob_file_addition;
-  Slice input(encoded);
-  const Status s = decoded_blob_file_addition.DecodeFrom(&input);
-
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible"));
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_file_builder.cc

@@ -1,359 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_builder.h"
-
-#include <cassert>
-
-#include "db/blob/blob_file_addition.h"
-#include "db/blob/blob_file_completion_callback.h"
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_writer.h"
-#include "db/version_set.h"
-#include "file/filename.h"
-#include "file/read_write_util.h"
-#include "file/writable_file_writer.h"
-#include "logging/logging.h"
-#include "options/cf_options.h"
-#include "options/options_helper.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "test_util/sync_point.h"
-#include "trace_replay/io_tracer.h"
-#include "util/compression.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlobFileBuilder::BlobFileBuilder(
-    VersionSet* versions, FileSystem* fs,
-    const ImmutableOptions* immutable_options,
-    const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
-    int job_id, uint32_t column_family_id,
-    const std::string& column_family_name, Env::IOPriority io_priority,
-    Env::WriteLifeTimeHint write_hint,
-    const std::shared_ptr<IOTracer>& io_tracer,
-    BlobFileCompletionCallback* blob_callback,
-    std::vector<std::string>* blob_file_paths,
-    std::vector<BlobFileAddition>* blob_file_additions)
-    : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs,
-                      immutable_options, mutable_cf_options, file_options,
-                      job_id, column_family_id, column_family_name, io_priority,
-                      write_hint, io_tracer, blob_callback, blob_file_paths,
-                      blob_file_additions) {}
-
-BlobFileBuilder::BlobFileBuilder(
-    std::function<uint64_t()> file_number_generator, FileSystem* fs,
-    const ImmutableOptions* immutable_options,
-    const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
-    int job_id, uint32_t column_family_id,
-    const std::string& column_family_name, Env::IOPriority io_priority,
-    Env::WriteLifeTimeHint write_hint,
-    const std::shared_ptr<IOTracer>& io_tracer,
-    BlobFileCompletionCallback* blob_callback,
-    std::vector<std::string>* blob_file_paths,
-    std::vector<BlobFileAddition>* blob_file_additions)
-    : file_number_generator_(std::move(file_number_generator)),
-      fs_(fs),
-      immutable_options_(immutable_options),
-      min_blob_size_(mutable_cf_options->min_blob_size),
-      blob_file_size_(mutable_cf_options->blob_file_size),
-      blob_compression_type_(mutable_cf_options->blob_compression_type),
-      file_options_(file_options),
-      job_id_(job_id),
-      column_family_id_(column_family_id),
-      column_family_name_(column_family_name),
-      io_priority_(io_priority),
-      write_hint_(write_hint),
-      io_tracer_(io_tracer),
-      blob_callback_(blob_callback),
-      blob_file_paths_(blob_file_paths),
-      blob_file_additions_(blob_file_additions),
-      blob_count_(0),
-      blob_bytes_(0) {
-  assert(file_number_generator_);
-  assert(fs_);
-  assert(immutable_options_);
-  assert(file_options_);
-  assert(blob_file_paths_);
-  assert(blob_file_paths_->empty());
-  assert(blob_file_additions_);
-  assert(blob_file_additions_->empty());
-}
-
-BlobFileBuilder::~BlobFileBuilder() = default;
-
-Status BlobFileBuilder::Add(const Slice& key, const Slice& value,
-                            std::string* blob_index) {
-  assert(blob_index);
-  assert(blob_index->empty());
-
-  if (value.size() < min_blob_size_) {
-    return Status::OK();
-  }
-
-  {
-    const Status s = OpenBlobFileIfNeeded();
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  Slice blob = value;
-  std::string compressed_blob;
-
-  {
-    const Status s = CompressBlobIfNeeded(&blob, &compressed_blob);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  uint64_t blob_file_number = 0;
-  uint64_t blob_offset = 0;
-
-  {
-    const Status s =
-        WriteBlobToFile(key, blob, &blob_file_number, &blob_offset);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  {
-    const Status s = CloseBlobFileIfNeeded();
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset, blob.size(),
-                        blob_compression_type_);
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::Finish() {
-  if (!IsBlobFileOpen()) {
-    return Status::OK();
-  }
-
-  return CloseBlobFile();
-}
-
-bool BlobFileBuilder::IsBlobFileOpen() const { return !!writer_; }
-
-Status BlobFileBuilder::OpenBlobFileIfNeeded() {
-  if (IsBlobFileOpen()) {
-    return Status::OK();
-  }
-
-  assert(!blob_count_);
-  assert(!blob_bytes_);
-
-  assert(file_number_generator_);
-  const uint64_t blob_file_number = file_number_generator_();
-
-  assert(immutable_options_);
-  assert(!immutable_options_->cf_paths.empty());
-  std::string blob_file_path =
-      BlobFileName(immutable_options_->cf_paths.front().path, blob_file_number);
-
-  std::unique_ptr<FSWritableFile> file;
-
-  {
-    assert(file_options_);
-    Status s = NewWritableFile(fs_, blob_file_path, &file, *file_options_);
-
-    TEST_SYNC_POINT_CALLBACK(
-        "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile", &s);
-
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  // Note: files get added to blob_file_paths_ right after the open, so they
-  // can be cleaned up upon failure. Contrast this with blob_file_additions_,
-  // which only contains successfully written files.
-  assert(blob_file_paths_);
-  blob_file_paths_->emplace_back(std::move(blob_file_path));
-
-  assert(file);
-  file->SetIOPriority(io_priority_);
-  file->SetWriteLifeTimeHint(write_hint_);
-  FileTypeSet tmp_set = immutable_options_->checksum_handoff_file_types;
-  Statistics* const statistics = immutable_options_->stats;
-  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-      std::move(file), blob_file_paths_->back(), *file_options_,
-      immutable_options_->clock, io_tracer_, statistics,
-      immutable_options_->listeners,
-      immutable_options_->file_checksum_gen_factory.get(),
-      tmp_set.Contains(FileType::kBlobFile), false));
-
-  constexpr bool do_flush = false;
-
-  std::unique_ptr<BlobLogWriter> blob_log_writer(new BlobLogWriter(
-      std::move(file_writer), immutable_options_->clock, statistics,
-      blob_file_number, immutable_options_->use_fsync, do_flush));
-
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-
-  BlobLogHeader header(column_family_id_, blob_compression_type_, has_ttl,
-                       expiration_range);
-
-  {
-    Status s = blob_log_writer->WriteHeader(header);
-
-    TEST_SYNC_POINT_CALLBACK(
-        "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", &s);
-
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  writer_ = std::move(blob_log_writer);
-
-  assert(IsBlobFileOpen());
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::CompressBlobIfNeeded(
-    Slice* blob, std::string* compressed_blob) const {
-  assert(blob);
-  assert(compressed_blob);
-  assert(compressed_blob->empty());
-  assert(immutable_options_);
-
-  if (blob_compression_type_ == kNoCompression) {
-    return Status::OK();
-  }
-
-  CompressionOptions opts;
-  CompressionContext context(blob_compression_type_);
-  constexpr uint64_t sample_for_compression = 0;
-
-  CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
-                       blob_compression_type_, sample_for_compression);
-
-  constexpr uint32_t compression_format_version = 2;
-
-  bool success = false;
-
-  {
-    StopWatch stop_watch(immutable_options_->clock, immutable_options_->stats,
-                         BLOB_DB_COMPRESSION_MICROS);
-    success =
-        CompressData(*blob, info, compression_format_version, compressed_blob);
-  }
-
-  if (!success) {
-    return Status::Corruption("Error compressing blob");
-  }
-
-  *blob = Slice(*compressed_blob);
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob,
-                                        uint64_t* blob_file_number,
-                                        uint64_t* blob_offset) {
-  assert(IsBlobFileOpen());
-  assert(blob_file_number);
-  assert(blob_offset);
-
-  uint64_t key_offset = 0;
-
-  Status s = writer_->AddRecord(key, blob, &key_offset, blob_offset);
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AddRecord", &s);
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  *blob_file_number = writer_->get_log_number();
-
-  ++blob_count_;
-  blob_bytes_ += BlobLogRecord::kHeaderSize + key.size() + blob.size();
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::CloseBlobFile() {
-  assert(IsBlobFileOpen());
-
-  BlobLogFooter footer;
-  footer.blob_count = blob_count_;
-
-  std::string checksum_method;
-  std::string checksum_value;
-
-  Status s = writer_->AppendFooter(footer, &checksum_method, &checksum_value);
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AppendFooter", &s);
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  const uint64_t blob_file_number = writer_->get_log_number();
-
-  assert(blob_file_additions_);
-  blob_file_additions_->emplace_back(blob_file_number, blob_count_, blob_bytes_,
-                                     std::move(checksum_method),
-                                     std::move(checksum_value));
-
-  assert(immutable_options_);
-  ROCKS_LOG_INFO(immutable_options_->logger,
-                 "[%s] [JOB %d] Generated blob file #%" PRIu64 ": %" PRIu64
-                 " total blobs, %" PRIu64 " total bytes",
-                 column_family_name_.c_str(), job_id_, blob_file_number,
-                 blob_count_, blob_bytes_);
-  if (blob_callback_) {
-    s = blob_callback_->OnBlobFileCompleted(blob_file_paths_->back());
-  }
-
-  writer_.reset();
-  blob_count_ = 0;
-  blob_bytes_ = 0;
-
-  return s;
-}
-
-Status BlobFileBuilder::CloseBlobFileIfNeeded() {
-  assert(IsBlobFileOpen());
-
-  const WritableFileWriter* const file_writer = writer_->file();
-  assert(file_writer);
-
-  if (file_writer->GetFileSize() < blob_file_size_) {
-    return Status::OK();
-  }
-
-  return CloseBlobFile();
-}
-
-void BlobFileBuilder::Abandon() {
-  if (!IsBlobFileOpen()) {
-    return;
-  }
-
-  if (blob_callback_) {
-    // BlobFileBuilder::Abandon() is called because of error while writing to
-    // Blob files. So we can ignore the below error.
-    blob_callback_->OnBlobFileCompleted(blob_file_paths_->back())
-        .PermitUncheckedError();
-  }
-
-  writer_.reset();
-  blob_count_ = 0;
-  blob_bytes_ = 0;
-}
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_builder.h

@@ -1,99 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <cinttypes>
-#include <functional>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "rocksdb/compression_type.h"
-#include "rocksdb/env.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class VersionSet;
-class FileSystem;
-class SystemClock;
-struct ImmutableOptions;
-struct MutableCFOptions;
-struct FileOptions;
-class BlobFileAddition;
-class Status;
-class Slice;
-class BlobLogWriter;
-class IOTracer;
-class BlobFileCompletionCallback;
-
-class BlobFileBuilder {
- public:
-  BlobFileBuilder(VersionSet* versions, FileSystem* fs,
-                  const ImmutableOptions* immutable_options,
-                  const MutableCFOptions* mutable_cf_options,
-                  const FileOptions* file_options, int job_id,
-                  uint32_t column_family_id,
-                  const std::string& column_family_name,
-                  Env::IOPriority io_priority,
-                  Env::WriteLifeTimeHint write_hint,
-                  const std::shared_ptr<IOTracer>& io_tracer,
-                  BlobFileCompletionCallback* blob_callback,
-                  std::vector<std::string>* blob_file_paths,
-                  std::vector<BlobFileAddition>* blob_file_additions);
-
-  BlobFileBuilder(std::function<uint64_t()> file_number_generator,
-                  FileSystem* fs, const ImmutableOptions* immutable_options,
-                  const MutableCFOptions* mutable_cf_options,
-                  const FileOptions* file_options, int job_id,
-                  uint32_t column_family_id,
-                  const std::string& column_family_name,
-                  Env::IOPriority io_priority,
-                  Env::WriteLifeTimeHint write_hint,
-                  const std::shared_ptr<IOTracer>& io_tracer,
-                  BlobFileCompletionCallback* blob_callback,
-                  std::vector<std::string>* blob_file_paths,
-                  std::vector<BlobFileAddition>* blob_file_additions);
-
-  BlobFileBuilder(const BlobFileBuilder&) = delete;
-  BlobFileBuilder& operator=(const BlobFileBuilder&) = delete;
-
-  ~BlobFileBuilder();
-
-  Status Add(const Slice& key, const Slice& value, std::string* blob_index);
-  Status Finish();
-  void Abandon();
-
- private:
-  bool IsBlobFileOpen() const;
-  Status OpenBlobFileIfNeeded();
-  Status CompressBlobIfNeeded(Slice* blob, std::string* compressed_blob) const;
-  Status WriteBlobToFile(const Slice& key, const Slice& blob,
-                         uint64_t* blob_file_number, uint64_t* blob_offset);
-  Status CloseBlobFile();
-  Status CloseBlobFileIfNeeded();
-
-  std::function<uint64_t()> file_number_generator_;
-  FileSystem* fs_;
-  const ImmutableOptions* immutable_options_;
-  uint64_t min_blob_size_;
-  uint64_t blob_file_size_;
-  CompressionType blob_compression_type_;
-  const FileOptions* file_options_;
-  int job_id_;
-  uint32_t column_family_id_;
-  std::string column_family_name_;
-  Env::IOPriority io_priority_;
-  Env::WriteLifeTimeHint write_hint_;
-  std::shared_ptr<IOTracer> io_tracer_;
-  BlobFileCompletionCallback* blob_callback_;
-  std::vector<std::string>* blob_file_paths_;
-  std::vector<BlobFileAddition>* blob_file_additions_;
-  std::unique_ptr<BlobLogWriter> writer_;
-  uint64_t blob_count_;
-  uint64_t blob_bytes_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_builder_test.cc

@@ -1,666 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_builder.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "db/blob/blob_file_addition.h"
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_sequential_reader.h"
-#include "env/mock_env.h"
-#include "file/filename.h"
-#include "file/random_access_file_reader.h"
-#include "options/cf_options.h"
-#include "rocksdb/env.h"
-#include "rocksdb/file_checksum.h"
-#include "rocksdb/options.h"
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-#include "util/compression.h"
-#include "utilities/fault_injection_env.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class TestFileNumberGenerator {
- public:
-  uint64_t operator()() { return ++next_file_number_; }
-
- private:
-  uint64_t next_file_number_ = 1;
-};
-
-class BlobFileBuilderTest : public testing::Test {
- protected:
-  BlobFileBuilderTest() : mock_env_(Env::Default()) {
-    fs_ = mock_env_.GetFileSystem().get();
-    clock_ = mock_env_.GetSystemClock().get();
-  }
-
-  void VerifyBlobFile(uint64_t blob_file_number,
-                      const std::string& blob_file_path,
-                      uint32_t column_family_id,
-                      CompressionType blob_compression_type,
-                      const std::vector<std::pair<std::string, std::string>>&
-                          expected_key_value_pairs,
-                      const std::vector<std::string>& blob_indexes) {
-    assert(expected_key_value_pairs.size() == blob_indexes.size());
-
-    std::unique_ptr<FSRandomAccessFile> file;
-    constexpr IODebugContext* dbg = nullptr;
-    ASSERT_OK(
-        fs_->NewRandomAccessFile(blob_file_path, file_options_, &file, dbg));
-
-    std::unique_ptr<RandomAccessFileReader> file_reader(
-        new RandomAccessFileReader(std::move(file), blob_file_path, clock_));
-
-    constexpr Statistics* statistics = nullptr;
-    BlobLogSequentialReader blob_log_reader(std::move(file_reader), clock_,
-                                            statistics);
-
-    BlobLogHeader header;
-    ASSERT_OK(blob_log_reader.ReadHeader(&header));
-    ASSERT_EQ(header.version, kVersion1);
-    ASSERT_EQ(header.column_family_id, column_family_id);
-    ASSERT_EQ(header.compression, blob_compression_type);
-    ASSERT_FALSE(header.has_ttl);
-    ASSERT_EQ(header.expiration_range, ExpirationRange());
-
-    for (size_t i = 0; i < expected_key_value_pairs.size(); ++i) {
-      BlobLogRecord record;
-      uint64_t blob_offset = 0;
-
-      ASSERT_OK(blob_log_reader.ReadRecord(
-          &record, BlobLogSequentialReader::kReadHeaderKeyBlob, &blob_offset));
-
-      // Check the contents of the blob file
-      const auto& expected_key_value = expected_key_value_pairs[i];
-      const auto& key = expected_key_value.first;
-      const auto& value = expected_key_value.second;
-
-      ASSERT_EQ(record.key_size, key.size());
-      ASSERT_EQ(record.value_size, value.size());
-      ASSERT_EQ(record.expiration, 0);
-      ASSERT_EQ(record.key, key);
-      ASSERT_EQ(record.value, value);
-
-      // Make sure the blob reference returned by the builder points to the
-      // right place
-      BlobIndex blob_index;
-      ASSERT_OK(blob_index.DecodeFrom(blob_indexes[i]));
-      ASSERT_FALSE(blob_index.IsInlined());
-      ASSERT_FALSE(blob_index.HasTTL());
-      ASSERT_EQ(blob_index.file_number(), blob_file_number);
-      ASSERT_EQ(blob_index.offset(), blob_offset);
-      ASSERT_EQ(blob_index.size(), value.size());
-    }
-
-    BlobLogFooter footer;
-    ASSERT_OK(blob_log_reader.ReadFooter(&footer));
-    ASSERT_EQ(footer.blob_count, expected_key_value_pairs.size());
-    ASSERT_EQ(footer.expiration_range, ExpirationRange());
-  }
-
-  MockEnv mock_env_;
-  FileSystem* fs_;
-  SystemClock* clock_;
-  FileOptions file_options_;
-};
-
-TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
-  // Build a single blob file
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 4;
-  constexpr size_t value_offset = 1234;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileBuilderTest_BuildAndCheckOneFile"),
-      0);
-  options.enable_blob_files = true;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
-      number_of_blobs);
-  std::vector<std::string> blob_indexes(number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    auto& expected_key_value = expected_key_value_pairs[i];
-
-    auto& key = expected_key_value.first;
-    key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    auto& value = expected_key_value.second;
-    value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    auto& blob_index = blob_indexes[i];
-
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_FALSE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(
-      blob_file_path,
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), number_of_blobs);
-  ASSERT_EQ(
-      blob_file_addition.GetTotalBlobBytes(),
-      number_of_blobs * (BlobLogRecord::kHeaderSize + key_size + value_size));
-
-  // Verify the contents of the new blob file as well as the blob references
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kNoCompression, expected_key_value_pairs, blob_indexes);
-}
-
-TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
-  // Build multiple blob files: file size limit is set to the size of a single
-  // value, so each blob ends up in a file of its own
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 10;
-  constexpr size_t value_offset = 1234567890;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileBuilderTest_BuildAndCheckMultipleFiles"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_file_size = value_size;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
-      number_of_blobs);
-  std::vector<std::string> blob_indexes(number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    auto& expected_key_value = expected_key_value_pairs[i];
-
-    auto& key = expected_key_value.first;
-    key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    auto& value = expected_key_value.second;
-    value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    auto& blob_index = blob_indexes[i];
-
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_FALSE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  ASSERT_EQ(blob_file_paths.size(), number_of_blobs);
-  ASSERT_EQ(blob_file_additions.size(), number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    const uint64_t blob_file_number = i + 2;
-
-    ASSERT_EQ(blob_file_paths[i],
-              BlobFileName(immutable_options.cf_paths.front().path,
-                           blob_file_number));
-
-    const auto& blob_file_addition = blob_file_additions[i];
-
-    ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-    ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-    ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-              BlobLogRecord::kHeaderSize + key_size + value_size);
-  }
-
-  // Verify the contents of the new blob files as well as the blob references
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    std::vector<std::pair<std::string, std::string>> expected_key_value_pair{
-        expected_key_value_pairs[i]};
-    std::vector<std::string> blob_index{blob_indexes[i]};
-
-    VerifyBlobFile(i + 2, blob_file_paths[i], column_family_id, kNoCompression,
-                   expected_key_value_pair, blob_index);
-  }
-}
-
-TEST_F(BlobFileBuilderTest, InlinedValues) {
-  // All values are below the min_blob_size threshold; no blob files get written
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 10;
-  constexpr size_t value_offset = 1234567890;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_InlinedValues"),
-      0);
-  options.enable_blob_files = true;
-  options.min_blob_size = 1024;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    const std::string key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    const std::string value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    std::string blob_index;
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_TRUE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  ASSERT_TRUE(blob_file_paths.empty());
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-TEST_F(BlobFileBuilderTest, Compression) {
-  // Build a blob file with a compressed blob
-  if (!Snappy_Supported()) {
-    return;
-  }
-
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 100;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_Compression"), 0);
-  options.enable_blob_files = true;
-  options.blob_compression_type = kSnappyCompression;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  const std::string key("1");
-  const std::string uncompressed_value(value_size, 'x');
-
-  std::string blob_index;
-
-  ASSERT_OK(builder.Add(key, uncompressed_value, &blob_index));
-  ASSERT_FALSE(blob_index.empty());
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(
-      blob_file_path,
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-
-  CompressionOptions opts;
-  CompressionContext context(kSnappyCompression);
-  constexpr uint64_t sample_for_compression = 0;
-
-  CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
-                       kSnappyCompression, sample_for_compression);
-
-  std::string compressed_value;
-  ASSERT_TRUE(Snappy_Compress(info, uncompressed_value.data(),
-                              uncompressed_value.size(), &compressed_value));
-
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-            BlobLogRecord::kHeaderSize + key_size + compressed_value.size());
-
-  // Verify the contents of the new blob file as well as the blob reference
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs{
-      {key, compressed_value}};
-  std::vector<std::string> blob_indexes{blob_index};
-
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kSnappyCompression, expected_key_value_pairs, blob_indexes);
-}
-
-TEST_F(BlobFileBuilderTest, CompressionError) {
-  // Simulate an error during compression
-  if (!Snappy_Supported()) {
-    return;
-  }
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_CompressionError"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_compression_type = kSnappyCompression;
-  options.env = &mock_env_;
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
-                                        [](void* arg) {
-                                          bool* ret = static_cast<bool*>(arg);
-                                          *ret = false;
-                                        });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr char key[] = "1";
-  constexpr char value[] = "deadbeef";
-
-  std::string blob_index;
-
-  ASSERT_TRUE(builder.Add(key, value, &blob_index).IsCorruption());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-  ASSERT_EQ(
-      blob_file_paths[0],
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
-
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-TEST_F(BlobFileBuilderTest, Checksum) {
-  // Build a blob file with checksum
-
-  class DummyFileChecksumGenerator : public FileChecksumGenerator {
-   public:
-    void Update(const char* /* data */, size_t /* n */) override {}
-
-    void Finalize() override {}
-
-    std::string GetChecksum() const override { return std::string("dummy"); }
-
-    const char* Name() const override { return "DummyFileChecksum"; }
-  };
-
-  class DummyFileChecksumGenFactory : public FileChecksumGenFactory {
-   public:
-    std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
-        const FileChecksumGenContext& /* context */) override {
-      return std::unique_ptr<FileChecksumGenerator>(
-          new DummyFileChecksumGenerator);
-    }
-
-    const char* Name() const override { return "DummyFileChecksumGenFactory"; }
-  };
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_Checksum"), 0);
-  options.enable_blob_files = true;
-  options.file_checksum_gen_factory =
-      std::make_shared<DummyFileChecksumGenFactory>();
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  const std::string key("1");
-  const std::string value("deadbeef");
-
-  std::string blob_index;
-
-  ASSERT_OK(builder.Add(key, value, &blob_index));
-  ASSERT_FALSE(blob_index.empty());
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(
-      blob_file_path,
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-            BlobLogRecord::kHeaderSize + key.size() + value.size());
-  ASSERT_EQ(blob_file_addition.GetChecksumMethod(), "DummyFileChecksum");
-  ASSERT_EQ(blob_file_addition.GetChecksumValue(), "dummy");
-
-  // Verify the contents of the new blob file as well as the blob reference
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs{
-      {key, value}};
-  std::vector<std::string> blob_indexes{blob_index};
-
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kNoCompression, expected_key_value_pairs, blob_indexes);
-}
-
-class BlobFileBuilderIOErrorTest
-    : public testing::Test,
-      public testing::WithParamInterface<std::string> {
- protected:
-  BlobFileBuilderIOErrorTest()
-      : mock_env_(Env::Default()),
-        fs_(mock_env_.GetFileSystem().get()),
-        sync_point_(GetParam()) {}
-
-  MockEnv mock_env_;
-  FileSystem* fs_;
-  FileOptions file_options_;
-  std::string sync_point_;
-};
-
-INSTANTIATE_TEST_CASE_P(
-    BlobFileBuilderTest, BlobFileBuilderIOErrorTest,
-    ::testing::ValuesIn(std::vector<std::string>{
-        "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile",
-        "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader",
-        "BlobFileBuilder::WriteBlobToFile:AddRecord",
-        "BlobFileBuilder::WriteBlobToFile:AppendFooter"}));
-
-TEST_P(BlobFileBuilderIOErrorTest, IOError) {
-  // Simulate an I/O error during the specified step of Add()
-  // Note: blob_file_size will be set to value_size in order for the first blob
-  // to trigger close
-  constexpr size_t value_size = 8;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderIOErrorTest_IOError"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_file_size = value_size;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {
-    Status* const s = static_cast<Status*>(arg);
-    assert(s);
-
-    (*s) = Status::IOError(sync_point_);
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr char key[] = "1";
-  constexpr char value[] = "deadbeef";
-
-  std::string blob_index;
-
-  ASSERT_TRUE(builder.Add(key, value, &blob_index).IsIOError());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  if (sync_point_ == "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile") {
-    ASSERT_TRUE(blob_file_paths.empty());
-  } else {
-    constexpr uint64_t blob_file_number = 2;
-
-    ASSERT_EQ(blob_file_paths.size(), 1);
-    ASSERT_EQ(blob_file_paths[0],
-              BlobFileName(immutable_options.cf_paths.front().path,
-                           blob_file_number));
-  }
-
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_file_cache.cc

@@ -1,102 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_cache.h"
-
-#include <cassert>
-#include <memory>
-
-#include "db/blob/blob_file_reader.h"
-#include "options/cf_options.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/slice.h"
-#include "test_util/sync_point.h"
-#include "trace_replay/io_tracer.h"
-#include "util/hash.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlobFileCache::BlobFileCache(Cache* cache,
-                             const ImmutableOptions* immutable_options,
-                             const FileOptions* file_options,
-                             uint32_t column_family_id,
-                             HistogramImpl* blob_file_read_hist,
-                             const std::shared_ptr<IOTracer>& io_tracer)
-    : cache_(cache),
-      mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
-      immutable_options_(immutable_options),
-      file_options_(file_options),
-      column_family_id_(column_family_id),
-      blob_file_read_hist_(blob_file_read_hist),
-      io_tracer_(io_tracer) {
-  assert(cache_);
-  assert(immutable_options_);
-  assert(file_options_);
-}
-
-Status BlobFileCache::GetBlobFileReader(
-    uint64_t blob_file_number,
-    CacheHandleGuard<BlobFileReader>* blob_file_reader) {
-  assert(blob_file_reader);
-  assert(blob_file_reader->IsEmpty());
-
-  const Slice key = GetSlice(&blob_file_number);
-
-  assert(cache_);
-
-  Cache::Handle* handle = cache_->Lookup(key);
-  if (handle) {
-    *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-    return Status::OK();
-  }
-
-  TEST_SYNC_POINT("BlobFileCache::GetBlobFileReader:DoubleCheck");
-
-  // Check again while holding mutex
-  MutexLock lock(mutex_.get(key));
-
-  handle = cache_->Lookup(key);
-  if (handle) {
-    *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-    return Status::OK();
-  }
-
-  assert(immutable_options_);
-  Statistics* const statistics = immutable_options_->stats;
-
-  RecordTick(statistics, NO_FILE_OPENS);
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  {
-    assert(file_options_);
-    const Status s = BlobFileReader::Create(
-        *immutable_options_, *file_options_, column_family_id_,
-        blob_file_read_hist_, blob_file_number, io_tracer_, &reader);
-    if (!s.ok()) {
-      RecordTick(statistics, NO_FILE_ERRORS);
-      return s;
-    }
-  }
-
-  {
-    constexpr size_t charge = 1;
-
-    const Status s = cache_->Insert(key, reader.get(), charge,
-                                    &DeleteCacheEntry<BlobFileReader>, &handle);
-    if (!s.ok()) {
-      RecordTick(statistics, NO_FILE_ERRORS);
-      return s;
-    }
-  }
-
-  reader.release();
-
-  *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-
-  return Status::OK();
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_cache.h

@@ -1,52 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cinttypes>
-
-#include "cache/cache_helpers.h"
-#include "rocksdb/rocksdb_namespace.h"
-#include "util/mutexlock.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Cache;
-struct ImmutableOptions;
-struct FileOptions;
-class HistogramImpl;
-class Status;
-class BlobFileReader;
-class Slice;
-class IOTracer;
-
-class BlobFileCache {
- public:
-  BlobFileCache(Cache* cache, const ImmutableOptions* immutable_options,
-                const FileOptions* file_options, uint32_t column_family_id,
-                HistogramImpl* blob_file_read_hist,
-                const std::shared_ptr<IOTracer>& io_tracer);
-
-  BlobFileCache(const BlobFileCache&) = delete;
-  BlobFileCache& operator=(const BlobFileCache&) = delete;
-
-  Status GetBlobFileReader(uint64_t blob_file_number,
-                           CacheHandleGuard<BlobFileReader>* blob_file_reader);
-
- private:
-  Cache* cache_;
-  // Note: mutex_ below is used to guard against multiple threads racing to open
-  // the same file.
-  Striped<port::Mutex, Slice> mutex_;
-  const ImmutableOptions* immutable_options_;
-  const FileOptions* file_options_;
-  uint32_t column_family_id_;
-  HistogramImpl* blob_file_read_hist_;
-  std::shared_ptr<IOTracer> io_tracer_;
-
-  static constexpr size_t kNumberOfMutexStripes = 1 << 7;
-};
-
-}  // namespace ROCKSDB_NAMESPACE