diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a47cafed7..bafe45fc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,11 +5,11 @@ on: [push, pull_request] jobs: test-ubuntu-latest: - runs-on: self-hosted + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: - submodules: true + submodules: recursive - name: make run: | sudo apt-get update @@ -36,19 +36,24 @@ jobs: ./runtest-rotation build-ubuntu-old: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + with: + submodules: recursive - name: make -j2 run: | sudo apt-get update - sudo apt-get -y install uuid-dev libcurl4-openssl-dev + sudo apt-get -y remove libzstd || true + sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev make -j2 build-macos-latest: runs-on: macos-latest steps: - uses: actions/checkout@v2 + with: + submodules: recursive - name: make run: make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' -j2 @@ -56,15 +61,20 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + submodules: recursive - name: make run: | sudo apt-get update - sudo apt-get -y install uuid-dev libcurl4-openssl-dev + sudo apt-get -y remove libzstd || true + sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' MALLOC=libc -j2 build-ubuntu-32bit: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + with: + submodules: recursive - name: make run: | sudo dpkg --add-architecture i386 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index fdfe4e122..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,164 +0,0 @@ -.standard-pipeline: - rules: - - if: '$COVERAGE' - when: never - - if: '$ENDURANCE' - when: never - - if: '$CI_PIPELINE_SOURCE == "push"' - -build: - extends: .standard-pipeline - tags: - - docker - stage: build - script: - - git submodule update --init - - make distclean - - make -j - artifacts: - paths: - - src/ - -runtest: - extends: .standard-pipeline - dependencies: - - build - tags: - - docker - stage: test - script: - - ./runtest --config server-threads 3 - -runtest-cluster: - extends: .standard-pipeline - dependencies: - - build - tags: - - docker - stage: test - script: - - ./runtest-cluster - -runtest-moduleapi: - extends: .standard-pipeline - dependencies: - - build - tags: - - docker - stage: test - script: - - ./runtest-moduleapi - -runtest-sentinel: - extends: .standard-pipeline - dependencies: - - build - tags: - - docker - stage: test - script: - - ./runtest-sentinel - -node-redis-test: - extends: .standard-pipeline - dependencies: - - build - rules: - - when: never - tags: - - docker - - ipv6 - stage: test - script: - - cp -pf src/keydb-server /usr/local/bin - - cp -pf src/keydb-cli /usr/local/bin - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/node-redis.git - - cd node-redis - - npm install - - npm run test - -jedis-test: - extends: .standard-pipeline - dependencies: - - build - tags: - - docker - - ipv4 - stage: test - script: - - cp -pf src/keydb-server /usr/local/bin - - cp -pf src/keydb-cli /usr/local/bin - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/jedis.git - - cd jedis - - make test - -redis-rs-test: - extends: .standard-pipeline - dependencies: - - build - tags: - - docker - stage: test - script: - - cp -pf src/keydb-server /usr/local/bin - - cp -pf src/keydb-cli /usr/local/bin - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/redis-rs.git - - cd redis-rs - - make test - -endurance-test: - rules: - - if: '$ENDURANCE' - tags: - - docker - stage: test - script: - - git submodule update --init - - make distclean - - make -j - - ./runtest --config server-threads 3 --loop --stop - -coverage-test: - rules: - - if: '$COVERAGE' - tags: - - docker - stage: test - script: - - git submodule update --init - - make distclean - - make gcov -j - - make install - - ./runtest || true - - pkill keydb-server || true - - pkill stunnel || true - - ./runtest-cluster || true - - pkill keydb-server || true - - pkill stunnel || true - - ./runtest-sentinel || true - - pkill keydb-server || true - - pkill stunnel || true - - ./runtest-moduleapi || true - - pkill keydb-server || true - - pkill stunnel || true - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/redis-rs.git - - cd redis-rs - - make test || true - - pkill keydb-server || true - - pkill stunnel || true - - cd .. - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/jedis.git - - cd jedis - - make test || true - - pkill keydb-server || true - - pkill stunnel || true - - cd .. - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/node-redis.git - - cd node-redis - - npm install - - npm run test || true - - pkill keydb-server || true - - pkill stunnel || true - - cd .. - - geninfo -o KeyDB.info --no-external . - - genhtml --legend -o lcov-html KeyDB.info \ No newline at end of file diff --git a/build.yaml b/build.yaml deleted file mode 100644 index 744ae947a..000000000 --- a/build.yaml +++ /dev/null @@ -1,48 +0,0 @@ -# Doc: https://wiki.sc-corp.net/pages/viewpage.action?pageId=121500284 -version: 1 -machamp: - keydb-build: - # Optional - build counter is linked to the build def - tag_template: 0.0.%build.counter% - # Optional - value in seconds before a build is terminated, default is 3600 seconds - timeout: 3600 - # Optional - update ghe or not, default to true - update_ghe: true - code_coverage: false - # Required - steps: - make-build: - type: cmd - # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c - command: ./machamp_scripts/build.sh - tls-test: - type: cmd - parent: make-build - # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c - command: ./runtest --clients 4 --verbose - cluster-test: - type: cmd - parent: make-build - # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c - command: ./runtest-cluster - sentinel-test: - type: cmd - parent: make-build - # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c - command: ./runtest-sentinel - module-test: - type: cmd - parent: make-build - # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c - command: ./runtest-moduleapi - rotation-test: - type: cmd - parent: make-build - # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04 - builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c - command: ./runtest-rotation diff --git a/ci.yaml b/ci.yaml deleted file mode 100644 index 89c048d1c..000000000 --- a/ci.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Doc: https://wiki.sc-corp.net/display/TOOL/ci.yaml+User+Guide -version: 1 -on: - pull_request: - - workflows: - # All builds that use machamp should use the defined `backend_workflow` - - workflow_type: backend_workflow - # references a build defined in build.yaml - build_name: keydb-build - arch_types: ["amd64", "arm64"] - push: - - branches: [keydbpro] - workflows: - - workflow_type: backend_workflow - build_name: keydb-build - arch_types: ["amd64", "arm64"] diff --git a/docker-internal/.gitignore b/docker-internal/.gitignore deleted file mode 100644 index cbee8387c..000000000 --- a/docker-internal/.gitignore +++ /dev/null @@ -1 +0,0 @@ -GHE_PersonalAccessToken.txt diff --git a/docker-internal/Dockerfile b/docker-internal/Dockerfile deleted file mode 100644 index 33993eea7..000000000 --- a/docker-internal/Dockerfile +++ /dev/null @@ -1,126 +0,0 @@ -FROM ubuntu:20.04 -SHELL ["/bin/bash","-c"] -RUN groupadd -r keydb && useradd -r -g keydb keydb -# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases -ENV GOSU_VERSION 1.14 -RUN set -eux; \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \ - rm -rf /var/lib/apt/lists/*; \ - dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ - wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ - wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - chmod +x /usr/local/bin/gosu; \ - gosu --version; \ - gosu nobody true -# build KeyDB -ARG KEYDB_DIR -ARG MODSTATSD_DIR -ARG MAKE_JOBS="" -COPY $KEYDB_DIR /tmp/keydb-internal/ -COPY $MODSTATSD_DIR /tmp/modstatsd/ -RUN set -eux; \ - \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \ - dpkg-dev \ - pkg-config \ - ca-certificates \ - build-essential \ - nasm \ - autotools-dev \ - autoconf \ - libjemalloc-dev \ - tcl \ - tcl-dev \ - uuid-dev \ - libcurl4-openssl-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ - libsnappy-dev \ - libssl-dev \ - git; \ - cd /tmp/modstatsd/; \ - make clean; \ - make; \ - mv modstatsd.so /usr/local/lib/; \ - cd /tmp/keydb-internal; \ - # disable protected mode as it relates to docker - grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ - sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \ - grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \ - make distclean; \ - make -j$([ -z "$MAKE_JOBS" ] && nproc || echo "$MAKE_JOBS") BUILD_TLS=yes NO_LICENSE_CHECK=yes; \ - cd src; \ - strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \ - mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ - # clean up unused dependencies - echo $savedAptMark; \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sed 's:.*/::' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - ; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - rm -rf /var/lib/apt/lists/*; \ -# create working directories and organize files -RUN \ - mkdir /data && chown keydb:keydb /data; \ - mkdir /flash && chown keydb:keydb /flash; \ - mkdir -p /etc/keydb; \ - cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \ - sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \ - sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \ - sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ - sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \ - sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \ - echo -e "\nloadmodule /usr/local/lib/modstatsd.so" >> /etc/keydb/keydb.conf; \ - ln -s keydb-cli redis-cli; \ - cd /etc/keydb; \ - ln -s keydb.conf redis.conf; \ - rm -rf /tmp/* -# generate entrypoint script -RUN set -eux; \ - echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ - echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ - echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ - echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \ - echo ' exec gosu keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ - chmod +x /usr/local/bin/docker-entrypoint.sh -# set remaining image properties -VOLUME /data -WORKDIR /data -ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/ -ENTRYPOINT ["docker-entrypoint.sh"] -EXPOSE 6379 -CMD ["keydb-server","/etc/keydb/keydb.conf"] - - - - - diff --git a/docker-internal/README.md b/docker-internal/README.md deleted file mode 100644 index 3eb861202..000000000 --- a/docker-internal/README.md +++ /dev/null @@ -1,132 +0,0 @@ -# KeyDB Docker Image (Snap Internal) - -This docker image builds KeyDB within the image and cleans up afterwards. A few notes about the image: -* use gosu to avoid running as root https://github.com/tianon/gosu -* packages are installed, tracked and cleaned up to minimize container sizes -* keydb-server binary symbols remain for troubleshooting. All other KeyDB binaries are stripped -* keydb.conf added and linked to redis.conf for legacy compatibility and as default config file -* use entrypoint and cmd for best practices. Remove protected-mode during build incase user specifies binary without .conf, or just wants append parameters - -## Building the Docker Image Using Local KeyDB Directory - -If you have a local keydb-internal repository you would like to generate the binaries from, use the command below. This will simply copy over all the files within the local keydb-internal repo and then build the image. - -One issue with COPY is that it creates an additional layer. If we remove the source directory in a later layer the size of the original COPY layer remains creating a large image, hence we need to squash the layers into one layer. This feature is available with experimental mode enabled. This can be done by modifying /etc/docker/daemon.json to include `"experimental": true,`. You may also be able to pass at the build line as shown below. - -Modify the KEYDB_DIR build argument to your local KeyDB repo and update your image tag in the line below - -``` -DOCKER_CLI_EXPERIMENTAL=enabled docker build --squash --build-arg KEYDB_DIR=. -t myImageName:imageTag -f ./docker-internal/Dockerfile . -``` - -Please note that directories are relative to the docker build context. You can use the `-f /path/to/Dockerfile` to specify Dockerfile which will also set the build context, your repo location will be relative to it. - -### Pushing - -#### AWS -There is a script ./build-and-publish.sh to build and push image. This script will push images to caching-infra AWS account and caching-infra GCP project. Make sure this script is not run using root/sudo otherwise you may not be able to access the correct AWS profile, even if everything else is configured correctly. - -If you are pushing to ECR, then you need to add this profile config in your ```~/.aws/config```: - -``` -[profile caching-infra-images-editor] -role_arn = arn:aws:iam::520173307535:role/_Snap_ContainerEditor -output = json -region = us-east-1 -source_profile = default -``` -and to get permission for assuming role [_Snap_ContainerEditor in account caching-infra](https://lease.sc-corp.net/v2/request_access/aws_resources/aws_account?resource=520173307535&roles=%5B_Snap_ContainerEditor%5D). - -Also, if you are using image different from "520173307535.dkr.ecr.us-east-1.amazonaws.com/keydb", then you need to give access to that image to snap-core-prod aws account. That is account where all mesh services are running. Go to your image in AWS Console and add policy: -``` -{ - "Version": "2008-10-17", - "Statement": [ - { - "Sid": "AllowImagePullApp", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::307862320347:root" - }, - "Action": [ - "ecr:BatchCheckLayerAvailability", - "ecr:BatchGetImage", - "ecr:GetDownloadUrlForLayer" - ] - } - ] -} -``` - -#### GCP - -In order to publish to GCP, you will need to get [Storage Admin Role in project caching-infra](https://lease.sc-corp.net/v2/request_access/gcp_resources/gcp_project?resource=caching-infra&roles=%5Broles/storage.admin%5D) - -For reading image you will need to add your service account to [caching-infra project](https://lease.sc-corp.net/v2/view_iam?resourceType=PRJ&resource=caching-infra) with "Container Registry Service Agent" role. - -#### Example - -``` -DOCKER_CLI_EXPERIMENTAL=enabled ./build-and-publish.sh -``` - -### Troubleshooting -If you see error: -``` -#11 354.1 g++: fatal error: Killed signal terminated program cc1plus -``` -most likely you are hitting memory constraint. If you are running docker build command from the above, then you can try to reduce number of jobs for "make" by adding "--build-arg MAKE_JOBS=" argument to lower value (i.e. 2). If you are running ./build-and-publish.sh you can reduce the number of jobs by passing it in args: -``` -DOCKER_CLI_EXPERIMENTAL=enabled ./build-and-publish.sh -j 2 -``` - -## Building the Docker Image Using PAT & Clone - -This image clones the keydb-internal repo, hence a GHE PAT token or SSH access is needed. See more on [obtaining GHE PAT](https://wiki.sc-corp.net/display/TOOL/Using+the+GHE+API#UsingtheGHEAPI-Step1:PersonalTokens). It is not secure to pass tokens/credentials as build-args, env variables, or COPYing then deleting, so we use secrets. This option is only available with the [Docker BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information)so the docker build kit must be enabled via `DOCKER_BUILDKIT=1`, or permanently by appending `"features": { "buildkit": true }` to /etc/docker/daemon.json. - -#### To Build: - -Run the command below updating your info as follows: -* Pass in the branch name as a build argument -* Tag your image to whatever you want to use -* Add your PAT info to GHE_PersonalAccessToken.txt - -``` -cd docker_PAT_build -DOCKER_BUILDKIT=1 docker build --no-cache --build-arg BRANCH=keydbpro --secret id=PAT,src=GHE_PersonalAccessToken.txt . -t myImageName:imageTag -``` - -For a detailed conventional build view pass `--progress=plain` while building. Note that `--no-cache` must be used to ensure your credentials are pulled in as they are not cached. - -#### Troubleshooting - -If you are building on macOS or with docker frontend you may need to append the following line to the top of the Dockerfile: - -``` -# syntax=docker/dockerfile:1.3 -``` - -## Using The Image - -#### Bind port - -Bind the container to the node/machine by passing the parameter `-p 6379:6379` when you run the docker container - -#### Configuration file - -By default KeyDB launches by specifiying its internal keydb.conf file at `/etc/keydb/keydb.conf`. If you would like to use your own config file you can link to the config file with `-v /path-to-config-file/keydb.conf:/etc/keydb/keydb.conf` then run the container whose default command is `keydb-server /etc/keydb/keydb.conf` - -Alternatively specify any config file location, by specifying the location following the binary to override default: `docker run ThisDockerImage keydb-server /path/to/my/config-file/`. You can also just append parameters to default image with `docker run ThisDockerImage keydb-server /etc/keydb/keydb.conf --dir /data --server-threads 4`. - -#### Persistent storage - -If persistence is enabled, data is stored in the VOLUME /data, which can be used with --volumes-from some-volume-container or -v /docker/host/dir:/data (see [docs.docker volumes](https://docs.docker.com/storage/volumes/)). - -#### Connect via keydb-cli - -you can grab the ip of the container with `docker inspect --format '{{ .NetworkSettings.IPAddress }}' mycontainername` then run the following: - -``` -docker run -it --rm ThisDockerImage keydb-cli -h -p 6379 -``` - diff --git a/docker-internal/build-and-publish.sh b/docker-internal/build-and-publish.sh deleted file mode 100755 index 816635265..000000000 --- a/docker-internal/build-and-publish.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -e - -set -o pipefail - -# Color for displaying error messages -red=`tput setaf 1` -reset=`tput sgr0` - -# Default value for the command line flag - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -FLAGS_tag="latest" -FLAGS_provider="both" -FLAGS_jobs= - -timestamp() { - date "+%m/%d %H:%M:%S" -} - -print_usage() { - echo "Usage:" - echo " ./{script} {modstatsd-path} [flags]" - echo "" - echo "Flags:" - echo " -p, --provider: name of the cloud provider, can be 'gcp', 'aws' or 'both' (default)" - echo " -t, --tag: tag of the image. Default is 'latest'" - echo " -j, --jobs: the number of jobs when making the build. Default is number of cores on this host" -} - -push_image () { # 1 - image, 2 - repo - export IMAGE_WITH_REPO="$2/$1" - docker tag $1 ${IMAGE_WITH_REPO} - echo "`timestamp` publishing image ${IMAGE_WITH_REPO}" - docker push "${IMAGE_WITH_REPO}" - echo "`timestamp` image ${IMAGE_WITH_REPO} is pushed" -} - -#Process Module Path Flag -MOD_STATSD_PATH="$1" -if [[ -z ${MOD_STATSD_PATH} ]] -then - echo "Missing path to statsd module" - # exit 1 -fi -shift - -# Processing flags -while [ ! $# -eq 0 ] -do - # The shift below ensures the unprocessed flag is always at $1 - case "$1" in - --help | -h) - print_usage - exit 0 - ;; - --provider | -p) - FLAGS_provider="$2" - if [[ ${FLAGS_provider} != "gcp" ]] && [[ ${FLAGS_provider} != "aws" ]] && [[ ${FLAGS_provider} != "both" ]]; - then - echo "${red} For the flag -p/--provider, only valid values are [\"gcp\", \"aws\", \"both\"]." - exit 1 - fi - shift - ;; - --tag | -t) - FLAGS_tag="$2" - shift - ;; - --jobs | -j) - FLAGS_jobs="$2" - shift - ;; - *) - echo "${red}" - echo "Unrecognized flag: $1." - echo "Run with '--help' flag to see the supported flags." - echo "${reset}" - exit 1 - ;; - esac - shift -done - -echo "MOD_STATSD_PATH: ${MOD_STATSD_PATH}" -echo "`timestamp` building image for ${FLAGS_tag}" -export IMAGE_SUFFIX="keydb:${FLAGS_tag}" -cp -r ${MOD_STATSD_PATH} ./modstatsd -docker build --squash --build-arg MODSTATSD_DIR=docker-internal/modstatsd --build-arg KEYDB_DIR=. --build-arg MAKE_JOBS=${FLAGS_jobs} -t keydb:${FLAGS_tag} -f ${DIR}/Dockerfile ${DIR}/.. -rm -rf ./modstatsd - -# Build and publish -if [[ ${FLAGS_provider} == "aws" ]] || [[ ${FLAGS_provider} == "both" ]] -then - export ECR="520173307535.dkr.ecr.us-east-1.amazonaws.com" - echo "`timestamp` Preparing to push image to AWS, ECR: ${ECR}" - aws ecr get-login-password --profile caching-infra-images-editor --region us-east-1 | docker login --username AWS --password-stdin ${ECR} - push_image ${IMAGE_SUFFIX} ${ECR} -fi - -if [[ ${FLAGS_provider} == "gcp" ]] || [[ ${FLAGS_provider} == "both" ]] -then - export GCR="gcr.io/caching-infra" - echo "`timestamp` Preparing to push to GCP, GCR: ${GCR}" - push_image ${IMAGE_SUFFIX} ${GCR} -fi diff --git a/docker-internal/docker_PAT_build/Dockerfile b/docker-internal/docker_PAT_build/Dockerfile deleted file mode 100644 index e27c4388a..000000000 --- a/docker-internal/docker_PAT_build/Dockerfile +++ /dev/null @@ -1,118 +0,0 @@ -FROM ubuntu:20.04 -SHELL ["/bin/bash","-c"] -RUN groupadd -r keydb && useradd -r -g keydb keydb -# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases -ENV GOSU_VERSION 1.14 -RUN set -eux; \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends ca-certificates dirmngr gnupg wget; \ - rm -rf /var/lib/apt/lists/*; \ - dpkgArch="$(dpkg --print-architecture | awk -F- '{ print $NF }')"; \ - wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch"; \ - wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$dpkgArch.asc"; \ - export GNUPGHOME="$(mktemp -d)"; \ - gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4; \ - gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \ - gpgconf --kill all; \ - rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - chmod +x /usr/local/bin/gosu; \ - gosu --version; \ - gosu nobody true -# build KeyDB -ARG BRANCH -RUN --mount=type=secret,id=PAT export $(cat /run/secrets/PAT | xargs); \ - set -eux; \ - \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - DEBIAN_FRONTEND=noninteractive apt-get install -qqy --no-install-recommends \ - dpkg-dev \ - pkg-config \ - ca-certificates \ - build-essential \ - nasm \ - autotools-dev \ - autoconf \ - libjemalloc-dev \ - tcl \ - tcl-dev \ - uuid-dev \ - libcurl4-openssl-dev \ - libbz2-dev \ - libzstd-dev \ - liblz4-dev \ - libsnappy-dev \ - libssl-dev \ - git; \ - cd /tmp && git clone --branch $BRANCH https://$PAT_ID:$TOKEN@github.sc-corp.net/Snapchat/keydb-internal.git --recursive; \ - cd /tmp/keydb-internal; \ - # disable protected mode as it relates to docker - grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ - sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \ - grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \ - make -j$(nproc) BUILD_TLS=yes NO_LICENSE_CHECK=yes; \ - cd src; \ - strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel; \ - mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ - # clean up unused dependencies - echo $savedAptMark; \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - find /usr/local -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sed 's:.*/::' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - ; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - rm -rf /var/lib/apt/lists/*; \ -# create working directories and organize files -RUN \ - mkdir /data && chown keydb:keydb /data; \ - mkdir /flash && chown keydb:keydb /flash; \ - mkdir -p /etc/keydb; \ - cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \ - sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \ - sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \ - sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ - sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \ - sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \ - ln -s keydb-cli redis-cli; \ - cd /etc/keydb; \ - ln -s keydb.conf redis.conf; \ - rm -rf /tmp/* -# generate entrypoint script -RUN set -eux; \ - echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ - echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ - echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ - echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ - echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \ - echo ' exec gosu keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ - echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ - chmod +x /usr/local/bin/docker-entrypoint.sh -# set remaining image properties -VOLUME /data -WORKDIR /data -ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/ -ENTRYPOINT ["docker-entrypoint.sh"] -EXPOSE 6379 -CMD ["keydb-server","/etc/keydb/keydb.conf"] - - - - - diff --git a/docker-internal/docker_PAT_build/GHE_PersonalAccessToken.txt b/docker-internal/docker_PAT_build/GHE_PersonalAccessToken.txt deleted file mode 100644 index 4266d9ddf..000000000 --- a/docker-internal/docker_PAT_build/GHE_PersonalAccessToken.txt +++ /dev/null @@ -1,2 +0,0 @@ -PAT_ID= -TOKEN= diff --git a/github-action-runner-docker/Dockerfile b/github-action-runner-docker/Dockerfile deleted file mode 100644 index c28faf44b..000000000 --- a/github-action-runner-docker/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM ubuntu:18.04 - -ARG RUNNER_VERSION=2.287.1 -ARG TOKEN -ARG NAME=internal-runner - -USER root - -WORKDIR /actions-runner - -RUN apt-get update -qq \ - && apt-get install -y curl \ - && curl -o actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \ - && tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \ - && ./bin/installdependencies.sh \ - && rm ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz \ - && apt-get purge -y \ - && chown 1000:1000 -R /actions-runner - -USER 1000 - -RUN ./config.sh --url https://github.com/EQ-Alpha/KeyDB --token ${TOKEN} logout --name ${NAME} --unattended - -ENTRYPOINT ["./run.sh"] diff --git a/github-action-runner-docker/README.md b/github-action-runner-docker/README.md deleted file mode 100644 index 3619051f3..000000000 --- a/github-action-runner-docker/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Github action runner. - -## Overview - All details are [here](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners). - In brief overview, self-hosted action runner is running on our hosts, connects to the github and waits for the - jobs to execute. Bellow is instruction on how to create image for the runner and start it in our Kubernetes - cluster. - -## Creating a new runner. - -1. Got [github](https://github.com/EQ-Alpha/KeyDB/settings/actions/runners/new?arch=x64&os=linux) to create a new runner. -2. The link above should display a script for installing runner. It should be aligned with what we have in a -[Dockerfile](https://github.sc-corp.net/Snapchat/keydb-internal/github-action-runner-docker/Dockerfile). -3. Take the token from that script. Should be in "Configure" section: - ``` - ./config.sh --url https://github.com/EQ-Alpha/KeyDB --token AUQJRVZIQCLO4ZQOZAOC3L3B7RHIU - ``` -4. Take the version of the runner from the "Download" section from file name. Example is "2.287.1", -5. Build and public the image: - ``` - ./build-and-publish.sh --token --version - ``` - Docker will register runner in Github during the build. If you got an error -```A runner exists with the same name```, it means you've already built the image -for runner with that name. In this case you can either give a new name to the runner -(using ```--name ``` parameter to the build script) or delete existing runner in -[github](https://github.com/EQ-Alpha/KeyDB/settings/actions/runners). Deleting the -runner will break existing runners running in our infrastructure. You may want to -delete them. -6. The default image is "[gcr.io/caching-infra/keydb-github-action-runner](https://console.cloud.google.com/gcr/images/caching-infra/global/keydb-github-action-runner?project=caching-infra)" -but you can set any other image full name to the building script (```--image ```). -7. Deploy new image to cluster [caching-infra--t-us-east4--staging](https://switchboard.sc-corp.net/#/services/caching-infra/cloud-resource/caching-infra--t-us-east4--staging/manage?region=us-east4&provider=GOOGLE&project_id=caching-infra): - * Configure kubectl for the cluster. - * Delete existing runner using [deployment manifest](https://github.sc-corp.net/Snapchat/keydb-internal/github-action-runner-docker/deployment.yaml): - ``` - kubectl delete -f deployment.yaml - ``` - * Start a new runner: - ``` - kubectl apply -f deployment.yaml - ``` - Deployment manifest uses [gcr.io/caching-infra/keydb-github-action-runner](https://console.cloud.google.com/gcr/images/caching-infra/global/keydb-github-action-runner?project=caching-infra). - If you specified another image name in build script, update deployment manifest with an appropriate image. -8. Validate on [github](https://github.com/EQ-Alpha/KeyDB/settings/actions/runners) that new runner is online. \ No newline at end of file diff --git a/github-action-runner-docker/build-and-publish.sh b/github-action-runner-docker/build-and-publish.sh deleted file mode 100755 index f94502523..000000000 --- a/github-action-runner-docker/build-and-publish.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -set -eu -set -o pipefail - -# Color for displaying error messages -red=`tput setaf 1` -reset=`tput sgr0` - -# Default value for the command line flag -DEFAULT_VERSION="2.287.1" -DEFAULT_IMG="gcr.io/caching-infra/keydb-github-action-runner:latest" -DEFAULT_NAME="internal-runner" -FLAGS_token="" -FLAGS_version=$DEFAULT_VERSION -FLAGS_img=$DEFAULT_IMG -FLAGS_name=$DEFAULT_NAME - - -timestamp() { - date "+%m/%d %H:%M:%S" -} - -print_usage() { - echo "Usage:" - echo " ./{script} [flags]" - echo "" - echo "Flags:" - echo " -t, --token: Token taken from onboarding script from github. (required)" - echo " -i, --image: Full docker image name you want to build and push. Default value is ${DEFAULT_IMG}" - echo " -v, --version: Action runner version. Can be taked from onboarding script from github. Default version is ${DEFAULT_VERSION}." - echo " -n, --name: Action runner name. Default name is ${DEFAULT_NAME}." -} - -# Processing flags -while [ ! $# -eq 0 ] -do - # The shift below ensures the unprocessed flag is always at $1 - case "$1" in - --help | -h) - print_usage - exit 0 - ;; - --image | -i) - FLAGS_img="$2" - shift - ;; - --token | -t) - FLAGS_token="$2" - shift - ;; - --version | -v) - FLAGS_version="$2" - shift - ;; - --name | -n) - FLAGS_name="$2" - shift - ;; - *) - echo "${red}" - echo "Unrecognized flag: $1." - echo "Run with '--help' flag to see the supported flags." - echo "${reset}" - exit 1 - ;; - esac - shift -done - -if [[ ${FLAGS_token} == "" ]]; then - echo "${red} ERROR:token is missing" - echo ${reset} - print_usage - exit 1 -fi - -echo "Building image ${FLAGS_img} ..." -docker build --build-arg TOKEN=${FLAGS_token} --build-arg RUNNER_VERSION=${FLAGS_version} --build-arg NAME=${FLAGS_name} -t ${FLAGS_img} . -echo "Pushing image ${FLAGS_img}...." -docker push ${FLAGS_img} diff --git a/github-action-runner-docker/deployment.yaml b/github-action-runner-docker/deployment.yaml deleted file mode 100644 index 0974fb00a..000000000 --- a/github-action-runner-docker/deployment.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: keydb-runner -spec: - replicas: 1 - selector: - matchLabels: - app: keydb-runner - template: - metadata: - labels: - app: keydb-runner - spec: - containers: - - name: keydb-runner - image: gcr.io/caching-infra/keydb-github-action-runner:latest - # Since it's 'latest' tag we're reusing it. - imagePullPolicy: Always diff --git a/keydb.conf b/keydb.conf index 355271150..3f9f1b663 100644 --- a/keydb.conf +++ b/keydb.conf @@ -184,7 +184,9 @@ tcp-keepalive 300 # # tls-client-key-file-pass secret -# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange: +# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange, +# required by older versions of OpenSSL (<3.0). Newer versions do not require +# this configuration and recommend against it. # # tls-dh-params-file keydb.dh diff --git a/machamp_scripts/build.sh b/machamp_scripts/build.sh deleted file mode 100755 index e05a9b3bd..000000000 --- a/machamp_scripts/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# make the build -git submodule init && git submodule update -make BUILD_TLS=yes -j$(nproc) KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' - -# gen-cert -./utils/gen-test-certs.sh \ No newline at end of file diff --git a/pkg/deb/deb-buildsource.sh b/pkg/deb/deb-buildsource.sh index 8d50035f1..39a2782e9 100755 --- a/pkg/deb/deb-buildsource.sh +++ b/pkg/deb/deb-buildsource.sh @@ -23,8 +23,8 @@ pkg_name=keydb-$majorv:$version$distname # create build tree cd ../../../ -tar -czvf keydb_$version.orig.tar.gz --force-local keydb-internal -cd keydb-internal/pkg/deb/ +tar -czvf keydb_$version.orig.tar.gz --force-local KeyDB +cd KeyDB/pkg/deb/ mkdir -p $pkg_name/tmp if [[ "$codename" == "xenial" ]] || [[ "$codename" == "stretch" ]]; then cp -r debian_dh9 $pkg_name/tmp/debian diff --git a/pkg/deb/debian/control b/pkg/deb/debian/control index 1091a4ba1..767f00cac 100644 --- a/pkg/deb/debian/control +++ b/pkg/deb/debian/control @@ -26,8 +26,8 @@ Build-Depends: libzstd-dev Standards-Version: 4.2.1 Homepage: https://docs.keydb.dev/ -Vcs-Git: https://github.com/EQ-Alpha/KeyDB.git -Vcs-Browser: https://github.com/EQ-Alpha/KeyDB +Vcs-Git: https://github.com/Snapchat/KeyDB.git +Vcs-Browser: https://github.com/Snapchat/KeyDB Package: keydb Architecture: all diff --git a/pkg/deb/debian_dh9/control b/pkg/deb/debian_dh9/control index 0feec1220..657ad2066 100644 --- a/pkg/deb/debian_dh9/control +++ b/pkg/deb/debian_dh9/control @@ -25,8 +25,8 @@ Build-Depends: libzstd-dev Standards-Version: 4.2.1 Homepage: https://docs.keydb.dev/ -Vcs-Git: https://github.com/EQ-Alpha/KeyDB.git -Vcs-Browser: https://github.com/EQ-Alpha/KeyDB +Vcs-Git: https://github.com/Snapchat/KeyDB.git +Vcs-Browser: https://github.com/Snapchat/KeyDB Package: keydb Architecture: all diff --git a/pkg/deb/master_changelog b/pkg/deb/master_changelog index d3bf48042..31990bf3b 100644 --- a/pkg/deb/master_changelog +++ b/pkg/deb/master_changelog @@ -1,3 +1,11 @@ +keydb (6:6.3.1-1distribution_placeholder) codename_placeholder; urgency=medium + + * This point release contains fixes to bugs related to expires, active-rep, and rdb saving + * Issues fixed: #419, #422, #428 + * PRs: #426, #429, #431, #433 + +-- Ben Schermel Mon, 23 May 2022 20:00:37 +0000 + keydb (6:6.3.0-1distribution_placeholder) codename_placeholder; urgency=medium * This release open sources KeyDB Enterprise features into the open source project along with PSYNC for active replication diff --git a/pkg/docker/Dockerfile b/pkg/docker/Dockerfile index f0bbe2cca..0f2a63ba8 100644 --- a/pkg/docker/Dockerfile +++ b/pkg/docker/Dockerfile @@ -77,7 +77,7 @@ RUN \ mkdir /data && chown keydb:keydb /data; \ mkdir /flash && chown keydb:keydb /flash; \ mkdir -p /etc/keydb; \ - cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \ + cp /tmp/KeyDB/keydb.conf /etc/keydb/; \ sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \ sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \ sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ @@ -110,8 +110,3 @@ ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/ ENTRYPOINT ["docker-entrypoint.sh"] EXPOSE 6379 CMD ["keydb-server","/etc/keydb/keydb.conf"] - - - - - diff --git a/pkg/docker/Dockerfile_Alpine b/pkg/docker/Dockerfile_Alpine new file mode 100644 index 000000000..8e5915c9a --- /dev/null +++ b/pkg/docker/Dockerfile_Alpine @@ -0,0 +1,77 @@ +FROM alpine:3.12 +# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added +RUN addgroup -S -g 1000 keydb && adduser -S -G keydb -u 999 keydb +RUN mkdir -p /etc/keydb +ARG BRANCH +RUN set -eux; \ + \ + apk add --no-cache su-exec; \ + apk add --no-cache --virtual .build-deps \ + coreutils \ + gcc \ + linux-headers \ + make \ + musl-dev \ + openssl-dev \ + git \ + util-linux-dev \ + curl-dev \ + g++ \ + libunwind-dev \ + bash \ + perl \ + git \ + ; \ + cd /tmp && git clone --branch $BRANCH https://github.com/Snapchat/KeyDB.git --recursive; \ + cd /tmp/KeyDB; \ + # disable protected mode as it relates to docker + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \ + sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \ + grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \ + make -j$(nproc) BUILD_TLS=yes; \ + cd src; \ + strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel keydb-server; \ + mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \ + runDeps="$( \ + scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \ + | tr ',' '\n' \ + | sort -u \ + | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \ + )"; \ + apk add --no-network --virtual .keydb-rundeps $runDeps; \ + apk del --no-network .build-deps; \ + # create working directories and organize files + mkdir /data && chown keydb:keydb /data; \ + mkdir /flash && chown keydb:keydb /flash; \ + mkdir -p /etc/keydb; \ + cp /tmp/KeyDB/keydb.conf /etc/keydb/; \ + sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \ + sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \ + sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \ + sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \ + ln -s keydb-cli redis-cli; \ + cd /etc/keydb; \ + ln -s keydb.conf redis.conf; \ + rm -rf /tmp/* +# generate entrypoint script +RUN set -eux; \ + echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \ + echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \ + echo " find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \ + echo ' exec su-exec keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \ + echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \ + chmod +x /usr/local/bin/docker-entrypoint.sh +VOLUME /data +WORKDIR /data +ENTRYPOINT ["docker-entrypoint.sh"] +EXPOSE 6379 +CMD ["keydb-server", "/etc/keydb/keydb.conf"] diff --git a/src/Makefile b/src/Makefile index d432c6fc5..0ab835ff5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -62,6 +62,7 @@ endif endif USEASM?=true +ENABLE_FLASH?=no ifneq ($(strip $(SANITIZE)),) CFLAGS+= -fsanitize=$(SANITIZE) -DSANITIZE -fno-omit-frame-pointer @@ -71,6 +72,15 @@ ifneq ($(strip $(SANITIZE)),) USEASM=false endif +ifeq ($(ENABLE_FLASH),yes) + FINAL_LIBS+= -lz -lcrypto -lbz2 -lzstd -llz4 -lsnappy + CXXFLAGS+= -I../deps/rocksdb/include/ -DENABLE_ROCKSDB + STORAGE_OBJ+= storage/rocksdb.o storage/rocksdbfactory.o + FINAL_LIBS+= ../deps/rocksdb/librocksdb.a + DEPENDENCY_TARGETS+= rocksdb +endif + + ifeq ($(CHECKED),true) CXXFLAGS+= -DCHECKED_BUILD endif diff --git a/src/StorageCache.cpp b/src/StorageCache.cpp index 45afe1e5c..9b89023aa 100644 --- a/src/StorageCache.cpp +++ b/src/StorageCache.cpp @@ -205,6 +205,7 @@ void StorageCache::beginWriteBatch() { } void StorageCache::emergencyFreeCache() { + std::unique_lock ul(m_lock); dict *d = m_pdict; m_pdict = nullptr; if (d != nullptr) { diff --git a/src/config.cpp b/src/config.cpp index adbb780fd..5e01630a5 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -355,11 +355,17 @@ bool initializeStorageProvider(const char **err) return true; if (!strcasecmp(g_sdsProvider, "flash") && g_sdsArgs != nullptr) { +#ifdef ENABLE_ROCKSDB // Create The Storage Factory (if necessary) serverLog(LL_NOTICE, "Initializing FLASH storage provider (this may take a long time)"); adjustOpenFilesLimit(); g_pserver->m_pstorageFactory = CreateRocksDBStorageFactory(g_sdsArgs, cserver.dbnum, cserver.storage_conf, cserver.storage_conf ? strlen(cserver.storage_conf) : 0); - } +#else + serverLog(LL_WARNING, "To use the flash storage provider please compile KeyDB with ENABLE_FLASH=yes"); + serverLog(LL_WARNING, "Exiting due to the use of an unsupported storage provider"); + exit(EXIT_FAILURE); +#endif + } else if (!strcasecmp(g_sdsProvider, "test") && g_sdsArgs == nullptr) { g_pserver->m_pstorageFactory = new (MALLOC_LOCAL) TestStorageFactory(); @@ -761,6 +767,12 @@ void loadServerConfigFromString(char *config) { g_sdsProvider = sdsdup(argv[1]); if (argc > 2) g_sdsArgs = sdsdup(argv[2]); + } else if (!strcasecmp(argv[0],"is-flash-enabled") && argc == 1) { +#ifdef ENABLE_ROCKSDB + exit(EXIT_SUCCESS); +#else + exit(EXIT_FAILURE); +#endif } else { err = "Bad directive or wrong number of arguments"; goto loaderr; } diff --git a/src/db.cpp b/src/db.cpp index e19dbbaf2..0446e224f 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -1573,6 +1573,12 @@ void moveCommand(client *c) { return; } + /* Return zero if the key already exists in the target DB */ + if (lookupKeyWrite(dst,c->argv[1]) != NULL) { + addReply(c,shared.czero); + return; + } + std::unique_ptr spexpire; { // scope pexpireOld std::unique_lock ul(g_expireLock); @@ -1587,12 +1593,6 @@ void moveCommand(client *c) { dbDelete(src,c->argv[1]); g_pserver->dirty++; - /* Return zero if the key already exists in the target DB */ - if (lookupKeyWrite(dst,c->argv[1]) != NULL) { - addReply(c,shared.czero); - decrRefCount(o); - return; - } dbAdd(dst,c->argv[1],o); if (spexpire != nullptr) setExpire(c,dst,c->argv[1],std::move(*spexpire)); @@ -2644,7 +2644,10 @@ bool redisDbPersistentData::insert(char *key, robj *o, bool fAssumeNew, dict_ite ensure(key); dictEntry *de; int res = dictAdd(m_pdict, key, o, &de); - serverAssert(FImplies(fAssumeNew, res == DICT_OK)); + if (!FImplies(fAssumeNew, res == DICT_OK)) { + serverLog(LL_WARNING, + "Assumed new key %s existed in DB.", key); + } if (res == DICT_OK) { #ifdef CHECKED_BUILD @@ -3297,6 +3300,10 @@ bool redisDbPersistentData::prefetchKeysAsync(client *c, parsed_command &command auto cmd = lookupCommand(szFromObj(command.argv[0])); if (cmd == nullptr) return false; // Bad command? It's not for us to judge, just bail + + if (command.argc < std::abs(cmd->arity)) + return false; // Invalid number of args + int numkeys = getKeysFromCommand(cmd, command.argv, command.argc, &result); for (int ikey = 0; ikey < numkeys; ++ikey) { diff --git a/src/gc.h b/src/gc.h index b60c562a3..eb7a42ff3 100644 --- a/src/gc.h +++ b/src/gc.h @@ -16,7 +16,17 @@ class GarbageCollector struct EpochHolder { uint64_t tstamp; - std::vector> m_vecObjs; + std::unique_ptr>> m_spvecObjs; + + EpochHolder() { + m_spvecObjs = std::make_unique>>(); + } + + // Support move operators + EpochHolder(EpochHolder &&other) = default; + EpochHolder &operator=(EpochHolder &&) = default; + + bool operator<(uint64_t tstamp) const { @@ -108,12 +118,12 @@ public: { EpochHolder e; e.tstamp = m_epochNext+1; - e.m_vecObjs.push_back(std::move(sp)); + e.m_spvecObjs->push_back(std::move(sp)); m_listepochs.emplace_back(std::move(e)); } else { - itr->m_vecObjs.push_back(std::move(sp)); + itr->m_spvecObjs->push_back(std::move(sp)); } } @@ -123,4 +133,4 @@ private: std::list m_listepochs; std::unordered_set m_setepochOutstanding; uint64_t m_epochNext = 0; -}; \ No newline at end of file +}; diff --git a/src/networking.cpp b/src/networking.cpp index 2bd3e5631..dc0abd79c 100644 --- a/src/networking.cpp +++ b/src/networking.cpp @@ -1999,9 +1999,9 @@ void ProcessPendingAsyncWrites() * writes may have been signalled without having been copied to the replyAsync buffer, * thus causing the buffer to be NULL */ if (c->replyAsync != nullptr){ - int size = c->replyAsync->used; + size_t size = c->replyAsync->used; - if (listLength(c->reply) == 0 && size <= (PROTO_REPLY_CHUNK_BYTES - c->bufpos)) { + if (listLength(c->reply) == 0 && size <= static_cast(PROTO_REPLY_CHUNK_BYTES - c->bufpos)) { memcpy(c->buf + c->bufpos, c->replyAsync->buf(), size); c->bufpos += size; } else { diff --git a/src/object.cpp b/src/object.cpp index 4f6a56338..b7d0ec15f 100644 --- a/src/object.cpp +++ b/src/object.cpp @@ -1091,7 +1091,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) { mem_total += g_pserver->initial_memory_usage; mem = 0; - if (g_pserver->repl_backlog) + if (g_pserver->repl_backlog && g_pserver->repl_backlog != g_pserver->repl_backlog_disk) mem += zmalloc_size(g_pserver->repl_backlog); mh->repl_backlog = mem; mem_total += mem; @@ -1140,7 +1140,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) { db->size() * sizeof(robj); mh->db[mh->num_dbs].overhead_ht_main = mem; mem_total+=mem; - + std::unique_lock ul(g_expireLock); mem = db->setexpire()->estimated_bytes_used(); mh->db[mh->num_dbs].overhead_ht_expires = mem; diff --git a/src/redis-cli.c b/src/redis-cli.c index a03194659..a2ed82d39 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -5485,8 +5485,8 @@ static int clusterManagerCommandRebalance(int argc, char **argv) { listAddNodeTail(lsrc, src); table = clusterManagerComputeReshardTable(lsrc, numslots); listRelease(lsrc); - int table_len = (int) listLength(table); - if (!table || table_len != numslots) { + int table_len = 0; + if (!table || (table_len = (int) listLength(table)) != numslots) { clusterManagerLogErr("*** Assertion failed: Reshard table " "!= number of slots"); result = 0; diff --git a/src/redis-cli.h b/src/redis-cli.h index 20ef1aba4..76b77d34d 100644 --- a/src/redis-cli.h +++ b/src/redis-cli.h @@ -97,9 +97,6 @@ extern "C" { #define CC_FORCE (1<<0) /* Re-connect if already connected. */ #define CC_QUIET (1<<1) /* Don't log connecting errors. */ -struct clusterManagerLink; -typedef struct clusterManagerLink clusterManagerLink; - /* Dict Helpers */ uint64_t dictSdsHash(const void *key); diff --git a/src/replication.cpp b/src/replication.cpp index 44ab62992..499bb533e 100644 --- a/src/replication.cpp +++ b/src/replication.cpp @@ -190,7 +190,7 @@ int bg_unlink(const char *filename) { bool createDiskBacklog() { // Lets create some disk backed pages and add them here std::string path = "./repl-backlog-temp" + std::to_string(gettid()); -#ifdef __APPLE__ +#if (defined __APPLE__ || defined __FreeBSD__) int fd = open(path.c_str(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); #else int fd = open(path.c_str(), O_CREAT | O_RDWR | O_LARGEFILE, S_IRUSR | S_IWUSR); @@ -1132,6 +1132,14 @@ public: replica->repl_put_online_on_ack = 1; } } + + void abort() { + for (auto replica : replicas) { + // Close the connection to force a resync + freeClientAsync(replica); + } + replicas.clear(); + } }; int rdbSaveSnapshotForReplication(struct rdbSaveInfo *rsi) { @@ -1227,7 +1235,11 @@ int rdbSaveSnapshotForReplication(struct rdbSaveInfo *rsi) { retval = C_ERR; break; } - serverAssert(count == snapshotDeclaredCount); + if (count != snapshotDeclaredCount) { + serverLog(LL_WARNING, "Replication BUG: Count of keys sent does not match actual count. Aborting full sync."); + replBuf.abort(); + break; + } } replBuf.end(); @@ -5640,7 +5652,10 @@ void flushReplBacklogToClients() replica->repl_end_off = g_pserver->master_repl_offset; /* Only if the there isn't already a pending write do we prepare the client to write */ - serverAssert(replica->repl_curr_off != g_pserver->master_repl_offset); + if (replica->repl_curr_off == g_pserver->master_repl_offset) { + serverLog(LL_DEBUG, "Pending write when it's on repl_offset=%lld", g_pserver->master_repl_offset); + continue; + } prepareClientToWrite(replica); } if (fAsyncWrite) diff --git a/src/server.cpp b/src/server.cpp index f727a82c4..59bc7c9a3 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -5070,6 +5070,8 @@ bool client::asyncCommand(std::function &&postFn) { serverAssert(FCorrectThread(this)); + if (serverTL->in_eval) + return false; // we cannot block clients in EVAL const redisDbPersistentDataSnapshot *snapshot = nullptr; if (!(this->flags & (CLIENT_MULTI | CLIENT_BLOCKED))) snapshot = this->db->createSnapshot(this->mvccCheckpoint, false /* fOptional */); diff --git a/src/storage/rocksdbfactory.cpp b/src/storage/rocksdbfactory.cpp index 701314d0f..398d6f68d 100644 --- a/src/storage/rocksdbfactory.cpp +++ b/src/storage/rocksdbfactory.cpp @@ -47,6 +47,7 @@ rocksdb::Options RocksDBStorageFactory::RocksDbOptions() options.create_if_missing = true; options.create_missing_column_families = true; options.info_log_level = rocksdb::ERROR_LEVEL; + options.max_total_wal_size = 1 * 1024 * 1024 * 1024; return options; } diff --git a/src/t_stream.cpp b/src/t_stream.cpp index b005cf600..ac895661e 100644 --- a/src/t_stream.cpp +++ b/src/t_stream.cpp @@ -789,13 +789,13 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) { * update it after (and if) we actually remove the entry */ unsigned char *pcopy = p; - int flags = lpGetInteger(p); + int64_t flags = lpGetInteger(p); p = lpNext(lp, p); /* Skip flags. */ - int to_skip; + int64_t to_skip; - int ms_delta = lpGetInteger(p); + int64_t ms_delta = lpGetInteger(p); p = lpNext(lp, p); /* Skip ID ms delta */ - int seq_delta = lpGetInteger(p); + int64_t seq_delta = lpGetInteger(p); p = lpNext(lp, p); /* Skip ID seq delta */ streamID currid = {0}; /* For MINID */ @@ -1125,7 +1125,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { * the first entry emitted for this listpack, then we already * emitted the current entry, and have to go back to the previous * one. */ - int lp_count = lpGetInteger(si->lp_ele); + int64_t lp_count = lpGetInteger(si->lp_ele); while(lp_count--) si->lp_ele = lpPrev(si->lp,si->lp_ele); /* Seek lp-count of prev entry. */ si->lp_ele = lpPrev(si->lp,si->lp_ele); @@ -1155,7 +1155,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { /* Get the flags entry. */ si->lp_flags = si->lp_ele; - int flags = lpGetInteger(si->lp_ele); + int64_t flags = lpGetInteger(si->lp_ele); si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek ID. */ /* Get the ID: it is encoded as difference between the master @@ -1264,7 +1264,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) { * deleted entries in the listpack header. * * We start flagging: */ - int flags = lpGetInteger(si->lp_flags); + int64_t flags = lpGetInteger(si->lp_flags); flags |= STREAM_ITEM_FLAG_DELETED; lp = lpReplaceInteger(lp,&si->lp_flags,flags); diff --git a/src/tls.cpp b/src/tls.cpp index 1527a5f91..7d49d2af4 100644 --- a/src/tls.cpp +++ b/src/tls.cpp @@ -46,6 +46,9 @@ #include #include +#if OPENSSL_VERSION_NUMBER >= 0x30000000L +#include +#endif #define REDIS_TLS_PROTO_TLSv1 (1<<0) #define REDIS_TLS_PROTO_TLSv1_1 (1<<1) @@ -154,14 +157,13 @@ void tlsInit(void) { */ #if OPENSSL_VERSION_NUMBER < 0x10100000L OPENSSL_config(NULL); + SSL_load_error_strings(); + SSL_library_init(); #elif OPENSSL_VERSION_NUMBER < 0x10101000L OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL); #else OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG|OPENSSL_INIT_ATFORK, NULL); #endif - ERR_load_crypto_strings(); - SSL_load_error_strings(); - SSL_library_init(); #ifdef USE_CRYPTO_LOCKS initCryptoLocks(); @@ -364,20 +366,46 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { if (ctx_config->prefer_server_ciphers) SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); -#if defined(SSL_CTX_set_ecdh_auto) +#if ((OPENSSL_VERSION_NUMBER < 0x30000000L) && defined(SSL_CTX_set_ecdh_auto)) SSL_CTX_set_ecdh_auto(ctx, 1); #endif SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE); if (ctx_config->dh_params_file) { FILE *dhfile = fopen(ctx_config->dh_params_file, "r"); - DH *dh = NULL; if (!dhfile) { serverLog(LL_WARNING, "Failed to load %s: %s", ctx_config->dh_params_file, strerror(errno)); goto error; } - dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL); +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + EVP_PKEY *pkey = NULL; + OSSL_DECODER_CTX *dctx = OSSL_DECODER_CTX_new_for_pkey( + &pkey, "PEM", NULL, "DH", OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS, NULL, NULL); + if (!dctx) { + serverLog(LL_WARNING, "No decoder for DH params."); + fclose(dhfile); + goto error; + } + if (!OSSL_DECODER_from_fp(dctx, dhfile)) { + serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file); + OSSL_DECODER_CTX_free(dctx); + fclose(dhfile); + goto error; + } + + OSSL_DECODER_CTX_free(dctx); + fclose(dhfile); + + if (SSL_CTX_set0_tmp_dh_pkey(ctx, pkey) <= 0) { + ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf)); + serverLog(LL_WARNING, "Failed to load DH params file: %s: %s", ctx_config->dh_params_file, errbuf); + EVP_PKEY_free(pkey); + goto error; + } + /* Not freeing pkey, it is owned by OpenSSL now */ +#else + DH *dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL); fclose(dhfile); if (!dh) { serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file); @@ -392,6 +420,11 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) { } DH_free(dh); +#endif + } else { +#if (OPENSSL_VERSION_NUMBER >= 0x30000000L) + SSL_CTX_set_dh_auto(ctx, 1); +#endif } /* If a client-side certificate is configured, create an explicit client context */ diff --git a/tests/integration/replication-fast.tcl b/tests/integration/replication-fast.tcl index a7368ff2c..e0da65edd 100644 --- a/tests/integration/replication-fast.tcl +++ b/tests/integration/replication-fast.tcl @@ -6,29 +6,31 @@ proc prepare_value {size} { return $_v } -start_server {tags {"replication-fast"} overrides {storage-provider {flash ./rocks.db.master} databases 256}} { - set slave [srv 0 client] - set slave_host [srv 0 host] - set slave_port [srv 0 port] - start_server {tags {} overrides {storage-provider {flash ./rocks.db.replica} databases 256}} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] +if {$::flash_enabled} { + start_server {tags {"replication-fast"} overrides {storage-provider {flash ./rocks.db.master} databases 256}} { + set slave [srv 0 client] + set slave_host [srv 0 host] + set slave_port [srv 0 port] + start_server {tags {} overrides {storage-provider {flash ./rocks.db.replica} databases 256}} { + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] - test "fast replication with large value" { - set _v [prepare_value [expr 64*1024*1024]] - # $master set key $_v + test "fast replication with large value" { + set _v [prepare_value [expr 64*1024*1024]] + # $master set key $_v - $slave replicaof $master_host $master_port - wait_for_condition 50 300 { - [lindex [$slave role] 0] eq {slave} && - [string match {*master_link_status:up*} [$slave info replication]] - } else { - fail "Can't turn the instance into a replica" + $slave replicaof $master_host $master_port + wait_for_condition 50 300 { + [lindex [$slave role] 0] eq {slave} && + [string match {*master_link_status:up*} [$slave info replication]] + } else { + fail "Can't turn the instance into a replica" + } + + assert_equal [$slave debug digest] [$master debug digest] + $slave replicaof no one } - - assert_equal [$slave debug digest] [$master debug digest] - $slave replicaof no one } } } diff --git a/tests/integration/replication-psync-flash.tcl b/tests/integration/replication-psync-flash.tcl index 8aa1ff010..84e877c6b 100644 --- a/tests/integration/replication-psync-flash.tcl +++ b/tests/integration/replication-psync-flash.tcl @@ -8,128 +8,130 @@ # If reconnect is > 0, the test actually try to break the connection and # reconnect with the master, otherwise just the initial synchronization is # checked for consistency. -proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} { - start_server [list tags {"repl"} overrides [list storage-provider {flash .rocks.db.m} repl-backlog-size 64m]] { - start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] { +if {$::flash_enabled} { + proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} { + start_server [list tags {"repl"} overrides [list storage-provider {flash .rocks.db.m} repl-backlog-size 64m]] { + start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] { - set master [srv -1 client] - set master_host [srv -1 host] - set master_port [srv -1 port] - set slave [srv 0 client] + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] - set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] - set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] - set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000] + set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] + set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] + set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000] - test {Slave should be able to synchronize with the master} { - $slave slaveof $master_host $master_port - wait_for_condition 50 100 { - [lindex [r role] 0] eq {slave} && - [lindex [r role] 3] eq {connected} - } else { - fail "Replication not started." + test {Slave should be able to synchronize with the master} { + $slave slaveof $master_host $master_port + wait_for_condition 50 100 { + [lindex [r role] 0] eq {slave} && + [lindex [r role] 3] eq {connected} + } else { + fail "Replication not started." + } } - } - # Check that the background clients are actually writing. - test {Detect write load to master} { - wait_for_condition 50 1000 { - [$master dbsize] > 100 - } else { - fail "Can't detect write load from background clients." + # Check that the background clients are actually writing. + test {Detect write load to master} { + wait_for_condition 50 1000 { + [$master dbsize] > 100 + } else { + fail "Can't detect write load from background clients." + } } - } - test "Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect)" { - # Now while the clients are writing data, break the maste-slave - # link multiple times. - if ($reconnect) { - for {set j 0} {$j < $duration*10} {incr j} { - after 100 - # catch {puts "MASTER [$master dbsize] keys, REPLICA [$slave dbsize] keys"} + test "Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect)" { + # Now while the clients are writing data, break the maste-slave + # link multiple times. + if ($reconnect) { + for {set j 0} {$j < $duration*10} {incr j} { + after 100 + # catch {puts "MASTER [$master dbsize] keys, REPLICA [$slave dbsize] keys"} - if {($j % 20) == 0} { - catch { - $slave debug restart + if {($j % 20) == 0} { + catch { + $slave debug restart + } } } } - } - stop_bg_complex_data $load_handle0 - stop_bg_complex_data $load_handle1 - stop_bg_complex_data $load_handle2 + stop_bg_complex_data $load_handle0 + stop_bg_complex_data $load_handle1 + stop_bg_complex_data $load_handle2 - # Wait for the slave to reach the "online" - # state from the POV of the master. - set retry 5000 - while {$retry} { - set info [$master info] - if {[string match {*slave0:*state=online*} $info]} { - break - } else { - incr retry -1 - after 100 + # Wait for the slave to reach the "online" + # state from the POV of the master. + set retry 5000 + while {$retry} { + set info [$master info] + if {[string match {*slave0:*state=online*} $info]} { + break + } else { + incr retry -1 + after 100 + } + } + if {$retry == 0} { + error "assertion:Slave not correctly synchronized" } - } - if {$retry == 0} { - error "assertion:Slave not correctly synchronized" - } - # Wait that slave acknowledge it is online so - # we are sure that DBSIZE and DEBUG DIGEST will not - # fail because of timing issues. (-LOADING error) - wait_for_condition 5000 100 { - [lindex [$slave role] 3] eq {connected} - } else { - fail "Slave still not connected after some time" - } + # Wait that slave acknowledge it is online so + # we are sure that DBSIZE and DEBUG DIGEST will not + # fail because of timing issues. (-LOADING error) + wait_for_condition 5000 100 { + [lindex [$slave role] 3] eq {connected} + } else { + fail "Slave still not connected after some time" + } - set retry 20 - while {$retry && ([$master debug digest] ne [$slave debug digest])}\ - { - after 1000 - incr retry -1 - } - assert {[$master dbsize] > 0} + set retry 20 + while {$retry && ([$master debug digest] ne [$slave debug digest])}\ + { + after 1000 + incr retry -1 + } + assert {[$master dbsize] > 0} - if {[$master debug digest] ne [$slave debug digest]} { - set csv1 [csvdump r] - set csv2 [csvdump {r -1}] - set fd [open /tmp/repldump1.txt w] - puts -nonewline $fd $csv1 - close $fd - set fd [open /tmp/repldump2.txt w] - puts -nonewline $fd $csv2 - close $fd - puts "Master - Replica inconsistency" - puts "Run diff -u against /tmp/repldump*.txt for more info" + if {[$master debug digest] ne [$slave debug digest]} { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Replica inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + } + assert_equal [r debug digest] [r -1 debug digest] + eval $cond } - assert_equal [r debug digest] [r -1 debug digest] - eval $cond } } } -} -foreach mdl {no yes} { - foreach sdl {disabled swapdb} { - test_psync {no reconnection, just sync} 6 1000000 3600 0 { - } $mdl $sdl 0 + foreach mdl {no yes} { + foreach sdl {disabled swapdb} { + test_psync {no reconnection, just sync} 6 1000000 3600 0 { + } $mdl $sdl 0 - test_psync {ok psync} 6 100000000 3600 0 { - assert {[s -1 sync_partial_ok] > 0} - } $mdl $sdl 1 + test_psync {ok psync} 6 100000000 3600 0 { + assert {[s -1 sync_partial_ok] > 0} + } $mdl $sdl 1 - test_psync {no backlog} 6 100 3600 0.5 { - assert {[s -1 sync_partial_err] > 0} - } $mdl $sdl 1 + test_psync {no backlog} 6 100 3600 0.5 { + assert {[s -1 sync_partial_err] > 0} + } $mdl $sdl 1 - test_psync {ok after delay} 3 100000000 3600 3 { - assert {[s -1 sync_partial_ok] > 0} - } $mdl $sdl 1 + test_psync {ok after delay} 3 100000000 3600 3 { + assert {[s -1 sync_partial_ok] > 0} + } $mdl $sdl 1 - test_psync {backlog expired} 3 100000000 1 3 { - assert {[s -1 sync_partial_err] > 0} - } $mdl $sdl 1 + test_psync {backlog expired} 3 100000000 1 3 { + assert {[s -1 sync_partial_err] > 0} + } $mdl $sdl 1 + } } } diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index d9c0e870e..d5e714449 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -739,6 +739,16 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } +# Check if we compiled with flash +set status [catch {exec src/keydb-server --is-flash-enabled}] +if {$status == 0} { + puts "KeyDB was built with FLASH, including FLASH tests" + set ::flash_enabled 1 +} else { + puts "KeyDB was not built with FLASH. Excluding FLASH tests" + set ::flash_enabled 0 +} + set filtered_tests {} # Set the filtered tests to be the short list (single_tests) if exists. diff --git a/tests/unit/flash.tcl b/tests/unit/flash.tcl index ea3c8e9fe..e66248fb2 100644 --- a/tests/unit/flash.tcl +++ b/tests/unit/flash.tcl @@ -1,170 +1,171 @@ -start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] { +if {$::flash_enabled} { + start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] { - test { FLASH - GET works after eviction } { - r set testkey foo - r flushall cache - assert_equal {foo} [r get testkey] - } + test { FLASH - GET works after eviction } { + r set testkey foo + r flushall cache + assert_equal {foo} [r get testkey] + } - test { DEL of nonexistant key returns 0 } { + test { DEL of nonexistant key returns 0 } { + r flushall + assert_equal {0} [r del foobar] + assert_equal {0} [r dbsize] "Key count is accurate after non-existant delete" + } + + test { DEL of flushed key works } { + r flushall + r set testkey foo + assert_equal {1} [r dbsize] "Only one key after first insert" + r flushall cache + assert_equal {foo} [r get testkey] "Value still there after flushing cache" + r del testkey + assert_equal {0} [r dbsize] "No keys after delete" + } + + test { SET of existing but flushed key works } { + r flushall + r set testkey foo + assert_equal {1} [r dbsize] "Only one key after first insert" + r flushall cache + assert_equal {1} [r dbsize] "Only one key after flushall cache" + r set testkey bar + assert_equal {1} [r dbsize] "Only one key after overwrite" + assert_equal {bar} [r get testkey] + } + + test { SET of existing but flushed key with EXPIRE works } { r flushall - assert_equal {0} [r del foobar] - assert_equal {0} [r dbsize] "Key count is accurate after non-existant delete" - } - - test { DEL of flushed key works } { - r flushall - r set testkey foo - assert_equal {1} [r dbsize] "Only one key after first insert" - r flushall cache - assert_equal {foo} [r get testkey] "Value still there after flushing cache" - r del testkey - assert_equal {0} [r dbsize] "No keys after delete" - } - - test { SET of existing but flushed key works } { - r flushall - r set testkey foo + assert_equal {0} [r dbsize] + r set testkey foo ex 10000 assert_equal {1} [r dbsize] "Only one key after first insert" r flushall cache assert_equal {1} [r dbsize] "Only one key after flushall cache" - r set testkey bar + r set testkey bar ex 10000 assert_equal {1} [r dbsize] "Only one key after overwrite" assert_equal {bar} [r get testkey] - } - - test { SET of existing but flushed key with EXPIRE works } { - r flushall - assert_equal {0} [r dbsize] - r set testkey foo ex 10000 - assert_equal {1} [r dbsize] "Only one key after first insert" - r flushall cache - assert_equal {1} [r dbsize] "Only one key after flushall cache" - r set testkey bar ex 10000 - assert_equal {1} [r dbsize] "Only one key after overwrite" - assert_equal {bar} [r get testkey] - assert [expr [r ttl testkey] > 0] - } - - test { EXPIRE of existing but flushed key } { - r flushall - assert_equal {0} [r dbsize] - r set testkey foo - assert_equal {1} [r dbsize] - r flushall cache - r expire testkey 10000 - assert_equal {1} [r dbsize] - assert_equal {foo} [r get testkey] - assert [expr [r ttl testkey] > 0] - } - - test { CREATE and UPDATE in transaction, key count is accurate } { - r flushall - r multi - r set testkey 2 - r incr testkey - r exec - assert_equal {1} [r dbsize] - assert_equal {3} [r get testkey] - } - - test { EXPIRE key count is accurate } { - r flushall - r set testkey foo ex 1 - r flushall cache - assert_equal {1} [r dbsize] - after 1500 - assert_equal {0} [r dbsize] - } - - test { SUBKEY EXPIRE persists after cache flush } { - r flushall - r sadd testkey foo bar baz - r expiremember testkey foo 10000 - r flushall cache - assert [expr [r ttl testkey foo] > 0] - } - - test { LIST pop works after flushing cache } { - r flushall - r lpush testkey foo - r flushall cache - assert_equal {foo} [r lpop testkey] - } - - test { DIGEST string the same after flushing cache } { - r flushall - r set testkey foo - r set testkey1 foo ex 10000 - set expectedDigest [r debug digest] - r flushall cache - assert_equal $expectedDigest [r debug digest] - } - - test { DIGEST list the same after flushing cache } { - r flushall - r lpush testkey foo bar - set expectedDigest [r debug digest] - r flushall cache - assert_equal $expectedDigest [r debug digest] - } - - test { DELETE of flushed set member persists after another flush } { - r flushall - r sadd set1 val1 val2 val3 - assert_equal {3} [r scard set1] - r flushall cache - r srem set1 val1 - r flushall cache - assert_equal {2} [r scard set1] - } - - r flushall - # If a weak storage memory model is set, wait for any pending snapshot writes to finish - after 500 - foreach policy { - allkeys-random allkeys-lru allkeys-lfu - } { - test "FLASH - is eviction working without data loss (successfully stored to flash)? (policy $policy)" { - # Get the current memory limit and calculate a new limit. - # Set limit to 100M. - set used [s used_memory] - set limit [expr {$used+60*1024*1024}] - r config set maxmemory $limit - r config set maxmemory-policy $policy - # Now add keys equivalent to 1024b until the limit is almost reached. - set numkeys 0 - r set first val - while 1 { - r set $numkeys xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - incr numkeys - if {[s used_memory]+1024 >= $limit} { - break - } - } - # Add additional keys to force eviction - # should still be under the limit for maxmemory, however all keys set should still exist between flash and memory - # check same number of keys exist in addition to values of first and last keys - set err 0 - set extra_keys [expr floor([expr ($limit * 0.4) / 1024])] - for {set j 0} {$j < $extra_keys} {incr j} { - catch { - r set p2$j xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - } err - assert {$err == {OK}} - } - if {[log_file_matches [srv 0 stdout] "*Failed to evict*"]} { - fail "Server did not evict cleanly (detected full flush)" - } - r set last val - set dbsize [r dbsize] - assert {[s used_memory] < ($limit*1.2)} - assert {$dbsize == $numkeys+$extra_keys+2} - assert {[r get first] == {val}} - assert {[r get last] == {val}} - r flushall + assert [expr [r ttl testkey] > 0] } + + test { EXPIRE of existing but flushed key } { + r flushall + assert_equal {0} [r dbsize] + r set testkey foo + assert_equal {1} [r dbsize] + r flushall cache + r expire testkey 10000 + assert_equal {1} [r dbsize] + assert_equal {foo} [r get testkey] + assert [expr [r ttl testkey] > 0] + } + + test { CREATE and UPDATE in transaction, key count is accurate } { + r flushall + r multi + r set testkey 2 + r incr testkey + r exec + assert_equal {1} [r dbsize] + assert_equal {3} [r get testkey] + } + + test { EXPIRE key count is accurate } { + r flushall + r set testkey foo ex 1 + r flushall cache + assert_equal {1} [r dbsize] + after 1500 + assert_equal {0} [r dbsize] + } + + test { SUBKEY EXPIRE persists after cache flush } { + r flushall + r sadd testkey foo bar baz + r expiremember testkey foo 10000 + r flushall cache + assert [expr [r ttl testkey foo] > 0] + } + + test { LIST pop works after flushing cache } { + r flushall + r lpush testkey foo + r flushall cache + assert_equal {foo} [r lpop testkey] + } + + test { DIGEST string the same after flushing cache } { + r flushall + r set testkey foo + r set testkey1 foo ex 10000 + set expectedDigest [r debug digest] + r flushall cache + assert_equal $expectedDigest [r debug digest] + } + + test { DIGEST list the same after flushing cache } { + r flushall + r lpush testkey foo bar + set expectedDigest [r debug digest] + r flushall cache + assert_equal $expectedDigest [r debug digest] + } + + test { DELETE of flushed set member persists after another flush } { + r flushall + r sadd set1 val1 val2 val3 + assert_equal {3} [r scard set1] + r flushall cache + r srem set1 val1 + r flushall cache + assert_equal {2} [r scard set1] + } + + r flushall + # If a weak storage memory model is set, wait for any pending snapshot writes to finish + after 500 + foreach policy { + allkeys-random allkeys-lru allkeys-lfu + } { + test "FLASH - is eviction working without data loss (successfully stored to flash)? (policy $policy)" { + # Get the current memory limit and calculate a new limit. + # Set limit to 100M. + set used [s used_memory] + set limit [expr {$used+60*1024*1024}] + r config set maxmemory $limit + r config set maxmemory-policy $policy + # Now add keys equivalent to 1024b until the limit is almost reached. + set numkeys 0 + r set first val + while 1 { + r set $numkeys xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + incr numkeys + if {[s used_memory]+1024 >= $limit} { + break + } + } + # Add additional keys to force eviction + # should still be under the limit for maxmemory, however all keys set should still exist between flash and memory + # check same number of keys exist in addition to values of first and last keys + set err 0 + set extra_keys [expr floor([expr ($limit * 0.4) / 1024])] + for {set j 0} {$j < $extra_keys} {incr j} { + catch { + r set p2$j xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + } err + assert {$err == {OK}} + } + if {[log_file_matches [srv 0 stdout] "*Failed to evict*"]} { + fail "Server did not evict cleanly (detected full flush)" + } + r set last val + set dbsize [r dbsize] + assert {[s used_memory] < ($limit*1.2)} + assert {$dbsize == $numkeys+$extra_keys+2} + assert {[r get first] == {val}} + assert {[r get last] == {val}} + r flushall + } + } + } - } - diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index e4af19d5c..2f4722d98 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -126,6 +126,10 @@ start_server {tags {"scripting"}} { r select 9 set res } {original value} + + test {EVAL background command} { + r eval {redis.call("SCAN", "0", "MATCH", "key*", "COUNT", 1000)} 0 + } {} if 0 { test {EVAL - Script can't run more than configured time limit} { diff --git a/tests/unit/type/stream.tcl b/tests/unit/type/stream.tcl index c299a1e97..2990f47ef 100644 --- a/tests/unit/type/stream.tcl +++ b/tests/unit/type/stream.tcl @@ -172,6 +172,15 @@ start_server { assert_equal [r XRANGE mystream - +] {{3-0 {f v}} {4-0 {f v}} {5-0 {f v}}} } + test {XTRIM with MINID option, big delta from master record} { + r DEL mystream + r XADD mystream 1-0 f v + r XADD mystream 1641544570597-0 f v + r XADD mystream 1641544570597-1 f v + r XTRIM mystream MINID 1641544570597-0 + assert_equal [r XRANGE mystream - +] {{1641544570597-0 {f v}} {1641544570597-1 {f v}}} + } + test {XADD mass insertion and XLEN} { r DEL mystream r multi diff --git a/utils/redis-copy.rb b/utils/redis-copy.rb index aa9e797b5..c32c0c022 100644 --- a/utils/redis-copy.rb +++ b/utils/redis-copy.rb @@ -1,27 +1,27 @@ -# keydb-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo +# redis-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo # BSD license, See the COPYING file for more information. # -# Copy the whole dataset from one Redis instance to another one +# Copy the whole dataset from one KeyDB/Redis instance to another one # # WARNING: this utility is deprecated and serves as a legacy adapter -# for the more-robust keydb-copy gem. +# for the more-robust redis-copy gem. require 'shellwords' def redisCopy(opts={}) src = "#{opts[:srchost]}:#{opts[:srcport]}" dst = "#{opts[:dsthost]}:#{opts[:dstport]}" - `keydb-copy #{src.shellescape} #{dst.shellescape}` + `redis-copy #{src.shellescape} #{dst.shellescape}` rescue Errno::ENOENT - $stderr.puts 'This utility requires the keydb-copy executable', - 'from the keydb-copy gem on https://rubygems.org', - 'To install it, run `gem install keydb-copy`.' + $stderr.puts 'This utility requires the redis-copy executable', + 'from the redis-copy gem on https://rubygems.org', + 'To install it, run `gem install redis-copy`.' exit 1 end -$stderr.puts "This utility is deprecated. Use the keydb-copy gem instead." +$stderr.puts "This utility is deprecated. Use the redis-copy gem instead." if ARGV.length != 4 - puts "Usage: keydb-copy.rb " + puts "Usage: redis-copy.rb " exit 1 end puts "WARNING: it's up to you to FLUSHDB the destination host before to continue, press any key when ready."