Compare commits

..
Author SHA1 Message Date
Paul ChenandPaul Chen 37fd5d25fe [BUG] Moves key to another DB, the source key was removed if the move failed due to the key exists in the destination db #497 (#498)
Co-authored-by: Paul Chen <mingchen@Mings-MacBook-Pro.local>
2022-10-20 18:06:58 -04:00
John Sully e8e3666dcb Workaround bug in snapshot sync - abort don't crash 2022-09-28 19:00:26 +00:00
John Sully 597b5669e3 Fix Issue #486 2022-09-14 14:48:28 +00:00
Ilya Shipitsin 772bf47ef7 src\redis-cli.c: fix potential null pointer dereference found by cppcheck
src\redis-cli.c:5488:35: warning: Either the condition
'!table' is redundant or there is possible null pointer dereference:
table. [nullPointerRedundantCheck]
2022-09-14 10:03:08 -04:00
root 7bf9c02954 avoid from instatiating EpochHolder multiple times to improve performance and cpu utilization 2022-09-14 10:02:02 -04:00
root 8661cfe7bd avoid from instatiating EpochHolder multiple times to improve performance and cpu utilization 2022-09-14 10:02:02 -04:00
Karthick Ariyaratnam (A) 2173975aaf Remove old assertion which is commented out. 2022-09-14 10:01:21 -04:00
Karthick Ariyaratnam (A) b989cd80c1 Fixes a crash where the server assertion failed when the key exists in DB during RDB load 2022-09-14 10:01:21 -04:00
John Sully 79b6c65185 Prevent crash on load in long running KeyDB instances 2022-09-01 15:26:26 +00:00
John Sully e114f41d5c Fix race removing key cache 2022-09-01 15:16:27 +00:00
John Sully a2e6e35d5e Fix crash running INFO command while a disk based backlog is set 2022-08-01 04:53:56 +00:00
John Sully 9e5d1c8ec9 Fix issue #480 2022-08-01 03:14:09 +00:00
John Sully 1948f21d0d Do not allow commands to run in background when in eval, Issue #452 2022-07-06 04:26:13 +00:00
John Sully 3cba0e0ba4 Fix issue #454 (BSD build break) 2022-07-06 04:04:47 +00:00
guybe7 3a4f46104c lpGetInteger returns int64_t, avoid overflow (#10068)
Fix #9410

Crucial for the ms and sequence deltas, but I changed all
calls, just in case (e.g. "flags")

Before this commit:
`ms_delta` and `seq_delta` could have overflown, causing `currid` to be wrong,
which in turn would cause `streamTrim` to trim the entire rax node (see new test)
2022-07-06 03:44:24 +00:00
benschermel aa032d3066 Merge pull request #438 from Snapchat/update_pkg
Packaging Updates
2022-06-06 22:38:28 -04:00
benschermel 7c9c448441 Merge pull request #443 from Snapchat/update_gem_ref
update gem reference
2022-06-06 22:35:11 -04:00
benschermel accd3871be update gem reference 2022-06-06 14:55:24 -04:00
benschermel 6ad5cabe15 update deb pkg references and changelog 2022-05-25 01:10:56 -04:00
benschermel a9ec2fc738 update dockerfiles 2022-05-25 01:08:24 -04:00
52 changed files with 658 additions and 1327 deletions

No files matched your search

+18 -23
View File
@@ -8,52 +8,40 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: make
run: |
sudo apt-get update
sudo apt-get -y remove libzstd || true
sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev
make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror'
sudo apt-get -y install uuid-dev libcurl4-openssl-dev
make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' BUILD_TLS=yes -j2
- name: gen-cert
run: ./utils/gen-test-certs.sh
- name: test-tls
run: |
sudo apt-get -y install tcl tcl-tls
./runtest --clients 1 --verbose --tls --config server-threads 3
./runtest --clients 2 --verbose --tls
- name: cluster-test
run: |
./runtest-cluster --tls --config server-threads 3
./runtest-cluster --tls
- name: sentinel test
run: |
./runtest-sentinel
- name: module tests
run: |
./runtest-moduleapi
- name: rotation test
run: |
./runtest-rotation
build-ubuntu-old:
runs-on: ubuntu-20.04
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- uses: actions/checkout@v1
- name: make -j2
run: |
sudo apt-get update
sudo apt-get -y remove libzstd || true
sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev
sudo apt-get -y install uuid-dev libcurl4-openssl-dev
make -j2
build-macos-latest:
runs-on: macos-latest
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: make
run: make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' -j2
@@ -61,11 +49,18 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
- name: make
run: |
sudo apt-get update
sudo apt-get -y remove libzstd || true
sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev
sudo apt-get -y install uuid-dev libcurl4-openssl-dev
make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' MALLOC=libc -j2
build-ubuntu-32bit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: make
run: |
sudo dpkg --add-architecture i386
sudo apt-get update
sudo apt-get -y install gcc-multilib g++-multilib libc6-dev-i386 lib32z1 uuid-dev:i386 libcurl4-openssl-dev:i386
make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' 32bit -j2
+164
View File
@@ -0,0 +1,164 @@
.standard-pipeline:
rules:
- if: '$COVERAGE'
when: never
- if: '$ENDURANCE'
when: never
- if: '$CI_PIPELINE_SOURCE == "push"'
build:
extends: .standard-pipeline
tags:
- docker
stage: build
script:
- git submodule update --init
- make distclean
- make -j
artifacts:
paths:
- src/
runtest:
extends: .standard-pipeline
dependencies:
- build
tags:
- docker
stage: test
script:
- ./runtest --config server-threads 3
runtest-cluster:
extends: .standard-pipeline
dependencies:
- build
tags:
- docker
stage: test
script:
- ./runtest-cluster
runtest-moduleapi:
extends: .standard-pipeline
dependencies:
- build
tags:
- docker
stage: test
script:
- ./runtest-moduleapi
runtest-sentinel:
extends: .standard-pipeline
dependencies:
- build
tags:
- docker
stage: test
script:
- ./runtest-sentinel
node-redis-test:
extends: .standard-pipeline
dependencies:
- build
rules:
- when: never
tags:
- docker
- ipv6
stage: test
script:
- cp -pf src/keydb-server /usr/local/bin
- cp -pf src/keydb-cli /usr/local/bin
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/node-redis.git
- cd node-redis
- npm install
- npm run test
jedis-test:
extends: .standard-pipeline
dependencies:
- build
tags:
- docker
- ipv4
stage: test
script:
- cp -pf src/keydb-server /usr/local/bin
- cp -pf src/keydb-cli /usr/local/bin
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/jedis.git
- cd jedis
- make test
redis-rs-test:
extends: .standard-pipeline
dependencies:
- build
tags:
- docker
stage: test
script:
- cp -pf src/keydb-server /usr/local/bin
- cp -pf src/keydb-cli /usr/local/bin
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/redis-rs.git
- cd redis-rs
- make test
endurance-test:
rules:
- if: '$ENDURANCE'
tags:
- docker
stage: test
script:
- git submodule update --init
- make distclean
- make -j
- ./runtest --config server-threads 3 --loop --stop
coverage-test:
rules:
- if: '$COVERAGE'
tags:
- docker
stage: test
script:
- git submodule update --init
- make distclean
- make gcov -j
- make install
- ./runtest || true
- pkill keydb-server || true
- pkill stunnel || true
- ./runtest-cluster || true
- pkill keydb-server || true
- pkill stunnel || true
- ./runtest-sentinel || true
- pkill keydb-server || true
- pkill stunnel || true
- ./runtest-moduleapi || true
- pkill keydb-server || true
- pkill stunnel || true
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/redis-rs.git
- cd redis-rs
- make test || true
- pkill keydb-server || true
- pkill stunnel || true
- cd ..
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/jedis.git
- cd jedis
- make test || true
- pkill keydb-server || true
- pkill stunnel || true
- cd ..
- git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/node-redis.git
- cd node-redis
- npm install
- npm run test || true
- pkill keydb-server || true
- pkill stunnel || true
- cd ..
- geninfo -o KeyDB.info --no-external .
- genhtml --legend -o lcov-html KeyDB.info
+1 -2
View File
@@ -1,7 +1,6 @@
[submodule "deps/rocksdb"]
path = deps/rocksdb
url = https://github.com/msotheeswaran/rocksdb.git
branch = keydb_6_3_2
url = https://github.com/facebook/rocksdb.git
[submodule "deps/depot_tools"]
path = deps/depot_tools
url = https://chromium.googlesource.com/chromium/tools/depot_tools.git
-12
View File
@@ -104,12 +104,6 @@ Avoid forwarding RREPLAY messages to other masters? WARNING: This setting is dan
If you would like KeyDB to dump and load directly to AWS S3 this option specifies the bucket. Using this option with the traditional RDB options will result in KeyDB backing up twice to both locations. If both are specified KeyDB will first attempt to load from the local dump file and if that fails load from S3. This requires the AWS CLI tools to be installed and configured which are used under the hood to transfer the data.
```
storage-provider flash /path/to/flash
```
If you would like to use KeyDB FLASH storage, specify the storage medium followed by the directory path on your local SSD volume. Note that this feature is still considered experimental and should be used with discretion. See [FLASH Documentation](https://docs.keydb.dev/docs/flash) for more details on configuration and setting up your FLASH volume.
Building KeyDB
--------------
@@ -151,12 +145,6 @@ installed):
% ./utils/gen-test-certs.sh
% ./runtest --tls
To build with KeyDB FLASH support, use:
% make ENABLE_FLASH=yes
***Note that the KeyDB FLASH feature is considered experimental (beta) and should used with discretion
Fixing build problems with dependencies or cached build options
---------
+48
View File
@@ -0,0 +1,48 @@
# Doc: https://wiki.sc-corp.net/pages/viewpage.action?pageId=121500284
version: 1
machamp:
keydb-build:
# Optional - build counter is linked to the build def
tag_template: 0.0.%build.counter%
# Optional - value in seconds before a build is terminated, default is 3600 seconds
timeout: 3600
# Optional - update ghe or not, default to true
update_ghe: true
code_coverage: false
# Required
steps:
make-build:
type: cmd
# https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
command: ./machamp_scripts/build.sh
tls-test:
type: cmd
parent: make-build
# https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
command: ./runtest --clients $(nproc) --verbose --tls
cluster-test:
type: cmd
parent: make-build
# https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
command: ./runtest-cluster --tls
sentinel-test:
type: cmd
parent: make-build
# https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
command: ./runtest-sentinel
module-test:
type: cmd
parent: make-build
# https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
command: ./runtest-moduleapi
rotation-test:
type: cmd
parent: make-build
# https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
command: ./runtest-rotation
+16
View File
@@ -0,0 +1,16 @@
# Doc: https://wiki.sc-corp.net/display/TOOL/ci.yaml+User+Guide
version: 1
on:
pull_request:
- workflows:
# All builds that use machamp should use the defined `backend_workflow`
- workflow_type: backend_workflow
# references a build defined in build.yaml
build_name: keydb-build
arch_types: ["amd64", "arm64"]
push:
- branches: [master]
workflows:
- workflow_type: backend_workflow
build_name: keydb-build
arch_types: ["amd64", "arm64"]
+3 -3
View File
@@ -101,11 +101,11 @@ jemalloc: .make-prerequisites
.PHONY: jemalloc
rocksdb: .make-prerequisites
@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
ifeq ($(uname_M),x86_64)
cd rocksdb && CFLAGS=-Wno-error PORTABLE=1 USE_SSE=1 FORCE_SSE42=1 $(MAKE) static_lib
# cd rocksdb && CFLAGS=-Wno-error PORTABLE=1 USE_SSE=1 FORCE_SSE42=1 $(MAKE) static_lib
else
cd rocksdb && PORTABLE=1 $(MAKE) static_lib
# cd rocksdb && PORTABLE=1 $(MAKE) static_lib
endif
.PHONY: rocksdb
+1 -1
+2 -18
View File
@@ -184,9 +184,7 @@ tcp-keepalive 300
#
# tls-client-key-file-pass secret
# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange,
# required by older versions of OpenSSL (<3.0). Newer versions do not require
# this configuration and recommend against it.
# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange:
#
# tls-dh-params-file keydb.dh
@@ -2073,18 +2071,4 @@ replica-weighting-factor 2
#
# By default this is enabled
#
active-client-balancing yes
# Enable FLASH support (Experimental Feature)
# storage-provider flash /path/to/flash/db
# Blob support is a way to store very large objects (>200MB) on disk
# The files are automatically cleaned up when KeyDB exits and are only
# for temporary use. This helps reduce memory pressure for very large
# data items at the cost of some performance.
#
# By default this config is disable. When enabled the disk associated
# with KeyDB's working directory will be used. If there is insufficient
# disk space or any other I/O error KeyDB will instead use memory.
#
# blob-support false
active-client-balancing yes
-14
View File
@@ -2063,17 +2063,3 @@ server-threads 2
#
# By default KeyDB sets this to 2.
replica-weighting-factor 2
# Enable FLASH support (Experimental Feature)
# storage-provider flash /path/to/flash/db
# Blob support is a way to store very large objects (>200MB) on disk
# The files are automatically cleaned up when KeyDB exits and are only
# for temporary use. This helps reduce memory pressure for very large
# data items at the cost of some performance.
#
# By default this config is disable. When enabled the disk associated
# with KeyDB's working directory will be used. If there is insufficient
# disk space or any other I/O error KeyDB will instead use memory.
#
# blob-support false
-5
View File
@@ -19,11 +19,6 @@ elif [ "$distributor" == "Ubuntu" ]; then
fi
codename=$(lsb_release --codename --short)
date=$(date +%a," "%d" "%b" "%Y" "%T)
# overwrite debian bookworm version until updated
if [ $codename == "bookworm" ]; then
distname=+deb12u1
fi
pkg_name=keydb-$majorv:$version$distname
# create build tree
-1
View File
@@ -4,7 +4,6 @@ include /usr/share/dpkg/buildflags.mk
export BUILD_TLS=yes
export USE_SYSTEMD=yes
export ENABLE_FLASH=yes
export CFLAGS CPPFLAGS LDFLAGS
export DEB_BUILD_MAINT_OPTIONS = hardening=+all
export DEB_LDFLAGS_MAINT_APPEND = -ldl -latomic $(LUA_LDFLAGS)
-1
View File
@@ -10,7 +10,6 @@ include /usr/share/dpkg/buildflags.mk
export BUILD_TLS=yes
export USE_SYSTEMD=yes
export ENABLE_FLASH=yes
export CFLAGS CPPFLAGS LDFLAGS
export DEB_BUILD_MAINT_OPTIONS = hardening=+all
export DEB_LDFLAGS_MAINT_APPEND = -ldl -latomic $(LUA_LDFLAGS)
+1 -1
View File
@@ -1,4 +1,4 @@
FROM ubuntu:20.04
FROM ubuntu:18.04
SHELL ["/bin/bash","-c"]
RUN groupadd -r keydb && useradd -r -g keydb keydb
# use gosu for easy step-down from root: https://github.com/tianon/gosu/releases
-4
View File
@@ -21,10 +21,6 @@ RUN set -eux; \
bash \
perl \
git \
bzip2-dev \
zstd-dev \
lz4-dev \
snappy-dev \
; \
cd /tmp && git clone --branch $BRANCH https://github.com/Snapchat/KeyDB.git --recursive; \
cd /tmp/KeyDB; \
@@ -2063,17 +2063,3 @@ server-threads 2
#
# By default KeyDB sets this to 2.
replica-weighting-factor 2
# Enable FLASH support (Experimental Feature)
# storage-provider flash /path/to/flash/db
# Blob support is a way to store very large objects (>200MB) on disk
# The files are automatically cleaned up when KeyDB exits and are only
# for temporary use. This helps reduce memory pressure for very large
# data items at the cost of some performance.
#
# By default this config is disable. When enabled the disk associated
# with KeyDB's working directory will be used. If there is insufficient
# disk space or any other I/O error KeyDB will instead use memory.
#
# blob-support false
+3 -13
View File
@@ -62,7 +62,6 @@ endif
endif
USEASM?=true
ENABLE_FLASH?=no
ifneq ($(strip $(SANITIZE)),)
CFLAGS+= -fsanitize=$(SANITIZE) -DSANITIZE -fno-omit-frame-pointer
@@ -72,15 +71,6 @@ ifneq ($(strip $(SANITIZE)),)
USEASM=false
endif
ifeq ($(ENABLE_FLASH),yes)
FINAL_LIBS+= -lz -lcrypto -lbz2 -lzstd -llz4 -lsnappy
CXXFLAGS+= -I../deps/rocksdb/include/ -DENABLE_ROCKSDB
STORAGE_OBJ+= storage/rocksdb.o storage/rocksdbfactory.o
FINAL_LIBS+= ../deps/rocksdb/librocksdb.a
DEPENDENCY_TARGETS+= rocksdb
endif
ifeq ($(CHECKED),true)
CXXFLAGS+= -DCHECKED_BUILD
endif
@@ -142,7 +132,7 @@ DEBUG=-g -ggdb
FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS)
FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CXXFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS)
FINAL_LDFLAGS=$(LDFLAGS) $(KEYDB_LDFLAGS) $(DEBUG)
FINAL_LIBS+=-lm -lz -lcrypto -lbz2 -lzstd -llz4 -lsnappy
FINAL_LIBS+=-lm -lz -lcrypto
ifneq ($(uname_S),Darwin)
FINAL_LIBS+=-latomic
@@ -376,7 +366,7 @@ endif
REDIS_SERVER_NAME=keydb-server$(PROG_SUFFIX)
REDIS_SENTINEL_NAME=keydb-sentinel$(PROG_SUFFIX)
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd_server.o timeout.o setcpuaffinity.o AsyncWorkQueue.o snapshot.o storage/rocksdb.o storage/rocksdbfactory.o storage/teststorageprovider.o keydbutils.o StorageCache.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)
REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd_server.o timeout.o setcpuaffinity.o AsyncWorkQueue.o snapshot.o storage/teststorageprovider.o keydbutils.o StorageCache.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)
KEYDB_SERVER_OBJ=SnapshotPayloadParseState.o
REDIS_CLI_NAME=keydb-cli$(PROG_SUFFIX)
REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crcspeed.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o motd_client.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)
@@ -437,7 +427,7 @@ endif
# keydb-server
$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(KEYDB_SERVER_OBJ)
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a ../deps/rocksdb/librocksdb.a $(FINAL_LIBS)
$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/lua/src/liblua.a $(FINAL_LIBS)
# keydb-sentinel
$(REDIS_SENTINEL_NAME): $(REDIS_SERVER_NAME)
-2
View File
@@ -43,8 +43,6 @@ void StorageCache::clear(void(callback)(void*))
void StorageCache::clearAsync()
{
std::unique_lock<fastlock> ul(m_lock);
if (count() == 0)
return;
if (m_pdict != nullptr) {
dict *dSav = m_pdict;
m_pdict = dictCreate(&dbStorageCacheType, nullptr);
+1 -1
View File
@@ -322,7 +322,7 @@ handle_wrap:
return 1;
}
int checkSignedBitfieldOverflow(int64_t value, int64_t incr, int bits, int owtype, int64_t *limit) {
int checkSignedBitfieldOverflow(int64_t value, int64_t incr, uint64_t bits, int owtype, int64_t *limit) {
int64_t max = (bits == 64) ? INT64_MAX : (((int64_t)1<<(bits-1))-1);
int64_t min = (-max)-1;
+2 -4
View File
@@ -771,7 +771,7 @@ unsigned long getClusterConnectionsCount(void) {
/* We decrement the number of nodes by one, since there is the
* "myself" node too in the list. Each node uses two file descriptors,
* one incoming and one outgoing, thus the multiplication by 2. */
return g_pserver->cluster_enabled && g_pserver->cluster != nullptr ?
return g_pserver->cluster_enabled ?
((dictSize(g_pserver->cluster->nodes)-1)*2) : 0;
}
@@ -1525,9 +1525,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
* it's greater than our view but is not in the future
* (with 500 milliseconds tolerance) from the POV of our
* clock. */
mstime_t mstime;
__atomic_load(&g_pserver->mstime, &mstime, __ATOMIC_RELAXED);
if (pongtime <= (mstime+500) &&
if (pongtime <= (g_pserver->mstime+500) &&
pongtime > node->pong_received)
{
node->pong_received = pongtime;
+8 -16
View File
@@ -355,17 +355,16 @@ bool initializeStorageProvider(const char **err)
return true;
if (!strcasecmp(g_sdsProvider, "flash") && g_sdsArgs != nullptr)
{
#ifdef ENABLE_ROCKSDB
// Temporary: Disable FLASH
serverLog(LL_WARNING, "FLASH Is Not yet Supported");
_Exit(EXIT_FAILURE);
// Create The Storage Factory (if necessary)
serverLog(LL_NOTICE, "Initializing FLASH storage provider (this may take a long time)");
adjustOpenFilesLimit();
g_pserver->m_pstorageFactory = CreateRocksDBStorageFactory(g_sdsArgs, cserver.dbnum, cserver.storage_conf, cserver.storage_conf ? strlen(cserver.storage_conf) : 0);
#else
serverLog(LL_WARNING, "To use the flash storage provider please compile KeyDB with ENABLE_FLASH=yes");
serverLog(LL_WARNING, "Exiting due to the use of an unsupported storage provider");
exit(EXIT_FAILURE);
#endif
}
}
else if (!strcasecmp(g_sdsProvider, "test") && g_sdsArgs == nullptr)
{
g_pserver->m_pstorageFactory = new (MALLOC_LOCAL) TestStorageFactory();
@@ -767,12 +766,6 @@ void loadServerConfigFromString(char *config) {
g_sdsProvider = sdsdup(argv[1]);
if (argc > 2)
g_sdsArgs = sdsdup(argv[2]);
} else if (!strcasecmp(argv[0],"is-flash-enabled") && argc == 1) {
#ifdef ENABLE_ROCKSDB
exit(EXIT_SUCCESS);
#else
exit(EXIT_FAILURE);
#endif
} else {
err = "Bad directive or wrong number of arguments"; goto loaderr;
}
@@ -2587,7 +2580,7 @@ static int updateMaxclients(long long val, long long prev, const char **err) {
adjustOpenFilesLimit();
if (g_pserver->maxclients != val) {
static char msg[128];
snprintf(msg, 128, "The operating system is not able to handle the specified number of clients, try with %d", g_pserver->maxclients);
sprintf(msg, "The operating system is not able to handle the specified number of clients, try with %d", g_pserver->maxclients);
*err = msg;
if (g_pserver->maxclients > prev) {
g_pserver->maxclients = prev;
@@ -2626,7 +2619,7 @@ static int updateMaxclients(long long val, long long prev, const char **err) {
if (res != AE_OK){
static char msg[128];
snprintf(msg, 128,"Failed to post the request to change setsize for Thread %d", iel);
sprintf(msg, "Failed to post the request to change setsize for Thread %d", iel);
*err = msg;
return 0;
}
@@ -2909,7 +2902,6 @@ standardConfig configs[] = {
createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL),
createBoolConfig("allow-write-during-load", NULL, MODIFIABLE_CONFIG, g_pserver->fWriteDuringActiveLoad, 0, NULL, NULL),
createBoolConfig("force-backlog-disk-reserve", NULL, MODIFIABLE_CONFIG, cserver.force_backlog_disk, 0, NULL, NULL),
createBoolConfig("soft-shutdown", NULL, MODIFIABLE_CONFIG, g_pserver->config_soft_shutdown, 0, NULL, NULL),
#ifdef USE_OPENSSL
createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */
-5
View File
@@ -1462,11 +1462,6 @@ void shutdownCommand(client *c) {
flags |= SHUTDOWN_NOSAVE;
} else if (!strcasecmp(szFromObj(c->argv[1]),"save")) {
flags |= SHUTDOWN_SAVE;
} else if (!strcasecmp(szFromObj(c->argv[1]), "soft")) {
g_pserver->soft_shutdown = true;
serverLog(LL_WARNING, "Soft Shutdown Initiated");
addReply(c, shared.ok);
return;
} else {
addReplyErrorObject(c,shared.syntaxerr);
return;
+1 -1
View File
@@ -1577,7 +1577,7 @@ char *stringFromLongLong(long long value) {
int len;
char *s;
len = snprintf(buf,32,"%lld",value);
len = sprintf(buf,"%lld",value);
s = zmalloc(len+1);
memcpy(s, buf, len);
s[len] = '\0';
+3 -3
View File
@@ -112,15 +112,15 @@ int endianconvTest(int argc, char *argv[], int accurate) {
UNUSED(argv);
UNUSED(accurate);
snprintf(buf,32,"ciaoroma");
sprintf(buf,"ciaoroma");
memrev16(buf);
printf("%s\n", buf);
snprintf(buf,32,"ciaoroma");
sprintf(buf,"ciaoroma");
memrev32(buf);
printf("%s\n", buf);
snprintf(buf,32,"ciaoroma");
sprintf(buf,"ciaoroma");
memrev64(buf);
printf("%s\n", buf);
+1 -1
View File
@@ -1071,7 +1071,7 @@ struct commandHelp {
1,
"2.2.0" },
{ "SHUTDOWN",
"[NOSAVE|SAVE|SOFT]",
"[NOSAVE|SAVE]",
"Synchronously save the dataset to disk and then shut down the server",
9,
"1.0.0" },
+1 -1
View File
@@ -904,7 +904,7 @@ int main(int argc, const char **argv) {
while (self_threads < config.max_threads) {
for (int i = 0; i < config.numclients; i++) {
snprintf(command, 63, "SET %d %s\r\n", self_threads * config.numclients + i, set_value);
sprintf(command, "SET %d %s\r\n", self_threads * config.numclients + i, set_value);
createClient(command, strlen(command), NULL,self_threads);
}
+3 -17
View File
@@ -145,7 +145,7 @@ client *createClient(connection *conn, int iel) {
client_id = g_pserver->next_client_id.fetch_add(1);
c->iel = iel;
c->id = client_id;
snprintf(c->lock.szName, 56, "client %" PRIu64, client_id);
sprintf(c->lock.szName, "client %" PRIu64, client_id);
c->resp = 2;
c->conn = conn;
c->name = NULL;
@@ -777,11 +777,11 @@ void setDeferredAggregateLen(client *c, void *node, long length, char prefix) {
* we return NULL in addReplyDeferredLen() */
if (node == NULL) return;
char lenstr[128];
size_t lenstr_len = snprintf(lenstr, 128, "%c%ld\r\n", prefix, length);
size_t lenstr_len = sprintf(lenstr, "%c%ld\r\n", prefix, length);
setDeferredReply(c, node, lenstr, lenstr_len);
} else {
char lenstr[128];
int lenstr_len = snprintf(lenstr, 128, "%c%ld\r\n", prefix, length);
int lenstr_len = sprintf(lenstr, "%c%ld\r\n", prefix, length);
size_t idxSplice = (size_t)node;
serverAssert(idxSplice <= c->replyAsync->used);
@@ -1256,20 +1256,6 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip, int iel)
return;
}
/* Prevent new connections if we're in a soft shutdown situation */
if (g_pserver->soft_shutdown) {
const char *err = "-SHUTDOWN";
/* That's a best effort error message, don't check write errors.
* Note that for TLS connections, no handshake was done yet so nothing
* is written and the connection will just drop. */
if (connWrite(conn,err,strlen(err)) == -1) {
/* Nothing to do, Just to avoid the warning... */
}
g_pserver->stat_rejected_conn++;
connClose(conn);
return;
}
/* Limit the number of connections we take at the same time.
*
* Admission control will happen before a client is created and connAccept()
+2 -1
View File
@@ -1142,7 +1142,8 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
mem_total+=mem;
std::unique_lock<fastlock> ul(g_expireLock);
mem = db->setexpire()->estimated_bytes_used();
mem = db->setexpire()->bytes_used();
mh->db[mh->num_dbs].overhead_ht_expires = mem;
mem_total+=mem;
+12 -9
View File
@@ -3115,12 +3115,13 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
std::unique_ptr<rdbInsertJob> spjob;
// If we're tracking changes we need to reset this
std::vector<bool> fTracking(cserver.dbnum);
// We don't want to track here because processChangesAsync is outside the normal scope handling
for (int idb = 0; idb < cserver.dbnum; ++idb) {
if ((fTracking[idb] = g_pserver->db[idb]->FTrackingChanges()))
bool fTracking = g_pserver->db[0]->FTrackingChanges();
if (fTracking) {
// We don't want to track here because processChangesAsync is outside the normal scope handling
for (int idb = 0; idb < cserver.dbnum; ++idb) {
if (g_pserver->db[idb]->processChanges(false))
g_pserver->db[idb]->commitChanges();
}
}
rdb->update_cksum = rdbLoadProgressCallback;
@@ -3488,10 +3489,11 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
}
wqueue.endWork();
// Reset track changes
for (int idb = 0; idb < cserver.dbnum; ++idb) {
if (fTracking[idb])
if (fTracking) {
// Reset track changes
for (int idb = 0; idb < cserver.dbnum; ++idb) {
g_pserver->db[idb]->trackChanges(false);
}
}
if (empty_keys_skipped) {
serverLog(LL_WARNING,
@@ -3509,10 +3511,11 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
* the RDB file from a socket during initial SYNC (diskless replica mode),
* we'll report the error to the caller, so that we can retry. */
eoferr:
if (fTracking) {
// Reset track changes
for (int idb = 0; idb < cserver.dbnum; ++idb) {
if (fTracking[idb])
for (int idb = 0; idb < cserver.dbnum; ++idb) {
g_pserver->db[idb]->trackChanges(false);
}
}
wqueue.endWork();
+10 -10
View File
@@ -6835,17 +6835,17 @@ void bytesToHuman(char *s, long long n) {
}
if (n < 1024) {
/* Bytes */
snprintf(s,256,"%lldB",n);
sprintf(s,"%lldB",n);
return;
} else if (n < (1024*1024)) {
d = (double)n/(1024);
snprintf(s,256,"%.2fK",d);
sprintf(s,"%.2fK",d);
} else if (n < (1024LL*1024*1024)) {
d = (double)n/(1024*1024);
snprintf(s,256,"%.2fM",d);
sprintf(s,"%.2fM",d);
} else if (n < (1024LL*1024*1024*1024)) {
d = (double)n/(1024LL*1024*1024);
snprintf(s,256,"%.2fG",d);
sprintf(s,"%.2fG",d);
}
}
@@ -6875,12 +6875,12 @@ static void statMode(void) {
for (j = 0; j < 20; j++) {
long k;
snprintf(buf,64,"db%d:keys",j);
sprintf(buf,"db%d:keys",j);
k = getLongInfoField(reply->str,buf);
if (k == LONG_MIN) continue;
aux += k;
}
snprintf(buf,64,"%ld",aux);
sprintf(buf,"%ld",aux);
printf("%-11s",buf);
/* Used memory */
@@ -6890,23 +6890,23 @@ static void statMode(void) {
/* Clients */
aux = getLongInfoField(reply->str,"connected_clients");
snprintf(buf,64,"%ld",aux);
sprintf(buf,"%ld",aux);
printf(" %-8s",buf);
/* Blocked (BLPOPPING) Clients */
aux = getLongInfoField(reply->str,"blocked_clients");
snprintf(buf,64,"%ld",aux);
sprintf(buf,"%ld",aux);
printf("%-8s",buf);
/* Requests */
aux = getLongInfoField(reply->str,"total_commands_processed");
snprintf(buf,64,"%ld (+%ld)",aux,requests == 0 ? 0 : aux-requests);
sprintf(buf,"%ld (+%ld)",aux,requests == 0 ? 0 : aux-requests);
printf("%-19s",buf);
requests = aux;
/* Connections */
aux = getLongInfoField(reply->str,"total_connections_received");
snprintf(buf,64,"%ld",aux);
sprintf(buf,"%ld",aux);
printf(" %-12s",buf);
/* Children */
+3
View File
@@ -97,6 +97,9 @@ extern "C" {
#define CC_FORCE (1<<0) /* Re-connect if already connected. */
#define CC_QUIET (1<<1) /* Don't log connecting errors. */
struct clusterManagerLink;
typedef struct clusterManagerLink clusterManagerLink;
/* Dict Helpers */
uint64_t dictSdsHash(const void *key);
+6 -19
View File
@@ -1142,7 +1142,7 @@ public:
}
};
int rdbSaveSnapshotForReplication(rdbSaveInfo *rsi) {
int rdbSaveSnapshotForReplication(struct rdbSaveInfo *rsi) {
// TODO: This needs to be on a background thread
int retval = C_OK;
serverAssert(GlobalLocksAcquired());
@@ -2989,11 +2989,10 @@ error:
}
void readSyncBulkPayload(connection *conn) {
serverAssert(GlobalLocksAcquired());
rdbSaveInfo rsi;
redisMaster *mi = (redisMaster*)connGetPrivateData(conn);
static int usemark = 0;
if (mi == nullptr || conn != mi->repl_transfer_s) {
if (mi == nullptr) {
// We're about to be free'd so bail out
return;
}
@@ -3006,9 +3005,6 @@ void readSyncBulkPayload(connection *conn) {
return;
}
if (conn != mi->repl_transfer_s)
return;
/* Final setup of the connected slave <- master link */
replicationCreateMasterClient(mi,mi->repl_transfer_s,rsi.repl_stream_db);
if (mi->isRocksdbSnapshotRepl) {
@@ -3817,14 +3813,12 @@ int connectWithMaster(redisMaster *mi) {
* Never call this function directly, use cancelReplicationHandshake() instead.
*/
void undoConnectWithMaster(redisMaster *mi) {
serverAssert(GlobalLocksAcquired());
auto conn = mi->repl_transfer_s;
connSetPrivateData(conn, nullptr);
mi->repl_transfer_s = NULL;
int result = aePostFunction(g_pserver->rgthreadvar[mi->ielReplTransfer].el, [conn]{
aePostFunction(g_pserver->rgthreadvar[mi->ielReplTransfer].el, [conn]{
connClose(conn);
}, false);
serverAssert(result == AE_OK);
});
mi->repl_transfer_s = NULL;
}
/* Abort the async download of the bulk dataset while SYNC-ing with master.
@@ -3979,10 +3973,6 @@ void freeMasterInfo(redisMaster *mi)
{
sdsfree(mi->masterauth);
zfree(mi->masteruser);
if (g_pserver->rdb_filename != nullptr && g_pserver->rdb_filename == mi->repl_transfer_tmpfile) {
unlink(g_pserver->rdb_filename);
g_pserver->rdb_filename = nullptr;
}
if (mi->repl_transfer_tmpfile)
zfree(mi->repl_transfer_tmpfile);
delete mi->staleKeyMap;
@@ -5652,10 +5642,7 @@ void flushReplBacklogToClients()
replica->repl_end_off = g_pserver->master_repl_offset;
/* Only if the there isn't already a pending write do we prepare the client to write */
if (replica->repl_curr_off == g_pserver->master_repl_offset) {
serverLog(LL_DEBUG, "Pending write when it's on repl_offset=%lld", g_pserver->master_repl_offset);
continue;
}
serverAssert(replica->repl_curr_off != g_pserver->master_repl_offset);
prepareClientToWrite(replica);
}
if (fAsyncWrite)
+7 -5
View File
@@ -240,12 +240,14 @@ public:
bool empty() const noexcept { return celem == 0; }
size_t size() const noexcept { return celem; }
size_t estimated_bytes_used() const
size_t bytes_used() const
{
// This estimate does't include all the overhead of the internal vectors
size_t cb = sizeof(this)
+ (m_data.capacity() * sizeof(m_data[0]))
+ sizeof(T) * size();
size_t cb = sizeof(this) + (m_data.capacity()-m_data.size())*sizeof(T);
for (auto &vec : m_data)
{
if (vec != nullptr)
cb += vec->bytes_used();
}
return cb;
}
+18 -43
View File
@@ -779,7 +779,7 @@ struct redisCommand redisCommandTable[] = {
0,NULL,0,0,0,0,0,0},
{"shutdown",shutdownCommand,-1,
"admin no-script ok-loading ok-stale noprop",
"admin no-script ok-loading ok-stale",
0,NULL,0,0,0,0,0,0},
{"lastsave",lastsaveCommand,1,
@@ -2648,26 +2648,6 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
}
}
if (g_pserver->soft_shutdown) {
/* Loop through our clients list and see if there are any active clients */
listIter li;
listNode *ln;
listRewind(g_pserver->clients, &li);
bool fActiveClient = false;
while ((ln = listNext(&li)) && !fActiveClient) {
client *c = (client*)listNodeValue(ln);
if (c->flags & CLIENT_IGNORE_SOFT_SHUTDOWN)
continue;
fActiveClient = true;
}
if (!fActiveClient) {
if (prepareForShutdown(SHUTDOWN_NOFLAGS) == C_OK) {
serverLog(LL_WARNING, "All active clients have disconnected while a soft shutdown is pending. Shutting down now.");
throw ShutdownException();
}
}
}
g_pserver->cronloops++;
return 1000/g_pserver->hz;
}
@@ -2795,8 +2775,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
AeLocker locker;
int iel = ielFromEventLoop(eventLoop);
tlsProcessPendingData();
locker.arm();
/* end any snapshots created by fast async commands */
@@ -2827,6 +2805,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
uint64_t processed = 0;
int aof_state = g_pserver->aof_state;
locker.disarm();
processed += tlsProcessPendingData();
processed += handleClientsWithPendingWrites(iel, aof_state);
locker.arm();
processed += freeClientsInAsyncFreeQueue(iel);
@@ -2837,6 +2816,13 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
/* Handle precise timeouts of blocked clients. */
handleBlockedClientsTimeout();
/* Handle TLS pending data. (must be done before flushAppendOnlyFile) */
if (tlsHasPendingData()) {
locker.disarm();
tlsProcessPendingData();
locker.arm();
}
/* If tls still has pending unread data don't sleep at all. */
aeSetDontWait(eventLoop, tlsHasPendingData());
@@ -3015,11 +3001,8 @@ void afterSleep(struct aeEventLoop *eventLoop) {
serverAssert(serverTL->gcEpoch.isReset());
serverTL->gcEpoch = g_pserver->garbageCollector.startEpoch();
aeAcquireLock();
for (int idb = 0; idb < cserver.dbnum; ++idb)
g_pserver->db[idb]->trackChanges(false);
aeReleaseLock();
serverTL->disable_async_commands = false;
}
@@ -5316,11 +5299,6 @@ void pingCommand(client *c) {
return;
}
if (g_pserver->soft_shutdown && !(c->flags & CLIENT_IGNORE_SOFT_SHUTDOWN)) {
addReplyError(c, "-SHUTDOWN PENDING");
return;
}
if (c->flags & CLIENT_PUBSUB && c->resp == 2) {
addReply(c,shared.mbulkhdr[2]);
addReplyBulkCBuffer(c,"pong",4);
@@ -5472,25 +5450,25 @@ void bytesToHuman(char *s, unsigned long long n) {
if (n < 1024) {
/* Bytes */
snprintf(s,256,"%lluB",n);
sprintf(s,"%lluB",n);
} else if (n < (1024*1024)) {
d = (double)n/(1024);
snprintf(s,256,"%.2fK",d);
sprintf(s,"%.2fK",d);
} else if (n < (1024LL*1024*1024)) {
d = (double)n/(1024*1024);
snprintf(s,256,"%.2fM",d);
sprintf(s,"%.2fM",d);
} else if (n < (1024LL*1024*1024*1024)) {
d = (double)n/(1024LL*1024*1024);
snprintf(s,256,"%.2fG",d);
sprintf(s,"%.2fG",d);
} else if (n < (1024LL*1024*1024*1024*1024)) {
d = (double)n/(1024LL*1024*1024*1024);
snprintf(s,256,"%.2fT",d);
sprintf(s,"%.2fT",d);
} else if (n < (1024LL*1024*1024*1024*1024*1024)) {
d = (double)n/(1024LL*1024*1024*1024*1024);
snprintf(s,256,"%.2fP",d);
sprintf(s,"%.2fP",d);
} else {
/* Let's hope we never need this */
snprintf(s,256,"%lluB",n);
sprintf(s,"%lluB",n);
}
}
@@ -6715,7 +6693,7 @@ static void sigShutdownHandler(int sig) {
* If we receive the signal the second time, we interpret this as
* the user really wanting to quit ASAP without waiting to persist
* on disk. */
if ((g_pserver->shutdown_asap || g_pserver->soft_shutdown) && sig == SIGINT) {
if (g_pserver->shutdown_asap && sig == SIGINT) {
serverLogFromHandler(LL_WARNING, "You insist... exiting now.");
rdbRemoveTempFile(g_pserver->rdbThreadVars.tmpfileNum, 1);
g_pserver->garbageCollector.shutdown();
@@ -6726,10 +6704,7 @@ static void sigShutdownHandler(int sig) {
}
serverLogFromHandler(LL_WARNING, msg);
if (g_pserver->config_soft_shutdown)
g_pserver->soft_shutdown = true;
else
g_pserver->shutdown_asap = 1;
g_pserver->shutdown_asap = 1;
}
void setupSignalHandlers(void) {
-10
View File
@@ -508,7 +508,6 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
#define CLIENT_PREVENT_AOF_PROP (1<<19) /* Don't propagate to AOF. */
#define CLIENT_PREVENT_REPL_PROP (1<<20) /* Don't propagate to slaves. */
#define CLIENT_PREVENT_PROP (CLIENT_PREVENT_AOF_PROP|CLIENT_PREVENT_REPL_PROP)
#define CLIENT_IGNORE_SOFT_SHUTDOWN (CLIENT_MASTER | CLIENT_SLAVE | CLIENT_BLOCKED | CLIENT_MONITOR)
#define CLIENT_PENDING_WRITE (1<<21) /* Client has output to send but a write
handler is yet not installed. */
#define CLIENT_REPLY_OFF (1<<22) /* Don't send replies to client. */
@@ -1909,12 +1908,6 @@ struct MasterSaveInfo {
masterhost = sdsstring(sdsdup(mi.masterhost));
masterport = mi.masterport;
}
MasterSaveInfo(const MasterSaveInfo &other) {
masterhost = other.masterhost;
masterport = other.masterport;
memcpy(master_replid, other.master_replid, sizeof(master_replid));
master_initial_offset = other.master_initial_offset;
}
MasterSaveInfo &operator=(const MasterSaveInfo &other) {
masterhost = other.masterhost;
@@ -2728,9 +2721,6 @@ struct redisServer {
long long repl_batch_offStart = -1;
long long repl_batch_idxStart = -1;
int config_soft_shutdown = false;
bool soft_shutdown = false;
/* Lock Contention Ring Buffer */
static const size_t s_lockContentionSamples = 64;
uint16_t rglockSamples[s_lockContentionSamples];
-234
View File
@@ -1,234 +0,0 @@
#include "rocksdb.h"
#include <string>
#include <sstream>
#include <mutex>
#include <unistd.h>
#include "rocksdbfactor_internal.h"
static const char keyprefix[] = INTERNAL_KEY_PREFIX;
rocksdb::Options DefaultRocksDBOptions();
extern "C" pid_t gettid();
bool FInternalKey(const char *key, size_t cch)
{
if (cch >= sizeof(INTERNAL_KEY_PREFIX))
{
if (memcmp(key, keyprefix, sizeof(INTERNAL_KEY_PREFIX)-1) == 0)
return true;
}
return false;
}
RocksDBStorageProvider::RocksDBStorageProvider(RocksDBStorageFactory *pfactory, std::shared_ptr<rocksdb::DB> &spdb, std::shared_ptr<rocksdb::ColumnFamilyHandle> &spcolfam, const rocksdb::Snapshot *psnapshot, size_t count)
: m_pfactory(pfactory), m_spdb(spdb), m_psnapshot(psnapshot), m_spcolfamily(spcolfam), m_count(count)
{
m_readOptionsTemplate = rocksdb::ReadOptions();
m_readOptionsTemplate.verify_checksums = false;
m_readOptionsTemplate.snapshot = m_psnapshot;
}
void RocksDBStorageProvider::insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwrite)
{
rocksdb::Status status;
std::unique_lock<fastlock> l(m_lock);
if (m_spbatch != nullptr)
status = m_spbatch->Put(m_spcolfamily.get(), rocksdb::Slice(key, cchKey), rocksdb::Slice((const char*)data, cb));
else
status = m_spdb->Put(WriteOptions(), m_spcolfamily.get(), rocksdb::Slice(key, cchKey), rocksdb::Slice((const char*)data, cb));
if (!status.ok())
throw status.ToString();
if (!fOverwrite)
++m_count;
}
void RocksDBStorageProvider::bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem)
{
if (celem >= 16384) {
rocksdb::Options options = DefaultRocksDBOptions();
rocksdb::SstFileWriter sst_file_writer(rocksdb::EnvOptions(), options, options.comparator);
std::string file_path = m_pfactory->getTempFolder() + "/tmpIngest.";
file_path += std::to_string(gettid());
file_path += ".sst";
rocksdb::Status s = sst_file_writer.Open(file_path);
if (!s.ok())
goto LFallback;
// Insert rows into the SST file, note that inserted keys must be
// strictly increasing (based on options.comparator)
for (size_t ielem = 0; ielem < celem; ++ielem) {
s = sst_file_writer.Put(rocksdb::Slice(rgkeys[ielem], rgcbkeys[ielem]), rocksdb::Slice(rgvals[ielem], rgcbvals[ielem]));
if (!s.ok()) {
unlink(file_path.c_str());
goto LFallback;
}
}
s = sst_file_writer.Finish();
if (!s.ok()) {
unlink(file_path.c_str());
goto LFallback;
}
auto ingestOptions = rocksdb::IngestExternalFileOptions();
ingestOptions.move_files = true;
ingestOptions.write_global_seqno = false;
ingestOptions.failed_move_fall_back_to_copy = false;
// Ingest the external SST file into the DB
s = m_spdb->IngestExternalFile(m_spcolfamily.get(), {file_path}, ingestOptions);
if (!s.ok()) {
unlink(file_path.c_str());
goto LFallback;
}
} else {
LFallback:
auto spbatch = std::make_unique<rocksdb::WriteBatch>();
for (size_t ielem = 0; ielem < celem; ++ielem) {
spbatch->Put(m_spcolfamily.get(), rocksdb::Slice(rgkeys[ielem], rgcbkeys[ielem]), rocksdb::Slice(rgvals[ielem], rgcbvals[ielem]));
}
m_spdb->Write(WriteOptions(), spbatch.get());
}
std::unique_lock<fastlock> l(m_lock);
m_count += celem;
}
bool RocksDBStorageProvider::erase(const char *key, size_t cchKey)
{
rocksdb::Status status;
std::unique_lock<fastlock> l(m_lock);
if (m_spbatch != nullptr)
{
status = m_spbatch->Delete(m_spcolfamily.get(), rocksdb::Slice(key, cchKey));
}
else
{
if (!FKeyExists(key, cchKey))
return false;
status = m_spdb->Delete(WriteOptions(), m_spcolfamily.get(), rocksdb::Slice(key, cchKey));
}
if (status.ok())
--m_count;
return status.ok();
}
void RocksDBStorageProvider::retrieve(const char *key, size_t cchKey, callbackSingle fn) const
{
rocksdb::PinnableSlice slice;
auto status = m_spdb->Get(ReadOptions(), m_spcolfamily.get(), rocksdb::Slice(key, cchKey), &slice);
if (status.ok())
fn(key, cchKey, slice.data(), slice.size());
}
size_t RocksDBStorageProvider::clear()
{
size_t celem = count();
auto status = m_spdb->DropColumnFamily(m_spcolfamily.get());
auto strName = m_spcolfamily->GetName();
rocksdb::ColumnFamilyHandle *handle = nullptr;
rocksdb::ColumnFamilyOptions cf_options(m_pfactory->RocksDbOptions());
m_spdb->CreateColumnFamily(cf_options, strName, &handle);
m_spcolfamily = std::shared_ptr<rocksdb::ColumnFamilyHandle>(handle);
if (!status.ok())
throw status.ToString();
m_count = 0;
return celem;
}
size_t RocksDBStorageProvider::count() const
{
std::unique_lock<fastlock> l(m_lock);
return m_count;
}
bool RocksDBStorageProvider::enumerate(callback fn) const
{
std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(ReadOptions(), m_spcolfamily.get()));
size_t count = 0;
for (it->SeekToFirst(); it->Valid(); it->Next()) {
if (FInternalKey(it->key().data(), it->key().size()))
continue;
++count;
bool fContinue = fn(it->key().data(), it->key().size(), it->value().data(), it->value().size());
if (!fContinue)
break;
}
if (!it->Valid() && count != m_count)
{
if (const_cast<RocksDBStorageProvider*>(this)->m_count != count)
printf("WARNING: rocksdb count mismatch");
const_cast<RocksDBStorageProvider*>(this)->m_count = count;
}
assert(it->status().ok()); // Check for any errors found during the scan
return !it->Valid();
}
const IStorage *RocksDBStorageProvider::clone() const
{
std::unique_lock<fastlock> l(m_lock);
const rocksdb::Snapshot *psnapshot = const_cast<RocksDBStorageProvider*>(this)->m_spdb->GetSnapshot();
return new RocksDBStorageProvider(m_pfactory, const_cast<RocksDBStorageProvider*>(this)->m_spdb, const_cast<RocksDBStorageProvider*>(this)->m_spcolfamily, psnapshot, m_count);
}
RocksDBStorageProvider::~RocksDBStorageProvider()
{
if (m_spbatch != nullptr)
endWriteBatch();
if (m_spdb != nullptr && m_psnapshot == nullptr)
{
insert(count_key, sizeof(count_key), &m_count, sizeof(m_count), false);
flush();
}
if (m_spdb != nullptr)
{
if (m_psnapshot != nullptr)
m_spdb->ReleaseSnapshot(m_psnapshot);
}
}
rocksdb::WriteOptions RocksDBStorageProvider::WriteOptions() const
{
auto opt = rocksdb::WriteOptions();
return opt;
}
void RocksDBStorageProvider::beginWriteBatch()
{
m_lock.lock();
m_spbatch = std::make_unique<rocksdb::WriteBatch>();
}
void RocksDBStorageProvider::endWriteBatch()
{
m_spdb->Write(WriteOptions(), m_spbatch.get());
m_spbatch = nullptr;
m_lock.unlock();
}
void RocksDBStorageProvider::batch_lock()
{
m_lock.lock();
}
void RocksDBStorageProvider::batch_unlock()
{
m_lock.unlock();
}
void RocksDBStorageProvider::flush()
{
m_spdb->SyncWAL();
}
bool RocksDBStorageProvider::FKeyExists(const char *key, size_t cch) const
{
rocksdb::PinnableSlice slice;
return m_spdb->Get(ReadOptions(), m_spcolfamily.get(), rocksdb::Slice(key, cch), &slice).ok();
}
-56
View File
@@ -1,56 +0,0 @@
#pragma once
#include <memory>
#include "../IStorage.h"
#include <rocksdb/db.h>
#include "../fastlock.h"
#define INTERNAL_KEY_PREFIX "\x00\x04\x03\x00\x05\x02\x04"
static const char count_key[] = INTERNAL_KEY_PREFIX "__keydb__count\1";
static const char version_key[] = INTERNAL_KEY_PREFIX "__keydb__version\1";
static const char meta_key[] = INTERNAL_KEY_PREFIX "__keydb__metadata\1";
class RocksDBStorageFactory;
class RocksDBStorageProvider : public IStorage
{
RocksDBStorageFactory *m_pfactory;
std::shared_ptr<rocksdb::DB> m_spdb; // Note: This must be first so it is deleted last
std::unique_ptr<rocksdb::WriteBatch> m_spbatch;
const rocksdb::Snapshot *m_psnapshot = nullptr;
std::shared_ptr<rocksdb::ColumnFamilyHandle> m_spcolfamily;
rocksdb::ReadOptions m_readOptionsTemplate;
size_t m_count = 0;
mutable fastlock m_lock {"RocksDBStorageProvider"};
public:
RocksDBStorageProvider(RocksDBStorageFactory *pfactory, std::shared_ptr<rocksdb::DB> &spdb, std::shared_ptr<rocksdb::ColumnFamilyHandle> &spcolfam, const rocksdb::Snapshot *psnapshot, size_t count);
~RocksDBStorageProvider();
virtual void insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwrite) override;
virtual bool erase(const char *key, size_t cchKey) override;
virtual void retrieve(const char *key, size_t cchKey, callbackSingle fn) const override;
virtual size_t clear() override;
virtual bool enumerate(callback fn) const override;
virtual const IStorage *clone() const override;
virtual void beginWriteBatch() override;
virtual void endWriteBatch() override;
virtual void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem) override;
virtual void batch_lock() override;
virtual void batch_unlock() override;
virtual void flush() override;
size_t count() const override;
protected:
bool FKeyExists(const char *key, size_t cchKey) const;
const rocksdb::ReadOptions &ReadOptions() const { return m_readOptionsTemplate; }
rocksdb::WriteOptions WriteOptions() const;
};
bool FInternalKey(const char *key, size_t cch);
-204
View File
@@ -1,204 +0,0 @@
#include "rocksdb.h"
#include "../version.h"
#include <rocksdb/filter_policy.h>
#include <rocksdb/table.h>
#include <rocksdb/utilities/options_util.h>
#include <rocksdb/sst_file_manager.h>
#include <rocksdb/utilities/convenience.h>
#include <rocksdb/slice_transform.h>
#include "rocksdbfactor_internal.h"
#include <sys/types.h>
#include <sys/stat.h>
rocksdb::Options DefaultRocksDBOptions() {
rocksdb::Options options;
options.max_background_compactions = 4;
options.max_background_flushes = 2;
options.bytes_per_sync = 1048576;
options.compaction_pri = rocksdb::kMinOverlappingRatio;
options.compression = rocksdb::kNoCompression;
options.enable_pipelined_write = true;
options.allow_mmap_reads = true;
options.avoid_unnecessary_blocking_io = true;
options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(0));
rocksdb::BlockBasedTableOptions table_options;
table_options.block_size = 16 * 1024;
table_options.cache_index_and_filter_blocks = true;
table_options.pin_l0_filter_and_index_blocks_in_cache = true;
table_options.data_block_index_type = rocksdb::BlockBasedTableOptions::kDataBlockBinaryAndHash;
table_options.checksum = rocksdb::kNoChecksum;
table_options.format_version = 4;
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
return options;
}
IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const char *rgchConfig, size_t cchConfig)
{
return new RocksDBStorageFactory(path, dbnum, rgchConfig, cchConfig);
}
rocksdb::Options RocksDBStorageFactory::RocksDbOptions()
{
rocksdb::Options options = DefaultRocksDBOptions();
options.max_open_files = filedsRequired();
options.sst_file_manager = m_pfilemanager;
options.create_if_missing = true;
options.create_missing_column_families = true;
options.info_log_level = rocksdb::ERROR_LEVEL;
options.max_total_wal_size = 1 * 1024 * 1024 * 1024;
return options;
}
RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig)
: m_path(dbfile)
{
dbnum++; // create an extra db for metadata
// Get the count of column families in the actual database
std::vector<std::string> vecT;
auto status = rocksdb::DB::ListColumnFamilies(rocksdb::Options(), dbfile, &vecT);
// RocksDB requires we know the count of col families before opening, if the user only wants to see less
// we still have to make room for all column family handles regardless
if (status.ok() && (int)vecT.size() > dbnum)
dbnum = (int)vecT.size();
std::vector<rocksdb::ColumnFamilyDescriptor> veccoldesc;
veccoldesc.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions())); // ignore default col family
m_pfilemanager = std::shared_ptr<rocksdb::SstFileManager>(rocksdb::NewSstFileManager(rocksdb::Env::Default()));
rocksdb::DB *db = nullptr;
auto options = RocksDbOptions();
for (int idb = 0; idb < dbnum; ++idb)
{
rocksdb::ColumnFamilyOptions cf_options(options);
cf_options.level_compaction_dynamic_level_bytes = true;
veccoldesc.push_back(rocksdb::ColumnFamilyDescriptor(std::to_string(idb), cf_options));
}
if (rgchConfig != nullptr)
{
std::string options_string(rgchConfig, cchConfig);
rocksdb::Status status;
if (!(status = rocksdb::GetDBOptionsFromString(options, options_string, &options)).ok())
{
fprintf(stderr, "Failed to parse FLASH options: %s\r\n", status.ToString().c_str());
exit(EXIT_FAILURE);
}
}
std::vector<rocksdb::ColumnFamilyHandle*> handles;
status = rocksdb::DB::Open(options, dbfile, veccoldesc, &handles, &db);
if (!status.ok())
throw status.ToString();
m_spdb = std::shared_ptr<rocksdb::DB>(db);
for (auto handle : handles)
{
std::string strVersion;
auto status = m_spdb->Get(rocksdb::ReadOptions(), handle, rocksdb::Slice(version_key, sizeof(version_key)), &strVersion);
if (!status.ok())
{
setVersion(handle);
}
else
{
SymVer ver = parseVersion(strVersion.c_str());
auto cmp = compareVersion(&ver);
if (cmp == NewerVersion)
throw "Cannot load FLASH database created by newer version of KeyDB";
if (cmp == OlderVersion)
setVersion(handle);
}
m_vecspcols.emplace_back(handle);
}
}
RocksDBStorageFactory::~RocksDBStorageFactory()
{
m_spdb->SyncWAL();
}
void RocksDBStorageFactory::setVersion(rocksdb::ColumnFamilyHandle *handle)
{
auto status = m_spdb->Put(rocksdb::WriteOptions(), handle, rocksdb::Slice(version_key, sizeof(version_key)), rocksdb::Slice(KEYDB_REAL_VERSION, strlen(KEYDB_REAL_VERSION)+1));
if (!status.ok())
throw status.ToString();
}
size_t RocksDBStorageFactory::filedsRequired() const {
return 256;
}
std::string RocksDBStorageFactory::getTempFolder()
{
auto path = m_path + "/keydb_tmp/";
if (!m_fCreatedTempFolder) {
if (!mkdir(path.c_str(), 0700))
m_fCreatedTempFolder = true;
}
return path;
}
IStorage *RocksDBStorageFactory::createMetadataDb()
{
IStorage *metadataDb = this->create(-1, nullptr, nullptr);
metadataDb->insert(meta_key, sizeof(meta_key), (void*)METADATA_DB_IDENTIFIER, strlen(METADATA_DB_IDENTIFIER), false);
return metadataDb;
}
IStorage *RocksDBStorageFactory::create(int db, key_load_iterator iter, void *privdata)
{
++db; // skip default col family
std::shared_ptr<rocksdb::ColumnFamilyHandle> spcolfamily(m_vecspcols[db].release());
size_t count = 0;
bool fUnclean = false;
std::string value;
auto status = m_spdb->Get(rocksdb::ReadOptions(), spcolfamily.get(), rocksdb::Slice(count_key, sizeof(count_key)), &value);
if (status.ok() && value.size() == sizeof(size_t))
{
count = *reinterpret_cast<const size_t*>(value.data());
m_spdb->Delete(rocksdb::WriteOptions(), spcolfamily.get(), rocksdb::Slice(count_key, sizeof(count_key)));
}
else
{
fUnclean = true;
}
if (fUnclean || iter != nullptr)
{
count = 0;
auto opts = rocksdb::ReadOptions();
opts.tailing = true;
std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(opts, spcolfamily.get()));
it->SeekToFirst();
bool fFirstRealKey = true;
for (;it->Valid(); it->Next()) {
if (FInternalKey(it->key().data(), it->key().size()))
continue;
if (fUnclean && it->Valid() && fFirstRealKey)
printf("\tDatabase %d was not shutdown cleanly, recomputing metrics\n", db);
fFirstRealKey = false;
if (iter != nullptr)
iter(it->key().data(), it->key().size(), privdata);
++count;
}
}
return new RocksDBStorageProvider(this, m_spdb, spcolfamily, nullptr, count);
}
const char *RocksDBStorageFactory::name() const
{
return "flash";
}
size_t RocksDBStorageFactory::totalDiskspaceUsed() const
{
return m_pfilemanager->GetTotalSize();
}
+2 -4
View File
@@ -26,12 +26,10 @@ TestStorageProvider::~TestStorageProvider()
{
}
void TestStorageProvider::insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwrite)
void TestStorageProvider::insert(const char *key, size_t cchKey, void *data, size_t cb, bool /*fOverwrite*/)
{
auto strkey = std::string(key, cchKey);
bool fActuallyExists = m_map.find(strkey) != m_map.end();
serverAssert(fActuallyExists == fOverwrite);
m_map.insert(std::make_pair(strkey, std::string((char*)data, cb)));
m_map[strkey] = std::string((char*)data, cb);
}
+1 -1
View File
@@ -263,7 +263,7 @@ sds writeJsonValue(sds output, const char *valIn, size_t cchIn) {
serverAssert(!FSimpleJsonEscapeCh(valIn[ich]));
if (FExtendedJsonEscapeCh(valIn[ich])) {
dst[ichDst++] = '\\'; dst[ichDst++] = 'u';
snprintf(dst + ichDst, cchIn+cchEscapeExtra-ichDst, "%4x", valIn[ich]);
sprintf(dst + ichDst, "%4x", valIn[ich]);
ichDst += 4;
} else {
dst[ichDst++] = valIn[ich];
+6 -54
View File
@@ -46,9 +46,6 @@
#include <sys/stat.h>
#include <arpa/inet.h>
#if OPENSSL_VERSION_NUMBER >= 0x30000000L
#include <openssl/decoder.h>
#endif
#define REDIS_TLS_PROTO_TLSv1 (1<<0)
#define REDIS_TLS_PROTO_TLSv1_1 (1<<1)
@@ -157,13 +154,14 @@ void tlsInit(void) {
*/
#if OPENSSL_VERSION_NUMBER < 0x10100000L
OPENSSL_config(NULL);
SSL_load_error_strings();
SSL_library_init();
#elif OPENSSL_VERSION_NUMBER < 0x10101000L
OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL);
#else
OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG|OPENSSL_INIT_ATFORK, NULL);
#endif
ERR_load_crypto_strings();
SSL_load_error_strings();
SSL_library_init();
#ifdef USE_CRYPTO_LOCKS
initCryptoLocks();
@@ -366,46 +364,20 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
if (ctx_config->prefer_server_ciphers)
SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE);
#if ((OPENSSL_VERSION_NUMBER < 0x30000000L) && defined(SSL_CTX_set_ecdh_auto))
#if defined(SSL_CTX_set_ecdh_auto)
SSL_CTX_set_ecdh_auto(ctx, 1);
#endif
SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE);
if (ctx_config->dh_params_file) {
FILE *dhfile = fopen(ctx_config->dh_params_file, "r");
DH *dh = NULL;
if (!dhfile) {
serverLog(LL_WARNING, "Failed to load %s: %s", ctx_config->dh_params_file, strerror(errno));
goto error;
}
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
EVP_PKEY *pkey = NULL;
OSSL_DECODER_CTX *dctx = OSSL_DECODER_CTX_new_for_pkey(
&pkey, "PEM", NULL, "DH", OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS, NULL, NULL);
if (!dctx) {
serverLog(LL_WARNING, "No decoder for DH params.");
fclose(dhfile);
goto error;
}
if (!OSSL_DECODER_from_fp(dctx, dhfile)) {
serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file);
OSSL_DECODER_CTX_free(dctx);
fclose(dhfile);
goto error;
}
OSSL_DECODER_CTX_free(dctx);
fclose(dhfile);
if (SSL_CTX_set0_tmp_dh_pkey(ctx, pkey) <= 0) {
ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
serverLog(LL_WARNING, "Failed to load DH params file: %s: %s", ctx_config->dh_params_file, errbuf);
EVP_PKEY_free(pkey);
goto error;
}
/* Not freeing pkey, it is owned by OpenSSL now */
#else
DH *dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL);
dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL);
fclose(dhfile);
if (!dh) {
serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file);
@@ -420,11 +392,6 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
}
DH_free(dh);
#endif
} else {
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
SSL_CTX_set_dh_auto(ctx, 1);
#endif
}
/* If a client-side certificate is configured, create an explicit client context */
@@ -536,26 +503,11 @@ bool tlsCheckAgainstAllowlist(const char * client){
#define ASN1_STRING_get0_data ASN1_STRING_data
#endif
class TCleanup {
std::function<void()> fn;
public:
TCleanup(std::function<void()> fn)
: fn(fn)
{}
~TCleanup() {
fn();
}
};
bool tlsValidateCertificateName(tls_connection* conn){
if (g_pserver->tls_allowlist.empty())
return true; // Empty list implies acceptance of all
X509 * cert = SSL_get_peer_certificate(conn->ssl);
TCleanup certClen([cert]{X509_free(cert);});
/* Check the common name (CN) of the certificate first */
X509_NAME_ENTRY * ne = X509_NAME_get_entry(X509_get_subject_name(cert), X509_NAME_get_index_by_NID(X509_get_subject_name(cert), NID_commonName, -1));
const char * commonName = reinterpret_cast<const char*>(ASN1_STRING_get0_data(X509_NAME_ENTRY_get_data(ne)));
+11 -11
View File
@@ -1704,17 +1704,17 @@ static unsigned char *createIntList() {
unsigned char *zl = ziplistNew();
char buf[32];
snprintf(buf, 32, "100");
sprintf(buf, "100");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
snprintf(buf, 32, "128000");
sprintf(buf, "128000");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
snprintf(buf, 32, "-100");
sprintf(buf, "-100");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_HEAD);
snprintf(buf, 32, "4294967296");
sprintf(buf, "4294967296");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_HEAD);
snprintf(buf, 32, "non integer");
sprintf(buf, "non integer");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
snprintf(buf, 32, "much much longer non integer");
sprintf(buf, "much much longer non integer");
zl = ziplistPush(zl, (unsigned char*)buf, strlen(buf), ZIPLIST_TAIL);
return zl;
}
@@ -2228,7 +2228,7 @@ int ziplistTest(int argc, char **argv, int accurate) {
char buf[32];
int i,len;
for (i = 0; i < 1000; i++) {
len = snprintf(buf,32,"%d",i);
len = sprintf(buf,"%d",i);
zl = ziplistPush(zl,(unsigned char*)buf,len,ZIPLIST_TAIL);
}
for (i = 0; i < 1000; i++) {
@@ -2375,13 +2375,13 @@ int ziplistTest(int argc, char **argv, int accurate) {
} else {
switch(rand() % 3) {
case 0:
buflen = snprintf(buf,1024,"%lld",(0LL + rand()) >> 20);
buflen = sprintf(buf,"%lld",(0LL + rand()) >> 20);
break;
case 1:
buflen = snprintf(buf,1024,"%lld",(0LL + rand()));
buflen = sprintf(buf,"%lld",(0LL + rand()));
break;
case 2:
buflen = snprintf(buf,1024,"%lld",(0LL + rand()) << 20);
buflen = sprintf(buf,"%lld",(0LL + rand()) << 20);
break;
default:
assert(NULL);
@@ -2410,7 +2410,7 @@ int ziplistTest(int argc, char **argv, int accurate) {
assert(ziplistGet(p,&sstr,&slen,&sval));
if (sstr == NULL) {
buflen = snprintf(buf,1024,"%lld",sval);
buflen = sprintf(buf,"%lld",sval);
} else {
buflen = slen;
memcpy(buf,sstr,buflen);
+1 -1
View File
@@ -521,7 +521,7 @@ int jemalloc_purge() {
unsigned narenas = 0;
size_t sz = sizeof(unsigned);
if (!je_mallctl("arenas.narenas", &narenas, &sz, NULL, 0)) {
snprintf(tmp, 32, "arena.%d.purge", narenas);
sprintf(tmp, "arena.%d.purge", narenas);
if (!je_mallctl(tmp, NULL, 0, NULL, 0))
return 0;
}
+2 -6
View File
@@ -53,12 +53,12 @@ start_server {overrides {active-replica {yes} multi-master {yes} client-output-b
set elapsed [expr {[clock milliseconds]-$cycle_start_time}]
if {$elapsed > $duration*1000} break
if {rand() < .05} {
test "PSYNC2 #3899 regression (multi-master): kill first replica" {
test "PSYNC2 #3899 regression: kill first replica" {
$R(1) client kill type master
}
}
if {rand() < .05} {
test "PSYNC2 #3899 regression (multi-master): kill chained replica" {
test "PSYNC2 #3899 regression: kill chained replica" {
$R(2) client kill type master
}
}
@@ -99,8 +99,4 @@ start_server {overrides {active-replica {yes} multi-master {yes} client-output-b
fail [format "The three instances have different data sets:\n\tnode 1: %s\n\tnode 2: %s\n\tnode 3: %s\nRun diff -u against /tmp/repldump*.txt for more info" [$R(0) debug digest] [$R(1) debug digest] [$R(2) debug digest] ]
}
}
assert {[s -2 sync_partial_ok] > 0}
assert {[s -1 sync_partial_ok] > 0}
assert {[s 0 sync_partial_ok] > 0}
}}}
+20 -22
View File
@@ -6,31 +6,29 @@ proc prepare_value {size} {
return $_v
}
if {$::flash_enabled} {
start_server {tags {"replication-fast"} overrides {storage-provider {flash ./rocks.db.master} databases 256}} {
set slave [srv 0 client]
set slave_host [srv 0 host]
set slave_port [srv 0 port]
start_server {tags {} overrides {storage-provider {flash ./rocks.db.replica} databases 256}} {
set master [srv 0 client]
set master_host [srv 0 host]
set master_port [srv 0 port]
start_server {tags {"replication-fast"} overrides {storage-provider {flash ./rocks.db.master} databases 256}} {
set slave [srv 0 client]
set slave_host [srv 0 host]
set slave_port [srv 0 port]
start_server {tags {} overrides {storage-provider {flash ./rocks.db.replica} databases 256}} {
set master [srv 0 client]
set master_host [srv 0 host]
set master_port [srv 0 port]
test "fast replication with large value" {
set _v [prepare_value [expr 64*1024*1024]]
# $master set key $_v
test "fast replication with large value" {
set _v [prepare_value [expr 64*1024*1024]]
# $master set key $_v
$slave replicaof $master_host $master_port
wait_for_condition 50 300 {
[lindex [$slave role] 0] eq {slave} &&
[string match {*master_link_status:up*} [$slave info replication]]
} else {
fail "Can't turn the instance into a replica"
}
assert_equal [$slave debug digest] [$master debug digest]
$slave replicaof no one
$slave replicaof $master_host $master_port
wait_for_condition 50 300 {
[lindex [$slave role] 0] eq {slave} &&
[string match {*master_link_status:up*} [$slave info replication]]
} else {
fail "Can't turn the instance into a replica"
}
assert_equal [$slave debug digest] [$master debug digest]
$slave replicaof no one
}
}
}
+103 -105
View File
@@ -8,130 +8,128 @@
# If reconnect is > 0, the test actually try to break the connection and
# reconnect with the master, otherwise just the initial synchronization is
# checked for consistency.
if {$::flash_enabled} {
proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
start_server [list tags {"repl"} overrides [list storage-provider {flash .rocks.db.m} repl-backlog-size 64m]] {
start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {
proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
start_server [list tags {"repl"} overrides [list storage-provider {flash .rocks.db.m} repl-backlog-size 64m]] {
start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {
set master [srv -1 client]
set master_host [srv -1 host]
set master_port [srv -1 port]
set slave [srv 0 client]
set master [srv -1 client]
set master_host [srv -1 host]
set master_port [srv -1 port]
set slave [srv 0 client]
set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000]
set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000]
set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000]
set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000]
set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000]
set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000]
test {Slave should be able to synchronize with the master} {
$slave slaveof $master_host $master_port
wait_for_condition 50 100 {
[lindex [r role] 0] eq {slave} &&
[lindex [r role] 3] eq {connected}
} else {
fail "Replication not started."
}
test {Slave should be able to synchronize with the master} {
$slave slaveof $master_host $master_port
wait_for_condition 50 100 {
[lindex [r role] 0] eq {slave} &&
[lindex [r role] 3] eq {connected}
} else {
fail "Replication not started."
}
}
# Check that the background clients are actually writing.
test {Detect write load to master} {
wait_for_condition 50 1000 {
[$master dbsize] > 100
} else {
fail "Can't detect write load from background clients."
}
# Check that the background clients are actually writing.
test {Detect write load to master} {
wait_for_condition 50 1000 {
[$master dbsize] > 100
} else {
fail "Can't detect write load from background clients."
}
}
test "Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect)" {
# Now while the clients are writing data, break the maste-slave
# link multiple times.
if ($reconnect) {
for {set j 0} {$j < $duration*10} {incr j} {
after 100
# catch {puts "MASTER [$master dbsize] keys, REPLICA [$slave dbsize] keys"}
test "Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect)" {
# Now while the clients are writing data, break the maste-slave
# link multiple times.
if ($reconnect) {
for {set j 0} {$j < $duration*10} {incr j} {
after 100
# catch {puts "MASTER [$master dbsize] keys, REPLICA [$slave dbsize] keys"}
if {($j % 20) == 0} {
catch {
$slave debug restart
}
if {($j % 20) == 0} {
catch {
$slave debug restart
}
}
}
stop_bg_complex_data $load_handle0
stop_bg_complex_data $load_handle1
stop_bg_complex_data $load_handle2
# Wait for the slave to reach the "online"
# state from the POV of the master.
set retry 5000
while {$retry} {
set info [$master info]
if {[string match {*slave0:*state=online*} $info]} {
break
} else {
incr retry -1
after 100
}
}
if {$retry == 0} {
error "assertion:Slave not correctly synchronized"
}
# Wait that slave acknowledge it is online so
# we are sure that DBSIZE and DEBUG DIGEST will not
# fail because of timing issues. (-LOADING error)
wait_for_condition 5000 100 {
[lindex [$slave role] 3] eq {connected}
} else {
fail "Slave still not connected after some time"
}
set retry 20
while {$retry && ([$master debug digest] ne [$slave debug digest])}\
{
after 1000
incr retry -1
}
assert {[$master dbsize] > 0}
if {[$master debug digest] ne [$slave debug digest]} {
set csv1 [csvdump r]
set csv2 [csvdump {r -1}]
set fd [open /tmp/repldump1.txt w]
puts -nonewline $fd $csv1
close $fd
set fd [open /tmp/repldump2.txt w]
puts -nonewline $fd $csv2
close $fd
puts "Master - Replica inconsistency"
puts "Run diff -u against /tmp/repldump*.txt for more info"
}
assert_equal [r debug digest] [r -1 debug digest]
eval $cond
}
stop_bg_complex_data $load_handle0
stop_bg_complex_data $load_handle1
stop_bg_complex_data $load_handle2
# Wait for the slave to reach the "online"
# state from the POV of the master.
set retry 5000
while {$retry} {
set info [$master info]
if {[string match {*slave0:*state=online*} $info]} {
break
} else {
incr retry -1
after 100
}
}
if {$retry == 0} {
error "assertion:Slave not correctly synchronized"
}
# Wait that slave acknowledge it is online so
# we are sure that DBSIZE and DEBUG DIGEST will not
# fail because of timing issues. (-LOADING error)
wait_for_condition 5000 100 {
[lindex [$slave role] 3] eq {connected}
} else {
fail "Slave still not connected after some time"
}
set retry 20
while {$retry && ([$master debug digest] ne [$slave debug digest])}\
{
after 1000
incr retry -1
}
assert {[$master dbsize] > 0}
if {[$master debug digest] ne [$slave debug digest]} {
set csv1 [csvdump r]
set csv2 [csvdump {r -1}]
set fd [open /tmp/repldump1.txt w]
puts -nonewline $fd $csv1
close $fd
set fd [open /tmp/repldump2.txt w]
puts -nonewline $fd $csv2
close $fd
puts "Master - Replica inconsistency"
puts "Run diff -u against /tmp/repldump*.txt for more info"
}
assert_equal [r debug digest] [r -1 debug digest]
eval $cond
}
}
}
}
foreach mdl {no yes} {
foreach sdl {disabled swapdb} {
test_psync {no reconnection, just sync} 6 1000000 3600 0 {
} $mdl $sdl 0
foreach mdl {no yes} {
foreach sdl {disabled swapdb} {
test_psync {no reconnection, just sync} 6 1000000 3600 0 {
} $mdl $sdl 0
test_psync {ok psync} 6 100000000 3600 0 {
assert {[s -1 sync_partial_ok] > 0}
} $mdl $sdl 1
test_psync {ok psync} 6 100000000 3600 0 {
assert {[s -1 sync_partial_ok] > 0}
} $mdl $sdl 1
test_psync {no backlog} 6 100 3600 0.5 {
assert {[s -1 sync_partial_err] > 0}
} $mdl $sdl 1
test_psync {no backlog} 6 100 3600 0.5 {
assert {[s -1 sync_partial_err] > 0}
} $mdl $sdl 1
test_psync {ok after delay} 3 100000000 3600 3 {
assert {[s -1 sync_partial_ok] > 0}
} $mdl $sdl 1
test_psync {ok after delay} 3 100000000 3600 3 {
assert {[s -1 sync_partial_ok] > 0}
} $mdl $sdl 1
test_psync {backlog expired} 3 100000000 1 3 {
assert {[s -1 sync_partial_err] > 0}
} $mdl $sdl 1
}
test_psync {backlog expired} 3 100000000 1 3 {
assert {[s -1 sync_partial_err] > 0}
} $mdl $sdl 1
}
}
+5 -36
View File
@@ -66,33 +66,6 @@ proc redis {{server 127.0.0.1} {port 6379} {defer 0} {tls 0} {tlsoptions {}} {re
interp alias {} ::redis::redisHandle$id {} ::redis::__dispatch__ $id
}
# On recent versions of tcl-tls/OpenSSL, reading from a dropped connection
# results with an error we need to catch and mimic the old behavior.
proc ::redis::redis_safe_read {fd len} {
if {$len == -1} {
set err [catch {set val [read $fd]} msg]
} else {
set err [catch {set val [read $fd $len]} msg]
}
if {!$err} {
return $val
}
if {[string match "*connection abort*" $msg]} {
return {}
}
error $msg
}
proc ::redis::redis_safe_gets {fd} {
if {[catch {set val [gets $fd]} msg]} {
if {[string match "*connection abort*" $msg]} {
return {}
}
error $msg
}
return $val
}
# This is a wrapper to the actual dispatching procedure that handles
# reconnection if needed.
proc ::redis::__dispatch__ {id method args} {
@@ -173,10 +146,6 @@ proc ::redis::__method__read {id fd} {
::redis::redis_read_reply $id $fd
}
proc ::redis::__method__rawread {id fd {len -1}} {
return [redis_safe_read $fd $len]
}
proc ::redis::__method__write {id fd buf} {
::redis::redis_write $fd $buf
}
@@ -223,8 +192,8 @@ proc ::redis::redis_writenl {fd buf} {
}
proc ::redis::redis_readnl {fd len} {
set buf [redis_safe_read $fd $len]
redis_safe_read $fd 2 ; # discard CR LF
set buf [read $fd $len]
read $fd 2 ; # discard CR LF
return $buf
}
@@ -270,11 +239,11 @@ proc ::redis::redis_read_map {id fd} {
}
proc ::redis::redis_read_line fd {
string trim [redis_safe_gets $fd]
string trim [gets $fd]
}
proc ::redis::redis_read_null fd {
redis_safe_gets $fd
gets $fd
return {}
}
@@ -291,7 +260,7 @@ proc ::redis::redis_read_reply {id fd} {
}
while {1} {
set type [redis_safe_read $fd 1]
set type [read $fd 1]
switch -exact -- $type {
_ {return [redis_read_null $fd]}
: -
-14
View File
@@ -47,7 +47,6 @@ set ::all_tests {
integration/replication-3
integration/replication-4
integration/replication-psync
integration/replication-psync-flash
integration/replication-active
integration/replication-multimaster
integration/replication-multimaster-connect
@@ -60,13 +59,11 @@ set ::all_tests {
integration/failover
integration/keydb-cli
integration/keydb-benchmark
integration/replication-fast
integration/replication-psync-multimaster
unit/pubsub
unit/slowlog
unit/scripting
unit/maxmemory
unit/flash
unit/introspection
unit/introspection-2
unit/limits
@@ -89,7 +86,6 @@ set ::all_tests {
integration/logging
integration/corrupt-dump
integration/corrupt-dump-fuzzer
unit/soft_shutdown
}
# Index to the next test to run in the ::all_tests list.
set ::next_test 0
@@ -739,16 +735,6 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
}
}
# Check if we compiled with flash
set status [catch {exec src/keydb-server --is-flash-enabled}]
if {$status == 0} {
puts "KeyDB was built with FLASH, including FLASH tests"
set ::flash_enabled 1
} else {
puts "KeyDB was not built with FLASH. Excluding FLASH tests"
set ::flash_enabled 0
}
set filtered_tests {}
# Set the filtered tests to be the short list (single_tests) if exists.
+160 -161
View File
@@ -1,171 +1,170 @@
if {$::flash_enabled} {
start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {
start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {
test { FLASH - GET works after eviction } {
r set testkey foo
r flushall cache
assert_equal {foo} [r get testkey]
}
test { FLASH - GET works after eviction } {
r set testkey foo
r flushall cache
assert_equal {foo} [r get testkey]
}
test { DEL of nonexistant key returns 0 } {
r flushall
assert_equal {0} [r del foobar]
assert_equal {0} [r dbsize] "Key count is accurate after non-existant delete"
}
test { DEL of flushed key works } {
r flushall
r set testkey foo
assert_equal {1} [r dbsize] "Only one key after first insert"
r flushall cache
assert_equal {foo} [r get testkey] "Value still there after flushing cache"
r del testkey
assert_equal {0} [r dbsize] "No keys after delete"
}
test { SET of existing but flushed key works } {
r flushall
r set testkey foo
assert_equal {1} [r dbsize] "Only one key after first insert"
r flushall cache
assert_equal {1} [r dbsize] "Only one key after flushall cache"
r set testkey bar
assert_equal {1} [r dbsize] "Only one key after overwrite"
assert_equal {bar} [r get testkey]
}
test { SET of existing but flushed key with EXPIRE works } {
test { DEL of nonexistant key returns 0 } {
r flushall
assert_equal {0} [r dbsize]
r set testkey foo ex 10000
assert_equal {0} [r del foobar]
assert_equal {0} [r dbsize] "Key count is accurate after non-existant delete"
}
test { DEL of flushed key works } {
r flushall
r set testkey foo
assert_equal {1} [r dbsize] "Only one key after first insert"
r flushall cache
assert_equal {foo} [r get testkey] "Value still there after flushing cache"
r del testkey
assert_equal {0} [r dbsize] "No keys after delete"
}
test { SET of existing but flushed key works } {
r flushall
r set testkey foo
assert_equal {1} [r dbsize] "Only one key after first insert"
r flushall cache
assert_equal {1} [r dbsize] "Only one key after flushall cache"
r set testkey bar ex 10000
r set testkey bar
assert_equal {1} [r dbsize] "Only one key after overwrite"
assert_equal {bar} [r get testkey]
assert [expr [r ttl testkey] > 0]
}
test { EXPIRE of existing but flushed key } {
r flushall
assert_equal {0} [r dbsize]
r set testkey foo
assert_equal {1} [r dbsize]
r flushall cache
r expire testkey 10000
assert_equal {1} [r dbsize]
assert_equal {foo} [r get testkey]
assert [expr [r ttl testkey] > 0]
}
test { CREATE and UPDATE in transaction, key count is accurate } {
r flushall
r multi
r set testkey 2
r incr testkey
r exec
assert_equal {1} [r dbsize]
assert_equal {3} [r get testkey]
}
test { EXPIRE key count is accurate } {
r flushall
r set testkey foo ex 1
r flushall cache
assert_equal {1} [r dbsize]
after 1500
assert_equal {0} [r dbsize]
}
test { SUBKEY EXPIRE persists after cache flush } {
r flushall
r sadd testkey foo bar baz
r expiremember testkey foo 10000
r flushall cache
assert [expr [r ttl testkey foo] > 0]
}
test { LIST pop works after flushing cache } {
r flushall
r lpush testkey foo
r flushall cache
assert_equal {foo} [r lpop testkey]
}
test { DIGEST string the same after flushing cache } {
r flushall
r set testkey foo
r set testkey1 foo ex 10000
set expectedDigest [r debug digest]
r flushall cache
assert_equal $expectedDigest [r debug digest]
}
test { DIGEST list the same after flushing cache } {
r flushall
r lpush testkey foo bar
set expectedDigest [r debug digest]
r flushall cache
assert_equal $expectedDigest [r debug digest]
}
test { DELETE of flushed set member persists after another flush } {
r flushall
r sadd set1 val1 val2 val3
assert_equal {3} [r scard set1]
r flushall cache
r srem set1 val1
r flushall cache
assert_equal {2} [r scard set1]
}
r flushall
# If a weak storage memory model is set, wait for any pending snapshot writes to finish
after 500
foreach policy {
allkeys-random allkeys-lru allkeys-lfu
} {
test "FLASH - is eviction working without data loss (successfully stored to flash)? (policy $policy)" {
# Get the current memory limit and calculate a new limit.
# Set limit to 100M.
set used [s used_memory]
set limit [expr {$used+60*1024*1024}]
r config set maxmemory $limit
r config set maxmemory-policy $policy
# Now add keys equivalent to 1024b until the limit is almost reached.
set numkeys 0
r set first val
while 1 {
r set $numkeys xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
incr numkeys
if {[s used_memory]+1024 >= $limit} {
break
}
}
# Add additional keys to force eviction
# should still be under the limit for maxmemory, however all keys set should still exist between flash and memory
# check same number of keys exist in addition to values of first and last keys
set err 0
set extra_keys [expr floor([expr ($limit * 0.4) / 1024])]
for {set j 0} {$j < $extra_keys} {incr j} {
catch {
r set p2$j xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
} err
assert {$err == {OK}}
}
if {[log_file_matches [srv 0 stdout] "*Failed to evict*"]} {
fail "Server did not evict cleanly (detected full flush)"
}
r set last val
set dbsize [r dbsize]
assert {[s used_memory] < ($limit*1.2)}
assert {$dbsize == $numkeys+$extra_keys+2}
assert {[r get first] == {val}}
assert {[r get last] == {val}}
r flushall
}
}
}
test { SET of existing but flushed key with EXPIRE works } {
r flushall
assert_equal {0} [r dbsize]
r set testkey foo ex 10000
assert_equal {1} [r dbsize] "Only one key after first insert"
r flushall cache
assert_equal {1} [r dbsize] "Only one key after flushall cache"
r set testkey bar ex 10000
assert_equal {1} [r dbsize] "Only one key after overwrite"
assert_equal {bar} [r get testkey]
assert [expr [r ttl testkey] > 0]
}
test { EXPIRE of existing but flushed key } {
r flushall
assert_equal {0} [r dbsize]
r set testkey foo
assert_equal {1} [r dbsize]
r flushall cache
r expire testkey 10000
assert_equal {1} [r dbsize]
assert_equal {foo} [r get testkey]
assert [expr [r ttl testkey] > 0]
}
test { CREATE and UPDATE in transaction, key count is accurate } {
r flushall
r multi
r set testkey 2
r incr testkey
r exec
assert_equal {1} [r dbsize]
assert_equal {3} [r get testkey]
}
test { EXPIRE key count is accurate } {
r flushall
r set testkey foo ex 1
r flushall cache
assert_equal {1} [r dbsize]
after 1500
assert_equal {0} [r dbsize]
}
test { SUBKEY EXPIRE persists after cache flush } {
r flushall
r sadd testkey foo bar baz
r expiremember testkey foo 10000
r flushall cache
assert [expr [r ttl testkey foo] > 0]
}
test { LIST pop works after flushing cache } {
r flushall
r lpush testkey foo
r flushall cache
assert_equal {foo} [r lpop testkey]
}
test { DIGEST string the same after flushing cache } {
r flushall
r set testkey foo
r set testkey1 foo ex 10000
set expectedDigest [r debug digest]
r flushall cache
assert_equal $expectedDigest [r debug digest]
}
test { DIGEST list the same after flushing cache } {
r flushall
r lpush testkey foo bar
set expectedDigest [r debug digest]
r flushall cache
assert_equal $expectedDigest [r debug digest]
}
test { DELETE of flushed set member persists after another flush } {
r flushall
r sadd set1 val1 val2 val3
assert_equal {3} [r scard set1]
r flushall cache
r srem set1 val1
r flushall cache
assert_equal {2} [r scard set1]
}
r flushall
# If a weak storage memory model is set, wait for any pending snapshot writes to finish
after 500
foreach policy {
allkeys-random allkeys-lru allkeys-lfu
} {
test "FLASH - is eviction working without data loss (successfully stored to flash)? (policy $policy)" {
# Get the current memory limit and calculate a new limit.
# Set limit to 100M.
set used [s used_memory]
set limit [expr {$used+60*1024*1024}]
r config set maxmemory $limit
r config set maxmemory-policy $policy
# Now add keys equivalent to 1024b until the limit is almost reached.
set numkeys 0
r set first val
while 1 {
r set $numkeys xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
incr numkeys
if {[s used_memory]+1024 >= $limit} {
break
}
}
# Add additional keys to force eviction
# should still be under the limit for maxmemory, however all keys set should still exist between flash and memory
# check same number of keys exist in addition to values of first and last keys
set err 0
set extra_keys [expr floor([expr ($limit * 0.4) / 1024])]
for {set j 0} {$j < $extra_keys} {incr j} {
catch {
r set p2$j xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
} err
assert {$err == {OK}}
}
if {[log_file_matches [srv 0 stdout] "*Failed to evict*"]} {
fail "Server did not evict cleanly (detected full flush)"
}
r set last val
set dbsize [r dbsize]
assert {[s used_memory] < ($limit*1.2)}
assert {$dbsize == $numkeys+$extra_keys+2}
assert {[r get first] == {val}}
assert {[r get last] == {val}}
r flushall
}
}
}
+9 -15
View File
@@ -49,13 +49,10 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
r config set maxmemory 100mb
r config set maxmemory-policy allkeys-lru
for {set i 0} {$i < 10} {incr i} {
populate 700000 asdf1 150
populate 170000 asdf2 300
after 120 ;# serverCron only updates the info once in 100ms
set frag [s allocator_frag_ratio]
if {$frag >= 1.4} {break}
}
populate 700000 asdf1 150
populate 170000 asdf2 300
after 120 ;# serverCron only updates the info once in 100ms
set frag [s allocator_frag_ratio]
if {$::verbose} {
puts "frag $frag"
}
@@ -510,14 +507,11 @@ start_server {tags {"defrag"} overrides {server-threads 1 active-replica yes} }
r config set active-defrag-ignore-bytes 2mb
r config set maxmemory 100mb
r config set maxmemory-policy allkeys-lru
for {set i 0} {$i < 10} {incr i} {
populate 700000 asdf1 150
populate 170000 asdf2 300
r ping ;# trigger eviction following the previous population
after 120 ;# serverCron only updates the info once in 100ms
set frag [s allocator_frag_ratio]
if {$frag >= 1.4} {break}
}
populate 700000 asdf1 150
populate 170000 asdf2 300
r ping ;# trigger eviction following the previous population
after 120 ;# serverCron only updates the info once in 100ms
set frag [s allocator_frag_ratio]
if {$::verbose} {
puts "frag $frag"
}
+3 -2
View File
@@ -111,7 +111,7 @@ start_server {tags {"obuf-limits"} overrides { server-threads 1 }} {
# Read nothing
set fd [$rd channel]
assert_equal {} [$rd rawread]
assert_equal {} [read $fd]
}
# Note: This test assumes that what's written with one write, will be read by redis in one read.
@@ -151,7 +151,8 @@ start_server {tags {"obuf-limits"} overrides { server-threads 1 }} {
assert_equal "PONG" [r ping]
set clients [r client list]
assert_no_match "*name=multicommands*" $clients
assert_equal {} [$rd2 rawread]
set fd [$rd2 channel]
assert_equal {} [read $fd]
}
test {Execute transactions completely even if client output buffer limit is enforced} {
-141
View File
@@ -1,141 +0,0 @@
start_server {tags {"soft_shutdown"} } {
test {soft shutdown command replies} {
assert_equal [r shutdown soft] "OK"
}
test {soft shutdown errors on ping} {
catch {[r ping]} e
assert_match {SHUTDOWN PENDING} $e
}
}
start_server {tags {"soft_shutdown"} } {
test {soft shutdown prevents new connections} {
assert_equal [r shutdown soft] "OK"
# reconnect
set catch_res [catch {set rd [redis_deferring_client]} e]
if {$::tls} {
assert_equal $catch_res 1
} else {
assert_match {*SHUTDOWN*} $e
}
}
}
start_server {tags {"soft_shutdown"} } {
test {soft shutdown allows commands to execute while waiting} {
assert_equal [r shutdown soft] "OK"
r set test val
assert_equal [r get test] {val}
}
}
start_server {tags {"soft_shutdown"} } {
test {soft shutdown shuts down after all clients exit} {
assert_equal [r shutdown soft] "OK"
r close
after 500
catch {set rd [redis_deferring_client]} e
assert_match {*refused*} $e
}
}
start_server {tags {"soft_shutdown"} overrides {soft-shutdown yes} } {
test {soft shutdown triggered by SIGINT} {
exec kill -SIGINT [s process_id]
after 100
catch {[r ping]} e
assert_match {SHUTDOWN PENDING} $e
}
test {second SIGINT forces a shutdown during a soft shutdown} {
exec kill -SIGINT [s process_id]
after 100
catch {[r ping]} e
assert_match {*I/O*} $e
}
}
start_server {tags {"soft_shutdown"} } {
test {monitor does not prevent soft shutdown} {
set monitor [redis_deferring_client]
$monitor monitor
assert_equal [r shutdown soft] "OK"
r close
after 500
catch {set rd [redis_deferring_client]} e
assert_match {*refused*} $e
}
}
start_server {tags {"soft_shutdown"} } {
start_server {} {
set node_0 [srv 0 client]
set node_0_host [srv 0 host]
set node_0_port [srv 0 port]
set node_0_pid [srv 0 pid]
set node_1 [srv -1 client]
set node_1_host [srv -1 host]
set node_1_port [srv -1 port]
set node_1_pid [srv -1 pid]
$node_0 replicaof $node_1_host $node_1_port
wait_for_sync $node_0
test {soft shutdown works for with master} {
$node_1 shutdown soft
} {OK}
test {soft shutdown on master doesn't affect replica} {
assert_equal [$node_0 ping] {PONG}
}
test {soft shutdown on master updates ping response} {
catch {$node_1 ping} e
assert_equal $e {SHUTDOWN PENDING}
}
test {master prevents new connections with soft shutdown} {
set c1 [redis $node_1_host $node_1_port 1 $::tls]
set catch_res [catch {$c1 read} e]
if {$::tls} {
assert_equal $catch_res 1
} else {
assert_match {*SHUTDOWN*} $e
}
}
test {master soft shutdown works after all clients disconnect} {
$node_1 close
after 500
catch {set c1 [redis $node_1_host $node_1_port 1 $::tls]} e
assert_match {*refused*} $e
}
}
}
start_server {tags {"soft_shutdown"} } {
start_server {} {
set node_0 [srv 0 client]
set node_0_host [srv 0 host]
set node_0_port [srv 0 port]
set node_0_pid [srv 0 pid]
set node_1 [srv -1 client]
set node_1_host [srv -1 host]
set node_1_port [srv -1 port]
set node_1_pid [srv -1 pid]
$node_0 replicaof $node_1_host $node_1_port
wait_for_sync $node_0
test {soft shutdown on replica is not blocked by master} {
assert_equal [$node_0 shutdown soft] {OK}
$node_0 close
after 500
catch {set c0 [redis $node_0_host $node_0_port 1 $::tls]} e
assert_match {*refused*} $e
}
}
}