Enable evction tenacity feature for storage providers

Replace mention of scratch-file-path with db-s3-object

.gitlab-ci.yml

@@ -0,0 +1,164 @@
+.standard-pipeline:
+  rules:
+    - if: '$COVERAGE'
+      when: never
+    - if: '$ENDURANCE'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "push"'
+
+build:
+  extends: .standard-pipeline
+  tags:
+    - docker
+  stage: build
+  script:
+    - git submodule update --init
+    - make distclean
+    - make -j
+  artifacts:
+    paths:
+      - src/
+
+runtest:
+  extends: .standard-pipeline
+  dependencies:
+    - build
+  tags:
+    - docker
+  stage: test
+  script:
+    - ./runtest --config server-threads 3
+
+runtest-cluster:
+  extends: .standard-pipeline
+  dependencies:
+    - build
+  tags:
+    - docker
+  stage: test
+  script:
+    - ./runtest-cluster
+
+runtest-moduleapi:
+  extends: .standard-pipeline
+  dependencies:
+    - build
+  tags:
+    - docker
+  stage: test
+  script:
+    - ./runtest-moduleapi
+
+runtest-sentinel:
+  extends: .standard-pipeline
+  dependencies:
+    - build
+  tags:
+    - docker
+  stage: test
+  script:
+    - ./runtest-sentinel
+
+node-redis-test:
+  extends: .standard-pipeline
+  dependencies:
+    - build
+  rules:
+    - when: never
+  tags:
+    - docker
+    - ipv6
+  stage: test
+  script:
+    - cp -pf src/keydb-server /usr/local/bin
+    - cp -pf src/keydb-cli /usr/local/bin
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/node-redis.git
+    - cd node-redis
+    - npm install
+    - npm run test
+
+jedis-test:
+  extends: .standard-pipeline
+  dependencies:
+    - build
+  tags:
+    - docker
+    - ipv4
+  stage: test
+  script:
+    - cp -pf src/keydb-server /usr/local/bin
+    - cp -pf src/keydb-cli /usr/local/bin
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/jedis.git
+    - cd jedis
+    - make test
+
+redis-rs-test:
+  extends: .standard-pipeline
+  dependencies:
+    - build
+  tags:
+    - docker
+  stage: test
+  script:
+    - cp -pf src/keydb-server /usr/local/bin
+    - cp -pf src/keydb-cli /usr/local/bin
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/redis-rs.git
+    - cd redis-rs
+    - make test
+
+endurance-test:
+  rules:
+    - if: '$ENDURANCE'
+  tags:
+    - docker
+  stage: test
+  script:
+    - git submodule update --init
+    - make distclean
+    - make -j
+    - ./runtest --config server-threads 3 --loop --stop
+
+coverage-test:
+  rules:
+    - if: '$COVERAGE'
+  tags:
+    - docker
+  stage: test
+  script:
+    - git submodule update --init
+    - make distclean
+    - make gcov -j
+    - make install
+    - ./runtest || true
+    - pkill keydb-server || true
+    - pkill stunnel || true
+    - ./runtest-cluster || true
+    - pkill keydb-server || true
+    - pkill stunnel || true
+    - ./runtest-sentinel || true
+    - pkill keydb-server || true
+    - pkill stunnel || true
+    - ./runtest-moduleapi || true
+    - pkill keydb-server || true
+    - pkill stunnel || true
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/redis-rs.git
+    - cd redis-rs
+    - make test || true
+    - pkill keydb-server || true
+    - pkill stunnel || true
+    - cd ..
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/jedis.git
+    - cd jedis
+    - make test || true
+    - pkill keydb-server || true
+    - pkill stunnel || true
+    - cd ..
+    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.eqalpha.com/keydb-dev/node-redis.git
+    - cd node-redis
+    - npm install
+    - npm run test || true
+    - pkill keydb-server || true
+    - pkill stunnel || true
+    - cd ..
+    - geninfo -o KeyDB.info --no-external .
+    - genhtml --legend -o lcov-html KeyDB.info

build.yaml

@@ -0,0 +1,48 @@
+# Doc: https://wiki.sc-corp.net/pages/viewpage.action?pageId=121500284
+version: 1
+machamp:
+  keydb-build:
+    # Optional - build counter is linked to the build def
+    tag_template: 0.0.%build.counter%
+    # Optional - value in seconds before a build is terminated, default is 3600 seconds
+    timeout: 3600
+    # Optional - update ghe or not, default to true
+    update_ghe: true
+    code_coverage: false
+    # Required
+    steps:
+      make-build:
+        type: cmd
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./machamp_scripts/build.sh
+      tls-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest --clients $(nproc) --verbose --tls
+      cluster-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest-cluster --tls
+      sentinel-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest-sentinel
+      module-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest-moduleapi
+      rotation-test:
+        type: cmd
+        parent: make-build
+        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
+        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
+        command: ./runtest-rotation

ci.yaml

@@ -0,0 +1,16 @@
+# Doc: https://wiki.sc-corp.net/display/TOOL/ci.yaml+User+Guide
+version: 1
+on:
+  pull_request:
+    - workflows:
+      # All builds that use machamp should use the defined `backend_workflow`
+      - workflow_type: backend_workflow
+        # references a build defined in build.yaml
+        build_name: keydb-build
+        arch_types: ["amd64", "arm64"]
+  push:
+    - branches: [master]
+      workflows:
+        - workflow_type: backend_workflow
+          build_name: keydb-build
+          arch_types: ["amd64", "arm64"]

deps/Makefile

@@ -101,11 +101,11 @@ jemalloc: .make-prerequisites
 .PHONY: jemalloc
 
 rocksdb: .make-prerequisites
-	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+#	@printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
 ifeq ($(uname_M),x86_64)
-	cd rocksdb && CFLAGS=-Wno-error PORTABLE=1 USE_SSE=1 FORCE_SSE42=1 $(MAKE) static_lib
+#	cd rocksdb && CFLAGS=-Wno-error PORTABLE=1 USE_SSE=1 FORCE_SSE42=1 $(MAKE) static_lib
 else
-	cd rocksdb && PORTABLE=1 $(MAKE) static_lib
+#	cd rocksdb && PORTABLE=1 $(MAKE) static_lib
 endif
 
 .PHONY: rocksdb

keydb.conf

@@ -184,9 +184,7 @@ tcp-keepalive 300
 #
 # tls-client-key-file-pass secret
 
-# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange,
-# required by older versions of OpenSSL (<3.0). Newer versions do not require
-# this configuration and recommend against it.
+# Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange:
 #
 # tls-dh-params-file keydb.dh
 

pkg/deb/deb-buildsource.sh

@@ -23,8 +23,8 @@ pkg_name=keydb-$majorv:$version$distname
 
 # create build tree
 cd ../../../
-tar -czvf keydb_$version.orig.tar.gz --force-local KeyDB
-cd KeyDB/pkg/deb/
+tar -czvf keydb_$version.orig.tar.gz --force-local keydb-internal
+cd keydb-internal/pkg/deb/
 mkdir -p $pkg_name/tmp
 if [[ "$codename" == "xenial" ]] || [[ "$codename" == "stretch" ]]; then
 	cp -r debian_dh9 $pkg_name/tmp/debian

pkg/deb/debian/control

@@ -26,8 +26,8 @@ Build-Depends:
  libzstd-dev <!nocheck>
 Standards-Version: 4.2.1
 Homepage: https://docs.keydb.dev/
-Vcs-Git: https://github.com/Snapchat/KeyDB.git
-Vcs-Browser: https://github.com/Snapchat/KeyDB
+Vcs-Git: https://github.com/EQ-Alpha/KeyDB.git
+Vcs-Browser: https://github.com/EQ-Alpha/KeyDB
 
 Package: keydb
 Architecture: all

pkg/deb/debian_dh9/control

@@ -25,8 +25,8 @@ Build-Depends:
  libzstd-dev <!nocheck>
 Standards-Version: 4.2.1
 Homepage: https://docs.keydb.dev/
-Vcs-Git: https://github.com/Snapchat/KeyDB.git
-Vcs-Browser: https://github.com/Snapchat/KeyDB
+Vcs-Git: https://github.com/EQ-Alpha/KeyDB.git
+Vcs-Browser: https://github.com/EQ-Alpha/KeyDB
 
 Package: keydb
 Architecture: all

pkg/deb/master_changelog

@@ -1,11 +1,3 @@
-keydb (6:6.3.1-1distribution_placeholder) codename_placeholder; urgency=medium
-
-  * This point release contains fixes to bugs related to expires, active-rep, and rdb saving
-  * Issues fixed: #419, #422, #428
-  * PRs: #426, #429, #431, #433
-
--- Ben Schermel <ben@eqalpha.com>  Mon, 23 May 2022 20:00:37 +0000
-
 keydb (6:6.3.0-1distribution_placeholder) codename_placeholder; urgency=medium
 
   * This release open sources KeyDB Enterprise features into the open source project along with PSYNC for active replication

pkg/docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 SHELL ["/bin/bash","-c"]
 RUN groupadd -r keydb && useradd -r -g keydb keydb
 # use gosu for easy step-down from root: https://github.com/tianon/gosu/releases
@@ -77,7 +77,7 @@ RUN \
         mkdir /data && chown keydb:keydb /data; \
         mkdir /flash && chown keydb:keydb /flash; \
         mkdir -p /etc/keydb; \
-        cp /tmp/KeyDB/keydb.conf /etc/keydb/; \
+        cp /tmp/keydb-internal/keydb.conf /etc/keydb/; \
         sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \
         sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \
         sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \
@@ -110,3 +110,8 @@ ENV KEYDB_PRO_DIRECTORY=/usr/local/bin/
 ENTRYPOINT ["docker-entrypoint.sh"]
 EXPOSE 6379
 CMD ["keydb-server","/etc/keydb/keydb.conf"]
+
+
+
+
+

pkg/docker/Dockerfile_Alpine

@@ -1,77 +0,0 @@
-FROM alpine:3.12
-# add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
-RUN addgroup -S -g 1000 keydb && adduser -S -G keydb -u 999 keydb
-RUN mkdir -p /etc/keydb
-ARG BRANCH
-RUN set -eux; \
-        \
-        apk add --no-cache su-exec; \
-        apk add --no-cache --virtual .build-deps \
-                coreutils \
-                gcc \
-                linux-headers \
-                make \
-                musl-dev \
-                openssl-dev \
-                git \
-                util-linux-dev \
-                curl-dev \
-                g++ \
-                libunwind-dev \
-                bash \
-                perl \
-                git \
-        ; \
-        cd /tmp && git clone --branch $BRANCH https://github.com/Snapchat/KeyDB.git --recursive; \
-        cd /tmp/KeyDB; \
-        # disable protected mode as it relates to docker
-        grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \
-        sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \
-        grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \
-        make -j$(nproc) BUILD_TLS=yes; \
-        cd src; \
-        strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel keydb-server; \
-        mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \
-        runDeps="$( \
-                scanelf --needed --nobanner --format '%n#p' --recursive /usr/local \
-                        | tr ',' '\n' \
-                        | sort -u \
-                        | awk 'system("[ -e /usr/local/lib/" $1 " ]") == 0 { next } { print "so:" $1 }' \
-        )"; \
-        apk add --no-network --virtual .keydb-rundeps $runDeps; \
-        apk del --no-network .build-deps; \
-        # create working directories and organize files
-        mkdir /data && chown keydb:keydb /data; \
-        mkdir /flash && chown keydb:keydb /flash; \
-        mkdir -p /etc/keydb; \
-        cp /tmp/KeyDB/keydb.conf /etc/keydb/; \
-        sed -i 's/^\(daemonize .*\)$/# \1/' /etc/keydb/keydb.conf; \
-        sed -i 's/^\(dir .*\)$/# \1\ndir \/data/' /etc/keydb/keydb.conf; \
-        sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \
-        sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \
-        sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \
-        ln -s keydb-cli redis-cli; \
-        cd /etc/keydb; \
-        ln -s keydb.conf redis.conf; \
-        rm -rf /tmp/*
-# generate entrypoint script
-RUN set -eux; \
-        echo '#!/bin/sh' > /usr/local/bin/docker-entrypoint.sh; \
-        echo 'set -e' >> /usr/local/bin/docker-entrypoint.sh; \
-        echo "# first arg is '-f' or '--some-option'" >> /usr/local/bin/docker-entrypoint.sh; \
-        echo "# or first arg is `something.conf`" >> /usr/local/bin/docker-entrypoint.sh; \
-        echo 'if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \
-        echo '        set -- keydb-server "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
-        echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \
-        echo "# allow the container to be started with `--user`" >> /usr/local/bin/docker-entrypoint.sh; \
-        echo 'if [ "$1" = "keydb-server" -a "$(id -u)" = "0" ]; then' >> /usr/local/bin/docker-entrypoint.sh; \
-        echo "        find . \! -user keydb -exec chown keydb '{}' +" >> /usr/local/bin/docker-entrypoint.sh; \
-        echo '        exec su-exec keydb "$0" "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
-        echo 'fi' >> /usr/local/bin/docker-entrypoint.sh; \
-        echo 'exec "$@"' >> /usr/local/bin/docker-entrypoint.sh; \
-        chmod +x /usr/local/bin/docker-entrypoint.sh
-VOLUME /data
-WORKDIR /data
-ENTRYPOINT ["docker-entrypoint.sh"]
-EXPOSE 6379
-CMD ["keydb-server", "/etc/keydb/keydb.conf"]

src/Makefile

@@ -20,7 +20,7 @@ release_hdr := $(shell sh -c './mkreleasehdr.sh')
 uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
 uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not')
 OPTIMIZATION?=-O2 -flto
-DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram
+DEPENDENCY_TARGETS=hiredis linenoise lua hdr_histogram rocksdb
 NODEPS:=clean distclean
 
 # Default settings
@@ -62,7 +62,6 @@ endif
 endif
 
 USEASM?=true
-ENABLE_FLASH?=no
 
 ifneq ($(strip $(SANITIZE)),)
 	CFLAGS+= -fsanitize=$(SANITIZE) -DSANITIZE  -fno-omit-frame-pointer
@@ -72,15 +71,6 @@ ifneq ($(strip $(SANITIZE)),)
 	USEASM=false
 endif
 
-ifeq ($(ENABLE_FLASH),yes)
-	FINAL_LIBS+= -lz -lcrypto -lbz2 -lzstd -llz4 -lsnappy
-	CXXFLAGS+= -I../deps/rocksdb/include/ -DENABLE_ROCKSDB
-	STORAGE_OBJ+= storage/rocksdb.o storage/rocksdbfactory.o
-	FINAL_LIBS+= ../deps/rocksdb/librocksdb.a 
-	DEPENDENCY_TARGETS+= rocksdb
-endif
-
-
 ifeq ($(CHECKED),true)
 	CXXFLAGS+= -DCHECKED_BUILD
 endif
@@ -142,7 +132,7 @@ DEBUG=-g -ggdb
 FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS)
 FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CXXFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS)
 FINAL_LDFLAGS=$(LDFLAGS) $(KEYDB_LDFLAGS) $(DEBUG)
-FINAL_LIBS+=-lm
+FINAL_LIBS+=-lm -lz -lcrypto
 
 ifneq ($(uname_S),Darwin)
 	FINAL_LIBS+=-latomic
@@ -260,7 +250,7 @@ endif
 
 # Include paths to dependencies
 FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram
-FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/concurrentqueue
+FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram -I../deps/rocksdb/include/ -I../deps/concurrentqueue
 
 # Determine systemd support and/or build preference (defaulting to auto-detection)
 BUILD_WITH_SYSTEMD=no
@@ -376,7 +366,7 @@ endif
 
 REDIS_SERVER_NAME=keydb-server$(PROG_SUFFIX)
 REDIS_SENTINEL_NAME=keydb-sentinel$(PROG_SUFFIX)
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd_server.o timeout.o setcpuaffinity.o AsyncWorkQueue.o snapshot.o storage/teststorageprovider.o keydbutils.o StorageCache.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ) $(STORAGE_OBJ)
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o t_nhash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crcspeed.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o connection.o tls.o sha256.o motd_server.o timeout.o setcpuaffinity.o AsyncWorkQueue.o snapshot.o storage/teststorageprovider.o keydbutils.o StorageCache.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)
 KEYDB_SERVER_OBJ=SnapshotPayloadParseState.o
 REDIS_CLI_NAME=keydb-cli$(PROG_SUFFIX)
 REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crcspeed.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o motd_client.o monotonic.o cli_common.o mt19937-64.o $(ASM_OBJ)

src/StorageCache.cpp

@@ -203,7 +203,6 @@ void StorageCache::beginWriteBatch() {
 }
 
 void StorageCache::emergencyFreeCache() {
-    std::unique_lock<fastlock> ul(m_lock);
     dict *d = m_pdict;
     m_pdict = nullptr;
     if (d != nullptr) {

src/cluster.cpp

@@ -771,7 +771,7 @@ unsigned long getClusterConnectionsCount(void) {
     /* We decrement the number of nodes by one, since there is the
      * "myself" node too in the list. Each node uses two file descriptors,
      * one incoming and one outgoing, thus the multiplication by 2. */
-    return g_pserver->cluster_enabled && g_pserver->cluster != nullptr ?
+    return g_pserver->cluster_enabled ?
            ((dictSize(g_pserver->cluster->nodes)-1)*2) : 0;
 }
 
@@ -1525,9 +1525,7 @@ void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) {
                  * it's greater than our view but is not in the future
                  * (with 500 milliseconds tolerance) from the POV of our
                  * clock. */
-                mstime_t mstime;
-                __atomic_load(&g_pserver->mstime, &mstime, __ATOMIC_RELAXED);
-                if (pongtime <= (mstime+500) &&
+                if (pongtime <= (g_pserver->mstime+500) &&
                     pongtime > node->pong_received)
                 {
                     node->pong_received = pongtime;

src/config.cpp

@@ -355,17 +355,16 @@ bool initializeStorageProvider(const char **err)
             return true;
         if (!strcasecmp(g_sdsProvider, "flash") && g_sdsArgs != nullptr)
         {
-#ifdef ENABLE_ROCKSDB
+            // Temporary: Disable FLASH
+            serverLog(LL_WARNING, "FLASH Is Not yet Supported");
+            _Exit(EXIT_FAILURE);
+
+
             // Create The Storage Factory (if necessary)
             serverLog(LL_NOTICE, "Initializing FLASH storage provider (this may take a long time)");
             adjustOpenFilesLimit();
             g_pserver->m_pstorageFactory = CreateRocksDBStorageFactory(g_sdsArgs, cserver.dbnum, cserver.storage_conf, cserver.storage_conf ? strlen(cserver.storage_conf) : 0);
-#else
-	    serverLog(LL_WARNING, "To use the flash storage provider please compile KeyDB with ENABLE_FLASH=yes");
-	    serverLog(LL_WARNING, "Exiting due to the use of an unsupported storage provider");
-	    exit(EXIT_FAILURE);
-#endif
-	}
+        }
         else if (!strcasecmp(g_sdsProvider, "test") && g_sdsArgs == nullptr)
         {
             g_pserver->m_pstorageFactory = new (MALLOC_LOCAL) TestStorageFactory();
@@ -767,12 +766,6 @@ void loadServerConfigFromString(char *config) {
             g_sdsProvider = sdsdup(argv[1]);
             if (argc > 2)
                 g_sdsArgs = sdsdup(argv[2]);
-	} else if (!strcasecmp(argv[0],"is-flash-enabled") && argc == 1) {
-#ifdef ENABLE_ROCKSDB
-            exit(EXIT_SUCCESS);
-#else
-            exit(EXIT_FAILURE);
-#endif
         } else {
             err = "Bad directive or wrong number of arguments"; goto loaderr;
         }
@@ -2909,7 +2902,6 @@ standardConfig configs[] = {
     createBoolConfig("multi-master-no-forward", NULL, MODIFIABLE_CONFIG, cserver.multimaster_no_forward, 0, validateMultiMasterNoForward, NULL),
     createBoolConfig("allow-write-during-load", NULL, MODIFIABLE_CONFIG, g_pserver->fWriteDuringActiveLoad, 0, NULL, NULL),
     createBoolConfig("force-backlog-disk-reserve", NULL, MODIFIABLE_CONFIG, cserver.force_backlog_disk, 0, NULL, NULL),
-    createBoolConfig("soft-shutdown", NULL, MODIFIABLE_CONFIG, g_pserver->config_soft_shutdown, 0, NULL, NULL),
 
 #ifdef USE_OPENSSL
     createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */

src/db.cpp

@@ -1462,11 +1462,6 @@ void shutdownCommand(client *c) {
             flags |= SHUTDOWN_NOSAVE;
         } else if (!strcasecmp(szFromObj(c->argv[1]),"save")) {
             flags |= SHUTDOWN_SAVE;
-        } else if (!strcasecmp(szFromObj(c->argv[1]), "soft")) {
-            g_pserver->soft_shutdown = true;
-            serverLog(LL_WARNING, "Soft Shutdown Initiated");
-            addReply(c, shared.ok);
-            return;
         } else {
             addReplyErrorObject(c,shared.syntaxerr);
             return;
@@ -1573,12 +1568,6 @@ void moveCommand(client *c) {
         return;
     }
 
-    /* Return zero if the key already exists in the target DB */
-    if (lookupKeyWrite(dst,c->argv[1]) != NULL) {
-        addReply(c,shared.czero);
-        return;
-    }
-
     std::unique_ptr<expireEntry> spexpire;
     {   // scope pexpireOld
     std::unique_lock<fastlock> ul(g_expireLock);
@@ -1593,6 +1582,12 @@ void moveCommand(client *c) {
     dbDelete(src,c->argv[1]);
     g_pserver->dirty++;
 
+    /* Return zero if the key already exists in the target DB */
+    if (lookupKeyWrite(dst,c->argv[1]) != NULL) {
+        addReply(c,shared.czero);
+        decrRefCount(o);
+        return;
+    }
     dbAdd(dst,c->argv[1],o);
     if (spexpire != nullptr) setExpire(c,dst,c->argv[1],std::move(*spexpire));
 
@@ -2644,10 +2639,7 @@ bool redisDbPersistentData::insert(char *key, robj *o, bool fAssumeNew, dict_ite
         ensure(key);
     dictEntry *de;
     int res = dictAdd(m_pdict, key, o, &de);
-    if (!FImplies(fAssumeNew, res == DICT_OK)) {
-        serverLog(LL_WARNING,
-            "Assumed new key %s existed in DB.", key);
-    }
+    serverAssert(FImplies(fAssumeNew, res == DICT_OK));
     if (res == DICT_OK)
     {
 #ifdef CHECKED_BUILD
@@ -3300,10 +3292,6 @@ bool redisDbPersistentData::prefetchKeysAsync(client *c, parsed_command &command
     auto cmd = lookupCommand(szFromObj(command.argv[0]));
     if (cmd == nullptr)
         return false; // Bad command? It's not for us to judge, just bail
-    
-    if (command.argc < std::abs(cmd->arity))
-        return false; // Invalid number of args
-    
     int numkeys = getKeysFromCommand(cmd, command.argv, command.argc, &result);
     for (int ikey = 0; ikey < numkeys; ++ikey)
     {

src/evict.cpp

@@ -552,7 +552,7 @@ static int evictionTimeProc(
     UNUSED(clientData);
     serverAssert(GlobalLocksAcquired());
 
-    if (performEvictions(false) == EVICT_RUNNING) return 0;  /* keep evicting */
+    if (performEvictions((bool)clientData) == EVICT_RUNNING) return 0;  /* keep evicting */
 
     /* For EVICT_OK - things are good, no need to keep evicting.
      * For EVICT_FAIL - there is nothing left to evict.  */
@@ -832,12 +832,12 @@ int performEvictions(bool fPreSnapshot) {
                 /* After some time, exit the loop early - even if memory limit
                  * hasn't been reached.  If we suddenly need to free a lot of
                  * memory, don't want to spend too much time here.  */
-                if (g_pserver->m_pstorageFactory == nullptr && elapsedUs(evictionTimer) > eviction_time_limit_us) {
+                if (elapsedUs(evictionTimer) > eviction_time_limit_us) {
                     // We still need to free memory - start eviction timer proc
                     if (!isEvictionProcRunning && serverTL->el != nullptr) {
                         isEvictionProcRunning = 1;
                         aeCreateTimeEvent(serverTL->el, 0,
-                                evictionTimeProc, NULL, NULL);
+                                evictionTimeProc, (void*)fPreSnapshot, NULL);
                     }
                     break;
                 }

src/gc.h

@@ -16,17 +16,7 @@ class GarbageCollector
     struct EpochHolder
     {
         uint64_t tstamp;
-        std::unique_ptr<std::vector<std::unique_ptr<T>>> m_spvecObjs;
-
-        EpochHolder() {
-            m_spvecObjs = std::make_unique<std::vector<std::unique_ptr<T>>>();
-        }
-
-        // Support move operators
-        EpochHolder(EpochHolder &&other) = default;
-        EpochHolder &operator=(EpochHolder &&) = default;
-
-
+        std::vector<std::unique_ptr<T>> m_vecObjs;
 
         bool operator<(uint64_t tstamp) const
         {
@@ -118,12 +108,12 @@ public:
         {
             EpochHolder e;
             e.tstamp = m_epochNext+1;
-            e.m_spvecObjs->push_back(std::move(sp));
+            e.m_vecObjs.push_back(std::move(sp));
             m_listepochs.emplace_back(std::move(e));
         }
         else
         {
-            itr->m_spvecObjs->push_back(std::move(sp));
+            itr->m_vecObjs.push_back(std::move(sp));
         }
     }
 
@@ -133,4 +123,4 @@ private:
     std::list<EpochHolder> m_listepochs;
     std::unordered_set<uint64_t> m_setepochOutstanding;
     uint64_t m_epochNext = 0;
-};
+};

src/help.h

@@ -1071,7 +1071,7 @@ struct commandHelp {
     1,
     "2.2.0" },
     { "SHUTDOWN",
-    "[NOSAVE|SAVE|SOFT]",
+    "[NOSAVE|SAVE]",
     "Synchronously save the dataset to disk and then shut down the server",
     9,
     "1.0.0" },

src/networking.cpp

@@ -1256,20 +1256,6 @@ static void acceptCommonHandler(connection *conn, int flags, char *ip, int iel)
         return;
     }
 
-    /* Prevent new connections if we're in a soft shutdown situation */
-    if (g_pserver->soft_shutdown) {
-        const char *err = "-SHUTDOWN";
-        /* That's a best effort error message, don't check write errors.
-         * Note that for TLS connections, no handshake was done yet so nothing
-         * is written and the connection will just drop. */
-        if (connWrite(conn,err,strlen(err)) == -1) {
-            /* Nothing to do, Just to avoid the warning... */
-        }
-        g_pserver->stat_rejected_conn++;
-        connClose(conn);
-        return;
-    }
-
     /* Limit the number of connections we take at the same time.
      *
      * Admission control will happen before a client is created and connAccept()
@@ -1999,9 +1985,9 @@ void ProcessPendingAsyncWrites()
          * writes may have been signalled without having been copied to the replyAsync buffer,
          * thus causing the buffer to be NULL */ 
         if (c->replyAsync != nullptr){
-            size_t size = c->replyAsync->used;
+            int size = c->replyAsync->used;
 
-            if (listLength(c->reply) == 0 && size <= static_cast<size_t>(PROTO_REPLY_CHUNK_BYTES - c->bufpos)) {
+            if (listLength(c->reply) == 0 && size <= (PROTO_REPLY_CHUNK_BYTES - c->bufpos)) {
                 memcpy(c->buf + c->bufpos, c->replyAsync->buf(), size);
                 c->bufpos += size;
             } else {

src/object.cpp

@@ -1091,7 +1091,7 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
     mem_total += g_pserver->initial_memory_usage;
 
     mem = 0;
-    if (g_pserver->repl_backlog && g_pserver->repl_backlog != g_pserver->repl_backlog_disk)
+    if (g_pserver->repl_backlog)
         mem += zmalloc_size(g_pserver->repl_backlog);
     mh->repl_backlog = mem;
     mem_total += mem;
@@ -1142,7 +1142,8 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
         mem_total+=mem;
         
         std::unique_lock<fastlock> ul(g_expireLock);
-        mem = db->setexpire()->estimated_bytes_used();
+        mem = db->setexpire()->bytes_used();
+        
         mh->db[mh->num_dbs].overhead_ht_expires = mem;
         mem_total+=mem;
 

src/rdb.cpp

@@ -3115,12 +3115,13 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
     std::unique_ptr<rdbInsertJob> spjob;
 
     // If we're tracking changes we need to reset this
-    std::vector<bool> fTracking(cserver.dbnum);
-    // We don't want to track here because processChangesAsync is outside the normal scope handling
-    for (int idb = 0; idb < cserver.dbnum; ++idb) {
-        if ((fTracking[idb] = g_pserver->db[idb]->FTrackingChanges()))
+    bool fTracking = g_pserver->db[0]->FTrackingChanges();
+    if (fTracking) {
+        // We don't want to track here because processChangesAsync is outside the normal scope handling
+        for (int idb = 0; idb < cserver.dbnum; ++idb) {
             if (g_pserver->db[idb]->processChanges(false))
                 g_pserver->db[idb]->commitChanges();
+        }
     }
 
     rdb->update_cksum = rdbLoadProgressCallback;
@@ -3488,10 +3489,11 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
     }
 
     wqueue.endWork();
-    // Reset track changes
-    for (int idb = 0; idb < cserver.dbnum; ++idb) {
-        if (fTracking[idb])
+    if (fTracking) {
+        // Reset track changes
+        for (int idb = 0; idb < cserver.dbnum; ++idb) {
             g_pserver->db[idb]->trackChanges(false);
+        }
     }
     if (empty_keys_skipped) {
         serverLog(LL_WARNING,
@@ -3509,10 +3511,11 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
      * the RDB file from a socket during initial SYNC (diskless replica mode),
      * we'll report the error to the caller, so that we can retry. */
 eoferr:
+    if (fTracking) {
         // Reset track changes
-    for (int idb = 0; idb < cserver.dbnum; ++idb) {
-        if (fTracking[idb])
+        for (int idb = 0; idb < cserver.dbnum; ++idb) {
             g_pserver->db[idb]->trackChanges(false);
+        }
     }
 
     wqueue.endWork();

src/redis-cli.c

@@ -5485,8 +5485,8 @@ static int clusterManagerCommandRebalance(int argc, char **argv) {
             listAddNodeTail(lsrc, src);
             table = clusterManagerComputeReshardTable(lsrc, numslots);
             listRelease(lsrc);
-            int table_len = 0;
-            if (!table || (table_len = (int) listLength(table)) != numslots) {
+            int table_len = (int) listLength(table);
+            if (!table || table_len != numslots) {
                 clusterManagerLogErr("*** Assertion failed: Reshard table "
                                      "!= number of slots");
                 result = 0;

src/redis-cli.h

@@ -98,6 +98,7 @@ extern "C" {
 #define CC_QUIET (1<<1)         /* Don't log connecting errors. */
 
 struct clusterManagerLink;
+typedef struct clusterManagerLink clusterManagerLink;
 
 /* Dict Helpers */
 
@@ -248,12 +249,12 @@ typedef struct clusterManagerReshardTableItem {
 
 /* Info about a cluster internal link. */
 
-struct clusterManagerLink {
+typedef struct clusterManagerLink {
     sds node_name;
     sds node_addr;
     int connected;
     int handshaking;
-};
+} clusterManagerLink;
 
 typedef struct typeinfo {
     char *name;

src/replication.cpp

@@ -190,7 +190,7 @@ int bg_unlink(const char *filename) {
 bool createDiskBacklog() {
     // Lets create some disk backed pages and add them here
     std::string path = "./repl-backlog-temp" + std::to_string(gettid());
-#if (defined __APPLE__ || defined __FreeBSD__)
+#ifdef __APPLE__
     int fd = open(path.c_str(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
 #else
     int fd = open(path.c_str(), O_CREAT | O_RDWR | O_LARGEFILE, S_IRUSR | S_IWUSR);
@@ -1132,14 +1132,6 @@ public:
             replica->repl_put_online_on_ack = 1;
         }
     }
-
-    void abort() {
-        for (auto replica : replicas) {
-            // Close the connection to force a resync
-            freeClientAsync(replica);
-        }
-        replicas.clear();
-    }
 };
 
 int rdbSaveSnapshotForReplication(struct rdbSaveInfo *rsi) {
@@ -1235,11 +1227,7 @@ int rdbSaveSnapshotForReplication(struct rdbSaveInfo *rsi) {
                 retval = C_ERR;
                 break;
             }
-            if (count != snapshotDeclaredCount) {
-                serverLog(LL_WARNING, "Replication BUG: Count of keys sent does not match actual count.  Aborting full sync.");
-                replBuf.abort();
-                break;
-            }
+            serverAssert(count == snapshotDeclaredCount);
         }
         
         replBuf.end();
@@ -2989,11 +2977,10 @@ error:
 }
 
 void readSyncBulkPayload(connection *conn) {
-    serverAssert(GlobalLocksAcquired());
     rdbSaveInfo rsi;
     redisMaster *mi = (redisMaster*)connGetPrivateData(conn);
     static int usemark = 0;
-    if (mi == nullptr || conn != mi->repl_transfer_s) {
+    if (mi == nullptr) {
         // We're about to be free'd so bail out
         return;
     }
@@ -3006,9 +2993,6 @@ void readSyncBulkPayload(connection *conn) {
             return;
     }
 
-    if (conn != mi->repl_transfer_s)
-        return;
-
     /* Final setup of the connected slave <- master link */
     replicationCreateMasterClient(mi,mi->repl_transfer_s,rsi.repl_stream_db);
     if (mi->isRocksdbSnapshotRepl) {
@@ -3817,14 +3801,12 @@ int connectWithMaster(redisMaster *mi) {
  * Never call this function directly, use cancelReplicationHandshake() instead.
  */
 void undoConnectWithMaster(redisMaster *mi) {
-    serverAssert(GlobalLocksAcquired());
     auto conn = mi->repl_transfer_s;
     connSetPrivateData(conn, nullptr);
-    mi->repl_transfer_s = NULL;
-    int result = aePostFunction(g_pserver->rgthreadvar[mi->ielReplTransfer].el, [conn]{
+    aePostFunction(g_pserver->rgthreadvar[mi->ielReplTransfer].el, [conn]{
         connClose(conn);
-    }, false);
-    serverAssert(result == AE_OK);
+    });
+    mi->repl_transfer_s = NULL;
 }
 
 /* Abort the async download of the bulk dataset while SYNC-ing with master.
@@ -3979,10 +3961,6 @@ void freeMasterInfo(redisMaster *mi)
 {
     sdsfree(mi->masterauth);
     zfree(mi->masteruser);
-    if (g_pserver->rdb_filename != nullptr && g_pserver->rdb_filename == mi->repl_transfer_tmpfile) {
-        unlink(g_pserver->rdb_filename);
-        g_pserver->rdb_filename = nullptr;
-    }
     if (mi->repl_transfer_tmpfile)
         zfree(mi->repl_transfer_tmpfile);
     delete mi->staleKeyMap;
@@ -5620,6 +5598,7 @@ void flushReplBacklogToClients()
 
         // Ensure no overflow if we get here
         serverAssert(g_pserver->master_repl_offset - g_pserver->repl_batch_offStart <= g_pserver->repl_backlog_size);
+        serverAssert(g_pserver->repl_batch_idxStart != g_pserver->repl_backlog_idx);
 
         // Repl backlog writes must become visible to all threads at this point
         std::atomic_thread_fence(std::memory_order_release);
@@ -5651,10 +5630,7 @@ void flushReplBacklogToClients()
             replica->repl_end_off = g_pserver->master_repl_offset;
 
             /* Only if the there isn't already a pending write do we prepare the client to write */
-            if (replica->repl_curr_off == g_pserver->master_repl_offset) {
-                serverLog(LL_DEBUG, "Pending write when it's on repl_offset=%lld", g_pserver->master_repl_offset);
-                continue;
-            }
+            serverAssert(replica->repl_curr_off != g_pserver->master_repl_offset);
             prepareClientToWrite(replica);
         }
         if (fAsyncWrite)

src/semiorderedset.h

@@ -240,12 +240,14 @@ public:
     bool empty() const noexcept { return celem == 0; }
     size_t size() const noexcept { return celem; }
 
-    size_t estimated_bytes_used() const
+    size_t bytes_used() const
     {
-        // This estimate does't include all the overhead of the internal vectors
-        size_t cb = sizeof(this)
-            + (m_data.capacity() * sizeof(m_data[0]))
-            + sizeof(T) * size();
+        size_t cb = sizeof(this) + (m_data.capacity()-m_data.size())*sizeof(T);
+        for (auto &vec : m_data)
+        {
+            if (vec != nullptr)
+                cb += vec->bytes_used();
+        }
         return cb;
     }
 

src/server.cpp

@@ -779,7 +779,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"shutdown",shutdownCommand,-1,
-     "admin no-script ok-loading ok-stale noprop",
+     "admin no-script ok-loading ok-stale",
      0,NULL,0,0,0,0,0,0},
 
     {"lastsave",lastsaveCommand,1,
@@ -2648,26 +2648,6 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
         }
     }
 
-    if (g_pserver->soft_shutdown) {
-        /* Loop through our clients list and see if there are any active clients */
-        listIter li;
-        listNode *ln;
-        listRewind(g_pserver->clients, &li);
-        bool fActiveClient = false;
-        while ((ln = listNext(&li)) && !fActiveClient) {
-            client *c = (client*)listNodeValue(ln);
-            if (c->flags & CLIENT_IGNORE_SOFT_SHUTDOWN)
-                continue;
-            fActiveClient = true;
-        }
-        if (!fActiveClient) {
-            if (prepareForShutdown(SHUTDOWN_NOFLAGS) == C_OK) {
-                serverLog(LL_WARNING, "All active clients have disconnected while a soft shutdown is pending.  Shutting down now.");
-                throw ShutdownException();
-            }
-        }
-    }
-
     g_pserver->cronloops++;
     return 1000/g_pserver->hz;
 }
@@ -2795,8 +2775,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     AeLocker locker;
     int iel = ielFromEventLoop(eventLoop);
 
-    tlsProcessPendingData();
-
     locker.arm();
 
     /* end any snapshots created by fast async commands */
@@ -2827,6 +2805,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
         uint64_t processed = 0;
         int aof_state = g_pserver->aof_state;
         locker.disarm();
+        processed += tlsProcessPendingData();
         processed += handleClientsWithPendingWrites(iel, aof_state);
         locker.arm();
         processed += freeClientsInAsyncFreeQueue(iel);
@@ -2837,6 +2816,13 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     /* Handle precise timeouts of blocked clients. */
     handleBlockedClientsTimeout();
 
+    /* Handle TLS pending data. (must be done before flushAppendOnlyFile) */
+    if (tlsHasPendingData()) {
+        locker.disarm();
+        tlsProcessPendingData();
+        locker.arm();
+    }
+
     /* If tls still has pending unread data don't sleep at all. */
     aeSetDontWait(eventLoop, tlsHasPendingData());
 
@@ -2902,7 +2888,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     static thread_local bool fFirstRun = true;
     // note: we also copy the DB pointer in case a DB swap is done while the lock is released
     std::vector<redisDb*> vecdb;    // note we cache the database pointer in case a dbswap is done while the lock is released
-    if (cserver.storage_memory_model == STORAGE_WRITETHROUGH && !g_pserver->loading)
+    if (cserver.storage_memory_model == STORAGE_WRITETHROUGH && g_pserver->m_pstorageFactory != nullptr && !g_pserver->loading)
     {
         if (!fFirstRun) {
             mstime_t storage_process_latency;
@@ -3015,11 +3001,8 @@ void afterSleep(struct aeEventLoop *eventLoop) {
 
         serverAssert(serverTL->gcEpoch.isReset());
         serverTL->gcEpoch = g_pserver->garbageCollector.startEpoch();
-
-        aeAcquireLock();
         for (int idb = 0; idb < cserver.dbnum; ++idb)
             g_pserver->db[idb]->trackChanges(false);
-        aeReleaseLock();
 
         serverTL->disable_async_commands = false;
     }
@@ -5070,8 +5053,6 @@ bool client::asyncCommand(std::function<void(const redisDbPersistentDataSnapshot
                             std::function<void(const redisDbPersistentDataSnapshot *)> &&postFn) 
 {
     serverAssert(FCorrectThread(this));
-    if (serverTL->in_eval)
-        return false;   // we cannot block clients in EVAL
     const redisDbPersistentDataSnapshot *snapshot = nullptr;
     if (!(this->flags & (CLIENT_MULTI | CLIENT_BLOCKED)))
         snapshot = this->db->createSnapshot(this->mvccCheckpoint, false /* fOptional */);
@@ -5316,11 +5297,6 @@ void pingCommand(client *c) {
         return;
     }
 
-    if (g_pserver->soft_shutdown && !(c->flags & CLIENT_IGNORE_SOFT_SHUTDOWN)) {
-        addReplyError(c, "-SHUTDOWN PENDING");
-        return;
-    }
-
     if (c->flags & CLIENT_PUBSUB && c->resp == 2) {
         addReply(c,shared.mbulkhdr[2]);
         addReplyBulkCBuffer(c,"pong",4);
@@ -6715,7 +6691,7 @@ static void sigShutdownHandler(int sig) {
      * If we receive the signal the second time, we interpret this as
      * the user really wanting to quit ASAP without waiting to persist
      * on disk. */
-    if ((g_pserver->shutdown_asap || g_pserver->soft_shutdown) && sig == SIGINT) {
+    if (g_pserver->shutdown_asap && sig == SIGINT) {
         serverLogFromHandler(LL_WARNING, "You insist... exiting now.");
         rdbRemoveTempFile(g_pserver->rdbThreadVars.tmpfileNum, 1);
         g_pserver->garbageCollector.shutdown();
@@ -6726,10 +6702,7 @@ static void sigShutdownHandler(int sig) {
     }
 
     serverLogFromHandler(LL_WARNING, msg);
-    if (g_pserver->config_soft_shutdown)
-        g_pserver->soft_shutdown = true;
-    else
-        g_pserver->shutdown_asap = 1;
+    g_pserver->shutdown_asap = 1;
 }
 
 void setupSignalHandlers(void) {

src/server.h

@@ -508,7 +508,6 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define CLIENT_PREVENT_AOF_PROP (1<<19)  /* Don't propagate to AOF. */
 #define CLIENT_PREVENT_REPL_PROP (1<<20)  /* Don't propagate to slaves. */
 #define CLIENT_PREVENT_PROP (CLIENT_PREVENT_AOF_PROP|CLIENT_PREVENT_REPL_PROP)
-#define CLIENT_IGNORE_SOFT_SHUTDOWN (CLIENT_MASTER | CLIENT_SLAVE | CLIENT_BLOCKED | CLIENT_MONITOR)
 #define CLIENT_PENDING_WRITE (1<<21) /* Client has output to send but a write
                                         handler is yet not installed. */
 #define CLIENT_REPLY_OFF (1<<22)   /* Don't send replies to client. */
@@ -2722,9 +2721,6 @@ struct redisServer {
     long long repl_batch_offStart = -1;
     long long repl_batch_idxStart = -1;
 
-    int config_soft_shutdown = false;
-    bool soft_shutdown = false;
-
     /* Lock Contention Ring Buffer */
     static const size_t s_lockContentionSamples = 64;
     uint16_t rglockSamples[s_lockContentionSamples];

src/storage/rocksdb.cpp

@@ -1,234 +0,0 @@
-#include "rocksdb.h"
-#include <string>
-#include <sstream>
-#include <mutex>
-#include <unistd.h>
-#include "rocksdbfactor_internal.h"
-
-static const char keyprefix[] = INTERNAL_KEY_PREFIX;
-
-rocksdb::Options DefaultRocksDBOptions();
-extern "C" pid_t gettid();
-
-bool FInternalKey(const char *key, size_t cch)
-{
-    if (cch >= sizeof(INTERNAL_KEY_PREFIX))
-    {
-        if (memcmp(key, keyprefix, sizeof(INTERNAL_KEY_PREFIX)-1) == 0)
-            return true;
-    }
-    return false;
-}
-
-RocksDBStorageProvider::RocksDBStorageProvider(RocksDBStorageFactory *pfactory, std::shared_ptr<rocksdb::DB> &spdb, std::shared_ptr<rocksdb::ColumnFamilyHandle> &spcolfam, const rocksdb::Snapshot *psnapshot, size_t count)
-    : m_pfactory(pfactory), m_spdb(spdb), m_psnapshot(psnapshot), m_spcolfamily(spcolfam), m_count(count)
-{
-    m_readOptionsTemplate = rocksdb::ReadOptions();
-    m_readOptionsTemplate.verify_checksums = false;
-    m_readOptionsTemplate.snapshot = m_psnapshot;
-}
-
-void RocksDBStorageProvider::insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwrite)
-{
-    rocksdb::Status status;
-    std::unique_lock<fastlock> l(m_lock);
-    if (m_spbatch != nullptr)
-        status = m_spbatch->Put(m_spcolfamily.get(), rocksdb::Slice(key, cchKey), rocksdb::Slice((const char*)data, cb));
-    else
-        status = m_spdb->Put(WriteOptions(), m_spcolfamily.get(), rocksdb::Slice(key, cchKey), rocksdb::Slice((const char*)data, cb));
-    if (!status.ok())
-        throw status.ToString();
-
-    if (!fOverwrite)
-        ++m_count;
-}
-
-void RocksDBStorageProvider::bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem)
-{
-    if (celem >= 16384) {
-        rocksdb::Options options = DefaultRocksDBOptions();
-        rocksdb::SstFileWriter sst_file_writer(rocksdb::EnvOptions(), options, options.comparator);
-        std::string file_path = m_pfactory->getTempFolder() + "/tmpIngest.";
-        file_path += std::to_string(gettid());
-        file_path += ".sst";
-
-        rocksdb::Status s = sst_file_writer.Open(file_path);
-        if (!s.ok())
-            goto LFallback;
-
-        // Insert rows into the SST file, note that inserted keys must be 
-        // strictly increasing (based on options.comparator)
-        for (size_t ielem = 0; ielem < celem; ++ielem) {
-            s = sst_file_writer.Put(rocksdb::Slice(rgkeys[ielem], rgcbkeys[ielem]), rocksdb::Slice(rgvals[ielem], rgcbvals[ielem]));
-            if (!s.ok()) {
-                unlink(file_path.c_str());
-                goto LFallback;
-            }
-        }
-
-        s = sst_file_writer.Finish();
-        if (!s.ok()) {
-            unlink(file_path.c_str());
-            goto LFallback;
-        }
-
-        auto ingestOptions = rocksdb::IngestExternalFileOptions();
-        ingestOptions.move_files = true;
-        ingestOptions.write_global_seqno = false;
-        ingestOptions.failed_move_fall_back_to_copy = false;
-
-        // Ingest the external SST file into the DB
-        s = m_spdb->IngestExternalFile(m_spcolfamily.get(), {file_path}, ingestOptions);
-        if (!s.ok()) {
-            unlink(file_path.c_str());
-            goto LFallback;
-        }
-    } else {
-    LFallback:
-        auto spbatch = std::make_unique<rocksdb::WriteBatch>();
-        for (size_t ielem = 0; ielem < celem; ++ielem) {
-            spbatch->Put(m_spcolfamily.get(), rocksdb::Slice(rgkeys[ielem], rgcbkeys[ielem]), rocksdb::Slice(rgvals[ielem], rgcbvals[ielem]));
-        }
-        m_spdb->Write(WriteOptions(), spbatch.get());
-    }
-
-    std::unique_lock<fastlock> l(m_lock);
-    m_count += celem;
-}
-
-bool RocksDBStorageProvider::erase(const char *key, size_t cchKey)
-{
-    rocksdb::Status status;
-    std::unique_lock<fastlock> l(m_lock);
-    if (m_spbatch != nullptr)
-    {
-        status = m_spbatch->Delete(m_spcolfamily.get(), rocksdb::Slice(key, cchKey));
-    }
-    else
-    {
-        if (!FKeyExists(key, cchKey))
-            return false;
-        status = m_spdb->Delete(WriteOptions(), m_spcolfamily.get(), rocksdb::Slice(key, cchKey));
-    }
-    if (status.ok())
-        --m_count;
-    return status.ok();
-}
-
-void RocksDBStorageProvider::retrieve(const char *key, size_t cchKey, callbackSingle fn) const
-{
-    rocksdb::PinnableSlice slice;
-    auto status = m_spdb->Get(ReadOptions(), m_spcolfamily.get(), rocksdb::Slice(key, cchKey), &slice);
-    if (status.ok())
-        fn(key, cchKey, slice.data(), slice.size());
-}
-
-size_t RocksDBStorageProvider::clear()
-{
-    size_t celem = count();
-    auto status = m_spdb->DropColumnFamily(m_spcolfamily.get());
-    auto strName = m_spcolfamily->GetName();
-
-    rocksdb::ColumnFamilyHandle *handle = nullptr;
-    rocksdb::ColumnFamilyOptions cf_options(m_pfactory->RocksDbOptions());
-    m_spdb->CreateColumnFamily(cf_options, strName, &handle);
-    m_spcolfamily = std::shared_ptr<rocksdb::ColumnFamilyHandle>(handle);
-
-    if (!status.ok())
-        throw status.ToString();
-    m_count = 0;
-    return celem;
-}
-
-size_t RocksDBStorageProvider::count() const
-{
-    std::unique_lock<fastlock> l(m_lock);
-    return m_count;
-}
-
-bool RocksDBStorageProvider::enumerate(callback fn) const
-{
-    std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(ReadOptions(), m_spcolfamily.get()));
-    size_t count = 0;
-    for (it->SeekToFirst(); it->Valid(); it->Next()) {
-        if (FInternalKey(it->key().data(), it->key().size()))
-            continue;
-        ++count;
-        bool fContinue = fn(it->key().data(), it->key().size(), it->value().data(), it->value().size());
-        if (!fContinue)
-            break;
-    }
-    if (!it->Valid() && count != m_count)
-    {
-        if (const_cast<RocksDBStorageProvider*>(this)->m_count != count)
-            printf("WARNING: rocksdb count mismatch");
-        const_cast<RocksDBStorageProvider*>(this)->m_count = count;
-    }
-    assert(it->status().ok()); // Check for any errors found during the scan
-    return !it->Valid();
-}
-
-const IStorage *RocksDBStorageProvider::clone() const
-{
-    std::unique_lock<fastlock> l(m_lock);
-    const rocksdb::Snapshot *psnapshot = const_cast<RocksDBStorageProvider*>(this)->m_spdb->GetSnapshot();
-    return new RocksDBStorageProvider(m_pfactory, const_cast<RocksDBStorageProvider*>(this)->m_spdb, const_cast<RocksDBStorageProvider*>(this)->m_spcolfamily, psnapshot, m_count);
-}
-
-RocksDBStorageProvider::~RocksDBStorageProvider()
-{
-    if (m_spbatch != nullptr)
-        endWriteBatch();
-    
-    if (m_spdb != nullptr && m_psnapshot == nullptr)
-    {
-        insert(count_key, sizeof(count_key), &m_count, sizeof(m_count), false);
-        flush();
-    }
-
-    if (m_spdb != nullptr)
-    {
-        if (m_psnapshot != nullptr)
-            m_spdb->ReleaseSnapshot(m_psnapshot);
-    }
-}
-
-rocksdb::WriteOptions RocksDBStorageProvider::WriteOptions() const
-{
-    auto opt = rocksdb::WriteOptions();
-    return opt;
-}
-
-void RocksDBStorageProvider::beginWriteBatch()
-{
-    m_lock.lock();
-    m_spbatch = std::make_unique<rocksdb::WriteBatch>();
-}
-
-void RocksDBStorageProvider::endWriteBatch()
-{
-    m_spdb->Write(WriteOptions(), m_spbatch.get());
-    m_spbatch = nullptr;
-    m_lock.unlock();
-}
-
-void RocksDBStorageProvider::batch_lock()
-{
-    m_lock.lock();
-}
-
-void RocksDBStorageProvider::batch_unlock()
-{
-    m_lock.unlock();
-}
-
-void RocksDBStorageProvider::flush()
-{
-    m_spdb->SyncWAL();
-}
-
-bool RocksDBStorageProvider::FKeyExists(const char *key, size_t cch) const
-{
-    rocksdb::PinnableSlice slice;
-    return m_spdb->Get(ReadOptions(), m_spcolfamily.get(), rocksdb::Slice(key, cch), &slice).ok();
-}

src/storage/rocksdb.h

@@ -1,56 +0,0 @@
-#pragma once
-
-#include <memory>
-#include "../IStorage.h"
-#include <rocksdb/db.h>
-#include "../fastlock.h"
-
-#define INTERNAL_KEY_PREFIX "\x00\x04\x03\x00\x05\x02\x04"
-static const char count_key[] = INTERNAL_KEY_PREFIX "__keydb__count\1";
-static const char version_key[] = INTERNAL_KEY_PREFIX "__keydb__version\1";
-static const char meta_key[] = INTERNAL_KEY_PREFIX "__keydb__metadata\1";
-class RocksDBStorageFactory;
-
-class RocksDBStorageProvider : public IStorage
-{
-    RocksDBStorageFactory *m_pfactory;
-    std::shared_ptr<rocksdb::DB> m_spdb;    // Note: This must be first so it is deleted last
-    std::unique_ptr<rocksdb::WriteBatch> m_spbatch;
-    const rocksdb::Snapshot *m_psnapshot = nullptr;
-    std::shared_ptr<rocksdb::ColumnFamilyHandle> m_spcolfamily;
-    rocksdb::ReadOptions m_readOptionsTemplate;
-    size_t m_count = 0;
-    mutable fastlock m_lock {"RocksDBStorageProvider"};
-
-public:
-    RocksDBStorageProvider(RocksDBStorageFactory *pfactory, std::shared_ptr<rocksdb::DB> &spdb, std::shared_ptr<rocksdb::ColumnFamilyHandle> &spcolfam, const rocksdb::Snapshot *psnapshot, size_t count);
-    ~RocksDBStorageProvider();
-
-    virtual void insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwrite) override;
-    virtual bool erase(const char *key, size_t cchKey) override;
-    virtual void retrieve(const char *key, size_t cchKey, callbackSingle fn) const override;
-    virtual size_t clear() override;
-    virtual bool enumerate(callback fn) const override;
-
-    virtual const IStorage *clone() const override;
-
-    virtual void beginWriteBatch() override;
-    virtual void endWriteBatch() override;
-
-    virtual void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem) override;
-
-    virtual void batch_lock() override;
-    virtual void batch_unlock() override;
-
-    virtual void flush() override;
-
-    size_t count() const override;
-
-protected:
-    bool FKeyExists(const char *key, size_t cchKey) const;
-
-    const rocksdb::ReadOptions &ReadOptions() const { return m_readOptionsTemplate; }
-    rocksdb::WriteOptions WriteOptions() const;
-};
-
-bool FInternalKey(const char *key, size_t cch);

src/storage/rocksdbfactory.cpp

@@ -1,204 +0,0 @@
-#include "rocksdb.h"
-#include "../version.h"
-#include <rocksdb/filter_policy.h>
-#include <rocksdb/table.h>
-#include <rocksdb/utilities/options_util.h>
-#include <rocksdb/sst_file_manager.h>
-#include <rocksdb/utilities/convenience.h>
-#include <rocksdb/slice_transform.h>
-#include "rocksdbfactor_internal.h"
-#include <sys/types.h>
-#include <sys/stat.h> 
-
-rocksdb::Options DefaultRocksDBOptions() {
-    rocksdb::Options options;
-    options.max_background_compactions = 4;
-    options.max_background_flushes = 2;
-    options.bytes_per_sync = 1048576;
-    options.compaction_pri = rocksdb::kMinOverlappingRatio;
-    options.compression = rocksdb::kNoCompression;
-    options.enable_pipelined_write = true;
-    options.allow_mmap_reads = true;
-    options.avoid_unnecessary_blocking_io = true;
-    options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(0));
-
-    rocksdb::BlockBasedTableOptions table_options;
-    table_options.block_size = 16 * 1024;
-    table_options.cache_index_and_filter_blocks = true;
-    table_options.pin_l0_filter_and_index_blocks_in_cache = true;
-    table_options.data_block_index_type = rocksdb::BlockBasedTableOptions::kDataBlockBinaryAndHash;
-    table_options.checksum = rocksdb::kNoChecksum;
-    table_options.format_version = 4;
-    options.table_factory.reset(NewBlockBasedTableFactory(table_options));
-    
-    return options;
-}
-
-IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const char *rgchConfig, size_t cchConfig)
-{
-    return new RocksDBStorageFactory(path, dbnum, rgchConfig, cchConfig);
-}
-
-rocksdb::Options RocksDBStorageFactory::RocksDbOptions()
-{
-    rocksdb::Options options = DefaultRocksDBOptions();
-    options.max_open_files = filedsRequired();
-    options.sst_file_manager = m_pfilemanager;
-    options.create_if_missing = true;
-    options.create_missing_column_families = true;
-    options.info_log_level = rocksdb::ERROR_LEVEL;
-    options.max_total_wal_size = 1 * 1024 * 1024 * 1024;
-    return options;
-}
-
-RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig)
-    : m_path(dbfile)
-{
-    dbnum++; // create an extra db for metadata
-    // Get the count of column families in the actual database
-    std::vector<std::string> vecT;
-    auto status = rocksdb::DB::ListColumnFamilies(rocksdb::Options(), dbfile, &vecT);
-    // RocksDB requires we know the count of col families before opening, if the user only wants to see less
-    //  we still have to make room for all column family handles regardless
-    if (status.ok() && (int)vecT.size() > dbnum)
-        dbnum = (int)vecT.size();
-
-    std::vector<rocksdb::ColumnFamilyDescriptor> veccoldesc;
-    veccoldesc.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions()));  // ignore default col family
-
-    m_pfilemanager = std::shared_ptr<rocksdb::SstFileManager>(rocksdb::NewSstFileManager(rocksdb::Env::Default()));
-
-    rocksdb::DB *db = nullptr;
-
-    auto options = RocksDbOptions();
-
-    for (int idb = 0; idb < dbnum; ++idb)
-    {
-        rocksdb::ColumnFamilyOptions cf_options(options);
-        cf_options.level_compaction_dynamic_level_bytes = true;
-        veccoldesc.push_back(rocksdb::ColumnFamilyDescriptor(std::to_string(idb), cf_options));
-    }
-
-    if (rgchConfig != nullptr)
-    {
-        std::string options_string(rgchConfig, cchConfig);
-        rocksdb::Status status;
-        if (!(status = rocksdb::GetDBOptionsFromString(options, options_string, &options)).ok())
-        {
-            fprintf(stderr, "Failed to parse FLASH options: %s\r\n", status.ToString().c_str());
-            exit(EXIT_FAILURE);
-        }
-    }
-    
-    std::vector<rocksdb::ColumnFamilyHandle*> handles;
-    status = rocksdb::DB::Open(options, dbfile, veccoldesc, &handles, &db);
-    if (!status.ok())
-        throw status.ToString();
-
-    m_spdb = std::shared_ptr<rocksdb::DB>(db);
-    for (auto handle : handles)
-    {
-        std::string strVersion;
-        auto status = m_spdb->Get(rocksdb::ReadOptions(), handle, rocksdb::Slice(version_key, sizeof(version_key)), &strVersion);
-        if (!status.ok())
-        {
-            setVersion(handle);
-        }
-        else
-        {
-            SymVer ver = parseVersion(strVersion.c_str());
-            auto cmp = compareVersion(&ver);
-            if (cmp == NewerVersion)
-                throw "Cannot load FLASH database created by newer version of KeyDB";
-            if (cmp == OlderVersion)
-                setVersion(handle);
-        }
-        m_vecspcols.emplace_back(handle);
-    }   
-}
-
-RocksDBStorageFactory::~RocksDBStorageFactory()
-{
-    m_spdb->SyncWAL();
-}
-
-void RocksDBStorageFactory::setVersion(rocksdb::ColumnFamilyHandle *handle)
-{
-    auto status = m_spdb->Put(rocksdb::WriteOptions(), handle, rocksdb::Slice(version_key, sizeof(version_key)), rocksdb::Slice(KEYDB_REAL_VERSION, strlen(KEYDB_REAL_VERSION)+1));
-    if (!status.ok())
-        throw status.ToString();
-}
-
-size_t RocksDBStorageFactory::filedsRequired() const {
-    return 256;
-}
-
-std::string RocksDBStorageFactory::getTempFolder()
-{
-    auto path = m_path + "/keydb_tmp/";
-    if (!m_fCreatedTempFolder) {
-        if (!mkdir(path.c_str(), 0700))
-            m_fCreatedTempFolder = true;
-    }
-    return path;
-}
-
-IStorage *RocksDBStorageFactory::createMetadataDb()
-{
-    IStorage *metadataDb = this->create(-1, nullptr, nullptr);
-    metadataDb->insert(meta_key, sizeof(meta_key), (void*)METADATA_DB_IDENTIFIER, strlen(METADATA_DB_IDENTIFIER), false);
-    return metadataDb;
-}
-
-IStorage *RocksDBStorageFactory::create(int db, key_load_iterator iter, void *privdata)
-{
-    ++db;   // skip default col family
-    std::shared_ptr<rocksdb::ColumnFamilyHandle> spcolfamily(m_vecspcols[db].release());
-    size_t count = 0;
-    bool fUnclean = false;
-    
-    std::string value;
-    auto status = m_spdb->Get(rocksdb::ReadOptions(), spcolfamily.get(), rocksdb::Slice(count_key, sizeof(count_key)), &value);
-    if (status.ok() && value.size() == sizeof(size_t))
-    {
-        count = *reinterpret_cast<const size_t*>(value.data());
-        m_spdb->Delete(rocksdb::WriteOptions(), spcolfamily.get(), rocksdb::Slice(count_key, sizeof(count_key)));
-    }
-    else
-    {
-        fUnclean = true;
-    }
-    
-    if (fUnclean || iter != nullptr)
-    {
-        count = 0;
-        auto opts = rocksdb::ReadOptions();
-        opts.tailing = true;
-        std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(opts, spcolfamily.get()));
-
-        it->SeekToFirst();
-        bool fFirstRealKey = true;
-        
-        for (;it->Valid(); it->Next()) {
-            if (FInternalKey(it->key().data(), it->key().size()))
-                continue;
-            if (fUnclean && it->Valid() && fFirstRealKey)
-                printf("\tDatabase %d was not shutdown cleanly, recomputing metrics\n", db);
-            fFirstRealKey = false;
-            if (iter != nullptr)
-                iter(it->key().data(), it->key().size(), privdata);
-            ++count;
-        }
-    }
-    return new RocksDBStorageProvider(this, m_spdb, spcolfamily, nullptr, count);
-}
-
-const char *RocksDBStorageFactory::name() const
-{
-    return "flash";
-}
-
-size_t RocksDBStorageFactory::totalDiskspaceUsed() const
-{
-    return m_pfilemanager->GetTotalSize();
-}

src/storage/teststorageprovider.cpp

@@ -26,10 +26,12 @@ TestStorageProvider::~TestStorageProvider()
 {
 }
 
-void TestStorageProvider::insert(const char *key, size_t cchKey, void *data, size_t cb, bool /*fOverwrite*/)
+void TestStorageProvider::insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwrite)
 {
     auto strkey = std::string(key, cchKey);
-    m_map[strkey] = std::string((char*)data, cb);
+    bool fActuallyExists = m_map.find(strkey) != m_map.end();
+    serverAssert(fActuallyExists == fOverwrite);
+    m_map.insert(std::make_pair(strkey, std::string((char*)data, cb)));
 }
 
 

src/t_stream.cpp

@@ -789,13 +789,13 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) {
              * update it after (and if) we actually remove the entry */
             unsigned char *pcopy = p;
 
-            int64_t flags = lpGetInteger(p);
+            int flags = lpGetInteger(p);
             p = lpNext(lp, p); /* Skip flags. */
-            int64_t to_skip;
+            int to_skip;
 
-            int64_t ms_delta = lpGetInteger(p);
+            int ms_delta = lpGetInteger(p);
             p = lpNext(lp, p); /* Skip ID ms delta */
-            int64_t seq_delta = lpGetInteger(p);
+            int seq_delta = lpGetInteger(p);
             p = lpNext(lp, p); /* Skip ID seq delta */
 
             streamID currid = {0}; /* For MINID */
@@ -1125,7 +1125,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
              * the first entry emitted for this listpack, then we already
              * emitted the current entry, and have to go back to the previous
              * one. */
-            int64_t lp_count = lpGetInteger(si->lp_ele);
+            int lp_count = lpGetInteger(si->lp_ele);
             while(lp_count--) si->lp_ele = lpPrev(si->lp,si->lp_ele);
             /* Seek lp-count of prev entry. */
             si->lp_ele = lpPrev(si->lp,si->lp_ele);
@@ -1155,7 +1155,7 @@ int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) {
 
             /* Get the flags entry. */
             si->lp_flags = si->lp_ele;
-            int64_t flags = lpGetInteger(si->lp_ele);
+            int flags = lpGetInteger(si->lp_ele);
             si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek ID. */
 
             /* Get the ID: it is encoded as difference between the master
@@ -1264,7 +1264,7 @@ void streamIteratorRemoveEntry(streamIterator *si, streamID *current) {
      * deleted entries in the listpack header.
      *
      * We start flagging: */
-    int64_t flags = lpGetInteger(si->lp_flags);
+    int flags = lpGetInteger(si->lp_flags);
     flags |= STREAM_ITEM_FLAG_DELETED;
     lp = lpReplaceInteger(lp,&si->lp_flags,flags);
 

src/tls.cpp

@@ -46,9 +46,6 @@
 
 #include <sys/stat.h>
 #include <arpa/inet.h>
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L
-#include <openssl/decoder.h>
-#endif
 
 #define REDIS_TLS_PROTO_TLSv1       (1<<0)
 #define REDIS_TLS_PROTO_TLSv1_1     (1<<1)
@@ -157,13 +154,14 @@ void tlsInit(void) {
      */
     #if OPENSSL_VERSION_NUMBER < 0x10100000L
     OPENSSL_config(NULL);
-    SSL_load_error_strings();
-    SSL_library_init();
     #elif OPENSSL_VERSION_NUMBER < 0x10101000L
     OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL);
     #else
     OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG|OPENSSL_INIT_ATFORK, NULL);
     #endif
+    ERR_load_crypto_strings();
+    SSL_load_error_strings();
+    SSL_library_init();
 
 #ifdef USE_CRYPTO_LOCKS
     initCryptoLocks();
@@ -366,46 +364,20 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
     if (ctx_config->prefer_server_ciphers)
         SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE);
 
-#if ((OPENSSL_VERSION_NUMBER < 0x30000000L) && defined(SSL_CTX_set_ecdh_auto))
+#if defined(SSL_CTX_set_ecdh_auto)
     SSL_CTX_set_ecdh_auto(ctx, 1);
 #endif
     SSL_CTX_set_options(ctx, SSL_OP_SINGLE_DH_USE);
 
     if (ctx_config->dh_params_file) {
         FILE *dhfile = fopen(ctx_config->dh_params_file, "r");
+        DH *dh = NULL;
         if (!dhfile) {
             serverLog(LL_WARNING, "Failed to load %s: %s", ctx_config->dh_params_file, strerror(errno));
             goto error;
         }
 
-#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
-        EVP_PKEY *pkey = NULL;
-        OSSL_DECODER_CTX *dctx = OSSL_DECODER_CTX_new_for_pkey(
-                &pkey, "PEM", NULL, "DH", OSSL_KEYMGMT_SELECT_DOMAIN_PARAMETERS, NULL, NULL);
-        if (!dctx) {
-            serverLog(LL_WARNING, "No decoder for DH params.");
-            fclose(dhfile);
-            goto error;
-        }
-        if (!OSSL_DECODER_from_fp(dctx, dhfile)) {
-            serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file);
-            OSSL_DECODER_CTX_free(dctx);
-            fclose(dhfile);
-            goto error;
-        }
-
-        OSSL_DECODER_CTX_free(dctx);
-        fclose(dhfile);
-
-        if (SSL_CTX_set0_tmp_dh_pkey(ctx, pkey) <= 0) {
-            ERR_error_string_n(ERR_get_error(), errbuf, sizeof(errbuf));
-            serverLog(LL_WARNING, "Failed to load DH params file: %s: %s", ctx_config->dh_params_file, errbuf);
-            EVP_PKEY_free(pkey);
-            goto error;
-        }
-        /* Not freeing pkey, it is owned by OpenSSL now */
-#else
-        DH *dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL);
+        dh = PEM_read_DHparams(dhfile, NULL, NULL, NULL);
         fclose(dhfile);
         if (!dh) {
             serverLog(LL_WARNING, "%s: failed to read DH params.", ctx_config->dh_params_file);
@@ -420,11 +392,6 @@ int tlsConfigure(redisTLSContextConfig *ctx_config) {
         }
 
         DH_free(dh);
-#endif
-    } else {
-#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
-        SSL_CTX_set_dh_auto(ctx, 1);
-#endif
     }
 
     /* If a client-side certificate is configured, create an explicit client context */
@@ -536,26 +503,11 @@ bool tlsCheckAgainstAllowlist(const char * client){
 #define ASN1_STRING_get0_data ASN1_STRING_data 
 #endif 
 
-class TCleanup {
-    std::function<void()> fn;
-
-public:
-    TCleanup(std::function<void()> fn)
-        : fn(fn)
-    {}
-
-    ~TCleanup() {
-        fn();
-    }
-};
-
 bool tlsValidateCertificateName(tls_connection* conn){
     if (g_pserver->tls_allowlist.empty())
         return true;    // Empty list implies acceptance of all
 
     X509 * cert = SSL_get_peer_certificate(conn->ssl);
-    TCleanup certClen([cert]{X509_free(cert);});
-    
     /* Check the common name (CN) of the certificate first */
     X509_NAME_ENTRY * ne = X509_NAME_get_entry(X509_get_subject_name(cert), X509_NAME_get_index_by_NID(X509_get_subject_name(cert), NID_commonName, -1));
     const char * commonName = reinterpret_cast<const char*>(ASN1_STRING_get0_data(X509_NAME_ENTRY_get_data(ne)));

tests/integration/psync2-reg-multimaster.tcl

@@ -53,12 +53,12 @@ start_server {overrides {active-replica {yes} multi-master {yes} client-output-b
         set elapsed [expr {[clock milliseconds]-$cycle_start_time}]
         if {$elapsed > $duration*1000} break
         if {rand() < .05} {
-            test "PSYNC2 #3899 regression (multi-master): kill first replica" {
+            test "PSYNC2 #3899 regression: kill first replica" {
                 $R(1) client kill type master
             }
         }
         if {rand() < .05} {
-            test "PSYNC2 #3899 regression (multi-master): kill chained replica" {
+            test "PSYNC2 #3899 regression: kill chained replica" {
                 $R(2) client kill type master
             }
         }
@@ -99,8 +99,4 @@ start_server {overrides {active-replica {yes} multi-master {yes} client-output-b
             fail [format "The three instances have different data sets:\n\tnode 1: %s\n\tnode 2: %s\n\tnode 3: %s\nRun diff -u against /tmp/repldump*.txt for more info" [$R(0) debug digest] [$R(1) debug digest] [$R(2) debug digest] ]
         }
     }
-
-    assert {[s -2 sync_partial_ok] > 0}
-    assert {[s -1 sync_partial_ok] > 0}
-    assert {[s 0 sync_partial_ok] > 0}
 }}}

tests/integration/replication-fast.tcl

@@ -6,7 +6,6 @@ proc prepare_value {size} {
     return $_v
 }
 
-if {$::flash_enabled} {
 start_server {tags {"replication-fast"} overrides {storage-provider {flash ./rocks.db.master} databases 256}} {
     set slave [srv 0 client]
     set slave_host [srv 0 host]
@@ -33,4 +32,3 @@ start_server {tags {"replication-fast"} overrides {storage-provider {flash ./roc
         }
     }
 }
-}

tests/integration/replication-psync-flash.tcl

@@ -8,7 +8,6 @@
 # If reconnect is > 0, the test actually try to break the connection and
 # reconnect with the master, otherwise just the initial synchronization is
 # checked for consistency.
-if {$::flash_enabled} {
 proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
     start_server [list tags {"repl"} overrides [list storage-provider {flash .rocks.db.m} repl-backlog-size 64m]] {
         start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {
@@ -134,4 +133,3 @@ foreach mdl {no yes} {
         } $mdl $sdl 1
     }
 }
-}

tests/test_helper.tcl

@@ -47,7 +47,6 @@ set ::all_tests {
     integration/replication-3
     integration/replication-4
     integration/replication-psync
-    integration/replication-psync-flash
     integration/replication-active
     integration/replication-multimaster
     integration/replication-multimaster-connect
@@ -60,13 +59,11 @@ set ::all_tests {
     integration/failover
     integration/keydb-cli
     integration/keydb-benchmark
-    integration/replication-fast
     integration/replication-psync-multimaster
     unit/pubsub
     unit/slowlog
     unit/scripting
     unit/maxmemory
-    unit/flash
     unit/introspection
     unit/introspection-2
     unit/limits
@@ -89,7 +86,6 @@ set ::all_tests {
     integration/logging
     integration/corrupt-dump
     integration/corrupt-dump-fuzzer
-    unit/soft_shutdown
 }
 # Index to the next test to run in the ::all_tests list.
 set ::next_test 0
@@ -739,16 +735,6 @@ for {set j 0} {$j < [llength $argv]} {incr j} {
     }
 }
 
-# Check if we compiled with flash
-set status [catch {exec src/keydb-server --is-flash-enabled}]
-if {$status == 0} {
-    puts "KeyDB was built with FLASH, including FLASH tests"
-    set ::flash_enabled 1
-} else {
-    puts "KeyDB was not built with FLASH.  Excluding FLASH tests"
-    set ::flash_enabled 0
-}
-
 set filtered_tests {}
 
 # Set the filtered tests to be the short list (single_tests) if exists.

tests/unit/flash.tcl

@@ -1,4 +1,3 @@
-if {$::flash_enabled} {
 start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {
 
     test { FLASH - GET works after eviction } {
@@ -168,4 +167,4 @@ start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.
     }
 
 }
-}
+

tests/unit/memefficiency.tcl

@@ -49,13 +49,10 @@ start_server {tags {"defrag"} overrides {appendonly yes auto-aof-rewrite-percent
             r config set maxmemory 100mb
             r config set maxmemory-policy allkeys-lru
 
-	    for {set i 0} {$i < 10} {incr i} {
-                populate 700000 asdf1 150
-                populate 170000 asdf2 300
-                after 120 ;# serverCron only updates the info once in 100ms
-                set frag [s allocator_frag_ratio]
-		if {$frag >= 1.4} {break}
-	    }
+            populate 700000 asdf1 150
+            populate 170000 asdf2 300
+            after 120 ;# serverCron only updates the info once in 100ms
+            set frag [s allocator_frag_ratio]
             if {$::verbose} {
                 puts "frag $frag"
             }
@@ -510,14 +507,11 @@ start_server {tags {"defrag"} overrides {server-threads 1 active-replica yes} }
             r config set active-defrag-ignore-bytes 2mb
             r config set maxmemory 100mb
             r config set maxmemory-policy allkeys-lru
-            for {set i 0} {$i < 10} {incr i} {
-	        populate 700000 asdf1 150
-                populate 170000 asdf2 300
-                r ping ;# trigger eviction following the previous population
-                after 120 ;# serverCron only updates the info once in 100ms
-                set frag [s allocator_frag_ratio]
-                if {$frag >= 1.4} {break}
-            }
+            populate 700000 asdf1 150
+            populate 170000 asdf2 300
+            r ping ;# trigger eviction following the previous population
+            after 120 ;# serverCron only updates the info once in 100ms
+            set frag [s allocator_frag_ratio]
             if {$::verbose} {
                 puts "frag $frag"
             }

tests/unit/scripting.tcl

@@ -126,10 +126,6 @@ start_server {tags {"scripting"}} {
         r select 9
         set res
     } {original value}
-    
-    test {EVAL background command} {
-	r eval {redis.call("SCAN", "0", "MATCH", "key*", "COUNT", 1000)} 0
-    } {}
 
     if 0 {
         test {EVAL - Script can't run more than configured time limit} {

tests/unit/soft_shutdown.tcl

@@ -1,141 +0,0 @@
-start_server {tags {"soft_shutdown"} } {
-    test {soft shutdown command replies} {
-        assert_equal [r shutdown soft] "OK"
-    }
-
-    test {soft shutdown errors on ping} {
-        catch {[r ping]} e
-        assert_match {SHUTDOWN PENDING} $e
-    }
-}
-
-start_server {tags {"soft_shutdown"} } {
-    test {soft shutdown prevents new connections} {
-        assert_equal [r shutdown soft] "OK"
-        # reconnect
-        set catch_res [catch {set rd [redis_deferring_client]} e]
-        if {$::tls} {
-            assert_equal $catch_res 1
-        } else {
-            assert_match {*SHUTDOWN*} $e
-        }
-    }
-}
-
-start_server {tags {"soft_shutdown"} } {
-    test {soft shutdown allows commands to execute while waiting} {
-        assert_equal [r shutdown soft] "OK"
-        r set test val
-        assert_equal [r get test] {val}
-    }
-}
-
-start_server {tags {"soft_shutdown"} } {
-    test {soft shutdown shuts down after all clients exit} {
-        assert_equal [r shutdown soft] "OK"
-        r close
-        after 500
-        catch {set rd [redis_deferring_client]} e
-        assert_match {*refused*} $e
-    }
-}
-
-start_server {tags {"soft_shutdown"} overrides {soft-shutdown yes} } {
-    test {soft shutdown triggered by SIGINT} {
-        exec kill -SIGINT [s process_id]
-        after 100
-        catch {[r ping]} e
-        assert_match {SHUTDOWN PENDING} $e
-    }
-
-    test {second SIGINT forces a shutdown during a soft shutdown} {
-        exec kill -SIGINT [s process_id]
-        after 100
-        catch {[r ping]} e
-        assert_match {*I/O*} $e
-    }
-}
-
-start_server {tags {"soft_shutdown"} } {
-    test {monitor does not prevent soft shutdown} {
-        set monitor [redis_deferring_client]
-        $monitor monitor
-        assert_equal [r shutdown soft] "OK"
-        r close
-        after 500
-        catch {set rd [redis_deferring_client]} e
-        assert_match {*refused*} $e
-    }
-}
-
-start_server {tags {"soft_shutdown"} } {
-    start_server {} {
-        set node_0 [srv 0 client]
-        set node_0_host [srv 0 host]
-        set node_0_port [srv 0 port]
-        set node_0_pid [srv 0 pid]
-
-        set node_1 [srv -1 client]
-        set node_1_host [srv -1 host]
-        set node_1_port [srv -1 port]
-        set node_1_pid [srv -1 pid]
-
-        $node_0 replicaof $node_1_host $node_1_port
-        wait_for_sync $node_0
-
-        test {soft shutdown works for with master} {
-            $node_1 shutdown soft
-        } {OK}
-
-        test {soft shutdown on master doesn't affect replica} {
-            assert_equal [$node_0 ping] {PONG}
-        }
-
-        test {soft shutdown on master updates ping response} {
-            catch {$node_1 ping} e
-            assert_equal $e {SHUTDOWN PENDING}
-        }
-
-        test {master prevents new connections with soft shutdown} {
-            set c1 [redis $node_1_host $node_1_port 1 $::tls]
-            set catch_res [catch {$c1 read} e]
-            if {$::tls} {
-                assert_equal $catch_res 1
-            } else {
-                assert_match {*SHUTDOWN*} $e
-            }
-        }
-
-        test {master soft shutdown works after all clients disconnect} {
-            $node_1 close
-            after 500
-            catch {set c1 [redis $node_1_host $node_1_port 1 $::tls]} e
-            assert_match {*refused*} $e
-        }
-    }
-}
-
-start_server {tags {"soft_shutdown"} } {
-    start_server {} {
-        set node_0 [srv 0 client]
-        set node_0_host [srv 0 host]
-        set node_0_port [srv 0 port]
-        set node_0_pid [srv 0 pid]
-
-        set node_1 [srv -1 client]
-        set node_1_host [srv -1 host]
-        set node_1_port [srv -1 port]
-        set node_1_pid [srv -1 pid]
-
-        $node_0 replicaof $node_1_host $node_1_port
-        wait_for_sync $node_0
-
-        test {soft shutdown on replica is not blocked by master} {
-            assert_equal [$node_0 shutdown soft] {OK}
-            $node_0 close
-            after 500
-            catch {set c0 [redis $node_0_host $node_0_port 1 $::tls]} e
-            assert_match {*refused*} $e
-        }
-    }
-}

tests/unit/type/stream.tcl

@@ -172,15 +172,6 @@ start_server {
         assert_equal [r XRANGE mystream - +] {{3-0 {f v}} {4-0 {f v}} {5-0 {f v}}}
     }
 
-    test {XTRIM with MINID option, big delta from master record} {
-        r DEL mystream
-        r XADD mystream 1-0 f v
-        r XADD mystream 1641544570597-0 f v
-        r XADD mystream 1641544570597-1 f v
-        r XTRIM mystream MINID 1641544570597-0
-        assert_equal [r XRANGE mystream - +] {{1641544570597-0 {f v}} {1641544570597-1 {f v}}}
-    }
-
     test {XADD mass insertion and XLEN} {
         r DEL mystream
         r multi

utils/redis-copy.rb

@@ -1,27 +1,27 @@
-# redis-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo
+# keydb-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo
 # BSD license, See the COPYING file for more information.
 #
-# Copy the whole dataset from one KeyDB/Redis instance to another one
+# Copy the whole dataset from one Redis instance to another one
 #
 # WARNING: this utility is deprecated and serves as a legacy adapter
-#          for the more-robust redis-copy gem.
+#          for the more-robust keydb-copy gem.
 
 require 'shellwords'
 
 def redisCopy(opts={})
   src = "#{opts[:srchost]}:#{opts[:srcport]}"
   dst = "#{opts[:dsthost]}:#{opts[:dstport]}"
-  `redis-copy #{src.shellescape} #{dst.shellescape}`
+  `keydb-copy #{src.shellescape} #{dst.shellescape}`
 rescue Errno::ENOENT
-  $stderr.puts 'This utility requires the redis-copy executable',
-               'from the redis-copy gem on https://rubygems.org',
-               'To install it, run `gem install redis-copy`.'
+  $stderr.puts 'This utility requires the keydb-copy executable',
+               'from the keydb-copy gem on https://rubygems.org',
+               'To install it, run `gem install keydb-copy`.'
   exit 1
 end
 
-$stderr.puts "This utility is deprecated. Use the redis-copy gem instead."
+$stderr.puts "This utility is deprecated. Use the keydb-copy gem instead."
 if ARGV.length != 4
-    puts "Usage: redis-copy.rb <srchost> <srcport> <dsthost> <dstport>"
+    puts "Usage: keydb-copy.rb <srchost> <srcport> <dsthost> <dstport>"
     exit 1
 end
 puts "WARNING: it's up to you to FLUSHDB the destination host before to continue, press any key when ready."