check ziplist len to avoid crash on empty ziplist convert

Change min tested version to 18.04

.github/ISSUE_TEMPLATE/question.md

@@ -1,21 +0,0 @@
----
-name: Question
-about: Ask the Redis developers
-title: '[QUESTION]'
-labels: ''
-assignees: ''
-
----
-
-Please keep in mind that this issue tracker should be used for reporting bugs or proposing improvements to the Redis server.
-
-Generally, questions about using Redis should be directed to the [community](https://redis.io/community):
-
-* [the mailing list](https://groups.google.com/forum/#!forum/redis-db)
-* [the `redis` tag at StackOverflow](http://stackoverflow.com/questions/tagged/redis)
-* [/r/redis subreddit](http://www.reddit.com/r/redis)
-* [the irc channel #redis](http://webchat.freenode.net/?channels=redis) on freenode
-
-It is also possible that your question was already asked here, so please do a quick issues search before submitting. Lastly, if your question is about one of Redis' [clients](https://redis.io/clients), you may to contact your client's developers for help.
-
-That said, please feel free to replace all this with your question :)

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
           ./runtest-moduleapi
         
   build-ubuntu-old:
-    runs-on: ubuntu-16.04
+    runs-on: ubuntu-18.04
     steps:
     - uses: actions/checkout@v1
     - name: make -j2
@@ -56,3 +56,14 @@ jobs:
           sudo apt-get -y install uuid-dev libcurl4-openssl-dev
           make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' MALLOC=libc -j2
 
+  build-ubuntu-32bit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: make
+      run: |
+          sudo dpkg --add-architecture i386
+          sudo apt-get update
+          sudo apt-get -y install gcc-multilib g++-multilib libc6-dev-i386 lib32z1 uuid-dev:i386 libcurl4-openssl-dev:i386
+          make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' 32bit -j2
+

.gitignore

@@ -59,4 +59,4 @@ Makefile.dep
 .ccls
 .ccls-cache/*
 compile_commands.json
-redis.code-workspace
+keydb.code-workspace

00-RELEASENOTES

@@ -11,6 +11,166 @@ CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP.
 SECURITY: There are security fixes in the release.
 --------------------------------------------------------------------------------
 
+================================================================================
+Redis 6.2.6 Released Mon Oct 4 12:00:00 IDT 2021
+================================================================================
+
+Upgrade urgency: SECURITY, contains fixes to security issues.
+
+Security Fixes:
+* (CVE-2021-41099) Integer to heap buffer overflow handling certain string
+  commands and network payloads, when proto-max-bulk-len is manually configured
+  to a non-default, very large value [reported by yiyuaner].
+* (CVE-2021-32762) Integer to heap buffer overflow issue in redis-cli and
+  redis-sentinel parsing large multi-bulk replies on some older and less common
+  platforms [reported by Microsoft Vulnerability Research].
+* (CVE-2021-32687) Integer to heap buffer overflow with intsets, when
+  set-max-intset-entries is manually configured to a non-default, very large
+  value [reported by Pawel Wieczorkiewicz, AWS].
+* (CVE-2021-32675) Denial Of Service when processing RESP request payloads with
+  a large number of elements on many connections.
+* (CVE-2021-32672) Random heap reading issue with Lua Debugger [reported by
+  Meir Shpilraien].
+* (CVE-2021-32628) Integer to heap buffer overflow handling ziplist-encoded
+  data types, when configuring a large, non-default value for
+  hash-max-ziplist-entries, hash-max-ziplist-value, zset-max-ziplist-entries
+  or zset-max-ziplist-value [reported by sundb].
+* (CVE-2021-32627) Integer to heap buffer overflow issue with streams, when
+  configuring a non-default, large value for proto-max-bulk-len and
+  client-query-buffer-limit [reported by sundb].
+* (CVE-2021-32626) Specially crafted Lua scripts may result with Heap buffer
+  overflow [reported by Meir Shpilraien].
+
+Bug fixes that involve behavior changes:
+* GEO* STORE with empty source key deletes the destination key and return 0 (#9271)
+  Previously it would have returned an empty array like the non-STORE variant.
+* PUBSUB NUMPAT replies with number of patterns rather than number of subscriptions (#9209)
+  This actually changed in 6.2.0 but was overlooked and omitted from the release notes.
+
+Bug fixes that are only applicable to previous releases of Redis 6.2:
+* Fix CLIENT PAUSE, used an old timeout from previous PAUSE (#9477)
+* Fix CLIENT PAUSE in a replica would mess the replication offset (#9448)
+* Add some missing error statistics in INFO errorstats (#9328)
+
+Other bug fixes:
+* Fix incorrect reply of COMMAND command key positions for MIGRATE command (#9455)
+* Fix appendfsync to always guarantee fsync before reply, on MacOS and FreeBSD (kqueue) (#9416)
+* Fix the wrong mis-detection of sync_file_range system call, affecting performance (#9371)
+
+CLI tools:
+* When redis-cli received ASK response, it didn't handle it (#8930)
+
+Improvements:
+* Add latency monitor sample when key is deleted via lazy expire (#9317)
+* Sanitize corrupt payload improvements (#9321, #9399)
+* Delete empty keys when loading RDB file or handling a RESTORE command (#9297, #9349)
+
+================================================================================
+Redis 6.2.5 Released Wed Jul 21 16:32:19 IDT 2021
+================================================================================
+
+Upgrade urgency: SECURITY, contains fixes to security issues that affect
+authenticated client connections on 32-bit versions. MODERATE otherwise.
+
+Fix integer overflow in BITFIELD on 32-bit versions (CVE-2021-32761).
+An integer overflow bug in Redis version 2.2 or newer can be exploited using the
+BITFIELD command to corrupt the heap and potentially result with remote code
+execution.
+
+Bug fixes that involve behavior changes:
+* Change reply type for ZPOPMAX/MIN with count in RESP3 to nested array (#8981).
+  Was using a flat array like in RESP2 instead of a nested array like ZRANGE does.
+* Fix reply type for HRANDFIELD and ZRANDMEMBER when key is missing (#9178).
+  Was using a null array instead of an empty array.
+* Fix reply type for ZRANGESTORE when source key is missing (#9089).
+  Was using an empty array like ZRANGE instead of 0 (used in the STORE variant).
+
+Bug fixes that are only applicable to previous releases of Redis 6.2:
+* ZRANDMEMBER WITHSCORES with negative COUNT may return bad score (#9162)
+* Fix crash after CLIENT UNPAUSE when threaded I/O config is enabled (#9041)
+* Fix XTRIM or XADD with LIMIT may delete more entries than the limit (#9048)
+* Fix build issue with OpenSSL 1.1.0 (#9233)
+
+Other bug fixes:
+* Fail EXEC command in case a watched key is expired (#9194)
+* Fix SMOVE not to invalidate dest key (WATCH and tracking) when member already exists (#9244)
+* Fix SINTERSTORE not to delete dest key when getting a wrong type error (#9032)
+* Fix overflows on 32-bit versions in GETBIT, SETBIT, BITCOUNT, BITPOS, and BITFIELD (#9191)
+* Improve MEMORY USAGE on stream keys (#9164)
+* Set TCP keepalive on inbound cluster bus connections (#9230)
+* Fix diskless replica loading to recover from RDB short read on module AUX data (#9199)
+* Fix race in client side tracking (#9116)
+* Fix ziplist length updates on big-endian platforms (#2080)
+
+CLI tools:
+* redis-cli cluster import command may issue wrong MIGRATE command, sending COPY instead of REPLACE (#8945)
+* redis-cli --rdb fixes when using "-" to write to stdout (#9136, #9135)
+* redis-cli support for RESP3 set type in CSV and RAW output (#7338)
+
+Modules:
+* Module API for getting current command name (#8792)
+* Fix RM_StringTruncate when newlen is 0 (#3718)
+* Fix CLIENT UNBLOCK crashing modules without timeout callback (#9167)
+
+================================================================================
+Redis 6.2.4 Released Tue June 1 12:00:00 IST 2021
+================================================================================
+
+Upgrade urgency: SECURITY, Contains fixes to security issues that affect
+authenticated client connections. MODERATE otherwise.
+
+Fix integer overflow in STRALGO LCS (CVE-2021-32625)
+An integer overflow bug in Redis version 6.0 or newer can be exploited using the
+STRALGO LCS command to corrupt the heap and potentially result with remote code
+execution. This is a result of an incomplete fix by CVE-2021-29477.
+
+Bug fixes that are only applicable to previous releases of Redis 6.2:
+* Fix crash after a diskless replication fork child is terminated (#8991)
+* Fix redis-benchmark crash on unsupported configs (#8916)
+
+Other bug fixes:
+* Fix crash in UNLINK on a stream key with deleted consumer groups (#8932)
+* SINTERSTORE: Add missing keyspace del event when none of the sources exist (#8949)
+* Sentinel: Fix CONFIG SET of empty string sentinel-user/sentinel-pass configs (#8958)
+* Enforce client output buffer soft limit when no traffic (#8833)
+
+Improvements:
+* Hide AUTH passwords in MIGRATE command from slowlog (#8859)
+
+================================================================================
+Redis 6.2.3 Released Mon May 3 19:00:00 IST 2021
+================================================================================
+
+Upgrade urgency: SECURITY, Contains fixes to security issues that affect
+authenticated client connections. LOW otherwise.
+
+Integer overflow in STRALGO LCS command (CVE-2021-29477):
+An integer overflow bug in Redis version 6.0 or newer could be exploited using
+the STRALGO LCS command to corrupt the heap and potentially result in remote
+code execution. The integer overflow bug exists in all versions of Redis
+starting with 6.0.
+
+Integer overflow in COPY command for large intsets (CVE-2021-29478):
+An integer overflow bug in Redis 6.2 could be exploited to corrupt the heap and
+potentially result with remote code execution. The vulnerability involves
+changing the default set-max-intset-entries configuration value, creating a
+large set key that consists of integer values and using the COPY command to
+duplicate it. The integer overflow bug exists in all versions of Redis starting
+with 2.6, where it could result with a corrupted RDB or DUMP payload, but not
+exploited through COPY (which did not exist before 6.2).
+
+Bug fixes that are only applicable to previous releases of Redis 6.2:
+* Fix memory leak in moduleDefragGlobals (#8853)
+* Fix memory leak when doing lazy freeing client tracking table (#8822)
+* Block abusive replicas from sending command that could assert and crash redis (#8868)
+
+Other bug fixes:
+* Use a monotonic clock to check for Lua script timeout (#8812)
+* redis-cli: Do not use unix socket when we got redirected in cluster mode (#8870)
+
+Modules:
+* Fix RM_GetClusterNodeInfo() to correctly populate master id (#8846)
+
 ================================================================================
 Redis 6.2.2 Released Mon April 19 19:00:00 IST 2021
 ================================================================================

README.md

@@ -33,19 +33,13 @@ Because of this difference of opinion features which are right for KeyDB may not
 Additional Resources
 --------------------
 
-Try our docker container: https://hub.docker.com/r/eqalpha/keydb
+Check out KeyDB's [Docker Image](https://hub.docker.com/r/eqalpha/keydb)
 
-Talk on Gitter: https://gitter.im/KeyDB
+Join us on [Slack](https://docs.keydb.dev/slack/)
 
-Visit our Website: https://keydb.dev
+Post to the [Community Forum](https://community.keydb.dev)
 
-See options for channel partners and support contracts: https://keydb.dev/support.html
-
-Learn with KeyDB’s official documentation site: https://docs.keydb.dev
-
-[Subscribe to the KeyDB mailing list](https://eqalpha.us20.list-manage.com/subscribe/post?u=978f486c2f95589b24591a9cc&id=4ab9220500)
-
-Management GUI: We recommend [FastoNoSQL](https://fastonosql.com/) which has official KeyDB support.
+Learn more through KeyDB's [Documentation & Learning Center](https://docs.keydb.dev)
 
 
 Benchmarking KeyDB
@@ -179,7 +173,7 @@ To compile against jemalloc on Mac OS X systems, use:
 Monotonic clock
 ---------------
 
-By default, Redis will build using the POSIX clock_gettime function as the
+By default, KeyDB will build using the POSIX clock_gettime function as the
 monotonic clock source.  On most modern systems, the internal processor clock
 can be used to improve performance.  Cautions can be found here: 
     http://oliveryang.net/2015/09/pitfalls-of-TSC-usage/

TLS.md

@@ -28,8 +28,8 @@ To manually run a Redis server with TLS mode (assuming `gen-test-certs.sh` was
 invoked so sample certificates/keys are available):
 
     ./src/keydb-server --tls-port 6379 --port 0 \
-        --tls-cert-file ./tests/tls/keydb.crt \
-        --tls-key-file ./tests/tls/keydb.key \
+        --tls-cert-file ./tests/tls/client.crt \
+        --tls-key-file ./tests/tls/client.key \
         --tls-ca-cert-file ./tests/tls/ca.crt
 
 To connect to this Redis server with `keydb-cli`:

deps/hiredis/Makefile

@@ -77,7 +77,12 @@ ifeq ($(USE_SSL),1)
 endif
 
 ifeq ($(uname_S),Linux)
-  SSL_LDFLAGS=-lssl -lcrypto
+  ifdef OPENSSL_PREFIX
+    CFLAGS+=-I$(OPENSSL_PREFIX)/include
+    SSL_LDFLAGS+=-L$(OPENSSL_PREFIX)/lib -lssl -lcrypto
+  else
+    SSL_LDFLAGS=-lssl -lcrypto
+  endif
 else
   OPENSSL_PREFIX?=/usr/local/opt/openssl
   CFLAGS+=-I$(OPENSSL_PREFIX)/include

deps/hiredis/hiredis.c

@@ -174,6 +174,7 @@ static void *createArrayObject(const redisReadTask *task, size_t elements) {
         return NULL;
 
     if (elements > 0) {
+        if (SIZE_MAX / sizeof(redisReply*) < elements) return NULL;  /* Don't overflow */
         r->element = hi_calloc(elements,sizeof(redisReply*));
         if (r->element == NULL) {
             freeReplyObject(r);

deps/hiredis/test.c

@@ -498,6 +498,20 @@ static void test_reply_reader(void) {
     freeReplyObject(reply);
     redisReaderFree(reader);
 
+    test("Multi-bulk never overflows regardless of maxelements: ");
+    size_t bad_mbulk_len = (SIZE_MAX / sizeof(void *)) + 3;
+    char bad_mbulk_reply[100];
+    snprintf(bad_mbulk_reply, sizeof(bad_mbulk_reply), "*%llu\r\n+asdf\r\n",
+        (unsigned long long) bad_mbulk_len);
+
+    reader = redisReaderCreate();
+    reader->maxelements = 0;    /* Don't rely on default limit */
+    redisReaderFeed(reader, bad_mbulk_reply, strlen(bad_mbulk_reply));
+    ret = redisReaderGetReply(reader,&reply);
+    test_cond(ret == REDIS_ERR && strcasecmp(reader->errstr, "Out of memory") == 0);
+    freeReplyObject(reply);
+    redisReaderFree(reader);
+
 #if LLONG_MAX > SIZE_MAX
     test("Set error when array > SIZE_MAX: ");
     reader = redisReaderCreate();

deps/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h

@@ -241,7 +241,7 @@ iget_defrag_hint(tsdn_t *tsdn, void* ptr) {
 			int free_in_slab = extent_nfree_get(slab);
 			if (free_in_slab) {
 				const bin_info_t *bin_info = &bin_infos[binind];
-				int curslabs = binshard->stats.curslabs;
+				ssize_t curslabs = binshard->stats.curslabs;
 				size_t curregs = binshard->stats.curregs;
 				if (binshard->slabcur) {
 					/* remove slabcur from the overall utilization */

keydb.conf

@@ -32,8 +32,17 @@
 # If instead you are interested in using includes to override configuration
 # options, it is better to use include as the last line.
 #
+# Included paths may contain wildcards. All files matching the wildcards will
+# be included in alphabetical order.
+# Note that if an include path contains a wildcards but no files match it when
+# the server is started, the include statement will be ignored and no error will
+# be emitted.  It is safe, therefore, to include wildcard files from empty
+# directories.
+#
 # include /path/to/local.conf
 # include /path/to/other.conf
+# include /path/to/fragments/*.conf
+#
 
 ################################## MODULES #####################################
 
@@ -49,23 +58,32 @@
 # for connections from all available network interfaces on the host machine.
 # It is possible to listen to just one or multiple selected interfaces using
 # the "bind" configuration directive, followed by one or more IP addresses.
+# Each address can be prefixed by "-", which means that redis will not fail to
+# start if the address is not available. Being not available only refers to
+# addresses that does not correspond to any network interfece. Addresses that
+# are already in use will always fail, and unsupported protocols will always BE
+# silently skipped.
 #
 # Examples:
 #
-# bind 192.168.1.100 10.0.0.1
-# bind 127.0.0.1 ::1
+# bind 192.168.1.100 10.0.0.1     # listens on two specific IPv4 addresses
+# bind 127.0.0.1 ::1              # listens on loopback IPv4 and IPv6
+# bind * -::*                     # like the default, all available interfaces
 #
 # ~~~ WARNING ~~~ If the computer running KeyDB is directly exposed to the
 # internet, binding to all the interfaces is dangerous and will expose the
 # instance to everybody on the internet. So by default we uncomment the
 # following bind directive, that will force KeyDB to listen only on the
-# IPv4 loopback interface address (this means KeyDB will only be able to
+# IPv4 and IPv6 (if available) loopback interface addresses (this means KeyDB will only be able to
 # accept client connections from the same host that it is running on).
 #
 # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES
 # JUST COMMENT OUT THE FOLLOWING LINE.
+#
+# You will also need to set a password unless you explicitly disable protected
+# mode.
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-bind 127.0.0.1
+bind 127.0.0.1 -::1
 
 # Protected mode is a layer of security protection, in order to avoid that
 # KeyDB instances left open on the internet are accessed and exploited.
@@ -125,7 +143,7 @@ timeout 0
 # On other kernels the period depends on the kernel configuration.
 #
 # A reasonable value for this option is 300 seconds, which is the new
-# KeyDB default starting with Redis 3.2.1.
+# KeyDB default starting with KeyDB 3.2.1.
 tcp-keepalive 300
 
 ################################# TLS/SSL #####################################
@@ -141,15 +159,37 @@ tcp-keepalive 300
 # server to connected clients, masters or cluster peers.  These files should be
 # PEM formatted.
 #
-# tls-cert-file redis.crt 
-# tls-key-file redis.key
+# tls-cert-file keydb.crt 
+# tls-key-file keydb.key
+#
+# If the key file is encrypted using a passphrase, it can be included here
+# as well.
+#
+# tls-key-file-pass secret
+
+# Normally KeyDB uses the same certificate for both server functions (accepting
+# connections) and client functions (replicating from a master, establishing
+# cluster bus connections, etc.).
+#
+# Sometimes certificates are issued with attributes that designate them as
+# client-only or server-only certificates. In that case it may be desired to use
+# different certificates for incoming (server) and outgoing (client)
+# connections. To do that, use the following directives:
+#
+# tls-client-cert-file client.crt
+# tls-client-key-file client.key
+#
+# If the key file is encrypted using a passphrase, it can be included here
+# as well.
+#
+# tls-client-key-file-pass secret
 
 # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange:
 #
-# tls-dh-params-file redis.dh
+# tls-dh-params-file keydb.dh
 
 # Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL
-# clients and peers.  Redis requires an explicit configuration of at least one
+# clients and peers. KeyDB requires an explicit configuration of at least one
 # of these, and will not implicitly use the system wide configuration.
 #
 # tls-ca-cert-file ca.crt
@@ -172,7 +212,7 @@ tcp-keepalive 300
 #
 # tls-replication yes
 
-# By default, the Redis Cluster bus uses a plain TCP connection. To enable
+# By default, the KeyDB Cluster bus uses a plain TCP connection. To enable
 # TLS for the bus protocol, use the following directive:
 #
 # tls-cluster yes
@@ -269,6 +309,16 @@ logfile ""
 # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7.
 # syslog-facility local0
 
+# To disable the built in crash log, which will possibly produce cleaner core
+# dumps when they are needed, uncomment the following:
+#
+# crash-log-enabled no
+
+# To disable the fast memory check that's run as part of the crash log, which
+# will possibly let keydb terminate sooner, uncomment the following:
+#
+# crash-memcheck-enabled no
+
 # Set the number of databases. The default database is DB 0, you can select
 # a different one on a per-connection basis using SELECT <dbid> where
 # dbid is a number between 0 and 'databases'-1
@@ -282,9 +332,31 @@ databases 16
 # ASCII art logo in startup logs by setting the following option to yes.
 always-show-logo yes
 
+# By default, KeyDB modifies the process title (as seen in 'top' and 'ps') to
+# provide some runtime information. It is possible to disable this and leave
+# the process name as executed by setting the following to no.
+set-proc-title yes
+
 # Retrieving "message of today" using CURL requests.
 #enable-motd yes
 
+# When changing the process title, KeyDB uses the following template to construct
+# the modified title.
+#
+# Template variables are specified in curly brackets. The following variables are
+# supported:
+#
+# {title}           Name of process as executed if parent, or type of child process.
+# {listen-addr}     Bind address or '*' followed by TCP or TLS port listening on, or
+#                   Unix socket if only that's available.
+# {server-mode}     Special mode, i.e. "[sentinel]" or "[cluster]".
+# {port}            TCP port listening on, or 0.
+# {tls-port}        TLS port listening on, or 0.
+# {unixsocket}      Unix domain socket listening on, or "".
+# {config-file}     Name of configuration file used.
+#
+proc-title-template "{title} {listen-addr} {server-mode}"
+
 ################################ SNAPSHOTTING  ################################
 #
 # Save the DB on disk:
@@ -299,8 +371,6 @@ always-show-logo yes
 #   after 300 sec (5 min) if at least 10 keys changed
 #   after 60 sec if at least 10000 keys changed
 #
-#   Note: you can disable saving completely by commenting out all "save" lines.
-#
 #   It is also possible to remove all the previously configured save
 #   points by adding a save directive with a single empty string argument
 #   like in the following example:
@@ -341,6 +411,21 @@ rdbcompression yes
 # tell the loading code to skip the check.
 rdbchecksum yes
 
+# Enables or disables full sanitation checks for ziplist and listpack etc when
+# loading an RDB or RESTORE payload. This reduces the chances of a assertion or
+# crash later on while processing commands.
+# Options:
+#   no         - Never perform full sanitation
+#   yes        - Always perform full sanitation
+#   clients    - Perform full sanitation only for user connections.
+#                Excludes: RDB files, RESTORE commands received from the master
+#                connection, and client connections which have the
+#                skip-sanitize-payload ACL flag.
+# The default should be 'clients' but since it currently affects cluster
+# resharding via MIGRATE, it is temporarily set to 'no' by default.
+#
+# sanitize-dump-payload no
+
 # The filename where to dump the DB
 dbfilename dump.rdb
 
@@ -397,7 +482,7 @@ dir ./
 #
 # masterauth <master-password>
 #
-# However this is not enough if you are using KeyDB ACLs (for Redis version
+# However this is not enough if you are using KeyDB ACLs (for KeyDB version
 # 6 or greater), and the default user is not capable of running the PSYNC
 # command and/or other commands needed for replication (gathered in the
 # @replication group). In this case it's better to configure a special user to
@@ -443,7 +528,7 @@ replica-serve-stale-data yes
 # may also cause problems if clients are writing to it because of a
 # misconfiguration.
 #
-# Since Redis 2.6 by default replicas are read-only.
+# Since KeyDB 2.6 by default replicas are read-only.
 #
 # Note: read only replicas are not designed to be exposed to untrusted clients
 # on the internet. It's just a protection layer against misuse of the instance.
@@ -595,6 +680,18 @@ repl-disable-tcp-nodelay no
 # By default the priority is 100.
 replica-priority 100
 
+# -----------------------------------------------------------------------------
+# By default, KeyDB Sentinel includes all replicas in its reports. A replica
+# can be excluded from KeyDB Sentinel's announcements. An unannounced replica
+# will be ignored by the 'sentinel replicas <master>' command and won't be
+# exposed to KeyDB Sentinel's clients.
+#
+# This option does not change the behavior of replica-priority. Even with
+# replica-announced set to 'no', the replica can be promoted to master. To
+# prevent this behavior, set replica-priority to 0.
+#
+# replica-announced yes
+
 # It is possible for a master to stop accepting writes if there are less than
 # N replicas connected, having a lag less or equal than M seconds.
 #
@@ -714,6 +811,8 @@ replica-priority 100
 #  off          Disable the user: it's no longer possible to authenticate
 #               with this user, however the already authenticated connections
 #               will still work.
+#  skip-sanitize-payload    RESTORE dump-payload sanitation is skipped.
+#  sanitize-payload         RESTORE dump-payload is sanitized (default).
 #  +<command>   Allow the execution of that command
 #  -<command>   Disallow the execution of that command
 #  +@<category> Allow the execution of all the commands in such category
@@ -736,6 +835,11 @@ replica-priority 100
 #               It is possible to specify multiple patterns.
 #  allkeys      Alias for ~*
 #  resetkeys    Flush the list of allowed keys patterns.
+#  &<pattern>   Add a glob-style pattern of Pub/Sub channels that can be
+#               accessed by the user. It is possible to specify multiple channel
+#               patterns.
+#  allchannels  Alias for &*
+#  resetchannels            Flush the list of allowed channel patterns.
 #  ><password>  Add this password to the list of valid password for the user.
 #               For example >mypass will add "mypass" to the list.
 #               This directive clears the "nopass" flag (see later).
@@ -775,6 +879,40 @@ replica-priority 100
 #
 # Basically ACL rules are processed left-to-right.
 #
+# The following is a list of command categories and their meanings:
+# * keyspace - Writing or reading from keys, databases, or their metadata 
+#     in a type agnostic way. Includes DEL, RESTORE, DUMP, RENAME, EXISTS, DBSIZE,
+#     KEYS, EXPIRE, TTL, FLUSHALL, etc. Commands that may modify the keyspace,
+#     key or metadata will also have `write` category. Commands that only read
+#     the keyspace, key or metadata will have the `read` category.
+# * read - Reading from keys (values or metadata). Note that commands that don't
+#     interact with keys, will not have either `read` or `write`.
+# * write - Writing to keys (values or metadata)
+# * admin - Administrative commands. Normal applications will never need to use
+#     these. Includes REPLICAOF, CONFIG, DEBUG, SAVE, MONITOR, ACL, SHUTDOWN, etc.
+# * dangerous - Potentially dangerous (each should be considered with care for
+#     various reasons). This includes FLUSHALL, MIGRATE, RESTORE, SORT, KEYS,
+#     CLIENT, DEBUG, INFO, CONFIG, SAVE, REPLICAOF, etc.
+# * connection - Commands affecting the connection or other connections.
+#     This includes AUTH, SELECT, COMMAND, CLIENT, ECHO, PING, etc.
+# * blocking - Potentially blocking the connection until released by another
+#     command.
+# * fast - Fast O(1) commands. May loop on the number of arguments, but not the
+#     number of elements in the key.
+# * slow - All commands that are not Fast.
+# * pubsub - PUBLISH / SUBSCRIBE related
+# * transaction - WATCH / MULTI / EXEC related commands.
+# * scripting - Scripting related.
+# * set - Data type: sets related.
+# * sortedset - Data type: zsets related.
+# * list - Data type: lists related.
+# * hash - Data type: hashes related.
+# * string - Data type: strings related.
+# * bitmap - Data type: bitmaps related.
+# * hyperloglog - Data type: hyperloglog related.
+# * geo - Data type: geo related.
+# * stream - Data type: streams related.
+#
 # For more information about ACL configuration please refer to
 # the Redis web site at https://redis.io/topics/acl
 
@@ -798,14 +936,38 @@ acllog-max-len 128
 #
 # aclfile /etc/keydb/users.acl
 
-# IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility
+# IMPORTANT NOTE: starting with KeyDB 6 "requirepass" is just a compatibility
 # layer on top of the new ACL system. The option effect will be just setting
 # the password for the default user. Clients will still authenticate using
 # AUTH <password> as usually, or more explicitly with AUTH default <password>
 # if they follow the new protocol: both will work.
 #
+# The requirepass is not compatible with aclfile option and the ACL LOAD
+# command, these will cause requirepass to be ignored.
+#
 # requirepass foobared
 
+# New users are initialized with restrictive permissions by default, via the
+# equivalent of this ACL rule 'off resetkeys -@all'. Starting with KeyDB 6.2, it
+# is possible to manage access to Pub/Sub channels with ACL rules as well. The
+# default Pub/Sub channels permission if new users is controlled by the
+# acl-pubsub-default configuration directive, which accepts one of these values:
+#
+# allchannels: grants access to all Pub/Sub channels
+# resetchannels: revokes access to all Pub/Sub channels
+#
+# To ensure backward compatibility while upgrading KeyDB 6.0, acl-pubsub-default
+# defaults to the 'allchannels' permission.
+#
+# Future compatibility note: it is very likely that in a future version of KeyDB
+# the directive's default of 'allchannels' will be changed to 'resetchannels' in
+# order to provide better out-of-the-box Pub/Sub security. Therefore, it is
+# recommended that you explicitly define Pub/Sub permissions for all users
+# rather then rely on implicit default values. Once you've set explicit
+# Pub/Sub for all existing users, you should uncomment the following line.
+#
+# acl-pubsub-default resetchannels
+
 # Command renaming (DEPRECATED).
 #
 # ------------------------------------------------------------------------
@@ -842,7 +1004,7 @@ acllog-max-len 128
 # Once the limit is reached KeyDB will close all the new connections sending
 # an error 'max number of clients reached'.
 #
-# IMPORTANT: When Redis Cluster is used, the max number of connections is also
+# IMPORTANT: When KeyDB Cluster is used, the max number of connections is also
 # shared with the cluster bus: every node in the cluster will use two
 # connections, one incoming and another outgoing. It is important to size the
 # limit accordingly in case of very large clusters.
@@ -918,7 +1080,15 @@ acllog-max-len 128
 #
 # maxmemory-samples 5
 
-# Starting from Redis 5, by default a replica will ignore its maxmemory setting
+# Eviction processing is designed to function well with the default setting.
+# If there is an unusually large amount of write traffic, this value may need to
+# be increased.  Decreasing this value may reduce latency at the risk of
+# eviction processing effectiveness
+#   0 = minimum latency, 10 = default, 100 = process without regard to latency
+#
+# maxmemory-eviction-tenacity 10
+
+# Starting from KeyDB 5, by default a replica will ignore its maxmemory setting
 # (unless it is promoted to master after a failover or manually). It means
 # that the eviction of keys will be just handled by the master, sending the
 # DEL commands to the replica as keys evict in the master side.
@@ -1011,6 +1181,13 @@ replica-lazy-flush no
 
 lazyfree-lazy-user-del no
 
+# FLUSHDB, FLUSHALL, and SCRIPT FLUSH support both asynchronous and synchronous
+# deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the
+# commands. When neither flag is passed, this directive will be used to determine
+# if the data should be deleted asynchronously.
+
+lazyfree-lazy-user-flush no
+
 ############################ KERNEL OOM CONTROL ##############################
 
 # On Linux, it is possible to hint the kernel OOM killer on what processes
@@ -1042,6 +1219,19 @@ oom-score-adj no
 # oom-score-adj-values to positive values will always succeed.
 oom-score-adj-values 0 200 800
 
+
+#################### KERNEL transparent hugepage CONTROL ######################
+
+# Usually the kernel Transparent Huge Pages control is set to "madvise" or
+# or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which
+# case this config has no effect. On systems in which it is set to "always",
+# KeyDB will attempt to disable it specifically for the KeyDB process in order
+# to avoid latency problems specifically with fork(2) and CoW.
+# If for some reason you prefer to keep it enabled, you can set this config to
+# "no" and the kernel global to "always".
+
+disable-thp yes
+
 ############################## APPEND ONLY MODE ###############################
 
 # By default KeyDB asynchronously dumps the dataset on disk. This mode is
@@ -1269,12 +1459,21 @@ lua-time-limit 5000
 # master in your cluster.
 #
 # Default is 1 (replicas migrate only if their masters remain with at least
-# one replica). To disable migration just set it to a very large value.
+# one replica). To disable migration just set it to a very large value or
+# set cluster-allow-replica-migration to 'no'.
 # A value of 0 can be set but is useful only for debugging and dangerous
 # in production.
 #
 # cluster-migration-barrier 1
 
+# Turning off this option allows to use less automatic cluster configuration.
+# It both disables migration to orphaned masters and migration from masters
+# that became empty.
+#
+# Default is 'yes' (allow automatic migrations).
+#
+# cluster-allow-replica-migration yes
+
 # By default KeyDB Cluster nodes stop accepting queries if they detect there
 # is at least a hash slot uncovered (no available node is serving it).
 # This way if the cluster is partially down (for example a range of hash slots
@@ -1325,17 +1524,23 @@ lua-time-limit 5000
 #
 # In order to make KeyDB Cluster working in such environments, a static
 # configuration where each node knows its public address is needed. The
-# following two options are used for this scope, and are:
+# following four options are used for this scope, and are:
 #
 # * cluster-announce-ip
 # * cluster-announce-port
+# * cluster-announce-tls-port
 # * cluster-announce-bus-port
 #
-# Each instructs the node about its address, client port, and cluster message
+# Each instructs the node about its address, client ports (for connections
+# without and with TLS), and cluster message
 # bus port. The information is then published in the header of the bus packets
 # so that other nodes will be able to correctly map the address of the node
 # publishing the information.
 #
+# If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set
+# to zero, then cluster-announce-port refers to the TLS port. Note also that
+# cluster-announce-tls-port has no effect if cluster-tls is set to no.
+#
 # If the above options are not used, the normal KeyDB Cluster auto-detection
 # will be used instead.
 #
@@ -1347,7 +1552,8 @@ lua-time-limit 5000
 # Example:
 #
 # cluster-announce-ip 10.1.1.5
-# cluster-announce-port 6379
+# cluster-announce-tls-port 6379
+# cluster-announce-port 0
 # cluster-announce-bus-port 6380
 
 ################################## SLOW LOG ###################################
@@ -1421,8 +1627,9 @@ latency-monitor-threshold 0
 #  x     Expired events (events generated every time a key expires)
 #  e     Evicted events (events generated when a key is evicted for maxmemory)
 #  t     Stream commands
+#  d     Module key type events
 #  m     Key-miss events (Note: It is not included in the 'A' class)
-#  A     Alias for g$lshzxet, so that the "AKE" string means all the events
+#  A     Alias for g$lshzxetd, so that the "AKE" string means all the events
 #        (Except key-miss events which are excluded from 'A' due to their
 #         unique nature).
 #

pkg/deb/conf/sentinel.conf

@@ -20,12 +20,12 @@
 # The port that this sentinel instance will run on
 port 26379
 
-# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/keydb-sentinel.pid when
+# By default KeyDB Sentinel does not run as a daemon. Use 'yes' if you need it.
+# Note that KeyDB will write a pid file in /var/run/keydb-sentinel.pid when
 # daemonized.
 daemonize yes
 
-# When running daemonized, Redis Sentinel writes a pid file in
+# When running daemonized, KeyDB Sentinel writes a pid file in
 # /var/run/keydb-sentinel.pid by default. You can specify a custom pid file
 # location here.
 pidfile /var/run/sentinel/keydb-sentinel.pid
@@ -59,7 +59,7 @@ logfile /var/log/keydb/keydb-sentinel.log
 
 # dir <working-directory>
 # Every long running process should have a well-defined working directory.
-# For Redis Sentinel to chdir to /tmp at startup is the simplest thing
+# For KeyDB Sentinel to chdir to /tmp at startup is the simplest thing
 # for the process to don't interfere with administrative tasks such as
 # unmounting filesystems.
 dir /var/lib/keydb
@@ -86,22 +86,34 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 # sentinel auth-pass <master-name> <password>
 #
 # Set the password to use to authenticate with the master and replicas.
-# Useful if there is a password set in the Redis instances to monitor.
+# Useful if there is a password set in the KeyDB instances to monitor.
 #
 # Note that the master password is also used for replicas, so it is not
 # possible to set a different password in masters and replicas instances
 # if you want to be able to monitor these instances with Sentinel.
 #
-# However you can have Redis instances without the authentication enabled
-# mixed with Redis instances requiring the authentication (as long as the
+# However you can have KeyDB instances without the authentication enabled
+# mixed with KeyDB instances requiring the authentication (as long as the
 # password set is the same for all the instances requiring the password) as
-# the AUTH command will have no effect in Redis instances with authentication
+# the AUTH command will have no effect in KeyDB instances with authentication
 # switched off.
 #
 # Example:
 #
 # sentinel auth-pass mymaster MySUPER--secret-0123passw0rd
 
+# sentinel auth-user <master-name> <username>
+#
+# This is useful in order to authenticate to instances having ACL capabilities,
+# that is, running KeyDB 6.0 or greater. When just auth-pass is provided the
+# Sentinel instance will authenticate to KeyDB using the old "AUTH <pass>"
+# method. When also an username is provided, it will use "AUTH <user> <pass>".
+# In the KeyDB servers side, the ACL to provide just minimal access to
+# Sentinel instances, should be configured along the following lines:
+#
+#     user sentinel-user >somepassword +client +subscribe +publish \
+#                        +ping +info +multi +slaveof +config +client +exec on
+
 # sentinel down-after-milliseconds <master-name> <milliseconds>
 #
 # Number of milliseconds the master (or any attached replica or sentinel) should
@@ -112,6 +124,73 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 # Default is 30 seconds.
 sentinel down-after-milliseconds mymaster 30000
 
+# IMPORTANT NOTE: starting with KeyDB 6.2 ACL capability is supported for
+# Sentinel mode, please refer to the Redis website https://redis.io/topics/acl
+# for more details.
+
+# Sentinel's ACL users are defined in the following format:
+#
+#   user <username> ... acl rules ...
+#
+# For example:
+#
+#   user worker +@admin +@connection ~* on >ffa9203c493aa99
+#
+# For more information about ACL configuration please refer to the Redis
+# website at https://redis.io/topics/acl and KeyDB server configuration 
+# template keydb.conf.
+
+# ACL LOG
+#
+# The ACL Log tracks failed commands and authentication events associated
+# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked 
+# by ACLs. The ACL Log is stored in memory. You can reclaim memory with 
+# ACL LOG RESET. Define the maximum entry length of the ACL Log below.
+acllog-max-len 128
+
+# Using an external ACL file
+#
+# Instead of configuring users here in this file, it is possible to use
+# a stand-alone file just listing users. The two methods cannot be mixed:
+# if you configure users here and at the same time you activate the external
+# ACL file, the server will refuse to start.
+#
+# The format of the external ACL user file is exactly the same as the
+# format that is used inside keydb.conf to describe users.
+#
+# aclfile /etc/keydb/sentinel-users.acl
+
+# requirepass <password>
+#
+# You can configure Sentinel itself to require a password, however when doing
+# so Sentinel will try to authenticate with the same password to all the
+# other Sentinels. So you need to configure all your Sentinels in a given
+# group with the same "requirepass" password. Check the following documentation
+# for more info: https://redis.io/topics/sentinel
+#
+# IMPORTANT NOTE: starting with KeyDB 6.2 "requirepass" is a compatibility
+# layer on top of the ACL system. The option effect will be just setting
+# the password for the default user. Clients will still authenticate using
+# AUTH <password> as usually, or more explicitly with AUTH default <password>
+# if they follow the new protocol: both will work.
+#
+# New config files are advised to use separate authentication control for
+# incoming connections (via ACL), and for outgoing connections (via
+# sentinel-user and sentinel-pass) 
+#
+# The requirepass is not compatable with aclfile option and the ACL LOAD
+# command, these will cause requirepass to be ignored.
+
+# sentinel sentinel-user <username>
+#
+# You can configure Sentinel to authenticate with other Sentinels with specific
+# user name. 
+
+# sentinel sentinel-pass <password>
+#
+# The password for Sentinel to authenticate with other Sentinels. If sentinel-user
+# is not configured, Sentinel will use 'default' user with sentinel-pass to authenticate.
+
 # sentinel parallel-syncs <master-name> <numreplicas>
 #
 # How many replicas we can reconfigure to point to the new replica simultaneously
@@ -172,7 +251,7 @@ sentinel failover-timeout mymaster 180000
 # generated in the WARNING level (for instance -sdown, -odown, and so forth).
 # This script should notify the system administrator via email, SMS, or any
 # other messaging system, that there is something wrong with the monitored
-# Redis systems.
+# KeyDB systems.
 #
 # The script is called with just two arguments: the first is the event type
 # and the second the event description.
@@ -182,7 +261,7 @@ sentinel failover-timeout mymaster 180000
 #
 # Example:
 #
-# sentinel notification-script mymaster /var/redis/notify.sh
+# sentinel notification-script mymaster /var/keydb/notify.sh
 
 # CLIENTS RECONFIGURATION SCRIPT
 #
@@ -207,7 +286,7 @@ sentinel failover-timeout mymaster 180000
 #
 # Example:
 #
-# sentinel client-reconfig-script mymaster /var/redis/reconfig.sh
+# sentinel client-reconfig-script mymaster /var/keydb/reconfig.sh
 
 # SECURITY
 #
@@ -218,11 +297,11 @@ sentinel failover-timeout mymaster 180000
 
 sentinel deny-scripts-reconfig yes
 
-# REDIS COMMANDS RENAMING
+# KEYDB COMMANDS RENAMING
 #
-# Sometimes the Redis server has certain commands, that are needed for Sentinel
+# Sometimes the KeyDB server has certain commands, that are needed for Sentinel
 # to work correctly, renamed to unguessable strings. This is often the case
-# of CONFIG and SLAVEOF in the context of providers that provide Redis as
+# of CONFIG and SLAVEOF in the context of providers that provide KeyDB as
 # a service, and don't want the customers to reconfigure the instances outside
 # of the administration console.
 #
@@ -239,6 +318,24 @@ sentinel deny-scripts-reconfig yes
 # SENTINEL SET can also be used in order to perform this configuration at runtime.
 #
 # In order to set a command back to its original name (undo the renaming), it
-# is possible to just rename a command to itsef:
+# is possible to just rename a command to itself:
 #
 # SENTINEL rename-command mymaster CONFIG CONFIG
+
+# HOSTNAMES SUPPORT
+#
+# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR
+# to specify an IP address. Also, it requires the KeyDB replica-announce-ip
+# keyword to specify only IP addresses.
+#
+# You may enable hostnames support by enabling resolve-hostnames. Note
+# that you must make sure your DNS is configured properly and that DNS
+# resolution does not introduce very long delays.
+#
+SENTINEL resolve-hostnames no
+
+# When resolve-hostnames is enabled, Sentinel still uses IP addresses
+# when exposing instances to users, configuration files, etc. If you want
+# to retain the hostnames when announced, enable announce-hostnames below.
+#
+SENTINEL announce-hostnames no

pkg/deb/debian/control

@@ -6,6 +6,7 @@ Build-Depends:
  debhelper (>= 9~),
  dpkg-dev (>= 1.17.5),
  systemd,
+ libsystemd-dev <!nocheck>,
  procps <!nocheck>,
  build-essential <!nocheck>,
  tcl <!nocheck>,

pkg/deb/debian/copyright

@@ -1,16 +1,94 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Contact: John Sully <john@eqalpha.com>
-Upstream-Name: keydb-pro
-Source: https://github.com/JohnSully/KeyDB-Pro
+Upstream-Name: KeyDB
+Source: https://github.com/EQ-Alpha/KeyDB
 
 Files: *
-Copyright: © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
-Copyright © 2019, John Sully
-License: Proprietary
+Copyright: 
+ © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
+ © 2019-2021 John Sully
+ © 2019-2021 EQ Alpha Technology Ltd.
 
+License: BSD-3-clause
 
-Files: debian/*
-Copyright: © 2009 Chris Lamb <lamby@debian.org>
+Files:
+ src/rio.*
+ src/t_zset.c
+ src/ziplist.h
+ src/intset.*
+ src/redis-check-aof.c
+ deps/hiredis/*
+ deps/linenoise/*
+Copyright:
+ © 2009-2012 Pieter Noordhuis <pcnoordhuis@gmail.com>
+ © 2009-2012 Salvatore Sanfilippo <antirez@gmail.com>
+License: BSD-3-clause
+
+Files:
+ src/lzf.h
+ src/lzfP.h
+ src/lzf_d.c
+ src/lzf_c.c
+Copyright:
+ © 2000-2007 Marc Alexander Lehmann <schmorp@schmorp.de>
+ © 2009-2012 Salvatore Sanfilippo <antirez@gmail.com>
+License: BSD-2-clause
+
+Files: src/setproctitle.c
+Copyright:
+ © 2010 William Ahern
+ © 2013 Salvatore Sanfilippo
+ © 2013 Stam He
+License: BSD-3-clause
+
+Files: src/ae_evport.c
+Copyright: © 2012 Joyent, Inc.
+License: BSD-3-clause
+
+Files: src/ae_kqueue.c
+Copyright: © 2009 Harish Mallipeddi <harish.mallipeddi@gmail.com>
+License: BSD-3-clause
+
+Files: utils/install_server.sh
+Copyright: © 2011 Dvir Volk <dvirsk@gmail.com>
+License: BSD-3-clause
+
+Files: deps/jemalloc/*
+Copyright:
+ © 2002-2012 Jason Evans <jasone@canonware.com>
+ © 2007-2012 Mozilla Foundation
+ © 2009-2012 Facebook, Inc.
+License: BSD-3-clause
+
+Files: src/pqsort.*
+Copyright: © 1992-1993 The Regents of the University of California
+License: BSD-3-clause
+
+Files: deps/lua/*
+Copyright: © 1994-2012 Lua.org, PUC-Ri
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+Files: pkg/deb/debian/*
+Copyright:
+ © 2009 Chris Lamb <lamby@debian.org>
+ © 2020-2021 EQ Alpha Technology Ltd. <support@eqalpha.com>
 License: BSD-3-clause
 
 License: BSD-2-clause

pkg/deb/debian/keydb-tools.install

@@ -5,3 +5,4 @@ src/keydb-check-rdb	/usr/bin
 src/keydb-cli		/usr/bin
 src/keydb-server	/usr/bin
 src/keydb-sentinel	/usr/bin
+src/keydb-diagnostic-tool	/usr/bin

pkg/deb/debian/rules

@@ -3,6 +3,7 @@
 include /usr/share/dpkg/buildflags.mk
 
 export BUILD_TLS=yes
+export USE_SYSTEMD=yes
 export CFLAGS CPPFLAGS LDFLAGS
 export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 export DEB_LDFLAGS_MAINT_APPEND = -ldl -latomic $(LUA_LDFLAGS)

pkg/deb/debian/zsh-completion/_keydb-cli

@@ -0,0 +1,53 @@
+#compdef keydb-cli
+local -a options
+options=(
+  '-h[Server hostname (default: 127.0.0.1).]: :_hosts'
+  '-p[Server port (default: 6379).]'
+  '-s[Server socket (overrides hostname and port).]'
+  '-a[Password to use when connecting to the server. You can also use the REDISCLI_AUTH environment variable to pass this password more safely (if both are used, this argument takes precedence).]'
+  '--user[Used to send ACL style "AUTH username pass". Needs -a.]'
+  '--pass[Alias of -a for consistency with the new --user option.]'
+  '--askpass[Force user to input password with mask from STDIN. If this argument is used, "-a" and REDISCLI_AUTH environment variable will be ignored.]'
+  '-u[Server URI.]'
+  '-r[Execute specified command N times.]'
+  '-i[When -r is used, waits <interval> seconds per command. It is possible to specify sub-second times like -i 0.1.]'
+  '-n[Database number.]'
+  '-3[Start session in RESP3 protocol mode.]'
+  '-x[Read last argument from STDIN.]'
+  '-d[Delimiter between response bulks for raw formatting (default: \n).]'
+  '-D[D <delimiter>     Delimiter between responses for raw formatting (default: \n).]'
+  '-c[Enable cluster mode (follow -ASK and -MOVED redirections).]'
+  '-e[Return exit error code when command execution fails.]'
+  '--raw[Use raw formatting for replies (default when STDOUT is not a tty).]'
+  '--no-raw[Force formatted output even when STDOUT is not a tty.]'
+  '--quoted-input[Force input to be handled as quoted strings.]'
+  '--csv[Output in CSV format.]'
+  '--show-pushes[Whether to print RESP3 PUSH messages.  Enabled by default when STDOUT is a tty but can be overriden with --show-pushes no.]'
+  '--stat[Print rolling stats about server: mem, clients, ...]'
+  '--latency[Enter a special mode continuously sampling latency. If you use this mode in an interactive session it runs forever displaying real-time stats. Otherwise if --raw or --csv is specified, or if you redirect the output to a non TTY, it samples the latency for 1 second (you can use -i to change the interval), then produces a single output and exits.]'
+  '--latency-history[Like --latency but tracking latency changes over time. Default time interval is 15 sec. Change it using -i.]'
+  '--latency-dist[Shows latency as a spectrum, requires xterm 256 colors. Default time interval is 1 sec. Change it using -i.]'
+  '--lru-test[Simulate a cache workload with an 80-20 distribution.]'
+  '--replica[Simulate a replica showing commands received from the master.]'
+  '--rdb[Transfer an RDB dump from remote server to local file.]'
+  '--pipe[Transfer raw KeyDB protocol from stdin to server.]'
+  '--pipe-timeout[In --pipe mode, abort with error if after sending all data. no reply is received within <n> seconds. Default timeout: 30. Use 0 to wait forever.]'
+  '--bigkeys[Sample KeyDB keys looking for keys with many elements (complexity).]'
+  '--memkeys[Sample KeyDB keys looking for keys consuming a lot of memory.]'
+  '--memkeys-samples[Sample KeyDB keys looking for keys consuming a lot of memory. And define number of key elements to sample]'
+  '--hotkeys[Sample KeyDB keys looking for hot keys. only works when maxmemory-policy is *lfu.]'
+  '--scan[List all keys using the SCAN command.]'
+  '--pattern[Keys pattern when using the --scan, --bigkeys or --hotkeys options (default: *).]'
+  '--quoted-pattern[Same as --pattern, but the specified string can be quoted, in order to pass an otherwise non binary-safe string.]'
+  '--intrinsic-latency[Run a test to measure intrinsic system latency. The test will run for the specified amount of seconds.]'
+  '--eval[Send an EVAL command using the Lua script at <file>.]'
+  '--ldb[Used with --eval enable the Redis Lua debugger.]'
+  '--ldb-sync-mode[Like --ldb but uses the synchronous Lua debugger, in this mode the server is blocked and script changes are not rolled back from the server memory.]'
+  '--cluster[<command> args... opts... Cluster Manager command and arguments (see below).]'
+  '--verbose[Verbose mode.]'
+  '--no-auth-warning[Dont show warning message when using password on command line interface.]'
+  '--help[Output this help and exit.]'
+   '--version[Output version and exit.]'
+)
+
+_arguments -s $options

pkg/deb/debian_dh9/control

@@ -6,6 +6,7 @@ Build-Depends:
  debhelper (>= 9~),
  dpkg-dev (>= 1.17.5),
  systemd,
+ libsystemd-dev <!nocheck>,
  procps <!nocheck>,
  build-essential <!nocheck>,
  tcl <!nocheck>,

pkg/deb/debian_dh9/copyright

@@ -1,11 +1,14 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Contact: John Sully <john@eqalpha.com>
-Upstream-Name: keydb
-Source: https://github.com/JohnSully/KeyDB
+Upstream-Name: KeyDB
+Source: https://github.com/EQ-Alpha/KeyDB
 
 Files: *
-Copyright: © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
-Copyright © 2019, John Sully
+Copyright: 
+ © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
+ © 2019-2021 John Sully
+ © 2019-2021 EQ Alpha Technology Ltd.
+
 License: BSD-3-clause
 
 Files:
@@ -82,8 +85,10 @@ License: MIT
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE.
 
-Files: debian/*
-Copyright: © 2009 Chris Lamb <lamby@debian.org>
+Files: pkg/deb/debian_dh9/*
+Copyright: 
+ © 2009 Chris Lamb <lamby@debian.org>
+ © 2020-2021 EQ Alpha Technology Ltd. <support@eqalpha.com>
 License: BSD-3-clause
 
 License: BSD-2-clause

pkg/deb/debian_dh9/keydb-tools.install

@@ -5,3 +5,4 @@ src/keydb-check-aof	/usr/bin
 src/keydb-check-rdb	/usr/bin
 src/keydb-cli		/usr/bin
 src/keydb-sentinel	/usr/bin
+src/keydb-diagnostic-tool	/usr/bin

pkg/deb/debian_dh9/rules

@@ -9,6 +9,7 @@ include /usr/share/dpkg/buildflags.mk
 #LUA_LDFLAGS = $(addprefix -llua5.1-,$(LUA_LIBS_DEBIAN)) $(addprefix ../deps/lua/src/,$(LUA_OBJECTS))
 
 export BUILD_TLS=yes
+export USE_SYSTEMD=yes
 export CFLAGS CPPFLAGS LDFLAGS
 export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 export DEB_LDFLAGS_MAINT_APPEND = -ldl -latomic $(LUA_LDFLAGS)

pkg/rpm/generate_rpms.sh

@@ -25,7 +25,13 @@ mkdir -p $DIR/keydb_build/keydb_rpm/var/log/keydb
 
 # move binaries to bin
 rm $DIR/keydb_build/keydb_rpm/usr/bin/*
-cp $DIR/../../src/keydb-* $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-server $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-sentinel $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-cli $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-benchmark $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-check-aof $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-check-rdb $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-diagnostic-tool $DIR/keydb_build/keydb_rpm/usr/bin/
 
 # update spec file with build info
 sed -i '2d' $DIR/keydb_build/keydb.spec

pkg/rpm/keydb_build/keydb_rpm/etc/keydb/sentinel.conf

@@ -20,20 +20,20 @@
 # The port that this sentinel instance will run on
 port 26379
 
-# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/redis-sentinel.pid when
+# By default KeyDB Sentinel does not run as a daemon. Use 'yes' if you need it.
+# Note that KeyDB will write a pid file in /var/run/keydb-sentinel.pid when
 # daemonized.
 daemonize no
 
-# When running daemonized, Redis Sentinel writes a pid file in
-# /var/run/redis-sentinel.pid by default. You can specify a custom pid file
+# When running daemonized, KeyDB Sentinel writes a pid file in
+# /var/run/keydb-sentinel.pid by default. You can specify a custom pid file
 # location here.
-pidfile /var/run/redis-sentinel.pid
+pidfile /var/run/sentinel/keydb-sentinel.pid
 
 # Specify the log file name. Also the empty string can be used to force
 # Sentinel to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
-logfile /var/log/redis/sentinel.log
+logfile /var/log/keydb/keydb-sentinel.log
 
 # sentinel announce-ip <ip>
 # sentinel announce-port <port>
@@ -59,12 +59,12 @@ logfile /var/log/redis/sentinel.log
 
 # dir <working-directory>
 # Every long running process should have a well-defined working directory.
-# For Redis Sentinel to chdir to /tmp at startup is the simplest thing
+# For KeyDB Sentinel to chdir to /tmp at startup is the simplest thing
 # for the process to don't interfere with administrative tasks such as
 # unmounting filesystems.
 dir /tmp
 
-# sentinel monitor <master-name> <ip> <redis-port> <quorum>
+# sentinel monitor <master-name> <ip> <keydb-port> <quorum>
 #
 # Tells Sentinel to monitor this master, and to consider it in O_DOWN
 # (Objectively Down) state only if at least <quorum> sentinels agree.
@@ -86,22 +86,34 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 # sentinel auth-pass <master-name> <password>
 #
 # Set the password to use to authenticate with the master and replicas.
-# Useful if there is a password set in the Redis instances to monitor.
+# Useful if there is a password set in the KeyDB instances to monitor.
 #
 # Note that the master password is also used for replicas, so it is not
 # possible to set a different password in masters and replicas instances
 # if you want to be able to monitor these instances with Sentinel.
 #
-# However you can have Redis instances without the authentication enabled
-# mixed with Redis instances requiring the authentication (as long as the
+# However you can have KeyDB instances without the authentication enabled
+# mixed with KeyDB instances requiring the authentication (as long as the
 # password set is the same for all the instances requiring the password) as
-# the AUTH command will have no effect in Redis instances with authentication
+# the AUTH command will have no effect in KeyDB instances with authentication
 # switched off.
 #
 # Example:
 #
 # sentinel auth-pass mymaster MySUPER--secret-0123passw0rd
 
+# sentinel auth-user <master-name> <username>
+#
+# This is useful in order to authenticate to instances having ACL capabilities,
+# that is, running KeyDB 6.0 or greater. When just auth-pass is provided the
+# Sentinel instance will authenticate to KeyDB using the old "AUTH <pass>"
+# method. When also an username is provided, it will use "AUTH <user> <pass>".
+# In the KeyDB servers side, the ACL to provide just minimal access to
+# Sentinel instances, should be configured along the following lines:
+#
+#     user sentinel-user >somepassword +client +subscribe +publish \
+#                        +ping +info +multi +slaveof +config +client +exec on
+
 # sentinel down-after-milliseconds <master-name> <milliseconds>
 #
 # Number of milliseconds the master (or any attached replica or sentinel) should
@@ -112,6 +124,73 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 # Default is 30 seconds.
 sentinel down-after-milliseconds mymaster 30000
 
+# IMPORTANT NOTE: starting with KeyDB 6.2 ACL capability is supported for
+# Sentinel mode, please refer to the Redis website https://redis.io/topics/acl
+# for more details.
+
+# Sentinel's ACL users are defined in the following format:
+#
+#   user <username> ... acl rules ...
+#
+# For example:
+#
+#   user worker +@admin +@connection ~* on >ffa9203c493aa99
+#
+# For more information about ACL configuration please refer to the Redis
+# website at https://redis.io/topics/acl and KeyDB server configuration 
+# template keydb.conf.
+
+# ACL LOG
+#
+# The ACL Log tracks failed commands and authentication events associated
+# with ACLs. The ACL Log is useful to troubleshoot failed commands blocked 
+# by ACLs. The ACL Log is stored in memory. You can reclaim memory with 
+# ACL LOG RESET. Define the maximum entry length of the ACL Log below.
+acllog-max-len 128
+
+# Using an external ACL file
+#
+# Instead of configuring users here in this file, it is possible to use
+# a stand-alone file just listing users. The two methods cannot be mixed:
+# if you configure users here and at the same time you activate the external
+# ACL file, the server will refuse to start.
+#
+# The format of the external ACL user file is exactly the same as the
+# format that is used inside keydb.conf to describe users.
+#
+# aclfile /etc/keydb/sentinel-users.acl
+
+# requirepass <password>
+#
+# You can configure Sentinel itself to require a password, however when doing
+# so Sentinel will try to authenticate with the same password to all the
+# other Sentinels. So you need to configure all your Sentinels in a given
+# group with the same "requirepass" password. Check the following documentation
+# for more info: https://redis.io/topics/sentinel
+#
+# IMPORTANT NOTE: starting with KeyDB 6.2 "requirepass" is a compatibility
+# layer on top of the ACL system. The option effect will be just setting
+# the password for the default user. Clients will still authenticate using
+# AUTH <password> as usually, or more explicitly with AUTH default <password>
+# if they follow the new protocol: both will work.
+#
+# New config files are advised to use separate authentication control for
+# incoming connections (via ACL), and for outgoing connections (via
+# sentinel-user and sentinel-pass) 
+#
+# The requirepass is not compatable with aclfile option and the ACL LOAD
+# command, these will cause requirepass to be ignored.
+
+# sentinel sentinel-user <username>
+#
+# You can configure Sentinel to authenticate with other Sentinels with specific
+# user name. 
+
+# sentinel sentinel-pass <password>
+#
+# The password for Sentinel to authenticate with other Sentinels. If sentinel-user
+# is not configured, Sentinel will use 'default' user with sentinel-pass to authenticate.
+
 # sentinel parallel-syncs <master-name> <numreplicas>
 #
 # How many replicas we can reconfigure to point to the new replica simultaneously
@@ -172,7 +251,7 @@ sentinel failover-timeout mymaster 180000
 # generated in the WARNING level (for instance -sdown, -odown, and so forth).
 # This script should notify the system administrator via email, SMS, or any
 # other messaging system, that there is something wrong with the monitored
-# Redis systems.
+# KeyDB systems.
 #
 # The script is called with just two arguments: the first is the event type
 # and the second the event description.
@@ -182,7 +261,7 @@ sentinel failover-timeout mymaster 180000
 #
 # Example:
 #
-# sentinel notification-script mymaster /var/redis/notify.sh
+# sentinel notification-script mymaster /var/keydb/notify.sh
 
 # CLIENTS RECONFIGURATION SCRIPT
 #
@@ -207,7 +286,7 @@ sentinel failover-timeout mymaster 180000
 #
 # Example:
 #
-# sentinel client-reconfig-script mymaster /var/redis/reconfig.sh
+# sentinel client-reconfig-script mymaster /var/keydb/reconfig.sh
 
 # SECURITY
 #
@@ -218,11 +297,11 @@ sentinel failover-timeout mymaster 180000
 
 sentinel deny-scripts-reconfig yes
 
-# REDIS COMMANDS RENAMING
+# KEYDB COMMANDS RENAMING
 #
-# Sometimes the Redis server has certain commands, that are needed for Sentinel
+# Sometimes the KeyDB server has certain commands, that are needed for Sentinel
 # to work correctly, renamed to unguessable strings. This is often the case
-# of CONFIG and SLAVEOF in the context of providers that provide Redis as
+# of CONFIG and SLAVEOF in the context of providers that provide KeyDB as
 # a service, and don't want the customers to reconfigure the instances outside
 # of the administration console.
 #
@@ -239,6 +318,24 @@ sentinel deny-scripts-reconfig yes
 # SENTINEL SET can also be used in order to perform this configuration at runtime.
 #
 # In order to set a command back to its original name (undo the renaming), it
-# is possible to just rename a command to itsef:
+# is possible to just rename a command to itself:
 #
 # SENTINEL rename-command mymaster CONFIG CONFIG
+
+# HOSTNAMES SUPPORT
+#
+# Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR
+# to specify an IP address. Also, it requires the KeyDB replica-announce-ip
+# keyword to specify only IP addresses.
+#
+# You may enable hostnames support by enabling resolve-hostnames. Note
+# that you must make sure your DNS is configured properly and that DNS
+# resolution does not introduce very long delays.
+#
+SENTINEL resolve-hostnames no
+
+# When resolve-hostnames is enabled, Sentinel still uses IP addresses
+# when exposing instances to users, configuration files, etc. If you want
+# to retain the hostnames when announced, enable announce-hostnames below.
+#
+SENTINEL announce-hostnames no

runtest

@@ -10,7 +10,7 @@ done
 
 if [ -z $TCLSH ]
 then
-    echo "You need tcl 8.5 or newer in order to run the Redis test"
+    echo "You need tcl 8.5 or newer in order to run the KeyDB test"
     exit 1
 fi
 $TCLSH tests/test_helper.tcl "${@}"

runtest-cluster

@@ -8,7 +8,7 @@ done
 
 if [ -z $TCLSH ]
 then
-    echo "You need tcl 8.5 or newer in order to run the Redis Cluster test"
+    echo "You need tcl 8.5 or newer in order to run the KeyDB Cluster test"
     exit 1
 fi
 $TCLSH tests/cluster/run.tcl $*

runtest-moduleapi

@@ -9,13 +9,14 @@ done
 
 if [ -z $TCLSH ]
 then
-    echo "You need tcl 8.5 or newer in order to run the Redis ModuleApi test"
+    echo "You need tcl 8.5 or newer in order to run the KeyDB ModuleApi test"
     exit 1
 fi
 
 $MAKE -C tests/modules && \
 $TCLSH tests/test_helper.tcl \
 --single unit/moduleapi/commandfilter \
+--single unit/moduleapi/basics \
 --single unit/moduleapi/fork \
 --single unit/moduleapi/testrdb \
 --single unit/moduleapi/infotest \

runtest-sentinel

@@ -8,7 +8,7 @@ done
 
 if [ -z $TCLSH ]
 then
-    echo "You need tcl 8.5 or newer in order to run the Redis Sentinel test"
+    echo "You need tcl 8.5 or newer in order to run the KeyDB Sentinel test"
     exit 1
 fi
 $TCLSH tests/sentinel/run.tcl $*

sentinel.conf

@@ -20,12 +20,12 @@
 # The port that this sentinel instance will run on
 port 26379
 
-# By default Redis Sentinel does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/keydb-sentinel.pid when
+# By default KeyDB Sentinel does not run as a daemon. Use 'yes' if you need it.
+# Note that KeyDB will write a pid file in /var/run/keydb-sentinel.pid when
 # daemonized.
 daemonize no
 
-# When running daemonized, Redis Sentinel writes a pid file in
+# When running daemonized, KeyDB Sentinel writes a pid file in
 # /var/run/keydb-sentinel.pid by default. You can specify a custom pid file
 # location here.
 pidfile /var/run/keydb-sentinel.pid
@@ -59,7 +59,7 @@ logfile ""
 
 # dir <working-directory>
 # Every long running process should have a well-defined working directory.
-# For Redis Sentinel to chdir to /tmp at startup is the simplest thing
+# For KeyDB Sentinel to chdir to /tmp at startup is the simplest thing
 # for the process to don't interfere with administrative tasks such as
 # unmounting filesystems.
 dir /tmp
@@ -86,16 +86,16 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 # sentinel auth-pass <master-name> <password>
 #
 # Set the password to use to authenticate with the master and replicas.
-# Useful if there is a password set in the Redis instances to monitor.
+# Useful if there is a password set in the KeyDB instances to monitor.
 #
 # Note that the master password is also used for replicas, so it is not
 # possible to set a different password in masters and replicas instances
 # if you want to be able to monitor these instances with Sentinel.
 #
-# However you can have Redis instances without the authentication enabled
-# mixed with Redis instances requiring the authentication (as long as the
+# However you can have KeyDB instances without the authentication enabled
+# mixed with KeyDB instances requiring the authentication (as long as the
 # password set is the same for all the instances requiring the password) as
-# the AUTH command will have no effect in Redis instances with authentication
+# the AUTH command will have no effect in KeyDB instances with authentication
 # switched off.
 #
 # Example:
@@ -105,10 +105,10 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 # sentinel auth-user <master-name> <username>
 #
 # This is useful in order to authenticate to instances having ACL capabilities,
-# that is, running Redis 6.0 or greater. When just auth-pass is provided the
-# Sentinel instance will authenticate to Redis using the old "AUTH <pass>"
+# that is, running KeyDB 6.0 or greater. When just auth-pass is provided the
+# Sentinel instance will authenticate to KeyDB using the old "AUTH <pass>"
 # method. When also an username is provided, it will use "AUTH <user> <pass>".
-# In the Redis servers side, the ACL to provide just minimal access to
+# In the KeyDB servers side, the ACL to provide just minimal access to
 # Sentinel instances, should be configured along the following lines:
 #
 #     user sentinel-user >somepassword +client +subscribe +publish \
@@ -125,7 +125,7 @@ sentinel monitor mymaster 127.0.0.1 6379 2
 sentinel down-after-milliseconds mymaster 30000
 
 # IMPORTANT NOTE: starting with KeyDB 6.2 ACL capability is supported for
-# Sentinel mode, please refer to the KeyDB website https://redis.io/topics/acl
+# Sentinel mode, please refer to the Redis website https://redis.io/topics/acl
 # for more details.
 
 # Sentinel's ACL users are defined in the following format:
@@ -137,8 +137,8 @@ sentinel down-after-milliseconds mymaster 30000
 #   user worker +@admin +@connection ~* on >ffa9203c493aa99
 #
 # For more information about ACL configuration please refer to the Redis
-# website at https://redis.io/topics/acl and redis server configuration 
-# template redis.conf.
+# website at https://redis.io/topics/acl and KeyDB server configuration 
+# template keydb.conf.
 
 # ACL LOG
 #
@@ -156,9 +156,9 @@ acllog-max-len 128
 # ACL file, the server will refuse to start.
 #
 # The format of the external ACL user file is exactly the same as the
-# format that is used inside redis.conf to describe users.
+# format that is used inside keydb.conf to describe users.
 #
-# aclfile /etc/redis/sentinel-users.acl
+# aclfile /etc/keydb/sentinel-users.acl
 
 # requirepass <password>
 #
@@ -168,7 +168,7 @@ acllog-max-len 128
 # group with the same "requirepass" password. Check the following documentation
 # for more info: https://redis.io/topics/sentinel
 #
-# IMPORTANT NOTE: starting with Redis 6.2 "requirepass" is a compatibility
+# IMPORTANT NOTE: starting with KeyDB 6.2 "requirepass" is a compatibility
 # layer on top of the ACL system. The option effect will be just setting
 # the password for the default user. Clients will still authenticate using
 # AUTH <password> as usually, or more explicitly with AUTH default <password>
@@ -251,7 +251,7 @@ sentinel failover-timeout mymaster 180000
 # generated in the WARNING level (for instance -sdown, -odown, and so forth).
 # This script should notify the system administrator via email, SMS, or any
 # other messaging system, that there is something wrong with the monitored
-# Redis systems.
+# KeyDB systems.
 #
 # The script is called with just two arguments: the first is the event type
 # and the second the event description.
@@ -261,7 +261,7 @@ sentinel failover-timeout mymaster 180000
 #
 # Example:
 #
-# sentinel notification-script mymaster /var/redis/notify.sh
+# sentinel notification-script mymaster /var/keydb/notify.sh
 
 # CLIENTS RECONFIGURATION SCRIPT
 #
@@ -286,7 +286,7 @@ sentinel failover-timeout mymaster 180000
 #
 # Example:
 #
-# sentinel client-reconfig-script mymaster /var/redis/reconfig.sh
+# sentinel client-reconfig-script mymaster /var/keydb/reconfig.sh
 
 # SECURITY
 #
@@ -297,11 +297,11 @@ sentinel failover-timeout mymaster 180000
 
 sentinel deny-scripts-reconfig yes
 
-# REDIS COMMANDS RENAMING
+# KEYDB COMMANDS RENAMING
 #
-# Sometimes the Redis server has certain commands, that are needed for Sentinel
+# Sometimes the KeyDB server has certain commands, that are needed for Sentinel
 # to work correctly, renamed to unguessable strings. This is often the case
-# of CONFIG and SLAVEOF in the context of providers that provide Redis as
+# of CONFIG and SLAVEOF in the context of providers that provide KeyDB as
 # a service, and don't want the customers to reconfigure the instances outside
 # of the administration console.
 #
@@ -325,7 +325,7 @@ sentinel deny-scripts-reconfig yes
 # HOSTNAMES SUPPORT
 #
 # Normally Sentinel uses only IP addresses and requires SENTINEL MONITOR
-# to specify an IP address. Also, it requires the Redis replica-announce-ip
+# to specify an IP address. Also, it requires the KeyDB replica-announce-ip
 # keyword to specify only IP addresses.
 #
 # You may enable hostnames support by enabling resolve-hostnames. Note

src/Makefile

@@ -3,11 +3,11 @@
 # This file is released under the BSD license, see the COPYING file
 #
 # The Makefile composes the final FINAL_CFLAGS and FINAL_LDFLAGS using
-# what is needed for Redis plus the standard CFLAGS and LDFLAGS passed.
+# what is needed for KeyDB plus the standard CFLAGS and LDFLAGS passed.
 # However when building the dependencies (Jemalloc, Lua, Hiredis, ...)
 # CFLAGS and LDFLAGS are propagated to the dependencies, so to pass
-# flags only to be used when compiling / linking Redis itself REDIS_CFLAGS
-# and REDIS_LDFLAGS are used instead (this is the case of 'make gcov').
+# flags only to be used when compiling / linking KeyDB itself KEYDB_CFLAGS
+# and KEYDB_LDFLAGS are used instead (this is the case of 'make gcov').
 #
 # Dependencies are stored in the Makefile.dep file. To rebuild this file
 # Just use 'make dep', but this is only needed by developers.
@@ -29,7 +29,7 @@ ifneq (,$(findstring FreeBSD,$(uname_S)))
   STD+=-Wno-c11-extensions
 endif
 endif
-WARN=-Wall -W -Wno-missing-field-initializers
+WARN=-Wall -W -Wno-missing-field-initializers -Wno-address-of-packed-member -Wno-atomic-alignment
 OPT=$(OPTIMIZATION)
 
 # Detect if the compiler supports C11 _Atomic
@@ -70,7 +70,7 @@ endif
 # Do we use our assembly spinlock? X64 only
 ifeq ($(uname_S),Linux)
 ifeq ($(uname_M),x86_64)
-ifneq ($(@@),32bit)
+ifneq ($(TARGET32), true)
 ifeq ($(USEASM),true)
 	ASM_OBJ+= fastlock_x64.o
 	CFLAGS+= -DASM_SPINLOCK
@@ -85,7 +85,7 @@ ifeq ($(COMPILER_NAME),clang)
 	LDFLAGS+= -latomic
 endif
 
-# To get ARM stack traces if Redis crashes we need a special C flag.
+# To get ARM stack traces if KeyDB crashes we need a special C flag.
 ifneq (,$(filter aarch64 armv,$(uname_M)))
         CFLAGS+=-funwind-tables
 	CXXFLAGS+=-funwind-tables
@@ -116,15 +116,19 @@ endif
 # Override default settings if possible
 -include .make-settings
 
-FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS)
-FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CXXFLAGS) $(REDIS_CFLAGS)
-FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG)
+FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS)
+FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CXXFLAGS) $(KEYDB_CFLAGS) $(REDIS_CFLAGS)
+FINAL_LDFLAGS=$(LDFLAGS) $(KEYDB_LDFLAGS) $(DEBUG)
 FINAL_LIBS+=-lm
 DEBUG=-g -ggdb
 
 ifneq ($(uname_S),Darwin)
 	FINAL_LIBS+=-latomic
 endif
+# Linux ARM32 needs -latomic at linking time
+ifneq (,$(findstring armv,$(uname_M)))
+	FINAL_LIBS+=-latomic
+endif
 
 
 ifeq ($(uname_S),SunOS)
@@ -152,12 +156,10 @@ ifeq ($(uname_S),Darwin)
 	# must be referenced explicitly during build.
 ifeq ($(uname_M),arm64)
 	# Homebrew arm64 uses /opt/homebrew as HOMEBREW_PREFIX
-	OPENSSL_CFLAGS=-I/opt/homebrew/opt/openssl/include
-	OPENSSL_LDFLAGS=-L/opt/homebrew/opt/openssl/lib
+	OPENSSL_PREFIX?=/opt/homebrew/opt/openssl
 else
 	# Homebrew x86/ppc uses /usr/local as HOMEBREW_PREFIX
-	OPENSSL_CFLAGS=-I/usr/local/opt/openssl/include
-	OPENSSL_LDFLAGS=-L/usr/local/opt/openssl/lib
+	OPENSSL_PREFIX?=/usr/local/opt/openssl
 endif
 else
 ifeq ($(uname_S),AIX)
@@ -173,7 +175,7 @@ ifeq ($(uname_S),OpenBSD)
 	    FINAL_CXXFLAGS+= -DUSE_BACKTRACE -I/usr/local/include
 	    FINAL_LDFLAGS+= -L/usr/local/lib
 	    FINAL_LIBS+= -lexecinfo
-    	endif
+	endif
 
 else
 ifeq ($(uname_S),NetBSD)
@@ -183,7 +185,7 @@ ifeq ($(uname_S),NetBSD)
 	    FINAL_CFLAGS+= -DUSE_BACKTRACE -I/usr/pkg/include
 	    FINAL_LDFLAGS+= -L/usr/pkg/lib
 	    FINAL_LIBS+= -lexecinfo
-    	endif
+	endif
 else
 ifeq ($(uname_S),FreeBSD)
 	# FreeBSD
@@ -225,6 +227,14 @@ endif
 endif
 endif
 endif
+
+ifdef OPENSSL_PREFIX
+	OPENSSL_CFLAGS=-I$(OPENSSL_PREFIX)/include
+	OPENSSL_LDFLAGS=-L$(OPENSSL_PREFIX)/lib
+	# Also export OPENSSL_PREFIX so it ends up in deps sub-Makefiles
+	export OPENSSL_PREFIX
+endif
+
 # Include paths to dependencies
 FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram
 FINAL_CXXFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src -I../deps/hdr_histogram
@@ -254,6 +264,7 @@ endif
 ifeq ($(BUILD_WITH_SYSTEMD),yes)
 	FINAL_LIBS+=$(LIBSYSTEMD_LIBS)
 	FINAL_CFLAGS+= -DHAVE_LIBSYSTEMD
+	FINAL_CXXFLAGS+= -DHAVE_LIBSYSTEMD
 endif
 
 ifeq ($(MALLOC),tcmalloc)
@@ -312,6 +323,14 @@ else
     endef
 endif
 
+# Alpine OS doesn't have support for the execinfo backtrace library we use for debug, so we provide an alternate implementation using libwunwind.
+OS := $(shell cat /etc/os-release | grep ID= | head -n 1 | cut -d'=' -f2)
+ifeq ($(OS),alpine)
+    FINAL_CXXFLAGS+=-DUNW_LOCAL_ONLY
+    FINAL_LIBS += -lunwind
+endif
+
+
 REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)
 REDIS_CXX=$(QUIET_CC)$(CXX) $(FINAL_CXXFLAGS)
 KEYDB_AS=$(QUIET_CC) as --64 -g
@@ -367,9 +386,9 @@ persist-settings: distclean
 	echo CFLAGS=$(CFLAGS) >> .make-settings
 	echo CXXFLAGS=$(CXXFLAGS) >> .make-settings
 	echo LDFLAGS=$(LDFLAGS) >> .make-settings
-	echo REDIS_CFLAGS=$(REDIS_CFLAGS) >> .make-settings
-	echo REDIS_CXXFLAGS=$(REDIS_CXXFLAGS) >> .make-settings
-	echo REDIS_LDFLAGS=$(REDIS_LDFLAGS) >> .make-settings
+	echo KEYDB_CFLAGS=$(KEYDB_CFLAGS) >> .make-settings
+	echo KEYDB_CXXFLAGS=$(KEYDB_CXXFLAGS) >> .make-settings
+	echo KEYDB_LDFLAGS=$(KEYDB_LDFLAGS) >> .make-settings
 	echo PREV_FINAL_CFLAGS=$(FINAL_CFLAGS) >> .make-settings
 	echo PREV_FINAL_CXXFLAGS=$(FINAL_CXXFLAGS) >> .make-settings
 	echo PREV_FINAL_LDFLAGS=$(FINAL_LDFLAGS) >> .make-settings
@@ -443,6 +462,7 @@ clean:
 distclean: clean
 	-(cd ../deps && $(MAKE) distclean)
 	-(cd modules && $(MAKE) clean)
+	-(cd ../tests/modules && $(MAKE) clean)
 	-(rm -f .make-*)
 
 .PHONY: distclean
@@ -450,6 +470,9 @@ distclean: clean
 test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME)
 	@(cd ..; ./runtest)
 
+test-modules: $(REDIS_SERVER_NAME)
+	@(cd ..; ./runtest-moduleapi)
+
 test-sentinel: $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME)
 	@(cd ..; ./runtest-sentinel)
 
@@ -471,10 +494,10 @@ bench: $(REDIS_BENCHMARK_NAME)
 	@echo ""
 	@echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386"
 	@echo ""
-	$(MAKE) CXXFLAGS="-m32" CFLAGS="-m32" LDFLAGS="-m32"
+	$(MAKE) CXXFLAGS="-m32" CFLAGS="-m32" LDFLAGS="-m32" TARGET32=true
 
 gcov:
-	$(MAKE) REDIS_CXXFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" REDIS_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" REDIS_LDFLAGS="-fprofile-arcs -ftest-coverage"
+	$(MAKE) KEYDB_CXXFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" KEYDB_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" KEYDB_LDFLAGS="-fprofile-arcs -ftest-coverage"
 
 noopt:
 	$(MAKE) OPTIMIZATION="-O0"
@@ -483,7 +506,7 @@ valgrind:
 	$(MAKE) OPTIMIZATION="-O0" USEASM="false" MALLOC="libc" CFLAGS="-DSANITIZE" CXXFLAGS="-DSANITIZE"
 
 helgrind:
-	$(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" REDIS_CFLAGS="-I/usr/local/include" REDIS_LDFLAGS="-L/usr/local/lib"
+	$(MAKE) OPTIMIZATION="-O0" MALLOC="libc" CFLAGS="-D__ATOMIC_VAR_FORCE_SYNC_MACROS" KEYDB_CFLAGS="-I/usr/local/include" KEYDB_LDFLAGS="-L/usr/local/lib"
 
 src/help.h:
 	@../utils/generate-command-help.rb > help.h

src/acl.cpp

@@ -223,12 +223,18 @@ uint64_t ACLGetCommandCategoryFlagByName(const char *name) {
     return 0; /* No match. */
 }
 
-/* Method for passwords/pattern comparison used for the user->passwords list
+/* Method for pattern comparison used for the user->patterns list
  * so that we can search for items with listSearchKey(). */
 int ACLListMatchSds(void *a, void *b) {
     return sdscmp((sds)a,(sds)b) == 0;
 }
 
+/* Method for password comparison used for the user->passwords list
+ * Like the above, but uses a time independant compare for security reasons */
+int ACLListMatchSdsSecure(void *a, void* b) {
+    return time_independent_strcmp((sds)a,(sds)b) == 0;
+}
+
 /* Method to free list elements from ACL users password/patterns lists. */
 void ACLListFreeSds(const void *item) {
     sdsfree((sds)item);
@@ -253,7 +259,7 @@ user *ACLCreateUser(const char *name, size_t namelen) {
     u->passwords = listCreate();
     u->patterns = listCreate();
     u->channels = listCreate();
-    listSetMatchMethod(u->passwords,ACLListMatchSds);
+    listSetMatchMethod(u->passwords,ACLListMatchSdsSecure);
     listSetFreeMethod(u->passwords,ACLListFreeSds);
     listSetDupMethod(u->passwords,ACLListDupSds);
     listSetMatchMethod(u->patterns,ACLListMatchSds);
@@ -1895,10 +1901,6 @@ void addACLLogEntry(client *c, int reason, int argpos, sds username) {
 void aclCommand(client *c) {
     char *sub = szFromObj(c->argv[1]);
     if (!strcasecmp(sub,"setuser") && c->argc >= 3) {
-        /* Consider information about passwords or permissions
-         * to be sensitive, which will be the arguments for this
-         * subcommand. */
-        preventCommandLogging(c); 
         sds username = szFromObj(c->argv[2]);
         /* Check username validity. */
         if (ACLStringHasSpaces(username,sdslen(username))) {
@@ -1915,6 +1917,12 @@ void aclCommand(client *c) {
         user *u = ACLGetUserByName(username,sdslen(username));
         if (u) ACLCopyUser(tempu, u);
 
+        /* Initially redact all of the arguments to not leak any information
+         * about the user. */
+        for (int j = 2; j < c->argc; j++) {
+            redactClientCommandArgument(c, j);
+        }
+
         for (int j = 3; j < c->argc; j++) {
             if (ACLSetUser(tempu,szFromObj(c->argv[j]),sdslen(szFromObj(c->argv[j]))) != C_OK) {
                 const char *errmsg = ACLSetUserStringError();
@@ -2248,6 +2256,8 @@ void authCommand(client *c) {
         addReplyErrorObject(c,shared.syntaxerr);
         return;
     }
+    /* Always redact the second argument */
+    redactClientCommandArgument(c, 1);
 
     /* Handle the two different forms here. The form with two arguments
      * will just use "default" as username. */
@@ -2267,6 +2277,7 @@ void authCommand(client *c) {
     } else {
         username = c->argv[1];
         password = c->argv[2];
+        redactClientCommandArgument(c, 2);
     }
 
     if (ACLAuthenticateUser(c,username,password) == C_OK) {

src/ae.cpp

@@ -53,6 +53,7 @@
 #include "zmalloc.h"
 #include "config.h"
 #include "serverassert.h"
+#include "readwritelock.h"
 
 #ifdef USE_MUTEX
 thread_local int cOwnLock = 0;
@@ -87,6 +88,7 @@ mutex_wrapper g_lock;
 #else
 fastlock g_lock("AE (global)");
 #endif
+readWriteLock g_forkLock("Fork (global)");
 thread_local aeEventLoop *g_eventLoopThisThread = NULL;
 
 /* Include the best multiplexing layer supported by this system.
@@ -151,8 +153,11 @@ void aeProcessCmd(aeEventLoop *eventLoop, int fd, void *, int )
         case AE_ASYNC_OP::PostFunction:
             {
             std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
-            if (cmd.fLock)
+            if (cmd.fLock) {
+                g_forkLock.releaseRead();
                 ulock.lock();
+                g_forkLock.acquireRead();
+            }
             ((aePostFunctionProc*)cmd.proc)(cmd.clientData);
             break;
             }
@@ -160,8 +165,11 @@ void aeProcessCmd(aeEventLoop *eventLoop, int fd, void *, int )
         case AE_ASYNC_OP::PostCppFunction:
         {
             std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
-            if (cmd.fLock)
+            if (cmd.fLock) {
+                g_forkLock.releaseRead();
                 ulock.lock();
+                g_forkLock.acquireRead();
+            }
             (*cmd.pfn)();
 
             delete cmd.pfn;
@@ -545,7 +553,11 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
             if (te->next)
                 te->next->prev = te->prev;
             if (te->finalizerProc) {
-                if (!ulock.owns_lock()) ulock.lock();
+                if (!ulock.owns_lock()) {
+                    g_forkLock.releaseRead();
+                    ulock.lock();
+                    g_forkLock.acquireRead();
+                }
                 te->finalizerProc(eventLoop, te->clientData);
                 now = getMonotonicUs();
             }
@@ -565,7 +577,11 @@ static int processTimeEvents(aeEventLoop *eventLoop) {
         }
 
         if (te->when <= now) {
-            if (!ulock.owns_lock()) ulock.lock();
+            if (!ulock.owns_lock()) {
+                g_forkLock.releaseRead();
+                ulock.lock();
+                g_forkLock.acquireRead();
+            }
             int retval;
 
             id = te->id;
@@ -589,8 +605,11 @@ extern "C" void ProcessEventCore(aeEventLoop *eventLoop, aeFileEvent *fe, int ma
 {
 #define LOCK_IF_NECESSARY(fe, tsmask) \
     std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock); \
-    if (!(fe->mask & tsmask)) \
-        ulock.lock()
+    if (!(fe->mask & tsmask)) { \
+        g_forkLock.releaseRead(); \
+        ulock.lock(); \
+        g_forkLock.acquireRead(); \
+    }
 
     int fired = 0; /* Number of events fired for current fd. */
 
@@ -702,8 +721,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
 
         if (eventLoop->beforesleep != NULL && flags & AE_CALL_BEFORE_SLEEP) {
             std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
-            if (!(eventLoop->beforesleepFlags & AE_SLEEP_THREADSAFE))
+            if (!(eventLoop->beforesleepFlags & AE_SLEEP_THREADSAFE)) {
+                g_forkLock.releaseRead();
                 ulock.lock();
+                g_forkLock.acquireRead();
+            }
             eventLoop->beforesleep(eventLoop);
         }
 
@@ -714,8 +736,11 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags)
         /* After sleep callback. */
         if (eventLoop->aftersleep != NULL && flags & AE_CALL_AFTER_SLEEP) {
             std::unique_lock<decltype(g_lock)> ulock(g_lock, std::defer_lock);
-            if (!(eventLoop->aftersleepFlags & AE_SLEEP_THREADSAFE))
+            if (!(eventLoop->aftersleepFlags & AE_SLEEP_THREADSAFE)) {
+                g_forkLock.releaseRead();
                 ulock.lock();
+                g_forkLock.acquireRead();
+            }
             eventLoop->aftersleep(eventLoop);
         }
 
@@ -789,9 +814,21 @@ void setAeLockSetThreadSpinWorker(spin_worker worker)
     tl_worker = worker;
 }
 
+void aeThreadOnline()
+{
+    g_forkLock.acquireRead();
+}
+
 void aeAcquireLock()
 {
+    g_forkLock.releaseRead();
     g_lock.lock(tl_worker);
+    g_forkLock.acquireRead();
+}
+
+void aeAcquireForkLock()
+{
+    g_forkLock.upgradeWrite();
 }
 
 int aeTryAcquireLock(int fWeak)
@@ -799,11 +836,21 @@ int aeTryAcquireLock(int fWeak)
     return g_lock.try_lock(!!fWeak);
 }
 
+void aeThreadOffline()
+{
+    g_forkLock.releaseRead();
+}
+
 void aeReleaseLock()
 {
     g_lock.unlock();
 }
 
+void aeReleaseForkLock()
+{
+    g_forkLock.downgradeWrite();
+}
+
 int aeThreadOwnsLock()
 {
     return g_lock.fOwnLock();

src/ae.h

@@ -164,9 +164,13 @@ void aeSetDontWait(aeEventLoop *eventLoop, int noWait);
 void aeClosePipesForForkChild(aeEventLoop *eventLoop);
 
 void setAeLockSetThreadSpinWorker(spin_worker worker);
+void aeThreadOnline();
 void aeAcquireLock();
+void aeAcquireForkLock();
 int aeTryAcquireLock(int fWeak);
+void aeThreadOffline();
 void aeReleaseLock();
+void aeReleaseForkLock();
 int aeThreadOwnsLock();
 
 #ifdef __cplusplus

src/ae_kqueue.c

@@ -36,8 +36,29 @@
 typedef struct aeApiState {
     int kqfd;
     struct kevent *events;
+
+    /* Events mask for merge read and write event.
+     * To reduce memory consumption, we use 2 bits to store the mask
+     * of an event, so that 1 byte will store the mask of 4 events. */
+    char *eventsMask; 
 } aeApiState;
 
+#define EVENT_MASK_MALLOC_SIZE(sz) (((sz) + 3) / 4)
+#define EVENT_MASK_OFFSET(fd) ((fd) % 4 * 2)
+#define EVENT_MASK_ENCODE(fd, mask) (((mask) & 0x3) << EVENT_MASK_OFFSET(fd))
+
+static inline int getEventMask(const char *eventsMask, int fd) {
+    return (eventsMask[fd/4] >> EVENT_MASK_OFFSET(fd)) & 0x3;
+}
+
+static inline void addEventMask(char *eventsMask, int fd, int mask) {
+    eventsMask[fd/4] |= EVENT_MASK_ENCODE(fd, mask);
+}
+
+static inline void resetEventMask(char *eventsMask, int fd) {
+    eventsMask[fd/4] &= ~EVENT_MASK_ENCODE(fd, 0x3);
+}
+
 static int aeApiCreate(aeEventLoop *eventLoop) {
     aeApiState *state = (aeApiState*)zmalloc(sizeof(aeApiState), MALLOC_LOCAL);
 
@@ -54,6 +75,8 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
         return -1;
     }
     anetCloexec(state->kqfd);
+    state->eventsMask = (char*)zmalloc(EVENT_MASK_MALLOC_SIZE(eventLoop->setsize), MALLOC_LOCAL);
+    memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(eventLoop->setsize));
     eventLoop->apidata = state;
     return 0;
 }
@@ -61,7 +84,9 @@ static int aeApiCreate(aeEventLoop *eventLoop) {
 static int aeApiResize(aeEventLoop *eventLoop, int setsize) {
     aeApiState *state = (aeApiState*)eventLoop->apidata;
 
-    state->events = (struct kevent*)zrealloc(state->events, sizeof(struct kevent)*setsize, MALLOC_LOCAL);
+    state->events = (struct kevent*)zrealloc(state->events, sizeof(struct kevent)*setsize, MALLOC_LOCAL);    
+    state->eventsMask = (char*)zrealloc(state->eventsMask, EVENT_MASK_MALLOC_SIZE(setsize), MALLOC_LOCAL);
+    memset(state->eventsMask, 0, EVENT_MASK_MALLOC_SIZE(setsize));
     return 0;
 }
 
@@ -70,6 +95,7 @@ static void aeApiFree(aeEventLoop *eventLoop) {
 
     close(state->kqfd);
     zfree(state->events);
+    zfree(state->eventsMask);
     zfree(state);
 }
 
@@ -120,15 +146,37 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) {
     if (retval > 0) {
         int j;
 
-        numevents = retval;
-        for(j = 0; j < numevents; j++) {
-            int mask = 0;
+        /* Normally we execute the read event first and then the write event.
+         * When the barrier is set, we will do it reverse.
+         * 
+         * However, under kqueue, read and write events would be separate
+         * events, which would make it impossible to control the order of
+         * reads and writes. So we store the event's mask we've got and merge
+         * the same fd events later. */
+        for (j = 0; j < retval; j++) {
             struct kevent *e = state->events+j;
+            int fd = e->ident;
+            int mask = 0; 
 
-            if (e->filter == EVFILT_READ) mask |= AE_READABLE;
-            if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE;
-            eventLoop->fired[j].fd = e->ident;
-            eventLoop->fired[j].mask = mask;
+            if (e->filter == EVFILT_READ) mask = AE_READABLE;
+            else if (e->filter == EVFILT_WRITE) mask = AE_WRITABLE;
+            addEventMask(state->eventsMask, fd, mask);
+        }
+
+        /* Re-traversal to merge read and write events, and set the fd's mask to
+         * 0 so that events are not added again when the fd is encountered again. */
+        numevents = 0;
+        for (j = 0; j < retval; j++) {
+            struct kevent *e = state->events+j;
+            int fd = e->ident;
+            int mask = getEventMask(state->eventsMask, fd);
+
+            if (mask) {
+                eventLoop->fired[numevents].fd = fd;
+                eventLoop->fired[numevents].mask = mask;
+                resetEventMask(state->eventsMask, fd);
+                numevents++;
+            }
         }
     }
     return numevents;

src/aof.cpp

@@ -131,7 +131,7 @@ void installAofRewriteEvent()
         g_pserver->aof_rewrite_pending = true;
         int res = aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, [] {
             g_pserver->aof_rewrite_pending = false;
-            if (g_pserver->aof_pipe_write_data_to_child >= 0)
+            if (!g_pserver->aof_stop_sending_diff && g_pserver->aof_pipe_write_data_to_child >= 0)
                 aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, g_pserver->aof_pipe_write_data_to_child, AE_WRITABLE, aofChildWriteDiffData, NULL);
         });
         if (res != AE_OK)
@@ -879,7 +879,7 @@ int loadAppendOnlyFile(char *filename) {
     } else {
         /* RDB preamble. Pass loading the RDB functions. */
         rio rdb;
-        rdbSaveInfo rsi = RDB_SAVE_INFO_INIT;
+        rdbSaveInfo rsi;
 
         serverLog(LL_NOTICE,"Reading RDB preamble from AOF file...");
         if (fseek(fp,0,SEEK_SET) == -1) goto readerr;

src/bitops.cpp

@@ -37,8 +37,8 @@
 /* Count number of bits set in the binary array pointed by 's' and long
  * 'count' bytes. The implementation of this function is required to
  * work with an input string length up to 512 MB or more (server.proto_max_bulk_len) */
-size_t redisPopcount(const void *s, long count) {
-    size_t bits = 0;
+long long redisPopcount(const void *s, long count) {
+    long long bits = 0;
     unsigned char *p = (unsigned char*)s;
     uint32_t *p4;
     static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8};
@@ -98,11 +98,11 @@ size_t redisPopcount(const void *s, long count) {
  * no zero bit is found, it returns count*8 assuming the string is zero
  * padded on the right. However if 'bit' is 1 it is possible that there is
  * not a single set bit in the bitmap. In this special case -1 is returned. */
-long redisBitpos(const void *s, unsigned long count, int bit) {
+long long redisBitpos(const void *s, unsigned long count, int bit) {
     unsigned long *l;
     unsigned char *c;
     unsigned long skipval, word = 0, one;
-    long pos = 0; /* Position of bit, to return to the caller. */
+    long long pos = 0; /* Position of bit, to return to the caller. */
     unsigned long j;
     int found;
 
@@ -405,6 +405,10 @@ void printBits(unsigned char *p, unsigned long count) {
 #define BITFIELDOP_SET 1
 #define BITFIELDOP_INCRBY 2
 
+/* Make sure we don't bit shift too many bits at a time.
+ * Even this amount is probably way too large. */
+#define BITOP_SHIFT_MAX (1<<30)
+
 /* This helper function used by GETBIT / SETBIT parses the bit offset argument
  * making sure an error is returned if it is negative or if it overflows
  * Redis 512 MB limit for the string value or more (server.proto_max_bulk_len).
@@ -412,7 +416,7 @@ void printBits(unsigned char *p, unsigned long count) {
  * If the 'hash' argument is true, and 'bits is positive, then the command
  * will also parse bit offsets prefixed by "#". In such a case the offset
  * is multiplied by 'bits'. This is useful for the BITFIELD command. */
-int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int bits) {
+int getBitOffsetFromArgument(client *c, robj *o, uint64_t *offset, int hash, int bits) {
     long long loffset;
     const char *err = "bit offset is not an integer or out of range";
     char *p = szFromObj(o);
@@ -437,7 +441,7 @@ int getBitOffsetFromArgument(client *c, robj *o, size_t *offset, int hash, int b
         return C_ERR;
     }
 
-    *offset = (size_t)loffset;
+    *offset = loffset;
     return C_OK;
 }
 
@@ -479,7 +483,7 @@ int getBitfieldTypeFromArgument(client *c, robj *o, int *sign, int *bits) {
  * so that the 'maxbit' bit can be addressed. The object is finally
  * returned. Otherwise if the key holds a wrong type NULL is returned and
  * an error is sent to the client. */
-robj *lookupStringForBitCommand(client *c, size_t maxbit) {
+robj *lookupStringForBitCommand(client *c, uint64_t maxbit) {
     size_t byte = maxbit >> 3;
     robj *o = lookupKeyWrite(c->db,c->argv[1]);
     if (checkType(c,o,OBJ_STRING)) return NULL;
@@ -529,7 +533,7 @@ const unsigned char *getObjectReadOnlyString(robj_roptr o, long *len, char *llbu
 void setbitCommand(client *c) {
     robj *o;
     const char *err = "bit is not an integer or out of range";
-    size_t bitoffset;
+    uint64_t bitoffset;
     ssize_t byte, bit;
     int byteval, bitval;
     long on;
@@ -568,7 +572,7 @@ void setbitCommand(client *c) {
 void getbitCommand(client *c) {
     robj_roptr o;
     char llbuf[32];
-    size_t bitoffset;
+    uint64_t bitoffset;
     size_t byte, bit;
     size_t bitval = 0;
 
@@ -642,6 +646,11 @@ void bitopCommand(client *c) {
             addReplyError(c, "BITOP SHIFT's last parameter must be an integer");
             return;
         }
+        if (shift > BITOP_SHIFT_MAX) {
+            addReplyError(c, "BITOP SHIFT value is too large.");
+            return;
+        }
+
         if (op == BITOP_RSHIFT)
             shift = -shift;
     }
@@ -682,7 +691,7 @@ void bitopCommand(client *c) {
 
     if (fShiftOp)
     {
-        long newlen = (long)maxlen + shift/CHAR_BIT;
+        long long newlen = (long long)maxlen + shift/CHAR_BIT;
         if (shift > 0 && (shift % CHAR_BIT) != 0)
             newlen++;
 
@@ -694,14 +703,14 @@ void bitopCommand(client *c) {
             res = (unsigned char*) sdsnewlen(NULL,newlen);
             if (shift >= 0)
             {   // left shift
-                long byteoffset = shift/CHAR_BIT;
+                long long byteoffset = shift/CHAR_BIT;
                 memset(res, 0, byteoffset);
-                long srcLen = newlen - byteoffset - ((shift % CHAR_BIT) ? 1 : 0);
+                long long srcLen = newlen - byteoffset - ((shift % CHAR_BIT) ? 1 : 0);
 
                 // now the bitshift+copy
                 unsigned bitshift = shift % CHAR_BIT;
                 unsigned char carry = 0;
-                for (long iSrc = 0; iSrc < srcLen; ++iSrc)
+                for (long long iSrc = 0; iSrc < srcLen; ++iSrc)
                 {
                     res[byteoffset+iSrc] = (src[0][iSrc] << bitshift) | carry;
                     carry = src[0][iSrc] >> (CHAR_BIT - bitshift);
@@ -711,14 +720,14 @@ void bitopCommand(client *c) {
             } 
             else 
             {   // right shift
-                long byteoffset = -shift/CHAR_BIT;
+                long long byteoffset = -shift/CHAR_BIT;
                 unsigned bitshift = -shift % CHAR_BIT;
                 if (bitshift)
                     ++byteoffset;
                 res[0] = (src[0][byteoffset] << (CHAR_BIT-bitshift));
                 if (byteoffset > 0)
                     res[0] |= (src[0][byteoffset-1] >> bitshift);
-                for (long idx = 1; idx < newlen; ++idx)
+                for (long long idx = 1; idx < newlen; ++idx)
                 {
                     res[idx] = (src[0][byteoffset+idx] << (CHAR_BIT-bitshift)) | (src[0][byteoffset+idx-1] >> bitshift);
                 }
@@ -800,8 +809,8 @@ void bitopCommand(client *c) {
                     }
                 }
             }
+            #endif
         }
-        #endif
 
         /* j is set to the next byte to process by the previous loop. */
         for (; j < maxlen; j++) {
@@ -961,7 +970,7 @@ void bitposCommand(client *c) {
         addReplyLongLong(c, -1);
     } else {
         long bytes = end-start+1;
-        long pos = redisBitpos(p+start,bytes,bit);
+        long long pos = redisBitpos(p+start,bytes,bit);
 
         /* If we are looking for clear bits, and the user specified an exact
          * range with start-end, we can't consider the right of the range as
@@ -970,11 +979,11 @@ void bitposCommand(client *c) {
          * So if redisBitpos() returns the first bit outside the range,
          * we return -1 to the caller, to mean, in the specified range there
          * is not a single "0" bit. */
-        if (end_given && bit == 0 && pos == bytes*8) {
+        if (end_given && bit == 0 && pos == (long long)bytes<<3) {
             addReplyLongLong(c,-1);
             return;
         }
-        if (pos != -1) pos += start*8; /* Adjust for the bytes we skipped. */
+        if (pos != -1) pos += (long long)start<<3; /* Adjust for the bytes we skipped. */
         addReplyLongLong(c,pos);
     }
 }
@@ -1006,12 +1015,12 @@ struct bitfieldOp {
  * GET subcommand is allowed, other subcommands will return an error. */
 void bitfieldGeneric(client *c, int flags) {
     robj_roptr o;
-    size_t bitoffset;
+    uint64_t bitoffset;
     int j, numops = 0, changes = 0;
     struct bitfieldOp *ops = NULL; /* Array of ops to execute at end. */
     int owtype = BFOVERFLOW_WRAP; /* Overflow type. */
     int readonly = 1;
-    size_t highest_write_offset = 0;
+    uint64_t highest_write_offset = 0;
 
     for (j = 2; j < c->argc; j++) {
         int remargs = c->argc-j-1; /* Remaining args other than current. */
@@ -1201,9 +1210,9 @@ void bitfieldGeneric(client *c, int flags) {
              * object boundaries. */
             memset(buf,0,9);
             int i;
-            size_t byte = thisop->offset >> 3;
+            uint64_t byte = thisop->offset >> 3;
             for (i = 0; i < 9; i++) {
-                if (src == NULL || i+byte >= (size_t)strlen) break;
+                if (src == NULL || i+byte >= (uint64_t)strlen) break;
                 buf[i] = src[i+byte];
             }
 

src/blocked.cpp

@@ -89,6 +89,11 @@ typedef struct bkinfo {
  * and will be processed when the client is unblocked. */
 void blockClient(client *c, int btype) {
     serverAssert(GlobalLocksAcquired());
+    /* Master client should never be blocked unless pause or module */
+    serverAssert(!(c->flags & CLIENT_MASTER &&
+                   btype != BLOCKED_MODULE &&
+                   btype != BLOCKED_PAUSE));
+
     c->flags |= CLIENT_BLOCKED;
     c->btype = btype;
     g_pserver->blocked_clients++;
@@ -173,12 +178,11 @@ void queueClientForReprocessing(client *c) {
     /* The client may already be into the unblocked list because of a previous
      * blocking operation, don't add back it into the list multiple times. */
     serverAssert(GlobalLocksAcquired());
-    fastlock_lock(&c->lock);
+    std::unique_lock<fastlock> ul(c->lock);
     if (!(c->flags & CLIENT_UNBLOCKED)) {
         c->flags |= CLIENT_UNBLOCKED;
         listAddNodeTail(g_pserver->rgthreadvar[c->iel].unblocked_clients,c);
     }
-    fastlock_unlock(&c->lock);
 }
 
 /* Unblock a client calling the right function depending on the kind
@@ -588,7 +592,6 @@ void handleClientsBlockedOnKeys(void) {
              * lookup, invalidating the first one.
              * See https://github.com/redis/redis/pull/6554. */
             serverTL->fixed_time_expire++;
-            updateCachedTime(0);
 
             /* Serve clients blocked on the key. */
             robj *o = lookupKeyWrite(rl->db,rl->key);
@@ -792,4 +795,4 @@ void signalKeyAsReady(redisDb *db, sds key, int type) {
     redisObjectStack o;
     initStaticStringObject(o, key);
     signalKeyAsReady(db, &o, type);
-}
+}

src/childinfo.cpp

@@ -93,7 +93,7 @@ void sendChildInfoGeneric(childInfoType info_type, size_t keys, double progress,
         if (cow) {
             serverLog((info_type == CHILD_INFO_TYPE_CURRENT_INFO) ? LL_VERBOSE : LL_NOTICE,
                       "%s: %zu MB of memory used by copy-on-write",
-                      pname, data.cow / (1024 * 1024));
+                      pname, cow / (1024 * 1024));
         }
     }
 

src/cluster.cpp

@@ -561,7 +561,7 @@ void clusterInit(void) {
     
     serverAssert(serverTL == &g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN]);
     if (createSocketAcceptHandler(&g_pserver->cfd, clusterAcceptHandler) != C_OK) {
-        serverPanic("Unrecoverable error creating Redis Cluster socket accept handler.");
+        serverPanic("Unrecoverable error creating KeyDB Cluster socket accept handler.");
     }
 
     /* The slots -> keys map is a radix tree. Initialize it here. */
@@ -746,6 +746,7 @@ void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
         }
         connNonBlock(conn);
         connEnableTcpNoDelay(conn);
+        connKeepAlive(conn,g_pserver->cluster_node_timeout * 2);
 
         /* Use non-blocking I/O for cluster messages. */
         serverLog(LL_VERBOSE,"Accepting cluster node connection from %s:%d", cip, cport);
@@ -2446,7 +2447,10 @@ void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) {
     if (sdslen(link->sndbuf) == 0 && msglen != 0)
     {
         aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, [link] {
-            connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1);
+            /* The connection could be timed out before this posted function executes (thanks to TCP keepalive).
+             * So check that the connection is still there before setting the write handler, otherwise you segfault */
+            if (link->conn != nullptr)
+                connSetWriteHandlerWithBarrier(link->conn, clusterWriteHandler, 1);
         });
     }
 
@@ -5167,11 +5171,12 @@ void dumpCommand(client *c) {
 
 /* KEYDB.MVCCRESTORE key mvcc expire serialized-value */
 void mvccrestoreCommand(client *c) {
-    long long mvcc, expire;
+    long long expire;
+    uint64_t mvcc;
     robj *key = c->argv[1], *obj = nullptr;
     int type;
     
-    if (getLongLongFromObjectOrReply(c, c->argv[2], &mvcc, "Invalid MVCC Tstamp") != C_OK)
+    if (getUnsignedLongLongFromObjectOrReply(c, c->argv[2], &mvcc, "Invalid MVCC Tstamp") != C_OK)
         return;
 
     if (getLongLongFromObjectOrReply(c, c->argv[3], &expire, "Invalid expire") != C_OK)
@@ -5189,7 +5194,7 @@ void mvccrestoreCommand(client *c) {
     rio payload;
     rioInitWithBuffer(&payload,szFromObj(c->argv[4]));
     if (((type = rdbLoadObjectType(&payload)) == -1) ||
-        ((obj = rdbLoadObject(type,&payload,szFromObj(key), OBJ_MVCC_INVALID)) == NULL))
+        ((obj = rdbLoadObject(type,&payload,szFromObj(key),NULL,OBJ_MVCC_INVALID)) == NULL))
     {
         addReplyError(c,"Bad data format");
         return;
@@ -5275,7 +5280,7 @@ void restoreCommand(client *c) {
 
     rioInitWithBuffer(&payload,szFromObj(c->argv[3]));
     if (((type = rdbLoadObjectType(&payload)) == -1) ||
-        ((obj = rdbLoadObject(type,&payload,szFromObj(key), OBJ_MVCC_INVALID)) == NULL))
+        ((obj = rdbLoadObject(type,&payload,szFromObj(key),NULL,OBJ_MVCC_INVALID)) == NULL))
     {
         addReplyError(c,"Bad data format");
         return;
@@ -5459,13 +5464,16 @@ void migrateCommand(client *c) {
             }
             j++;
             password = szFromObj(c->argv[j]);
+            redactClientCommandArgument(c,j);
         } else if (!strcasecmp(szFromObj(c->argv[j]),"auth2")) {
             if (moreargs < 2) {
                 addReply(c,shared.syntaxerr);
                 return;
             }
             username = szFromObj(c->argv[++j]);
+            redactClientCommandArgument(c,j);
             password = szFromObj(c->argv[++j]);
+            redactClientCommandArgument(c,j);
         } else if (!strcasecmp(szFromObj(c->argv[j]),"keys")) {
             if (sdslen(szFromObj(c->argv[3])) != 0) {
                 addReplyError(c,
@@ -5555,11 +5563,11 @@ try_again:
     for (j = 0; j < num_keys; j++) {
         long long ttl = 0;
         expireEntry *pexpire = getExpire(c->db,kv[j]);
-        long long expireat = -1;
+        long long expireat = INVALID_EXPIRE;
         if (pexpire != nullptr)
             pexpire->FGetPrimaryExpire(&expireat);
 
-        if (expireat != -1) {
+        if (expireat != INVALID_EXPIRE) {
             ttl = expireat-mstime();
             if (ttl < 0) {
                 continue;
@@ -5567,9 +5575,10 @@ try_again:
             if (ttl < 1) ttl = 1;
         }
 
-        /* Relocate valid (non expired) keys into the array in successive
+        /* Relocate valid (non expired) keys and values into the array in successive
          * positions to remove holes created by the keys that were present
          * in the first lookup but are now expired after the second lookup. */
+        ov[non_expired] = ov[j];
         kv[non_expired++] = kv[j];
 
         serverAssertWithInfo(c,NULL,

src/config.cpp

@@ -359,7 +359,7 @@ static int updateOOMScoreAdjValues(sds *args, const char **err, int apply) {
 
     if (values[CONFIG_OOM_REPLICA] < values[CONFIG_OOM_MASTER] ||
         values[CONFIG_OOM_BGCHILD] < values[CONFIG_OOM_REPLICA]) {
-            serverLog(LOG_WARNING,
+            serverLog(LL_WARNING,
                     "The oom-score-adj-values configuration may not work for non-privileged processes! "
                     "Please consult the documentation.");
     }
@@ -800,7 +800,7 @@ void configSetCommand(client *c) {
             (config->alias && !strcasecmp(szFromObj(c->argv[2]),config->alias))))
         {
             if (config->flags & SENSITIVE_CONFIG) {
-                preventCommandLogging(c);
+                redactClientCommandArgument(c,3);
             }
             if (!config->interface.set(config->data,szFromObj(o),1,&errstr)) {
                 goto badfmt;

src/config.h

@@ -36,6 +36,7 @@
 
 #ifdef __linux__
 #include <features.h>
+#include <fcntl.h>
 #endif
 
 #define CONFIG_DEFAULT_RDB_FILENAME "dump.rdb"

src/connection.cpp

@@ -437,6 +437,9 @@ void connSetThreadAffinity(connection *conn, int cpu) {
     {
         serverLog(LL_WARNING, "Failed to set socket affinity");
     }
+#else
+    (void)conn;
+    (void)cpu;
 #endif
 }
 

src/cron.cpp

@@ -51,6 +51,7 @@ void cronCommand(client *c)
         
         if (c->argc < (6 + numkeys)) {
             addReplyError(c, "Missing arguments or numkeys is too big");
+            return;
         }
     }
 

src/db.cpp

@@ -31,6 +31,7 @@
 #include "cluster.h"
 #include "atomicvar.h"
 #include "aelocker.h"
+#include "latency.h"
 
 #include <signal.h>
 #include <ctype.h>
@@ -194,7 +195,7 @@ robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags) {
 robj *lookupKeyWrite(redisDb *db, robj *key) {
     return lookupKeyWriteWithFlags(db, key, LOOKUP_NONE);
 }
-static void SentReplyOnKeyMiss(client *c, robj *reply){
+void SentReplyOnKeyMiss(client *c, robj *reply){
     serverAssert(sdsEncodedObject(reply));
     sds rep = szFromObj(reply);
     if (sdslen(rep) > 1 && rep[0] == '-'){
@@ -1403,9 +1404,8 @@ void copyCommand(client *c) {
     }
 
     dbAdd(dst,newkey,newobj);
-    if (expire != nullptr) {
-        if (expire != nullptr) setExpire(c, dst, newkey, expire->duplicate());
-    }
+    if (expire != nullptr)
+        setExpire(c, dst, newkey, expire->duplicate());
 
     /* OK! key copied */
     signalModifiedKey(c,dst,c->argv[2]);
@@ -1671,6 +1671,23 @@ expireEntry *getExpire(redisDb *db, robj_roptr key) {
     return itr.operator->();
 }
 
+/* Delete the specified expired key and propagate expire. */
+void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj) {
+    mstime_t expire_latency;
+    latencyStartMonitor(expire_latency);
+    if (g_pserver->lazyfree_lazy_expire) {
+        dbAsyncDelete(db,keyobj);
+    } else {
+        dbSyncDelete(db,keyobj);
+    }
+    latencyEndMonitor(expire_latency);
+    latencyAddSampleIfNeeded("expire-del",expire_latency);
+    notifyKeyspaceEvent(NOTIFY_EXPIRED,"expired",keyobj,db->id);
+    signalModifiedKey(NULL,db,keyobj);
+    propagateExpire(db,keyobj,g_pserver->lazyfree_lazy_expire);
+    g_pserver->stat_expiredkeys++;
+}
+
 /* Propagate expires into slaves and the AOF file.
  * When a key expires in the master, a DEL operation for this key is sent
  * to all the slaves and the AOF file if enabled.
@@ -1752,17 +1769,9 @@ int keyIsExpired(redisDb *db, robj *key) {
     /* Don't expire anything while loading. It will be done later. */
     if (g_pserver->loading) return 0;
 
-    long long when = -1;
-    for (auto &exp : *pexpire)
-    {
-        if (exp.subkey() == nullptr)
-        {
-            when = exp.when();
-            break;
-        }
-    }
+    long long when = pexpire->FGetPrimaryExpire();
 
-    if (when == -1)
+    if (when == INVALID_EXPIRE)
         return 0;
 
     /* If we are in the context of a Lua script, we pretend that time is
@@ -1771,7 +1780,7 @@ int keyIsExpired(redisDb *db, robj *key) {
      * script execution, making propagation to slaves / AOF consistent.
      * See issue #1525 on Github for more information. */
     if (g_pserver->lua_caller) {
-        now = g_pserver->lua_time_start;
+        now = g_pserver->lua_time_snapshot;
     }
     /* If we are in the middle of a command execution, we still want to use
      * a reference time that does not change: in that case we just use the
@@ -1832,14 +1841,8 @@ int expireIfNeeded(redisDb *db, robj *key) {
     if (checkClientPauseTimeoutAndReturnIfPaused()) return 1;
 
     /* Delete the key */
-    g_pserver->stat_expiredkeys++;
-    propagateExpire(db,key,g_pserver->lazyfree_lazy_expire);
-    notifyKeyspaceEvent(NOTIFY_EXPIRED,
-        "expired",key,db->id);
-    int retval = g_pserver->lazyfree_lazy_expire ? dbAsyncDelete(db,key) :
-                                               dbSyncDelete(db,key);
-    if (retval) signalModifiedKey(NULL,db,key);
-    return retval;
+    deleteExpiredKeyAndPropagate(db,key);
+    return 1;
 }
 
 /* -----------------------------------------------------------------------------
@@ -1903,7 +1906,6 @@ int getKeysUsingCommandTable(struct redisCommand *cmd,robj **argv, int argc, get
              * return no keys and expect the command implementation to report
              * an arity or syntax error. */
             if (cmd->flags & CMD_MODULE || cmd->arity < 0) {
-                getKeysFreeResult(result);
                 result->numkeys = 0;
                 return 0;
             } else {

src/debug.cpp

@@ -51,6 +51,12 @@ typedef ucontext_t sigcontext_t;
 #include <cxxabi.h>
 #endif /* HAVE_BACKTRACE */
 
+//UNW_LOCAL_ONLY being set means we use libunwind for backtraces instead of execinfo
+#ifdef UNW_LOCAL_ONLY
+#include <libunwind.h>
+#include <cxxabi.h>
+#endif
+
 #ifdef __CYGWIN__
 #ifndef SA_ONSTACK
 #define SA_ONSTACK 0x08000000
@@ -143,7 +149,7 @@ void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj
     uint32_t aux = htonl(o->type);
     mixDigest(digest,&aux,sizeof(aux));
     expireEntry *pexpire = getExpire(db,keyobj);
-    long long expiretime = -1;
+    long long expiretime = INVALID_EXPIRE;
     char buf[128];
 
     if (pexpire != nullptr)
@@ -275,7 +281,7 @@ void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj
         serverPanic("Unknown object type");
     }
     /* If the key has an expire, add it to the mix */
-    if (expiretime != -1) xorDigest(digest,"!!expire!!",10);
+    if (expiretime != INVALID_EXPIRE) xorDigest(digest,"!!expire!!",10);
 }
 
 /* Compute the dataset digest. Since keys, sets elements, hashes elements
@@ -735,7 +741,7 @@ NULL
         } else if (!strcasecmp(name,"double")) {
             addReplyDouble(c,3.14159265359);
         } else if (!strcasecmp(name,"bignum")) {
-            addReplyProto(c,"(1234567999999999999999999999999999999\r\n",40);
+            addReplyBigNum(c,"1234567999999999999999999999999999999",37);
         } else if (!strcasecmp(name,"null")) {
             addReplyNull(c);
         } else if (!strcasecmp(name,"array")) {
@@ -751,11 +757,13 @@ NULL
                 addReplyBool(c, j == 1);
             }
         } else if (!strcasecmp(name,"attrib")) {
-            addReplyAttributeLen(c,1);
-            addReplyBulkCString(c,"key-popularity");
-            addReplyArrayLen(c,2);
-            addReplyBulkCString(c,"key:123");
-            addReplyLongLong(c,90);
+            if (c->resp >= 3) {
+                addReplyAttributeLen(c,1);
+                addReplyBulkCString(c,"key-popularity");
+                addReplyArrayLen(c,2);
+                addReplyBulkCString(c,"key:123");
+                addReplyLongLong(c,90);
+            }
             /* Attributes are not real replies, so a well formed reply should
              * also have a normal reply type after the attribute. */
             addReplyBulkCString(c,"Some real reply following the attribute");
@@ -947,7 +955,7 @@ void _serverAssert(const char *estr, const char *file, int line) {
     serverLog(LL_WARNING,"==> %s:%d '%s' is not true",file,line,estr);
 
     if (g_pserver->crashlog_enabled) {
-#ifdef HAVE_BACKTRACE
+#if defined HAVE_BACKTRACE || defined UNW_LOCAL_ONLY
         logStackTrace(NULL, 1);
 #endif
         printCrashReport();
@@ -1038,14 +1046,13 @@ void _serverPanic(const char *file, int line, const char *msg, ...) {
     vsnprintf(fmtmsg,sizeof(fmtmsg),msg,ap);
     va_end(ap);
 
-    g_fInCrash = true;
     bugReportStart();
     serverLog(LL_WARNING,"------------------------------------------------");
     serverLog(LL_WARNING,"!!! Software Failure. Press left mouse button to continue");
     serverLog(LL_WARNING,"Guru Meditation: %s #%s:%d",fmtmsg,file,line);
 
     if (g_pserver->crashlog_enabled) {
-#ifdef HAVE_BACKTRACE
+#if defined HAVE_BACKTRACE || defined UNW_LOCAL_ONLY
         logStackTrace(NULL, 1);
 #endif
         printCrashReport();
@@ -1600,6 +1607,65 @@ void safe_write(int fd, const void *pv, ssize_t cb)
     } while (offset < cb);
 }
 
+#ifdef UNW_LOCAL_ONLY
+
+/* Logs the stack trace using the libunwind call.
+ * The eip argument is unused as libunwind only gets local context.
+ * The uplevel argument indicates how many of the calling functions to skip.
+ */
+void logStackTrace(void * eip, int uplevel) {
+    (void)eip;//UNUSED
+    const char *msg;
+    int fd = openDirectLogFiledes();
+
+    if (fd == -1) return; /* If we can't log there is anything to do. */
+
+    msg = "\n------ STACK TRACE ------\n";
+    if (write(fd,msg,strlen(msg)) == -1) {/* Avoid warning. */};
+    unw_cursor_t cursor;
+    unw_context_t context;
+
+    unw_getcontext(&context);
+    unw_init_local(&cursor, &context);
+
+    /* Write symbols to log file */
+    msg = "\nBacktrace:\n";
+    if (write(fd,msg,strlen(msg)) == -1) {/* Avoid warning. */};
+
+    for (int i = 0; i < uplevel; i++) {
+        unw_step(&cursor);
+    }
+
+    while ( unw_step(&cursor) ) {
+    unw_word_t ip, sp, off;
+
+    unw_get_reg(&cursor, UNW_REG_IP, &ip);
+    unw_get_reg(&cursor, UNW_REG_SP, &sp);
+
+    char symbol[256] = {"<unknown>"};
+    char *name = symbol;
+
+    if ( !unw_get_proc_name(&cursor, symbol, sizeof(symbol), &off) ) {
+        int status;
+        if ( (name = abi::__cxa_demangle(symbol, NULL, NULL, &status)) == 0 )
+        name = symbol;
+    }
+
+    dprintf(fd, "%s(+0x%" PRIxPTR ") [0x%016" PRIxPTR "] sp=0x%016" PRIxPTR "\n",
+        name,
+        static_cast<uintptr_t>(off),
+        static_cast<uintptr_t>(ip),
+        static_cast<uintptr_t>(sp));
+
+    if ( name != symbol )
+        free(name);
+    }
+}
+
+#endif /* UNW_LOCAL_ONLY */
+
+#ifdef HAVE_BACKTRACE
+
 void backtrace_symbols_demangle_fd(void **trace, size_t csym, int fd)
 {
     char **syms = backtrace_symbols(trace, csym);
@@ -1643,8 +1709,6 @@ void backtrace_symbols_demangle_fd(void **trace, size_t csym, int fd)
     free(syms);
 }
 
-#ifdef HAVE_BACKTRACE
-
 /* Logs the stack trace using the backtrace() call. This function is designed
  * to be called from signal handlers safely.
  * The eip argument is optional (can take NULL).
@@ -1935,6 +1999,9 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) {
 
     logRegisters(uc);
 #endif
+#ifdef UNW_LOCAL_ONLY
+    logStackTrace(NULL, 1);
+#endif 
 
     printCrashReport();
 
@@ -2029,6 +2096,8 @@ void watchdogSignalHandler(int sig, siginfo_t *info, void *secret) {
     serverLogFromHandler(LL_WARNING,"\n--- WATCHDOG TIMER EXPIRED ---");
 #ifdef HAVE_BACKTRACE
     logStackTrace(getMcontextEip(uc), 1);
+#elif defined UNW_LOCAL_ONLY
+    logStackTrace(NULL, 1);
 #else
     serverLogFromHandler(LL_WARNING,"Sorry: no support for backtrace().");
 #endif

src/dict.cpp

@@ -154,6 +154,10 @@ int _dictExpand(dict *d, unsigned long size, int* malloc_failed)
     dictht n; /* the new hash table */
     unsigned long realsize = _dictNextPower(size);
 
+    /* Detect overflows */
+    if (realsize < size || realsize * sizeof(dictEntry*) < realsize)
+        return DICT_ERR;
+
     /* Rehashing to the same table size is not useful. */
     if (realsize == d->ht[0].size) return DICT_ERR;
 

src/expire.cpp

@@ -72,11 +72,12 @@ void activeExpireCycleExpireFullKey(redisDb *db, const char *key) {
  *----------------------------------------------------------------------------*/
 
 
-void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
+int activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now, size_t &tried) {
     if (!e.FFat())
     {
         activeExpireCycleExpireFullKey(db, e.key());
-        return;
+        ++tried;
+        return 1;
     }
 
     expireEntryFat *pfat = e.pfatentry();
@@ -90,6 +91,7 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
 
     while (!pfat->FEmpty())
     {
+        ++tried;
         if (pfat->nextExpireEntry().when > now)
             break;
 
@@ -97,7 +99,7 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
         if (pfat->nextExpireEntry().spsubkey == nullptr)
         {
             activeExpireCycleExpireFullKey(db, e.key());
-            return;
+            return ++deleted;
         }
 
         switch (val->type)
@@ -107,7 +109,7 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
                 deleted++;
                 if (setTypeSize(val) == 0) {
                     activeExpireCycleExpireFullKey(db, e.key());
-                    return;
+                    return deleted;
                 }
             }
             break;
@@ -117,7 +119,7 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
                 deleted++;
                 if (hashTypeLength(val) == 0) {
                     activeExpireCycleExpireFullKey(db, e.key());
-                    return;
+                    return deleted;
                 }
             }
             break;
@@ -127,7 +129,7 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
                 deleted++;
                 if (zsetLength(val) == 0) {
                     activeExpireCycleExpireFullKey(db, e.key());
-                    return;
+                    return deleted;
                 }
             }
             break;
@@ -142,7 +144,7 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
                 decrRefCount(val);
             }, true /*fLock*/, true /*fForceQueue*/);
         }
-            return;
+            return deleted;
 
         case OBJ_LIST:
         default:
@@ -155,6 +157,9 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
         
         pfat->popfrontExpireEntry();
         fTtlChanged = true;
+        if ((tried % ACTIVE_EXPIRE_CYCLE_SUBKEY_LOOKUPS_PER_LOOP) == 0) {
+            break;
+        }
     }
 
     if (!pfat->FEmpty() && fTtlChanged)
@@ -181,6 +186,7 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
     {
         removeExpire(db, &objKey);
     }
+    return deleted;
 }
 
 int parseUnitString(const char *sz)
@@ -392,10 +398,8 @@ void activeExpireCycleCore(int type) {
         db->expireitr = db->setexpire->enumerate(db->expireitr, now, [&](expireEntry &e) __attribute__((always_inline)) {
             if (e.when() < now)
             {
-                activeExpireCycleExpire(db, e, now);
-                ++expired;
+                expired += activeExpireCycleExpire(db, e, now, tried);
             }
-            ++tried;
 
             if ((tried % ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP) == 0)
             {
@@ -503,8 +507,8 @@ void expireSlaveKeys(void) {
                 if (itr != db->setexpire->end())
                 {
                     if (itr->when() < start) {
-                        activeExpireCycleExpire(g_pserver->db+dbid,*itr,start);
-                        expired = 1;
+                        size_t tried = 0;
+                        expired = activeExpireCycleExpire(g_pserver->db+dbid,*itr,start,tried);
                     }
                 }
 
@@ -680,7 +684,7 @@ void pexpireatCommand(client *c) {
 
 /* Implements TTL and PTTL */
 void ttlGenericCommand(client *c, int output_ms) {
-    long long expire = -1, ttl = -1;
+    long long expire = INVALID_EXPIRE, ttl = -1;
 
     /* If the key does not exist at all, return -2 */
     if (lookupKeyReadWithFlags(c->db,c->argv[1],LOOKUP_NOTOUCH) == nullptr) {
@@ -714,7 +718,7 @@ void ttlGenericCommand(client *c, int output_ms) {
     }
 
     
-    if (expire != -1) {
+    if (expire != INVALID_EXPIRE) {
         ttl = expire-mstime();
         if (ttl < 0) ttl = 0;
     }
@@ -821,8 +825,8 @@ void expireEntryFat::expireSubKey(const char *szSubkey, long long when)
                     fFound = true;
                 }
                 if (fFound) {
-                    m_vecexpireEntries.erase(itr);
                     dictDelete(m_dictIndex, szSubkey);
+                    m_vecexpireEntries.erase(itr);
                     break;
                 }
                 ++itr;

src/expire.h

@@ -1,5 +1,12 @@
 #pragma once
 
+/* 
+ * INVALID_EXPIRE is the value we set the expireEntry's m_when value to when the main key is not expired and the value we return when we try to get the expire time of a key or subkey that is not expired
+ * Want this value to be LLONG_MAX however we use the most significant bit of m_when as a flag to see if the expireEntry is Fat or not so we want to ensure that it is unset hence the (& ~((1LL) << (sizeof(long long)*CHAR_BIT - 1)))
+ * Eventually this number might actually end up being a valid expire time, this could cause bugs so at that time it might be a good idea to use a larger data type.
+ */
+#define INVALID_EXPIRE (LLONG_MAX & ~((1LL) << (sizeof(long long)*CHAR_BIT - 1)))
+
 class expireEntryFat
 {
     friend class expireEntry;
@@ -47,6 +54,7 @@ public:
     ~expireEntryFat();
 
     long long when() const noexcept { return m_vecexpireEntries.front().when; }
+
     const char *key() const noexcept { return m_keyPrimary; }
 
     bool operator<(long long when) const noexcept { return this->when() <  when; }
@@ -66,7 +74,12 @@ class expireEntry {
         sds m_key;
         expireEntryFat *m_pfatentry;
     } u;
-    long long m_when;   // LLONG_MIN means this is a fat entry and we should use the pointer
+    long long m_when;   // bit wise and with FFatMask means this is a fat entry and we should use the pointer
+
+    /* Mask to check if an entry is Fat, most significant bit of m_when being set means it is Fat otherwise it is not */
+    long long FFatMask() const noexcept {
+        return (1LL) << (sizeof(long long)*CHAR_BIT - 1);
+    }
 
     expireEntry() = default;
 public:
@@ -108,7 +121,7 @@ public:
     {
         if (subkey != nullptr)
         {
-            m_when = LLONG_MIN;
+            m_when = FFatMask() | INVALID_EXPIRE;
             u.m_pfatentry = new (MALLOC_LOCAL) expireEntryFat(key);
             u.m_pfatentry->expireSubKey(subkey, when);
         }
@@ -122,7 +135,15 @@ public:
     expireEntry(expireEntryFat *pfatentry)
     {
         u.m_pfatentry = pfatentry;
-        m_when = LLONG_MIN;
+        m_when = FFatMask() | INVALID_EXPIRE;
+        for (auto itr : *this)
+        {
+            if (itr.subkey() == nullptr)
+            {
+                m_when = FFatMask() | itr.when();
+                break;
+            }
+        }
     }
 
     expireEntry(expireEntry &&e)
@@ -159,7 +180,7 @@ public:
             u.m_key = key;
     }
 
-    inline bool FFat() const noexcept { return m_when == LLONG_MIN; }
+    inline bool FFat() const noexcept { return m_when & FFatMask(); }
     expireEntryFat *pfatentry() { assert(FFat()); return u.m_pfatentry; }
 
 
@@ -187,7 +208,7 @@ public:
     { 
         if (FFat())
             return u.m_pfatentry->when();
-        return m_when; 
+        return FGetPrimaryExpire();
     }
 
     void update(const char *subkey, long long when)
@@ -204,12 +225,14 @@ public:
                 // we have to upgrade to a fat entry
                 long long whenT = m_when;
                 sds keyPrimary = u.m_key;
-                m_when = LLONG_MIN;
+                m_when |= FFatMask();
                 u.m_pfatentry = new (MALLOC_LOCAL) expireEntryFat(keyPrimary);
                 u.m_pfatentry->expireSubKey(nullptr, whenT);
                 // at this point we're fat so fall through
             }
         }
+        if (subkey == nullptr)
+            m_when = when | FFatMask();
         u.m_pfatentry->expireSubKey(subkey, when);
     }
     
@@ -229,18 +252,15 @@ public:
             pfatentry()->m_vecexpireEntries.begin() + itr.m_idx);
     }
 
-    bool FGetPrimaryExpire(long long *pwhen)
-    {
-        *pwhen = -1;
-        for (auto itr : *this)
-        {
-            if (itr.subkey() == nullptr)
-            {
-                *pwhen = itr.when();
-                return true;
-            }
-        }
-        return false;
+    long long FGetPrimaryExpire() const noexcept
+    { 
+        return m_when & (~FFatMask()); 
+    }
+
+    bool FGetPrimaryExpire(long long *pwhen) const noexcept
+    { 
+        *pwhen = FGetPrimaryExpire();
+        return *pwhen != INVALID_EXPIRE;
     }
 
     explicit operator const char*() const noexcept { return key(); }

src/geo.cpp

@@ -509,7 +509,7 @@ void geoaddCommand(client *c) {
  *                               [COUNT count [ANY]] [STORE key] [STOREDIST key]
  * GEORADIUSBYMEMBER key member radius unit ... options ...
  * GEOSEARCH key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit]
- *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC]
+ *               [BYBOX width height unit] [WITHCOORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC]
  * GEOSEARCHSTORE dest_key src_key [FROMMEMBER member] [FROMLONLAT long lat] [BYRADIUS radius unit]
  *               [BYBOX width height unit] [WITHCORD] [WITHDIST] [WITHASH] [COUNT count [ANY]] [ASC|DESC] [STOREDIST]
  *  */
@@ -518,21 +518,24 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
     int storedist = 0; /* 0 for STORE, 1 for STOREDIST. */
 
     /* Look up the requested zset */
-    robj_roptr zobj = nullptr;
-    if ((zobj = lookupKeyReadOrReply(c, c->argv[srcKeyIndex], shared.emptyarray)) == nullptr ||
-        checkType(c, zobj, OBJ_ZSET)) {
-        return;
-    }
+    robj_roptr zobj = lookupKeyRead(c->db, c->argv[srcKeyIndex]);
+    if (checkType(c, zobj, OBJ_ZSET)) return;
 
     /* Find long/lat to use for radius or box search based on inquiry type */
     int base_args;
     GeoShape shape = {0};
     if (flags & RADIUS_COORDS) {
+        /* GEORADIUS or GEORADIUS_RO */
         base_args = 6;
         shape.type = CIRCULAR_TYPE;
         if (extractLongLatOrReply(c, c->argv + 2, shape.xy) == C_ERR) return;
         if (extractDistanceOrReply(c, c->argv+base_args-2, &shape.conversion, &shape.t.radius) != C_OK) return;
+    } else if ((flags & RADIUS_MEMBER) && !zobj) {
+        /* We don't have a source key, but we need to proceed with argument
+         * parsing, so we know which reply to use depending on the STORE flag. */
+        base_args = 5;
     } else if (flags & RADIUS_MEMBER) {
+        /* GEORADIUSBYMEMBER or GEORADIUSBYMEMBER_RO */
         base_args = 5;
         shape.type = CIRCULAR_TYPE;
         robj *member = c->argv[2];
@@ -542,6 +545,7 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
         }
         if (extractDistanceOrReply(c, c->argv+base_args-2, &shape.conversion, &shape.t.radius) != C_OK) return;
     } else if (flags & GEOSEARCH) {
+        /* GEOSEARCH or GEOSEARCHSTORE */
         base_args = 2;
         if (flags & GEOSEARCHSTORE) {
             base_args = 3;
@@ -608,6 +612,13 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
                       flags & GEOSEARCH &&
                       !fromloc)
             {
+                /* No source key, proceed with argument parsing and return an error when done. */
+                if (!zobj) {
+                    frommember = 1;
+                    i++;
+                    continue;
+                }
+
                 if (longLatFromMember(zobj, c->argv[base_args+i+1], shape.xy) == C_ERR) {
                     addReplyError(c, "could not decode requested zset member");
                     return;
@@ -676,6 +687,23 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
         return;
     }
 
+    /* Return ASAP when src key does not exist. */
+    if (!zobj) {
+        if (storekey) {
+            /* store key is not NULL, try to delete it and return 0. */
+            if (dbDelete(c->db, storekey)) {
+                signalModifiedKey(c, c->db, storekey);
+                notifyKeyspaceEvent(NOTIFY_GENERIC, "del", storekey, c->db->id);
+                g_pserver->dirty++;
+            }
+            addReply(c, shared.czero);
+        } else {
+            /* Otherwise we return an empty array. */
+            addReply(c, shared.emptyarray);
+        }
+        return;
+    }
+
     /* COUNT without ordering does not make much sense (we need to
      * sort in order to return the closest N entries),
      * force ASC ordering if COUNT was specified but no sorting was
@@ -770,7 +798,7 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
         robj *zobj;
         zset *zs;
         int i;
-        size_t maxelelen = 0;
+        size_t maxelelen = 0, totelelen = 0;
 
         if (returned_items) {
             zobj = createZsetObject();
@@ -785,13 +813,14 @@ void georadiusGeneric(client *c, int srcKeyIndex, int flags) {
             size_t elelen = sdslen(gp->member);
 
             if (maxelelen < elelen) maxelelen = elelen;
+            totelelen += elelen;
             znode = zslInsert(zs->zsl,score,gp->member);
             serverAssert(dictAdd(zs->dict,gp->member,&znode->score) == DICT_OK);
             gp->member = NULL;
         }
 
         if (returned_items) {
-            zsetConvertToZiplistIfNeeded(zobj,maxelelen);
+            zsetConvertToZiplistIfNeeded(zobj,maxelelen,totelelen);
             setKey(c,c->db,storekey,zobj);
             decrRefCount(zobj);
             notifyKeyspaceEvent(NOTIFY_ZSET,flags & GEOSEARCH ? "geosearchstore" : "georadiusstore",storekey,

src/intset.c

@@ -104,7 +104,8 @@ intset *intsetNew(void) {
 
 /* Resize the intset */
 static intset *intsetResize(intset *is, uint32_t len) {
-    uint32_t size = len*intrev32ifbe(is->encoding);
+    uint64_t size = (uint64_t)len*intrev32ifbe(is->encoding);
+    assert(size <= SIZE_MAX - sizeof(intset));
     is = zrealloc(is,sizeof(intset)+size, MALLOC_SHARED);
     return is;
 }
@@ -281,7 +282,7 @@ uint32_t intsetLen(const intset *is) {
 
 /* Return intset blob size in bytes. */
 size_t intsetBlobLen(intset *is) {
-    return sizeof(intset)+intrev32ifbe(is->length)*intrev32ifbe(is->encoding);
+    return sizeof(intset)+(size_t)intrev32ifbe(is->length)*intrev32ifbe(is->encoding);
 }
 
 /* Validate the integrity of the data structure.

src/latency.cpp

@@ -426,7 +426,7 @@ sds createLatencyReport(void) {
         }
 
         if (advise_slowlog_inspect) {
-            report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check https://redis.io/commands/slowlog for more information.\n");
+            report = sdscat(report,"- Check your Slow Log to understand what are the commands you are running which are too slow to execute. Please check https://docs.keydb.dev/docs/commands#slowlog for more information.\n");
         }
 
         /* Intrinsic latency. */

src/lazyfree.cpp

@@ -39,12 +39,11 @@ void lazyfreeFreeSlotsMap(void *args[]) {
     atomicIncr(lazyfreed_objects,len);
 }
 
-/* Release the rax mapping Redis Cluster keys to slots in the
- * lazyfree thread. */
+/* Release the key tracking table. */
 void lazyFreeTrackingTable(void *args[]) {
     rax *rt = (rax*)args[0];
     size_t len = rt->numele;
-    raxFree(rt);
+    freeTrackingRadixTree(rt);
     atomicDecr(lazyfree_objects,len);
     atomicIncr(lazyfreed_objects,len);
 }
@@ -110,7 +109,7 @@ size_t lazyfreeGetFreeEffort(robj *key, robj *obj) {
         /* Every consumer group is an allocation and so are the entries in its
          * PEL. We use size of the first group's PEL as an estimate for all
          * others. */
-        if (s->cgroups) {
+        if (s->cgroups && raxSize(s->cgroups)) {
             raxIterator ri;
             streamCG *cg;
             raxStart(&ri,s->cgroups);

src/listpack.c

@@ -131,6 +131,8 @@
     assert((p) >= (lp)+LP_HDR_SIZE && (p)+(len) < (lp)+lpGetTotalBytes((lp))); \
 } while (0)
 
+static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigned char *p);
+
 /* Convert a string into a signed 64 bit integer.
  * The function returns 1 if the string could be parsed into a (non-overflowing)
  * signed 64 bit int, 0 otherwise. The 'value' will be set to the parsed value
@@ -313,7 +315,7 @@ int lpEncodeGetType(unsigned char *ele, uint32_t size, unsigned char *intenc, ui
     } else {
         if (size < 64) *enclen = 1+size;
         else if (size < 4096) *enclen = 2+size;
-        else *enclen = 5+size;
+        else *enclen = 5+(uint64_t)size;
         return LP_ENCODING_STRING;
     }
 }
@@ -453,8 +455,8 @@ unsigned char *lpSkip(unsigned char *p) {
 unsigned char *lpNext(unsigned char *lp, unsigned char *p) {
     assert(p);
     p = lpSkip(p);
-    ASSERT_INTEGRITY(lp, p);
     if (p[0] == LP_EOF) return NULL;
+    lpAssertValidEntry(lp, lpBytes(lp), p);
     return p;
 }
 
@@ -468,16 +470,17 @@ unsigned char *lpPrev(unsigned char *lp, unsigned char *p) {
     uint64_t prevlen = lpDecodeBacklen(p);
     prevlen += lpEncodeBacklen(NULL,prevlen);
     p -= prevlen-1; /* Seek the first byte of the previous entry. */
-    ASSERT_INTEGRITY(lp, p);
+    lpAssertValidEntry(lp, lpBytes(lp), p);
     return p;
 }
 
 /* Return a pointer to the first element of the listpack, or NULL if the
  * listpack has no elements. */
 unsigned char *lpFirst(unsigned char *lp) {
-    lp += LP_HDR_SIZE; /* Skip the header. */
-    if (lp[0] == LP_EOF) return NULL;
-    return lp;
+    unsigned char *p = lp + LP_HDR_SIZE; /* Skip the header. */
+    if (p[0] == LP_EOF) return NULL;
+    lpAssertValidEntry(lp, lpBytes(lp), p);
+    return p;
 }
 
 /* Return a pointer to the last element of the listpack, or NULL if the
@@ -861,6 +864,13 @@ unsigned char *lpSeek(unsigned char *lp, long index) {
     }
 }
 
+/* Same as lpFirst but without validation assert, to be used right before lpValidateNext. */
+unsigned char *lpValidateFirst(unsigned char *lp) {
+    unsigned char *p = lp + LP_HDR_SIZE; /* Skip the header. */
+    if (p[0] == LP_EOF) return NULL;
+    return p;
+}
+
 /* Validate the integrity of a single listpack entry and move to the next one.
  * The input argument 'pp' is a reference to the current record and is advanced on exit.
  * Returns 1 if valid, 0 if invalid. */
@@ -872,6 +882,10 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) {
     if (!p)
         return 0;
 
+    /* Before accessing p, make sure it's valid. */
+    if (OUT_OF_RANGE(p))
+        return 0;
+
     if (*p == LP_EOF) {
         *pp = NULL;
         return 1;
@@ -908,6 +922,11 @@ int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes) {
 #undef OUT_OF_RANGE
 }
 
+/* Validate that the entry doesn't reach outside the listpack allocation. */
+static inline void lpAssertValidEntry(unsigned char* lp, size_t lpbytes, unsigned char *p) {
+    assert(lpValidateNext(lp, &p, lpbytes));
+}
+
 /* Validate the integrity of the data structure.
  * when `deep` is 0, only the integrity of the header is validated.
  * when `deep` is 1, we scan all the entries one by one. */
@@ -930,8 +949,8 @@ int lpValidateIntegrity(unsigned char *lp, size_t size, int deep){
 
     /* Validate the invividual entries. */
     uint32_t count = 0;
-    unsigned char *p = lpFirst(lp);
-    while(p) {
+    unsigned char *p = lp + LP_HDR_SIZE;
+    while(p && p[0] != LP_EOF) {
         if (!lpValidateNext(lp, &p, bytes))
             return 0;
         count++;

src/listpack.h

@@ -64,6 +64,7 @@ unsigned char *lpPrev(unsigned char *lp, unsigned char *p);
 uint32_t lpBytes(unsigned char *lp);
 unsigned char *lpSeek(unsigned char *lp, long index);
 int lpValidateIntegrity(unsigned char *lp, size_t size, int deep);
+unsigned char *lpValidateFirst(unsigned char *lp);
 int lpValidateNext(unsigned char *lp, unsigned char **pp, size_t lpbytes);
 
 #ifdef __cplusplus

src/lolwut.c

@@ -94,8 +94,8 @@ lwCanvas *lwCreateCanvas(int width, int height, int bgcolor) {
     lwCanvas *canvas = zmalloc(sizeof(*canvas));
     canvas->width = width;
     canvas->height = height;
-    canvas->pixels = zmalloc(width*height);
-    memset(canvas->pixels,bgcolor,width*height);
+    canvas->pixels = zmalloc((size_t)width*height);
+    memset(canvas->pixels,bgcolor,(size_t)width*height);
     return canvas;
 }
 

src/memtest.c

@@ -71,7 +71,7 @@ void memtest_progress_start(char *title, int pass) {
     printf("\x1b[H\x1b[2K");          /* Cursor home, clear current line.  */
     printf("%s [%d]\n", title, pass); /* Print title. */
     progress_printed = 0;
-    progress_full = ws.ws_col*(ws.ws_row-3);
+    progress_full = (size_t)ws.ws_col*(ws.ws_row-3);
     fflush(stdout);
 }
 

src/module.cpp

@@ -365,11 +365,7 @@ typedef struct RedisModuleCommandFilter {
 static list *moduleCommandFilters;
 
 /* Module GIL Variables */
-static int s_cAcquisitionsServer = 0;
-static int s_cAcquisitionsModule = 0;
-static std::mutex s_mutex;
-static std::condition_variable s_cv;
-static std::recursive_mutex s_mutexModule;
+static readWriteLock s_moduleGIL("Module GIL");
 thread_local bool g_fModuleThread = false;
 
 typedef void (*RedisModuleForkDoneHandler) (int exitcode, int bysignal, void *user_data);
@@ -612,6 +608,19 @@ int moduleDelKeyIfEmpty(RedisModuleKey *key) {
     }
 }
 
+/* This function is used to set the thread local variables (serverTL) for 
+ * arbitrary module threads. All incoming module threads share the same set of 
+ * thread local variables (modulethreadvar).
+ *
+ * This is needed as some KeyDB functions use thread local variables to do things,
+ * and we don't want to share the thread local variables of existing server threads */
+void moduleSetThreadVariablesIfNeeded(void) {
+    if (serverTL == nullptr) {
+        serverTL = &g_pserver->modulethreadvar; 
+        g_fModuleThread = true;
+    }
+}
+
 /* --------------------------------------------------------------------------
  * Service API exported to modules
  *
@@ -2265,6 +2274,7 @@ int RM_GetContextFlags(RedisModuleCtx *ctx) {
  * periodically in timer callbacks or other periodic callbacks.
  */
 int RM_AvoidReplicaTraffic() {
+    moduleSetThreadVariablesIfNeeded();
     return checkClientPauseTimeoutAndReturnIfPaused();
 }
 
@@ -2341,8 +2351,11 @@ void *RM_OpenKey(RedisModuleCtx *ctx, robj *keyname, int mode) {
 /* Destroy a RedisModuleKey struct (freeing is the responsibility of the caller). */
 static void moduleCloseKey(RedisModuleKey *key) {
     int signal = SHOULD_SIGNAL_MODIFIED_KEYS(key->ctx);
+    moduleAcquireGIL(false);
     if ((key->mode & REDISMODULE_WRITE) && signal)
         signalModifiedKey(key->ctx->client,key->db,key->key);
+    /* TODO: if (key->iter) RM_KeyIteratorStop(kp); */
+    moduleReleaseGIL(false);
     if (key->iter) zfree(key->iter);
     RM_ZsetRangeStop(key);
     if (key && key->value && key->value->type == OBJ_STREAM &&
@@ -2429,11 +2442,11 @@ int RM_UnlinkKey(RedisModuleKey *key) {
  * REDISMODULE_NO_EXPIRE is returned. */
 mstime_t RM_GetExpire(RedisModuleKey *key) {
     expireEntry *pexpire = getExpire(key->db,key->key);
-    mstime_t expire = -1;
+    mstime_t expire = INVALID_EXPIRE;
     if (pexpire != nullptr)
         pexpire->FGetPrimaryExpire(&expire);
     
-    if (expire == -1 || key->value == NULL)
+    if (expire == INVALID_EXPIRE || key->value == NULL)
         return REDISMODULE_NO_EXPIRE;
     expire -= mstime();
     return expire >= 0 ? expire : 0;
@@ -2615,7 +2628,7 @@ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) {
         if (newlen > curlen) {
             key->value->m_ptr = sdsgrowzero(szFromObj(key->value),newlen);
         } else if (newlen < curlen) {
-            sdsrange(szFromObj(key->value),0,newlen-1);
+            sdssubstr(szFromObj(key->value),0,newlen);
             /* If the string is too wasteful, reallocate it. */
             if (sdslen(szFromObj(key->value)) < sdsavail(szFromObj(key->value)))
                 key->value->m_ptr = sdsRemoveFreeSpace(szFromObj(key->value));
@@ -3392,6 +3405,7 @@ int RM_HashGet(RedisModuleKey *key, int flags, ...) {
  * - EDOM if the given ID was 0-0 or not greater than all other IDs in the
  *   stream (only if the AUTOID flag is unset)
  * - EFBIG if the stream has reached the last possible ID
+ * - ERANGE if the elements are too large to be stored.
  */
 int RM_StreamAdd(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisModuleString **argv, long numfields) {
     /* Validate args */
@@ -3435,8 +3449,9 @@ int RM_StreamAdd(RedisModuleKey *key, int flags, RedisModuleStreamID *id, RedisM
         use_id_ptr = &use_id;
     }
     if (streamAppendItem(s, argv, numfields, &added_id, use_id_ptr) == C_ERR) {
-        /* ID not greater than all existing IDs in the stream */
-        errno = EDOM;
+        /* Either the ID not greater than all existing IDs in the stream, or
+         * the elements are too large to be stored. either way, errno is already
+         * set by streamAppendItem. */
         return REDISMODULE_ERR;
     }
     /* Postponed signalKeyAsReady(). Done implicitly by moduleCreateEmptyKey()
@@ -4270,7 +4285,14 @@ RedisModuleCallReply *RM_Call(RedisModuleCtx *ctx, const char *cmdname, const ch
         if (!(flags & REDISMODULE_ARGV_NO_REPLICAS))
             call_flags |= CMD_CALL_PROPAGATE_REPL;
     }
+
+    {
+    aeAcquireLock();
+    std::unique_lock<fastlock> ul(c->lock);
     call(c,call_flags);
+    aeReleaseLock();
+    }
+
     g_pserver->replication_allowed = prev_replication_allowed;
 
     serverAssert((c->flags & CLIENT_BLOCKED) == 0);
@@ -5450,8 +5472,8 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
  *     reply_callback:   called after a successful RedisModule_UnblockClient()
  *                       call in order to reply to the client and unblock it.
  *
- *     timeout_callback: called when the timeout is reached in order to send an
- *                       error to the client.
+ *     timeout_callback: called when the timeout is reached or if `CLIENT UNBLOCK`
+ *                       is invoked, in order to send an error to the client.
  *
  *     free_privdata:    called in order to free the private data that is passed
  *                       by RedisModule_UnblockClient() call.
@@ -5468,6 +5490,12 @@ int moduleTryServeClientBlockedOnKey(client *c, robj *key) {
  * In these cases, a call to RedisModule_BlockClient() will **not** block the
  * client, but instead produce a specific error reply.
  *
+ * A module that registers a timeout_callback function can also be unblocked
+ * using the `CLIENT UNBLOCK` command, which will trigger the timeout callback.
+ * If a callback function is not registered, then the blocked client will be
+ * treated as if it is not in a blocked state and `CLIENT UNBLOCK` will return
+ * a zero value.
+ *
  * Measuring background time: By default the time spent in the blocked command
  * is not account for the total command duration. To include such time you should
  * use RM_BlockedClientMeasureTimeStart() and RM_BlockedClientMeasureTimeEnd() one,
@@ -5595,10 +5623,7 @@ int moduleClientIsBlockedOnKeys(client *c) {
  * RedisModule_BlockClientOnKeys() is accessible from the timeout
  * callback via RM_GetBlockedClientPrivateData). */
 int RM_UnblockClient(RedisModuleBlockedClient *bc, void *privdata) {
-    if (serverTL == nullptr) {
-        serverTL = &g_pserver->modulethreadvar; 
-        g_fModuleThread = true;
-    }
+    moduleSetThreadVariablesIfNeeded();
     if (bc->blocked_on_keys) {
         /* In theory the user should always pass the timeout handler as an
          * argument, but better to be safe than sorry. */
@@ -5757,6 +5782,17 @@ void moduleHandleBlockedClients(int iel) {
     pthread_mutex_unlock(&moduleUnblockedClientsMutex);
 }
 
+/* Check if the specified client can be safely timed out using
+ * moduleBlockedClientTimedOut().
+ */
+int moduleBlockedClientMayTimeout(client *c) {
+    if (c->btype != BLOCKED_MODULE)
+        return 1;
+
+    RedisModuleBlockedClient *bc = (RedisModuleBlockedClient*)c->bpop.module_blocked_handle;
+    return (bc && bc->timeout_callback != NULL);
+}
+
 /* Called when our client timed out. After this function unblockClient()
  * is called, and it will invalidate the blocked client. So this function
  * does not need to do any cleanup. Eventually the module will call the
@@ -5898,10 +5934,7 @@ void RM_FreeThreadSafeContext(RedisModuleCtx *ctx) {
  * a blocked client connected to the thread safe context. */
 void RM_ThreadSafeContextLock(RedisModuleCtx *ctx) {
     UNUSED(ctx);
-    if (serverTL == nullptr) {
-        serverTL = &g_pserver->modulethreadvar;
-        g_fModuleThread = true;
-    }
+    moduleSetThreadVariablesIfNeeded();
     moduleAcquireGIL(FALSE /*fServerThread*/, true /*fExclusive*/);
 }
 
@@ -5932,95 +5965,58 @@ void RM_ThreadSafeContextUnlock(RedisModuleCtx *ctx) {
 //  as the server thread acquisition is sufficient.  If we did try to lock we would deadlock
 static bool FModuleCallBackLock(bool fServerThread)
 {
-    return !fServerThread && aeThreadOwnsLock() && !g_fModuleThread && s_cAcquisitionsServer > 0;
+    return !fServerThread && aeThreadOwnsLock() && !g_fModuleThread && s_moduleGIL.hasReader();
 }
 void moduleAcquireGIL(int fServerThread, int fExclusive) {
-    std::unique_lock<std::mutex> lock(s_mutex);
-    int *pcheck = fServerThread ? &s_cAcquisitionsModule : &s_cAcquisitionsServer;
-
     if (FModuleCallBackLock(fServerThread)) {
         return;
     }
 
-    while (*pcheck > 0)
-        s_cv.wait(lock);
-
     if (fServerThread)
     {
-        ++s_cAcquisitionsServer;
+        s_moduleGIL.acquireRead();
     }
     else
     {
-        // only try to acquire the mutexModule in exclusive mode 
-        if (fExclusive){
-            // It is possible that another module thread holds the GIL (and s_mutexModule as a result). 
-            // When said thread goes to release the GIL, it will wait for s_mutex, which this thread owns. 
-            // This thread is however waiting for the GIL (and s_mutexModule) that the other thread owns.
-            // As a result, a deadlock has occured. 
-            // We release the lock on s_mutex and wait until we are able to safely acquire the GIL 
-            // in order to prevent this deadlock from occuring. 
-            while (!s_mutexModule.try_lock())
-                s_cv.wait(lock);     
-        }    
-        ++s_cAcquisitionsModule;
-        fModuleGILWlocked++;
+        s_moduleGIL.acquireWrite(fExclusive);
     }
 }
 
 int moduleTryAcquireGIL(bool fServerThread, int fExclusive) {
-    std::unique_lock<std::mutex> lock(s_mutex, std::defer_lock);
-    if (!lock.try_lock())
-        return 1;
-    int *pcheck = fServerThread ? &s_cAcquisitionsModule : &s_cAcquisitionsServer;
-
     if (FModuleCallBackLock(fServerThread)) {
         return 0;
     }
 
-    if (*pcheck > 0)
-        return 1;
-
     if (fServerThread)
     {
-        ++s_cAcquisitionsServer;
+        if (!s_moduleGIL.tryAcquireRead())
+            return 1;
     }
     else
     {
-        // only try to acquire the mutexModule in exclusive mode 
-        if (fExclusive){
-            if (!s_mutexModule.try_lock())
-                return 1;
-        }
-        ++s_cAcquisitionsModule;
-        fModuleGILWlocked++;
+        if (!s_moduleGIL.tryAcquireWrite(fExclusive))
+            return 1;
     }
     return 0;
 }
 
 void moduleReleaseGIL(int fServerThread, int fExclusive) {
-    std::unique_lock<std::mutex> lock(s_mutex);
-
     if (FModuleCallBackLock(fServerThread)) {
         return;
     }
-
+    
     if (fServerThread)
     {
-        --s_cAcquisitionsServer;
+        s_moduleGIL.releaseRead();
     }
     else
     {
-        // only try to release the mutexModule in exclusive mode
-        if (fExclusive)
-            s_mutexModule.unlock();
-        --s_cAcquisitionsModule;
-        fModuleGILWlocked--;
+        s_moduleGIL.releaseWrite(fExclusive);
     }
-    s_cv.notify_all();
 }
 
 int moduleGILAcquiredByModule(void) {
-    return fModuleGILWlocked > 0;
+    return s_moduleGIL.hasWriter();
 }
 
 
@@ -6366,7 +6362,7 @@ int RM_GetClusterNodeInfo(RedisModuleCtx *ctx, const char *id, char *ip, char *m
         /* If the information is not available, the function will set the
          * field to zero bytes, so that when the field can't be populated the
          * function kinda remains predictable. */
-        if (node->flags & CLUSTER_NODE_MASTER && node->slaveof)
+        if (node->flags & CLUSTER_NODE_SLAVE && node->slaveof)
             memcpy(master_id,node->slaveof->name,REDISMODULE_NODE_ID_LEN);
         else
             memset(master_id,0,REDISMODULE_NODE_ID_LEN);
@@ -8897,8 +8893,9 @@ sds genModulesInfoStringRenderModulesList(list *l) {
     while((ln = listNext(&li))) {
         RedisModule *module = (RedisModule*)ln->value;
         output = sdscat(output,module->name);
+        if (ln != listLast(l))
+            output = sdscat(output,"|");
     }
-    output = sdstrim(output,"|");
     output = sdscat(output,"]");
     return output;
 }
@@ -9226,6 +9223,14 @@ int *RM_GetCommandKeys(RedisModuleCtx *ctx, RedisModuleString **argv, int argc,
     return res;
 }
 
+/* Return the name of the command currently running */
+const char *RM_GetCurrentCommandName(RedisModuleCtx *ctx) {
+    if (!ctx || !ctx->client || !ctx->client->cmd)
+        return NULL;
+
+    return (const char*)ctx->client->cmd->name;
+}
+
 /* --------------------------------------------------------------------------
  * ## Defrag API
  * -------------------------------------------------------------------------- */
@@ -9425,6 +9430,7 @@ long moduleDefragGlobals(void) {
         module->defrag_cb(&defrag_ctx);
         defragged += defrag_ctx.defragged;
     }
+    dictReleaseIterator(di);
 
     return defragged;
 }
@@ -9686,6 +9692,7 @@ void moduleRegisterCoreAPI(void) {
     REGISTER_API(GetServerVersion);
     REGISTER_API(GetClientCertificate);
     REGISTER_API(GetCommandKeys);
+    REGISTER_API(GetCurrentCommandName);
     REGISTER_API(GetTypeMethodVersion);
     REGISTER_API(RegisterDefragFunc);
     REGISTER_API(DefragAlloc);

src/modules/Makefile

@@ -13,7 +13,7 @@ endif
 
 .SUFFIXES: .c .so .xo .o
 
-all: helloworld.so hellotype.so helloblock.so testmodule.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so
+all: helloworld.so hellotype.so helloblock.so hellocluster.so hellotimer.so hellodict.so hellohook.so helloacl.so
 
 .c.xo:
 	$(CC) -I. $(CFLAGS) $(SHOBJ_CFLAGS) -fPIC -c $< -o $@
@@ -58,10 +58,5 @@ helloacl.xo: ../redismodule.h
 helloacl.so: helloacl.xo
 	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
 
-testmodule.xo: ../redismodule.h
-
-testmodule.so: testmodule.xo
-	$(LD) -o $@ $< $(SHOBJ_LDFLAGS) $(LIBS) -lc
-
 clean:
 	rm -rf *.xo *.so

src/multi.cpp

@@ -156,8 +156,7 @@ void execCommandAbort(client *c, sds error) {
     /* Send EXEC to clients waiting data from MONITOR. We did send a MULTI
      * already, and didn't send any of the queued commands, now we'll just send
      * EXEC so it is clear that the transaction is over. */
-    if (listLength(g_pserver->monitors) && !g_pserver->loading)
-        replicationFeedMonitors(c,g_pserver->monitors,c->db->id,c->argv,c->argc);
+    replicationFeedMonitors(c,g_pserver->monitors,c->db->id,c->argv,c->argc);
 }
 
 void execCommand(client *c) {
@@ -172,20 +171,28 @@ void execCommand(client *c) {
         return;
     }
 
+    /* EXEC with expired watched key is disallowed*/
+    if (isWatchedKeyExpired(c)) {
+        c->flags |= (CLIENT_DIRTY_CAS);
+    }
+
     /* Check if we need to abort the EXEC because:
      * 1) Some WATCHed key was touched.
      * 2) There was a previous error while queueing commands.
      * A failed EXEC in the first case returns a multi bulk nil object
      * (technically it is not an error but a special behavior), while
      * in the second an EXECABORT error is returned. */
-    if (c->flags & (CLIENT_DIRTY_CAS|CLIENT_DIRTY_EXEC)) {
-        addReply(c, c->flags & CLIENT_DIRTY_EXEC ? shared.execaborterr :
-                                                   shared.nullarray[c->resp]);
+    if (c->flags & (CLIENT_DIRTY_CAS | CLIENT_DIRTY_EXEC)) {
+        if (c->flags & CLIENT_DIRTY_EXEC) {
+            addReplyErrorObject(c, shared.execaborterr);
+        } else {
+            addReply(c, shared.nullarray[c->resp]);
+        }
+
         discardTransaction(c);
-        goto handle_monitor;
+        return;
     }
 
-{   // GOTO Protectect Variable Scope
     uint64_t old_flags = c->flags;
 
     /* we do not want to allow blocking commands inside multi */
@@ -273,16 +280,6 @@ void execCommand(client *c) {
     }
 
     serverTL->in_exec = 0;
-} // END Goto Variable Protection Scope
-
-handle_monitor:
-    /* Send EXEC to clients waiting data from MONITOR. We do it here
-     * since the natural order of commands execution is actually:
-     * MUTLI, EXEC, ... commands inside transaction ...
-     * Instead EXEC is flagged as CMD_SKIP_MONITOR in the command
-     * table, and we do it here with correct ordering. */
-    if (listLength(g_pserver->monitors) && !g_pserver->loading)
-        replicationFeedMonitors(c,g_pserver->monitors,c->db->id,c->argv,c->argc);
 }
 
 /* ===================== WATCH (CAS alike for MULTI/EXEC) ===================
@@ -360,6 +357,22 @@ void unwatchAllKeys(client *c) {
     }
 }
 
+/* iterates over the watched_keys list and
+ * look for an expired key . */
+int isWatchedKeyExpired(client *c) {
+    listIter li;
+    listNode *ln;
+    watchedKey *wk;
+    if (listLength(c->watched_keys) == 0) return 0;
+    listRewind(c->watched_keys,&li);
+    while ((ln = listNext(&li))) {
+        wk = (watchedKey*)listNodeValue(ln);
+        if (keyIsExpired(wk->db, wk->key)) return 1;
+    }
+
+    return 0;
+}
+
 /* "Touch" a key, so that if this key is being WATCHed by some client the
  * next EXEC will fail. */
 void touchWatchedKey(redisDb *db, robj *key) {

src/networking.cpp

@@ -112,6 +112,15 @@ static void clientSetDefaultAuth(client *c) {
                        !(c->user->flags & USER_FLAG_DISABLED);
 }
 
+int authRequired(client *c) {
+    /* Check if the user is authenticated. This check is skipped in case
+     * the default user is flagged as "nopass" and is active. */
+    int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) ||
+                          (DefaultUser->flags & USER_FLAG_DISABLED)) &&
+                        !c->authenticated;
+    return auth_required;
+}
+
 client *createClient(connection *conn, int iel) {
     client *c = (client*)zmalloc(sizeof(client), MALLOC_LOCAL);
     serverAssert(conn == nullptr || (iel == (serverTL - g_pserver->rgthreadvar)));
@@ -403,7 +412,7 @@ void _addReplyProtoToList(client *c, const char *s, size_t len) {
         listAddNodeTail(c->reply, tail);
         c->reply_bytes += tail->size;
 
-        asyncCloseClientOnOutputBufferLimitReached(c);
+        closeClientOnOutputBufferLimitReached(c, 1);
     }
 }
 
@@ -735,7 +744,7 @@ void setDeferredReply(client *c, void *node, const char *s, size_t length) {
         listNodeValue(ln) = buf;
         c->reply_bytes += buf->size;
 
-        asyncCloseClientOnOutputBufferLimitReached(c);
+        closeClientOnOutputBufferLimitReached(c, 1);
     }
 }
 
@@ -789,14 +798,13 @@ void setDeferredSetLen(client *c, void *node, long length) {
 }
 
 void setDeferredAttributeLen(client *c, void *node, long length) {
-    int prefix = c->resp == 2 ? '*' : '|';
-    if (c->resp == 2) length *= 2;
-    setDeferredAggregateLen(c,node,length,prefix);
+    serverAssert(c->resp >= 3);
+    setDeferredAggregateLen(c,node,length,'|');
 }
 
 void setDeferredPushLen(client *c, void *node, long length) {
-    int prefix = c->resp == 2 ? '*' : '>';
-    setDeferredAggregateLen(c,node,length,prefix);
+    serverAssert(c->resp >= 3);
+    setDeferredAggregateLen(c,node,length,'>');
 }
 
 /* Add a double as a bulk reply */
@@ -825,6 +833,16 @@ void addReplyDouble(client *c, double d) {
     }
 }
 
+void addReplyBigNum(client *c, const char* num, size_t len) {
+    if (c->resp == 2) {
+        addReplyBulkCBuffer(c, num, len);
+    } else {
+        addReplyProto(c,"(",1);
+        addReplyProto(c,num,len);
+        addReply(c,shared.crlf);
+    }
+}
+
 /* Add a long double as a bulk reply, but uses a human readable formatting
  * of the double instead of exposing the crude behavior of doubles to the
  * dear user. */
@@ -896,14 +914,13 @@ void addReplySetLen(client *c, long length) {
 }
 
 void addReplyAttributeLen(client *c, long length) {
-    int prefix = c->resp == 2 ? '*' : '|';
-    if (c->resp == 2) length *= 2;
-    addReplyAggregateLen(c,length,prefix);
+    serverAssert(c->resp >= 3);
+    addReplyAggregateLen(c,length,'|');
 }
 
 void addReplyPushLen(client *c, long length) {
-    int prefix = c->resp == 2 ? '*' : '>';
-    addReplyAggregateLen(c,length,prefix);
+    serverAssert(c->resp >= 3);
+    addReplyAggregateLen(c,length,'>');
 }
 
 void addReplyNull(client *c) {
@@ -1093,7 +1110,7 @@ void AddReplyFromClient(client *dst, client *src) {
     src->bufpos = 0;
 
     /* Check output buffer limits */
-    asyncCloseClientOnOutputBufferLimitReached(dst);
+    closeClientOnOutputBufferLimitReached(dst, 1);
 }
 
 /* Copy 'src' client output buffers into 'dst' client output buffers.
@@ -1304,11 +1321,10 @@ void acceptOnThread(connection *conn, int flags, char *cip)
             szT = (char*)zmalloc(NET_IP_STR_LEN, MALLOC_LOCAL);
             memcpy(szT, cip, NET_IP_STR_LEN);
         }
-        int res = aePostFunction(g_pserver->rgthreadvar[ielTarget].el, [conn, flags, ielTarget, szT, fBootLoad] {
+        int res = aePostFunction(g_pserver->rgthreadvar[ielTarget].el, [conn, flags, ielTarget, szT] {
             connMarshalThread(conn);
             acceptCommonHandler(conn,flags,szT,ielTarget);
-            if (!g_fTestMode && !fBootLoad)
-                rgacceptsInFlight[ielTarget].fetch_sub(1, std::memory_order_relaxed);
+            rgacceptsInFlight[ielTarget].fetch_sub(1, std::memory_order_relaxed);
             zfree(szT);
         });
 
@@ -1316,8 +1332,7 @@ void acceptOnThread(connection *conn, int flags, char *cip)
             return;
         // If res != AE_OK we can still try to accept on the local thread
     }
-    if (!g_fTestMode && !fBootLoad)
-        rgacceptsInFlight[ielTarget].fetch_sub(1, std::memory_order_relaxed);
+    rgacceptsInFlight[ielTarget].fetch_sub(1, std::memory_order_relaxed);
 
     aeAcquireLock();
     acceptCommonHandler(conn,flags,cip,ielCur);
@@ -1892,25 +1907,12 @@ void ProcessPendingAsyncWrites()
         zfree(c->replyAsync);
         c->replyAsync = nullptr;
         c->fPendingAsyncWrite = FALSE;
-
-        // Now install the write event handler
-        int ae_flags = AE_WRITABLE|AE_WRITE_THREADSAFE;
-        /* For the fsync=always policy, we want that a given FD is never
-            * served for reading and writing in the same event loop iteration,
-            * so that in the middle of receiving the query, and serving it
-            * to the client, we'll call beforeSleep() that will do the
-            * actual fsync of AOF to disk. AE_BARRIER ensures that. */
-        if (g_pserver->aof_state == AOF_ON &&
-            g_pserver->aof_fsync == AOF_FSYNC_ALWAYS)
-        {
-            ae_flags |= AE_BARRIER;
-        }
         
         if (!((c->replstate == REPL_STATE_NONE ||
          (c->replstate == SLAVE_STATE_ONLINE && !c->repl_put_online_on_ack))))
             continue;
 
-        asyncCloseClientOnOutputBufferLimitReached(c);
+        closeClientOnOutputBufferLimitReached(c, 1);
         if (c->flags & CLIENT_CLOSE_ASAP)
             continue;   // we will never write this so don't post an op
         
@@ -1927,7 +1929,9 @@ void ProcessPendingAsyncWrites()
                 bool fResult = c->postFunction([](client *c) {
                     c->fPendingAsyncWriteHandler = false;
                     clientInstallWriteHandler(c);
+                    c->lock.unlock();
                     handleClientsWithPendingWrites(c->iel, g_pserver->aof_state);
+                    c->lock.lock();
                 }, false);
 
                 if (!fResult)
@@ -2035,9 +2039,6 @@ void resetClient(client *c) {
         c->flags |= CLIENT_REPLY_SKIP;
         c->flags &= ~CLIENT_REPLY_SKIP_NEXT;
     }
-
-    /* Always clear the prevent logging field. */
-    c->flags &= ~CLIENT_PREVENT_LOGGING;
 }
 
 /* This function is used when we want to re-enter the event loop but there
@@ -2229,6 +2230,10 @@ int processMultibulkBuffer(client *c) {
             addReplyError(c,"Protocol error: invalid multibulk length");
             setProtocolError("invalid mbulk count",c);
             return C_ERR;
+        } else if (ll > 10 && authRequired(c)) {
+            addReplyError(c, "Protocol error: unauthenticated multibulk length");
+            setProtocolError("unauth mbulk count", c);
+            return C_ERR;
         }
 
         c->qb_pos = (newline-c->querybuf)+2;
@@ -2276,6 +2281,10 @@ int processMultibulkBuffer(client *c) {
                 addReplyError(c,"Protocol error: invalid bulk length");
                 setProtocolError("invalid bulk length",c);
                 return C_ERR;
+            } else if (ll > 16384 && authRequired(c)) {
+                addReplyError(c, "Protocol error: unauthenticated bulk length");
+                setProtocolError("unauth bulk length", c);
+                return C_ERR;
             }
 
             c->qb_pos = newline-c->querybuf+2;
@@ -2339,23 +2348,27 @@ int processMultibulkBuffer(client *c) {
 
 /* Perform necessary tasks after a command was executed:
  *
- * 1. The client is reset unless there are reasons to avoid doing it.
+ * 1. The client is reset unless there are reasons to avoid doing it. 
  * 2. In the case of master clients, the replication offset is updated.
  * 3. Propagate commands we got from our master to replicas down the line. */
 void commandProcessed(client *c, int flags) {
+        /* If client is blocked(including paused), just return avoid reset and replicate.
+     *
+     * 1. Don't reset the client structure for blocked clients, so that the reply
+     *    callback will still be able to access the client argv and argc fields.
+     *    The client will be reset in unblockClient().
+     * 2. Don't update replication offset or propagate commands to replicas,
+     *    since we have not applied the command. */
+    if (c->flags & CLIENT_BLOCKED) return;
+
+    resetClient(c);
+
     long long prev_offset = c->reploff;
     if (c->flags & CLIENT_MASTER && !(c->flags & CLIENT_MULTI)) {
         /* Update the applied replication offset of our master. */
         c->reploff = c->read_reploff - sdslen(c->querybuf) + c->qb_pos;
     }
 
-    /* Don't reset the client structure for blocked clients, so that the reply
-     * callback will still be able to access the client argv and argc fields.
-     * The client will be reset in unblockClient(). */
-    if (!(c->flags & CLIENT_BLOCKED)) {
-        resetClient(c);
-    }
-
     /* If the client is a master we need to compute the difference
      * between the applied offset before and after processing the buffer,
      * to understand how much of the replication stream was actually
@@ -3087,7 +3100,7 @@ NULL
         if (getLongLongFromObjectOrReply(c,c->argv[2],&id,NULL)
             != C_OK) return;
         struct client *target = lookupClientByID(id);
-        if (target && target->flags & CLIENT_BLOCKED) {
+        if (target && target->flags & CLIENT_BLOCKED && moduleBlockedClientMayTimeout(target)) {
             std::unique_lock<fastlock> ul(target->lock);
             if (unblock_error)
                 addReplyError(target,
@@ -3413,7 +3426,8 @@ void helloCommand(client *c) {
         int moreargs = (c->argc-1) - j;
         const char *opt = (const char*)ptrFromObj(c->argv[j]);
         if (!strcasecmp(opt,"AUTH") && moreargs >= 2) {
-            preventCommandLogging(c);
+            redactClientCommandArgument(c, j+1);
+            redactClientCommandArgument(c, j+2);
             if (ACLAuthenticateUser(c, c->argv[j+1], c->argv[j+2]) == C_ERR) {
                 addReplyError(c,"-WRONGPASS invalid username-password pair or user is disabled.");
                 return;
@@ -3502,6 +3516,15 @@ static void retainOriginalCommandVector(client *c) {
     }
 }
 
+/* Redact a given argument to prevent it from being shown
+ * in the slowlog. This information is stored in the
+ * original_argv array. */
+void redactClientCommandArgument(client *c, int argc) {
+    retainOriginalCommandVector(c);
+    decrRefCount(c->argv[argc]);
+    c->original_argv[argc] = shared.redacted;
+}
+
 /* Rewrite the command vector of the client. All the new objects ref count
  * is incremented. The old command vector is freed, and the old objects
  * ref count is decremented. */
@@ -3671,18 +3694,33 @@ int checkClientOutputBufferLimits(client *c) {
  *
  * Note: we need to close the client asynchronously because this function is
  * called from contexts where the client can't be freed safely, i.e. from the
- * lower level functions pushing data inside the client output buffers. */
-void asyncCloseClientOnOutputBufferLimitReached(client *c) {
-    if (!c->conn) return; /* It is unsafe to free fake clients. */
+ * lower level functions pushing data inside the client output buffers.
+ * When `async` is set to 0, we close the client immediately, this is
+ * useful when called from cron.
+ *
+ * Returns 1 if client was (flagged) closed. */
+int closeClientOnOutputBufferLimitReached(client *c, int async) {
+    if (!c->conn) return 0; /* It is unsafe to free fake clients. */
     serverAssert(c->reply_bytes < SIZE_MAX-(1024*64));
-    if (c->reply_bytes == 0 || c->flags & CLIENT_CLOSE_ASAP) return;
+    if (c->reply_bytes == 0 || c->flags & CLIENT_CLOSE_ASAP) return 0;
     if (checkClientOutputBufferLimits(c)) {
         sds client = catClientInfoString(sdsempty(),c);
 
-        freeClientAsync(c);
-        serverLog(LL_WARNING,"Client %s scheduled to be closed ASAP for overcoming of output buffer limits.", client);
+        if (async) {
+            freeClientAsync(c);
+            serverLog(LL_WARNING,
+                      "Client %s scheduled to be closed ASAP for overcoming of output buffer limits.",
+                      client);
+        } else {
+            freeClient(c);
+            serverLog(LL_WARNING,
+                      "Client %s closed for overcoming of output buffer limits.",
+                      client);
+        }
         sdsfree(client);
+        return  1;
     }
+    return 0;
 }
 
 /* Helper function used by performEvictions() in order to flush slaves
@@ -3769,6 +3807,7 @@ void unpauseClients(void) {
     client *c;
     
     g_pserver->client_pause_type = CLIENT_PAUSE_OFF;
+    g_pserver->client_pause_end_time = 0;
 
     /* Unblock all of the clients so they are reprocessed. */
     listRewind(g_pserver->paused_clients,&li);
@@ -3809,86 +3848,37 @@ int checkClientPauseTimeoutAndReturnIfPaused(void) {
  *
  * The function returns the total number of events processed. */
 void processEventsWhileBlocked(int iel) {
-    serverAssert(GlobalLocksAcquired());
-    int iterations = 4; /* See the function top-comment. */
 
-    std::vector<client*> vecclients;
-    listIter li;
-    listNode *ln;
-    listRewind(g_pserver->clients, &li);
-
-    /* Update our cached time since it is used to create and update the last
-     * interaction time with clients and for other important things. */
-    updateCachedTime(0);
-
-    // All client locks must be acquired *after* the global lock is reacquired to prevent deadlocks
-    //  so unlock here, and save them for reacquisition later
-    while ((ln = listNext(&li)) != nullptr)
-    {
-        client *c = (client*)listNodeValue(ln);
-        if (c->lock.fOwnLock()) {
-            serverAssert(c->flags & CLIENT_PROTECTED);  // If the client is not protected we have no gurantee they won't be free'd in the event loop
-            c->lock.unlock();
-            vecclients.push_back(c);
+    int eventsCount = 0;
+    executeWithoutGlobalLock([&](){
+        int iterations = 4; /* See the function top-comment. */
+        try
+        {
+            ProcessingEventsWhileBlocked = 1;
+            while (iterations--) {
+                long long startval = g_pserver->events_processed_while_blocked;
+                long long ae_events = aeProcessEvents(g_pserver->rgthreadvar[iel].el,
+                    AE_FILE_EVENTS|AE_DONT_WAIT|
+                    AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP);
+                /* Note that g_pserver->events_processed_while_blocked will also get
+                * incremeted by callbacks called by the event loop handlers. */
+                eventsCount += ae_events;
+                long long events = eventsCount - startval;
+                if (!events) break;
+            }
+            ProcessingEventsWhileBlocked = 0;
         }
-    }
-    
-    /* Since we're about to release our lock we need to flush the repl backlog queue */
-    bool fReplBacklog = g_pserver->repl_batch_offStart >= 0;
-    if (fReplBacklog) {
-        flushReplBacklogToClients();
-        g_pserver->repl_batch_idxStart = -1;
-        g_pserver->repl_batch_offStart = -1;
-    }
-
-    long long eventsCount = 0;
-    aeReleaseLock();
-    serverAssert(!GlobalLocksAcquired());
-    try
-    {
-        ProcessingEventsWhileBlocked = 1;
-        while (iterations--) {
-            long long startval = g_pserver->events_processed_while_blocked;
-            long long ae_events = aeProcessEvents(g_pserver->rgthreadvar[iel].el,
-                AE_FILE_EVENTS|AE_DONT_WAIT|
-                AE_CALL_BEFORE_SLEEP|AE_CALL_AFTER_SLEEP);
-            /* Note that g_pserver->events_processed_while_blocked will also get
-            * incremeted by callbacks called by the event loop handlers. */
-            eventsCount += ae_events;
-            long long events = eventsCount - startval;
-            if (!events) break;
+        catch (...)
+        {
+            ProcessingEventsWhileBlocked = 0;
+            throw;
         }
-        ProcessingEventsWhileBlocked = 0;
-    }
-    catch (...)
-    {
-        // Caller expects us to be locked so fix and rethrow
-        ProcessingEventsWhileBlocked = 0;
-        AeLocker locker;
-        locker.arm(nullptr);
-        locker.release();
-        for (client *c : vecclients)
-            c->lock.lock();
-        throw;
-    }
-    
-    AeLocker locker;
-    locker.arm(nullptr);
-    locker.release();
+    });
 
     g_pserver->events_processed_while_blocked += eventsCount;
 
     whileBlockedCron();
 
-    // Restore it so the calling code is not confused
-    if (fReplBacklog) {
-        g_pserver->repl_batch_idxStart = g_pserver->repl_backlog_idx;
-        g_pserver->repl_batch_offStart = g_pserver->master_repl_offset;
-    }
-
-    for (client *c : vecclients)
-        c->lock.lock();
-
     // If a different thread processed the shutdown we need to abort the lua command or we will hang
     if (serverTL->el->stop)
         throw ShutdownException();

src/new.cpp

@@ -27,14 +27,17 @@ void *operator new(std::size_t size, const std::nothrow_t &) noexcept
     return zmalloc(size, MALLOC_LOCAL);
 }
 
+//need to do null checks for delete since the compiler can optimize out null checks in zfree
 void operator delete(void * p) noexcept
 {
-    zfree(p);
+    if (p != nullptr)
+        zfree(p);
 }
 
 void operator delete(void *p, std::size_t) noexcept
 {
-    zfree(p);
+    if (p != nullptr)
+        zfree(p);
 }
 
 #endif

src/object.cpp

@@ -136,7 +136,7 @@ robj *createEmbeddedStringObject(const char *ptr, size_t len) {
  *
  * The current limit of 52 is chosen so that the biggest string object
  * we allocate as EMBSTR will still fit into the 64 byte arena of jemalloc. */
-#define OBJ_ENCODING_EMBSTR_SIZE_LIMIT 52
+size_t OBJ_ENCODING_EMBSTR_SIZE_LIMIT = 52;
 
 //static_assert((sizeof(redisObject)+OBJ_ENCODING_EMBSTR_SIZE_LIMIT-8) == 64, "Max EMBSTR obj should be 64 bytes total");
 robj *createStringObject(const char *ptr, size_t len) {
@@ -146,6 +146,21 @@ robj *createStringObject(const char *ptr, size_t len) {
         return createRawStringObject(ptr,len);
 }
 
+/* Same as CreateRawStringObject, can return NULL if allocation fails */
+robj *tryCreateRawStringObject(const char *ptr, size_t len) {
+    sds str = sdstrynewlen(ptr,len);
+    if (!str) return NULL;
+    return createObject(OBJ_STRING, str);
+}
+
+/* Same as createStringObject, can return NULL if allocation fails */
+robj *tryCreateStringObject(const char *ptr, size_t len) {
+    if (len <= OBJ_ENCODING_EMBSTR_SIZE_LIMIT)
+        return createEmbeddedStringObject(ptr,len);
+    else
+        return tryCreateRawStringObject(ptr,len);
+}
+
 /* Create a string object from a long long value. When possible returns a
  * shared integer object, or at least an integer encoded one.
  *
@@ -763,6 +778,20 @@ int getLongLongFromObjectOrReply(client *c, robj *o, long long *target, const ch
     return C_OK;
 }
 
+int getUnsignedLongLongFromObjectOrReply(client *c, robj *o, uint64_t *target, const char *msg) {
+    uint64_t value;
+    if (getUnsignedLongLongFromObject(o, &value) != C_OK) {
+        if (msg != NULL) {
+            addReplyError(c,(char*)msg);
+        } else {
+            addReplyError(c,"value is not an integer or out of range");
+        }
+        return C_ERR;
+    }
+    *target = value;
+    return C_OK;
+}
+
 int getLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg) {
     long long value;
 
@@ -902,7 +931,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
             if (samples) asize += (double)elesize/samples*dictSize(d);
         } else if (o->encoding == OBJ_ENCODING_INTSET) {
             intset *is = (intset*)ptrFromObj(o);
-            asize = sizeof(*o)+sizeof(*is)+is->encoding*is->length;
+            asize = sizeof(*o)+sizeof(*is)+(size_t)is->encoding*is->length;
         } else {
             serverPanic("Unknown set encoding");
         }
@@ -947,7 +976,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) {
         }
     } else if (o->type == OBJ_STREAM) {
         stream *s = (stream*)ptrFromObj(o);
-        asize = sizeof(*o);
+        asize = sizeof(*o)+sizeof(*s);
         asize += streamRadixTreeMemoryUsage(s->rax);
 
         /* Now we have to add the listpacks. The last listpack is often non
@@ -1298,8 +1327,7 @@ robj_roptr objectCommandLookup(client *c, robj *key) {
 
 robj_roptr objectCommandLookupOrReply(client *c, robj *key, robj *reply) {
     robj_roptr o = objectCommandLookup(c,key);
-
-    if (!o) addReply(c, reply);
+    if (!o) SentReplyOnKeyMiss(c, reply);
     return o;
 }
 

src/quicklist.c

@@ -45,11 +45,16 @@
 #define REDIS_STATIC static
 #endif
 
-/* Optimization levels for size-based filling */
+/* Optimization levels for size-based filling.
+ * Note that the largest possible limit is 16k, so even if each record takes
+ * just one byte, it still won't overflow the 16 bit count field. */
 static const size_t optimization_level[] = {4096, 8192, 16384, 32768, 65536};
 
 /* Maximum size in bytes of any multi-element ziplist.
- * Larger values will live in their own isolated ziplists. */
+ * Larger values will live in their own isolated ziplists.
+ * This is used only if we're limited by record count. when we're limited by
+ * size, the maximum limit is bigger, but still safe.
+ * 8k is a recommended / default size limit */
 #define SIZE_SAFETY_LIMIT 8192
 
 /* Minimum ziplist size in bytes for attempting compression. */
@@ -444,6 +449,8 @@ REDIS_STATIC int _quicklistNodeAllowInsert(const quicklistNode *node,
     unsigned int new_sz = node->sz + sz + ziplist_overhead;
     if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(new_sz, fill)))
         return 1;
+    /* when we return 1 above we know that the limit is a size limit (which is
+     * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
     else if (!sizeMeetsSafetyLimit(new_sz))
         return 0;
     else if ((int)node->count < fill)
@@ -463,6 +470,8 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
     unsigned int merge_sz = a->sz + b->sz - 11;
     if (likely(_quicklistNodeSizeMeetsOptimizationRequirement(merge_sz, fill)))
         return 1;
+    /* when we return 1 above we know that the limit is a size limit (which is
+     * safe, see comments next to optimization_level and SIZE_SAFETY_LIMIT) */
     else if (!sizeMeetsSafetyLimit(merge_sz))
         return 0;
     else if ((int)(a->count + b->count) <= fill)
@@ -482,6 +491,7 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a,
  * Returns 1 if new head created. */
 int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {
     quicklistNode *orig_head = quicklist->head;
+    assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
     if (likely(
             _quicklistNodeAllowInsert(quicklist->head, quicklist->fill, sz))) {
         quicklist->head->zl =
@@ -505,6 +515,7 @@ int quicklistPushHead(quicklist *quicklist, void *value, size_t sz) {
  * Returns 1 if new tail created. */
 int quicklistPushTail(quicklist *quicklist, void *value, size_t sz) {
     quicklistNode *orig_tail = quicklist->tail;
+    assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
     if (likely(
             _quicklistNodeAllowInsert(quicklist->tail, quicklist->fill, sz))) {
         quicklist->tail->zl =
@@ -847,6 +858,7 @@ REDIS_STATIC void _quicklistInsert(quicklist *quicklist, quicklistEntry *entry,
     int fill = quicklist->fill;
     quicklistNode *node = entry->node;
     quicklistNode *new_node = NULL;
+    assert(sz < UINT32_MAX); /* TODO: add support for quicklist nodes that are sds encoded (not zipped) */
 
     if (!node) {
         /* we have no reference node, so let's create only node in the list */

src/rdb.cpp

@@ -547,8 +547,12 @@ void *rdbGenericLoadStringObject(rio *rdb, int flags, size_t *lenptr) {
         }
         return buf;
     } else {
-        robj *o = encode ? createStringObject(SDS_NOINIT,len) :
-                           createRawStringObject(SDS_NOINIT,len);
+        robj *o = encode ? tryCreateStringObject(SDS_NOINIT,len) :
+                           tryCreateRawStringObject(SDS_NOINIT,len);
+        if (!o) {
+            serverLog(g_pserver->loading? LL_WARNING: LL_VERBOSE, "rdbGenericLoadStringObject failed allocating %llu bytes", len);
+            return NULL;
+        }
         if (len && rioRead(rdb,ptrFromObj(o),len) == 0) {
             decrRefCount(o);
             return NULL;
@@ -1124,7 +1128,7 @@ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, expireEntry *pexpire) {
     int savelfu = g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LFU;
 
     /* Save the expire time */
-    long long expiretime = -1;
+    long long expiretime = INVALID_EXPIRE;
     if (pexpire != nullptr && pexpire->FGetPrimaryExpire(&expiretime)) {
         if (rdbSaveType(rdb,RDB_OPCODE_EXPIRETIME_MS) == -1) return -1;
         if (rdbSaveMillisecondTime(rdb,expiretime) == -1) return -1;
@@ -1201,6 +1205,40 @@ int rdbSaveInfoAuxFields(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             == -1) return -1;
         if (rdbSaveAuxFieldStrInt(rdb,"repl-offset",g_pserver->master_repl_offset)
             == -1) return -1;
+        if (g_pserver->fActiveReplica && listLength(g_pserver->masters) > 0) {
+            sdsstring val = sdsstring(sdsempty());
+            listNode *ln;
+            listIter li;
+            redisMaster* mi;
+            listRewind(g_pserver->masters,&li);
+            while ((ln = listNext(&li)) != NULL) {
+                mi = (redisMaster*)listNodeValue(ln);
+                if (!mi->master) {
+                    // If master client is not available, use info from master struct - better than nothing
+                    serverLog(LL_NOTICE, "saving master %s", mi->master_replid);
+                    if (mi->master_replid[0] == 0) {
+                        // if replid is null, there's no reason to save it
+                        continue;
+                    }
+                    val = val.catfmt("%s:%I:%s:%i;", mi->master_replid,
+                        mi->master_initial_offset,
+                        mi->masterhost,
+                        mi->masterport);
+                }
+                else {
+                    serverLog(LL_NOTICE, "saving master %s", mi->master->replid);
+                    if (mi->master->replid[0] == 0) {
+                        // if replid is null, there's no reason to save it
+                        continue;
+                    }
+                    val = val.catfmt("%s:%I:%s:%i;", mi->master->replid,
+                        mi->master->reploff,
+                        mi->masterhost,
+                        mi->masterport);
+                }
+            }
+            if (rdbSaveAuxFieldStrStr(rdb, "repl-masters",val.get()) == -1) return -1;
+        }
     }
     if (rdbSaveAuxFieldStrInt(rdb,"aof-preamble",aof_preamble) == -1) return -1;
     return 1;
@@ -1601,12 +1639,17 @@ robj *rdbLoadCheckModuleValue(rio *rdb, char *modulename) {
 }
 
 /* Load a Redis object of the specified type from the specified file.
- * On success a newly allocated object is returned, otherwise NULL. */
-robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
+ * On success a newly allocated object is returned, otherwise NULL.
+ * When the function returns NULL and if 'error' is not NULL, the
+ * integer pointed by 'error' is set to the type of error that occurred */
+robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error, uint64_t mvcc_tstamp) {
     robj *o = NULL, *ele, *dec;
     uint64_t len;
     unsigned int i;
 
+    /* Set default error of load object, it will be set to 0 on success. */
+    if (error) *error = RDB_LOAD_ERR_OTHER;
+
     int deep_integrity_validation = cserver.sanitize_dump_payload == SANITIZE_DUMP_YES;
     if (cserver.sanitize_dump_payload == SANITIZE_DUMP_CLIENTS) {
         /* Skip sanitization when loading (an RDB), or getting a RESTORE command
@@ -1625,6 +1668,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
     } else if (rdbtype == RDB_TYPE_LIST) {
         /* Read list value */
         if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
+        if (len == 0) goto emptykey;
 
         o = createQuicklistObject();
         quicklistSetOptions((quicklist*)ptrFromObj(o), g_pserver->list_max_ziplist_size,
@@ -1645,9 +1689,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
     } else if (rdbtype == RDB_TYPE_SET) {
         /* Read Set value */
         if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
+        if (len == 0) goto emptykey;
 
         /* Use a regular set when there are too many entries. */
-        if (len > g_pserver->set_max_intset_entries) {
+        size_t max_entries = g_pserver->set_max_intset_entries;
+        if (max_entries >= 1<<30) max_entries = 1<<30;
+        if (len > max_entries) {
             o = createSetObject();
             /* It's faster to expand the dict to the right size asap in order
              * to avoid rehashing */
@@ -1708,10 +1755,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
     } else if (rdbtype == RDB_TYPE_ZSET_2 || rdbtype == RDB_TYPE_ZSET) {
         /* Read list/set value. */
         uint64_t zsetlen;
-        size_t maxelelen = 0;
+        size_t maxelelen = 0, totelelen = 0;
         zset *zs;
 
         if ((zsetlen = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
+        if (zsetlen == 0) goto emptykey;
+
         o = createZsetObject();
         zs = (zset*)ptrFromObj(o);
 
@@ -1748,6 +1797,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
 
             /* Don't care about integer-encoded strings. */
             if (sdslen(sdsele) > maxelelen) maxelelen = sdslen(sdsele);
+            totelelen += sdslen(sdsele);
 
             znode = zslInsert(zs->zsl,score,sdsele);
             if (dictAdd(zs->dict,sdsele,&znode->score) != DICT_OK) {
@@ -1760,8 +1810,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
 
         /* Convert *after* loading, since sorted sets are not stored ordered. */
         if (zsetLength(o) <= g_pserver->zset_max_ziplist_entries &&
-            maxelelen <= g_pserver->zset_max_ziplist_value)
+            maxelelen <= g_pserver->zset_max_ziplist_value &&
+            ziplistSafeToAdd(NULL, totelelen))
                 zsetConvert(o,OBJ_ENCODING_ZIPLIST);
+
     } else if (rdbtype == RDB_TYPE_HASH) {
         uint64_t len;
         int ret;
@@ -1770,6 +1822,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
 
         len = rdbLoadLen(rdb, NULL);
         if (len == RDB_LENERR) return NULL;
+        if (len == 0) goto emptykey;
 
         o = createHashObject();
 
@@ -1814,21 +1867,31 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                 }
             }
 
+            /* Convert to hash table if size threshold is exceeded */
+            if (sdslen(field) > g_pserver->hash_max_ziplist_value ||
+                sdslen(value) > g_pserver->hash_max_ziplist_value || 
+                !ziplistSafeToAdd((unsigned char*)ptrFromObj(o), sdslen(field)+sdslen(value)))
+            {
+                hashTypeConvert(o, OBJ_ENCODING_HT);
+                ret = dictAdd((dict*)ptrFromObj(o), field, value);
+                if (ret == DICT_ERR) {
+                    rdbReportCorruptRDB("Duplicate hash fields detected");
+                    if (dupSearchDict) dictRelease(dupSearchDict);
+                    sdsfree(value);
+                    sdsfree(field);
+                    decrRefCount(o);
+                    return NULL;
+                }
+                break;
+            }
+
             /* Add pair to ziplist */
             o->m_ptr = ziplistPush((unsigned char*)ptrFromObj(o), (unsigned char*)field,
                     sdslen(field), ZIPLIST_TAIL);
             o->m_ptr = ziplistPush((unsigned char*)ptrFromObj(o), (unsigned char*)value,
                     sdslen(value), ZIPLIST_TAIL);
 
-            /* Convert to hash table if size threshold is exceeded */
-            if (sdslen(field) > g_pserver->hash_max_ziplist_value ||
-                sdslen(value) > g_pserver->hash_max_ziplist_value)
-            {
-                sdsfree(field);
-                sdsfree(value);
-                hashTypeConvert(o, OBJ_ENCODING_HT);
-                break;
-            }
+
             sdsfree(field);
             sdsfree(value);
         }
@@ -1877,6 +1940,8 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
         serverAssert(len == 0);
     } else if (rdbtype == RDB_TYPE_LIST_QUICKLIST) {
         if ((len = rdbLoadLen(rdb,NULL)) == RDB_LENERR) return NULL;
+        if (len == 0) goto emptykey;
+
         o = createQuicklistObject();
         quicklistSetOptions((quicklist*)ptrFromObj(o), g_pserver->list_max_ziplist_size,
                             g_pserver->list_compress_depth);
@@ -1896,7 +1961,19 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                 zfree(zl);
                 return NULL;
             }
-            quicklistAppendZiplist((quicklist*)ptrFromObj(o), zl);
+
+            /* Silently skip empty ziplists, if we'll end up with empty quicklist we'll fail later. */
+            if (ziplistLen(zl) == 0) {
+                zfree(zl);
+                continue;
+            } else {
+                quicklistAppendZiplist((quicklist*)ptrFromObj(o), zl);
+            }
+        }
+
+        if (quicklistCount((quicklist*)ptrFromObj(o)) == 0) {
+            decrRefCount(o);
+            goto emptykey;
         }
     } else if (rdbtype == RDB_TYPE_HASH_ZIPMAP  ||
                rdbtype == RDB_TYPE_LIST_ZIPLIST ||
@@ -1941,12 +2018,11 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                     while ((zi = zipmapNext(zi, &fstr, &flen, &vstr, &vlen)) != NULL) {
                         if (flen > maxlen) maxlen = flen;
                         if (vlen > maxlen) maxlen = vlen;
-                        zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL);
-                        zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL);
 
                         /* search for duplicate records */
                         sds field = sdstrynewlen(fstr, flen);
-                        if (!field || dictAdd(dupSearchDict, field, NULL) != DICT_OK) {
+                        if (!field || dictAdd(dupSearchDict, field, NULL) != DICT_OK ||
+                            !ziplistSafeToAdd(zl, (size_t)flen + vlen)) {
                             rdbReportCorruptRDB("Hash zipmap with dup elements, or big length (%u)", flen);
                             dictRelease(dupSearchDict);
                             sdsfree(field);
@@ -1955,6 +2031,9 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                             decrRefCount(o);
                             return NULL;
                         }
+
+                        zl = ziplistPush(zl, fstr, flen, ZIPLIST_TAIL);
+                        zl = ziplistPush(zl, vstr, vlen, ZIPLIST_TAIL);
                     }
 
                     dictRelease(dupSearchDict);
@@ -1979,6 +2058,14 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                     decrRefCount(o);
                     return NULL;
                 }
+
+                if (ziplistLen(encoded) == 0) {
+                    zfree(encoded);
+                    o->m_ptr = NULL;
+                    decrRefCount(o);
+                    goto emptykey;
+                }
+
                 o->type = OBJ_LIST;
                 o->encoding = OBJ_ENCODING_ZIPLIST;
                 listTypeConvert(o,OBJ_ENCODING_QUICKLIST);
@@ -2008,6 +2095,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                 }
                 o->type = OBJ_ZSET;
                 o->encoding = OBJ_ENCODING_ZIPLIST;
+                if (zsetLength(o) == 0) {
+                    zfree(encoded);
+                    o->m_ptr = NULL;
+                    decrRefCount(o);
+                    goto emptykey;
+                }
+
                 if (zsetLength(o) > g_pserver->zset_max_ziplist_entries)
                     zsetConvert(o,OBJ_ENCODING_SKIPLIST);
                 break;
@@ -2022,6 +2116,13 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                 }
                 o->type = OBJ_HASH;
                 o->encoding = OBJ_ENCODING_ZIPLIST;
+                if (hashTypeLength(o) == 0) {
+                    zfree(encoded);
+                    o->m_ptr = NULL;
+                    decrRefCount(o);
+                    goto emptykey;
+                }
+
                 if (hashTypeLength(o) > g_pserver->hash_max_ziplist_entries)
                     hashTypeConvert(o, OBJ_ENCODING_HT);
                 break;
@@ -2090,7 +2191,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
             }
 
             /* Insert the key in the radix tree. */
-            int retval = raxInsert(s->rax,
+            int retval = raxTryInsert(s->rax,
                 (unsigned char*)nodekey,sizeof(streamID),lp,NULL);
             sdsfree(nodekey);
             if (!retval) {
@@ -2179,7 +2280,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                     streamFreeNACK(nack);
                     return NULL;
                 }
-                if (!raxInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) {
+                if (!raxTryInsert(cgroup->pel,rawid,sizeof(rawid),nack,NULL)) {
                     rdbReportCorruptRDB("Duplicated global PEL entry "
                                             "loading stream consumer group");
                     decrRefCount(o);
@@ -2243,15 +2344,33 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
                      * loading the global PEL. Then set the same shared
                      * NACK structure also in the consumer-specific PEL. */
                     nack->consumer = consumer;
-                    if (!raxInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL)) {
+                    if (!raxTryInsert(consumer->pel,rawid,sizeof(rawid),nack,NULL)) {
                         rdbReportCorruptRDB("Duplicated consumer PEL entry "
                                                 " loading a stream consumer "
                                                 "group");
                         decrRefCount(o);
+                        streamFreeNACK(nack);
                         return NULL;
                     }
                 }
             }
+
+            /* Verify that each PEL eventually got a consumer assigned to it. */
+            if (deep_integrity_validation) {
+                raxIterator ri_cg_pel;
+                raxStart(&ri_cg_pel,cgroup->pel);
+                raxSeek(&ri_cg_pel,"^",NULL,0);
+                while(raxNext(&ri_cg_pel)) {
+                    streamNACK *nack = (streamNACK *)ri_cg_pel.data;
+                    if (!nack->consumer) {
+                        raxStop(&ri_cg_pel);
+                        rdbReportCorruptRDB("Stream CG PEL entry without consumer");
+                        decrRefCount(o);
+                        return NULL;
+                    }
+                }
+                raxStop(&ri_cg_pel);
+            }
         }
     } else if (rdbtype == RDB_TYPE_MODULE || rdbtype == RDB_TYPE_MODULE_2) {
         uint64_t moduleid = rdbLoadLen(rdb,NULL);
@@ -2333,7 +2452,12 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, uint64_t mvcc_tstamp) {
 
     setMvccTstamp(o, mvcc_tstamp);
     serverAssert(!o->FExpires());
+    if (error) *error = 0;
     return o;
+
+emptykey:
+    if (error) *error = RDB_LOAD_ERR_EMPTY_KEY;
+    return NULL;
 }
 
 /* Mark that we are loading in the global state and setup the fields
@@ -2459,6 +2583,8 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
     robj *subexpireKey = nullptr;
     sds key = nullptr;
     bool fLastKeyExpired = false;
+    int error;
+    long long empty_keys_skipped = 0, expired_keys_skipped = 0, keys_loaded = 0;
 
     rdb->update_cksum = rdbLoadProgressCallback;
     rdb->max_processing_chunk = g_pserver->loading_process_events_interval_bytes;
@@ -2561,6 +2687,23 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                     memcpy(rsi->repl_id,ptrFromObj(auxval),CONFIG_RUN_ID_SIZE+1);
                     rsi->repl_id_is_set = 1;
                 }
+            } else if (!strcasecmp(szFromObj(auxkey),"repl-masters")) {
+                if (rsi) {
+                    struct redisMaster mi;
+                    char *masters = szFromObj(auxval);
+                    char *entry = strtok(masters, ":");
+                    while (entry != NULL) {
+                        memcpy(mi.master_replid, entry, sizeof(mi.master_replid));
+                        entry = strtok(NULL, ":");
+                        mi.master_initial_offset = atoi(entry);
+                        entry = strtok(NULL, ":");
+                        mi.masterhost = entry;
+                        entry = strtok(NULL, ";");
+                        mi.masterport = atoi(entry);
+                        entry = strtok(NULL, ":");
+                        rsi->addMaster(mi);
+                    }
+                }
             } else if (!strcasecmp(szFromObj(auxkey),"repl-offset")) {
                 if (rsi) rsi->repl_offset = strtoll(szFromObj(auxval),NULL,10);
             } else if (!strcasecmp(szFromObj(auxkey),"lua")) {
@@ -2636,8 +2779,10 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             int when_opcode = rdbLoadLen(rdb,NULL);
             int when = rdbLoadLen(rdb,NULL);
             if (rioGetReadError(rdb)) goto eoferr;
-            if (when_opcode != RDB_MODULE_OPCODE_UINT)
+            if (when_opcode != RDB_MODULE_OPCODE_UINT) {
                 rdbReportReadError("bad when_opcode");
+                goto eoferr;
+            }
             moduleType *mt = moduleTypeLookupModuleByID(moduleid);
             char name[10];
             moduleTypeNameByID(name,moduleid);
@@ -2661,7 +2806,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                 if (mt->aux_load(&io,moduleid&1023, when) != REDISMODULE_OK || io.error) {
                     moduleTypeNameByID(name,moduleid);
                     serverLog(LL_WARNING,"The RDB file contains module AUX data for the module type '%s', that the responsible module is not able to load. Check for modules log above for additional clues.", name);
-                    exit(1);
+                    goto eoferr;
                 }
                 if (io.ctx) {
                     moduleFreeContext(io.ctx);
@@ -2670,7 +2815,7 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
                 uint64_t eof = rdbLoadLen(rdb,NULL);
                 if (eof != RDB_MODULE_OPCODE_EOF) {
                     serverLog(LL_WARNING,"The RDB file contains module AUX data for the module '%s' that is not terminated by the proper module value EOF marker", name);
-                    exit(1);
+                    goto eoferr;
                 }
                 continue;
             } else {
@@ -2691,64 +2836,79 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
         if ((key = (sds)rdbGenericLoadStringObject(rdb,RDB_LOAD_SDS,NULL)) == NULL)
             goto eoferr;
         /* Read value */
-        if ((val = rdbLoadObject(type,rdb,key,mvcc_tstamp)) == NULL) {
-            sdsfree(key);
-            key = nullptr;
-            goto eoferr;
-        }
-        bool fStaleMvccKey = (rsi) ? mvccFromObj(val) < rsi->mvccMinThreshold : false;
-
-        /* Check if the key already expired. This function is used when loading
-         * an RDB file from disk, either at startup, or when an RDB was
-         * received from the master. In the latter case, the master is
-         * responsible for key expiry. If we would expire keys here, the
-         * snapshot taken by the master may not be reflected on the replica.
-         * Similarly if the RDB is the preamble of an AOF file, we want to
-         * load all the keys as they are, since the log of operations later
-         * assume to work in an exact keyspace state. */
-        redisObjectStack keyobj;
-        initStaticStringObject(keyobj,key);
-        bool fExpiredKey = iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now;
-        if (fStaleMvccKey || fExpiredKey) {
-            if (fStaleMvccKey && !fExpiredKey && rsi != nullptr && rsi->mi != nullptr && rsi->mi->staleKeyMap != nullptr && lookupKeyRead(db, &keyobj) == nullptr) {
-                // We have a key that we've already deleted and is not back in our database.
-                //  We'll need to inform the sending master of the delete if it is also a replica of us
-                robj_sharedptr objKeyDup(createStringObject(key, sdslen(key)));
-                rsi->mi->staleKeyMap->operator[](db - g_pserver->db).push_back(objKeyDup);                
+        val = rdbLoadObject(type,rdb,key,&error,mvcc_tstamp);
+        if (val == NULL) {
+            /* Since we used to have bug that could lead to empty keys
+             * (See #8453), we rather not fail when empty key is encountered
+             * in an RDB file, instead we will silently discard it and
+             * continue loading. */
+            if (error == RDB_LOAD_ERR_EMPTY_KEY) {
+                if(empty_keys_skipped++ < 10)
+                    serverLog(LL_WARNING, "rdbLoadObject skipping empty key: %s", key);
+                sdsfree(key);
+                key = nullptr;
+            } else {
+                sdsfree(key);
+                key = nullptr;
+                goto eoferr;
             }
-            fLastKeyExpired = true;
-            sdsfree(key);
-            key = nullptr;
-            decrRefCount(val);
-            val = nullptr;
         } else {
+            bool fStaleMvccKey = (rsi) ? mvccFromObj(val) < rsi->mvccMinThreshold : false;
+
+            /* Check if the key already expired. This function is used when loading
+            * an RDB file from disk, either at startup, or when an RDB was
+            * received from the master. In the latter case, the master is
+            * responsible for key expiry. If we would expire keys here, the
+            * snapshot taken by the master may not be reflected on the replica.
+            * Similarly if the RDB is the preamble of an AOF file, we want to
+            * load all the keys as they are, since the log of operations later
+            * assume to work in an exact keyspace state. */
             redisObjectStack keyobj;
             initStaticStringObject(keyobj,key);
-
-            /* Add the new object in the hash table */
-            int fInserted = dbMerge(db, key, val, (rsi && rsi->fForceSetKey) || (rdbflags & RDBFLAGS_ALLOW_DUP));   // Note: dbMerge will incrRef
-            fLastKeyExpired = false;
-
-            if (fInserted)
-            {
-                /* Set the expire time if needed */
-                if (expiretime != -1)
-                {
-                    setExpire(NULL,db,&keyobj,nullptr,expiretime);
+            bool fExpiredKey = iAmMaster() && !(rdbflags&RDBFLAGS_AOF_PREAMBLE) && expiretime != -1 && expiretime < now;
+        
+            if (fStaleMvccKey || fExpiredKey) {
+                if (fStaleMvccKey && !fExpiredKey && rsi != nullptr && rsi->masters != nullptr && rsi->masters[0].staleKeyMap != nullptr && lookupKeyRead(db, &keyobj) == nullptr) {
+                    // We have a key that we've already deleted and is not back in our database.
+                    //  We'll need to inform the sending master of the delete if it is also a replica of us
+                    robj_sharedptr objKeyDup(createStringObject(key, sdslen(key)));
+                    rsi->masters[0].staleKeyMap->operator[](db - g_pserver->db).push_back(objKeyDup);                
                 }
-
-                /* Set usage information (for eviction). */
-                objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock,1000);
-
-                /* call key space notification on key loaded for modules only */
-                moduleNotifyKeyspaceEvent(NOTIFY_LOADED, "loaded", &keyobj, db->id);
-
-                replicationNotifyLoadedKey(db, &keyobj, val, expiretime);
-            }
-            else
-            {
+                fLastKeyExpired = true;
+                sdsfree(key);
+                key = nullptr;
                 decrRefCount(val);
                 val = nullptr;
+                expired_keys_skipped++;
+            } else {
+                redisObjectStack keyobj;
+                initStaticStringObject(keyobj,key);
+
+                /* Add the new object in the hash table */
+                int fInserted = dbMerge(db, key, val, (rsi && rsi->fForceSetKey) || (rdbflags & RDBFLAGS_ALLOW_DUP));   // Note: dbMerge will incrRef
+                fLastKeyExpired = false;
+                keys_loaded++;
+                if (fInserted)
+                {
+                    /* Set the expire time if needed */
+                    if (expiretime != -1)
+                    {
+                        setExpire(NULL,db,&keyobj,nullptr,expiretime);
+                    }
+
+                    /* Set usage information (for eviction). */
+                    objectSetLRUOrLFU(val,lfu_freq,lru_idle,lru_clock,1000);
+
+                    /* call key space notification on key loaded for modules only */
+                    moduleNotifyKeyspaceEvent(NOTIFY_LOADED, "loaded", &keyobj, db->id);
+
+                    replicationNotifyLoadedKey(db, &keyobj, val, expiretime);
+                }
+                else
+                {
+                    decrRefCount(val);
+                    val = nullptr;
+                }
             }
         }
 
@@ -2795,6 +2955,16 @@ int rdbLoadRio(rio *rdb, int rdbflags, rdbSaveInfo *rsi) {
             }
         }
     }
+
+    if (empty_keys_skipped) {
+        serverLog(LL_WARNING,
+            "Done loading RDB, keys loaded: %lld, keys expired: %lld, empty keys skipped: %lld.",
+                keys_loaded, expired_keys_skipped, empty_keys_skipped);
+    } else {
+        serverLog(LL_WARNING,
+            "Done loading RDB, keys loaded: %lld, keys expired: %lld.",
+                keys_loaded, expired_keys_skipped);
+    }
     return C_OK;
 
     /* Unexpected end of file is handled here calling rdbReportReadError():
@@ -3127,8 +3297,8 @@ void bgsaveCommand(client *c) {
  * is returned, and the RDB saving will not persist any replication related
  * information. */
 rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi) {
-    rdbSaveInfo rsi_init = RDB_SAVE_INFO_INIT;
-    *rsi = rsi_init;
+    rdbSaveInfo rsi_init;
+    *rsi = std::move(rsi_init);
 
     /* If the instance is a master, we can populate the replication info
      * only when repl_backlog is not NULL. If the repl_backlog is NULL,

src/rdb.h

@@ -130,6 +130,11 @@
 #define RDBFLAGS_REPLICATION (1<<1)     /* Load/save for SYNC. */
 #define RDBFLAGS_ALLOW_DUP (1<<2)       /* Allow duplicated keys when loading.*/
 
+/* When rdbLoadObject() returns NULL, the err flag is
+ * set to hold the type of error that occurred */
+#define RDB_LOAD_ERR_EMPTY_KEY  1   /* Error of empty key */
+#define RDB_LOAD_ERR_OTHER      2   /* Any other errors */
+
 int rdbSaveType(rio *rdb, unsigned char type);
 int rdbLoadType(rio *rdb);
 int rdbSaveTime(rio *rdb, time_t t);
@@ -153,7 +158,7 @@ int rdbSaveS3(char *path, rdbSaveInfo *rsi);
 int rdbLoadS3(char *path, rdbSaveInfo *rsi, int rdbflags);
 ssize_t rdbSaveObject(rio *rdb, robj_roptr o, robj *key);
 size_t rdbSavedObjectLen(robj *o, robj *key);
-robj *rdbLoadObject(int type, rio *rdb, sds key, uint64_t mvcc_tstamp);
+robj *rdbLoadObject(int type, rio *rdb, sds key, int *error, uint64_t mvcc_tstamp);
 void backgroundSaveDoneHandler(int exitcode, int bysignal);
 int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime);
 ssize_t rdbSaveSingleModuleAux(rio *rdb, int when, moduleType *mt);

src/readwritelock.h

@@ -0,0 +1,96 @@
+#pragma once
+#include <condition_variable>
+
+class readWriteLock {
+    fastlock m_readLock;
+    fastlock m_writeLock;
+    std::condition_variable_any m_cv;
+    int m_readCount = 0;
+    int m_writeCount = 0;
+    bool m_writeWaiting = false;
+public:
+    readWriteLock(const char *name) : m_readLock(name), m_writeLock(name) {}
+
+    void acquireRead() {
+        std::unique_lock<fastlock> rm(m_readLock);
+        while (m_writeCount > 0 || m_writeWaiting)
+            m_cv.wait(rm);
+        m_readCount++;
+    }
+    
+    bool tryAcquireRead() {
+        std::unique_lock<fastlock> rm(m_readLock, std::defer_lock);
+        if (!rm.try_lock())
+            return false;
+        if (m_writeCount > 0 || m_writeWaiting)
+            return false;
+        m_readCount++;
+        return true;
+    }
+
+    void acquireWrite(bool exclusive = true) {
+        std::unique_lock<fastlock> rm(m_readLock);
+        m_writeWaiting = true;
+        while (m_readCount > 0)
+            m_cv.wait(rm);
+        if (exclusive) {
+            /* Another thread might have the write lock while we have the read lock
+               but won't be able to release it until they can acquire the read lock
+               so release the read lock and try again instead of waiting to avoid deadlock */
+            while(!m_writeLock.try_lock())
+                m_cv.wait(rm);
+        }
+        m_writeCount++;
+        m_writeWaiting = false;
+    }
+
+    void upgradeWrite(bool exclusive = true) {
+        releaseRead();
+        acquireWrite(exclusive);
+    }
+
+    bool tryAcquireWrite(bool exclusive = true) {
+        std::unique_lock<fastlock> rm(m_readLock, std::defer_lock);
+        if (!rm.try_lock())
+            return false;
+        if (m_readCount > 0)
+            return false;
+        if (exclusive)
+            if (!m_writeLock.try_lock())
+                return false;
+        m_writeCount++;
+        return true;
+    }
+
+    void releaseRead() {
+        std::unique_lock<fastlock> rm(m_readLock);
+        m_readCount--;
+        m_cv.notify_all();
+    }
+
+    void releaseWrite(bool exclusive = true) {
+        std::unique_lock<fastlock> rm(m_readLock);
+        serverAssert(m_writeCount > 0);
+        if (exclusive)
+            m_writeLock.unlock();
+        m_writeCount--;
+        m_cv.notify_all();
+    }
+
+    void downgradeWrite(bool exclusive = true) {
+        releaseWrite(exclusive);
+        acquireRead();
+    }
+
+    bool hasReader() {
+        return m_readCount > 0;
+    }
+
+    bool hasWriter() {
+        return m_writeCount > 0;
+    }
+
+    bool writeWaiting() {
+        return m_writeWaiting;
+    }
+};

src/redis-benchmark.cpp

@@ -103,7 +103,6 @@ static struct config {
     int randomkeys_keyspacelen;
     int keepalive;
     int pipeline;
-    int showerrors;
     long long start;
     long long totlatency;
     const char *title;
@@ -313,7 +312,9 @@ static redisContext *getRedisContext(const char *ip, int port,
                 fprintf(stderr, "Node %s:%d replied with error:\n%s\n", ip, port, reply->str);
             else
                 fprintf(stderr, "Node %s replied with error:\n%s\n", hostsocket, reply->str);
-            goto cleanup;
+            freeReplyObject(reply);
+            redisFree(ctx);
+            exit(1);
         }
         freeReplyObject(reply);
         return ctx;
@@ -370,9 +371,16 @@ fail:
     fprintf(stderr, "ERROR: failed to fetch CONFIG from ");
     if (hostsocket == NULL) fprintf(stderr, "%s:%d\n", ip, port);
     else fprintf(stderr, "%s\n", hostsocket);
+    int abort_test = 0;
+    if (reply && reply->type == REDIS_REPLY_ERROR &&
+        (!strncmp(reply->str,"NOAUTH",5) ||
+         !strncmp(reply->str,"WRONGPASS",9) ||
+         !strncmp(reply->str,"NOPERM",5)))
+        abort_test = 1;
     freeReplyObject(reply);
     redisFree(c);
     freeRedisConfig(cfg);
+    if (abort_test) exit(1);
     return NULL;
 }
 static void freeRedisConfig(redisConfig *cfg) {
@@ -517,44 +525,39 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
                     exit(1);
                 }
                 redisReply *r = (redisReply*)reply;
-                int is_err = (r->type == REDIS_REPLY_ERROR);
-
-                if (is_err && config.showerrors) {
-                    /* TODO: static lasterr_time not thread-safe */
-                    static time_t lasterr_time = 0;
-                    time_t now = time(NULL);
-                    if (lasterr_time != now) {
-                        lasterr_time = now;
-                        if (c->cluster_node) {
-                            printf("Error from server %s:%d: %s\n",
+                if (r->type == REDIS_REPLY_ERROR) {
+                    /* Try to update slots configuration if reply error is
+                    * MOVED/ASK/CLUSTERDOWN and the key(s) used by the command
+                    * contain(s) the slot hash tag.
+                    * If the error is not topology-update related then we
+                    * immediately exit to avoid false results. */
+                    if (c->cluster_node && c->staglen) {
+                        int fetch_slots = 0, do_wait = 0;
+                        if (!strncmp(r->str,"MOVED",5) || !strncmp(r->str,"ASK",3))
+                            fetch_slots = 1;
+                        else if (!strncmp(r->str,"CLUSTERDOWN",11)) {
+                            /* Usually the cluster is able to recover itself after
+                            * a CLUSTERDOWN error, so try to sleep one second
+                            * before requesting the new configuration. */
+                            fetch_slots = 1;
+                            do_wait = 1;
+                            printf("Error from server %s:%d: %s.\n",
                                    c->cluster_node->ip,
                                    c->cluster_node->port,
                                    r->str);
+                        }
+                        if (do_wait) sleep(1);
+                        if (fetch_slots && !fetchClusterSlotsConfiguration(c))
+                            exit(1);
+                    } else {
+                        if (c->cluster_node) {
+                            printf("Error from server %s:%d: %s\n",
+                                c->cluster_node->ip,
+                                c->cluster_node->port,
+                                r->str);
                         } else printf("Error from server: %s\n", r->str);
-                    }
-                }
-
-                /* Try to update slots configuration if reply error is
-                 * MOVED/ASK/CLUSTERDOWN and the key(s) used by the command
-                 * contain(s) the slot hash tag. */
-                if (is_err && c->cluster_node && c->staglen) {
-                    int fetch_slots = 0, do_wait = 0;
-                    if (!strncmp(r->str,"MOVED",5) || !strncmp(r->str,"ASK",3))
-                        fetch_slots = 1;
-                    else if (!strncmp(r->str,"CLUSTERDOWN",11)) {
-                        /* Usually the cluster is able to recover itself after
-                         * a CLUSTERDOWN error, so try to sleep one second
-                         * before requesting the new configuration. */
-                        fetch_slots = 1;
-                        do_wait = 1;
-                        printf("Error from server %s:%d: %s\n",
-                               c->cluster_node->ip,
-                               c->cluster_node->port,
-                               r->str);
-                    }
-                    if (do_wait) sleep(1);
-                    if (fetch_slots && !fetchClusterSlotsConfiguration(c))
                         exit(1);
+                    }
                 }
 
                 freeReplyObject(reply);
@@ -1298,8 +1301,7 @@ static int fetchClusterSlotsConfiguration(client c) {
     atomicGetIncr(config.is_fetching_slots, is_fetching_slots, 1);
     if (is_fetching_slots) return -1; //TODO: use other codes || errno ?
     atomicSet(config.is_fetching_slots, 1);
-    if (config.showerrors)
-        printf("Cluster slots configuration changed, fetching new one...\n");
+    printf("WARNING: Cluster slots configuration changed, fetching new one...\n");
     const char *errmsg = "Failed to update cluster slots configuration";
     static dictType dtype = {
         dictSdsHash,               /* hash function */
@@ -1475,7 +1477,8 @@ int parseOptions(int argc, const char **argv) {
         } else if (!strcmp(argv[i],"-I")) {
             config.idlemode = 1;
         } else if (!strcmp(argv[i],"-e")) {
-            config.showerrors = 1;
+            printf("WARNING: -e option has been deprecated. "
+                   "We now immediatly exit on error to avoid false results.\n");
         } else if (!strcmp(argv[i],"-t")) {
             if (lastarg) goto invalid;
             /* We get the list of tests to run as a string in the form
@@ -1578,8 +1581,6 @@ usage:
 "  is executed. Default tests use this to hit random keys in the\n"
 "  specified range.\n"
 " -P <numreq>        Pipeline <numreq> requests. Default 1 (no pipeline).\n"
-" -e                 If server replies with errors, show them on stdout.\n"
-"                    (no more than 1 error per second is displayed)\n"
 " -q                 Quiet. Just show query/sec values\n"
 " --precision        Number of decimal places to display in latency output (default 0)\n"
 " --csv              Output in CSV format\n"
@@ -1707,7 +1708,6 @@ int main(int argc, const char **argv) {
     config.keepalive = 1;
     config.datasize = 3;
     config.pipeline = 1;
-    config.showerrors = 0;
     config.randomkeys = 0;
     config.randomkeys_keyspacelen = 0;
     config.quiet = 0;
@@ -1790,8 +1790,9 @@ int main(int argc, const char **argv) {
     } else {
         config.redis_config =
             getRedisConfig(config.hostip, config.hostport, config.hostsocket);
-        if (config.redis_config == NULL)
+        if (config.redis_config == NULL) {
             fprintf(stderr, "WARN: could not fetch server CONFIG\n");
+        }
     }
     if (config.num_threads > 0) {
         pthread_mutex_init(&(config.liveclients_mutex), NULL);
@@ -1954,8 +1955,8 @@ int main(int argc, const char **argv) {
         }
 
         if (test_is_selected("lrange") || test_is_selected("lrange_500")) {
-            len = redisFormatCommand(&cmd,"LRANGE mylist%s 0 449",tag);
-            benchmark("LRANGE_500 (first 450 elements)",cmd,len);
+            len = redisFormatCommand(&cmd,"LRANGE mylist%s 0 499",tag);
+            benchmark("LRANGE_500 (first 500 elements)",cmd,len);
             free(cmd);
         }
 

src/redis-check-aof.cpp

@@ -39,12 +39,14 @@
 
 static char error[1044];
 static off_t epos;
+static long long line = 1;
 
 int consumeNewline(char *buf) {
     if (strncmp(buf,"\r\n",2) != 0) {
         ERROR("Expected \\r\\n, got: %02x%02x",buf[0],buf[1]);
         return 0;
     }
+    line += 1;
     return 1;
 }
 
@@ -201,8 +203,8 @@ int redis_check_aof_main(int argc, char **argv) {
 
     off_t pos = process(fp);
     off_t diff = size-pos;
-    printf("AOF analyzed: size=%lld, ok_up_to=%lld, diff=%lld\n",
-        (long long) size, (long long) pos, (long long) diff);
+    printf("AOF analyzed: size=%lld, ok_up_to=%lld, ok_up_to_line=%lld, diff=%lld\n",
+        (long long) size, (long long) pos, line, (long long) diff);
     if (diff > 0) {
         if (fix) {
             char buf[2];

src/redis-check-rdb.cpp

@@ -250,7 +250,7 @@ int redis_check_rdb(const char *rdbfilename, FILE *fp) {
             rdbstate.doing = RDB_CHECK_DOING_READ_LEN;
             if ((dbid = rdbLoadLen(&rdb,NULL)) == RDB_LENERR)
                 goto eoferr;
-            rdbCheckInfo("Selecting DB ID %d", dbid);
+            rdbCheckInfo("Selecting DB ID %llu", (unsigned long long)dbid);
             continue; /* Read type again. */
         } else if (type == RDB_OPCODE_RESIZEDB) {
             /* RESIZEDB: Hint about the size of the keys in the currently
@@ -308,7 +308,7 @@ int redis_check_rdb(const char *rdbfilename, FILE *fp) {
         rdbstate.keys++;
         /* Read value */
         rdbstate.doing = RDB_CHECK_DOING_READ_OBJECT_VALUE;
-        if ((val = rdbLoadObject(type,&rdb,szFromObj(key),OBJ_MVCC_INVALID)) == NULL) goto eoferr;
+        if ((val = rdbLoadObject(type,&rdb,szFromObj(key),NULL,OBJ_MVCC_INVALID)) == NULL) goto eoferr;
         /* Check if the key already expired. */
         if (expiretime != -1 && expiretime < now)
             rdbstate.already_expired++;

src/redis-cli.c

@@ -277,7 +277,7 @@ static void parseRedisUri(const char *uri) {
     if (curr == end) return;
 
     /* Extract database number. */
-    config.dbnum = atoi(curr);
+    config.input_dbnum = atoi(curr);
 }
 
 /* _serverAssert is needed by dict */
@@ -467,7 +467,7 @@ static void cliOutputHelp(int argc, char **argv) {
         help = entry->org;
         if (group == -1) {
             /* Compare all arguments */
-            if (argc == entry->argc) {
+            if (argc <= entry->argc) {
                 for (j = 0; j < argc; j++) {
                     if (strcasecmp(argv[j],entry->argv[j]) != 0) break;
                 }
@@ -596,28 +596,42 @@ static int cliAuth(redisContext *ctx, char *user, char *auth) {
         reply = redisCommand(ctx,"AUTH %s",auth);
     else
         reply = redisCommand(ctx,"AUTH %s %s",user,auth);
-    if (reply != NULL) {
-        if (reply->type == REDIS_REPLY_ERROR)
-            fprintf(stderr,"Warning: AUTH failed\n");
-        freeReplyObject(reply);
-        return REDIS_OK;
+
+    if (reply == NULL) {
+        fprintf(stderr, "\nI/O error\n");
+        return REDIS_ERR;
     }
-    return REDIS_ERR;
+
+    int result = REDIS_OK;
+    if (reply->type == REDIS_REPLY_ERROR) {
+        result = REDIS_ERR;
+        fprintf(stderr, "AUTH failed: %s\n", reply->str);
+    }
+    freeReplyObject(reply);
+    return result;
 }
 
-/* Send SELECT dbnum to the server */
+/* Send SELECT input_dbnum to the server */
 static int cliSelect(void) {
     redisReply *reply;
-    if (config.dbnum == 0) return REDIS_OK;
+    if (config.input_dbnum == config.dbnum) return REDIS_OK;
 
-    reply = redisCommand(context,"SELECT %d",config.dbnum);
-    if (reply != NULL) {
-        int result = REDIS_OK;
-        if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR;
-        freeReplyObject(reply);
-        return result;
+    reply = redisCommand(context,"SELECT %d",config.input_dbnum);
+    if (reply == NULL) {
+        fprintf(stderr, "\nI/O error\n");
+        return REDIS_ERR;
     }
-    return REDIS_ERR;
+
+    int result = REDIS_OK;
+    if (reply->type == REDIS_REPLY_ERROR) {
+        result = REDIS_ERR;
+        fprintf(stderr,"SELECT %d failed: %s\n",config.input_dbnum,reply->str);
+    } else {
+        config.dbnum = config.input_dbnum;
+        cliRefreshPrompt();
+    }
+    freeReplyObject(reply);
+    return result;
 }
 
 /* Select RESP3 mode if redis-cli was started with the -3 option.  */
@@ -626,13 +640,18 @@ static int cliSwitchProto(void) {
     if (config.resp3 == 0) return REDIS_OK;
 
     reply = redisCommand(context,"HELLO 3");
-    if (reply != NULL) {
-        int result = REDIS_OK;
-        if (reply->type == REDIS_REPLY_ERROR) result = REDIS_ERR;
-        freeReplyObject(reply);
-        return result;
+    if (reply == NULL) {
+        fprintf(stderr, "\nI/O error\n");
+        return REDIS_ERR;
     }
-    return REDIS_ERR;
+
+    int result = REDIS_OK;
+    if (reply->type == REDIS_REPLY_ERROR) {
+        result = REDIS_ERR;
+        fprintf(stderr,"HELLO 3 failed: %s\n",reply->str);
+    }
+    freeReplyObject(reply);
+    return result;
 }
 
 /* Connect to the server. It is possible to pass certain flags to the function:
@@ -648,7 +667,9 @@ static int cliConnect(int flags) {
             cliRefreshPrompt();
         }
 
-        if (config.hostsocket == NULL) {
+        /* Do not use hostsocket when we got redirected in cluster mode */
+        if (config.hostsocket == NULL ||
+            (config.cluster_mode && config.cluster_reissue_command)) {
             context = redisConnect(config.hostip,config.hostport);
         } else {
             context = redisConnectUnix(config.hostsocket);
@@ -667,12 +688,15 @@ static int cliConnect(int flags) {
         if (context->err) {
             if (!(flags & CC_QUIET)) {
                 fprintf(stderr,"Could not connect to Redis at ");
-                if (config.hostsocket == NULL)
-                    fprintf(stderr,"%s:%d: %s\n",
+                if (config.hostsocket == NULL ||
+                    (config.cluster_mode && config.cluster_reissue_command))
+                {
+                    fprintf(stderr, "%s:%d: %s\n",
                         config.hostip,config.hostport,context->errstr);
-                else
+                } else {
                     fprintf(stderr,"%s: %s\n",
                         config.hostsocket,context->errstr);
+                }
             }
             redisFree(context);
             context = NULL;
@@ -703,6 +727,29 @@ static int cliConnect(int flags) {
     return REDIS_OK;
 }
 
+/* In cluster, if server replies ASK, we will redirect to a different node.
+ * Before sending the real command, we need to send ASKING command first. */
+static int cliSendAsking() {
+    redisReply *reply;
+
+    config.cluster_send_asking = 0;
+    if (context == NULL) {
+        return REDIS_ERR;
+    }
+    reply = redisCommand(context,"ASKING");
+    if (reply == NULL) {
+        fprintf(stderr, "\nI/O error\n");
+        return REDIS_ERR;
+    }
+    int result = REDIS_OK;
+    if (reply->type == REDIS_REPLY_ERROR) {
+        result = REDIS_ERR;
+        fprintf(stderr,"ASKING failed: %s\n",reply->str);
+    }
+    freeReplyObject(reply);
+    return result;
+}
+
 static void cliPrintContextError(void) {
     if (context == NULL) return;
     fprintf(stderr,"Error: %s\n",context->errstr);
@@ -931,6 +978,7 @@ static sds cliFormatReplyRaw(redisReply *r) {
     case REDIS_REPLY_DOUBLE:
         out = sdscatprintf(out,"%s",r->str);
         break;
+    case REDIS_REPLY_SET:
     case REDIS_REPLY_ARRAY:
     case REDIS_REPLY_PUSH:
         for (i = 0; i < r->elements; i++) {
@@ -989,6 +1037,7 @@ static sds cliFormatReplyCSV(redisReply *r) {
         out = sdscat(out,r->integer ? "true" : "false");
     break;
     case REDIS_REPLY_ARRAY:
+    case REDIS_REPLY_SET:
     case REDIS_REPLY_PUSH:
     case REDIS_REPLY_MAP: /* CSV has no map type, just output flat list. */
         for (i = 0; i < r->elements; i++) {
@@ -1076,7 +1125,7 @@ static int cliReadReply(int output_raw_strings) {
     /* Check if we need to connect to a different node and reissue the
      * request. */
     if (config.cluster_mode && reply->type == REDIS_REPLY_ERROR &&
-        (!strncmp(reply->str,"MOVED",5) || !strcmp(reply->str,"ASK")))
+        (!strncmp(reply->str,"MOVED ",6) || !strncmp(reply->str,"ASK ",4)))
     {
         char *p = reply->str, *s;
         int slot;
@@ -1100,6 +1149,9 @@ static int cliReadReply(int output_raw_strings) {
             printf("-> Redirected to slot [%d] located at %s:%d\n",
                 slot, config.hostip, config.hostport);
         config.cluster_reissue_command = 1;
+        if (!strncmp(reply->str,"ASK ",4)) {
+            config.cluster_send_asking = 1;
+        }
         cliRefreshPrompt();
     } else if (!config.interactive && config.set_errcode && 
         reply->type == REDIS_REPLY_ERROR) 
@@ -1112,6 +1164,7 @@ static int cliReadReply(int output_raw_strings) {
     if (output) {
         out = cliFormatReply(reply, config.output, output_raw_strings);
         fwrite(out,sdslen(out),1,stdout);
+        fflush(stdout);
         sdsfree(out);
     }
     freeReplyObject(reply);
@@ -1233,7 +1286,7 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
             if (!strcasecmp(command,"select") && argc == 2 && 
                 config.last_cmd_type != REDIS_REPLY_ERROR) 
             {
-                config.dbnum = atoi(argv[1]);
+                config.input_dbnum = config.dbnum = atoi(argv[1]);
                 cliRefreshPrompt();
             } else if (!strcasecmp(command,"auth") && (argc == 2 || argc == 3)) {
                 cliSelect();
@@ -1245,21 +1298,23 @@ static int cliSendCommand(int argc, char **argv, long repeat) {
                 cliRefreshPrompt();
             } else if (!strcasecmp(command,"exec") && argc == 1 && config.in_multi) {
                 config.in_multi = 0;
-                if (config.last_cmd_type == REDIS_REPLY_ERROR) {
-                    config.dbnum = config.pre_multi_dbnum;
+                if (config.last_cmd_type == REDIS_REPLY_ERROR ||
+                    config.last_cmd_type == REDIS_REPLY_NIL)
+                {
+                    config.input_dbnum = config.dbnum = config.pre_multi_dbnum;
                 }
                 cliRefreshPrompt();
             } else if (!strcasecmp(command,"discard") && argc == 1 && 
                 config.last_cmd_type != REDIS_REPLY_ERROR) 
             {
                 config.in_multi = 0;
-                config.dbnum = config.pre_multi_dbnum;
+                config.input_dbnum = config.dbnum = config.pre_multi_dbnum;
                 cliRefreshPrompt();
             } 
         }
         if (config.cluster_reissue_command){
             /* If we need to reissue the command, break to prevent a
-               further 'repeat' number of dud interations */
+               further 'repeat' number of dud interactions */
             break;
         }
         if (config.interval) usleep(config.interval);
@@ -1340,7 +1395,7 @@ static int parseOptions(int argc, char **argv) {
             double seconds = atof(argv[++i]);
             config.interval = seconds*1000000;
         } else if (!strcmp(argv[i],"-n") && !lastarg) {
-            config.dbnum = atoi(argv[++i]);
+            config.input_dbnum = atoi(argv[++i]);
         } else if (!strcmp(argv[i], "--no-auth-warning")) {
             config.no_auth_warning = 1;
         } else if (!strcmp(argv[i], "--askpass")) {
@@ -1580,6 +1635,11 @@ static int parseOptions(int argc, char **argv) {
         }
     }
 
+    if (config.hostsocket && config.cluster_mode) {
+        fprintf(stderr,"Options -c and -s are mutually exclusive.\n");
+        exit(1);
+    }
+
     /* --ldb requires --eval. */
     if (config.eval_ldb && config.eval == NULL) {
         fprintf(stderr,"Options --ldb and --ldb-sync-mode require --eval.\n");
@@ -1698,6 +1758,7 @@ static void usage(void) {
 "  --lru-test <keys>  Simulate a cache workload with an 80-20 distribution.\n"
 "  --replica          Simulate a replica showing commands received from the master.\n"
 "  --rdb <filename>   Transfer an RDB dump from remote server to local file.\n"
+"                     Use filename of \"-\" to write to stdout.\n"
 "  --pipe             Transfer raw KeyDB protocol from stdin to server.\n"
 "  --pipe-timeout <n> In --pipe mode, abort with error if after sending all data.\n"
 "                     no reply is received within <n> seconds.\n"
@@ -1796,23 +1857,32 @@ static sds *getSdsArrayFromArgv(int argc, char **argv, int quoted) {
 
 static int issueCommandRepeat(int argc, char **argv, long repeat) {
     while (1) {
+        if (config.cluster_reissue_command || context == NULL ||
+            context->err == REDIS_ERR_IO || context->err == REDIS_ERR_EOF)
+        {
+            if (cliConnect(CC_FORCE) != REDIS_OK) {
+                cliPrintContextError();
+                config.cluster_reissue_command = 0;
+                return REDIS_ERR;
+            }
+        }
         config.cluster_reissue_command = 0;
-        if (cliSendCommand(argc,argv,repeat) != REDIS_OK) {
-            cliConnect(CC_FORCE);
-
-            /* If we still cannot send the command print error.
-             * We'll try to reconnect the next time. */
-            if (cliSendCommand(argc,argv,repeat) != REDIS_OK) {
+        if (config.cluster_send_asking) {
+            if (cliSendAsking() != REDIS_OK) {
                 cliPrintContextError();
                 return REDIS_ERR;
             }
         }
+        if (cliSendCommand(argc,argv,repeat) != REDIS_OK) {
+            cliPrintContextError();
+            return REDIS_ERR;
+        }
+
         /* Issue the command again if we got redirected in cluster mode */
         if (config.cluster_mode && config.cluster_reissue_command) {
-            cliConnect(CC_FORCE);
-        } else {
-            break;
+            continue;
         }
+        break;
     }
     return REDIS_OK;
 }
@@ -4554,7 +4624,7 @@ static void clusterManagerNodeArrayReset(clusterManagerNodeArray *array) {
 static void clusterManagerNodeArrayShift(clusterManagerNodeArray *array,
                                          clusterManagerNode **nodeptr)
 {
-    assert(array->nodes < (array->nodes + array->len));
+    assert(array->len > 0);
     /* If the first node to be shifted is not NULL, decrement count. */
     if (*array->nodes != NULL) array->count--;
     /* Store the first node to be shifted into 'nodeptr'. */
@@ -4567,7 +4637,7 @@ static void clusterManagerNodeArrayShift(clusterManagerNodeArray *array,
 static void clusterManagerNodeArrayAdd(clusterManagerNodeArray *array,
                                        clusterManagerNode *node)
 {
-    assert(array->nodes < (array->nodes + array->len));
+    assert(array->len > 0);
     assert(node != NULL);
     assert(array->count < array->len);
     array->nodes[array->count++] = node;
@@ -5607,9 +5677,9 @@ static int clusterManagerCommandImport(int argc, char **argv) {
     }
 
     if (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_COPY)
-        strcat(cmdfmt, " %s");
+        cmdfmt = sdscat(cmdfmt," COPY");
     if (config.cluster_manager_command.flags & CLUSTER_MANAGER_CMD_FLAG_REPLACE)
-        strcat(cmdfmt, " %s");
+        cmdfmt = sdscat(cmdfmt," REPLACE");
 
     /* Use SCAN to iterate over the keys, migrating to the
      * right node as needed. */
@@ -5641,8 +5711,7 @@ static int clusterManagerCommandImport(int argc, char **argv) {
             printf("Migrating %s to %s:%d: ", key, target->ip, target->port);
             redisReply *r = reconnectingRedisCommand(src_ctx, cmdfmt,
                                                      target->ip, target->port,
-                                                     key, 0, timeout,
-                                                     "COPY", "REPLACE");
+                                                     key, 0, timeout);
             if (!r || r->type == REDIS_REPLY_ERROR) {
                 if (r && r->str) {
                     clusterManagerLogErr("Source %s:%d replied with "
@@ -5944,7 +6013,7 @@ void showLatencyDistSamples(struct distsamples *samples, long long tot) {
     printf("\033[38;5;0m"); /* Set foreground color to black. */
     for (j = 0; ; j++) {
         int coloridx =
-            ceil((float) samples[j].count / tot * (spectrum_palette_size-1));
+            ceil((double) samples[j].count / tot * (spectrum_palette_size-1));
         int color = spectrum_palette[coloridx];
         printf("\033[48;5;%dm%c", (int)color, samples[j].character);
         samples[j].count = 0;
@@ -6058,7 +6127,7 @@ static void latencyDistMode(void) {
 #define RDB_EOF_MARK_SIZE 40
 
 void sendReplconf(const char* arg1, const char* arg2) {
-    printf("sending REPLCONF %s %s\n", arg1, arg2);
+    fprintf(stderr, "sending REPLCONF %s %s\n", arg1, arg2);
     redisReply *reply = redisCommand(context, "REPLCONF %s %s", arg1, arg2);
 
     /* Handle any error conditions */
@@ -6118,7 +6187,7 @@ unsigned long long sendSync(redisContext *c, char *out_eof) {
     }
     *p = '\0';
     if (buf[0] == '-') {
-        printf("SYNC with master failed: %s\n", buf);
+        fprintf(stderr, "SYNC with master failed: %s\n", buf);
         exit(1);
     }
     if (strncmp(buf+1,"EOF:",4) == 0 && strlen(buf+5) >= RDB_EOF_MARK_SIZE) {
@@ -6223,8 +6292,9 @@ static void getRDB(clusterManagerNode *node) {
             payload, filename);
     }
 
+    int write_to_stdout = !strcmp(filename,"-");
     /* Write to file. */
-    if (!strcmp(filename,"-")) {
+    if (write_to_stdout) {
         fd = STDOUT_FILENO;
     } else {
         fd = open(filename, O_CREAT|O_WRONLY, 0644);
@@ -6266,7 +6336,7 @@ static void getRDB(clusterManagerNode *node) {
     }
     if (usemark) {
         payload = ULLONG_MAX - payload - RDB_EOF_MARK_SIZE;
-        if (ftruncate(fd, payload) == -1)
+        if (!write_to_stdout && ftruncate(fd, payload) == -1)
             fprintf(stderr,"ftruncate failed: %s.\n", strerror(errno));
         fprintf(stderr,"Transfer finished with success after %llu bytes\n", payload);
     } else {
@@ -6275,7 +6345,7 @@ static void getRDB(clusterManagerNode *node) {
     redisFree(s); /* Close the connection ASAP as fsync() may take time. */
     if (node)
         node->context = NULL;
-    if (fsync(fd) == -1) {
+    if (!write_to_stdout && fsync(fd) == -1) {
         fprintf(stderr,"Fail to fsync '%s': %s\n", filename, strerror(errno));
         exit(1);
     }
@@ -7073,6 +7143,7 @@ int main(int argc, char **argv) {
     config.repeat = 1;
     config.interval = 0;
     config.dbnum = 0;
+    config.input_dbnum = 0;
     config.interactive = 0;
     config.shutdown = 0;
     config.monitor_mode = 0;
@@ -7083,6 +7154,7 @@ int main(int argc, char **argv) {
     config.lru_test_mode = 0;
     config.lru_test_sample_size = 0;
     config.cluster_mode = 0;
+    config.cluster_send_asking = 0;
     config.slave_mode = 0;
     config.getrdb_mode = 0;
     config.stat_mode = 0;

src/redis-cli.h

@@ -146,7 +146,8 @@ extern struct config {
     char *key;
     long repeat;
     long interval;
-    int dbnum;
+    int dbnum; /* db num currently selected */
+    int input_dbnum; /* db num user input */
     int interactive;
     int shutdown;
     int monitor_mode;
@@ -158,6 +159,7 @@ extern struct config {
     long long lru_test_sample_size;
     int cluster_mode;
     int cluster_reissue_command;
+    int cluster_send_asking;
     int slave_mode;
     int pipe_mode;
     int pipe_timeout;

src/redismodule.h

@@ -840,6 +840,7 @@ REDISMODULE_API int (*RedisModule_AuthenticateClientWithUser)(RedisModuleCtx *ct
 REDISMODULE_API int (*RedisModule_DeauthenticateAndCloseClient)(RedisModuleCtx *ctx, uint64_t client_id) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleString * (*RedisModule_GetClientCertificate)(RedisModuleCtx *ctx, uint64_t id) REDISMODULE_ATTR;
 REDISMODULE_API int *(*RedisModule_GetCommandKeys)(RedisModuleCtx *ctx, RedisModuleString **argv, int argc, int *num_keys) REDISMODULE_ATTR;
+REDISMODULE_API const char *(*RedisModule_GetCurrentCommandName)(RedisModuleCtx *ctx) REDISMODULE_ATTR;
 REDISMODULE_API int (*RedisModule_RegisterDefragFunc)(RedisModuleCtx *ctx, RedisModuleDefragFunc func) REDISMODULE_ATTR;
 REDISMODULE_API void *(*RedisModule_DefragAlloc)(RedisModuleDefragCtx *ctx, void *ptr) REDISMODULE_ATTR;
 REDISMODULE_API RedisModuleString *(*RedisModule_DefragRedisModuleString)(RedisModuleDefragCtx *ctx, RedisModuleString *str) REDISMODULE_ATTR;
@@ -1112,6 +1113,7 @@ static int RedisModule_Init(RedisModuleCtx *ctx, const char *name, int ver, int
     REDISMODULE_GET_API(AuthenticateClientWithUser);
     REDISMODULE_GET_API(GetClientCertificate);
     REDISMODULE_GET_API(GetCommandKeys);
+    REDISMODULE_GET_API(GetCurrentCommandName);
     REDISMODULE_GET_API(RegisterDefragFunc);
     REDISMODULE_GET_API(DefragAlloc);
     REDISMODULE_GET_API(DefragRedisModuleString);

src/replication.cpp

@@ -609,6 +609,7 @@ void replicationFeedSlavesFromMasterStream(char *buf, size_t buflen) {
 }
 
 void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv, int argc) {
+    if (!(listLength(g_pserver->monitors) && !g_pserver->loading)) return;
     listNode *ln;
     listIter li;
     int j;
@@ -1986,7 +1987,7 @@ void readSyncBulkPayload(connection *conn) {
     ssize_t nread, readlen, nwritten;
     int use_diskless_load = useDisklessLoad();
     dbBackup *diskless_load_backup = NULL;
-    rdbSaveInfo rsi = RDB_SAVE_INFO_INIT;
+    rdbSaveInfo rsi;
     int empty_db_flags = g_pserver->repl_slave_lazy_flush ? EMPTYDB_ASYNC :
                                                         EMPTYDB_NO_FLAGS;
     off_t left;
@@ -2307,7 +2308,7 @@ void readSyncBulkPayload(connection *conn) {
                 mi->staleKeyMap->clear();
             else
                 mi->staleKeyMap = new (MALLOC_LOCAL) std::map<int, std::vector<robj_sharedptr>>();
-            rsi.mi = mi;
+            rsi.addMaster(*mi);
         }
         if (rdbLoadFile(rdb_filename,&rsi,RDBFLAGS_REPLICATION) != C_OK) {
             serverLog(LL_WARNING,
@@ -2358,7 +2359,7 @@ void readSyncBulkPayload(connection *conn) {
     {
         mergeReplicationId(mi->master->replid);
     }
-    else
+    else if (!g_pserver->fActiveReplica)
     {
         /* After a full resynchroniziation we use the replication ID and
         * offset of the master. The secondary ID / offset are cleared since
@@ -2554,7 +2555,7 @@ int slaveTryPartialResynchronization(redisMaster *mi, connection *conn, int read
          * client structure representing the master into g_pserver->master. */
         mi->master_initial_offset = -1;
 
-        if (mi->cached_master && !g_pserver->fActiveReplica) {
+        if (mi->cached_master) {
             psync_replid = mi->cached_master->replid;
             snprintf(psync_offset,sizeof(psync_offset),"%lld", mi->cached_master->reploff+1);
             serverLog(LL_NOTICE,"Trying a partial resynchronization (request %s:%s).", psync_replid, psync_offset);
@@ -2653,14 +2654,15 @@ int slaveTryPartialResynchronization(redisMaster *mi, connection *conn, int read
                     sizeof(g_pserver->replid2));
                 g_pserver->second_replid_offset = g_pserver->master_repl_offset+1;
 
-                /* Update the cached master ID and our own primary ID to the
-                 * new one. */
-                memcpy(g_pserver->replid,sznew,sizeof(g_pserver->replid));
-                memcpy(mi->cached_master->replid,sznew,sizeof(g_pserver->replid));
+                if (!g_pserver->fActiveReplica) {
+                    /* Update the cached master ID and our own primary ID to the
+                     * new one. */
+                    memcpy(g_pserver->replid,sznew,sizeof(g_pserver->replid));
+                    memcpy(mi->cached_master->replid,sznew,sizeof(g_pserver->replid));
 
-                /* Disconnect all the sub-slaves: they need to be notified. */
-                if (!g_pserver->fActiveReplica)
+                    /* Disconnect all the sub-slaves: they need to be notified. */
                     disconnectSlaves();
+                }
             }
         }
 
@@ -2759,18 +2761,19 @@ void syncWithMaster(connection *conn) {
         goto error;
     }
 
+retry_connect:
     /* Send a PING to check the master is able to reply without errors. */
-    if (mi->repl_state == REPL_STATE_CONNECTING) {
+    if (mi->repl_state == REPL_STATE_CONNECTING || mi->repl_state == REPL_STATE_RETRY_NOREPLPING) {
         serverLog(LL_NOTICE,"Non blocking connect for SYNC fired the event.");
         /* Delete the writable event so that the readable event remains
          * registered and we can wait for the PONG reply. */
         connSetReadHandler(conn, syncWithMaster);
         connSetWriteHandler(conn, NULL);
-        mi->repl_state = REPL_STATE_RECEIVE_PING_REPLY;
         /* Send the PING, don't check for errors at all, we have the timeout
          * that will take care about this. */
-        err = sendCommand(conn,"PING",NULL);
+        err = sendCommand(conn,mi->repl_state == REPL_STATE_RETRY_NOREPLPING ? "PING" : "REPLPING",NULL);
         if (err) goto write_error;
+        mi->repl_state = REPL_STATE_RECEIVE_PING_REPLY;
         return;
     }
 
@@ -2783,7 +2786,13 @@ void syncWithMaster(connection *conn) {
          * Note that older versions of Redis replied with "operation not
          * permitted" instead of using a proper error code, so we test
          * both. */
-        if (err[0] != '+' &&
+        if (strncmp(err,"-ERR unknown command",20) == 0) {
+            serverLog(LL_NOTICE,"Master does not support REPLPING, sending PING instead...");
+            mi->repl_state = REPL_STATE_RETRY_NOREPLPING;
+            sdsfree(err);
+            err = NULL;
+            goto retry_connect;
+        } else if (err[0] != '+' &&
             strncmp(err,"-NOAUTH",7) != 0 &&
             strncmp(err,"-NOPERM",7) != 0 &&
             strncmp(err,"-ERR operation not permitted",28) != 0)
@@ -3014,18 +3023,6 @@ void syncWithMaster(connection *conn) {
         disconnectSlavesExcept(mi->master_uuid); /* Force our slaves to resync with us as well. */
         freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */
     }
-    else
-    {
-        if (listLength(g_pserver->slaves))
-        {
-            changeReplicationId();
-            clearReplicationId2();
-        }
-        else
-        {
-            freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */
-        }
-    }
 
     /* Fall back to SYNC if needed. Otherwise psync_result == PSYNC_FULLRESYNC
      * and the g_pserver->master_replid and master_initial_offset are
@@ -3671,6 +3668,26 @@ void replicationCacheMasterUsingMyself(redisMaster *mi) {
     mi->master = NULL;
 }
 
+/* This function is called when reloading master info from an RDB in Active Replica mode.
+ * It creates a cached master client using the info contained in the redisMaster struct.
+ *
+ * Assumes that the passed struct contains valid master info. */
+void replicationCacheMasterUsingMaster(redisMaster *mi) {
+    if (mi->cached_master) {
+        freeClient(mi->cached_master);
+    }
+
+    replicationCreateMasterClient(mi, NULL, -1);
+    std::lock_guard<decltype(mi->master->lock)> lock(mi->master->lock);
+
+    memcpy(mi->master->replid, mi->master_replid, sizeof(mi->master_replid));
+    mi->master->reploff = mi->master_initial_offset;
+
+    unlinkClient(mi->master);
+    mi->cached_master = mi->master;
+    mi->master = NULL;
+}
+
 /* Free a cached master, called when there are no longer the conditions for
  * a partial resync on reconnection. */
 void replicationDiscardCachedMaster(redisMaster *mi) {
@@ -4803,7 +4820,7 @@ void replicationNotifyLoadedKey(redisDb *db, robj_roptr key, robj_roptr val, lon
     redisObjectStack objTtl;
     initStaticStringObject(objTtl, sdscatprintf(sdsempty(), "%lld", expire));
     redisObjectStack objMvcc;
-    initStaticStringObject(objMvcc, sdscatprintf(sdsempty(), "%lu", mvccFromObj(val)));
+    initStaticStringObject(objMvcc, sdscatprintf(sdsempty(), "%" PRIu64, mvccFromObj(val)));
     redisObject *argv[5] = {shared.mvccrestore, key.unsafe_robjcast(), &objMvcc, &objTtl, &objPayload};
 
     replicationFeedSlaves(g_pserver->slaves, db - g_pserver->db, argv, 5);

src/rio.h

@@ -63,7 +63,7 @@ struct _rio {
     uint64_t cksum, flags;
 
     /* number of keys loaded since last rdbLoadProgressCallback */
-    long int keys_since_last_callback;
+    unsigned long int keys_since_last_callback;
 
     /* number of bytes read or written */
     size_t processed_bytes;

src/scripting.cpp

@@ -31,6 +31,7 @@
 #include "sha1.h"
 #include "rand.h"
 #include "cluster.h"
+#include "monotonic.h"
 
 extern "C" {
 #include <lua.h>
@@ -131,6 +132,16 @@ void sha1hex(char *digest, char *script, size_t len) {
  */
 
 char *redisProtocolToLuaType(lua_State *lua, char* reply) {
+
+    if (!lua_checkstack(lua, 5)) {
+        /*
+         * Increase the Lua stack if needed, to make sure there is enough room
+         * to push 5 elements to the stack. On failure, exit with panic.
+         * Notice that we need, in the worst case, 5 elements because redisProtocolToLuaType_Aggregate
+         * might push 5 elements to the Lua stack.*/
+        serverPanic("lua stack limit reach when parsing redis.call reply");
+    }
+
     char *p = reply;
 
     switch(*p) {
@@ -223,6 +234,11 @@ char *redisProtocolToLuaType_Aggregate(lua_State *lua, char *reply, int atype) {
             if (atype == '%') {
                 p = redisProtocolToLuaType(lua,p);
             } else {
+                if (!lua_checkstack(lua, 1)) {
+                    /* Notice that here we need to check the stack again because the recursive
+                     * call to redisProtocolToLuaType might have use the room allocated in the stack */
+                    serverPanic("lua stack limit reach when parsing redis.call reply");
+                }
                 lua_pushboolean(lua,1);
             }
             lua_settable(lua,-3);
@@ -342,6 +358,17 @@ void luaSortArray(lua_State *lua) {
 /* Reply to client 'c' converting the top element in the Lua stack to a
  * Redis reply. As a side effect the element is consumed from the stack.  */
 void luaReplyToRedisReply(client *c, lua_State *lua) {
+
+    if (!lua_checkstack(lua, 4)) {
+        /* Increase the Lua stack if needed to make sure there is enough room
+         * to push 4 elements to the stack. On failure, return error.
+         * Notice that we need, in the worst case, 4 elements because returning a map might
+         * require push 4 elements to the Lua stack.*/
+        addReplyErrorFormat(c, "reached lua stack limit");
+        lua_pop(lua,1); /* pop the element from the stack */
+        return;
+    }
+
     int t = lua_type(lua,-1);
 
     switch(t) {
@@ -365,6 +392,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
          * field. */
 
         /* Handle error reply. */
+        /* we took care of the stack size on function start */
         lua_pushstring(lua,"err");
         lua_gettable(lua,-2);
         t = lua_type(lua,-1);
@@ -407,6 +435,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
         if (t == LUA_TTABLE) {
             int maplen = 0;
             void *replylen = addReplyDeferredLen(c);
+            /* we took care of the stack size on function start */
             lua_pushnil(lua); /* Use nil to start iteration. */
             while (lua_next(lua,-2)) {
                 /* Stack now: table, key, value */
@@ -429,6 +458,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
         if (t == LUA_TTABLE) {
             int setlen = 0;
             void *replylen = addReplyDeferredLen(c);
+            /* we took care of the stack size on function start */
             lua_pushnil(lua); /* Use nil to start iteration. */
             while (lua_next(lua,-2)) {
                 /* Stack now: table, key, true */
@@ -449,6 +479,7 @@ void luaReplyToRedisReply(client *c, lua_State *lua) {
         void *replylen = addReplyDeferredLen(c);
         int j = 1, mbulklen = 0;
         while(1) {
+            /* we took care of the stack size on function start */
             lua_pushnumber(lua,j++);
             lua_gettable(lua,-2);
             t = lua_type(lua,-1);
@@ -1437,7 +1468,7 @@ sds luaCreateFunction(client *c, lua_State *lua, robj *body) {
 
 /* This is the Lua script "count" hook that we use to detect scripts timeout. */
 void luaMaskCountHook(lua_State *lua, lua_Debug *ar) {
-    long long elapsed = mstime() - g_pserver->lua_time_start;
+    long long elapsed = elapsedMs(g_pserver->lua_time_start);
     UNUSED(ar);
     UNUSED(lua);
 
@@ -1588,7 +1619,8 @@ void evalGenericCommand(client *c, int evalsha) {
     serverTL->in_eval = 1;
     g_pserver->lua_caller = c;
     g_pserver->lua_cur_script = funcname + 2;
-    g_pserver->lua_time_start = mstime();
+    g_pserver->lua_time_start = getMonotonicUs();
+    g_pserver->lua_time_snapshot = mstime();
     g_pserver->lua_kill = 0;
     if (g_pserver->lua_time_limit > 0 && ldb.active == 0) {
         lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000);
@@ -1705,6 +1737,9 @@ void evalGenericCommand(client *c, int evalsha) {
 }
 
 void evalCommand(client *c) {
+    /* Explicitly feed monitor here so that lua commands appear after their
+     * script command. */
+    replicationFeedMonitors(c,g_pserver->monitors,c->db->id,c->argv,c->argc);
     if (!(c->flags & CLIENT_LUA_DEBUG))
         evalGenericCommand(c,0);
     else
@@ -1712,6 +1747,9 @@ void evalCommand(client *c) {
 }
 
 void evalShaCommand(client *c) {
+    /* Explicitly feed monitor here so that lua commands appear after their
+     * script command. */
+    replicationFeedMonitors(c,g_pserver->monitors,c->db->id,c->argv,c->argc);
     if (sdslen((sds)ptrFromObj(c->argv[1])) != 40) {
         /* We know that a match is not possible if the provided SHA is
          * not the right length. So we return an error ASAP, this way
@@ -2090,7 +2128,8 @@ int ldbDelBreakpoint(int line) {
 /* Expect a valid multi-bulk command in the debugging client query buffer.
  * On success the command is parsed and returned as an array of SDS strings,
  * otherwise NULL is returned and there is to read more buffer. */
-sds *ldbReplParseCommand(int *argcp) {
+sds *ldbReplParseCommand(int *argcp, char** err) {
+    static sds protocol_error = sdsnew("protocol error");
     sds *argv = NULL;
     int argc = 0;
     char *plen = NULL;
@@ -2108,7 +2147,7 @@ sds *ldbReplParseCommand(int *argcp) {
     /* Seek and parse *<count>\r\n. */
     p = strchr(p,'*'); if (!p) goto protoerr;
     plen = p+1; /* Multi bulk len pointer. */
-    p = strstr(p,"\r\n"); if (!p) goto protoerr;
+    p = strstr(p,"\r\n"); if (!p) goto keep_reading;
     *p = '\0'; p += 2;
     *argcp = atoi(plen);
     if (*argcp <= 0 || *argcp > 1024) goto protoerr;
@@ -2117,12 +2156,16 @@ sds *ldbReplParseCommand(int *argcp) {
     argv = (sds*)zmalloc(sizeof(sds)*(*argcp), MALLOC_LOCAL);
     argc = 0;
     while(argc < *argcp) {
+        /* reached the end but there should be more data to read */
+        if (*p == '\0') goto keep_reading;
+
         if (*p != '$') goto protoerr;
         plen = p+1; /* Bulk string len pointer. */
-        p = strstr(p,"\r\n"); if (!p) goto protoerr;
+        p = strstr(p,"\r\n"); if (!p) goto keep_reading;
         *p = '\0'; p += 2;
         int slen = atoi(plen); /* Length of this arg. */
         if (slen <= 0 || slen > 1024) goto protoerr;
+        if ((size_t)(p + slen + 2 - copy) > sdslen(copy) ) goto keep_reading;
         argv[argc++] = sdsnewlen(p,slen);
         p += slen; /* Skip the already parsed argument. */
         if (p[0] != '\r' || p[1] != '\n') goto protoerr;
@@ -2132,6 +2175,8 @@ sds *ldbReplParseCommand(int *argcp) {
     return argv;
 
 protoerr:
+    *err = protocol_error;
+keep_reading:
     sdsfreesplitres(argv,argc);
     sdsfree(copy);
     return NULL;
@@ -2564,6 +2609,19 @@ void ldbEval(lua_State *lua, sds *argv, int argc) {
 void ldbRedis(lua_State *lua, sds *argv, int argc) {
     int j, saved_rc = g_pserver->lua_replicate_commands;
 
+    if (!lua_checkstack(lua, argc + 1)) {
+        /* Increase the Lua stack if needed to make sure there is enough room
+         * to push 'argc + 1' elements to the stack. On failure, return error.
+         * Notice that we need, in worst case, 'argc + 1' elements because we push all the arguments
+         * given by the user (without the first argument) and we also push the 'redis' global table and
+         * 'redis.call' function so:
+         * (1 (redis table)) + (1 (redis.call function)) + (argc - 1 (all arguments without the first)) = argc + 1*/
+        sds reply = sdsnew("max lua stack reached");
+        ldbLogRedisReply(reply);
+        sdsfree(reply);
+        return;
+    }
+
     lua_getglobal(lua,"redis");
     lua_pushstring(lua,"call");
     lua_gettable(lua,-2);       /* Stack: redis, redis.call */
@@ -2620,12 +2678,17 @@ void ldbMaxlen(sds *argv, int argc) {
 int ldbRepl(lua_State *lua) {
     sds *argv;
     int argc;
+    char* err = NULL;
 
     /* We continue processing commands until a command that should return
      * to the Lua interpreter is found. */
     while(1) {
-        while((argv = ldbReplParseCommand(&argc)) == NULL) {
+        while((argv = ldbReplParseCommand(&argc, &err)) == NULL) {
             char buf[1024];
+            if (err) {
+                lua_pushstring(lua, err);
+                lua_error(lua);
+            }
             int nread = connRead(ldb.conn,buf,sizeof(buf));
             if (nread <= 0) {
                 /* Make sure the script runs without user input since the
@@ -2635,6 +2698,15 @@ int ldbRepl(lua_State *lua) {
                 return C_ERR;
             }
             ldb.cbuf = sdscatlen(ldb.cbuf,buf,nread);
+            /* after 1M we will exit with an error
+             * so that the client will not blow the memory
+             */
+            if (sdslen(ldb.cbuf) > 1<<20) {
+                sdsfree(ldb.cbuf);
+                ldb.cbuf = sdsempty();
+                lua_pushstring(lua, "max client buffer reached");
+                lua_error(lua);
+            }
         }
 
         /* Flush the old buffer. */
@@ -2747,7 +2819,7 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) {
 
     /* Check if a timeout occurred. */
     if (ar->event == LUA_HOOKCOUNT && ldb.step == 0 && bp == 0) {
-        mstime_t elapsed = mstime() - g_pserver->lua_time_start;
+        mstime_t elapsed = elapsedMs(g_pserver->lua_time_start);
         mstime_t timelimit = g_pserver->lua_time_limit ?
                              g_pserver->lua_time_limit : 5000;
         if (elapsed >= timelimit) {
@@ -2777,6 +2849,7 @@ void luaLdbLineHook(lua_State *lua, lua_Debug *ar) {
             lua_pushstring(lua, "timeout during Lua debugging with client closing connection");
             lua_error(lua);
         }
-        g_pserver->lua_time_start = mstime();
+        g_pserver->lua_time_start = getMonotonicUs();
+        g_pserver->lua_time_snapshot = mstime();
     }
 }

src/sds.c

@@ -265,7 +265,7 @@ void sdsclear(sds s) {
 sds sdsMakeRoomFor(sds s, size_t addlen) {
     void *sh, *newsh;
     size_t avail = sdsavail(s);
-    size_t len, newlen;
+    size_t len, newlen, reqlen;
     char type, oldtype = s[-1] & SDS_TYPE_MASK;
     int hdrlen;
     size_t usable;
@@ -275,7 +275,7 @@ sds sdsMakeRoomFor(sds s, size_t addlen) {
 
     len = sdslen(s);
     sh = (char*)s-sdsHdrSize(oldtype);
-    newlen = (len+addlen);
+    reqlen = newlen = (len+addlen);
     assert(newlen > len);   /* Catch size_t overflow */
     if (newlen < SDS_MAX_PREALLOC)
         newlen *= 2;
@@ -290,10 +290,8 @@ sds sdsMakeRoomFor(sds s, size_t addlen) {
     if (type == SDS_TYPE_5) type = SDS_TYPE_8;
 
     hdrlen = sdsHdrSize(type);
-    assert(hdrlen + newlen + 1 > len);  /* Catch size_t overflow */
-    if (oldtype==type && (len+1024) >= avail) {
-        // note: if we have a lot of free space don't use this as we don't want s_realloc copying
-        //  uninitialized data
+    assert(hdrlen + newlen + 1 > reqlen);  /* Catch size_t overflow */
+    if (oldtype==type) {
         newsh = s_realloc_usable(sh, hdrlen+newlen+1, &usable);
         if (newsh == NULL) return NULL;
         s = (char*)newsh+hdrlen;
@@ -798,6 +796,21 @@ sds sdstrim(sds s, const char *cset) {
     return s;
 }
 
+/* Changes the input string to be a subset of the original.
+ * It does not release the free space in the string, so a call to
+ * sdsRemoveFreeSpace may be wise after. */
+void sdssubstr(sds s, size_t start, size_t len) {
+    /* Clamp out of range input */
+    size_t oldlen = sdslen(s);
+    if (start >= oldlen) start = len = 0;
+    if (len > oldlen-start) len = oldlen-start;
+
+    /* Move the data */
+    if (len) memmove(s, s+start, len);
+    s[len] = 0;
+    sdssetlen(s,len);
+}
+
 /* Turn the string into a smaller (or equal) string containing only the
  * substring specified by the 'start' and 'end' indexes.
  *
@@ -809,6 +822,11 @@ sds sdstrim(sds s, const char *cset) {
  *
  * The string is modified in-place.
  *
+ * NOTE: this function can be misleading and can have unexpected behaviour,
+ * specifically when you want the length of the new string to be 0.
+ * Having start==end will result in a string with one character.
+ * please consider using sdssubstr instead.
+ *
  * Example:
  *
  * s = sdsnew("Hello World");
@@ -816,28 +834,13 @@ sds sdstrim(sds s, const char *cset) {
  */
 void sdsrange(sds s, ssize_t start, ssize_t end) {
     size_t newlen, len = sdslen(s);
-
     if (len == 0) return;
-    if (start < 0) {
-        start = len+start;
-        if (start < 0) start = 0;
-    }
-    if (end < 0) {
-        end = len+end;
-        if (end < 0) end = 0;
-    }
+    if (start < 0)
+        start = len + start;
+    if (end < 0)
+        end = len + end;
     newlen = (start > end) ? 0 : (end-start)+1;
-    if (newlen != 0) {
-        if (start >= (ssize_t)len) {
-            newlen = 0;
-        } else if (end >= (ssize_t)len) {
-            end = len-1;
-            newlen = (start > end) ? 0 : (end-start)+1;
-        }
-    }
-    if (start && newlen) memmove(s, s+start, newlen);
-    s[newlen] = 0;
-    sdssetlen(s,newlen);
+    sdssubstr(s, start, newlen);
 }
 
 /* Apply tolower() to every character of the sds string 's'. */
@@ -1408,6 +1411,18 @@ int sdsTest(int argc, char **argv, int accurate) {
         test_cond("sdsrange(...,100,100)",
             sdslen(y) == 0 && memcmp(y,"\0",1) == 0);
 
+        sdsfree(y);
+        y = sdsdup(x);
+        sdsrange(y,4,6);
+        test_cond("sdsrange(...,4,6)",
+            sdslen(y) == 0 && memcmp(y,"\0",1) == 0);
+
+        sdsfree(y);
+        y = sdsdup(x);
+        sdsrange(y,3,6);
+        test_cond("sdsrange(...,3,6)",
+            sdslen(y) == 1 && memcmp(y,"o\0",2) == 0);
+
         sdsfree(y);
         sdsfree(x);
         x = sdsnew("foo");

src/sds.h

@@ -297,6 +297,7 @@ sds sdscatprintf(sds s, const char *fmt, ...);
 
 sds sdscatfmt(sds s, char const *fmt, ...);
 sds sdstrim(sds s, const char *cset);
+void sdssubstr(sds s, size_t start, size_t len);
 void sdsrange(sds s, ssize_t start, ssize_t end);
 void sdsupdatelen(sds s);
 void sdsclear(sds s);
@@ -423,6 +424,21 @@ public:
         return *this;
     }
 
+    sdsstring &operator=(sdsstring &&other)
+    {
+        sds tmp = m_str;
+        m_str = other.m_str;
+        other.m_str = tmp;
+        return *this;
+    }
+
+    template<typename... Args>
+    sdsstring catfmt(const char *fmt, Args... args)
+    {
+        m_str = sdscatfmt(m_str, fmt, args...);
+        return *this;
+    }
+
     sds release() {
         sds sdsT = m_str;
         m_str = nullptr;

src/semiorderedset.h

@@ -247,8 +247,8 @@ public:
         /* Generate human readable stats. */
         l += snprintf(buf+l,bufsize-l,
             "semiordered set stats:\n"
-            " table size: %ld\n"
-            " number of slots: %ld\n"
+            " table size: %zu\n"
+            " number of slots: %zu\n"
             " used slots: %ld\n"
             " max chain length: %ld\n"
             " avg chain length (counted): %.02f\n"

src/sentinel.cpp

@@ -3187,11 +3187,13 @@ void sentinelConfigSetCommand(client *c) {
         sentinel.announce_port = numval;
     } else if (!strcasecmp(szFromObj(o), "sentinel-user")) {
         sdsfree(sentinel.sentinel_auth_user);
-        sentinel.sentinel_auth_user = sdsnew(szFromObj(val));
+        sentinel.sentinel_auth_user = sdslen(szFromObj(val)) == 0 ?
+            NULL : sdsdup(szFromObj(val));
         drop_conns = 1;
     } else if (!strcasecmp(szFromObj(o), "sentinel-pass")) {
         sdsfree(sentinel.sentinel_auth_pass);
-        sentinel.sentinel_auth_pass = sdsnew(szFromObj(val));
+        sentinel.sentinel_auth_pass = sdslen(szFromObj(val)) == 0 ?
+            NULL : sdsdup(szFromObj(val));
         drop_conns = 1;
     } else {
         addReplyErrorFormat(c, "Invalid argument '%s' to SENTINEL CONFIG SET",
@@ -4131,16 +4133,16 @@ void sentinelSetCommand(client *c) {
         int numargs = j-old_j+1;
         switch(numargs) {
         case 2:
-            sentinelEvent(LL_WARNING,"+set",ri,"%@ %s %s",ptrFromObj(c->argv[old_j]),
-                                                          ptrFromObj(c->argv[old_j+1]));
+            sentinelEvent(LL_WARNING,"+set",ri,"%@ %s %s",szFromObj(c->argv[old_j]),
+                                                          szFromObj(c->argv[old_j+1]));
             break;
         case 3:
-            sentinelEvent(LL_WARNING,"+set",ri,"%@ %s %s %s",ptrFromObj(c->argv[old_j]),
-                                                             ptrFromObj(c->argv[old_j+1]),
-                                                             ptrFromObj(c->argv[old_j+2]));
+            sentinelEvent(LL_WARNING,"+set",ri,"%@ %s %s %s",szFromObj(c->argv[old_j]),
+                                                             szFromObj(c->argv[old_j+1]),
+                                                             szFromObj(c->argv[old_j+2]));
             break;
         default:
-            sentinelEvent(LL_WARNING,"+set",ri,"%@ %s",ptrFromObj(c->argv[old_j]));
+            sentinelEvent(LL_WARNING,"+set",ri,"%@ %s",szFromObj(c->argv[old_j]));
             break;
         }
     }

src/server.cpp

@@ -57,17 +57,16 @@
 #include <limits.h>
 #include <float.h>
 #include <math.h>
-#include <sys/resource.h>
 #include <sys/utsname.h>
 #include <locale.h>
 #include <sys/socket.h>
 #include <algorithm>
 #include <uuid/uuid.h>
-#include <mutex>
+#include <condition_variable>
 #include "aelocker.h"
 #include "motd.h"
 #include "t_nhash.h"
-#include <sys/resource.h>
+#include "readwritelock.h"
 #ifdef __linux__
 #include <sys/prctl.h>
 #include <sys/mman.h>
@@ -96,6 +95,10 @@ struct redisServer server; /* Server global state */
 redisServer *g_pserver = &GlobalHidden::server;
 struct redisServerConst cserver;
 __thread struct redisServerThreadVars *serverTL = NULL;   // thread local server vars
+fastlock time_thread_lock("Time thread lock");
+std::condition_variable_any time_thread_cv;
+int sleeping_threads = 0;
+void wakeTimeThread();
 
 /* Our command table.
  *
@@ -745,7 +748,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"auth",authCommand,-2,
-     "no-auth no-script ok-loading ok-stale fast no-monitor no-slowlog @connection",
+     "no-auth no-script ok-loading ok-stale fast @connection",
      0,NULL,0,0,0,0,0,0},
 
     /* We don't allow PING during loading since in Redis PING is used as
@@ -755,6 +758,10 @@ struct redisCommand redisCommandTable[] = {
      "ok-stale ok-loading fast @connection @replication",
      0,NULL,0,0,0,0,0,0},
 
+     {"replping",pingCommand,-1,
+     "ok-stale fast @connection @replication",
+     0,NULL,0,0,0,0,0,0},
+
     {"echo",echoCommand,2,
      "fast @connection",
      0,NULL,0,0,0,0,0,0},
@@ -788,7 +795,7 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"exec",execCommand,1,
-     "no-script no-monitor no-slowlog ok-loading ok-stale @transaction",
+     "no-script no-slowlog ok-loading ok-stale @transaction",
      0,NULL,0,0,0,0,0,0},
 
     {"discard",discardCommand,1,
@@ -909,7 +916,7 @@ struct redisCommand redisCommandTable[] = {
 
     {"migrate",migrateCommand,-6,
      "write random @keyspace @dangerous",
-     0,migrateGetKeys,0,0,0,0,0,0},
+     0,migrateGetKeys,3,3,1,0,0,0},
 
     {"asking",askingCommand,1,
      "fast @keyspace",
@@ -940,17 +947,21 @@ struct redisCommand redisCommandTable[] = {
      0,NULL,0,0,0,0,0,0},
 
     {"hello",helloCommand,-1,
-     "no-auth no-script fast no-monitor ok-loading ok-stale @connection",
+     "no-auth no-script fast ok-loading ok-stale @connection",
      0,NULL,0,0,0,0,0,0},
 
     /* EVAL can modify the dataset, however it is not flagged as a write
-     * command since we do the check while running commands from Lua. */
+     * command since we do the check while running commands from Lua.
+     * 
+     * EVAL and EVALSHA also feed monitors before the commands are executed,
+     * as opposed to after.
+      */
     {"eval",evalCommand,-3,
-     "no-script may-replicate @scripting",
+     "no-script no-monitor may-replicate @scripting",
      0,evalGetKeys,0,0,0,0,0,0},
 
     {"evalsha",evalShaCommand,-3,
-     "no-script may-replicate @scripting",
+     "no-script no-monitor may-replicate @scripting",
      0,evalGetKeys,0,0,0,0,0,0},
 
     {"slowlog",slowlogCommand,-2,
@@ -1915,6 +1926,7 @@ void clientsCron(int iel) {
             if (clientsCronResizeQueryBuffer(c)) goto LContinue;
             if (clientsCronTrackExpansiveClients(c, curr_peak_mem_usage_slot)) goto LContinue;
             if (clientsCronTrackClientsMemUsage(c)) goto LContinue;
+            if (closeClientOnOutputBufferLimitReached(c, 0)) continue; // Client also free'd
         LContinue:
             fastlock_unlock(&c->lock);
         }        
@@ -2000,7 +2012,7 @@ void databasesCron(bool fMainThread) {
  * info or not using the 'update_daylight_info' argument. Normally we update
  * such info only when calling this function from serverCron() but not when
  * calling it from call(). */
-void updateCachedTime(int update_daylight_info) {
+void updateCachedTime() {
     long long t = ustime();
     __atomic_store(&g_pserver->ustime, &t, __ATOMIC_RELAXED);
     t /= 1000;
@@ -2013,12 +2025,10 @@ void updateCachedTime(int update_daylight_info) {
      * context is safe since we will never fork() while here, in the main
      * thread. The logging function will call a thread safe version of
      * localtime that has no locks. */
-    if (update_daylight_info) {
-        struct tm tm;
-        time_t ut = g_pserver->unixtime;
-        localtime_r(&ut,&tm);
-        __atomic_store(&g_pserver->daylight_active, &tm.tm_isdst, __ATOMIC_RELAXED);
-    }
+    struct tm tm;
+    time_t ut = g_pserver->unixtime;
+    localtime_r(&ut,&tm);
+    __atomic_store(&g_pserver->daylight_active, &tm.tm_isdst, __ATOMIC_RELAXED);
 }
 
 void checkChildrenDone(void) {
@@ -2145,9 +2155,6 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
      * handler if we don't return here fast enough. */
     if (g_pserver->watchdog_period) watchdogScheduleSignal(g_pserver->watchdog_period);
 
-    /* Update the time cache. */
-    updateCachedTime(1);
-
     g_pserver->hz = g_pserver->config_hz;
     /* Adapt the g_pserver->hz value to the number of configured clients. If we have
      * many clients, we want to call serverCron() with an higher frequency. */
@@ -2405,7 +2412,6 @@ int serverCronLite(struct aeEventLoop *eventLoop, long long id, void *clientData
 
 void blockingOperationStarts() {
     if(!g_pserver->blocking_op_nesting++){
-        updateCachedTime(0);
         g_pserver->blocked_last_cron = g_pserver->mstime;
     }
 }
@@ -2616,6 +2622,15 @@ void beforeSleep(struct aeEventLoop *eventLoop) {
     locker.disarm();
     if (!fSentReplies)
         handleClientsWithPendingWrites(iel, aof_state);
+
+    aeThreadOffline();
+    // Scope lock_guard
+    {
+        std::unique_lock<fastlock> lock(time_thread_lock);
+        sleeping_threads++;
+        serverAssert(sleeping_threads <= cserver.cthreads);
+    }
+    
     /* Determine whether the modules are enabled before sleeping, and use that result
        both here, and after wakeup to avoid double acquire or release of the GIL */
     serverTL->modulesEnabledThisAeLoop = !!moduleCount();
@@ -2636,6 +2651,8 @@ void afterSleep(struct aeEventLoop *eventLoop) {
        Otherwise you may double acquire the GIL and cause deadlocks in the module */
     if (!ProcessingEventsWhileBlocked) {
         if (serverTL->modulesEnabledThisAeLoop) moduleAcquireGIL(TRUE /*fServerThread*/);
+        aeThreadOnline();
+        wakeTimeThread();
     }
 }
 
@@ -2770,6 +2787,7 @@ void createSharedObjects(void) {
     shared.lastid = makeObjectShared("LASTID",6);
     shared.default_username = makeObjectShared("default",7);
     shared.ping = makeObjectShared("ping",4);
+    shared.replping = makeObjectShared("replping", 8);
     shared.setid = makeObjectShared("SETID",5);
     shared.keepttl = makeObjectShared("KEEPTTL",7);
     shared.load = makeObjectShared("LOAD",4);
@@ -2777,6 +2795,7 @@ void createSharedObjects(void) {
     shared.getack = makeObjectShared("GETACK",6);
     shared.special_asterick = makeObjectShared("*",1);
     shared.special_equals = makeObjectShared("=",1);
+    shared.redacted = makeObjectShared("(redacted)",10);
 
     /* KeyDB Specific */
     shared.hdel = makeObjectShared(createStringObject("HDEL", 4));
@@ -2830,7 +2849,7 @@ void initMasterInfo(redisMaster *master)
 void initServerConfig(void) {
     int j;
 
-    updateCachedTime(true);
+    updateCachedTime();
     getRandomHexChars(g_pserver->runid,CONFIG_RUN_ID_SIZE);
     g_pserver->runid[CONFIG_RUN_ID_SIZE] = '\0';
     changeReplicationId();
@@ -2967,6 +2986,10 @@ void initServerConfig(void) {
     cserver.cthreads = CONFIG_DEFAULT_THREADS;
     cserver.fThreadAffinity = CONFIG_DEFAULT_THREAD_AFFINITY;
     cserver.threadAffinityOffset = 0;
+
+    /* Client Pause related */
+    g_pserver->client_pause_type = CLIENT_PAUSE_OFF;
+    g_pserver->client_pause_end_time = 0; 
     initConfigValues();
 }
 
@@ -3088,7 +3111,7 @@ int setOOMScoreAdj(int process_class) {
 
     fd = open("/proc/self/oom_score_adj", O_WRONLY);
     if (fd < 0 || write(fd, buf, strlen(buf)) < 0) {
-        serverLog(LOG_WARNING, "Unable to write oom_score_adj: %s", strerror(errno));
+        serverLog(LL_WARNING, "Unable to write oom_score_adj: %s", strerror(errno));
         if (fd != -1) close(fd);
         return C_ERR;
     }
@@ -3495,11 +3518,6 @@ void initServer(void) {
             selectDb(mi->cached_master, 0);
     }
 
-    if (g_pserver->syslog_enabled) {
-        openlog(g_pserver->syslog_ident, LOG_PID | LOG_NDELAY | LOG_NOWAIT,
-            g_pserver->syslog_facility);
-    }
-
     g_pserver->aof_state = g_pserver->aof_enabled ? AOF_ON : AOF_OFF;
     g_pserver->hz = g_pserver->config_hz;
     cserver.pid = getpid();
@@ -3912,12 +3930,6 @@ void preventCommandPropagation(client *c) {
     c->flags |= CLIENT_PREVENT_PROP;
 }
 
-/* Avoid logging any information about this client's arguments
- * since they contain sensitive information. */
-void preventCommandLogging(client *c) {
-    c->flags |= CLIENT_PREVENT_LOGGING;
-}
-
 /* AOF specific version of preventCommandPropagation(). */
 void preventCommandAOF(client *c) {
     c->flags |= CLIENT_PREVENT_AOF_PROP;
@@ -3931,7 +3943,7 @@ void preventCommandReplication(client *c) {
 /* Log the last command a client executed into the slowlog. */
 void slowlogPushCurrentCommand(client *c, struct redisCommand *cmd, ustime_t duration) {
     /* Some commands may contain sensitive data that should not be available in the slowlog. */
-    if ((c->flags & CLIENT_PREVENT_LOGGING) || (cmd->flags & CMD_SKIP_SLOWLOG))
+    if (cmd->flags & CMD_SKIP_SLOWLOG)
         return;
 
     /* If command argument vector was rewritten, use the original
@@ -3986,17 +3998,6 @@ void call(client *c, int flags) {
     serverAssert(GlobalLocksAcquired());
     static long long prev_err_count;
 
-    serverTL->fixed_time_expire++;
-
-    /* Send the command to clients in MONITOR mode if applicable.
-     * Administrative commands are considered too dangerous to be shown. */
-    if (listLength(g_pserver->monitors) &&
-        !g_pserver->loading.load(std::memory_order_relaxed) &&
-        !(c->cmd->flags & (CMD_SKIP_MONITOR|CMD_ADMIN)))
-    {
-        replicationFeedMonitors(c,g_pserver->monitors,c->db->id,c->argv,c->argc);
-    }
-
     /* We need to transfer async writes before a client's repl state gets changed.  Otherwise
         we won't be able to propogate them correctly. */
     if (c->cmd->flags & CMD_CATEGORY_REPLICATION) {
@@ -4013,7 +4014,8 @@ void call(client *c, int flags) {
     /* Call the command. */
     dirty = g_pserver->dirty;
     prev_err_count = g_pserver->stat_total_error_replies;
-    updateCachedTime(0);
+    g_pserver->fixed_time_expire++;
+
     incrementMvccTstamp();
     elapsedStart(&call_timer);
     try {
@@ -4079,6 +4081,14 @@ void call(client *c, int flags) {
     if ((flags & CMD_CALL_SLOWLOG) && !(c->flags & CLIENT_BLOCKED))
         slowlogPushCurrentCommand(c, real_cmd, duration);
 
+    /* Send the command to clients in MONITOR mode if applicable.
+     * Administrative commands are considered too dangerous to be shown. */
+    if (!(c->cmd->flags & (CMD_SKIP_MONITOR|CMD_ADMIN))) {
+        robj **argv = c->original_argv ? c->original_argv : c->argv;
+        int argc = c->original_argv ? c->original_argc : c->argc;
+        replicationFeedMonitors(c,g_pserver->monitors,c->db->id,argv,argc);
+    }
+
     /* Clear the original argv.
      * If the client is blocked we will handle slowlog when it is unblocked. */
     if (!(c->flags & CLIENT_BLOCKED))
@@ -4301,6 +4311,8 @@ int processCommand(client *c, int callFlags) {
         return C_OK;
     }
 
+    int is_read_command = (c->cmd->flags & CMD_READONLY) ||
+                           (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_READONLY));
     int is_write_command = (c->cmd->flags & CMD_WRITE) ||
                            (c->cmd->proc == execCommand && (c->mstate.cmd_flags & CMD_WRITE));
     int is_denyoom_command = (c->cmd->flags & CMD_DENYOOM) ||
@@ -4312,13 +4324,8 @@ int processCommand(client *c, int callFlags) {
     int is_may_replicate_command = (c->cmd->flags & (CMD_WRITE | CMD_MAY_REPLICATE)) ||
                                    (c->cmd->proc == execCommand && (c->mstate.cmd_flags & (CMD_WRITE | CMD_MAY_REPLICATE)));
 
-    /* Check if the user is authenticated. This check is skipped in case
-     * the default user is flagged as "nopass" and is active. */
-    int auth_required = (!(DefaultUser->flags & USER_FLAG_NOPASS) ||
-                          (DefaultUser->flags & USER_FLAG_DISABLED)) &&
-                        !c->authenticated;
-    if (auth_required) {
-        /* AUTH and HELLO and no auth modules are valid even in
+    if (authRequired(c)) {
+        /* AUTH and HELLO and no auth commands are valid even in
          * non-authenticated state. */
         if (!(c->cmd->flags & CMD_NO_AUTH)) {
             rejectCommand(c,shared.noautherr);
@@ -4515,7 +4522,7 @@ int processCommand(client *c, int callFlags) {
           c->cmd->proc != discardCommand &&
           c->cmd->proc != watchCommand &&
           c->cmd->proc != unwatchCommand &&
-	  c->cmd->proc != resetCommand &&
+          c->cmd->proc != resetCommand &&
         !(c->cmd->proc == shutdownCommand &&
           c->argc == 2 &&
           tolower(((char*)ptrFromObj(c->argv[1]))[0]) == 'n') &&
@@ -4527,6 +4534,14 @@ int processCommand(client *c, int callFlags) {
         return C_OK;
     }
 
+    /* Prevent a replica from sending commands that access the keyspace.
+     * The main objective here is to prevent abuse of client pause check
+     * from which replicas are exempt. */
+    if ((c->flags & CLIENT_SLAVE) && (is_may_replicate_command || is_write_command || is_read_command)) {
+        rejectCommandFormat(c, "Replica can't interract with the keyspace");
+        return C_OK;
+    }
+
     /* If the server is paused, block the client until
      * the pause has ended. Replicas are never paused. */
     if (!(c->flags & CLIENT_SLAVE) && 
@@ -4635,10 +4650,6 @@ int prepareForShutdown(int flags) {
        overwrite the synchronous saving did by SHUTDOWN. */
     if (g_pserver->child_type == CHILD_TYPE_RDB) {
         serverLog(LL_WARNING,"There is a child saving an .rdb. Killing it!");
-        /* Note that, in killRDBChild, we call rdbRemoveTempFile that will
-         * do close fd(in order to unlink file actully) in background thread.
-         * The temp rdb file fd may won't be closed when redis exits quickly,
-         * but OS will close this fd when process exits. */
         killRDBChild();
         /* Note that, in killRDBChild normally has backgroundSaveDoneHandler
          * doing it's cleanup, but in this case this code will not be reached,
@@ -5446,12 +5457,16 @@ sds genRedisInfoString(const char *section) {
             while ((ln = listNext(&li)))
             {
                 long long slave_repl_offset = 1;
+                long long slave_read_repl_offset = 1;
                 redisMaster *mi = (redisMaster*)listNodeValue(ln);
 
-                if (mi->master)
+                if (mi->master){
                     slave_repl_offset = mi->master->reploff;
-                else if (mi->cached_master)
+                    slave_read_repl_offset = mi->master->read_reploff;
+                } else if (mi->cached_master){
                     slave_repl_offset = mi->cached_master->reploff;
+                    slave_read_repl_offset = mi->cached_master->read_reploff;
+                }
 
                 char master_prefix[128] = "";
                 if (cmasters != 0) {
@@ -5464,6 +5479,7 @@ sds genRedisInfoString(const char *section) {
                     "master%s_link_status:%s\r\n"
                     "master%s_last_io_seconds_ago:%d\r\n"
                     "master%s_sync_in_progress:%d\r\n"
+                    "slave_read_repl_offset:%lld\r\n"
                     "slave_repl_offset:%lld\r\n"
                     ,master_prefix, mi->masterhost,
                     master_prefix, mi->masterport,
@@ -5472,6 +5488,7 @@ sds genRedisInfoString(const char *section) {
                     master_prefix, mi->master ?
                     ((int)(g_pserver->unixtime-mi->master->lastinteraction)) : -1,
                     master_prefix, mi->repl_state == REPL_STATE_TRANSFER,
+                    slave_read_repl_offset, 
                     slave_repl_offset
                 );
 
@@ -6203,6 +6220,63 @@ void closeChildUnusedResourceAfterFork() {
     cserver.pidfile = NULL;
 }
 
+void executeWithoutGlobalLock(std::function<void()> func) {
+    serverAssert(GlobalLocksAcquired());
+
+    std::vector<client*> vecclients;
+    listIter li;
+    listNode *ln;
+    listRewind(g_pserver->clients, &li);
+
+    // All client locks must be acquired *after* the global lock is reacquired to prevent deadlocks
+    //  so unlock here, and save them for reacquisition later
+    while ((ln = listNext(&li)) != nullptr)
+    {
+        client *c = (client*)listNodeValue(ln);
+        if (c->lock.fOwnLock()) {
+            serverAssert(c->flags & CLIENT_PROTECTED || c->flags & CLIENT_EXECUTING_COMMAND);  // If the client is not protected we have no gurantee they won't be free'd in the event loop
+            c->lock.unlock();
+            vecclients.push_back(c);
+        }
+    }
+    
+    /* Since we're about to release our lock we need to flush the repl backlog queue */
+    bool fReplBacklog = g_pserver->repl_batch_offStart >= 0;
+    if (fReplBacklog) {
+        flushReplBacklogToClients();
+        g_pserver->repl_batch_idxStart = -1;
+        g_pserver->repl_batch_offStart = -1;
+    }
+
+    aeReleaseLock();
+    serverAssert(!GlobalLocksAcquired());
+    try {
+        func();
+    }
+    catch (...) {
+        // Caller expects us to be locked so fix and rethrow
+        AeLocker locker;
+        locker.arm(nullptr);
+        locker.release();
+        for (client *c : vecclients)
+            c->lock.lock();
+        throw;
+    }
+    
+    AeLocker locker;
+    locker.arm(nullptr);
+    locker.release();
+
+    // Restore it so the calling code is not confused
+    if (fReplBacklog) {
+        g_pserver->repl_batch_idxStart = g_pserver->repl_backlog_idx;
+        g_pserver->repl_batch_offStart = g_pserver->master_repl_offset;
+    }
+
+    for (client *c : vecclients)
+        c->lock.lock();
+}
+
 /* purpose is one of CHILD_TYPE_ types */
 int redisFork(int purpose) {
     int childpid;
@@ -6214,7 +6288,9 @@ int redisFork(int purpose) {
 
         openChildInfoPipe();
     }
-    
+    long long startWriteLock = ustime();
+    aeAcquireForkLock();
+    latencyAddSampleIfNeeded("fork-lock",(ustime()-startWriteLock)/1000);
     if ((childpid = fork()) == 0) {
         /* Child */
         g_pserver->in_fork_child = purpose;
@@ -6223,6 +6299,7 @@ int redisFork(int purpose) {
         closeChildUnusedResourceAfterFork();
     } else {
         /* Parent */
+        aeReleaseForkLock();
         g_pserver->stat_total_forks++;
         g_pserver->stat_fork_time = ustime()-start;
         g_pserver->stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / g_pserver->stat_fork_time / (1024*1024*1024); /* GB per second. */
@@ -6285,7 +6362,7 @@ void loadDataFromDisk(void) {
         if (loadAppendOnlyFile(g_pserver->aof_filename) == C_OK)
             serverLog(LL_NOTICE,"DB loaded from append only file: %.3f seconds",(float)(ustime()-start)/1000000);
     } else if (g_pserver->rdb_filename != NULL || g_pserver->rdb_s3bucketpath != NULL) {
-        rdbSaveInfo rsi = RDB_SAVE_INFO_INIT;
+        rdbSaveInfo rsi;
         errno = 0; /* Prevent a stale value from affecting error checking */
         if (rdbLoad(&rsi,RDBFLAGS_NONE) == C_OK) {
             serverLog(LL_NOTICE,"DB loaded from disk: %.3f seconds",
@@ -6313,11 +6390,23 @@ void loadDataFromDisk(void) {
                 while ((ln = listNext(&li)))
                 {
                     redisMaster *mi = (redisMaster*)listNodeValue(ln);
-                    /* If we are a replica, create a cached master from this
-                    * information, in order to allow partial resynchronizations
-                    * with masters. */
-                    replicationCacheMasterUsingMyself(mi);
-                    selectDb(mi->cached_master,rsi.repl_stream_db);
+                    if (g_pserver->fActiveReplica) {
+                        for (size_t i = 0; i < rsi.numMasters(); i++) {
+                            if (!strcmp(mi->masterhost, rsi.masters[i].masterhost) && mi->masterport == rsi.masters[i].masterport) {
+                                memcpy(mi->master_replid, rsi.masters[i].master_replid, sizeof(mi->master_replid));
+                                mi->master_initial_offset = rsi.masters[i].master_initial_offset;
+                                replicationCacheMasterUsingMaster(mi);
+                                serverLog(LL_NOTICE, "Cached master recovered from RDB for %s:%d", mi->masterhost, mi->masterport);
+                            }
+                        }
+                    }
+                    else {
+                        /* If we are a replica, create a cached master from this
+                        * information, in order to allow partial resynchronizations
+                        * with masters. */
+                        replicationCacheMasterUsingMyself(mi);
+                        selectDb(mi->cached_master,rsi.repl_stream_db);
+                    }
                 }
             }
         } else if (errno != ENOENT) {
@@ -6526,10 +6615,55 @@ void OnTerminate()
     serverPanic("std::teminate() called");
 }
 
+void wakeTimeThread() {
+    updateCachedTime();
+    aeThreadOffline();
+    std::unique_lock<fastlock> lock(time_thread_lock);
+    aeThreadOnline();
+    if (sleeping_threads >= cserver.cthreads)
+        time_thread_cv.notify_one();
+    sleeping_threads--;
+    serverAssert(sleeping_threads >= 0);
+}
+
+void *timeThreadMain(void*) {
+    timespec delay;
+    delay.tv_sec = 0;
+    delay.tv_nsec = 100;
+    int cycle_count = 0;
+    aeThreadOnline();
+    while (true) {
+        {
+            aeThreadOffline();
+            std::unique_lock<fastlock> lock(time_thread_lock);
+            aeThreadOnline();
+            if (sleeping_threads >= cserver.cthreads) {
+                aeThreadOffline();
+                time_thread_cv.wait(lock);
+                aeThreadOnline();
+                cycle_count = 0;
+            }
+        }
+        updateCachedTime();
+        if (cycle_count == MAX_CYCLES_TO_HOLD_FORK_LOCK) {
+            aeThreadOffline();
+            aeThreadOnline();
+            cycle_count = 0;
+        }
+#if defined(__APPLE__)
+        nanosleep(&delay, nullptr);
+#else
+        clock_nanosleep(CLOCK_MONOTONIC, 0, &delay, NULL);
+#endif
+        cycle_count++;
+    }
+    aeThreadOffline();
+}
+
 void *workerThreadMain(void *parg)
 {
     int iel = (int)((int64_t)parg);
-    serverLog(LOG_INFO, "Thread %d alive.", iel);
+    serverLog(LL_NOTICE, "Thread %d alive.", iel);
     serverTL = g_pserver->rgthreadvar+iel;  // set the TLS threadsafe global
     tlsInitThread();
 
@@ -6541,6 +6675,7 @@ void *workerThreadMain(void *parg)
     }
 
     moduleAcquireGIL(true); // Normally afterSleep acquires this, but that won't be called on the first run
+    aeThreadOnline();
     aeEventLoop *el = g_pserver->rgthreadvar[iel].el;
     try
     {
@@ -6549,6 +6684,7 @@ void *workerThreadMain(void *parg)
     catch (ShutdownException)
     {
     }
+    aeThreadOffline();
     moduleReleaseGIL(true);
     serverAssert(!GlobalLocksAcquired());
     aeDeleteEventLoop(el);
@@ -6686,8 +6822,8 @@ int main(int argc, char **argv) {
     g_pserver->sentinel_mode = checkForSentinelMode(argc,argv);
     initServerConfig();
     serverTL = &g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN];
+    aeThreadOnline();
     aeAcquireLock();    // We own the lock on boot
-
     ACLInit(); /* The ACL subsystem must be initialized ASAP because the
                   basic networking code and client creation depends on it. */
     moduleInitModulesSystem();
@@ -6773,6 +6909,11 @@ int main(int argc, char **argv) {
         sdsfree(options);
     }
 
+    if (g_pserver->syslog_enabled) {
+        openlog(g_pserver->syslog_ident, LOG_PID | LOG_NDELAY | LOG_NOWAIT,
+            g_pserver->syslog_facility);
+    }
+
     if (cserver.fUsePro) {
         const char *keydb_pro_dir = getenv("KEYDB_PRO_DIRECTORY");
         sds path = sdsnew(keydb_pro_dir);
@@ -6834,7 +6975,7 @@ int main(int argc, char **argv) {
                 serverLog(LL_WARNING, "Failed to test the kernel for a bug that could lead to data corruption during background save. "
                                       "Your system could be affected, please report this error.");
             if (!checkIgnoreWarning("ARM64-COW-BUG")) {
-                serverLog(LL_WARNING,"Redis will now exit to prevent data corruption. "
+                serverLog(LL_WARNING,"KeyDB will now exit to prevent data corruption. "
                                      "Note that it is possible to suppress this warning by setting the following config: ignore-warnings ARM64-COW-BUG");
                 exit(1);
             }
@@ -6853,7 +6994,7 @@ int main(int argc, char **argv) {
             __AFL_INIT();
 #endif
             rio rdb;
-            rdbSaveInfo rsi = RDB_SAVE_INFO_INIT;
+            rdbSaveInfo rsi;
             startLoadingFile(stdin, (char*)"stdin", 0);
             rioInitWithFile(&rdb,stdin);
             rdbLoadRio(&rdb,0,&rsi);
@@ -6912,11 +7053,14 @@ int main(int argc, char **argv) {
 
     redisSetCpuAffinity(g_pserver->server_cpulist);
     aeReleaseLock();    //Finally we can dump the lock
+    aeThreadOffline();
     moduleReleaseGIL(true);
     
     setOOMScoreAdj(-1);
     serverAssert(cserver.cthreads > 0 && cserver.cthreads <= MAX_EVENT_LOOPS);
 
+    pthread_create(&cserver.time_thread_id, nullptr, timeThreadMain, nullptr);
+
     pthread_attr_t tattr;
     pthread_attr_init(&tattr);
     pthread_attr_setstacksize(&tattr, 1 << 23); // 8 MB
@@ -6931,7 +7075,7 @@ int main(int argc, char **argv) {
             CPU_SET(iel + cserver.threadAffinityOffset, &cpuset);
             if (pthread_setaffinity_np(g_pserver->rgthread[iel], sizeof(cpu_set_t), &cpuset) == 0)
             {
-                serverLog(LOG_INFO, "Binding thread %d to cpu %d", iel, iel + cserver.threadAffinityOffset + 1);
+                serverLog(LL_NOTICE, "Binding thread %d to cpu %d", iel, iel + cserver.threadAffinityOffset + 1);
             }
 #else
 			serverLog(LL_WARNING, "CPU pinning not available on this platform");

src/server.h

@@ -96,6 +96,7 @@ typedef long long ustime_t; /* microsecond time type. */
 #include "connection.h" /* Connection abstraction */
 #include "serverassert.h"
 #include "expire.h"
+#include "readwritelock.h"
 
 #define REDISMODULE_CORE 1
 #include "redismodule.h"    /* Redis modules API defines. */
@@ -306,6 +307,7 @@ inline bool operator!=(const void *p, const robj_sharedptr &rhs)
 #define CONFIG_DEFAULT_ENABLE_MULTIMASTER 0
 
 #define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 64 /* Loopkups per loop. */
+#define ACTIVE_EXPIRE_CYCLE_SUBKEY_LOOKUPS_PER_LOOP 16384 /* Subkey loopkups per loop. */
 #define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */
 #define ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC 25 /* CPU max % for keys collection */
 #define ACTIVE_EXPIRE_CYCLE_SLOW 0
@@ -470,7 +472,6 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
                                            and AOF client */
 #define CLIENT_REPL_RDBONLY (1ULL<<42) /* This client is a replica that only wants
                                           RDB without replication buffer. */
-#define CLIENT_PREVENT_LOGGING (1ULL<<43)  /* Prevent logging of command to slowlog */
 #define CLIENT_FORCE_REPLY (1ULL<<44) /* Should addReply be forced to write the text? */
 
 /* Client block type (btype field in client structure)
@@ -505,6 +506,7 @@ typedef enum {
     REPL_STATE_NONE = 0,            /* No active replication */
     REPL_STATE_CONNECT,             /* Must connect to master */
     REPL_STATE_CONNECTING,          /* Connecting to master */
+    REPL_STATE_RETRY_NOREPLPING,    /* Master does not support REPLPING, retry with PING */
     /* --- Handshake states, must be ordered --- */
     REPL_STATE_RECEIVE_PING_REPLY,  /* Wait for PING reply */
     REPL_STATE_SEND_HANDSHAKE,      /* Send handshake sequance to master */
@@ -732,6 +734,9 @@ typedef enum {
 #define REDISMODULE_AUX_BEFORE_RDB (1<<0)
 #define REDISMODULE_AUX_AFTER_RDB (1<<1)
 
+/* Number of cycles before time thread gives up fork lock */
+#define MAX_CYCLES_TO_HOLD_FORK_LOCK 10
+
 struct RedisModule;
 struct RedisModuleIO;
 struct RedisModuleDigest;
@@ -900,7 +905,7 @@ public:
     void addref() const { refcount.fetch_add(1, std::memory_order_relaxed); }
     unsigned release() const { return refcount.fetch_sub(1, std::memory_order_seq_cst) & ~(1U << 31); }
 } robj;
-static_assert(sizeof(redisObject) == 16, "object size is critical, don't increase");
+static_assert(sizeof(redisObject) <= 16, "object size is critical, don't increase");
 
 class redisObjectStack : public redisObjectExtended, public redisObject
 {
@@ -1283,9 +1288,9 @@ struct sharedObjectsStruct {
     *emptyscan, *multi, *exec, *left, *right, *hset, *srem, *xgroup, *xclaim,  
     *script, *replconf, *eval, *persist, *set, *pexpireat, *pexpire, 
     *time, *pxat, *px, *retrycount, *force, *justid, 
-    *lastid, *ping, *setid, *keepttl, *load, *createconsumer,
+    *lastid, *ping, *replping, *setid, *keepttl, *load, *createconsumer,
     *getack, *special_asterick, *special_equals, *default_username,
-    *hdel, *zrem, *mvccrestore, *pexpirememberat,
+    *hdel, *zrem, *mvccrestore, *pexpirememberat, *redacted,
     *select[PROTO_SHARED_SELECT_CMDS],
     *integers[OBJ_SHARED_INTEGERS],
     *mbulkhdr[OBJ_SHARED_BULKHDR_LEN], /* "*<value>\r\n" */
@@ -1388,6 +1393,38 @@ struct redisMemOverhead {
     } *db;
 };
 
+struct redisMaster {
+    char *masteruser;               /* AUTH with this user and masterauth with master */
+    char *masterauth;               /* AUTH with this password with master */
+    char *masterhost;               /* Hostname of master */
+    int masterport;                 /* Port of master */
+    client *cached_master;          /* Cached master to be reused for PSYNC. */
+    client *master;
+    /* The following two fields is where we store master PSYNC replid/offset
+     * while the PSYNC is in progress. At the end we'll copy the fields into
+     * the server->master client structure. */
+    char master_replid[CONFIG_RUN_ID_SIZE+1];  /* Master PSYNC runid. */
+    long long master_initial_offset;           /* Master PSYNC offset. */
+
+    bool isActive = false;
+    int repl_state;          /* Replication status if the instance is a replica */
+    off_t repl_transfer_size; /* Size of RDB to read from master during sync. */
+    off_t repl_transfer_read; /* Amount of RDB read from master during sync. */
+    off_t repl_transfer_last_fsync_off; /* Offset when we fsync-ed last time. */
+    connection *repl_transfer_s;     /* Slave -> Master SYNC socket */
+    int repl_transfer_fd;    /* Slave -> Master SYNC temp file descriptor */
+    char *repl_transfer_tmpfile; /* Slave-> master SYNC temp file name */
+    time_t repl_transfer_lastio; /* Unix time of the latest read, for timeout */
+    time_t repl_down_since; /* Unix time at which link with master went down */
+
+    unsigned char master_uuid[UUID_BINARY_LEN];  /* Used during sync with master, this is our master's UUID */
+                                                /* After we've connected with our master use the UUID in g_pserver->master */
+    uint64_t mvccLastSync;
+    /* During a handshake the server may have stale keys, we track these here to share once a reciprocal connection is made */
+    std::map<int, std::vector<robj_sharedptr>> *staleKeyMap;
+    int ielReplTransfer = -1;
+};
+
 /* This structure can be optionally passed to RDB save/load functions in
  * order to implement additional functionalities, by storing and loading
  * metadata to the RDB file.
@@ -1396,7 +1433,66 @@ struct redisMemOverhead {
  * replication in order to make sure that chained slaves (slaves of slaves)
  * select the correct DB and are able to accept the stream coming from the
  * top-level master. */
-typedef struct rdbSaveInfo {
+class rdbSaveInfo {
+public:
+    rdbSaveInfo() {
+        repl_stream_db = -1;
+        repl_id_is_set = 0;
+        memcpy(repl_id, "0000000000000000000000000000000000000000", sizeof(repl_id));
+        repl_offset = -1;
+        fForceSetKey = TRUE;
+        mvccMinThreshold = 0;
+        masters = nullptr;
+        masterCount = 0;
+    }
+    rdbSaveInfo(const rdbSaveInfo &other) {
+        repl_stream_db = other.repl_stream_db;
+        repl_id_is_set = other.repl_id_is_set;
+        memcpy(repl_id, other.repl_id, sizeof(repl_id));
+        repl_offset = other.repl_offset;
+        fForceSetKey = other.fForceSetKey;
+        mvccMinThreshold = other.mvccMinThreshold;
+        masters = (struct redisMaster*)malloc(sizeof(struct redisMaster) * other.masterCount);
+        memcpy(masters, other.masters, sizeof(struct redisMaster) * other.masterCount);
+        masterCount = other.masterCount;
+    }
+    rdbSaveInfo(rdbSaveInfo &&other) : rdbSaveInfo() {
+        swap(*this, other);
+    }
+    rdbSaveInfo &operator=(rdbSaveInfo other) {
+        swap(*this, other);
+        return *this;
+    }
+    ~rdbSaveInfo() {
+        free(masters);
+    }
+    friend void swap(rdbSaveInfo &first, rdbSaveInfo &second) {
+        std::swap(first.repl_stream_db, second.repl_stream_db);
+        std::swap(first.repl_id_is_set, second.repl_id_is_set);
+        std::swap(first.repl_id, second.repl_id);
+        std::swap(first.repl_offset, second.repl_offset);
+        std::swap(first.fForceSetKey, second.fForceSetKey);
+        std::swap(first.mvccMinThreshold, second.mvccMinThreshold);
+        std::swap(first.masters, second.masters);
+        std::swap(first.masterCount, second.masterCount);
+
+    }
+
+    void addMaster(const struct redisMaster &mi) {
+        masterCount++;
+        if (masters == nullptr) {
+            masters = (struct redisMaster*)malloc(sizeof(struct redisMaster));
+        }
+        else {
+            masters = (struct redisMaster*)realloc(masters, sizeof(struct redisMaster) * masterCount);
+        }
+        memcpy(masters + masterCount - 1, &mi, sizeof(struct redisMaster));
+    }
+
+    size_t numMasters() {
+        return masterCount;
+    }
+
     /* Used saving and loading. */
     int repl_stream_db;  /* DB to select in g_pserver->master client. */
 
@@ -1406,10 +1502,11 @@ typedef struct rdbSaveInfo {
     long long repl_offset;                  /* Replication offset. */
     int fForceSetKey;
     uint64_t mvccMinThreshold;
-    struct redisMaster *mi;
-} rdbSaveInfo;
+    struct redisMaster *masters;
 
-#define RDB_SAVE_INFO_INIT {-1,0,"0000000000000000000000000000000000000000",-1, TRUE, 0, nullptr}
+private:
+    size_t masterCount;
+};
 
 struct malloc_stats {
     size_t zmalloc_used;
@@ -1498,43 +1595,12 @@ struct redisServerThreadVars {
     std::vector<client*> vecclientsProcess;
 };
 
-struct redisMaster {
-    char *masteruser;               /* AUTH with this user and masterauth with master */
-    char *masterauth;               /* AUTH with this password with master */
-    char *masterhost;               /* Hostname of master */
-    int masterport;                 /* Port of master */
-    client *cached_master;          /* Cached master to be reused for PSYNC. */
-    client *master;
-    /* The following two fields is where we store master PSYNC replid/offset
-     * while the PSYNC is in progress. At the end we'll copy the fields into
-     * the server->master client structure. */
-    char master_replid[CONFIG_RUN_ID_SIZE+1];  /* Master PSYNC runid. */
-    long long master_initial_offset;           /* Master PSYNC offset. */
-
-    bool isActive = false;
-    int repl_state;          /* Replication status if the instance is a replica */
-    off_t repl_transfer_size; /* Size of RDB to read from master during sync. */
-    off_t repl_transfer_read; /* Amount of RDB read from master during sync. */
-    off_t repl_transfer_last_fsync_off; /* Offset when we fsync-ed last time. */
-    connection *repl_transfer_s;     /* Slave -> Master SYNC socket */
-    int repl_transfer_fd;    /* Slave -> Master SYNC temp file descriptor */
-    char *repl_transfer_tmpfile; /* Slave-> master SYNC temp file name */
-    time_t repl_transfer_lastio; /* Unix time of the latest read, for timeout */
-    time_t repl_down_since; /* Unix time at which link with master went down */
-
-    unsigned char master_uuid[UUID_BINARY_LEN];  /* Used during sync with master, this is our master's UUID */
-                                                /* After we've connected with our master use the UUID in g_pserver->master */
-    uint64_t mvccLastSync;
-    /* During a handshake the server may have stale keys, we track these here to share once a reciprocal connection is made */
-    std::map<int, std::vector<robj_sharedptr>> *staleKeyMap;
-    int ielReplTransfer = -1;
-};
-
 // Const vars are not changed after worker threads are launched
 struct redisServerConst {
     pid_t pid;                  /* Main process pid. */
     time_t stat_starttime;          /* Server start time */
     pthread_t main_thread_id;         /* Main thread id */
+    pthread_t time_thread_id;
     char *configfile;           /* Absolute config file path, or NULL */
     char *executable;           /* Absolute executable file path. */
     char **exec_argv;           /* Executable argv vector (copy). */
@@ -1934,7 +2000,8 @@ struct redisServer {
     ::dict *lua_scripts;         /* A dictionary of SHA1 -> Lua scripts */
     unsigned long long lua_scripts_mem;  /* Cached scripts' memory + oh */
     mstime_t lua_time_limit;  /* Script timeout in milliseconds */
-    mstime_t lua_time_start;  /* Start time of script, milliseconds time */
+    monotime lua_time_start;  /* monotonic timer to detect timed-out script */
+    mstime_t lua_time_snapshot; /* Snapshot of mstime when script is started */
     int lua_write_dirty;  /* True if a write command was called during the
                              execution of the current script. */
     int lua_random_dirty; /* True if a random command was called during the
@@ -2162,6 +2229,7 @@ void moduleFireServerEvent(uint64_t eid, int subid, void *data);
 void processModuleLoadingProgressEvent(int is_aof);
 int moduleTryServeClientBlockedOnKey(client *c, robj *key);
 void moduleUnblockClient(client *c);
+int moduleBlockedClientMayTimeout(client *c);
 int moduleClientIsBlockedOnKeys(client *c);
 void moduleNotifyUserChanged(client *c);
 void moduleNotifyKeyUnlink(robj *key, robj *val);
@@ -2177,7 +2245,7 @@ extern "C" void getRandomHexChars(char *p, size_t len);
 extern "C" void getRandomBytes(unsigned char *p, size_t len);
 uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l);
 void exitFromChild(int retcode);
-size_t redisPopcount(const void *s, long count);
+long long redisPopcount(const void *s, long count);
 int redisSetProcTitle(const char *title);
 int validateProcTitleTemplate(const char *_template);
 int redisCommunicateSystemd(const char *sd_notify_msg);
@@ -2223,6 +2291,7 @@ void addReplyErrorSds(client *c, sds err);
 void addReplyError(client *c, const char *err);
 void addReplyStatus(client *c, const char *status);
 void addReplyDouble(client *c, double d);
+void addReplyBigNum(client *c, const char* num, size_t len);
 void addReplyHumanLongDouble(client *c, long double d);
 void addReplyLongLong(client *c, long long ll);
 #ifdef __cplusplus
@@ -2250,9 +2319,10 @@ sds getAllClientsInfoString(int type);
 void rewriteClientCommandVector(client *c, int argc, ...);
 void rewriteClientCommandArgument(client *c, int i, robj *newval);
 void replaceClientCommandVector(client *c, int argc, robj **argv);
+void redactClientCommandArgument(client *c, int argc);
 unsigned long getClientOutputBufferMemoryUsage(client *c);
 int freeClientsInAsyncFreeQueue(int iel);
-void asyncCloseClientOnOutputBufferLimitReached(client *c);
+int closeClientOnOutputBufferLimitReached(client *c, int async);
 int getClientType(client *c);
 int getClientTypeByName(const char *name);
 const char *getClientTypeName(int cclass);
@@ -2279,6 +2349,7 @@ void unprotectClient(client *c);
 
 void ProcessPendingAsyncWrites(void);
 client *lookupClientByID(uint64_t id);
+int authRequired(client *c);
 
 #ifdef __GNUC__
 void addReplyErrorFormat(client *c, const char *fmt, ...)
@@ -2296,6 +2367,7 @@ void disableTracking(client *c);
 void trackingRememberKeys(client *c);
 void trackingInvalidateKey(client *c, robj *keyobj);
 void trackingInvalidateKeysOnFlush(int async);
+void freeTrackingRadixTree(rax *rt);
 void freeTrackingRadixTreeAsync(rax *rt);
 void trackingLimitUsedSlots(void);
 uint64_t trackingGetTotalItems(void);
@@ -2328,6 +2400,7 @@ void initClientMultiState(client *c);
 void freeClientMultiState(client *c);
 void queueMultiCommand(client *c);
 void touchWatchedKey(redisDb *db, robj *key);
+int isWatchedKeyExpired(client *c);
 void touchAllWatchedKeysInDb(redisDb *emptied, redisDb *replaced_with);
 void discardTransaction(client *c);
 void flagTransaction(client *c);
@@ -2353,6 +2426,8 @@ robj *createObject(int type, void *ptr);
 robj *createStringObject(const char *ptr, size_t len);
 robj *createRawStringObject(const char *ptr, size_t len);
 robj *createEmbeddedStringObject(const char *ptr, size_t len);
+robj *tryCreateRawStringObject(const char *ptr, size_t len);
+robj *tryCreateStringObject(const char *ptr, size_t len);
 robj *dupStringObject(const robj *o);
 int isSdsRepresentableAsLongLong(const char *s, long long *llval);
 int isObjectRepresentableAsLongLong(robj *o, long long *llongval);
@@ -2373,6 +2448,7 @@ robj *createZsetZiplistObject(void);
 robj *createStreamObject(void);
 robj *createModuleObject(moduleType *mt, void *value);
 int getLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg);
+int getUnsignedLongLongFromObjectOrReply(client *c, robj *o, uint64_t *target, const char *msg);
 int getPositiveLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg);
 int getRangeLongFromObjectOrReply(client *c, robj *o, long min, long max, long *target, const char *msg);
 int checkType(client *c, robj_roptr o, int type);
@@ -2429,6 +2505,7 @@ void clearReplicationId2(void);
 void mergeReplicationId(const char *);
 void chopReplicationBacklog(void);
 void replicationCacheMasterUsingMyself(struct redisMaster *mi);
+void replicationCacheMasterUsingMaster(struct redisMaster *mi);
 void feedReplicationBacklog(const void *ptr, size_t len);
 void updateMasterAuth();
 void showLatestBacklog();
@@ -2484,6 +2561,7 @@ void sendChildInfo(childInfoType info_type, size_t keys, const char *pname);
 void receiveChildInfo(void);
 
 /* Fork helpers */
+void executeWithoutGlobalLock(std::function<void()> func);
 int redisFork(int type);
 int hasActiveChildProcess();
 void resetChildState();
@@ -2561,7 +2639,7 @@ unsigned char *zzlFirstInRange(unsigned char *zl, zrangespec *range);
 unsigned char *zzlLastInRange(unsigned char *zl, zrangespec *range);
 unsigned long zsetLength(robj_roptr zobj);
 void zsetConvert(robj *zobj, int encoding);
-void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen);
+void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen);
 int zsetScore(robj_roptr zobj, sds member, double *score);
 unsigned long zslGetRank(zskiplist *zsl, double score, sds o);
 int zsetAdd(robj *zobj, double score, sds ele, int in_flags, int *out_flags, double *newscore);
@@ -2605,7 +2683,6 @@ void redisOpArrayInit(redisOpArray *oa);
 void redisOpArrayFree(redisOpArray *oa);
 void forceCommandPropagation(client *c, int flags);
 void preventCommandPropagation(client *c);
-void preventCommandLogging(client *c);
 void preventCommandAOF(client *c);
 void preventCommandReplication(client *c);
 void slowlogPushCurrentCommand(client *c, struct redisCommand *cmd, ustime_t duration);
@@ -2627,7 +2704,7 @@ void resetErrorTableStats(void);
 void adjustOpenFilesLimit(void);
 void incrementErrorCount(const char *fullerr, size_t namelen);
 void closeListeningSockets(int unlink_unix_socket);
-void updateCachedTime(int update_daylight_info);
+void updateCachedTime(void);
 void resetServerStats(void);
 void activeDefragCycle(void);
 unsigned int getLRUClock(void);
@@ -2714,6 +2791,8 @@ int removeExpireCore(redisDb *db, robj *key, dictEntry *de);
 int removeSubkeyExpire(redisDb *db, robj *key, robj *subkey);
 void propagateExpire(redisDb *db, robj *key, int lazy);
 void propagateSubkeyExpire(redisDb *db, int type, robj *key, robj *subkey);
+void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj);
+int keyIsExpired(redisDb *db, robj *key);
 int expireIfNeeded(redisDb *db, robj *key);
 expireEntry *getExpire(redisDb *db, robj_roptr key);
 void setExpire(client *c, redisDb *db, robj *key, robj *subkey, long long when);
@@ -2727,6 +2806,7 @@ robj_roptr lookupKeyReadWithFlags(redisDb *db, robj *key, int flags);
 robj *lookupKeyWriteWithFlags(redisDb *db, robj *key, int flags);
 robj_roptr objectCommandLookup(client *c, robj *key);
 robj_roptr objectCommandLookupOrReply(client *c, robj *key, robj *reply);
+void SentReplyOnKeyMiss(client *c, robj *reply);
 int objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
                        long long lru_clock, int lru_multiplier);
 #define LOOKUP_NONE 0

src/storage.cpp

@@ -109,11 +109,12 @@ void pool_free(struct alloc_pool *ppool, void *pv)
         }
         cur = cur->pnext;
     }
-    serverLog(LOG_CRIT, "obj not from pool");
+    serverLog(LL_WARNING, "obj not from pool");
     sfree(obj); // we don't know where it came from
     return;
 }
 
+extern size_t OBJ_ENCODING_EMBSTR_SIZE_LIMIT;
 #define EMBSTR_ROBJ_SIZE (sizeof(robj)+sizeof(struct sdshdr8)+OBJ_ENCODING_EMBSTR_SIZE_LIMIT+1)
 struct alloc_pool poolobj;
 struct alloc_pool poolembstrobj;
@@ -151,14 +152,14 @@ void storage_init(const char *tmpfilePath, size_t cbFileReserve)
         int errv = memkind_create_pmem(PMEM_DIR, 0, &mkdisk);
         if (errv == MEMKIND_ERROR_INVALID)
         {
-            serverLog(LOG_CRIT, "Memory pool creation failed: %s", strerror(errno));
+            serverLog(LL_WARNING, "Memory pool creation failed: %s", strerror(errno));
             exit(EXIT_FAILURE);
         }
         else if (errv)
         {
             char msgbuf[1024];
             memkind_error_message(errv, msgbuf, 1024);
-            serverLog(LOG_CRIT, "Memory pool creation failed: %s", msgbuf);
+            serverLog(LL_WARNING, "Memory pool creation failed: %s", msgbuf);
             exit(EXIT_FAILURE);
         }
 
@@ -166,7 +167,7 @@ void storage_init(const char *tmpfilePath, size_t cbFileReserve)
         int fdTest = forkFile();
         if (fdTest < 0)
         {
-            serverLog(LOG_ERR, "Scratch file system does not support Copy on Write.  To fix this scratch-file-path must point to a path on a filesystem which supports copy on write, such as btrfs.");
+            serverLog(LL_WARNING, "Scratch file system does not support Copy on Write.  To fix this scratch-file-path must point to a path on a filesystem which supports copy on write, such as btrfs.");
             exit(EXIT_FAILURE);
         }
         close(fdTest);
@@ -254,7 +255,7 @@ void handle_prefork()
 {
     fdNew = forkFile();
     if (fdNew < 0)
-        serverLog(LOG_ERR, "Failed to clone scratch file");
+        serverLog(LL_WARNING, "Failed to clone scratch file");
 }
 
 void handle_postfork_parent()

src/t_hash.cpp

@@ -39,17 +39,22 @@
  * as their string length can be queried in constant time. */
 void hashTypeTryConversion(robj *o, robj **argv, int start, int end) {
     int i;
+    size_t sum = 0;
 
     if (o->encoding != OBJ_ENCODING_ZIPLIST) return;
 
     for (i = start; i <= end; i++) {
-        if (sdsEncodedObject(argv[i]) &&
-            sdslen(szFromObj(argv[i])) > g_pserver->hash_max_ziplist_value)
-        {
+        if (!sdsEncodedObject(argv[i]))
+            continue;
+        size_t len = sdslen(szFromObj(argv[i]));
+        if (len > g_pserver->hash_max_ziplist_value) {
             hashTypeConvert(o, OBJ_ENCODING_HT);
-            break;
+            return;
         }
+        sum += len;
     }
+    if (!ziplistSafeToAdd((unsigned char *)ptrFromObj(o), sum))
+        hashTypeConvert(o, OBJ_ENCODING_HT);
 }
 
 /* Get the value from a ziplist encoded hash, identified by field.
@@ -1015,7 +1020,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
     int uniq = 1;
     robj_roptr hash;
 
-    if ((hash = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp]))
+    if ((hash = lookupKeyReadOrReply(c,c->argv[1],shared.emptyarray))
         == nullptr || checkType(c,hash,OBJ_HASH)) return;
     size = hashTypeLength(hash);
 
@@ -1204,7 +1209,7 @@ void hrandfieldWithCountCommand(client *c, long l, int withvalues) {
     }
 }
 
-/* HRANDFIELD [<count> WITHVALUES] */
+/* HRANDFIELD key [<count> [WITHVALUES]] */
 void hrandfieldCommand(client *c) {
     long l;
     int withvalues = 0;

src/t_list.cpp

@@ -29,6 +29,8 @@
 
 #include "server.h"
 
+#define LIST_MAX_ITEM_SIZE ((1ull<<32)-1024)
+
 /*-----------------------------------------------------------------------------
  * List API
  *----------------------------------------------------------------------------*/
@@ -224,6 +226,13 @@ robj *listTypeDup(robj *o) {
 void pushGenericCommand(client *c, int where, int xx) {
     int j;
 
+    for (j = 2; j < c->argc; j++) {
+        if (sdslen(szFromObj(c->argv[j])) > LIST_MAX_ITEM_SIZE) {
+            addReplyError(c, "Element too large");
+            return;
+        }
+    }
+
     robj *lobj = lookupKeyWrite(c->db, c->argv[1]);
     if (checkType(c,lobj,OBJ_LIST)) return;
     if (!lobj) {
@@ -287,6 +296,11 @@ void linsertCommand(client *c) {
         return;
     }
 
+    if (sdslen(szFromObj(c->argv[4])) > LIST_MAX_ITEM_SIZE) {
+        addReplyError(c, "Element too large");
+        return;
+    }
+
     if ((subject = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL ||
         checkType(c,subject,OBJ_LIST)) return;
 
@@ -354,6 +368,11 @@ void lsetCommand(client *c) {
     long index;
     robj *value = c->argv[3];
 
+    if (sdslen(szFromObj(value)) > LIST_MAX_ITEM_SIZE) {
+        addReplyError(c, "Element too large");
+        return;
+    }
+
     if ((getLongFromObjectOrReply(c, c->argv[2], &index, NULL) != C_OK))
         return;
 
@@ -576,6 +595,11 @@ void lposCommand(client *c) {
     int direction = LIST_TAIL;
     long rank = 1, count = -1, maxlen = 0; /* Count -1: option not given. */
 
+    if (sdslen(szFromObj(ele)) > LIST_MAX_ITEM_SIZE) {
+        addReplyError(c, "Element too large");
+        return;
+    }
+
     /* Parse the optional arguments. */
     for (int j = 3; j < c->argc; j++) {
         char *opt = szFromObj(c->argv[j]);
@@ -671,6 +695,11 @@ void lremCommand(client *c) {
     long toremove;
     long removed = 0;
 
+    if (sdslen(szFromObj(obj)) > LIST_MAX_ITEM_SIZE) {
+        addReplyError(c, "Element too large");
+        return;
+    }
+
     if ((getLongFromObjectOrReply(c, c->argv[2], &toremove, NULL) != C_OK))
         return;
 

src/t_set.cpp

@@ -66,7 +66,10 @@ int setTypeAdd(robj *subject, const char *value) {
             if (success) {
                 /* Convert to regular set when the intset contains
                  * too many entries. */
-                if (intsetLen((intset*)subject->m_ptr) > g_pserver->set_max_intset_entries)
+                size_t max_entries = g_pserver->set_max_intset_entries;
+                /* limit to 1G entries due to intset internals. */
+                if (max_entries >= 1<<30) max_entries = 1<<30;
+                if (intsetLen((intset*)subject->m_ptr) > max_entries)
                     setTypeConvert(subject,OBJ_ENCODING_HT);
                 return 1;
             }
@@ -397,12 +400,12 @@ void smoveCommand(client *c) {
     }
 
     signalModifiedKey(c,c->db,c->argv[1]);
-    signalModifiedKey(c,c->db,c->argv[2]);
     g_pserver->dirty++;
 
     /* An extra key has changed when ele was successfully added to dstset */
     if (setTypeAdd(dstset,szFromObj(ele))) {
         g_pserver->dirty++;
+        signalModifiedKey(c,c->db,c->argv[2]);
         notifyKeyspaceEvent(NOTIFY_SET,"sadd",c->argv[2],c->db->id);
     }
     addReply(c,shared.cone);
@@ -512,7 +515,7 @@ void spopWithCountCommand(client *c) {
     const char *sdsele;
     robj *objele;
     int encoding;
-    int64_t llele;
+    int64_t llele = 0;
     unsigned long remaining = size-count; /* Elements left after SPOP. */
 
     /* If we are here, the number of requested elements is less than the
@@ -664,7 +667,7 @@ void srandmemberWithCountCommand(client *c) {
     int uniq = 1;
     robj_roptr set;
     const char *ele;
-    int64_t llele;
+    int64_t llele = 0;
     int encoding;
 
     dict *d;
@@ -813,7 +816,7 @@ void srandmemberWithCountCommand(client *c) {
 void srandmemberCommand(client *c) {
     robj_roptr set;
     const char *ele;
-    int64_t llele;
+    int64_t llele = 0;
     int encoding;
 
     if (c->argc == 3) {
@@ -863,24 +866,17 @@ void sinterGenericCommand(client *c, robj **setkeys,
     int64_t intobj;
     void *replylen = NULL;
     unsigned long j, cardinality = 0;
-    int encoding;
+    int encoding, empty = 0;
 
     for (j = 0; j < setnum; j++) {
         robj *setobj = dstkey ?
             lookupKeyWrite(c->db,setkeys[j]) :
             lookupKeyRead(c->db,setkeys[j]).unsafe_robjcast();
         if (!setobj) {
-            zfree(sets);
-            if (dstkey) {
-                if (dbDelete(c->db,dstkey)) {
-                    signalModifiedKey(c,c->db,dstkey);
-                    g_pserver->dirty++;
-                }
-                addReply(c,shared.czero);
-            } else {
-                addReply(c,shared.emptyset[c->resp]);
-            }
-            return;
+            /* A NULL is considered an empty set */
+            empty += 1;
+            sets[j] = NULL;
+            continue;
         }
         if (checkType(c,setobj,OBJ_SET)) {
             zfree(sets);
@@ -888,6 +884,24 @@ void sinterGenericCommand(client *c, robj **setkeys,
         }
         sets[j] = setobj;
     }
+
+    /* Set intersection with an empty set always results in an empty set.
+     * Return ASAP if there is an empty set. */
+    if (empty > 0) {
+        zfree(sets);
+        if (dstkey) {
+            if (dbDelete(c->db,dstkey)) {
+                signalModifiedKey(c,c->db,dstkey);
+                notifyKeyspaceEvent(NOTIFY_GENERIC,"del",dstkey,c->db->id);
+                g_pserver->dirty++;
+            }
+            addReply(c,shared.czero);
+        } else {
+            addReply(c,shared.emptyset[c->resp]);
+        }
+        return;
+    }
+
     /* Sort sets from the smallest to largest, this will improve our
      * algorithm's performance */
     qsort(sets,setnum,sizeof(robj*),qsortCompareSetsByCardinality);
@@ -981,10 +995,12 @@ void sinterGenericCommand(client *c, robj **setkeys,
     zfree(sets);
 }
 
+/* SINTER key [key ...] */
 void sinterCommand(client *c) {
     sinterGenericCommand(c,c->argv+1,c->argc-1,NULL);
 }
 
+/* SINTERSTORE destination key [key ...] */
 void sinterstoreCommand(client *c) {
     sinterGenericCommand(c,c->argv+2,c->argc-2,c->argv[1]);
 }
@@ -1154,18 +1170,22 @@ void sunionDiffGenericCommand(client *c, robj **setkeys, int setnum,
     zfree(sets);
 }
 
+/* SUNION key [key ...] */
 void sunionCommand(client *c) {
     sunionDiffGenericCommand(c,c->argv+1,c->argc-1,NULL,SET_OP_UNION);
 }
 
+/* SUNIONSTORE destination key [key ...] */
 void sunionstoreCommand(client *c) {
     sunionDiffGenericCommand(c,c->argv+2,c->argc-2,c->argv[1],SET_OP_UNION);
 }
 
+/* SDIFF key [key ...] */
 void sdiffCommand(client *c) {
     sunionDiffGenericCommand(c,c->argv+1,c->argc-1,NULL,SET_OP_DIFF);
 }
 
+/* SDIFFSTORE destination key [key ...] */
 void sdiffstoreCommand(client *c) {
     sunionDiffGenericCommand(c,c->argv+2,c->argc-2,c->argv[1],SET_OP_DIFF);
 }

src/t_stream.cpp

@@ -47,6 +47,12 @@
  * setting stream_node_max_bytes to a huge number. */
 #define STREAM_LISTPACK_MAX_PRE_ALLOCATE 4096
 
+/* Don't let listpacks grow too big, even if the user config allows it.
+ * doing so can lead to an overflow (trying to store more than 32bit length
+ * into the listpack header), or actually an assertion since lpInsert
+ * will return NULL. */
+#define STREAM_LISTPACK_MAX_SIZE (1<<30)
+
 void streamFreeCG(streamCG *cg);
 void streamFreeNACK(streamNACK *na);
 size_t streamReplyWithRangeFromConsumerPEL(client *c, stream *s, streamID *start, streamID *end, size_t count, streamConsumer *consumer);
@@ -434,8 +440,11 @@ void streamGetEdgeID(stream *s, int first, streamID *edge_id)
  *
  * The function returns C_OK if the item was added, this is always true
  * if the ID was generated by the function. However the function may return
- * C_ERR if an ID was given via 'use_id', but adding it failed since the
- * current top ID is greater or equal. */
+ * C_ERR in several cases:
+ * 1. If an ID was given via 'use_id', but adding it failed since the
+ *    current top ID is greater or equal. errno will be set to EDOM.
+ * 2. If a size of a single element or the sum of the elements is too big to
+ *    be stored into the stream. errno will be set to ERANGE. */
 int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_id, streamID *use_id) {
 
     /* Generate the new entry ID. */
@@ -449,7 +458,23 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
      * or return an error. Automatically generated IDs might
      * overflow (and wrap-around) when incrementing the sequence
        part. */
-    if (streamCompareID(&id,&s->last_id) <= 0) return C_ERR;
+    if (streamCompareID(&id,&s->last_id) <= 0) {
+        errno = EDOM;
+        return C_ERR;
+    }
+
+    /* Avoid overflow when trying to add an element to the stream (listpack
+     * can only host up to 32bit length sttrings, and also a total listpack size
+     * can't be bigger than 32bit length. */
+    size_t totelelen = 0;
+    for (int64_t i = 0; i < numfields*2; i++) {
+        sds ele = szFromObj(argv[i]);
+        totelelen += sdslen(ele);
+    }
+    if (totelelen > STREAM_LISTPACK_MAX_SIZE) {
+        errno = ERANGE;
+        return C_ERR;
+    }
 
     /* Add the new entry. */
     raxIterator ri;
@@ -508,9 +533,10 @@ int streamAppendItem(stream *s, robj **argv, int64_t numfields, streamID *added_
      * if we need to switch to the next one. 'lp' will be set to NULL if
      * the current node is full. */
     if (lp != NULL) {
-        if (g_pserver->stream_node_max_bytes &&
-            lp_bytes >= g_pserver->stream_node_max_bytes)
-        {
+        size_t node_max_bytes = g_pserver->stream_node_max_bytes;
+        if (node_max_bytes == 0 || node_max_bytes > STREAM_LISTPACK_MAX_SIZE)
+            node_max_bytes = STREAM_LISTPACK_MAX_SIZE;
+        if (lp_bytes + totelelen >= node_max_bytes) {
             lp = NULL;
         } else if (g_pserver->stream_node_max_entries) {
             unsigned char *lp_ele = lpFirst(lp);
@@ -703,16 +729,16 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) {
 
     int64_t deleted = 0;
     while (raxNext(&ri)) {
-        /* Check if we exceeded the amount of work we could do */
-        if (limit && deleted >= limit)
-            break;
-
         if (trim_strategy == TRIM_STRATEGY_MAXLEN && s->length <= maxlen)
             break;
 
         unsigned char *lp = (unsigned char*)ri.data, *p = lpFirst(lp);
         int64_t entries = lpGetInteger(p);
 
+        /* Check if we exceeded the amount of work we could do */
+        if (limit && (deleted + entries) > limit)
+            break;
+
         /* Check if we can remove the whole node. */
         int remove_node;
         streamID master_id = {0}; /* For MINID */
@@ -813,7 +839,7 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) {
         }
         deleted += deleted_from_lp;
 
-        /* Now we the entries/deleted counters. */
+        /* Now we update the entries/deleted counters. */
         p = lpFirst(lp);
         lp = lpReplaceInteger(lp,&p,entries-deleted_from_lp);
         p = lpNext(lp,p); /* Skip deleted field. */
@@ -842,7 +868,7 @@ int64_t streamTrim(stream *s, streamAddTrimArgs *args) {
 
 /* Trims a stream by length. Returns the number of deleted items. */
 int64_t streamTrimByLength(stream *s, long long maxlen, int approx) {
-    streamAddTrimArgs args = {0};
+    streamAddTrimArgs args = {{0}};
     args.trim_strategy = TRIM_STRATEGY_MAXLEN;
     args.approx_trim = approx;
     args.limit = approx ? 100 * g_pserver->stream_node_max_entries : 0;
@@ -852,7 +878,7 @@ int64_t streamTrimByLength(stream *s, long long maxlen, int approx) {
 
 /* Trims a stream by minimum ID. Returns the number of deleted items. */
 int64_t streamTrimByID(stream *s, streamID minid, int approx) {
-    streamAddTrimArgs args = {0};
+    streamAddTrimArgs args = {{0}};
     args.trim_strategy = TRIM_STRATEGY_MINID;
     args.approx_trim = approx;
     args.limit = approx ? 100 * g_pserver->stream_node_max_entries : 0;
@@ -1799,11 +1825,13 @@ void xaddCommand(client *c) {
     /* Append using the low level function and return the ID. */
     streamID id;
     if (streamAppendItem(s,c->argv+field_pos,(c->argc-field_pos)/2,
-        &id, parsed_args.id_given ? &parsed_args.id : NULL)
-        == C_ERR)
+        &id, parsed_args.id_given ? &parsed_args.id : NULL) == C_ERR)
     {
-        addReplyError(c,"The ID specified in XADD is equal or smaller than the "
-                        "target stream top item");
+        if (errno == EDOM)
+            addReplyError(c,"The ID specified in XADD is equal or smaller than "
+                            "the target stream top item");
+        else
+            addReplyError(c,"Elements are too large to be stored");
         return;
     }
     addReplyStreamID(c,&id);
@@ -3241,7 +3269,7 @@ void xtrimCommand(client *c) {
 
     /* Argument parsing. */
     streamAddTrimArgs parsed_args;
-    if (streamParseAddOrTrimArgsOrReply(c, &parsed_args, 1) < 0)
+    if (streamParseAddOrTrimArgsOrReply(c, &parsed_args, 0) < 0)
         return; /* streamParseAddOrTrimArgsOrReply already replied. */
 
     /* If the key does not exist, we are ok returning zero, that is, the
@@ -3570,7 +3598,7 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) {
     /* In non-deep mode we just validated the listpack header (encoded size) */
     if (!deep) return 1;
 
-    next = p = lpFirst(lp);
+    next = p = lpValidateFirst(lp);
     if (!lpValidateNext(lp, &next, size)) return 0;
     if (!p) return 0;
 
@@ -3608,7 +3636,11 @@ int streamValidateListpackIntegrity(unsigned char *lp, size_t size, int deep) {
         p = next; if (!lpValidateNext(lp, &next, size)) return 0;
 
         /* entry id */
+        lpGetIntegerIfValid(p, &valid_record);
+        if (!valid_record) return 0;
         p = next; if (!lpValidateNext(lp, &next, size)) return 0;
+        lpGetIntegerIfValid(p, &valid_record);
+        if (!valid_record) return 0;
         p = next; if (!lpValidateNext(lp, &next, size)) return 0;
 
         if (!(flags & STREAM_ITEM_FLAG_SAMEFIELDS)) {

src/t_string.cpp

@@ -802,6 +802,12 @@ void stralgoLCS(client *c) {
 
     {   // Scope variables below for the goto
     
+    /* Detect string truncation or later overflows. */
+    if (sdslen(a) >= UINT32_MAX-1 || sdslen(b) >= UINT32_MAX-1) {
+        addReplyError(c, "String too long for LCS");
+        goto cleanup;
+    }
+
     /* Compute the LCS using the vanilla dynamic programming technique of
      * building a table of LCS(x,y) substrings. */
     uint32_t alen = sdslen(a);
@@ -810,9 +816,19 @@ void stralgoLCS(client *c) {
     /* Setup an uint32_t array to store at LCS[i,j] the length of the
      * LCS A0..i-1, B0..j-1. Note that we have a linear array here, so
      * we index it as LCS[j+(blen+1)*j] */
-    uint32_t *lcs = (uint32_t*)zmalloc((alen+1)*(blen+1)*sizeof(uint32_t));
     #define LCS(A,B) lcs[(B)+((A)*(blen+1))]
 
+    /* Try to allocate the LCS table, and abort on overflow or insufficient memory. */
+    unsigned long long lcssize = (unsigned long long)(alen+1)*(blen+1); /* Can't overflow due to the size limits above. */
+    unsigned long long lcsalloc = lcssize * sizeof(uint32_t);
+    uint32_t *lcs = NULL;
+    if (lcsalloc < SIZE_MAX && lcsalloc / lcssize == sizeof(uint32_t))
+        lcs = (uint32_t *)ztrymalloc(lcsalloc);
+    if (!lcs) {
+        addReplyError(c, "Insufficient memory");
+        goto cleanup;
+    }
+
     /* Start building the LCS table. */
     for (uint32_t i = 0; i <= alen; i++) {
         for (uint32_t j = 0; j <= blen; j++) {

src/t_zset.cpp

@@ -1191,24 +1191,26 @@ void zsetConvert(robj *zobj, int encoding) {
         zs->dict = dictCreate(&zsetDictType,NULL);
         zs->zsl = zslCreate();
 
-        eptr = ziplistIndex(zl,0);
-        serverAssertWithInfo(NULL,zobj,eptr != NULL);
-        sptr = ziplistNext(zl,eptr);
-        serverAssertWithInfo(NULL,zobj,sptr != NULL);
+        if (ziplistLen(zl) > 0) {
+            eptr = ziplistIndex(zl,0);
+            serverAssertWithInfo(NULL,zobj,eptr != NULL);
+            sptr = ziplistNext(zl,eptr);
+            serverAssertWithInfo(NULL,zobj,sptr != NULL);
 
-        while (eptr != NULL) {
-            score = zzlGetScore(sptr);
-            serverAssertWithInfo(NULL,zobj,ziplistGet(eptr,&vstr,&vlen,&vlong));
-            if (vstr == NULL)
-                ele = sdsfromlonglong(vlong);
-            else
-                ele = sdsnewlen((char*)vstr,vlen);
+            while (eptr != NULL) {
+                score = zzlGetScore(sptr);
+                serverAssertWithInfo(NULL,zobj,ziplistGet(eptr,&vstr,&vlen,&vlong));
+                if (vstr == NULL)
+                    ele = sdsfromlonglong(vlong);
+                else
+                    ele = sdsnewlen((char*)vstr,vlen);
 
-            node = zslInsert(zs->zsl,score,ele);
-            serverAssert(dictAdd(zs->dict,ele,&node->score) == DICT_OK);
-            zzlNext(zl,&eptr,&sptr);
+                node = zslInsert(zs->zsl,score,ele);
+                serverAssert(dictAdd(zs->dict,ele,&node->score) == DICT_OK);
+                zzlNext(zl,&eptr,&sptr);
+            }
         }
-
+        
         zfree(zobj->m_ptr);
         zobj->m_ptr = zs;
         zobj->encoding = OBJ_ENCODING_SKIPLIST;
@@ -1242,14 +1244,15 @@ void zsetConvert(robj *zobj, int encoding) {
 }
 
 /* Convert the sorted set object into a ziplist if it is not already a ziplist
- * and if the number of elements and the maximum element size is within the
- * expected ranges. */
-void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen) {
+ * and if the number of elements and the maximum element size and total elements size
+ * are within the expected ranges. */
+void zsetConvertToZiplistIfNeeded(robj *zobj, size_t maxelelen, size_t totelelen) {
     if (zobj->encoding == OBJ_ENCODING_ZIPLIST) return;
     zset *set = (zset*)zobj->m_ptr;
 
     if (set->zsl->length <= g_pserver->zset_max_ziplist_entries &&
-        maxelelen <= g_pserver->zset_max_ziplist_value)
+        maxelelen <= g_pserver->zset_max_ziplist_value &&
+        ziplistSafeToAdd(NULL, totelelen))
             zsetConvert(zobj,OBJ_ENCODING_ZIPLIST);
 }
 
@@ -1370,20 +1373,28 @@ int zsetAdd(robj *zobj, double score, sds ele, int in_flags, int *out_flags, dou
             }
             return 1;
         } else if (!xx) {
-            /* Optimize: check if the element is too large or the list
+            /* check if the element is too large or the list
              * becomes too long *before* executing zzlInsert. */
-            zobj->m_ptr = zzlInsert((unsigned char*)zobj->m_ptr,ele,score);
-            if (zzlLength((unsigned char*)zobj->m_ptr) > g_pserver->zset_max_ziplist_entries ||
-                sdslen(ele) > g_pserver->zset_max_ziplist_value)
+            if (zzlLength((unsigned char*)ptrFromObj(zobj))+1 > g_pserver->zset_max_ziplist_entries ||
+                sdslen(ele) > g_pserver->zset_max_ziplist_value ||
+                !ziplistSafeToAdd((unsigned char *)ptrFromObj(zobj), sdslen(ele)))
+            {
                 zsetConvert(zobj,OBJ_ENCODING_SKIPLIST);
-            if (newscore) *newscore = score;
-            *out_flags |= ZADD_OUT_ADDED;
-            return 1;
+            } else {
+                zobj->m_ptr = zzlInsert((unsigned char *)ptrFromObj(zobj),ele,score);
+                if (newscore) *newscore = score;
+                *out_flags |= ZADD_OUT_ADDED;
+                return 1;
+            }
         } else {
             *out_flags |= ZADD_OUT_NOP;
             return 1;
         }
-    } else if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
+    }
+
+    /* Note that the above block handling ziplist would have either returned or
+     * converted the key to skiplist. */
+    if (zobj->encoding == OBJ_ENCODING_SKIPLIST) {
         zset *zs = (zset*)zobj->m_ptr;
         zskiplistNode *znode;
         dictEntry *de;
@@ -2360,7 +2371,7 @@ inline static void zunionInterAggregate(double *target, double val, int aggregat
     }
 }
 
-static int zsetDictGetMaxElementLength(dict *d) {
+static size_t zsetDictGetMaxElementLength(dict *d, size_t *totallen) {
     dictIterator *di;
     dictEntry *de;
     size_t maxelelen = 0;
@@ -2370,6 +2381,8 @@ static int zsetDictGetMaxElementLength(dict *d) {
     while((de = dictNext(di)) != NULL) {
         sds ele = (sds)dictGetKey(de);
         if (sdslen(ele) > maxelelen) maxelelen = sdslen(ele);
+        if (totallen)
+            (*totallen) += sdslen(ele);
     }
 
     dictReleaseIterator(di);
@@ -2377,7 +2390,7 @@ static int zsetDictGetMaxElementLength(dict *d) {
     return maxelelen;
 }
 
-static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) {
+static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) {
     /* DIFF Algorithm 1:
      *
      * We perform the diff by iterating all the elements of the first set,
@@ -2425,13 +2438,14 @@ static void zdiffAlgorithm1(zsetopsrc *src, long setnum, zset *dstzset, size_t *
             znode = zslInsert(dstzset->zsl,zval.score,tmp);
             dictAdd(dstzset->dict,tmp,&znode->score);
             if (sdslen(tmp) > *maxelelen) *maxelelen = sdslen(tmp);
+            (*totelelen) += sdslen(tmp);
         }
     }
     zuiClearIterator(&src[0]);
 }
 
 
-static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) {
+static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) {
     /* DIFF Algorithm 2:
      *
      * Add all the elements of the first set to the auxiliary set.
@@ -2485,7 +2499,7 @@ static void zdiffAlgorithm2(zsetopsrc *src, long setnum, zset *dstzset, size_t *
 
     /* Using this algorithm, we can't calculate the max element as we go,
      * we have to iterate through all elements to find the max one after. */
-    *maxelelen = zsetDictGetMaxElementLength(dstzset->dict);
+    *maxelelen = zsetDictGetMaxElementLength(dstzset->dict, totelelen);
 }
 
 static int zsetChooseDiffAlgorithm(zsetopsrc *src, long setnum) {
@@ -2522,14 +2536,14 @@ static int zsetChooseDiffAlgorithm(zsetopsrc *src, long setnum) {
     return (algo_one_work <= algo_two_work) ? 1 : 2;
 }
 
-static void zdiff(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen) {
+static void zdiff(zsetopsrc *src, long setnum, zset *dstzset, size_t *maxelelen, size_t *totelelen) {
     /* Skip everything if the smallest input is empty. */
     if (zuiLength(&src[0]) > 0) {
         int diff_algo = zsetChooseDiffAlgorithm(src, setnum);
         if (diff_algo == 1) {
-            zdiffAlgorithm1(src, setnum, dstzset, maxelelen);
+            zdiffAlgorithm1(src, setnum, dstzset, maxelelen, totelelen);
         } else if (diff_algo == 2) {
-            zdiffAlgorithm2(src, setnum, dstzset, maxelelen);
+            zdiffAlgorithm2(src, setnum, dstzset, maxelelen, totelelen);
         } else if (diff_algo != 0) {
             serverPanic("Unknown algorithm");
         }
@@ -2564,7 +2578,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
     zsetopsrc *src;
     zsetopval zval;
     sds tmp;
-    size_t maxelelen = 0;
+    size_t maxelelen = 0, totelelen = 0;
     robj *dstobj;
     zset *dstzset;
     zskiplistNode *znode;
@@ -2700,6 +2714,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
                     tmp = zuiNewSdsFromValue(&zval);
                     znode = zslInsert(dstzset->zsl,score,tmp);
                     dictAdd(dstzset->dict,tmp,&znode->score);
+                    totelelen += sdslen(tmp);
                     if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp);
                 }
             }
@@ -2736,6 +2751,7 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
                     /* Remember the longest single element encountered,
                      * to understand if it's possible to convert to ziplist
                      * at the end. */
+                     totelelen += sdslen(tmp);
                      if (sdslen(tmp) > maxelelen) maxelelen = sdslen(tmp);
                     /* Update the element with its initial score. */
                     dictSetKey(accumulator, de, tmp);
@@ -2770,14 +2786,14 @@ void zunionInterDiffGenericCommand(client *c, robj *dstkey, int numkeysIndex, in
         dictReleaseIterator(di);
         dictRelease(accumulator);
     } else if (op == SET_OP_DIFF) {
-        zdiff(src, setnum, dstzset, &maxelelen);
+        zdiff(src, setnum, dstzset, &maxelelen, &totelelen);
     } else {
         serverPanic("Unknown operator");
     }
 
     if (dstkey) {
         if (dstzset->zsl->length) {
-            zsetConvertToZiplistIfNeeded(dstobj, maxelelen);
+            zsetConvertToZiplistIfNeeded(dstobj, maxelelen, totelelen);
             setKey(c, c->db, dstkey, dstobj);
             addReplyLongLong(c, zsetLength(dstobj));
             notifyKeyspaceEvent(NOTIFY_ZSET,
@@ -3662,7 +3678,12 @@ void zrangeGenericCommand(zrange_result_handler *handler, int argc_start, int st
         lookupKeyWrite(c->db,key) :
         lookupKeyRead(c->db,key);
     if (zobj == nullptr) {
-        addReply(c,shared.emptyarray);
+        if (store) {
+            handler->beginResultEmission(handler);
+            handler->finalizeResultEmission(handler, 0);
+        } else {
+            addReply(c, shared.emptyarray);
+        }
         goto cleanup;
     }
 
@@ -3818,11 +3839,16 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
     }
 
     void *arraylen_ptr = addReplyDeferredLen(c);
-    long arraylen = 0;
+    long result_count = 0;
 
     /* We emit the key only for the blocking variant. */
     if (emitkey) addReplyBulk(c,key);
 
+    /* Respond with a single (flat) array in RESP2 or if countarg is not
+     * provided (returning a single element). In RESP3, when countarg is
+     * provided, use nested array.  */
+    int use_nested_array = c->resp > 2 && countarg != NULL;
+
     /* Remove the element. */
     do {
         if (zobj->encoding == OBJ_ENCODING_ZIPLIST) {
@@ -3865,16 +3891,19 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
         serverAssertWithInfo(c,zobj,zsetDel(zobj,ele));
         g_pserver->dirty++;
 
-        if (arraylen == 0) { /* Do this only for the first iteration. */
+        if (result_count == 0) { /* Do this only for the first iteration. */
             const char *events[2] = {"zpopmin","zpopmax"};
             notifyKeyspaceEvent(NOTIFY_ZSET,events[where],key,c->db->id);
             signalModifiedKey(c,c->db,key);
         }
 
+        if (use_nested_array) {
+            addReplyArrayLen(c,2);
+        }
         addReplyBulkCBuffer(c,ele,sdslen(ele));
         addReplyDouble(c,score);
         sdsfree(ele);
-        arraylen += 2;
+        ++result_count;
 
         /* Remove the key, if indeed needed. */
         if (zsetLength(zobj) == 0) {
@@ -3884,7 +3913,10 @@ void genericZpopCommand(client *c, robj **keyv, int keyc, int where, int emitkey
         }
     } while(--count);
 
-    setDeferredArrayLen(c,arraylen_ptr,arraylen + (emitkey != 0));
+    if (!use_nested_array) {
+        result_count *= 2;
+    }
+    setDeferredArrayLen(c,arraylen_ptr,result_count + (emitkey != 0));
 }
 
 /* ZPOPMIN key [<count>] */
@@ -3985,7 +4017,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
     int uniq = 1;
     robj_roptr zsetobj;
 
-    if ((zsetobj = lookupKeyReadOrReply(c, c->argv[1], shared.null[c->resp]))
+    if ((zsetobj = lookupKeyReadOrReply(c, c->argv[1], shared.emptyarray))
         == nullptr || checkType(c, zsetobj, OBJ_ZSET)) return;
     size = zsetLength(zsetobj);
 
@@ -4021,7 +4053,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
                     addReplyArrayLen(c,2);
                 addReplyBulkCBuffer(c, key, sdslen(key));
                 if (withscores)
-                    addReplyDouble(c, dictGetDoubleVal(de));
+                    addReplyDouble(c, *(double*)dictGetVal(de));
             }
         } else if (zsetobj->encoding == OBJ_ENCODING_ZIPLIST) {
             ziplistEntry *keys, *vals = NULL;
@@ -4170,7 +4202,7 @@ void zrandmemberWithCountCommand(client *c, long l, int withscores) {
     }
 }
 
-/* ZRANDMEMBER [<count> WITHSCORES] */
+/* ZRANDMEMBER key [<count> [WITHSCORES]] */
 void zrandmemberCommand(client *c) {
     long l;
     int withscores = 0;

src/tls.cpp

@@ -140,47 +140,6 @@ static void initCryptoLocks(void) {
 }
 #endif /* USE_CRYPTO_LOCKS */
 
-/**
- * OpenSSL global initialization and locking handling callbacks.
- * Note that this is only required for OpenSSL < 1.1.0.
- */
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000L
-#define USE_CRYPTO_LOCKS
-#endif
-
-#ifdef USE_CRYPTO_LOCKS
-
-static pthread_mutex_t *openssl_locks;
-
-static void sslLockingCallback(int mode, int lock_id, const char *f, int line) {
-    pthread_mutex_t *mt = openssl_locks + lock_id;
-
-    if (mode & CRYPTO_LOCK) {
-        pthread_mutex_lock(mt);
-    } else {
-        pthread_mutex_unlock(mt);
-    }
-
-    (void)f;
-    (void)line;
-}
-
-static void initCryptoLocks(void) {
-    unsigned i, nlocks;
-    if (CRYPTO_get_locking_callback() != NULL) {
-        /* Someone already set the callback before us. Don't destroy it! */
-        return;
-    }
-    nlocks = CRYPTO_num_locks();
-    openssl_locks = zmalloc(sizeof(*openssl_locks) * nlocks);
-    for (i = 0; i < nlocks; i++) {
-        pthread_mutex_init(openssl_locks + i, NULL);
-    }
-    CRYPTO_set_locking_callback(sslLockingCallback);
-}
-#endif /* USE_CRYPTO_LOCKS */
-
 void tlsInit(void) {
     /* Enable configuring OpenSSL using the standard openssl.cnf
      * OPENSSL_config()/OPENSSL_init_crypto() should be the first 
@@ -190,6 +149,8 @@ void tlsInit(void) {
      */
     #if OPENSSL_VERSION_NUMBER < 0x10100000L
     OPENSSL_config(NULL);
+    #elif OPENSSL_VERSION_NUMBER < 0x10101000L
+    OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL);
     #else
     OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG|OPENSSL_INIT_ATFORK, NULL);
     #endif
@@ -224,7 +185,8 @@ void tlsCleanup(void) {
         redis_tls_client_ctx = NULL;
     }
 
-    #if OPENSSL_VERSION_NUMBER >= 0x10100000L
+    #if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER)
+    // unavailable on LibreSSL
     OPENSSL_cleanup();
     #endif
 }

src/tracking.cpp

@@ -330,7 +330,7 @@ void trackingRememberKeyToBroadcast(client *c, char *keyname, size_t keylen) {
          * tree. This way we know who was the client that did the last
          * change to the key, and can avoid sending the notification in the
          * case the client is in NOLOOP mode. */
-        raxTryInsert(bs->keys,(unsigned char*)keyname,keylen,c,NULL);
+        raxInsert(bs->keys,(unsigned char*)keyname,keylen,c,NULL);
     }
     raxStop(&ri);
 }

src/ziplist.c

@@ -263,10 +263,21 @@
  * to stay there to signal that a full scan is needed to get the number of
  * items inside the ziplist. */
 #define ZIPLIST_INCR_LENGTH(zl,incr) { \
-    if (ZIPLIST_LENGTH(zl) < UINT16_MAX) \
+    if (intrev16ifbe(ZIPLIST_LENGTH(zl)) < UINT16_MAX) \
         ZIPLIST_LENGTH(zl) = intrev16ifbe(intrev16ifbe(ZIPLIST_LENGTH(zl))+incr); \
 }
 
+/* Don't let ziplists grow over 1GB in any case, don't wanna risk overflow in
+ * zlbytes*/
+#define ZIPLIST_MAX_SAFETY_SIZE (1<<30)
+int ziplistSafeToAdd(unsigned char* zl, size_t add) {
+    size_t len = zl? ziplistBlobLen(zl): 0;
+    if (len + add > ZIPLIST_MAX_SAFETY_SIZE)
+        return 0;
+    return 1;
+}
+
+
 /* We use this function to receive information about a ziplist entry.
  * Note that this is not how the data is actually encoded, is just what we
  * get filled by a function in order to operate more easily. */
@@ -709,7 +720,8 @@ unsigned char *ziplistNew(void) {
 }
 
 /* Resize the ziplist. */
-unsigned char *ziplistResize(unsigned char *zl, unsigned int len) {
+unsigned char *ziplistResize(unsigned char *zl, size_t len) {
+    assert(len < UINT32_MAX);
     zl = zrealloc(zl,len, MALLOC_SHARED);
     ZIPLIST_BYTES(zl) = intrev32ifbe(len);
     zl[len-1] = ZIP_END;
@@ -1070,6 +1082,9 @@ unsigned char *ziplistMerge(unsigned char **first, unsigned char **second) {
     /* Combined zl length should be limited within UINT16_MAX */
     zllength = zllength < UINT16_MAX ? zllength : UINT16_MAX;
 
+    /* larger values can't be stored into ZIPLIST_BYTES */
+    assert(zlbytes < UINT32_MAX);
+
     /* Save offset positions before we start ripping memory apart. */
     size_t first_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*first));
     size_t second_offset = intrev32ifbe(ZIPLIST_TAIL_OFFSET(*second));
@@ -1522,8 +1537,12 @@ int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep,
         count++;
     }
 
+    /* Make sure 'p' really does point to the end of the ziplist. */
+    if (p != zl + bytes - ZIPLIST_END_SIZE)
+        return 0;
+
     /* Make sure the <zltail> entry really do point to the start of the last entry. */
-    if (prev != ZIPLIST_ENTRY_TAIL(zl))
+    if (prev != NULL && prev != ZIPLIST_ENTRY_TAIL(zl))
         return 0;
 
     /* Check that the count in the header is correct */

src/ziplist.h

@@ -69,6 +69,7 @@ int ziplistValidateIntegrity(unsigned char *zl, size_t size, int deep,
 void ziplistRandomPair(unsigned char *zl, unsigned long total_count, ziplistEntry *key, ziplistEntry *val);
 void ziplistRandomPairs(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals);
 unsigned int ziplistRandomPairsUnique(unsigned char *zl, unsigned int count, ziplistEntry *keys, ziplistEntry *vals);
+int ziplistSafeToAdd(unsigned char* zl, size_t add);
 
 #ifdef REDIS_TEST
 int ziplistTest(int argc, char *argv[], int accurate);

src/zipmap.c

@@ -430,6 +430,9 @@ int zipmapValidateIntegrity(unsigned char *zm, size_t size, int deep) {
             return 0;
     }
 
+    /* check that the zipmap is not empty. */
+    if (count == 0) return 0;
+
     /* check that the count in the header is correct */
     if (zm[0] != ZIPMAP_BIGLEN && zm[0] != count)
         return 0;

src/zmalloc.cpp

@@ -243,7 +243,7 @@ void *ztryrealloc_usable(void *ptr, size_t size, size_t *usable) {
 #else
     realptr = (char*)ptr-PREFIX_SIZE;
     oldsize = *((size_t*)realptr);
-    newptr = realloc(realptr,size+PREFIX_SIZE);
+    newptr = realloc(realptr,size+PREFIX_SIZE, MALLOC_LOCAL);
     if (newptr == NULL) {
         if (usable) *usable = 0;
         return NULL;

src/zmalloc.h

@@ -42,7 +42,7 @@
     #define USE_MALLOC_CLASS 1
 #elif defined(USE_TCMALLOC)
 #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR))
-#include <google/tcmalloc.h>
+#include <gperftools/tcmalloc.h>
 #if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1)
 #define HAVE_MALLOC_SIZE 1
 #define zmalloc_size(p) tc_malloc_size(p)

tests/assets/default.conf

@@ -1,9 +1,9 @@
-# Redis configuration for testing.
+# KeyDB configuration for testing.
 
 always-show-logo yes
 notify-keyspace-events KEA
 daemonize no
-pidfile /var/run/redis.pid
+pidfile /var/run/keydb.pid
 port 6379
 timeout 0
 bind 127.0.0.1

tests/assets/minimal.conf

@@ -1,5 +1,6 @@
 # Minimal configuration for testing.
+bind 127.0.0.1
 always-show-logo yes
 daemonize no
-pidfile /var/run/redis.pid
+pidfile /var/run/keydb.pid
 loglevel verbose

tests/cluster/run.tcl

@@ -13,8 +13,9 @@ set ::tlsdir "../../tls"
 proc main {} {
     parse_options
     spawn_instance redis $::redis_base_port $::instances_count {
-        "cluster-enabled yes"
-        "appendonly yes"
+	"bind 127.0.0.1"
+	"cluster-enabled yes"
+	"appendonly yes"
 	"testmode yes"
 	"server-threads 3"
     }