update release packaging

don't use flags that aren't supported by certain versions of OPENSSL

.github/workflows/ci.yml

@@ -56,14 +56,3 @@ jobs:
           sudo apt-get -y install uuid-dev libcurl4-openssl-dev
           make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' MALLOC=libc -j2
 
-  build-ubuntu-32bit:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make
-      run: |
-          sudo dpkg --add-architecture i386
-          sudo apt-get update
-          sudo apt-get -y install gcc-multilib g++-multilib libc6-dev-i386 lib32z1 uuid-dev:i386 libcurl4-openssl-dev:i386
-          make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' 32bit -j2
-

.github/workflows/endurance.yml

@@ -2,7 +2,7 @@ name: Endurance
 
 on:
   schedule:
-    - cron: '30 * * * *'
+    - cron: '0 * * * *'
 
 jobs:
 

pkg/deb/conf/keydb.conf

@@ -261,7 +261,7 @@ tcp-keepalive 300
 
 # By default KeyDB does not run as a daemon. Use 'yes' if you need it.
 # Note that KeyDB will write a pid file in /var/run/keydb.pid when daemonized.
-daemonize no
+daemonize yes
 
 # If you run KeyDB from upstart or systemd, KeyDB can interact with your
 # supervision tree. Options:
@@ -284,7 +284,7 @@ daemonize no
 #
 # Creating a pid file is best effort: if KeyDB is not able to create it
 # nothing bad happens, the server will start and run normally.
-pidfile /var/run/keydb_6379.pid
+pidfile /var/run/keydb/keydb-server.pid
 
 # Specify the server verbosity level.
 # This can be one of:

pkg/deb/debian/control

@@ -6,6 +6,7 @@ Build-Depends:
  debhelper (>= 9~),
  dpkg-dev (>= 1.17.5),
  systemd,
+ libsystemd-dev <!nocheck>,
  procps <!nocheck>,
  build-essential <!nocheck>,
  tcl <!nocheck>,

pkg/deb/debian/copyright

@@ -1,16 +1,94 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Contact: John Sully <john@eqalpha.com>
-Upstream-Name: keydb-pro
-Source: https://github.com/JohnSully/KeyDB-Pro
+Upstream-Name: KeyDB
+Source: https://github.com/EQ-Alpha/KeyDB
 
 Files: *
-Copyright: © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
-Copyright © 2019, John Sully
-License: Proprietary
+Copyright: 
+ © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
+ © 2019-2021 John Sully
+ © 2019-2021 EQ Alpha Technology Ltd.
 
+License: BSD-3-clause
 
-Files: debian/*
-Copyright: © 2009 Chris Lamb <lamby@debian.org>
+Files:
+ src/rio.*
+ src/t_zset.c
+ src/ziplist.h
+ src/intset.*
+ src/redis-check-aof.c
+ deps/hiredis/*
+ deps/linenoise/*
+Copyright:
+ © 2009-2012 Pieter Noordhuis <pcnoordhuis@gmail.com>
+ © 2009-2012 Salvatore Sanfilippo <antirez@gmail.com>
+License: BSD-3-clause
+
+Files:
+ src/lzf.h
+ src/lzfP.h
+ src/lzf_d.c
+ src/lzf_c.c
+Copyright:
+ © 2000-2007 Marc Alexander Lehmann <schmorp@schmorp.de>
+ © 2009-2012 Salvatore Sanfilippo <antirez@gmail.com>
+License: BSD-2-clause
+
+Files: src/setproctitle.c
+Copyright:
+ © 2010 William Ahern
+ © 2013 Salvatore Sanfilippo
+ © 2013 Stam He
+License: BSD-3-clause
+
+Files: src/ae_evport.c
+Copyright: © 2012 Joyent, Inc.
+License: BSD-3-clause
+
+Files: src/ae_kqueue.c
+Copyright: © 2009 Harish Mallipeddi <harish.mallipeddi@gmail.com>
+License: BSD-3-clause
+
+Files: utils/install_server.sh
+Copyright: © 2011 Dvir Volk <dvirsk@gmail.com>
+License: BSD-3-clause
+
+Files: deps/jemalloc/*
+Copyright:
+ © 2002-2012 Jason Evans <jasone@canonware.com>
+ © 2007-2012 Mozilla Foundation
+ © 2009-2012 Facebook, Inc.
+License: BSD-3-clause
+
+Files: src/pqsort.*
+Copyright: © 1992-1993 The Regents of the University of California
+License: BSD-3-clause
+
+Files: deps/lua/*
+Copyright: © 1994-2012 Lua.org, PUC-Ri
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+Files: pkg/deb/debian/*
+Copyright:
+ © 2009 Chris Lamb <lamby@debian.org>
+ © 2020-2021 EQ Alpha Technology Ltd. <support@eqalpha.com>
 License: BSD-3-clause
 
 License: BSD-2-clause

pkg/deb/debian/keydb-tools.install

@@ -5,3 +5,4 @@ src/keydb-check-rdb	/usr/bin
 src/keydb-cli		/usr/bin
 src/keydb-server	/usr/bin
 src/keydb-sentinel	/usr/bin
+src/keydb-diagnostic-tool	/usr/bin

pkg/deb/debian/rules

@@ -3,6 +3,7 @@
 include /usr/share/dpkg/buildflags.mk
 
 export BUILD_TLS=yes
+export USE_SYSTEMD=yes
 export CFLAGS CPPFLAGS LDFLAGS
 export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 export DEB_LDFLAGS_MAINT_APPEND = -ldl -latomic $(LUA_LDFLAGS)

pkg/deb/debian_dh9/control

@@ -6,6 +6,7 @@ Build-Depends:
  debhelper (>= 9~),
  dpkg-dev (>= 1.17.5),
  systemd,
+ libsystemd-dev <!nocheck>,
  procps <!nocheck>,
  build-essential <!nocheck>,
  tcl <!nocheck>,

pkg/deb/debian_dh9/copyright

@@ -1,11 +1,14 @@
 Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Upstream-Contact: John Sully <john@eqalpha.com>
-Upstream-Name: keydb
-Source: https://github.com/JohnSully/KeyDB
+Upstream-Name: KeyDB
+Source: https://github.com/EQ-Alpha/KeyDB
 
 Files: *
-Copyright: © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
-Copyright © 2019, John Sully
+Copyright: 
+ © 2006-2014 Salvatore Sanfilippo <antirez@gmail.com>
+ © 2019-2021 John Sully
+ © 2019-2021 EQ Alpha Technology Ltd.
+
 License: BSD-3-clause
 
 Files:
@@ -82,8 +85,10 @@ License: MIT
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE.
 
-Files: debian/*
-Copyright: © 2009 Chris Lamb <lamby@debian.org>
+Files: pkg/deb/debian_dh9/*
+Copyright: 
+ © 2009 Chris Lamb <lamby@debian.org>
+ © 2020-2021 EQ Alpha Technology Ltd. <support@eqalpha.com>
 License: BSD-3-clause
 
 License: BSD-2-clause

pkg/deb/debian_dh9/keydb-tools.install

@@ -5,3 +5,4 @@ src/keydb-check-aof	/usr/bin
 src/keydb-check-rdb	/usr/bin
 src/keydb-cli		/usr/bin
 src/keydb-sentinel	/usr/bin
+src/keydb-diagnostic-tool	/usr/bin

pkg/deb/debian_dh9/rules

@@ -9,6 +9,7 @@ include /usr/share/dpkg/buildflags.mk
 #LUA_LDFLAGS = $(addprefix -llua5.1-,$(LUA_LIBS_DEBIAN)) $(addprefix ../deps/lua/src/,$(LUA_OBJECTS))
 
 export BUILD_TLS=yes
+export USE_SYSTEMD=yes
 export CFLAGS CPPFLAGS LDFLAGS
 export DEB_BUILD_MAINT_OPTIONS = hardening=+all
 export DEB_LDFLAGS_MAINT_APPEND = -ldl -latomic $(LUA_LDFLAGS)

pkg/deb/master_changelog

@@ -1,3 +1,48 @@
+keydb (6:6.2.0-1distribution_placeholder) codename_placeholder; urgency=medium
+
+  * Removed unused command line options(-a, -d, -P, -r, -q, --csv, -l, -I, -e, --precision, --cluster, --enable-tracking)
+  * --ms option changed to --time, added --clients, --host, --port, and --threads options
+  * Added keydb-diagnostic-tool
+  * Improved test reliability
+  * Fixed memory leak in keydb-benchmark
+  * Fixed a number of race conditions
+  * Fixed OSX build issues
+  * Feature parity with Redis 6.2.3
+  * Fixed Issues: #323 #325 #276
+
+ -- Ben Schermel <ben@eqalpha.com>  Thu, 12 Aug 2021 20:00:37 +0000
+
+keydb (6:6.0.18-1distribution_placeholder) codename_placeholder; urgency=medium
+
+  * 6.0.18 - This release contains a number of improvements to stability and performance. 
+  * Additional tests were added with improvements to existing test reliability
+  * Merged Redis 6.0.10 into KeyDB
+  * Issues resolved in this release: #214, #222, #238, #257, #273, #276, #285
+
+ -- Ben Schermel <ben@eqalpha.com>  Fri, 26 Mar 2021 16:00:00 +0000
+
+keydb (6:6.0.16-1distribution_placeholder) codename_placeholder; urgency=medium
+
+  * 6.0.16 - This release contains significant performance improvements as well as fixes to the following issues: #234, #236, #233, #207, #229, #231
+  * Reduced memory consumption when Active Replication is not used
+  * Reduced CPU consumption
+  * Improved TLS performance
+  * replica-quorum config for multi-master.Intended to be used with "replica-serve-stale-data no", if at least N replicas are connected we will consider our master link up and serve data.
+  * Subkey expires performance improvement for big sets
+
+ -- Ben Schermel <ben@eqalpha.com>  Mon, 28 Sep 2020 16:00:00 +0000
+
+keydb (6:6.0.13-1distribution_placeholder) codename_placeholder; urgency=medium
+
+  * 6.0.13 New features added: KEYDB.MEXISTS, REPLICAOF REMOVE, multi-master-no-forward configuration option
+  * KEYDB.MEXISTS - return exactly which keys exist in the database not just the count
+  * REPLICAOF REMOVE - Remove specific masters in a multi master setup
+  * multi-master-no-forward configuration option - Reduce network traffic in mesh topology multimaster setups
+  * Redis 6.0.5 feature parity
+  * Fixed Issues: 194 Bad directive or wrong number of arguments error on 'lazyfree-lazy-user-del no'; 209 Timeout using modules; 210 Databases are not merged on multi-master sync; 211 rpm packages now signed; 212 rpm file permissions updated
+
+ -- Ben Schermel <ben@eqalpha.com>  Mon, 13 Jul 2020 8:00:00 +0000
+
 keydb (6:6.0.9-1distribution_placeholder) codename_placeholder; urgency=medium
 
   * 6.0.9 Addressing issues 187 - cpu lockup with subkey expire, 190 - missing sentinel binary in keydb-tools. 

pkg/rpm/generate_rpms.sh

@@ -25,7 +25,13 @@ mkdir -p $DIR/keydb_build/keydb_rpm/var/log/keydb
 
 # move binaries to bin
 rm $DIR/keydb_build/keydb_rpm/usr/bin/*
-cp $DIR/../../src/keydb-* $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-server $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-sentinel $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-cli $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-benchmark $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-check-aof $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-check-rdb $DIR/keydb_build/keydb_rpm/usr/bin/
+cp $DIR/../../src/keydb-diagnostic-tool $DIR/keydb_build/keydb_rpm/usr/bin/
 
 # update spec file with build info
 sed -i '2d' $DIR/keydb_build/keydb.spec

src/Makefile

@@ -70,7 +70,7 @@ endif
 # Do we use our assembly spinlock? X64 only
 ifeq ($(uname_S),Linux)
 ifeq ($(uname_M),x86_64)
-ifneq ($(TARGET32), true)
+ifneq ($(@@),32bit)
 ifeq ($(USEASM),true)
 	ASM_OBJ+= fastlock_x64.o
 	CFLAGS+= -DASM_SPINLOCK
@@ -484,7 +484,7 @@ bench: $(REDIS_BENCHMARK_NAME)
 	@echo ""
 	@echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386"
 	@echo ""
-	$(MAKE) CXXFLAGS="-m32" CFLAGS="-m32" LDFLAGS="-m32" TARGET32=true
+	$(MAKE) CXXFLAGS="-m32" CFLAGS="-m32" LDFLAGS="-m32"
 
 gcov:
 	$(MAKE) KEYDB_CXXFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" KEYDB_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" KEYDB_LDFLAGS="-fprofile-arcs -ftest-coverage"

src/networking.cpp

@@ -2437,6 +2437,8 @@ void processInputBuffer(client *c, int callFlags) {
 
         if (c->flags & CLIENT_EXECUTING_COMMAND) break;
 
+        if (c->flags & CLIENT_EXECUTING_COMMAND) break;
+
         /* Don't process input from the master while there is a busy script
          * condition on the replica. We want just to accumulate the replication
          * stream (instead of replying -BUSY like we do with other clients) and

src/rio.h

@@ -63,7 +63,7 @@ struct _rio {
     uint64_t cksum, flags;
 
     /* number of keys loaded since last rdbLoadProgressCallback */
-    unsigned long int keys_since_last_callback;
+    long int keys_since_last_callback;
 
     /* number of bytes read or written */
     size_t processed_bytes;

src/semiorderedset.h

@@ -247,8 +247,8 @@ public:
         /* Generate human readable stats. */
         l += snprintf(buf+l,bufsize-l,
             "semiordered set stats:\n"
-            " table size: %zu\n"
-            " number of slots: %zu\n"
+            " table size: %ld\n"
+            " number of slots: %ld\n"
             " used slots: %ld\n"
             " max chain length: %ld\n"
             " avg chain length (counted): %.02f\n"

src/server.h

@@ -901,7 +901,7 @@ public:
     void addref() const { refcount.fetch_add(1, std::memory_order_relaxed); }
     unsigned release() const { return refcount.fetch_sub(1, std::memory_order_seq_cst) & ~(1U << 31); }
 } robj;
-static_assert(sizeof(redisObject) <= 16, "object size is critical, don't increase");
+static_assert(sizeof(redisObject) == 16, "object size is critical, don't increase");
 
 class redisObjectStack : public redisObjectExtended, public redisObject
 {

src/tls.cpp

@@ -140,47 +140,6 @@ static void initCryptoLocks(void) {
 }
 #endif /* USE_CRYPTO_LOCKS */
 
-/**
- * OpenSSL global initialization and locking handling callbacks.
- * Note that this is only required for OpenSSL < 1.1.0.
- */
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000L
-#define USE_CRYPTO_LOCKS
-#endif
-
-#ifdef USE_CRYPTO_LOCKS
-
-static pthread_mutex_t *openssl_locks;
-
-static void sslLockingCallback(int mode, int lock_id, const char *f, int line) {
-    pthread_mutex_t *mt = openssl_locks + lock_id;
-
-    if (mode & CRYPTO_LOCK) {
-        pthread_mutex_lock(mt);
-    } else {
-        pthread_mutex_unlock(mt);
-    }
-
-    (void)f;
-    (void)line;
-}
-
-static void initCryptoLocks(void) {
-    unsigned i, nlocks;
-    if (CRYPTO_get_locking_callback() != NULL) {
-        /* Someone already set the callback before us. Don't destroy it! */
-        return;
-    }
-    nlocks = CRYPTO_num_locks();
-    openssl_locks = zmalloc(sizeof(*openssl_locks) * nlocks);
-    for (i = 0; i < nlocks; i++) {
-        pthread_mutex_init(openssl_locks + i, NULL);
-    }
-    CRYPTO_set_locking_callback(sslLockingCallback);
-}
-#endif /* USE_CRYPTO_LOCKS */
-
 void tlsInit(void) {
     /* Enable configuring OpenSSL using the standard openssl.cnf
      * OPENSSL_config()/OPENSSL_init_crypto() should be the first 
@@ -190,6 +149,8 @@ void tlsInit(void) {
      */
     #if OPENSSL_VERSION_NUMBER < 0x10100000L
     OPENSSL_config(NULL);
+    #elif OPENSSL_VERSION_NUMBER < 0x10101000L
+    OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG, NULL);
     #else
     OPENSSL_init_crypto(OPENSSL_INIT_LOAD_CONFIG|OPENSSL_INIT_ATFORK, NULL);
     #endif

src/version.h

@@ -1,4 +1,4 @@
-#define KEYDB_REAL_VERSION "255.255.255"
-#define KEYDB_VERSION_NUM 0x00ffffff
+#define KEYDB_REAL_VERSION "6.2.0"
+#define KEYDB_VERSION_NUM 0x00060200
 extern const char *KEYDB_SET_VERSION;   // Unlike real version, this can be overriden by the config
 

tests/unit/scripting.tcl

@@ -485,6 +485,21 @@ start_server {tags {"scripting"}} {
     }
 
 
+    test {EVAL with pipelined command (No crash)} {
+        r flushall
+        r config set lua-time-limit 1
+        set rd [redis_deferring_client]
+        $rd eval {for i=1,1000000 do redis.call('set', i, 'sdfdsfd') end} 0
+        $rd set testkey foo
+        $rd get testkey
+        after 1200
+        catch {r echo "foo"} err
+        assert_match {BUSY*} $err
+        $rd read
+        $rd close
+    }
+
+
     test {EVAL timeout from AOF} {
         # generate a long running script that is propagated to the AOF as script
         # make sure that the script times out during loading