Initial implementation of TSXRTM

This reverts commit 41b678814b.

src/IStorage.h

@@ -1,14 +0,0 @@
-#pragma once
-#include <functional>
-
-class IStorage
-{
-public:
-    typedef std::function<void(const char *, size_t, const void *, size_t)> callback;
-
-    virtual void insert(const char *key, size_t cchKey, void *data, size_t cb) = 0;
-    virtual void erase(const char *key, size_t cchKey) = 0;
-    virtual void retrieve(const char *key, size_t cchKey, bool fDelete, callback fn) = 0;
-    virtual size_t clear() = 0;
-    virtual void enumerate(callback fn) = 0;
-};

src/Makefile

@@ -47,8 +47,6 @@ endif
 USEASM?=true
 
 ifneq ($(SANITIZE),)
-	CC=clang
-	CXX=clang++
 	CFLAGS+= -fsanitize=$(SANITIZE) -DSANITIZE
 	CXXFLAGS+= -fsanitize=$(SANITIZE) -DSANITIZE
 	LDFLAGS+= -fsanitize=$(SANITIZE)
@@ -190,7 +188,7 @@ endif
 
 REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS)
 REDIS_CXX=$(QUIET_CC)$(CC) $(FINAL_CXXFLAGS)
-REDIS_NASM=$(QUIET_CC)nasm -felf64 
+KEYDB_AS=$(QUIET_CC) as --64 -g
 REDIS_LD=$(QUIET_LINK)$(CXX) $(FINAL_LDFLAGS)
 REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL)
 
@@ -295,7 +293,7 @@ dict-benchmark: dict.cpp zmalloc.cpp sds.c siphash.c
 	$(REDIS_CXX) -c $<
 
 %.o: %.asm .make-prerequisites
-	$(REDIS_NASM) $<
+	$(KEYDB_AS) $< -o $@
 
 clean:
 	rm -rf $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_RDB_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html Makefile.dep dict-benchmark

src/ae.cpp

@@ -80,11 +80,11 @@ public:
 mutex_wrapper g_lock;
 
 #else
-fastlock g_lock;
+fastlock g_lock("AE (global)");
 #endif
 thread_local aeEventLoop *g_eventLoopThisThread = NULL;
 
-#define AE_ASSERT(x) if (!(x)) do { fprintf(stderr, "AE_ASSER FAILURE\n"); *((volatile int*)0) = 1; } while(0)
+#define AE_ASSERT(x) if (!(x)) do { fprintf(stderr, "AE_ASSERT FAILURE %s: %d\n", __FILE__, __LINE__); *((volatile int*)0) = 1; } while(0)
 
 /* Include the best multiplexing layer supported by this system.
  * The following should be ordered by performances, descending. */
@@ -327,7 +327,7 @@ aeEventLoop *aeCreateEventLoop(int setsize) {
     for (i = 0; i < setsize; i++)
         eventLoop->events[i].mask = AE_NONE;
 
-    fastlock_init(&eventLoop->flock);
+    fastlock_init(&eventLoop->flock, "event loop");
     int rgfd[2];
     if (pipe(rgfd) < 0)
         goto err;

src/aof.cpp

@@ -264,7 +264,7 @@ int startAppendOnly(void) {
             strerror(errno));
         return C_ERR;
     }
-    if (g_pserver->FRdbSaveInProgress()) {
+    if (g_pserver->rdb_child_pid != -1) {
         g_pserver->aof_rewrite_scheduled = 1;
         serverLog(LL_WARNING,"AOF was enabled but there is already a child process saving an RDB file on disk. An AOF background was scheduled to start when possible.");
     } else {
@@ -397,7 +397,7 @@ void flushAppendOnlyFile(int force) {
      * useful for graphing / monitoring purposes. */
     if (sync_in_progress) {
         latencyAddSampleIfNeeded("aof-write-pending-fsync",latency);
-    } else if (g_pserver->aof_child_pid != -1 || g_pserver->FRdbSaveInProgress()) {
+    } else if (g_pserver->aof_child_pid != -1 || g_pserver->rdb_child_pid != -1) {
         latencyAddSampleIfNeeded("aof-write-active-child",latency);
     } else {
         latencyAddSampleIfNeeded("aof-write-alone",latency);
@@ -494,7 +494,7 @@ try_fsync:
     /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are
      * children doing I/O in the background. */
     if (g_pserver->aof_no_fsync_on_rewrite &&
-        (g_pserver->aof_child_pid != -1 || g_pserver->FRdbSaveInProgress()))
+        (g_pserver->aof_child_pid != -1 || g_pserver->rdb_child_pid != -1))
             return;
 
     /* Perform the fsync if needed. */
@@ -678,7 +678,7 @@ client *createFakeClient(void) {
     c->puser = NULL;
     listSetFreeMethod(c->reply,freeClientReplyValue);
     listSetDupMethod(c->reply,dupClientReplyValue);
-    fastlock_init(&c->lock);
+    fastlock_init(&c->lock, "fake client");
     fastlock_lock(&c->lock);
     initClientMultiState(c);
     return c;
@@ -1300,46 +1300,52 @@ ssize_t aofReadDiffFromParent(void) {
 
 int rewriteAppendOnlyFileRio(rio *aof) {
     dictIterator *di = NULL;
+    dictEntry *de;
     size_t processed = 0;
     int j;
 
     for (j = 0; j < cserver.dbnum; j++) {
         char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
         redisDb *db = g_pserver->db+j;
-        if (db->size() == 0) continue;
+        dict *d = db->pdict;
+        if (dictSize(d) == 0) continue;
+        di = dictGetSafeIterator(d);
 
         /* SELECT the new DB */
         if (rioWrite(aof,selectcmd,sizeof(selectcmd)-1) == 0) goto werr;
         if (rioWriteBulkLongLong(aof,j) == 0) goto werr;
 
         /* Iterate this DB writing every entry */
-        bool fComplete = db->iterate([&](const char *keystr, robj *o)->bool{
-            robj key;
+        while((de = dictNext(di)) != NULL) {
+            sds keystr;
+            robj key, *o;
 
-            initStaticStringObject(key,(sds)keystr);
+            keystr = (sds)dictGetKey(de);
+            o = (robj*)dictGetVal(de);
+            initStaticStringObject(key,keystr);
 
-            expireEntry *pexpire = db->getExpire(&key);
+            expireEntry *pexpire = getExpire(db,&key);
 
             /* Save the key and associated value */
             if (o->type == OBJ_STRING) {
                 /* Emit a SET command */
                 char cmd[]="*3\r\n$3\r\nSET\r\n";
-                if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) return false;
+                if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) goto werr;
                 /* Key and value */
-                if (rioWriteBulkObject(aof,&key) == 0) return false;
-                if (rioWriteBulkObject(aof,o) == 0) return false;
+                if (rioWriteBulkObject(aof,&key) == 0) goto werr;
+                if (rioWriteBulkObject(aof,o) == 0) goto werr;
             } else if (o->type == OBJ_LIST) {
-                if (rewriteListObject(aof,&key,o) == 0) return false;
+                if (rewriteListObject(aof,&key,o) == 0) goto werr;
             } else if (o->type == OBJ_SET) {
-                if (rewriteSetObject(aof,&key,o) == 0) return false;
+                if (rewriteSetObject(aof,&key,o) == 0) goto werr;
             } else if (o->type == OBJ_ZSET) {
-                if (rewriteSortedSetObject(aof,&key,o) == 0) return false;
+                if (rewriteSortedSetObject(aof,&key,o) == 0) goto werr;
             } else if (o->type == OBJ_HASH) {
-                if (rewriteHashObject(aof,&key,o) == 0) return false;
+                if (rewriteHashObject(aof,&key,o) == 0) goto werr;
             } else if (o->type == OBJ_STREAM) {
-                if (rewriteStreamObject(aof,&key,o) == 0) return false;
+                if (rewriteStreamObject(aof,&key,o) == 0) goto werr;
             } else if (o->type == OBJ_MODULE) {
-                if (rewriteModuleObject(aof,&key,o) == 0) return false;
+                if (rewriteModuleObject(aof,&key,o) == 0) goto werr;
             } else {
                 serverPanic("Unknown object type");
             }
@@ -1349,17 +1355,17 @@ int rewriteAppendOnlyFileRio(rio *aof) {
                     if (subExpire.subkey() == nullptr)
                     {
                         char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n";
-                        if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) return false;
-                        if (rioWriteBulkObject(aof,&key) == 0) return false;
+                        if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(aof,&key) == 0) goto werr;
                     }
                     else
                     {
                         char cmd[]="*4\r\n$12\r\nEXPIREMEMBER\r\n";
-                        if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) return false;
-                        if (rioWriteBulkObject(aof,&key) == 0) return false;
-                        if (rioWrite(aof,subExpire.subkey(),sdslen(subExpire.subkey())) == 0) return false;
+                        if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) goto werr;
+                        if (rioWriteBulkObject(aof,&key) == 0) goto werr;
+                        if (rioWrite(aof,subExpire.subkey(),sdslen(subExpire.subkey())) == 0) goto werr;
                     }
-                    if (rioWriteBulkLongLong(aof,subExpire.when()) == 0) return false; // common
+                    if (rioWriteBulkLongLong(aof,subExpire.when()) == 0) goto werr; // common
                 }
             }
             /* Read some diff from the parent process from time to time. */
@@ -1367,10 +1373,9 @@ int rewriteAppendOnlyFileRio(rio *aof) {
                 processed = aof->processed_bytes;
                 aofReadDiffFromParent();
             }
-            return true;
-        });
-        if (!fComplete)
-            goto werr;
+        }
+        dictReleaseIterator(di);
+        di = NULL;
     }
     return C_OK;
 
@@ -1411,12 +1416,7 @@ int rewriteAppendOnlyFile(char *filename) {
 
     if (g_pserver->aof_use_rdb_preamble) {
         int error;
-        std::vector<redisDbPersistentData*> vecpdb;
-        for (int idb = 0; idb < cserver.dbnum; ++idb)
-        {
-            vecpdb.push_back(&g_pserver->db[idb]);
-        }
-        if (rdbSaveRio(&aof,vecpdb.data(),&error,RDB_SAVE_AOF_PREAMBLE,NULL) == C_ERR) {
+        if (rdbSaveRio(&aof,&error,RDB_SAVE_AOF_PREAMBLE,NULL) == C_ERR) {
             errno = error;
             goto werr;
         }
@@ -1593,7 +1593,7 @@ int rewriteAppendOnlyFileBackground(void) {
     pid_t childpid;
     long long start;
 
-    if (g_pserver->aof_child_pid != -1 || g_pserver->FRdbSaveInProgress()) return C_ERR;
+    if (g_pserver->aof_child_pid != -1 || g_pserver->rdb_child_pid != -1) return C_ERR;
     if (aofCreatePipes() != C_OK) return C_ERR;
     openChildInfoPipe();
     start = ustime();
@@ -1652,7 +1652,7 @@ int rewriteAppendOnlyFileBackground(void) {
 void bgrewriteaofCommand(client *c) {
     if (g_pserver->aof_child_pid != -1) {
         addReplyError(c,"Background append only file rewriting already in progress");
-    } else if (g_pserver->FRdbSaveInProgress()) {
+    } else if (g_pserver->rdb_child_pid != -1) {
         g_pserver->aof_rewrite_scheduled = 1;
         addReplyStatus(c,"Background append only file rewriting scheduled");
     } else if (rewriteAppendOnlyFileBackground() == C_OK) {

src/cluster.cpp

@@ -617,6 +617,17 @@ clusterLink *createClusterLink(clusterNode *node) {
  * This function will just make sure that the original node associated
  * with this link will have the 'link' field set to NULL. */
 void freeClusterLink(clusterLink *link) {
+    if (ielFromEventLoop(serverTL->el) != IDX_EVENT_LOOP_MAIN)
+    {
+        // we can't perform this operation on this thread, queue it on the main thread
+        if (link->node)
+            link->node->link = NULL;
+        link->node = nullptr;
+        aePostFunction(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, [link]{
+            freeClusterLink(link);
+        });
+        return;
+    }
     if (link->fd != -1) {
         aeDeleteFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, link->fd, AE_READABLE|AE_WRITABLE);
     }
@@ -2139,21 +2150,35 @@ void handleLinkIOError(clusterLink *link) {
  * consumed by write(). We don't try to optimize this for speed too much
  * as this is a very low traffic channel. */
 void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
+    serverAssert(ielFromEventLoop(el) == IDX_EVENT_LOOP_MAIN);
     clusterLink *link = (clusterLink*) privdata;
     ssize_t nwritten;
     UNUSED(el);
     UNUSED(mask);
 
-    nwritten = write(fd, link->sndbuf, sdslen(link->sndbuf));
+    // We're about to release the lock, so the link's sndbuf needs to be owned fully by us
+    //  allocate a new one in case anyone tries to write while we're waiting
+    sds sndbuf = link->sndbuf;
+    link->sndbuf = sdsempty();
+
+    aeReleaseLock();
+    nwritten = write(fd, sndbuf, sdslen(sndbuf));
+    aeAcquireLock();
+
     if (nwritten <= 0) {
         serverLog(LL_DEBUG,"I/O error writing to node link: %s",
             (nwritten == -1) ? strerror(errno) : "short write");
+        sdsfree(sndbuf);
         handleLinkIOError(link);
         return;
     }
-    sdsrange(link->sndbuf,nwritten,-1);
+    sdsrange(sndbuf,nwritten,-1);
+    // Restore our send buffer, ensuring any unsent data is first
+    sndbuf = sdscat(sndbuf, link->sndbuf);
+    sdsfree(link->sndbuf);
+    link->sndbuf = sndbuf;
     if (sdslen(link->sndbuf) == 0)
-        aeDeleteFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el, link->fd, AE_WRITABLE);
+        aeDeleteFileEvent(el, link->fd, AE_WRITABLE);
 }
 
 /* Read data. Try to read the first field of the header first to check the
@@ -2228,9 +2253,10 @@ void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) {
  * the link to be invalidated, so it is safe to call this function
  * from event handlers that will do stuff with the same link later. */
 void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) {
+    serverAssert(GlobalLocksAcquired());
     if (sdslen(link->sndbuf) == 0 && msglen != 0)
-        aeCreateFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el,link->fd,AE_WRITABLE|AE_BARRIER,
-                    clusterWriteHandler,link);
+        aeCreateRemoteFileEvent(g_pserver->rgthreadvar[IDX_EVENT_LOOP_MAIN].el,link->fd,AE_WRITABLE|AE_BARRIER,
+                    clusterWriteHandler,link,false);
 
     link->sndbuf = sdscatlen(link->sndbuf, msg, msglen);
 
@@ -3284,7 +3310,7 @@ void clusterHandleSlaveMigration(int max_slaves) {
 void resetManualFailover(void) {
     if (g_pserver->cluster->mf_end && clientsArePaused()) {
         g_pserver->clients_pause_end_time = 0;
-        clientsArePaused(); /* Just use the side effect of the function. */
+        unpauseClientsIfNecessary();
     }
     g_pserver->cluster->mf_end = 0; /* No manual failover in progress. */
     g_pserver->cluster->mf_can_start = 0;
@@ -3910,7 +3936,7 @@ int verifyClusterConfigWithData(void) {
 
     /* Make sure we only have keys in DB0. */
     for (j = 1; j < cserver.dbnum; j++) {
-        if (g_pserver->db[j].size()) return C_ERR;
+        if (dictSize(g_pserver->db[j].pdict)) return C_ERR;
     }
 
     /* Check that all the slots we see populated memory have a corresponding
@@ -4154,57 +4180,76 @@ void clusterReplyMultiBulkSlots(client *c) {
     dictIterator *di = dictGetSafeIterator(g_pserver->cluster->nodes);
     while((de = dictNext(di)) != NULL) {
         clusterNode *node = (clusterNode*)dictGetVal(de);
-        int j = 0, start = -1;
+        int start = -1;
 
         /* Skip slaves (that are iterated when producing the output of their
          * master) and  masters not serving any slot. */
         if (!nodeIsMaster(node) || node->numslots == 0) continue;
+        
+        static_assert((CLUSTER_SLOTS % (sizeof(uint32_t)*8)) == 0, "code below assumes the bitfield is a multiple of sizeof(unsinged)");
 
-        for (j = 0; j < CLUSTER_SLOTS; j++) {
-            int bit, i;
-
-            if ((bit = clusterNodeGetSlotBit(node,j)) != 0) {
-                if (start == -1) start = j;
+        for (unsigned iw = 0; iw < (CLUSTER_SLOTS/sizeof(uint32_t)/8); ++iw)
+        {
+            uint32_t wordCur = reinterpret_cast<uint32_t*>(node->slots)[iw];
+            if (iw != ((CLUSTER_SLOTS/sizeof(uint32_t)/8)-1))
+            {
+                if (start == -1 && wordCur == 0)
+                    continue;
+                if (start != -1 && (wordCur+1)==0)
+                    continue;
             }
-            if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) {
-                int nested_elements = 3; /* slots (2) + master addr (1). */
-                void *nested_replylen = addReplyDeferredLen(c);
 
-                if (bit && j == CLUSTER_SLOTS-1) j++;
-
-                /* If slot exists in output map, add to it's list.
-                 * else, create a new output map for this slot */
-                if (start == j-1) {
-                    addReplyLongLong(c, start); /* only one slot; low==high */
-                    addReplyLongLong(c, start);
-                } else {
-                    addReplyLongLong(c, start); /* low */
-                    addReplyLongLong(c, j-1);   /* high */
+            unsigned ibitStartLoop = iw*sizeof(uint32_t)*8;
+    
+            for (unsigned j = ibitStartLoop; j < (iw+1)*sizeof(uint32_t)*8; j++) {
+                int i;
+                int bit = (int)(wordCur & 1);
+                wordCur >>= 1;
+                if (bit != 0) {
+                    if (start == -1) start = j;
                 }
-                start = -1;
+                if (start != -1 && (!bit || j == CLUSTER_SLOTS-1)) {
+                    int nested_elements = 3; /* slots (2) + master addr (1). */
+                    void *nested_replylen = addReplyDeferredLen(c);
 
-                /* First node reply position is always the master */
-                addReplyArrayLen(c, 3);
-                addReplyBulkCString(c, node->ip);
-                addReplyLongLong(c, node->port);
-                addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN);
+                    if (bit && j == CLUSTER_SLOTS-1) j++;
 
-                /* Remaining nodes in reply are replicas for slot range */
-                for (i = 0; i < node->numslaves; i++) {
-                    /* This loop is copy/pasted from clusterGenNodeDescription()
-                     * with modifications for per-slot node aggregation */
-                    if (nodeFailed(node->slaves[i])) continue;
+                    /* If slot exists in output map, add to it's list.
+                    * else, create a new output map for this slot */
+                    if (start == j-1) {
+                        addReplyLongLong(c, start); /* only one slot; low==high */
+                        addReplyLongLong(c, start);
+                    } else {
+                        addReplyLongLong(c, start); /* low */
+                        addReplyLongLong(c, j-1);   /* high */
+                    }
+                    start = -1;
+
+                    /* First node reply position is always the master */
                     addReplyArrayLen(c, 3);
-                    addReplyBulkCString(c, node->slaves[i]->ip);
-                    addReplyLongLong(c, node->slaves[i]->port);
-                    addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN);
-                    nested_elements++;
+                    addReplyBulkCString(c, node->ip);
+                    addReplyLongLong(c, node->port);
+                    addReplyBulkCBuffer(c, node->name, CLUSTER_NAMELEN);
+
+                    /* Remaining nodes in reply are replicas for slot range */
+                    for (i = 0; i < node->numslaves; i++) {
+                        /* This loop is copy/pasted from clusterGenNodeDescription()
+                        * with modifications for per-slot node aggregation */
+                        if (nodeFailed(node->slaves[i])) continue;
+                        addReplyArrayLen(c, 3);
+                        addReplyBulkCString(c, node->slaves[i]->ip);
+                        addReplyLongLong(c, node->slaves[i]->port);
+                        addReplyBulkCBuffer(c, node->slaves[i]->name, CLUSTER_NAMELEN);
+                        nested_elements++;
+                    }
+                    setDeferredArrayLen(c, nested_replylen, nested_elements);
+                    num_masters++;
                 }
-                setDeferredArrayLen(c, nested_replylen, nested_elements);
-                num_masters++;
             }
         }
+        serverAssert(start == -1);
     }
+    
     dictReleaseIterator(di);
     setDeferredArrayLen(c, slot_replylen, num_masters);
 }
@@ -4286,7 +4331,7 @@ NULL
         clusterReplyMultiBulkSlots(c);
     } else if (!strcasecmp(szFromObj(c->argv[1]),"flushslots") && c->argc == 2) {
         /* CLUSTER FLUSHSLOTS */
-        if (g_pserver->db[0].size() != 0) {
+        if (dictSize(g_pserver->db[0].pdict) != 0) {
             addReplyError(c,"DB must be empty to perform CLUSTER FLUSHSLOTS.");
             return;
         }
@@ -4621,7 +4666,7 @@ NULL
          * slots nor keys to accept to replicate some other node.
          * Slaves can switch to another master without issues. */
         if (nodeIsMaster(myself) &&
-            (myself->numslots != 0 || g_pserver->db[0].size() != 0)) {
+            (myself->numslots != 0 || dictSize(g_pserver->db[0].pdict) != 0)) {
             addReplyError(c,
                 "To set a master the node must be empty and "
                 "without assigned slots.");
@@ -4778,7 +4823,7 @@ NULL
 
         /* Slaves can be reset while containing data, but not master nodes
          * that must be empty. */
-        if (nodeIsMaster(myself) && c->db->size() != 0) {
+        if (nodeIsMaster(myself) && dictSize(c->db->pdict) != 0) {
             addReplyError(c,"CLUSTER RESET can't be called with "
                             "master nodes containing keys");
             return;
@@ -4794,7 +4839,6 @@ NULL
 /* -----------------------------------------------------------------------------
  * DUMP, RESTORE and MIGRATE commands
  * -------------------------------------------------------------------------- */
-ssize_t rdbSaveAuxFieldStrStr(rio *rdb, const char *key, const char *val);
 
 /* Generates a DUMP-format representation of the object 'o', adding it to the
  * io stream pointed by 'rio'. This function can't fail. */
@@ -4807,9 +4851,6 @@ void createDumpPayload(rio *payload, robj_roptr o, robj *key) {
     rioInitWithBuffer(payload,sdsempty());
     serverAssert(rdbSaveObjectType(payload,o));
     serverAssert(rdbSaveObject(payload,o,key));
-    char szT[32];
-    snprintf(szT, 32, "%" PRIu64, o->mvcc_tstamp);
-    serverAssert(rdbSaveAuxFieldStrStr(payload,"mvcc-tstamp", szT) != -1);
 
     /* Write the footer, this is how it looks like:
      * ----------------+---------------------+---------------+
@@ -4945,21 +4986,6 @@ void restoreCommand(client *c) {
         addReplyError(c,"Bad data format");
         return;
     }
-    if (rdbLoadType(&payload) == RDB_OPCODE_AUX)
-    {
-        robj *auxkey, *auxval;
-        if ((auxkey = rdbLoadStringObject(&payload)) == NULL) goto eoferr;
-        if ((auxval = rdbLoadStringObject(&payload)) == NULL) {
-            decrRefCount(auxkey);
-            goto eoferr;
-        }
-        if (strcasecmp(szFromObj(auxkey), "mvcc-tstamp") == 0) {
-            obj->mvcc_tstamp = strtoull(szFromObj(auxval), nullptr, 10);
-        }
-        decrRefCount(auxkey);
-        decrRefCount(auxval);
-    }
-eoferr:
 
     /* Remove the old key if needed. */
     if (replace) dbDelete(c->db,c->argv[1]);
@@ -5213,7 +5239,7 @@ try_again:
     /* Create RESTORE payload and generate the protocol to call the command. */
     for (j = 0; j < num_keys; j++) {
         long long ttl = 0;
-        expireEntry *pexpire = c->db->getExpire(kv[j]);
+        expireEntry *pexpire = getExpire(c->db,kv[j]);
         long long expireat = -1;
         if (pexpire != nullptr)
             pexpire->FGetPrimaryExpire(&expireat);

src/compactvector.h

@@ -30,16 +30,7 @@ public:
         zfree(m_data);
     }
 
-    compactvector(const compactvector &src)
-    {
-        m_celem = src.m_celem;
-        m_max = src.m_max;
-        m_data = (T*)zmalloc(sizeof(T) * m_max, MALLOC_LOCAL);
-        for (size_t ielem = 0; ielem < m_celem; ++ielem)
-        {
-            new (m_data+ielem) T(src[ielem]);
-        }
-    }
+    compactvector(compactvector &) noexcept = delete;
 
     compactvector(compactvector &&src) noexcept
     {
@@ -51,6 +42,7 @@ public:
         src.m_max = 0;
     }
 
+    compactvector &operator=(const compactvector&) noexcept = delete;
     compactvector &operator=(compactvector &&src) noexcept
     {
         zfree(m_data);

src/db.cpp

@@ -41,6 +41,7 @@
 
 int keyIsExpired(redisDb *db, robj *key);
 int expireIfNeeded(redisDb *db, robj *key, robj *o);
+void dbOverwriteCore(redisDb *db, dictEntry *de, robj *key, robj *val, bool fUpdateMvcc, bool fRemoveExpire);
 
 /* Update LFU when an object is accessed.
  * Firstly, decrement the counter if the decrement time is reached.
@@ -56,7 +57,8 @@ void updateExpire(redisDb *db, sds key, robj *valOld, robj *valNew)
     serverAssert(valOld->FExpires());
     serverAssert(!valNew->FExpires());
     
-    serverAssert(db->FKeyExpires((const char*)key));
+    auto itr = db->setexpire->find(key);
+    serverAssert(itr != db->setexpire->end());
     
     valNew->SetFExpires(true);
     valOld->SetFExpires(false);
@@ -68,13 +70,14 @@ void updateExpire(redisDb *db, sds key, robj *valOld, robj *valNew)
  * implementations that should instead rely on lookupKeyRead(),
  * lookupKeyWrite() and lookupKeyReadWithFlags(). */
 static robj *lookupKey(redisDb *db, robj *key, int flags) {
-    auto itr = db->find(key);
-    if (itr) {
-        robj *val = itr.val();
+    dictEntry *de = dictFind(db->pdict,ptrFromObj(key));
+    if (de) {
+        robj *val = (robj*)dictGetVal(de);
+
         /* Update the access time for the ageing algorithm.
          * Don't do it if we have a saving child, as this will trigger
          * a copy on write madness. */
-        if (!g_pserver->FRdbSaveInProgress() &&
+        if (g_pserver->rdb_child_pid == -1 &&
             g_pserver->aof_child_pid == -1 &&
             !(flags & LOOKUP_NOTOUCH))
         {
@@ -87,7 +90,6 @@ static robj *lookupKey(redisDb *db, robj *key, int flags) {
 
         if (flags & LOOKUP_UPDATEMVCC) {
             val->mvcc_tstamp = getMvccTstamp();
-            db->trackkey(key);
         }
         return val;
     } else {
@@ -193,14 +195,13 @@ robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply) {
     return o;
 }
 
-bool dbAddCore(redisDb *db, robj *key, robj *val) {
+int dbAddCore(redisDb *db, robj *key, robj *val) {
     serverAssert(!val->FExpires());
     sds copy = sdsdup(szFromObj(key));
-    bool fInserted = db->insert(copy, val);
-    if (g_pserver->fActiveReplica)
-        val->mvcc_tstamp = key->mvcc_tstamp = getMvccTstamp();
+    int retval = dictAdd(db->pdict, copy, val);
+    val->mvcc_tstamp = key->mvcc_tstamp = getMvccTstamp();
 
-    if (fInserted)
+    if (retval == DICT_OK)
     {
         if (val->type == OBJ_LIST ||
             val->type == OBJ_ZSET)
@@ -212,7 +213,7 @@ bool dbAddCore(redisDb *db, robj *key, robj *val) {
         sdsfree(copy);
     }
 
-    return fInserted;
+    return retval;
 }
 
 /* Add the key to the DB. It's up to the caller to increment the reference
@@ -221,22 +222,23 @@ bool dbAddCore(redisDb *db, robj *key, robj *val) {
  * The program is aborted if the key already exists. */
 void dbAdd(redisDb *db, robj *key, robj *val)
 {
-    bool fInserted = dbAddCore(db, key, val);
-    serverAssertWithInfo(NULL,key,fInserted);
+    int retval = dbAddCore(db, key, val);
+    serverAssertWithInfo(NULL,key,retval == DICT_OK);
 }
 
-void redisDb::dbOverwriteCore(redisDb::iter itr, robj *key, robj *val, bool fUpdateMvcc, bool fRemoveExpire)
+void dbOverwriteCore(redisDb *db, dictEntry *de, robj *key, robj *val, bool fUpdateMvcc, bool fRemoveExpire)
 {
-    robj *old = itr.val();
+    dictEntry auxentry = *de;
+    robj *old = (robj*)dictGetVal(de);
 
     if (old->FExpires()) {
         if (fRemoveExpire) {
-            ::removeExpire(this, key);
+            removeExpire(db, key);
         }
         else {
             if (val->getrefcount(std::memory_order_relaxed) == OBJ_SHARED_REFCOUNT)
                 val = dupStringObject(val);
-            ::updateExpire(this, itr.key(), old, val);
+            updateExpire(db, (sds)dictGetKey(de), old, val);
         }
     }
 
@@ -249,12 +251,14 @@ void redisDb::dbOverwriteCore(redisDb::iter itr, robj *key, robj *val, bool fUpd
         val->mvcc_tstamp = getMvccTstamp();
     }
 
-    if (g_pserver->lazyfree_lazy_server_del)
-        freeObjAsync(itr.val());
-    else
-        decrRefCount(itr.val());
+    dictSetVal(db->pdict, de, val);
 
-    updateValue(itr, val);
+    if (g_pserver->lazyfree_lazy_server_del) {
+        freeObjAsync(old);
+        dictSetVal(db->pdict, &auxentry, NULL);
+    }
+
+    dictFreeVal(db->pdict, &auxentry);
 }
 
 /* Overwrite an existing key with a new value. Incrementing the reference
@@ -263,10 +267,10 @@ void redisDb::dbOverwriteCore(redisDb::iter itr, robj *key, robj *val, bool fUpd
  *
  * The program is aborted if the key was not already present. */
 void dbOverwrite(redisDb *db, robj *key, robj *val) {
-    auto itr = db->find(key);
+    dictEntry *de = dictFind(db->pdict,ptrFromObj(key));
 
-    serverAssertWithInfo(NULL,key,itr != nullptr);
-    db->dbOverwriteCore(itr, key, val, !!g_pserver->fActiveReplica, false);
+    serverAssertWithInfo(NULL,key,de != NULL);
+    dbOverwriteCore(db, de, key, val, !!g_pserver->fActiveReplica, false);
 }
 
 /* Insert a key, handling duplicate keys according to fReplace */
@@ -274,14 +278,14 @@ int dbMerge(redisDb *db, robj *key, robj *val, int fReplace)
 {
     if (fReplace)
     {
-        auto itr = db->find(key);
-        if (itr == nullptr)
-            return (dbAddCore(db, key, val) == true);
+        dictEntry *de = dictFind(db->pdict, ptrFromObj(key));
+        if (de == nullptr)
+            return (dbAddCore(db, key, val) == DICT_OK);
 
-        robj *old = itr.val();
+        robj *old = (robj*)dictGetVal(de);
         if (old->mvcc_tstamp <= val->mvcc_tstamp)
         {
-            db->dbOverwriteCore(itr, key, val, false, true);
+            dbOverwriteCore(db, de, key, val, false, true);
             return true;
         }
         
@@ -289,7 +293,7 @@ int dbMerge(redisDb *db, robj *key, robj *val, int fReplace)
     }
     else
     {
-        return (dbAddCore(db, key, val) == true);
+        return (dbAddCore(db, key, val) == DICT_OK);
     }
 }
 
@@ -302,18 +306,18 @@ int dbMerge(redisDb *db, robj *key, robj *val, int fReplace)
  *
  * All the new keys in the database should be created via this interface. */
 void setKey(redisDb *db, robj *key, robj *val) {
-    auto itr = db->find(key);
-    if (itr == NULL) {
+    dictEntry *de = dictFind(db->pdict, ptrFromObj(key));
+    if (de == NULL) {
         dbAdd(db,key,val);
     } else {
-        db->dbOverwriteCore(itr,key,val,!!g_pserver->fActiveReplica,true);
+        dbOverwriteCore(db,de,key,val,!!g_pserver->fActiveReplica,true);
     }
     incrRefCount(val);
     signalModifiedKey(db,key);
 }
 
 int dbExists(redisDb *db, robj *key) {
-    return (db->find(key) != nullptr);
+    return dictFind(db->pdict,ptrFromObj(key)) != NULL;
 }
 
 /* Return a random key, in form of a Redis object.
@@ -321,20 +325,21 @@ int dbExists(redisDb *db, robj *key) {
  *
  * The function makes sure to return keys not already expired. */
 robj *dbRandomKey(redisDb *db) {
+    dictEntry *de;
     int maxtries = 100;
-    bool allvolatile = db->expireSize() == db->size();
+    int allvolatile = dictSize(db->pdict) == db->setexpire->size();
 
     while(1) {
         sds key;
         robj *keyobj;
 
-        auto itr = db->random();
-        if (itr == nullptr) return NULL;
+        de = dictGetRandomKey(db->pdict);
+        if (de == NULL) return NULL;
 
-        key = itr.key();
+        key = (sds)dictGetKey(de);
         keyobj = createStringObject(key,sdslen(key));
 
-        if (itr.val()->FExpires())
+        if (((robj*)dictGetVal(de))->FExpires())
         {
             if (allvolatile && listLength(g_pserver->masters) && --maxtries == 0) {
                 /* If the DB is composed only of keys with an expire set,
@@ -349,7 +354,7 @@ robj *dbRandomKey(redisDb *db) {
             }
         }
             
-        if (itr.val()->FExpires())
+        if (((robj*)dictGetVal(de))->FExpires())
         {
              if (expireIfNeeded(db,keyobj)) {
                 decrRefCount(keyobj);
@@ -361,16 +366,15 @@ robj *dbRandomKey(redisDb *db) {
     }
 }
 
-bool redisDbPersistentData::syncDelete(robj *key)
-{
+/* Delete a key, value, and associated expiration entry if any, from the DB */
+int dbSyncDelete(redisDb *db, robj *key) {
     /* Deleting an entry from the expires dict will not free the sds of
      * the key, because it is shared with the main dictionary. */
 
-    auto itr = find(szFromObj(key));
-    trackkey(szFromObj(key));
-    if (itr != nullptr && itr.val()->FExpires())
-        removeExpire(key, itr);
-    if (dictDelete(m_pdict,ptrFromObj(key)) == DICT_OK) {
+    dictEntry *de = dictFind(db->pdict, szFromObj(key));
+    if (de != nullptr && ((robj*)dictGetVal(de))->FExpires())
+        removeExpireCore(db, key, de);
+    if (dictDelete(db->pdict,ptrFromObj(key)) == DICT_OK) {
         if (g_pserver->cluster_enabled) slotToKeyDel(key);
         return 1;
     } else {
@@ -378,11 +382,6 @@ bool redisDbPersistentData::syncDelete(robj *key)
     }
 }
 
-/* Delete a key, value, and associated expiration entry if any, from the DB */
-int dbSyncDelete(redisDb *db, robj *key) {
-    return db->syncDelete(key);
-}
-
 /* This is a wrapper whose behavior depends on the Redis lazy free
  * configuration. Deletes the key synchronously or asynchronously. */
 int dbDelete(redisDb *db, robj *key) {
@@ -428,7 +427,7 @@ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) {
     return o;
 }
 
-/* Remove all keys from all the databases in a Redis DB
+/* Remove all keys from all the databases in a Redis g_pserver->
  * If callback is given the function is called from time to time to
  * signal that work is in progress.
  *
@@ -460,7 +459,15 @@ long long emptyDb(int dbnum, int flags, void(callback)(void*)) {
     }
 
     for (int j = startdb; j <= enddb; j++) {
-        removed += g_pserver->db[j].clear(!!async, callback); 
+        removed += dictSize(g_pserver->db[j].pdict);
+        if (async) {
+            emptyDbAsync(&g_pserver->db[j]);
+        } else {
+            dictEmpty(g_pserver->db[j].pdict,callback);
+            delete g_pserver->db[j].setexpire;
+            g_pserver->db[j].setexpire = new (MALLOC_LOCAL) expireset();
+            g_pserver->db[j].expireitr = g_pserver->db[j].setexpire->end();
+        }
     }
     if (g_pserver->cluster_enabled) {
         if (async) {
@@ -546,14 +553,14 @@ void flushallCommand(client *c) {
     signalFlushedDb(-1);
     g_pserver->dirty += emptyDb(-1,flags,NULL);
     addReply(c,shared.ok);
-    if (g_pserver->FRdbSaveInProgress()) killRDBChild();
+    if (g_pserver->rdb_child_pid != -1) killRDBChild();
     if (g_pserver->saveparamslen > 0) {
         /* Normally rdbSave() will reset dirty, but we don't want this here
          * as otherwise FLUSHALL will not be replicated nor put into the AOF. */
         int saved_dirty = g_pserver->dirty;
         rdbSaveInfo rsi, *rsiptr;
         rsiptr = rdbPopulateSaveInfo(&rsi);
-        rdbSave(nullptr, rsiptr);
+        rdbSave(rsiptr);
         g_pserver->dirty = saved_dirty;
     }
     g_pserver->dirty++;
@@ -628,35 +635,18 @@ void randomkeyCommand(client *c) {
     decrRefCount(key);
 }
 
-
-bool redisDbPersistentData::iterate(std::function<bool(const char*, robj*)> fn)
-{
-    dictIterator *di = dictGetSafeIterator(m_pdict);
-    dictEntry *de = nullptr;
-    bool fResult = true;
-    while((de = dictNext(di)) != nullptr)
-    {
-        ensure(de);
-        if (!fn((const char*)dictGetKey(de), (robj*)dictGetVal(de)))
-        {
-            fResult = false;
-            break;
-        }
-    }
-    dictReleaseIterator(di);
-    return fResult;
-}
-
 void keysCommand(client *c) {
+    dictIterator *di;
+    dictEntry *de;
     sds pattern = szFromObj(c->argv[1]);
     int plen = sdslen(pattern), allkeys;
     unsigned long numkeys = 0;
     void *replylen = addReplyDeferredLen(c);
 
-    aeReleaseLock();
-
+    di = dictGetSafeIterator(c->db->pdict);
     allkeys = (pattern[0] == '*' && pattern[1] == '\0');
-    c->db->iterate([&](const char *key, robj *)->bool {
+    while((de = dictNext(di)) != NULL) {
+        sds key = (sds)dictGetKey(de);
         robj *keyobj;
 
         if (allkeys || stringmatchlen(pattern,plen,key,sdslen(key),0)) {
@@ -667,15 +657,9 @@ void keysCommand(client *c) {
             }
             decrRefCount(keyobj);
         }
-        return true;
-    });
+    }
+    dictReleaseIterator(di);
     setDeferredArrayLen(c,replylen,numkeys);
-    
-    fastlock_unlock(&c->db->lock);  // we must release the DB lock before acquiring the AE lock to prevent deadlocks
-    AeLocker lock;
-    lock.arm(c);
-    fastlock_lock(&c->db->lock);    // we still need the DB lock
-    lock.release();
 }
 
 /* This callback is used by scanGenericCommand in order to collect elements
@@ -803,7 +787,7 @@ void scanGenericCommand(client *c, robj_roptr o, unsigned long cursor) {
     /* Handle the case of a hash table. */
     ht = NULL;
     if (o == nullptr) {
-        ht = c->db->dictUnsafeKeyOnly();
+        ht = c->db->pdict;
     } else if (o->type == OBJ_SET && o->encoding == OBJ_ENCODING_HT) {
         ht = (dict*)ptrFromObj(o);
     } else if (o->type == OBJ_HASH && o->encoding == OBJ_ENCODING_HT) {
@@ -936,7 +920,7 @@ void scanCommand(client *c) {
 }
 
 void dbsizeCommand(client *c) {
-    addReplyLongLong(c,c->db->size());
+    addReplyLongLong(c,dictSize(c->db->pdict));
 }
 
 void lastsaveCommand(client *c) {
@@ -1019,7 +1003,7 @@ void renameGenericCommand(client *c, int nx) {
     std::unique_ptr<expireEntry> spexpire;
 
     {   // scope pexpireOld since it will be invalid soon
-    expireEntry *pexpireOld = c->db->getExpire(c->argv[1]);
+    expireEntry *pexpireOld = getExpire(c->db,c->argv[1]);
     if (pexpireOld != nullptr)
         spexpire = std::make_unique<expireEntry>(std::move(*pexpireOld));
     }
@@ -1097,7 +1081,7 @@ void moveCommand(client *c) {
 
     std::unique_ptr<expireEntry> spexpire;
     {   // scope pexpireOld
-    expireEntry *pexpireOld = c->db->getExpire(c->argv[1]);
+    expireEntry *pexpireOld = getExpire(c->db,c->argv[1]);
     if (pexpireOld != nullptr)
         spexpire = std::make_unique<expireEntry>(std::move(*pexpireOld));
     }
@@ -1156,14 +1140,17 @@ int dbSwapDatabases(int id1, int id2) {
     /* Swap hash tables. Note that we don't swap blocking_keys,
      * ready_keys and watched_keys, since we want clients to
      * remain in the same DB they were. */
-    redisDbPersistentData::swap(db1, db2);
+    db1->pdict = db2->pdict;
+    db1->setexpire = db2->setexpire;
+    db1->expireitr = db2->expireitr;
     db1->avg_ttl = db2->avg_ttl;
     db1->last_expire_set = db2->last_expire_set;
-    db1->expireitr = db2->expireitr;
 
+    db2->pdict = aux.pdict;
+    db2->setexpire = aux.setexpire;
+    db2->expireitr = aux.expireitr;
     db2->avg_ttl = aux.avg_ttl;
     db2->last_expire_set = aux.last_expire_set;
-    db2->expireitr = aux.expireitr;
 
     /* Now we need to handle clients blocked on lists: as an effect
      * of swapping the two DBs, a client that was waiting for list
@@ -1212,38 +1199,37 @@ void swapdbCommand(client *c) {
  * Expires API
  *----------------------------------------------------------------------------*/
 int removeExpire(redisDb *db, robj *key) {
-    auto itr = db->find(key);
-    return db->removeExpire(key, itr);
+    dictEntry *de = dictFind(db->pdict,ptrFromObj(key));
+    return removeExpireCore(db, key, de);
 }
-int redisDbPersistentData::removeExpire(robj *key, dict_iter itr) {
+int removeExpireCore(redisDb *db, robj *key, dictEntry *de) {
     /* An expire may only be removed if there is a corresponding entry in the
      * main dict. Otherwise, the key will never be freed. */
-    serverAssertWithInfo(NULL,key,itr != nullptr);
+    serverAssertWithInfo(NULL,key,de != NULL);
 
-    robj *val = itr.val();
+    robj *val = (robj*)dictGetVal(de);
     if (!val->FExpires())
         return 0;
 
-    trackkey(key);
-    auto itrExpire = m_setexpire->find(itr.key());
-    serverAssert(itrExpire != m_setexpire->end());
-    serverAssert(itrExpire->key() == itr.key());
-    m_setexpire->erase(itrExpire);
+    auto itr = db->setexpire->find((sds)dictGetKey(de));
+    serverAssert(itr != db->setexpire->end());
+    serverAssert(itr->key() == (sds)dictGetKey(de));
+    db->setexpire->erase(itr);
     val->SetFExpires(false);
     return 1;
 }
 
-int redisDbPersistentData::removeSubkeyExpire(robj *key, robj *subkey) {
-    auto de = find(szFromObj(key));
-    serverAssertWithInfo(NULL,key,de != nullptr);
+int removeSubkeyExpire(redisDb *db, robj *key, robj *subkey) {
+    dictEntry *de = dictFind(db->pdict,ptrFromObj(key));
+    serverAssertWithInfo(NULL,key,de != NULL);
     
-    robj *val = de.val();
+    robj *val = (robj*)dictGetVal(de);
     if (!val->FExpires())
         return 0;
     
-    auto itr = m_setexpire->find(de.key());
-    serverAssert(itr != m_setexpire->end());
-    serverAssert(itr->key() == de.key());
+    auto itr = db->setexpire->find((sds)dictGetKey(de));
+    serverAssert(itr != db->setexpire->end());
+    serverAssert(itr->key() == (sds)dictGetKey(de));
     if (!itr->FFat())
         return 0;
 
@@ -1261,7 +1247,7 @@ int redisDbPersistentData::removeSubkeyExpire(robj *key, robj *subkey) {
     }
 
     if (itr->pfatentry()->size() == 0)
-        this->removeExpire(key, de);
+        removeExpireCore(db, key, de);
 
     return found;
 }
@@ -1271,23 +1257,49 @@ int redisDbPersistentData::removeSubkeyExpire(robj *key, robj *subkey) {
  * to NULL. The 'when' parameter is the absolute unix time in milliseconds
  * after which the key will no longer be considered valid. */
 void setExpire(client *c, redisDb *db, robj *key, robj *subkey, long long when) {
+    dictEntry *kde;
+
     serverAssert(GlobalLocksAcquired());
 
+    /* Reuse the sds from the main dict in the expire dict */
+    kde = dictFind(db->pdict,ptrFromObj(key));
+    serverAssertWithInfo(NULL,key,kde != NULL);
+
+    if (((robj*)dictGetVal(kde))->getrefcount(std::memory_order_relaxed) == OBJ_SHARED_REFCOUNT)
+    {
+        // shared objects cannot have the expire bit set, create a real object
+        dictSetVal(db->pdict, kde, dupStringObject((robj*)dictGetVal(kde)));
+    }
+
     /* Update TTL stats (exponential moving average) */
     /*  Note: We never have to update this on expiry since we reduce it by the current elapsed time here */
     long long now = g_pserver->mstime;
     db->avg_ttl -= (now - db->last_expire_set); // reduce the TTL by the time that has elapsed
-    if (db->expireSize() == 0)
+    if (db->setexpire->empty())
         db->avg_ttl = 0;
     else
-        db->avg_ttl -= db->avg_ttl / db->expireSize(); // slide one entry out the window
+        db->avg_ttl -= db->avg_ttl / db->setexpire->size(); // slide one entry out the window
     if (db->avg_ttl < 0)
         db->avg_ttl = 0;    // TTLs are never negative
-    db->avg_ttl += (double)(when-now) / (db->expireSize()+1);    // add the new entry
+    db->avg_ttl += (double)(when-now) / (db->setexpire->size()+1);    // add the new entry
     db->last_expire_set = now;
 
     /* Update the expire set */
-    db->setExpire(key, subkey, when);
+    const char *szSubKey = (subkey != nullptr) ? szFromObj(subkey) : nullptr;
+    if (((robj*)dictGetVal(kde))->FExpires()) {
+        auto itr = db->setexpire->find((sds)dictGetKey(kde));
+        serverAssert(itr != db->setexpire->end());
+        expireEntry eNew(std::move(*itr));
+        eNew.update(szSubKey, when);
+        db->setexpire->erase(itr);
+        db->setexpire->insert(eNew);
+    }
+    else
+    {
+        expireEntry e((sds)dictGetKey(kde), szSubKey, when);
+        ((robj*)dictGetVal(kde))->SetFExpires(true);
+        db->setexpire->insert(e);
+    }
 
     int writable_slave = listLength(g_pserver->masters) && g_pserver->repl_slave_ro == 0;
     if (c && writable_slave && !(c->flags & CLIENT_MASTER))
@@ -1296,24 +1308,26 @@ void setExpire(client *c, redisDb *db, robj *key, robj *subkey, long long when)
 
 void setExpire(client *c, redisDb *db, robj *key, expireEntry &&e)
 {
+    dictEntry *kde;
+
     serverAssert(GlobalLocksAcquired());
 
     /* Reuse the sds from the main dict in the expire dict */
-    auto kde = db->find(key);
+    kde = dictFind(db->pdict,ptrFromObj(key));
     serverAssertWithInfo(NULL,key,kde != NULL);
 
-    if (kde.val()->getrefcount(std::memory_order_relaxed) == OBJ_SHARED_REFCOUNT)
+    if (((robj*)dictGetVal(kde))->getrefcount(std::memory_order_relaxed) == OBJ_SHARED_REFCOUNT)
     {
         // shared objects cannot have the expire bit set, create a real object
-        db->updateValue(kde, dupStringObject(kde.val()));
+        dictSetVal(db->pdict, kde, dupStringObject((robj*)dictGetVal(kde)));
     }
 
-    if (kde.val()->FExpires())
+    if (((robj*)dictGetVal(kde))->FExpires())
         removeExpire(db, key);
 
-    e.setKeyUnsafe(kde.key());
-    db->setExpire(std::move(e));
-    kde.val()->SetFExpires(true);
+    e.setKeyUnsafe((sds)dictGetKey(kde));
+    db->setexpire->insert(e);
+    ((robj*)dictGetVal(kde))->SetFExpires(true);
 
 
     int writable_slave = listLength(g_pserver->masters) && g_pserver->repl_slave_ro == 0;
@@ -1323,19 +1337,22 @@ void setExpire(client *c, redisDb *db, robj *key, expireEntry &&e)
 
 /* Return the expire time of the specified key, or null if no expire
  * is associated with this key (i.e. the key is non volatile) */
-expireEntry *redisDbPersistentData::getExpire(robj_roptr key) {
+expireEntry *getExpire(redisDb *db, robj_roptr key) {
+    dictEntry *de;
+
     /* No expire? return ASAP */
-    if (expireSize() == 0)
+    if (db->setexpire->size() == 0)
         return nullptr;
 
-    auto itr = find(szFromObj(key));
-    if (itr == nullptr)
+    de = dictFind(db->pdict, ptrFromObj(key));
+    if (de == NULL)
         return nullptr;
-    if (!itr.val()->FExpires())
+    robj *obj = (robj*)dictGetVal(de);
+    if (!obj->FExpires())
         return nullptr;
 
-    auto itrExpire = findExpire(itr.key());
-    return itrExpire.operator->();
+    auto itr = db->setexpire->find((sds)dictGetKey(de));
+    return itr.operator->();
 }
 
 /* Propagate expires into slaves and the AOF file.
@@ -1365,7 +1382,7 @@ void propagateExpire(redisDb *db, robj *key, int lazy) {
 
 /* Check if the key is expired. Note, this does not check subexpires */
 int keyIsExpired(redisDb *db, robj *key) {
-    expireEntry *pexpire = db->getExpire(key);
+    expireEntry *pexpire = getExpire(db,key);
 
     if (pexpire == nullptr) return 0; /* No expire for this key */
 
@@ -1797,304 +1814,3 @@ unsigned int delKeysInSlot(unsigned int hashslot) {
 unsigned int countKeysInSlot(unsigned int hashslot) {
     return g_pserver->cluster->slots_keys_count[hashslot];
 }
-
-void redisDbPersistentData::initialize()
-{
-    m_pdict = dictCreate(&dbDictType,this);
-    m_setexpire = new(MALLOC_LOCAL) expireset();
-    m_fAllChanged = false;
-    m_fTrackingChanges = 0;
-}
-
-void redisDb::initialize(int id)
-{
-    redisDbPersistentData::initialize();
-    this->expireitr = setexpire()->end();
-    this->blocking_keys = dictCreate(&keylistDictType,NULL);
-    this->ready_keys = dictCreate(&objectKeyPointerValueDictType,NULL);
-    this->watched_keys = dictCreate(&keylistDictType,NULL);
-    this->id = id;
-    this->avg_ttl = 0;
-    this->last_expire_set = 0;
-    this->defrag_later = listCreate();
-}
-
-bool redisDbPersistentData::insert(char *key, robj *o)
-{
-    int res = dictAdd(m_pdict, key, o);
-    if (res == DICT_OK)
-        trackkey(key);
-    return (res == DICT_OK);
-}
-
-void redisDbPersistentData::tryResize()
-{
-    if (htNeedsResize(m_pdict))
-        dictResize(m_pdict);
-}
-
-size_t redisDb::clear(bool fAsync, void(callback)(void*))
-{
-    size_t removed = size();
-    if (fAsync) {
-        redisDbPersistentData::emptyDbAsync();
-    } else {
-        redisDbPersistentData::clear(callback);
-    }
-    expireitr = setexpire()->end();
-    return removed;
-}
-
-void redisDbPersistentData::clear(void(callback)(void*))
-{
-    dictEmpty(m_pdict,callback);
-    if (m_fTrackingChanges)
-        m_fAllChanged = true;
-    delete m_setexpire;
-    m_setexpire = new (MALLOC_LOCAL) expireset();
-    if (m_pstorage != nullptr)
-        m_pstorage->clear();
-}
-
-/* static */ void redisDbPersistentData::swap(redisDbPersistentData *db1, redisDbPersistentData *db2)
-{
-    redisDbPersistentData aux = *db1;
-    db1->m_pdict = db2->m_pdict;
-    db1->m_fTrackingChanges = db2->m_fTrackingChanges;
-    db1->m_fAllChanged = db2->m_fAllChanged;
-    db1->m_setexpire = db2->m_setexpire;
-    db1->m_pstorage = db2->m_pstorage;
-    db1->m_spdbSnapshot = db2->m_spdbSnapshot;
-
-    db2->m_pdict = aux.m_pdict;
-    db2->m_fTrackingChanges = aux.m_fTrackingChanges;
-    db2->m_fAllChanged = aux.m_fAllChanged;
-    db2->m_setexpire = aux.m_setexpire;
-    db2->m_pstorage = aux.m_pstorage;
-    db2->m_spdbSnapshot = aux.m_spdbSnapshot;
-
-    db1->m_pdict->privdata = static_cast<redisDbPersistentData*>(db1);
-    db2->m_pdict->privdata = static_cast<redisDbPersistentData*>(db2);
-}
-
-void redisDbPersistentData::setExpire(robj *key, robj *subkey, long long when)
-{
-    /* Reuse the sds from the main dict in the expire dict */
-    dictEntry *kde = dictFind(m_pdict,ptrFromObj(key));
-    serverAssertWithInfo(NULL,key,kde != NULL);
-    trackkey(key);
-
-    if (((robj*)dictGetVal(kde))->getrefcount(std::memory_order_relaxed) == OBJ_SHARED_REFCOUNT)
-    {
-        // shared objects cannot have the expire bit set, create a real object
-        dictSetVal(m_pdict, kde, dupStringObject((robj*)dictGetVal(kde)));
-    }
-
-    const char *szSubKey = (subkey != nullptr) ? szFromObj(subkey) : nullptr;
-    if (((robj*)dictGetVal(kde))->FExpires()) {
-        auto itr = m_setexpire->find((sds)dictGetKey(kde));
-        serverAssert(itr != m_setexpire->end());
-        expireEntry eNew(std::move(*itr));
-        eNew.update(szSubKey, when);
-        m_setexpire->erase(itr);
-        m_setexpire->insert(eNew);
-    }
-    else
-    {
-        expireEntry e((sds)dictGetKey(kde), szSubKey, when);
-        ((robj*)dictGetVal(kde))->SetFExpires(true);
-        m_setexpire->insert(e);
-    }
-}
-
-void redisDbPersistentData::setExpire(expireEntry &&e)
-{
-    trackkey(e.key());
-    m_setexpire->insert(e);
-}
-
-bool redisDb::FKeyExpires(const char *key)
-{
-    return setexpireUnsafe()->find(key) != setexpire()->end();
-}
-
-void redisDbPersistentData::updateValue(dict_iter itr, robj *val)
-{
-    trackkey(itr.key());
-    dictSetVal(m_pdict, itr.de, val);
-}
-
-void redisDbPersistentData::ensure(const char *key)
-{
-    dictEntry *de = dictFind(m_pdict, key);
-    ensure(de);
-}
-
-void redisDbPersistentData::ensure(dictEntry *de)
-{
-    if (de != nullptr && dictGetVal(de) == nullptr)
-    {
-        if (m_spdbSnapshot != nullptr)
-        {
-            auto itr = m_spdbSnapshot->find((const char*)dictGetKey(de));
-            serverAssert(itr != m_spdbSnapshot->end());
-            if (itr.val()->getrefcount(std::memory_order_relaxed) == OBJ_SHARED_REFCOUNT)
-            {
-                dictSetVal(m_pdict, de, itr.val());
-            }
-            else
-            {
-                sds strT = serializeStoredObject(itr.val());
-                robj *objNew = deserializeStoredObject(strT, sdslen(strT));
-                sdsfree(strT);
-                dictSetVal(m_pdict, de, objNew);
-                serverAssert(objNew->getrefcount(std::memory_order_relaxed) == 1);
-                serverAssert(objNew->mvcc_tstamp == itr.val()->mvcc_tstamp);
-            }
-        }
-        else
-        {
-            serverAssert(m_pstorage != nullptr);
-            sds key = (sds)dictGetKey(de);
-            m_pstorage->retrieve(key, sdslen(key), true, [&](const char *, size_t, const void *data, size_t cb){
-                robj *o = deserializeStoredObject(data, cb);
-                serverAssert(o != nullptr);
-                dictSetVal(m_pdict, de, o);
-            });
-        }
-    }
-}
-
-void redisDbPersistentData::storeKey(const char *szKey, size_t cchKey, robj *o)
-{
-    sds temp = serializeStoredObject(o);
-    m_pstorage->insert(szKey, cchKey, temp, sdslen(temp));
-    sdsfree(temp);
-}
-
-void redisDbPersistentData::storeDatabase()
-{
-    dictIterator *di = dictGetIterator(m_pdict);
-    dictEntry *de;
-    while ((de = dictNext(di)) != NULL) {
-        sds key = (sds)dictGetKey(de);
-        robj *o = (robj*)dictGetVal(de);
-        storeKey(key, sdslen(key), o);
-    }
-    dictReleaseIterator(di);
-}
-
-void redisDbPersistentData::processChanges()
-{
-    --m_fTrackingChanges;
-    serverAssert(m_fTrackingChanges >= 0);
-
-    if (m_pstorage != nullptr)
-    {
-        if (m_fTrackingChanges == 0)
-        {
-            if (m_fAllChanged)
-            {
-                m_pstorage->clear();
-                storeDatabase();
-            }
-            else
-            {
-                for (auto &str : m_setchanged)
-                {
-                    sds sdsKey = sdsnewlen(str.data(), str.size());
-                    robj *o = find(sdsKey);
-                    if (o != nullptr)
-                    {
-                        storeKey(str.data(), str.size(), o);
-                    }
-                    else
-                    {
-                        m_pstorage->erase(str.data(), str.size());
-                    }
-                    sdsfree(sdsKey);
-                }
-            }
-        }
-    }
-    m_setchanged.clear();
-}
-
-std::shared_ptr<redisDbPersistentData> redisDbPersistentData::createSnapshot()
-{
-    serverAssert(GlobalLocksAcquired());
-    serverAssert(m_spdbSnapshot == nullptr);
-    auto spdb = std::make_shared<redisDbPersistentData>();
-    spdb->m_pdict = dictCreate(&dbDictType,spdb.get());
-    spdb->m_fAllChanged = false;
-    spdb->m_fTrackingChanges = 0;
-    spdb->m_pdict->rehashidx = m_pdict->rehashidx;
-    spdb->m_pdict->iterators++; // fake an iterator so it doesn't rehash
-    if (m_setexpire != nullptr)
-        spdb->m_setexpire = new (MALLOC_LOCAL) expireset(*m_setexpire);
-
-    for (unsigned iht = 0; iht < 2; ++iht)
-    {
-        spdb->m_pdict->ht[iht] = m_pdict->ht[iht];
-        if (m_pdict->ht[iht].size)
-            m_pdict->ht[iht].table = (dictEntry**)zcalloc(m_pdict->ht[iht].size*sizeof(dictEntry*), MALLOC_SHARED);
-        else
-            m_pdict->ht[iht].table = nullptr;
-
-        for (size_t idx = 0; idx < m_pdict->ht[iht].size; ++idx)
-        {
-            const dictEntry *deSrc = spdb->m_pdict->ht[iht].table[idx];
-            dictEntry **pdeDst = &m_pdict->ht[iht].table[idx];
-            while (deSrc != nullptr)
-            {
-                *pdeDst = (dictEntry*)zmalloc(sizeof(dictEntry), MALLOC_SHARED);
-                (*pdeDst)->key = deSrc->key;
-                (*pdeDst)->v.val = nullptr;
-                (*pdeDst)->next = nullptr;
-                pdeDst = &(*pdeDst)->next;
-                deSrc = deSrc->next;
-            }
-        }
-    }
-    
-    m_spdbSnapshot = std::move(spdb);
-    return m_spdbSnapshot;
-}
-
-void redisDbPersistentData::endSnapshot(const redisDbPersistentData *psnapshot)
-{
-    serverAssert(GlobalLocksAcquired());
-    serverAssert(m_spdbSnapshot.get() == psnapshot);
-
-    dictIterator *di = dictGetIterator(m_pdict);
-    dictEntry *de;
-    while ((de = dictNext(di)) != NULL)
-    {
-        dictEntry *deSnapshot = dictFind(m_spdbSnapshot->m_pdict, dictGetKey(de));
-        if (dictGetVal(de) == nullptr)
-        {
-            if (deSnapshot != nullptr)
-            {
-                de->v.val = deSnapshot->v.val;
-                deSnapshot->v.val = nullptr;
-            }
-        }
-        if (deSnapshot && (dictGetKey(deSnapshot) == dictGetKey(de)))
-        {
-            // The key is owned by the parent snapshot, so we modify the DB key dtor
-            //  to ensure the key is not free'd during the delete
-            m_spdbSnapshot->m_pdict->type = &dbSnapshotDictType;
-            dictDelete(m_spdbSnapshot->m_pdict, dictGetKey(de));
-            m_spdbSnapshot->m_pdict->type = &dbDictType;
-        }
-    }
-    dictReleaseIterator(di);
-    m_spdbSnapshot->m_pdict->iterators--;
-    m_spdbSnapshot = nullptr;
-}
-
-redisDbPersistentData::~redisDbPersistentData()
-{
-    dictRelease(m_pdict);
-    delete m_setexpire;
-}

src/debug.cpp

@@ -55,7 +55,7 @@ typedef ucontext_t sigcontext_t;
 #endif
 #endif
 
-bool g_fInCrash = false;
+int g_fInCrash = false;
 
 /* ================================= Debugging ============================== */
 
@@ -126,7 +126,7 @@ void mixStringObjectDigest(unsigned char *digest, robj_roptr o) {
 void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj_roptr o) {
     uint32_t aux = htonl(o->type);
     mixDigest(digest,&aux,sizeof(aux));
-    expireEntry *pexpire = db->getExpire(keyobj);
+    expireEntry *pexpire = getExpire(db,keyobj);
     long long expiretime = -1;
     char buf[128];
 
@@ -266,6 +266,8 @@ void xorObjectDigest(redisDb *db, robj_roptr keyobj, unsigned char *digest, robj
  * a different digest. */
 void computeDatasetDigest(unsigned char *final) {
     unsigned char digest[20];
+    dictIterator *di = NULL;
+    dictEntry *de;
     int j;
     uint32_t aux;
 
@@ -274,7 +276,8 @@ void computeDatasetDigest(unsigned char *final) {
     for (j = 0; j < cserver.dbnum; j++) {
         redisDb *db = g_pserver->db+j;
 
-        if (db->size() == 0) continue;
+        if (dictSize(db->pdict) == 0) continue;
+        di = dictGetSafeIterator(db->pdict);
 
         /* hash the DB id, so the same dataset moved in a different
          * DB will lead to a different digest */
@@ -282,21 +285,24 @@ void computeDatasetDigest(unsigned char *final) {
         mixDigest(final,&aux,sizeof(aux));
 
         /* Iterate this DB writing every entry */
-        db->iterate([&](const char *key, robj *o)->bool {
-            robj *keyobj;
+        while((de = dictNext(di)) != NULL) {
+            sds key;
+            robj *keyobj, *o;
 
             memset(digest,0,20); /* This key-val digest */
+            key = (sds)dictGetKey(de);
             keyobj = createStringObject(key,sdslen(key));
 
             mixDigest(digest,key,sdslen(key));
 
+            o = (robj*)dictGetVal(de);
             xorObjectDigest(db,keyobj,digest,o);
 
             /* We can finally xor the key-val digest to the final digest */
             xorDigest(final,digest,20);
             decrRefCount(keyobj);
-            return true;
-        });
+        }
+        dictReleaseIterator(di);
     }
 }
 
@@ -359,7 +365,7 @@ NULL
     } else if (!strcasecmp(szFromObj(c->argv[1]),"reload")) {
         rdbSaveInfo rsi, *rsiptr;
         rsiptr = rdbPopulateSaveInfo(&rsi);
-        if (rdbSave(nullptr, rsiptr) != C_OK) {
+        if (rdbSave(rsiptr) != C_OK) {
             addReply(c,shared.err);
             return;
         }
@@ -388,14 +394,15 @@ NULL
         serverLog(LL_WARNING,"Append Only File loaded by DEBUG LOADAOF");
         addReply(c,shared.ok);
     } else if (!strcasecmp(szFromObj(c->argv[1]),"object") && c->argc == 3) {
+        dictEntry *de;
         robj *val;
         const char *strenc;
 
-        val = c->db->find(c->argv[2]);
-        if (val == NULL) {
+        if ((de = dictFind(c->db->pdict,ptrFromObj(c->argv[2]))) == NULL) {
             addReply(c,shared.nokeyerr);
             return;
         }
+        val = (robj*)dictGetVal(de);
         strenc = strEncoding(val->encoding);
 
         char extra[138] = {0};
@@ -439,14 +446,16 @@ NULL
             strenc, rdbSavedObjectLen(val),
             val->lru, estimateObjectIdleTime(val)/1000, extra);
     } else if (!strcasecmp(szFromObj(c->argv[1]),"sdslen") && c->argc == 3) {
-        auto itr = c->db->find(c->argv[2]);
-        robj *val = itr.val();
-        const char *key = itr.key();
+        dictEntry *de;
+        robj *val;
+        sds key;
 
-        if (val == NULL) {
+        if ((de = dictFind(c->db->pdict,ptrFromObj(c->argv[2]))) == NULL) {
             addReply(c,shared.nokeyerr);
             return;
         }
+        val = (robj*)dictGetVal(de);
+        key = (sds)dictGetKey(de);
 
         if (val->type != OBJ_STRING || !sdsEncodedObject(val)) {
             addReplyError(c,"Not an sds encoded string.");
@@ -456,16 +465,16 @@ NULL
                 "val_sds_len:%lld, val_sds_avail:%lld, val_zmalloc: %lld",
                 (long long) sdslen(key),
                 (long long) sdsavail(key),
-                (long long) sdsZmallocSize((sds)key),
+                (long long) sdsZmallocSize(key),
                 (long long) sdslen(szFromObj(val)),
                 (long long) sdsavail(szFromObj(val)),
                 (long long) getStringObjectSdsUsedMemory(val));
         }
     } else if (!strcasecmp(szFromObj(c->argv[1]),"ziplist") && c->argc == 3) {
-        robj_roptr o;
+        robj *o;
 
         if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr))
-                == nullptr) return;
+                == NULL) return;
 
         if (o->encoding != OBJ_ENCODING_ZIPLIST) {
             addReplyError(c,"Not an sds encoded string.");
@@ -481,7 +490,7 @@ NULL
 
         if (getLongFromObjectOrReply(c, c->argv[2], &keys, NULL) != C_OK)
             return;
-        c->db->expand(keys);
+        dictExpand(c->db->pdict,keys);
         for (j = 0; j < keys; j++) {
             long valsize = 0;
             snprintf(buf,sizeof(buf),"%s:%lu",
@@ -632,20 +641,20 @@ NULL
         }
 
         stats = sdscatprintf(stats,"[Dictionary HT]\n");
-        g_pserver->db[dbid].getStats(buf,sizeof(buf));
+        dictGetStats(buf,sizeof(buf),g_pserver->db[dbid].pdict);
         stats = sdscat(stats,buf);
 
         stats = sdscatprintf(stats,"[Expires set]\n");
-        g_pserver->db[dbid].getExpireStats(buf, sizeof(buf));
+        g_pserver->db[dbid].setexpire->getstats(buf, sizeof(buf));
         stats = sdscat(stats, buf);
 
         addReplyBulkSds(c,stats);
     } else if (!strcasecmp(szFromObj(c->argv[1]),"htstats-key") && c->argc == 3) {
-        robj_roptr o;
+        robj *o;
         dict *ht = NULL;
 
         if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.nokeyerr))
-                == nullptr) return;
+                == NULL) return;
 
         /* Get the hash table reference from the object, if possible. */
         switch (o->encoding) {
@@ -1247,10 +1256,12 @@ void logCurrentClient(void) {
      * selected DB, and if so print info about the associated object. */
     if (cc->argc >= 1) {
         robj *val, *key;
+        dictEntry *de;
 
         key = getDecodedObject(cc->argv[1]);
-        val = cc->db->find(key);
-        if (val) {
+        de = dictFind(cc->db->pdict, ptrFromObj(key));
+        if (de) {
+            val = (robj*)dictGetVal(de);
             serverLog(LL_WARNING,"key '%s' found in DB containing the following object:", (char*)ptrFromObj(key));
             serverLogObjectDebugInfo(val);
         }

src/defrag.cpp

@@ -781,14 +781,12 @@ long defragKey(redisDb *db, dictEntry *de) {
     newsds = activeDefragSds(keysds);
     if (newsds)
         defragged++, de->key = newsds;
-    if (!db->setexpire()->empty()) {
-        replaceSateliteOSetKeyPtr(*const_cast<expireset*>(db->setexpire()), keysds, newsds);
+    if (!db->setexpire->empty()) {
+        replaceSateliteOSetKeyPtr(*db->setexpire, keysds, newsds);
     }
 
     /* Try to defrag robj and / or string value. */
     ob = (robj*)dictGetVal(de);
-    if (ob == nullptr)
-        return defragged;
     if ((newob = activeDefragStringOb(ob, &defragged))) {
         de->v.val = newob;
         ob = newob;
@@ -855,7 +853,7 @@ void defragScanCallback(void *privdata, const dictEntry *de) {
     g_pserver->stat_active_defrag_scanned++;
 }
 
-/* Defrag scan callback for each hash table bucket,
+/* Defrag scan callback for each hash table bicket,
  * used in order to defrag the dictEntry allocations. */
 void defragDictBucketCallback(void *privdata, dictEntry **bucketref) {
     UNUSED(privdata); /* NOTE: this function is also used by both activeDefragCycle and scanLaterHash, etc. don't use privdata */
@@ -904,8 +902,9 @@ long defragOtherGlobals() {
 
 /* returns 0 more work may or may not be needed (see non-zero cursor),
  * and 1 if time is up and more work is needed. */
-int defragLaterItem(robj *ob, unsigned long *cursor, long long endtime) {
-    if (ob) {
+int defragLaterItem(dictEntry *de, unsigned long *cursor, long long endtime) {
+    if (de) {
+        robj *ob = (robj*)dictGetVal(de);
         if (ob->type == OBJ_LIST) {
             g_pserver->stat_active_defrag_hits += scanLaterList(ob);
             *cursor = 0; /* list has no scan, we must finish it in one go */
@@ -960,11 +959,11 @@ int defragLaterStep(redisDb *db, long long endtime) {
         }
 
         /* each time we enter this function we need to fetch the key from the dict again (if it still exists) */
-        robj *o = db->find(current_key);
+        dictEntry *de = dictFind(db->pdict, current_key);
         key_defragged = g_pserver->stat_active_defrag_hits;
         do {
             int quit = 0;
-            if (defragLaterItem(o, &cursor, endtime))
+            if (defragLaterItem(de, &cursor, endtime))
                 quit = 1; /* time is up, we didn't finish all the work */
 
             /* Don't start a new BIG key in this loop, this is because the
@@ -1046,7 +1045,7 @@ void activeDefragCycle(void) {
     mstime_t latency;
     int quit = 0;
 
-    if (g_pserver->aof_child_pid!=-1 || g_pserver->FRdbSaveInProgress())
+    if (g_pserver->aof_child_pid!=-1 || g_pserver->rdb_child_pid!=-1)
         return; /* Defragging memory while there's a fork will just do damage. */
 
     /* Once a second, check if we the fragmentation justfies starting a scan
@@ -1113,8 +1112,7 @@ void activeDefragCycle(void) {
                 break; /* this will exit the function and we'll continue on the next cycle */
             }
 
-            // we actually look at the objects too but defragScanCallback can handle missing values
-            cursor = dictScan(db->dictUnsafeKeyOnly(), cursor, defragScanCallback, defragDictBucketCallback, db);
+            cursor = dictScan(db->pdict, cursor, defragScanCallback, defragDictBucketCallback, db);
 
             /* Once in 16 scan iterations, 512 pointer reallocations. or 64 keys
              * (if we have a lot of pointers in one hash bucket or rehasing),

src/evict.cpp

@@ -87,7 +87,7 @@ unsigned int LRU_CLOCK(void) {
 
 /* Given an object returns the min number of milliseconds the object was never
  * requested, using an approximated LRU algorithm. */
-unsigned long long estimateObjectIdleTime(robj_roptr o) {
+unsigned long long estimateObjectIdleTime(robj *o) {
     unsigned long long lruclock = LRU_CLOCK();
     if (lruclock >= o->lru) {
         return (lruclock - o->lru) * LRU_CLOCK_RESOLUTION;
@@ -252,20 +252,19 @@ struct visitFunctor
         return count < g_pserver->maxmemory_samples;
     }
 };
-void evictionPoolPopulate(int dbid, redisDb *db, expireset *setexpire, struct evictionPoolEntry *pool)
+void evictionPoolPopulate(int dbid, dict *dbdict, expireset *setexpire, struct evictionPoolEntry *pool)
 {
     if (setexpire != nullptr)
     {
-        visitFunctor visitor { dbid, db->dictUnsafeKeyOnly(), pool, 0 };
+        visitFunctor visitor { dbid, dbdict, pool, 0 };
         setexpire->random_visit(visitor);
     }
     else
     {
         dictEntry **samples = (dictEntry**)alloca(g_pserver->maxmemory_samples * sizeof(dictEntry*));
-        int count = dictGetSomeKeys(db->dictUnsafeKeyOnly(),samples,g_pserver->maxmemory_samples);
+        int count = dictGetSomeKeys(dbdict,samples,g_pserver->maxmemory_samples);
         for (int j = 0; j < count; j++) {
             robj *o = (robj*)dictGetVal(samples[j]);
-            serverAssert(o != nullptr); // BUG!!! We have to get the info we need here without permanently rehydrating the obj
             processEvictionCandidate(dbid, (sds)dictGetKey(samples[j]), o, nullptr, pool);
         }
     }
@@ -505,16 +504,16 @@ int freeMemoryIfNeeded(void) {
                     db = g_pserver->db+i;
                     if (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS)
                     {
-                        if ((keys = db->size()) != 0) {
-                            evictionPoolPopulate(i, db, nullptr, pool);
+                        if ((keys = dictSize(db->pdict)) != 0) {
+                            evictionPoolPopulate(i, db->pdict, nullptr, pool);
                             total_keys += keys;
                         }
                     }
                     else
                     {
-                        keys = db->expireSize();
+                        keys = db->setexpire->size();
                         if (keys != 0)
-                            evictionPoolPopulate(i, db, db->setexpireUnsafe(), pool);
+                            evictionPoolPopulate(i, db->pdict, db->setexpire, pool);
                         total_keys += keys;
                     }
                 }
@@ -526,9 +525,9 @@ int freeMemoryIfNeeded(void) {
                     bestdbid = pool[k].dbid;
                     sds key = nullptr;
 
-                    auto itr = g_pserver->db[pool[k].dbid].find(pool[k].key);
-                    if (itr != nullptr && (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS || itr.val()->FExpires()))
-                        key = itr.key();
+                    dictEntry *de = dictFind(g_pserver->db[pool[k].dbid].pdict,pool[k].key);
+                    if (de != nullptr && (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_ALLKEYS || ((robj*)dictGetVal(de))->FExpires()))
+                        key = (sds)dictGetKey(de);
 
                     /* Remove the entry from the pool. */
                     if (pool[k].key != pool[k].cached)
@@ -560,18 +559,18 @@ int freeMemoryIfNeeded(void) {
                 db = g_pserver->db+j;
                 if (g_pserver->maxmemory_policy == MAXMEMORY_ALLKEYS_RANDOM)
                 {
-                    if (db->size() != 0) {
-                        auto itr = db->random();
-                        bestkey = itr.key();
+                    if (dictSize(db->pdict) != 0) {
+                        dictEntry *de = dictGetRandomKey(db->pdict);
+                        bestkey = (sds)dictGetKey(de);
                         bestdbid = j;
                         break;
                     }
                 }
                 else
                 {
-                    if (db->expireSize())
+                    if (!db->setexpire->empty())
                     {
-                        bestkey = (sds)db->random_expire().key();
+                        bestkey = (sds)db->setexpire->random_value().key();
                         bestdbid = j;
                         break;
                     }
@@ -582,17 +581,16 @@ int freeMemoryIfNeeded(void) {
         /* Finally remove the selected key. */
         if (bestkey) {
             db = g_pserver->db+bestdbid;
-
             robj *keyobj = createStringObject(bestkey,sdslen(bestkey));
             propagateExpire(db,keyobj,g_pserver->lazyfree_lazy_eviction);
             /* We compute the amount of memory freed by db*Delete() alone.
-            * It is possible that actually the memory needed to propagate
-            * the DEL in AOF and replication link is greater than the one
-            * we are freeing removing the key, but we can't account for
-            * that otherwise we would never exit the loop.
-            *
-            * AOF and Output buffer memory will be freed eventually so
-            * we only care about memory used by the key space. */
+             * It is possible that actually the memory needed to propagate
+             * the DEL in AOF and replication link is greater than the one
+             * we are freeing removing the key, but we can't account for
+             * that otherwise we would never exit the loop.
+             *
+             * AOF and Output buffer memory will be freed eventually so
+             * we only care about memory used by the key space. */
             delta = (long long) zmalloc_used_memory();
             latencyStartMonitor(eviction_latency);
             if (g_pserver->lazyfree_lazy_eviction)
@@ -611,18 +609,18 @@ int freeMemoryIfNeeded(void) {
             keys_freed++;
 
             /* When the memory to free starts to be big enough, we may
-            * start spending so much time here that is impossible to
-            * deliver data to the slaves fast enough, so we force the
-            * transmission here inside the loop. */
+             * start spending so much time here that is impossible to
+             * deliver data to the slaves fast enough, so we force the
+             * transmission here inside the loop. */
             if (slaves) flushSlavesOutputBuffers();
 
             /* Normally our stop condition is the ability to release
-            * a fixed, pre-computed amount of memory. However when we
-            * are deleting objects in another thread, it's better to
-            * check, from time to time, if we already reached our target
-            * memory, since the "mem_freed" amount is computed only
-            * across the dbAsyncDelete() call, while the thread can
-            * release the memory all the time. */
+             * a fixed, pre-computed amount of memory. However when we
+             * are deleting objects in another thread, it's better to
+             * check, from time to time, if we already reached our target
+             * memory, since the "mem_freed" amount is computed only
+             * across the dbAsyncDelete() call, while the thread can
+             * release the memory all the time. */
             if (g_pserver->lazyfree_lazy_eviction && !(keys_freed % 16)) {
                 if (getMaxmemoryState(NULL,NULL,NULL,NULL) == C_OK) {
                     /* Let's satisfy our stop condition. */

src/expire.cpp

@@ -74,7 +74,8 @@ void activeExpireCycleExpire(redisDb *db, expireEntry &e, long long now) {
     }
 
     expireEntryFat *pfat = e.pfatentry();
-    robj *val = db->find(e.key());
+    dictEntry *de = dictFind(db->pdict, e.key());
+    robj *val = (robj*)dictGetVal(de);
     int deleted = 0;
     while (!pfat->FEmpty())
     {
@@ -156,7 +157,7 @@ void expireMemberCore(client *c, robj *key, robj *subkey, long long basetime, lo
 
     /* No key, return zero. */
     robj *val = lookupKeyWriteOrReply(c, key, shared.czero);
-    if (val == nullptr) {
+    if (val == NULL) {
         return;
     }
 
@@ -295,7 +296,7 @@ void activeExpireCycle(int type) {
         now = mstime();
 
         /* If there is nothing to expire try next DB ASAP. */
-        if (db->setexpireUnsafe()->empty())
+        if (db->setexpire->empty())
         {
             db->avg_ttl = 0;
             db->last_expire_set = now;
@@ -305,7 +306,7 @@ void activeExpireCycle(int type) {
         size_t expired = 0;
         size_t tried = 0;
         long long check = ACTIVE_EXPIRE_CYCLE_FAST_DURATION;    // assume a check is roughly 1us.  It isn't but good enough
-        db->expireitr = db->setexpireUnsafe()->enumerate(db->expireitr, now, [&](expireEntry &e) __attribute__((always_inline)) {
+        db->expireitr = db->setexpire->enumerate(db->expireitr, now, [&](expireEntry &e) __attribute__((always_inline)) {
             if (e.when() < now)
             {
                 activeExpireCycleExpire(db, e, now);
@@ -405,16 +406,16 @@ void expireSlaveKeys(void) {
                 redisDb *db = g_pserver->db+dbid;
 
                 // the expire is hashed based on the key pointer, so we need the point in the main db
-                auto itrDB = db->find(keyname);
-                auto itrExpire = db->setexpire()->end();
-                if (itrDB != nullptr)
-                    itrExpire = db->setexpireUnsafe()->find(itrDB.key());
+                dictEntry *deMain = dictFind(db->pdict, keyname);
+                auto itr = db->setexpire->end();
+                if (deMain != nullptr)
+                    itr = db->setexpire->find((sds)dictGetKey(deMain));
                 int expired = 0;
 
-                if (itrExpire != db->setexpire()->end())
+                if (itr != db->setexpire->end())
                 {
-                    if (itrExpire->when() < start) {
-                        activeExpireCycleExpire(g_pserver->db+dbid,*itrExpire,start);
+                    if (itr->when() < start) {
+                        activeExpireCycleExpire(g_pserver->db+dbid,*itr,start);
                         expired = 1;
                     }
                 }
@@ -423,7 +424,7 @@ void expireSlaveKeys(void) {
                  * corresponding bit in the new bitmap we set as value.
                  * At the end of the loop if the bitmap is zero, it means we
                  * no longer need to keep track of this key. */
-                if (itrExpire != db->setexpire()->end() && !expired) {
+                if (itr != db->setexpire->end() && !expired) {
                     noexpire++;
                     new_dbids |= (uint64_t)1 << dbid;
                 }
@@ -591,7 +592,7 @@ void ttlGenericCommand(client *c, int output_ms) {
 
     /* The key exists. Return -1 if it has no expire, or the actual
         * TTL value otherwise. */
-    expireEntry *pexpire = c->db->getExpire(c->argv[1]);
+    expireEntry *pexpire = getExpire(c->db,c->argv[1]);
 
     if (c->argc == 2) {
         // primary expire    
@@ -647,7 +648,7 @@ void persistCommand(client *c) {
                 addReply(c,shared.czero);
             }
         } else if (c->argc == 3) {
-            if (c->db->removeSubkeyExpire(c->argv[1], c->argv[2])) {
+            if (removeSubkeyExpire(c->db, c->argv[1], c->argv[2])) {
                 addReply(c,shared.cone);
                 g_pserver->dirty++;
             } else {

src/fastlock.cpp

@@ -36,10 +36,13 @@
 #include <assert.h>
 #include <pthread.h>
 #include <limits.h>
+#include <map>
 #ifdef __linux__
 #include <linux/futex.h>
 #endif
 #include <string.h>
+#include <stdarg.h>
+#include <stdio.h>
 
 #ifdef __APPLE__
 #include <TargetConditionals.h>
@@ -57,6 +60,7 @@
 #define UNUSED(x) ((void)x)
 #endif
 
+extern int g_fInCrash;
 
 /****************************************************
  *
@@ -125,6 +129,80 @@
 
 #endif
 
+#pragma weak _serverPanic
+extern "C"  __attribute__((weak)) void _serverPanic(const char * /*file*/, int /*line*/, const char * /*msg*/, ...)
+{
+    *((char*)-1) = 'x';
+}
+
+#pragma weak serverLog
+__attribute__((weak)) void serverLog(int , const char *fmt, ...)
+{
+    va_list args;
+    va_start(args, fmt);
+    vprintf(fmt, args);
+    va_end(args);
+    printf("\n");
+}
+
+class DeadlockDetector
+{
+    std::map<pid_t, fastlock *> m_mapwait;
+    fastlock m_lock { "deadlock detector" };
+public:
+    void registerwait(fastlock *lock, pid_t thispid)
+    {
+        if (lock == &m_lock || g_fInCrash)
+            return;
+        fastlock_lock(&m_lock);
+        m_mapwait.insert(std::make_pair(thispid, lock));
+
+        // Detect cycles
+        pid_t pidCheck = thispid;
+        size_t cchecks = 0;
+        for (;;)
+        {
+            auto itr = m_mapwait.find(pidCheck);
+            if (itr == m_mapwait.end())
+                break;
+            pidCheck = itr->second->m_pidOwner;
+            if (pidCheck == thispid)
+            {
+                // Deadlock detected, printout some debugging info and crash
+                serverLog(3 /*LL_WARNING*/, "\n\n");
+                serverLog(3 /*LL_WARNING*/, "!!! ERROR: Deadlock detected !!!");
+                pidCheck = thispid;
+                for (;;)
+                {
+                    auto itr = m_mapwait.find(pidCheck);
+                    serverLog(3 /* LL_WARNING */, "\t%d: (%p) %s", pidCheck, itr->second, itr->second->szName);
+                    pidCheck = itr->second->m_pidOwner;
+                    if (pidCheck == thispid)
+                        break;
+                }
+                serverLog(3 /*LL_WARNING*/, "!!! KeyDB Will Now Crash !!!");
+                _serverPanic(__FILE__, __LINE__, "Deadlock detected");
+            }
+
+            if (cchecks > m_mapwait.size())
+                break;  // There is a cycle but we're not in it
+            ++cchecks;
+        }
+        fastlock_unlock(&m_lock);
+    }
+
+    void clearwait(fastlock *lock, pid_t thispid)
+    {
+        if (lock == &m_lock || g_fInCrash)
+            return;
+        fastlock_lock(&m_lock);
+        m_mapwait.erase(thispid);
+        fastlock_unlock(&m_lock);
+    }
+};
+
+DeadlockDetector g_dlock;
+
 static_assert(sizeof(pid_t) <= sizeof(fastlock::m_pidOwner), "fastlock::m_pidOwner not large enough");
 uint64_t g_longwaits = 0;
 
@@ -135,7 +213,6 @@ uint64_t fastlock_getlongwaitcount()
     return rval;
 }
 
-#ifndef ASM_SPINLOCK
 #ifdef __linux__
 static int futex(volatile unsigned *uaddr, int futex_op, int val,
     const struct timespec *timeout, int val3)
@@ -144,7 +221,6 @@ static int futex(volatile unsigned *uaddr, int futex_op, int val,
                     timeout, uaddr, val3);
 }
 #endif
-#endif
 
 extern "C" pid_t gettid()
 {
@@ -163,13 +239,29 @@ extern "C" pid_t gettid()
     return pidCache;
 }
 
-extern "C" void fastlock_init(struct fastlock *lock)
+extern "C" void fastlock_sleep(fastlock *lock, pid_t pid, unsigned wake, unsigned mask)
+{
+#ifdef __linux__
+    g_dlock.registerwait(lock, pid);
+    __atomic_fetch_or(&lock->futex, mask, __ATOMIC_ACQUIRE);
+    futex(&lock->m_ticket.u, FUTEX_WAIT_BITSET_PRIVATE, wake, nullptr, mask);
+    __atomic_fetch_and(&lock->futex, ~mask, __ATOMIC_RELEASE);
+    g_dlock.clearwait(lock, pid);
+#endif
+    __atomic_fetch_add(&g_longwaits, 1, __ATOMIC_RELAXED);
+}
+
+extern "C" void fastlock_init(struct fastlock *lock, const char *name)
 {
     lock->m_ticket.m_active = 0;
     lock->m_ticket.m_avail = 0;
     lock->m_depth = 0;
     lock->m_pidOwner = -1;
     lock->futex = 0;
+    int cch = strlen(name);
+    cch = std::min<int>(cch, sizeof(lock->szName)-1);
+    memcpy(lock->szName, name, cch);
+    lock->szName[cch] = '\0';
     ANNOTATE_RWLOCK_CREATE(lock);
 }
 
@@ -184,12 +276,12 @@ extern "C" void fastlock_lock(struct fastlock *lock)
         return;
     }
 
+    int tid = gettid();
     unsigned myticket = __atomic_fetch_add(&lock->m_ticket.m_avail, 1, __ATOMIC_RELEASE);
-#ifdef __linux__
     unsigned mask = (1U << (myticket % 32));
-#endif
     int cloops = 0;
     ticket ticketT;
+
     for (;;)
     {
         __atomic_load(&lock->m_ticket.u, &ticketT.u, __ATOMIC_ACQUIRE);
@@ -201,17 +293,11 @@ extern "C" void fastlock_lock(struct fastlock *lock)
 #endif
         if ((++cloops % 1024*1024) == 0)
         {
-#ifdef __linux__
-            __atomic_fetch_or(&lock->futex, mask, __ATOMIC_ACQUIRE);
-            futex(&lock->m_ticket.u, FUTEX_WAIT_BITSET_PRIVATE, ticketT.u, nullptr, mask);
-            __atomic_fetch_and(&lock->futex, ~mask, __ATOMIC_RELEASE);
-#endif
-            __atomic_fetch_add(&g_longwaits, 1, __ATOMIC_RELAXED);
+            fastlock_sleep(lock, tid, ticketT.u, mask);
         }
     }
 
     lock->m_depth = 1;
-    int tid = gettid();
     __atomic_store(&lock->m_pidOwner, &tid, __ATOMIC_RELEASE);
     ANNOTATE_RWLOCK_ACQUIRED(lock, true);
     std::atomic_thread_fence(std::memory_order_acquire);

src/fastlock.h

@@ -1,5 +1,6 @@
 #pragma once
 #include <inttypes.h>
+#include <stddef.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -7,7 +8,7 @@ extern "C" {
 
 /* Begin C API */
 struct fastlock;
-void fastlock_init(struct fastlock *lock);
+void fastlock_init(struct fastlock *lock, const char *name);
 void fastlock_lock(struct fastlock *lock);
 int fastlock_trylock(struct fastlock *lock, int fWeak);
 void fastlock_unlock(struct fastlock *lock);
@@ -40,29 +41,31 @@ struct ticket
 
 struct fastlock
 {
-    volatile struct ticket m_ticket;
-
     volatile int m_pidOwner;
     volatile int m_depth;
+    char szName[56];
+    /* Volatile data on seperate cache line */
+    volatile struct ticket m_ticket;
     unsigned futex;
+    char padding[56];   // ensure ticket and futex are on their own independent cache line
 
 #ifdef __cplusplus
-    fastlock()
+    fastlock(const char *name)
     {
-        fastlock_init(this);
+        fastlock_init(this, name);
     }
 
-    void lock()
+    inline void lock()
     {
         fastlock_lock(this);
     }
 
-    bool try_lock(bool fWeak = false)
+    inline bool try_lock(bool fWeak = false)
     {
         return !!fastlock_trylock(this, fWeak);
     }
 
-    void unlock()
+    inline void unlock()
     {
         fastlock_unlock(this);
     }
@@ -80,3 +83,5 @@ struct fastlock
     bool fOwnLock();   // true if this thread owns the lock, NOTE: not 100% reliable, use for debugging only
 #endif
 };
+
+static_assert(offsetof(struct fastlock, m_ticket) == 64, "ensure padding is correct");

src/fastlock_x64.asm

@@ -1,156 +1,233 @@
-section .text
+.intel_syntax noprefix
+.section .note.GNU-stack,"",@progbits
 
-extern gettid
-extern sched_yield
-extern g_longwaits
+.extern gettid
+.extern fastlock_sleep
 
-;	This is the first use of assembly in this codebase, a valid question is WHY?
-;	The spinlock we implement here is performance critical, and simply put GCC
-;	emits awful code.  The original C code is left in fastlock.cpp for reference
-;	and x-plat.
+.data
+.align 8
+fastlock_unlock_vec:
+	.quad fastlock_unlock_vanilla
+fastlock_lock_vec:
+	.quad cpuid_check_tsxrtm_polyphill
+fastlock_trylock_vec:
+	.quad fastlock_trylock_vanilla
 
-ALIGN 16
-global fastlock_lock
+.text
+.global fastlock_unlock
+fastlock_unlock:
+	mov rax, [rip + fastlock_unlock_vec]
+	jmp rax
+.global fastlock_lock
+.type   fastlock_lock,@function
 fastlock_lock:
-	; RDI points to the struct:
-	;	uint16_t active
-	;	uint16_t avail
-	;	int32_t m_pidOwner
-	;	int32_t m_depth
+	mov rax, [rip + fastlock_lock_vec]
+	jmp rax
+.global fastlock_trylock
+.type   fastlock_trylock,@function
+fastlock_trylock:
+	mov rax, [rip + fastlock_trylock_vec]
+	jmp rax
+
+#	This is the first use of assembly in this codebase, a valid question is WHY?
+#	The spinlock we implement here is performance critical, and simply put GCC
+#	emits awful code.  The original C code is left in fastlock.cpp for reference
+#	and x-plat.
+
+cpuid_check_tsxrtm_polyphill:
+	mov eax, 7
+	xor ecx, ecx
+	cpuid
+	and ebx, 0x800
+	jnz .LHasTSX
+	# CPU does not have TSX RTM support
+	lea rax, [rip+fastlock_lock_vanilla]
+	mov [rip + fastlock_lock_vec], rax
+	jmp rax
+.LHasTSX:
+	lea rax, [rip+fastlock_lock_tsxrtm]
+	mov [rip + fastlock_lock_vec], rax
+	lea rax, [rip+fastlock_trylock_tsxrtm]
+	mov [rip + fastlock_trylock_vec], rax
+	lea rax, [rip+fastlock_unlock_tsxrtm]
+	mov [rip + fastlock_unlock_vec], rax
+	jmp fastlock_lock_tsxrtm
+
+.ALIGN 16
+fastlock_lock_tsxrtm:
+	xbegin fastlock_lock_vanilla
+fastlock_lock_tsxrtmcore:
+	# make sure the lock is not taken (do this inside the TX)
+	mov eax, [rdi+64]
+	mov ecx, eax            # duplicate in ecx
+	ror ecx, 16             # swap upper and lower 16-bits
+	cmp eax, ecx            # are the upper and lower 16-bits the same?
+	jne .LContested
+	mov eax, 1				# this is not needed for the regular lock, but is for try lock which shares this body
+	ret
+.LContested:
+	xabort 0
+
+.ALIGN 16
+fastlock_trylock_tsxrtm:
+	xbegin fastlock_trylock_vanilla
+	jmp fastlock_lock_tsxrtmcore
+
+.ALIGN 16
+fastlock_unlock_tsxrtm:
+	mov eax, [rdi+64]
+	mov ecx, eax            # duplicate in ecx
+	ror ecx, 16             # swap upper and lower 16-bits
+	cmp eax, ecx            # are the upper and lower 16-bits the same?
+	jne fastlock_unlock_vanilla
+	xend
+	ret
+
+.ALIGN 16
+fastlock_lock_vanilla:
+	.cfi_startproc
+	.cfi_def_cfa rsp, 8
+	# RDI points to the struct:
+	#	int32_t m_pidOwner
+	#	int32_t m_depth
+	# [rdi+64] ...
+	#	uint16_t active
+	#	uint16_t avail
 	
-	; First get our TID and put it in ecx
-	push rdi                ; we need our struct pointer (also balance the stack for the call)
-	call gettid             ; get our thread ID (TLS is nasty in ASM so don't bother inlining)
-	mov esi, eax            ; back it up in esi
-	pop rdi                 ; get our pointer back
+	# First get our TID and put it in ecx
+	push rdi                # we need our struct pointer (also balance the stack for the call)
+	.cfi_adjust_cfa_offset 8
+	call gettid             # get our thread ID (TLS is nasty in ASM so don't bother inlining)
+	mov esi, eax            # back it up in esi
+	pop rdi                 # get our pointer back
+	.cfi_adjust_cfa_offset -8
 
-	cmp [rdi+4], esi        ; Is the TID we got back the owner of the lock?
-	je .LLocked             ; Don't spin in that case
+	cmp [rdi], esi          # Is the TID we got back the owner of the lock?
+	je .LLocked             # Don't spin in that case
 
-	xor eax, eax            ; eliminate partial register dependency
-	inc eax                 ; we want to add one
-	lock xadd [rdi+2], ax   ; do the xadd, ax contains the value before the addition
-	; ax now contains the ticket
-ALIGN 16
+	xor eax, eax            # eliminate partial register dependency
+	inc eax                 # we want to add one
+	lock xadd [rdi+66], ax  # do the xadd, ax contains the value before the addition
+	# ax now contains the ticket
+	# OK Start the wait loop
+	xor ecx, ecx
+.ALIGN 16
 .LLoop:
-	mov edx, [rdi]
-	cmp dx, ax              ; is our ticket up?
-	je .LLocked             ; leave the loop
+	mov edx, [rdi+64]
+	cmp dx, ax              # is our ticket up?
+	je .LLocked             # leave the loop
 	pause
-	add ecx, 1000h          ; Have we been waiting a long time? (oflow if we have)
-	                        ;	1000h is set so we overflow on the 1024*1024'th iteration (like the C code)
-	jnc .LLoop              ; If so, give up our timeslice to someone who's doing real work
-	; Like the compiler, you're probably thinking: "Hey! I should take these pushs out of the loop"
-	;	But the compiler doesn't know that we rarely hit this, and when we do we know the lock is
-	;	taking a long time to be released anyways.  We optimize for the common case of short
-	;	lock intervals.  That's why we're using a spinlock in the first place
-	; If we get here we're going to sleep in the kernel with a futex
+	add ecx, 0x1000         # Have we been waiting a long time? (oflow if we have)
+	                        #	1000h is set so we overflow on the 1024*1024'th iteration (like the C code)
+	jnc .LLoop              # If so, give up our timeslice to someone who's doing real work
+	# Like the compiler, you're probably thinking: "Hey! I should take these pushs out of the loop"
+	#	But the compiler doesn't know that we rarely hit this, and when we do we know the lock is
+	#	taking a long time to be released anyways.  We optimize for the common case of short
+	#	lock intervals.  That's why we're using a spinlock in the first place
+	# If we get here we're going to sleep in the kernel with a futex
+	push rdi
 	push rsi
 	push rax
-	; Setup the syscall args
-                            ; rdi ARG1 futex (already in rdi)
-	mov esi, (9 | 128)      ; rsi ARG2 FUTEX_WAIT_BITSET_PRIVATE
-                            ; rdx ARG3 ticketT.u (already in edx)
-	xor r10d, r10d          ; r10 ARG4 NULL
-	mov r8, rdi	            ; r8 ARG5 dup rdi
-	xor r9d, r9d
-	bts r9d, eax            ; r9 ARG6 mask
-	mov eax, 202            ; sys_futex
-	; Do the syscall
-	lock or [rdi+12], r9d   ; inform the unlocking thread we're waiting
-	syscall	                ; wait for the futex
-	not r9d	                ; convert our flag into a mask of bits not to touch
-	lock and [rdi+12], r9d  ; clear the flag in the futex control mask
-	; cleanup and continue
-	mov rcx, g_longwaits
-	inc qword [rcx]         ; increment our long wait counter
+	.cfi_adjust_cfa_offset 24
+	# Setup the syscall args
+
+                            # rdi ARG1 futex (already in rdi)
+	                        # rsi ARG2 tid (already in esi)
+                            # rdx ARG3 ticketT.u (already in edx)
+	bts ecx, eax            # rcx ARG4 mask
+	call fastlock_sleep
+	# cleanup and continue
 	pop rax
 	pop rsi
-	xor ecx, ecx            ; Reset our loop counter
-	jmp .LLoop              ; Get back in the game
-ALIGN 16
+	pop rdi
+	.cfi_adjust_cfa_offset -24
+	xor ecx, ecx            # Reset our loop counter
+	jmp .LLoop              # Get back in the game
+.ALIGN 16
 .LLocked:
-	mov [rdi+4], esi        ; lock->m_pidOwner = gettid()
-	inc dword [rdi+8]       ; lock->m_depth++
+	mov [rdi], esi          # lock->m_pidOwner = gettid()
+	inc dword ptr [rdi+4]   # lock->m_depth++
 	ret
+.cfi_endproc
 
-ALIGN 16
-global fastlock_trylock
-fastlock_trylock:
-	; RDI points to the struct:
-	;	uint16_t active
-	;	uint16_t avail
-	;	int32_t m_pidOwner
-	;	int32_t m_depth
+.ALIGN 16
+fastlock_trylock_vanilla:
+	# RDI points to the struct:
+	#	int32_t m_pidOwner
+	#	int32_t m_depth
+	# [rdi+64] ...
+	#	uint16_t active
+	#	uint16_t avail
 	
-	; First get our TID and put it in ecx
-	push rdi                ; we need our struct pointer (also balance the stack for the call)
-	call gettid             ; get our thread ID (TLS is nasty in ASM so don't bother inlining)
-	mov esi, eax            ; back it up in esi
-	pop rdi                 ; get our pointer back
+	# First get our TID and put it in ecx
+	push rdi                # we need our struct pointer (also balance the stack for the call)
+	call gettid             # get our thread ID (TLS is nasty in ASM so don't bother inlining)
+	mov esi, eax            # back it up in esi
+	pop rdi                 # get our pointer back
 
-	cmp [rdi+4], esi        ; Is the TID we got back the owner of the lock?
-	je .LRecursive          ; Don't spin in that case
+	cmp [rdi], esi          # Is the TID we got back the owner of the lock?
+	je .LRecursive          # Don't spin in that case
 
-	mov eax, [rdi]          ; get both active and avail counters
-	mov ecx, eax            ; duplicate in ecx
-	ror ecx, 16             ; swap upper and lower 16-bits
-	cmp eax, ecx            ; are the upper and lower 16-bits the same?
-	jnz .LAlreadyLocked     ;	If not return failure
+	mov eax, [rdi+64]       # get both active and avail counters
+	mov ecx, eax            # duplicate in ecx
+	ror ecx, 16             # swap upper and lower 16-bits
+	cmp eax, ecx            # are the upper and lower 16-bits the same?
+	jnz .LAlreadyLocked     #	If not return failure
 
-	; at this point we know eax+ecx have [avail][active] and they are both the same
-	add ecx, 10000h         ; increment avail, ecx is now our wanted value
-	lock cmpxchg [rdi], ecx ;	If rdi still contains the value in eax, put in ecx (inc avail)
-	jnz .LAlreadyLocked     ; If Z is not set then someone locked it while we were preparing
+	# at this point we know eax+ecx have [avail][active] and they are both the same
+	add ecx, 0x10000            # increment avail, ecx is now our wanted value
+	lock cmpxchg [rdi+64], ecx  #	If rdi still contains the value in eax, put in ecx (inc avail)
+	jnz .LAlreadyLocked         # If Z is not set then someone locked it while we were preparing
 	xor eax, eax
-	inc eax                 ; return SUCCESS! (eax=1)
-	mov [rdi+4], esi        ; lock->m_pidOwner = gettid()
-	mov dword [rdi+8], eax  ; lock->m_depth = 1
+	inc eax                     # return SUCCESS! (eax=1)
+	mov [rdi], esi              # lock->m_pidOwner = gettid()
+	mov dword ptr [rdi+4], eax  # lock->m_depth = 1
 	ret
-ALIGN 16
+.ALIGN 16
 .LRecursive:
 	xor eax, eax
-	inc eax                 ; return SUCCESS! (eax=1)
-	inc dword [rdi+8]       ; lock->m_depth++
+	inc eax                 # return SUCCESS! (eax=1)
+	inc dword ptr [rdi+4]   # lock->m_depth++
 	ret
-ALIGN 16
+.ALIGN 16
 .LAlreadyLocked:
-	xor eax, eax            ; return 0;
+	xor eax, eax            # return 0
 	ret
 
-ALIGN 16
-global fastlock_unlock
-fastlock_unlock:
-	; RDI points to the struct:
-	;	uint16_t active
-	;	uint16_t avail
-	;	int32_t m_pidOwner
-	;	int32_t m_depth
+.ALIGN 16
+fastlock_unlock_vanilla:
+	# RDI points to the struct:
+	#	int32_t m_pidOwner
+	#	int32_t m_depth
+	# [rdi+64] ...
+	#	uint16_t active
+	#	uint16_t avail
 	push r11
-	sub dword [rdi+8], 1         ; decrement m_depth, don't use dec because it partially writes the flag register and we don't know its state
-	jnz .LDone                   ; if depth is non-zero this is a recursive unlock, and we still hold it
-	mov dword [rdi+4], -1        ; pidOwner = -1 (we don't own it anymore)
-	mov ecx, [rdi]               ; get current active (this one)
-	inc ecx                      ; bump it to the next thread
-	mov [rdi], cx                ; give up our ticket (note: lock is not required here because the spinlock itself guards this variable)
-	; At this point the lock is removed, however we must wake up any pending futexs
-	mov r9d, 1                   ; eax is the bitmask for 2 threads
-	rol r9d, cl                  ; place the mask in the right spot for the next 2 threads
-ALIGN 16
+	sub dword ptr [rdi+4], 1     # decrement m_depth, don't use dec because it partially writes the flag register and we don't know its state
+	jnz .LDone                   # if depth is non-zero this is a recursive unlock, and we still hold it
+	mov dword ptr [rdi], -1      # pidOwner = -1 (we don't own it anymore)
+	mov ecx, [rdi+64]            # get current active (this one)
+	inc ecx                      # bump it to the next thread
+	mov [rdi+64], cx             # give up our ticket (note: lock is not required here because the spinlock itself guards this variable)
+	# At this point the lock is removed, however we must wake up any pending futexs
+	mov r9d, 1                   # eax is the bitmask for 2 threads
+	rol r9d, cl                  # place the mask in the right spot for the next 2 threads
+	add rdi, 64                  # rdi now points to the token
+.ALIGN 16
 .LRetryWake:
-	mov r11d, [rdi+12]           ; load the futex mask
-	and r11d, r9d                ; are any threads waiting on a futex?
-	jz .LDone                    ; if not we're done.
-	; we have to wake the futexs
-                                 ; rdi ARG1 futex (already in rdi)
-	mov esi, (10 | 128)          ; rsi ARG2 FUTEX_WAKE_BITSET_PRIVATE
-	mov edx, 0x7fffffff          ; rdx ARG3 INT_MAX (number of threads to wake)
-	xor r10d, r10d               ; r10 ARG4 NULL
-	mov r8, rdi                  ; r8 ARG5 dup rdi
-                                 ; r9 ARG6 mask (already set above)
-	mov eax, 202                 ; sys_futex
+	mov r11d, [rdi+4]            # load the futex mask
+	and r11d, r9d                # are any threads waiting on a futex?
+	jz .LDone                    # if not we're done.
+	# we have to wake the futexs
+                                 # rdi ARG1 futex (already in rdi)
+	mov esi, (10 | 128)          # rsi ARG2 FUTEX_WAKE_BITSET_PRIVATE
+	mov edx, 0x7fffffff          # rdx ARG3 INT_MAX (number of threads to wake)
+	xor r10d, r10d               # r10 ARG4 NULL
+	mov r8, rdi                  # r8 ARG5 dup rdi
+                                 # r9 ARG6 mask (already set above)
+	mov eax, 202                 # sys_futex
 	syscall
-	cmp eax, 1                   ; did we wake as many as we expected?
+	cmp eax, 1                   # did we wake as many as we expected?
 	jnz .LRetryWake
 .LDone:
 	pop r11

src/geohash_helper.cpp

@@ -46,13 +46,13 @@
 #define ECCENT (sqrt(1.0 - (RATIO *RATIO)))
 #define COM (0.5 * ECCENT)
 
+/// @brief The usual PI/180 constant
+const double DEG_TO_RAD = 0.017453292519943295769236907684886;
 /// @brief Earth's quatratic mean radius for WGS-84
 const double EARTH_RADIUS_IN_METERS = 6372797.560856;
 
 const double MERCATOR_MAX = 20037726.37;
-#if 0 //unused
 const double MERCATOR_MIN = -20037726.37;
-#endif
 
 static inline double deg_rad(double ang) { return ang * D_R; }
 static inline double rad_deg(double ang) { return ang / D_R; }

src/lazyfree.cpp

@@ -51,18 +51,18 @@ size_t lazyfreeGetFreeEffort(robj *obj) {
  * a lazy free list instead of being freed synchronously. The lazy free list
  * will be reclaimed in a different bio.c thread. */
 #define LAZYFREE_THRESHOLD 64
-bool redisDbPersistentData::asyncDelete(robj *key) {
+int dbAsyncDelete(redisDb *db, robj *key) {
     /* If the value is composed of a few allocations, to free in a lazy way
      * is actually just slower... So under a certain limit we just free
      * the object synchronously. */
-    dictEntry *de = dictUnlink(m_pdict,ptrFromObj(key));
+    dictEntry *de = dictUnlink(db->pdict,ptrFromObj(key));
     if (de) {
         robj *val = (robj*)dictGetVal(de);
         if (val->FExpires())
         {
             /* Deleting an entry from the expires dict will not free the sds of
              * the key, because it is shared with the main dictionary. */
-            removeExpire(key,dict_iter(de));
+            removeExpireCore(db,key,de);
         }
 
         size_t free_effort = lazyfreeGetFreeEffort(val);
@@ -78,25 +78,21 @@ bool redisDbPersistentData::asyncDelete(robj *key) {
         if (free_effort > LAZYFREE_THRESHOLD && val->getrefcount(std::memory_order_relaxed) == 1) {
             atomicIncr(lazyfree_objects,1);
             bioCreateBackgroundJob(BIO_LAZY_FREE,val,NULL,NULL);
-            dictSetVal(m_pdict,de,NULL);
+            dictSetVal(db->pdict,de,NULL);
         }
     }
 
     /* Release the key-val pair, or just the key if we set the val
      * field to NULL in order to lazy free it later. */
     if (de) {
-        dictFreeUnlinkedEntry(m_pdict,de);
+        dictFreeUnlinkedEntry(db->pdict,de);
         if (g_pserver->cluster_enabled) slotToKeyDel(key);
-        return true;
+        return 1;
     } else {
-        return false;
+        return 0;
     }
 }
 
-int dbAsyncDelete(redisDb *db, robj *key) {
-    return db->asyncDelete(key);
-}
-
 /* Free an object, if the object is huge enough, free it in async way. */
 void freeObjAsync(robj *o) {
     size_t free_effort = lazyfreeGetFreeEffort(o);
@@ -111,15 +107,12 @@ void freeObjAsync(robj *o) {
 /* Empty a Redis DB asynchronously. What the function does actually is to
  * create a new empty set of hash tables and scheduling the old ones for
  * lazy freeing. */
-void redisDbPersistentData::emptyDbAsync() {
-    dict *oldht1 = m_pdict;
-    auto *set = m_setexpire;
-    m_setexpire = new (MALLOC_LOCAL) expireset();
-    m_pdict = dictCreate(&dbDictType,this);
-    if (m_pstorage != nullptr)
-        m_pstorage->clear();
-    if (m_fTrackingChanges)
-        m_fAllChanged = true;
+void emptyDbAsync(redisDb *db) {
+    dict *oldht1 = db->pdict;
+    auto *set = db->setexpire;
+    db->setexpire = new (MALLOC_LOCAL) expireset();
+    db->expireitr = db->setexpire->end();
+    db->pdict = dictCreate(&dbDictType,NULL);
     atomicIncr(lazyfree_objects,dictSize(oldht1));
     bioCreateBackgroundJob(BIO_LAZY_FREE,NULL,oldht1,set);
 }

src/module.cpp

@@ -1644,7 +1644,7 @@ int RM_UnlinkKey(RedisModuleKey *key) {
  * If no TTL is associated with the key or if the key is empty,
  * REDISMODULE_NO_EXPIRE is returned. */
 mstime_t RM_GetExpire(RedisModuleKey *key) {
-    expireEntry *pexpire = key->db->getExpire(key->key);
+    expireEntry *pexpire = getExpire(key->db,key->key);
     mstime_t expire = -1;
     if (pexpire != nullptr)
         pexpire->FGetPrimaryExpire(&expire);

src/multi.cpp

@@ -333,7 +333,7 @@ void touchWatchedKeysOnFlush(int dbid) {
              * key exists, mark the client as dirty, as the key will be
              * removed. */
             if (dbid == -1 || wk->db->id == dbid) {
-                if (wk->db->find(wk->key) != NULL)
+                if (dictFind(wk->db->pdict, ptrFromObj(wk->key)) != NULL)
                     c->flags |= CLIENT_DIRTY_CAS;
             }
         }

src/networking.cpp

@@ -116,7 +116,7 @@ client *createClient(int fd, int iel) {
     uint64_t client_id;
     client_id = g_pserver->next_client_id.fetch_add(1);
     c->iel = iel;
-    fastlock_init(&c->lock);
+    fastlock_init(&c->lock, "client");
     c->id = client_id;
     c->resp = 2;
     c->fd = fd;
@@ -248,7 +248,11 @@ void clientInstallAsyncWriteHandler(client *c) {
 int prepareClientToWrite(client *c, bool fAsync) {
     fAsync = fAsync && !FCorrectThread(c);  // Not async if we're on the right thread
     serverAssert(FCorrectThread(c) || fAsync);
-    serverAssert(c->fd <= 0 || c->lock.fOwnLock());
+	if (FCorrectThread(c)) {
+		serverAssert(c->fd <= 0 || c->lock.fOwnLock());
+	} else {
+		serverAssert(GlobalLocksAcquired());
+	}
 
     if (c->flags & CLIENT_FORCE_REPLY) return C_OK; // FORCE REPLY means we're doing something else with the buffer.
                                                 // do not install a write handler
@@ -1509,7 +1513,6 @@ int writeToClient(int fd, client *c, int handler_installed) {
         } else {
             serverLog(LL_VERBOSE,
                 "Error writing to client: %s", strerror(errno));
-            lock.unlock();
             freeClientAsync(c);
             
             return C_ERR;
@@ -1528,7 +1531,6 @@ int writeToClient(int fd, client *c, int handler_installed) {
 
         /* Close connection after entire reply has been sent. */
         if (c->flags & CLIENT_CLOSE_AFTER_REPLY) {
-            lock.unlock();
             freeClientAsync(c);
             return C_ERR;
         }
@@ -2519,9 +2521,17 @@ NULL
                 close_this_client = 1;
             } else {
                 if (FCorrectThread(client))
+                {
                     freeClient(client);
+                }
                 else
+                {
+                    int iel = client->iel;
                     freeClientAsync(client);
+                    aePostFunction(g_pserver->rgthreadvar[client->iel].el, [iel] {
+                        freeClientsInAsyncFreeQueue(iel);
+                    });
+                }
             }
             killed++;
         }
@@ -2950,38 +2960,48 @@ void flushSlavesOutputBuffers(void) {
  * than the time left for the previous pause, no change is made to the
  * left duration. */
 void pauseClients(mstime_t end) {
-    if (!g_pserver->clients_paused || end > g_pserver->clients_pause_end_time)
+    serverAssert(GlobalLocksAcquired());
+    if (!serverTL->clients_paused || end > g_pserver->clients_pause_end_time)
         g_pserver->clients_pause_end_time = end;
-    g_pserver->clients_paused = 1;
+    
+    for (int iel = 0; iel < cserver.cthreads; ++iel)
+    {
+        g_pserver->rgthreadvar[iel].clients_paused = true;
+    }
 }
 
 /* Return non-zero if clients are currently paused. As a side effect the
  * function checks if the pause time was reached and clear it. */
 int clientsArePaused(void) {
-    if (g_pserver->clients_paused &&
+    return serverTL->clients_paused;
+}
+
+void unpauseClientsIfNecessary()
+{
+    serverAssert(GlobalLocksAcquired());
+    if (serverTL->clients_paused &&
         g_pserver->clients_pause_end_time < g_pserver->mstime)
     {
-        aeAcquireLock();
         listNode *ln;
         listIter li;
         client *c;
 
-        g_pserver->clients_paused = 0;
+        serverTL->clients_paused = 0;
 
         /* Put all the clients in the unblocked clients queue in order to
          * force the re-processing of the input buffer if any. */
         listRewind(g_pserver->clients,&li);
         while ((ln = listNext(&li)) != NULL) {
             c = (client*)listNodeValue(ln);
+            if (!FCorrectThread(c))
+                continue;
 
             /* Don't touch slaves and blocked clients.
              * The latter pending requests will be processed when unblocked. */
             if (c->flags & (CLIENT_SLAVE|CLIENT_BLOCKED)) continue;
             queueClientForReprocessing(c);
         }
-        aeReleaseLock();
     }
-    return g_pserver->clients_paused;
 }
 
 /* This function is called by Redis in order to process a few events from
@@ -3000,6 +3020,12 @@ int processEventsWhileBlocked(int iel) {
     int iterations = 4; /* See the function top-comment. */
     int count = 0;
 
+    client *c = serverTL->current_client;
+    if (c != nullptr)
+    {
+        serverAssert(c->flags & CLIENT_PROTECTED);
+        c->lock.unlock();
+    }
     aeReleaseLock();
     while (iterations--) {
         int events = 0;
@@ -3008,7 +3034,11 @@ int processEventsWhileBlocked(int iel) {
         if (!events) break;
         count += events;
     }
-    aeAcquireLock();
+    AeLocker locker;
+    if (c != nullptr)
+        c->lock.lock();
+    locker.arm(c);
+    locker.release();
     return count;
 }
 

src/object.cpp

@@ -789,7 +789,7 @@ size_t streamRadixTreeMemoryUsage(rax *rax) {
  * case of aggregated data types where only "sample_size" elements
  * are checked and averaged to estimate the total size. */
 #define OBJ_COMPUTE_SIZE_DEF_SAMPLES 5 /* Default sample size. */
-size_t objectComputeSize(robj_roptr o, size_t sample_size) {
+size_t objectComputeSize(robj *o, size_t sample_size) {
     sds ele, ele2;
     dict *d;
     dictIterator *di;
@@ -800,7 +800,7 @@ size_t objectComputeSize(robj_roptr o, size_t sample_size) {
         if(o->encoding == OBJ_ENCODING_INT) {
             asize = sizeof(*o);
         } else if(o->encoding == OBJ_ENCODING_RAW) {
-            asize = sdsAllocSize((sds)szFromObj(o))+sizeof(*o);
+            asize = sdsAllocSize(szFromObj(o))+sizeof(*o);
         } else if(o->encoding == OBJ_ENCODING_EMBSTR) {
             asize = sdslen(szFromObj(o))+2+sizeof(*o);
         } else {
@@ -1054,20 +1054,20 @@ struct redisMemOverhead *getMemoryOverheadData(void) {
 
     for (j = 0; j < cserver.dbnum; j++) {
         redisDb *db = g_pserver->db+j;
-        long long keyscount = db->size();
+        long long keyscount = dictSize(db->pdict);
         if (keyscount==0) continue;
 
         mh->total_keys += keyscount;
         mh->db = (decltype(mh->db))zrealloc(mh->db,sizeof(mh->db[0])*(mh->num_dbs+1), MALLOC_LOCAL);
         mh->db[mh->num_dbs].dbid = j;
 
-        mem = db->size() * sizeof(dictEntry) +
-              db->slots() * sizeof(dictEntry*) +
-              db->size() * sizeof(robj);
+        mem = dictSize(db->pdict) * sizeof(dictEntry) +
+              dictSlots(db->pdict) * sizeof(dictEntry*) +
+              dictSize(db->pdict) * sizeof(robj);
         mh->db[mh->num_dbs].overhead_ht_main = mem;
         mem_total+=mem;
 
-        mem = db->setexpire()->bytes_used();
+        mem = db->setexpire->bytes_used();
         mh->db[mh->num_dbs].overhead_ht_expires = mem;
         mem_total+=mem;
 
@@ -1246,12 +1246,15 @@ void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
 
 /* This is a helper function for the OBJECT command. We need to lookup keys
  * without any modification of LRU or other parameters. */
-robj_roptr objectCommandLookup(client *c, robj *key) {
-    return c->db->find(key);
+robj *objectCommandLookup(client *c, robj *key) {
+    dictEntry *de;
+
+    if ((de = dictFind(c->db->pdict,ptrFromObj(key))) == NULL) return NULL;
+    return (robj*) dictGetVal(de);
 }
 
-robj_roptr objectCommandLookupOrReply(client *c, robj *key, robj *reply) {
-    robj_roptr o = objectCommandLookup(c,key);
+robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply) {
+    robj *o = objectCommandLookup(c,key);
 
     if (!o) addReply(c, reply);
     return o;
@@ -1260,7 +1263,7 @@ robj_roptr objectCommandLookupOrReply(client *c, robj *key, robj *reply) {
 /* Object command allows to inspect the internals of an Redis Object.
  * Usage: OBJECT <refcount|encoding|idletime|freq> <key> */
 void objectCommand(client *c) {
-    robj_roptr o;
+    robj *o;
 
     if (c->argc == 2 && !strcasecmp(szFromObj(c->argv[1]),"help")) {
         const char *help[] = {
@@ -1273,15 +1276,15 @@ NULL
         addReplyHelp(c, help);
     } else if (!strcasecmp(szFromObj(c->argv[1]),"refcount") && c->argc == 3) {
         if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.null[c->resp]))
-                == nullptr) return;
+                == NULL) return;
         addReplyLongLong(c,o->getrefcount(std::memory_order_relaxed));
     } else if (!strcasecmp(szFromObj(c->argv[1]),"encoding") && c->argc == 3) {
         if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.null[c->resp]))
-                == nullptr) return;
+                == NULL) return;
         addReplyBulkCString(c,strEncoding(o->encoding));
     } else if (!strcasecmp(szFromObj(c->argv[1]),"idletime") && c->argc == 3) {
         if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.null[c->resp]))
-                == nullptr) return;
+                == NULL) return;
         if (g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LFU) {
             addReplyError(c,"An LFU maxmemory policy is selected, idle time not tracked. Please note that when switching between policies at runtime LRU and LFU data will take some time to adjust.");
             return;
@@ -1289,7 +1292,7 @@ NULL
         addReplyLongLong(c,estimateObjectIdleTime(o)/1000);
     } else if (!strcasecmp(szFromObj(c->argv[1]),"freq") && c->argc == 3) {
         if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.null[c->resp]))
-                == nullptr) return;
+                == NULL) return;
         if (!(g_pserver->maxmemory_policy & MAXMEMORY_FLAG_LFU)) {
             addReplyError(c,"An LFU maxmemory policy is not selected, access frequency not tracked. Please note that when switching between policies at runtime LRU and LFU data will take some time to adjust.");
             return;
@@ -1298,10 +1301,10 @@ NULL
          * in case of the key has not been accessed for a long time,
          * because we update the access time only
          * when the key is read or overwritten. */
-        addReplyLongLong(c,LFUDecrAndReturn(o.unsafe_robjcast()));
+        addReplyLongLong(c,LFUDecrAndReturn(o));
     } else if (!strcasecmp(szFromObj(c->argv[1]), "lastmodified") && c->argc == 3) {
         if ((o = objectCommandLookupOrReply(c,c->argv[2],shared.null[c->resp]))
-                == nullptr) return;
+                == NULL) return;
         addReplyLongLong(c, (g_pserver->mstime - (o->mvcc_tstamp >> MVCC_MS_SHIFT)) / 1000);
     } else {
         addReplySubcommandSyntaxError(c);
@@ -1324,6 +1327,7 @@ NULL
         };
         addReplyHelp(c, help);
     } else if (!strcasecmp(szFromObj(c->argv[1]),"usage") && c->argc >= 3) {
+        dictEntry *de;
         long long samples = OBJ_COMPUTE_SIZE_DEF_SAMPLES;
         for (int j = 3; j < c->argc; j++) {
             if (!strcasecmp(szFromObj(c->argv[j]),"samples") &&
@@ -1342,14 +1346,12 @@ NULL
                 return;
             }
         }
-
-        auto itr = c->db->find(c->argv[2]);
-        if (itr == nullptr) {
+        if ((de = dictFind(c->db->pdict,ptrFromObj(c->argv[2]))) == NULL) {
             addReplyNull(c, shared.nullbulk);
             return;
         }
-        size_t usage = objectComputeSize(itr.val(),samples);
-        usage += sdsAllocSize(itr.key());
+        size_t usage = objectComputeSize((robj*)dictGetVal(de),samples);
+        usage += sdsAllocSize((sds)dictGetKey(de));
         usage += sizeof(dictEntry);
         addReplyLongLong(c,usage);
     } else if (!strcasecmp(szFromObj(c->argv[1]),"stats") && c->argc == 2) {
@@ -1493,110 +1495,3 @@ void redisObject::setrefcount(unsigned ref)
     serverAssert(!FExpires());
     refcount.store(ref, std::memory_order_relaxed); 
 }
-
-sds serializeStoredStringObject(sds str, robj_roptr o)
-{
-    str = sdscatlen(str, &(*o), sizeof(robj));
-    switch (o->encoding)
-    {
-    case OBJ_ENCODING_RAW:
-        str = sdscatsds(str, (sds)szFromObj(o));
-        break;
-
-    case OBJ_ENCODING_INT:
-        break;  //nop
-
-    case OBJ_ENCODING_EMBSTR:
-        size_t cb = zmalloc_size(o.unsafe_robjcast());
-        if (cb > sizeof(robj))
-        {
-            str = sdscatlen(str, o.unsafe_robjcast() + 1, cb - sizeof(robj));
-        }
-        break;
-    }
-        
-    return str;
-}
-
-robj *deserializeStoredStringObject(const char *data, size_t cb)
-{
-    const robj *oT = (const robj*)data;
-    robj *newObject = nullptr;
-    switch (oT->encoding)
-    {
-    case OBJ_ENCODING_INT:
-        serverAssert(cb == sizeof(robj));
-        [[fallthrough]];
-    case OBJ_ENCODING_EMBSTR:
-        newObject = (robj*)zmalloc(cb, MALLOC_LOCAL);
-        memcpy(newObject, data, cb);
-        newObject->setrefcount(1);
-        return newObject;
-
-    case OBJ_ENCODING_RAW:
-        newObject = (robj*)zmalloc(sizeof(robj), MALLOC_SHARED);
-        memcpy(newObject, data, sizeof(robj));
-        newObject->m_ptr = sdsnewlen(SDS_NOINIT,cb-sizeof(robj));
-        memcpy(newObject->m_ptr, data+sizeof(robj), cb-sizeof(robj));
-        newObject->setrefcount(1);
-        return newObject;
-    }
-    serverPanic("Unknown string object encoding from storage");
-    return nullptr;
-}
-
-robj *deserializeStoredObject(const void *data, size_t cb)
-{
-    switch (((char*)data)[0])
-    {
-        case RDB_TYPE_STRING:
-            return deserializeStoredStringObject(((char*)data)+1, cb-1);
-
-        default:
-            rio payload;
-            int type;
-            robj *obj;
-            rioInitWithConstBuffer(&payload,data,cb);
-            if (((type = rdbLoadObjectType(&payload)) == -1) ||
-                ((obj = rdbLoadObject(type,&payload,nullptr, OBJ_MVCC_INVALID)) == nullptr))
-            {
-                serverPanic("Bad data format");
-            }
-            if (rdbLoadType(&payload) == RDB_OPCODE_AUX)
-            {
-                robj *auxkey, *auxval;
-                if ((auxkey = rdbLoadStringObject(&payload)) == NULL) goto eoferr;
-                if ((auxval = rdbLoadStringObject(&payload)) == NULL) {
-                    decrRefCount(auxkey);
-                    goto eoferr;
-                }
-                if (strcasecmp(szFromObj(auxkey), "mvcc-tstamp") == 0) {
-                    obj->mvcc_tstamp = strtoull(szFromObj(auxval), nullptr, 10);
-                }
-                decrRefCount(auxkey);
-                decrRefCount(auxval);
-            }
-        eoferr:
-            return obj;
-    }
-    serverPanic("Unknown object type loading from storage");
-}
-
-sds serializeStoredObject(robj_roptr o)
-{
-    switch (o->type)
-    {
-        case OBJ_STRING:
-        {
-            sds sdsT = sdsnewlen(nullptr, 1);
-            sdsT[0] = RDB_TYPE_STRING;
-            return serializeStoredStringObject(sdsT, o);
-        }
-            
-        default:
-            rio rdb;
-            createDumpPayload(&rdb,o,nullptr);
-            return (sds)rdb.io.buffer.ptr;
-    }
-    serverPanic("Attempting to store unknown object type");
-}

src/pubsub.cpp

@@ -301,9 +301,11 @@ int pubsubPublishMessage(robj *channel, robj *message) {
             client *c = reinterpret_cast<client*>(ln->value);
             if (c->flags & CLIENT_CLOSE_ASAP)   // avoid blocking if the write will be ignored
                 continue;
-            fastlock_lock(&c->lock);
+            if (FCorrectThread(c))
+                fastlock_lock(&c->lock);
             addReplyPubsubMessage(c,channel,message);
-            fastlock_unlock(&c->lock);
+            if (FCorrectThread(c))
+                fastlock_unlock(&c->lock);
             receivers++;
         }
     }
@@ -321,10 +323,12 @@ int pubsubPublishMessage(robj *channel, robj *message) {
             {
                 if (pat->pclient->flags & CLIENT_CLOSE_ASAP)
                     continue;
-                fastlock_lock(&pat->pclient->lock);
+                if (FCorrectThread(pat->pclient))
+                    fastlock_lock(&pat->pclient->lock);
                 addReplyPubsubPatMessage(pat->pclient,
                     pat->pattern,channel,message);
-                fastlock_unlock(&pat->pclient->lock);
+                if (FCorrectThread(pat->pclient))
+                    fastlock_unlock(&pat->pclient->lock);
                 receivers++;
             }
         }

src/rax.c

@@ -705,7 +705,7 @@ int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void **
             nodesize = sizeof(raxNode)+trimmedlen+raxPadding(trimmedlen)+
                        sizeof(raxNode*);
             if (h->iskey && !h->isnull) nodesize += sizeof(void*);
-            trimmed = zcalloc(nodesize, MALLOC_LOCAL);//rax_malloc(nodesize);
+            trimmed = rax_malloc(nodesize);
         }
 
         if (postfixlen) {

src/rdb-s3.cpp

@@ -6,7 +6,7 @@ extern "C" {
 #include <sys/wait.h>
 
 /* Save the DB on disk. Return C_ERR on error, C_OK on success. */
-int rdbSaveS3(char *s3bucket, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi)
+int rdbSaveS3(char *s3bucket, rdbSaveInfo *rsi)
 {
     int status = EXIT_FAILURE;
     int fd[2];
@@ -40,7 +40,7 @@ int rdbSaveS3(char *s3bucket, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi)
             return C_ERR;
         }
 
-        if (rdbSaveFp(fp, rgpdb, rsi) != C_OK)
+        if (rdbSaveFp(fp, rsi) != C_OK)
         {
             fclose(fp);
             return C_ERR;

src/rdb.cpp

@@ -42,9 +42,6 @@
 #include <arpa/inet.h>
 #include <sys/stat.h>
 #include <sys/param.h>
-#include <thread>
-#include <future>
-#include "aelocker.h"
 
 #define rdbExitReportCorruptRDB(...) rdbCheckThenExit(__LINE__,__VA_ARGS__)
 
@@ -1106,21 +1103,21 @@ int rdbSaveInfoAuxFields(rio *rdb, int flags, rdbSaveInfo *rsi) {
     if (rsi) {
         if (rdbSaveAuxFieldStrInt(rdb,"repl-stream-db",rsi->repl_stream_db)
             == -1) return -1;
-        if (rdbSaveAuxFieldStrStr(rdb,"repl-id",rsi->repl_id)
+        if (rdbSaveAuxFieldStrStr(rdb,"repl-id",g_pserver->replid)
             == -1) return -1;
-        if (rdbSaveAuxFieldStrInt(rdb,"repl-offset",rsi->master_repl_offset)
+        if (rdbSaveAuxFieldStrInt(rdb,"repl-offset",g_pserver->master_repl_offset)
             == -1) return -1;
     }
     if (rdbSaveAuxFieldStrInt(rdb,"aof-preamble",aof_preamble) == -1) return -1;
     return 1;
 }
 
-int saveKey(rio *rdb, redisDbPersistentData *db, int flags, size_t *processed, const char *keystr, robj *o)
+int saveKey(rio *rdb, redisDb *db, int flags, size_t *processed, const char *keystr, robj *o)
 {    
     robj key;
 
     initStaticStringObject(key,(char*)keystr);
-    expireEntry *pexpire = db->getExpire(&key);
+    expireEntry *pexpire = getExpire(db, &key);
 
     if (rdbSaveKeyValuePair(rdb,&key,o,pexpire) == -1)
         return 0;
@@ -1145,9 +1142,9 @@ int saveKey(rio *rdb, redisDbPersistentData *db, int flags, size_t *processed, c
  * When the function returns C_ERR and if 'error' is not NULL, the
  * integer pointed by 'error' is set to the value of errno just after the I/O
  * error. */
-int rdbSaveRio(rio *rdb, redisDbPersistentData **rgpdb, int *error, int flags, rdbSaveInfo *rsi) {
-    dictEntry *de;
+int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi) {
     dictIterator *di = NULL;
+    dictEntry *de;
     char magic[10];
     int j;
     uint64_t cksum;
@@ -1157,12 +1154,13 @@ int rdbSaveRio(rio *rdb, redisDbPersistentData **rgpdb, int *error, int flags, r
         rdb->update_cksum = rioGenericUpdateChecksum;
     snprintf(magic,sizeof(magic),"REDIS%04d",RDB_VERSION);
     if (rdbWriteRaw(rdb,magic,9) == -1) goto werr;
-    
     if (rdbSaveInfoAuxFields(rdb,flags,rsi) == -1) goto werr;
 
     for (j = 0; j < cserver.dbnum; j++) {
-        redisDbPersistentData *db = rgpdb[j];
-        if (db->size() == 0) continue;
+        redisDb *db = g_pserver->db+j;
+        dict *d = db->pdict;
+        if (dictSize(d) == 0) continue;
+        di = dictGetSafeIterator(d);
 
         /* Write the SELECT DB opcode */
         if (rdbSaveType(rdb,RDB_OPCODE_SELECTDB) == -1) goto werr;
@@ -1173,47 +1171,43 @@ int rdbSaveRio(rio *rdb, redisDbPersistentData **rgpdb, int *error, int flags, r
          * However this does not limit the actual size of the DB to load since
          * these sizes are just hints to resize the hash tables. */
         uint64_t db_size, expires_size;
-        db_size = db->size();
-        expires_size = db->expireSize();
+        db_size = dictSize(db->pdict);
+        expires_size = db->setexpire->size();
         if (rdbSaveType(rdb,RDB_OPCODE_RESIZEDB) == -1) goto werr;
         if (rdbSaveLen(rdb,db_size) == -1) goto werr;
         if (rdbSaveLen(rdb,expires_size) == -1) goto werr;
         
         /* Iterate this DB writing every entry */
         size_t ckeysExpired = 0;
-        bool fSavedAll = db->iterate([&](const char *keystr, robj *o)->bool {
+        while((de = dictNext(di)) != NULL) {
+            sds keystr = (sds)dictGetKey(de);
+            robj *o = (robj*)dictGetVal(de);
+
             if (o->FExpires())
                 ++ckeysExpired;
             
             if (!saveKey(rdb, db, flags, &processed, keystr, o))
-                return false;
-            return !g_pserver->rdbThreadVars.fRdbThreadCancel;
-        });
-        if (!fSavedAll)
-            goto werr;
-        serverAssert(ckeysExpired == db->expireSize());
+                goto werr;
+        }
+        serverAssert(ckeysExpired == db->setexpire->size());
+        dictReleaseIterator(di);
+        di = NULL; /* So that we don't release it again on error. */
     }
 
     /* If we are storing the replication information on disk, persist
      * the script cache as well: on successful PSYNC after a restart, we need
      * to be able to process any EVALSHA inside the replication backlog the
      * master will send us. */
-    {
-    AeLocker lock;
-    lock.arm(nullptr);
     if (rsi && dictSize(g_pserver->lua_scripts)) {
         di = dictGetIterator(g_pserver->lua_scripts);
         while((de = dictNext(di)) != NULL) {
             robj *body = (robj*)dictGetVal(de);
             if (rdbSaveAuxField(rdb,"lua",3,szFromObj(body),sdslen(szFromObj(body))) == -1)
                 goto werr;
-            if (g_pserver->rdbThreadVars.fRdbThreadCancel)
-                goto werr;
         }
         dictReleaseIterator(di);
         di = NULL; /* So that we don't release it again on error. */
     }
-    }   // AeLocker end scope
 
     /* EOF opcode */
     if (rdbSaveType(rdb,RDB_OPCODE_EOF) == -1) goto werr;
@@ -1239,7 +1233,7 @@ werr:
  * While the suffix is the 40 bytes hex string we announced in the prefix.
  * This way processes receiving the payload can understand when it ends
  * without doing any processing of the content. */
-int rdbSaveRioWithEOFMark(rio *rdb, redisDbPersistentData **rgpdb, int *error, rdbSaveInfo *rsi) {
+int rdbSaveRioWithEOFMark(rio *rdb, int *error, rdbSaveInfo *rsi) {
     char eofmark[RDB_EOF_MARK_SIZE];
 
     getRandomHexChars(eofmark,RDB_EOF_MARK_SIZE);
@@ -1247,7 +1241,7 @@ int rdbSaveRioWithEOFMark(rio *rdb, redisDbPersistentData **rgpdb, int *error, r
     if (rioWrite(rdb,"$EOF:",5) == 0) goto werr;
     if (rioWrite(rdb,eofmark,RDB_EOF_MARK_SIZE) == 0) goto werr;
     if (rioWrite(rdb,"\r\n",2) == 0) goto werr;
-    if (rdbSaveRio(rdb,rgpdb,error,RDB_SAVE_NONE,rsi) == C_ERR) goto werr;
+    if (rdbSaveRio(rdb,error,RDB_SAVE_NONE,rsi) == C_ERR) goto werr;
     if (rioWrite(rdb,eofmark,RDB_EOF_MARK_SIZE) == 0) goto werr;
     return C_OK;
 
@@ -1257,7 +1251,7 @@ werr: /* Write error. */
     return C_ERR;
 }
 
-int rdbSaveFp(FILE *fp, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi)
+int rdbSaveFp(FILE *fp, rdbSaveInfo *rsi)
 {
     int error = 0;
     rio rdb;
@@ -1267,41 +1261,31 @@ int rdbSaveFp(FILE *fp, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi)
     if (g_pserver->rdb_save_incremental_fsync)
         rioSetAutoSync(&rdb,REDIS_AUTOSYNC_BYTES);
 
-    if (rdbSaveRio(&rdb,rgpdb,&error,RDB_SAVE_NONE,rsi) == C_ERR) {
+    if (rdbSaveRio(&rdb,&error,RDB_SAVE_NONE,rsi) == C_ERR) {
         errno = error;
         return C_ERR;
     }
     return C_OK;
 }
 
-int rdbSave(redisDbPersistentData **rgpdb, rdbSaveInfo *rsi)
+int rdbSave(rdbSaveInfo *rsi)
 {
-    std::vector<redisDbPersistentData*> vecdb;
-    if (rgpdb == nullptr)
-    {
-        for (int idb = 0; idb < cserver.dbnum; ++idb)
-        {
-            vecdb.push_back(&g_pserver->db[idb]);
-        }
-        rgpdb = vecdb.data();
-    }
-
     int err = C_OK;
     if (g_pserver->rdb_filename != NULL)
-        err = rdbSaveFile(g_pserver->rdb_filename, rgpdb, rsi);
+        err = rdbSaveFile(g_pserver->rdb_filename, rsi);
 
     if (err == C_OK && g_pserver->rdb_s3bucketpath != NULL)
-        err = rdbSaveS3(g_pserver->rdb_s3bucketpath, rgpdb, rsi);
+        err = rdbSaveS3(g_pserver->rdb_s3bucketpath, rsi);
     return err;
 }
 
 /* Save the DB on disk. Return C_ERR on error, C_OK on success. */
-int rdbSaveFile(char *filename, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi) {
+int rdbSaveFile(char *filename, rdbSaveInfo *rsi) {
     char tmpfile[256];
     char cwd[MAXPATHLEN]; /* Current working dir path for error messages. */
     FILE *fp;
 
-    snprintf(tmpfile,256,"temp-%d.rdb", g_pserver->rdbThreadVars.tmpfileNum);
+    snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid());
     fp = fopen(tmpfile,"w");
     if (!fp) {
         char *cwdp = getcwd(cwd,MAXPATHLEN);
@@ -1314,7 +1298,7 @@ int rdbSaveFile(char *filename, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi)
         return C_ERR;
     }
 
-    if (rdbSaveFp(fp, rgpdb, rsi) == C_ERR){
+    if (rdbSaveFp(fp, rsi) == C_ERR){
         goto werr;
     }
 
@@ -1339,12 +1323,9 @@ int rdbSaveFile(char *filename, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi)
     }
 
     serverLog(LL_NOTICE,"DB saved on disk");
-    if (serverTL != nullptr)
-    {
-        g_pserver->dirty = 0;
-        g_pserver->lastsave = time(NULL);
-        g_pserver->lastbgsave_status = C_OK;
-    }
+    g_pserver->dirty = 0;
+    g_pserver->lastsave = time(NULL);
+    g_pserver->lastbgsave_status = C_OK;
     return C_OK;
 
 werr:
@@ -1354,99 +1335,63 @@ werr:
     return C_ERR;
 }
 
-struct rdbSaveThreadArgs
-{
-    rdbSaveInfo rsi;
-    redisDbPersistentData *rgpdb[1];    // NOTE: Variable Length
-};
-
-void *rdbSaveThread(void *vargs)
-{
-    rdbSaveThreadArgs *args = reinterpret_cast<rdbSaveThreadArgs*>(vargs);
-    serverAssert(serverTL == nullptr);
-    int retval = rdbSave(args->rgpdb, &args->rsi);
-    if (retval == C_OK) {
-        size_t private_dirty = zmalloc_get_private_dirty(-1);
-
-        if (private_dirty) {
-            serverLog(LL_NOTICE,
-                "RDB: %zu MB of memory used by copy-on-write",
-                private_dirty/(1024*1024));
-        }
-
-        g_pserver->child_info_data.cow_size = private_dirty;
-        sendChildInfo(CHILD_INFO_TYPE_RDB);
-    }
-
-    aeAcquireLock();
-    for (int idb = 0; idb < cserver.dbnum; ++idb)
-        g_pserver->db[idb].endSnapshot(args->rgpdb[idb]);
-    aeReleaseLock();
-    zfree(args);
-    return (retval == C_OK) ? (void*)0 : (void*)1;
-}
-
-int launchRdbSaveThread(pthread_t &child, rdbSaveInfo *rsi)
-{
-    rdbSaveThreadArgs *args = (rdbSaveThreadArgs*)zmalloc(sizeof(rdbSaveThreadArgs) + ((cserver.dbnum-1)*sizeof(redisDbPersistentData*)), MALLOC_LOCAL);
-    rdbSaveInfo rsiT = RDB_SAVE_INFO_INIT;
-    if (rsi == nullptr)
-        rsi = &rsiT;
-    memcpy(&args->rsi, rsi, sizeof(rdbSaveInfo));
-    memcpy(&args->rsi.repl_id, g_pserver->replid, sizeof(g_pserver->replid));
-    args->rsi.master_repl_offset = g_pserver->master_repl_offset;
-        
-    for (int idb = 0; idb < cserver.dbnum; ++idb)
-        args->rgpdb[idb] = g_pserver->db[idb].createSnapshot().get();
-
-    g_pserver->rdbThreadVars.tmpfileNum++;
-    g_pserver->rdbThreadVars.fRdbThreadCancel = false;
-    if (pthread_create(&child, NULL, rdbSaveThread, args)) {
-        for (int idb = 0; idb < cserver.dbnum; ++idb)
-            g_pserver->db[idb].endSnapshot(args->rgpdb[idb]);
-        zfree(args);
-        return C_ERR;
-    }
-    return C_OK;
-}
-
 int rdbSaveBackground(rdbSaveInfo *rsi) {
-    pthread_t child;
+    pid_t childpid;
     long long start;
 
-    if (g_pserver->aof_child_pid != -1 || g_pserver->FRdbSaveInProgress()) return C_ERR;
+    if (g_pserver->aof_child_pid != -1 || g_pserver->rdb_child_pid != -1) return C_ERR;
 
     g_pserver->dirty_before_bgsave = g_pserver->dirty;
     g_pserver->lastbgsave_try = time(NULL);
     openChildInfoPipe();
 
     start = ustime();
+    if ((childpid = fork()) == 0) {
+        int retval;
 
-    
-    g_pserver->stat_fork_time = ustime()-start;
-    g_pserver->stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / g_pserver->stat_fork_time / (1024*1024*1024); /* GB per second. */
-    latencyAddSampleIfNeeded("fork",g_pserver->stat_fork_time/1000);
-    if (launchRdbSaveThread(child, rsi) != C_OK) {
-        closeChildInfoPipe();
-        g_pserver->lastbgsave_status = C_ERR;
-        serverLog(LL_WARNING,"Can't save in background: fork: %s",
-            strerror(errno));
-        return C_ERR;
+        /* Child */
+        closeListeningSockets(0);
+        redisSetProcTitle("keydb-rdb-bgsave");
+        retval = rdbSave(rsi);
+        if (retval == C_OK) {
+            size_t private_dirty = zmalloc_get_private_dirty(-1);
+
+            if (private_dirty) {
+                serverLog(LL_NOTICE,
+                    "RDB: %zu MB of memory used by copy-on-write",
+                    private_dirty/(1024*1024));
+            }
+
+            g_pserver->child_info_data.cow_size = private_dirty;
+            sendChildInfo(CHILD_INFO_TYPE_RDB);
+        }
+        exitFromChild((retval == C_OK) ? 0 : 1);
+    } else {
+        /* Parent */
+        g_pserver->stat_fork_time = ustime()-start;
+        g_pserver->stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / g_pserver->stat_fork_time / (1024*1024*1024); /* GB per second. */
+        latencyAddSampleIfNeeded("fork",g_pserver->stat_fork_time/1000);
+        if (childpid == -1) {
+            closeChildInfoPipe();
+            g_pserver->lastbgsave_status = C_ERR;
+            serverLog(LL_WARNING,"Can't save in background: fork: %s",
+                strerror(errno));
+            return C_ERR;
+        }
+        serverLog(LL_NOTICE,"Background saving started by pid %d",childpid);
+        g_pserver->rdb_save_time_start = time(NULL);
+        g_pserver->rdb_child_pid = childpid;
+        g_pserver->rdb_child_type = RDB_CHILD_TYPE_DISK;
+        updateDictResizePolicy();
+        return C_OK;
     }
-    serverLog(LL_NOTICE,"Background saving started");
-    g_pserver->rdb_save_time_start = time(NULL);
-    g_pserver->rdbThreadVars.fRdbThreadActive = true;
-    g_pserver->rdbThreadVars.rdb_child_thread = child;
-    g_pserver->rdb_child_type = RDB_CHILD_TYPE_DISK;
-    updateDictResizePolicy();
-
-    return C_OK;
+    return C_OK; /* unreached */
 }
 
-void rdbRemoveTempFile(int tmpfileNum) {
+void rdbRemoveTempFile(pid_t childpid) {
     char tmpfile[256];
 
-    snprintf(tmpfile,sizeof(tmpfile),"temp-%d.rdb", tmpfileNum);
+    snprintf(tmpfile,sizeof(tmpfile),"temp-%d.rdb", (int) childpid);
     unlink(tmpfile);
 }
 
@@ -2053,7 +1998,7 @@ int rdbLoadRio(rio *rdb, rdbSaveInfo *rsi, int loading_aof) {
                 goto eoferr;
             if ((expires_size = rdbLoadLen(rdb,NULL)) == RDB_LENERR)
                 goto eoferr;
-            db->expand(db_size);
+            dictExpand(db->pdict,db_size);
             continue; /* Read next opcode. */
         } else if (type == RDB_OPCODE_AUX) {
             /* AUX: generic string-string fields. Use to add state to RDB
@@ -2291,7 +2236,7 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
         serverLog(LL_WARNING,
             "Background saving terminated by signal %d", bysignal);
         latencyStartMonitor(latency);
-        rdbRemoveTempFile(g_pserver->rdbThreadVars.tmpfileNum);
+        rdbRemoveTempFile(g_pserver->rdb_child_pid);
         latencyEndMonitor(latency);
         latencyAddSampleIfNeeded("rdb-unlink-temp-file",latency);
         /* SIGUSR1 is whitelisted, so we have a way to kill a child without
@@ -2299,7 +2244,7 @@ void backgroundSaveDoneHandlerDisk(int exitcode, int bysignal) {
         if (bysignal != SIGUSR1)
             g_pserver->lastbgsave_status = C_ERR;
     }
-    g_pserver->rdbThreadVars.fRdbThreadActive = false;
+    g_pserver->rdb_child_pid = -1;
     g_pserver->rdb_child_type = RDB_CHILD_TYPE_NONE;
     g_pserver->rdb_save_time_last = time(NULL)-g_pserver->rdb_save_time_start;
     g_pserver->rdb_save_time_start = -1;
@@ -2324,7 +2269,7 @@ void backgroundSaveDoneHandlerSocket(int exitcode, int bysignal) {
         serverLog(LL_WARNING,
             "Background transfer terminated by signal %d", bysignal);
     }
-    g_pserver->rdbThreadVars.fRdbThreadActive = false;
+    g_pserver->rdb_child_pid = -1;
     g_pserver->rdb_child_type = RDB_CHILD_TYPE_NONE;
     g_pserver->rdb_save_time_start = -1;
 
@@ -2423,114 +2368,26 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) {
  * the child did not exit for an error, but because we wanted), and performs
  * the cleanup needed. */
 void killRDBChild(void) {
-    g_pserver->rdbThreadVars.fRdbThreadCancel = true;
-    void *rval;
-    pthread_join(g_pserver->rdbThreadVars.rdb_child_thread,&rval);
-    g_pserver->rdbThreadVars.fRdbThreadActive = false;
-    g_pserver->rdbThreadVars.fRdbThreadCancel = false;
-    rdbRemoveTempFile(g_pserver->rdbThreadVars.tmpfileNum);
+    kill(g_pserver->rdb_child_pid,SIGUSR1);
+    rdbRemoveTempFile(g_pserver->rdb_child_pid);
     closeChildInfoPipe();
     updateDictResizePolicy();
 }
 
-struct rdbSaveSocketThreadArgs
-{
-    rdbSaveInfo rsi;
-    int *fds;
-    int numfds;
-    uint64_t *clientids;
-    redisDbPersistentData *rgpdb[1];
-};
-void *rdbSaveToSlavesSocketsThread(void *vargs)
-{
-    /* Child */
-    serverAssert(serverTL == nullptr);
-    rdbSaveSocketThreadArgs *args = (rdbSaveSocketThreadArgs*)vargs;
-    int retval;
-    rio slave_sockets;
-
-    rioInitWithFdset(&slave_sockets,args->fds,args->numfds);
-    zfree(args->fds);
-    args->fds = nullptr;
-
-    retval = rdbSaveRioWithEOFMark(&slave_sockets,args->rgpdb,NULL,&args->rsi);
-    if (retval == C_OK && rioFlush(&slave_sockets) == 0)
-        retval = C_ERR;
-
-    if (retval == C_OK) {
-        size_t private_dirty = zmalloc_get_private_dirty(-1);
-
-        if (private_dirty) {
-            serverLog(LL_NOTICE,
-                "RDB: %zu MB of memory used by copy-on-write",
-                private_dirty/(1024*1024));
-        }
-
-        g_pserver->child_info_data.cow_size = private_dirty;
-        sendChildInfo(CHILD_INFO_TYPE_RDB);
-
-        /* If we are returning OK, at least one replica was served
-            * with the RDB file as expected, so we need to send a report
-            * to the parent via the pipe. The format of the message is:
-            *
-            * <len> <replica[0].id> <replica[0].error> ...
-            *
-            * len, replica IDs, and replica errors, are all uint64_t integers,
-            * so basically the reply is composed of 64 bits for the len field
-            * plus 2 additional 64 bit integers for each entry, for a total
-            * of 'len' entries.
-            *
-            * The 'id' represents the replica's client ID, so that the master
-            * can match the report with a specific replica, and 'error' is
-            * set to 0 if the replication process terminated with a success
-            * or the error code if an error occurred. */
-        void *msg = zmalloc(sizeof(uint64_t)*(1+2*args->numfds), MALLOC_LOCAL);
-        uint64_t *len = (uint64_t*)msg;
-        uint64_t *ids = len+1;
-        int j, msglen;
-
-        *len = args->numfds;
-        for (j = 0; j < args->numfds; j++) {
-            *ids++ = args->clientids[j];
-            *ids++ = slave_sockets.io.fdset.state[j];
-        }
-
-        /* Write the message to the parent. If we have no good slaves or
-            * we are unable to transfer the message to the parent, we exit
-            * with an error so that the parent will abort the replication
-            * process with all the childre that were waiting. */
-        msglen = sizeof(uint64_t)*(1+2*args->numfds);
-        if (*len == 0 ||
-            write(g_pserver->rdb_pipe_write_result_to_parent,msg,msglen)
-            != msglen)
-        {
-            retval = C_ERR;
-        }
-        zfree(msg);
-    }
-    aeAcquireLock();
-    for (int idb = 0; idb < cserver.dbnum; ++idb)
-        g_pserver->db[idb].endSnapshot(args->rgpdb[idb]);
-    aeReleaseLock();
-    zfree(args->clientids);
-    zfree(args);
-    rioFreeFdset(&slave_sockets);
-
-    return (retval == C_OK) ? (void*)0 : (void*)1;
-}
-
 /* Spawn an RDB child that writes the RDB to the sockets of the slaves
  * that are currently in SLAVE_STATE_WAIT_BGSAVE_START state. */
 int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
     serverAssert(GlobalLocksAcquired());
+    int *fds;
+    uint64_t *clientids;
+    int numfds;
     listNode *ln;
     listIter li;
-    pthread_t child;
+    pid_t childpid;
     long long start;
     int pipefds[2];
-    rdbSaveSocketThreadArgs *args = nullptr;
 
-    if (g_pserver->aof_child_pid != -1 || g_pserver->FRdbSaveInProgress()) return C_ERR;
+    if (g_pserver->aof_child_pid != -1 || g_pserver->rdb_child_pid != -1) return C_ERR;
 
     /* Before to fork, create a pipe that will be used in order to
      * send back to the parent the IDs of the slaves that successfully
@@ -2539,27 +2396,22 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
     g_pserver->rdb_pipe_read_result_from_child = pipefds[0];
     g_pserver->rdb_pipe_write_result_to_parent = pipefds[1];
 
-    args = (rdbSaveSocketThreadArgs*)zmalloc(sizeof(rdbSaveSocketThreadArgs) + sizeof(redisDbPersistentData*)*(cserver.dbnum-1), MALLOC_LOCAL);
-
     /* Collect the file descriptors of the slaves we want to transfer
      * the RDB to, which are i WAIT_BGSAVE_START state. */
-    args->fds = (int*)zmalloc(sizeof(int)*listLength(g_pserver->slaves), MALLOC_LOCAL);
+    fds = (int*)zmalloc(sizeof(int)*listLength(g_pserver->slaves), MALLOC_LOCAL);
     /* We also allocate an array of corresponding client IDs. This will
      * be useful for the child process in order to build the report
      * (sent via unix pipe) that will be sent to the parent. */
-    args->clientids = (uint64_t*)zmalloc(sizeof(uint64_t)*listLength(g_pserver->slaves), MALLOC_LOCAL);
-    args->numfds = 0;
-    memcpy(&args->rsi, rsi, sizeof(rdbSaveInfo));
-    memcpy(&args->rsi.repl_id, g_pserver->replid, sizeof(g_pserver->replid));
-    args->rsi.master_repl_offset = g_pserver->master_repl_offset;
+    clientids = (uint64_t*)zmalloc(sizeof(uint64_t)*listLength(g_pserver->slaves), MALLOC_LOCAL);
+    numfds = 0;
 
     listRewind(g_pserver->slaves,&li);
     while((ln = listNext(&li))) {
         client *replica = (client*)ln->value;
 
         if (replica->replstate == SLAVE_STATE_WAIT_BGSAVE_START) {
-            args->clientids[args->numfds] = replica->id;
-            args->fds[args->numfds++] = replica->fd;
+            clientids[numfds] = replica->id;
+            fds[numfds++] = replica->fd;
             replicationSetupSlaveForFullResync(replica,getPsyncInitialOffset());
             /* Put the socket in blocking mode to simplify RDB transfer.
              * We'll restore it when the children returns (since duped socket
@@ -2572,63 +2424,126 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
     /* Create the child process. */
     openChildInfoPipe();
     start = ustime();
+    if ((childpid = fork()) == 0) {
+        /* Child */
+        int retval;
+        rio slave_sockets;
 
-    for (int idb = 0; idb < cserver.dbnum; ++idb)
-        args->rgpdb[idb] = g_pserver->db[idb].createSnapshot().get();
+        rioInitWithFdset(&slave_sockets,fds,numfds);
+        zfree(fds);
 
-    g_pserver->rdbThreadVars.tmpfileNum++;
-    g_pserver->rdbThreadVars.fRdbThreadCancel = false;
-    if (pthread_create(&child, nullptr, rdbSaveToSlavesSocketsThread, args)) {
-        serverLog(LL_WARNING,"Can't save in background: fork: %s",
-            strerror(errno));
+        closeListeningSockets(0);
+        redisSetProcTitle("keydb-rdb-to-slaves");
 
-        /* Undo the state change. The caller will perform cleanup on
-            * all the slaves in BGSAVE_START state, but an early call to
-            * replicationSetupSlaveForFullResync() turned it into BGSAVE_END */
-        listRewind(g_pserver->slaves,&li);
-        while((ln = listNext(&li))) {
-            client *replica = (client*)ln->value;
-            int j;
+        retval = rdbSaveRioWithEOFMark(&slave_sockets,NULL,rsi);
+        if (retval == C_OK && rioFlush(&slave_sockets) == 0)
+            retval = C_ERR;
 
-            for (j = 0; j < args->numfds; j++) {
-                if (replica->id == args->clientids[j]) {
-                    replica->replstate = SLAVE_STATE_WAIT_BGSAVE_START;
-                    break;
+        if (retval == C_OK) {
+            size_t private_dirty = zmalloc_get_private_dirty(-1);
+
+            if (private_dirty) {
+                serverLog(LL_NOTICE,
+                    "RDB: %zu MB of memory used by copy-on-write",
+                    private_dirty/(1024*1024));
+            }
+
+            g_pserver->child_info_data.cow_size = private_dirty;
+            sendChildInfo(CHILD_INFO_TYPE_RDB);
+
+            /* If we are returning OK, at least one replica was served
+             * with the RDB file as expected, so we need to send a report
+             * to the parent via the pipe. The format of the message is:
+             *
+             * <len> <replica[0].id> <replica[0].error> ...
+             *
+             * len, replica IDs, and replica errors, are all uint64_t integers,
+             * so basically the reply is composed of 64 bits for the len field
+             * plus 2 additional 64 bit integers for each entry, for a total
+             * of 'len' entries.
+             *
+             * The 'id' represents the replica's client ID, so that the master
+             * can match the report with a specific replica, and 'error' is
+             * set to 0 if the replication process terminated with a success
+             * or the error code if an error occurred. */
+            void *msg = zmalloc(sizeof(uint64_t)*(1+2*numfds), MALLOC_LOCAL);
+            uint64_t *len = (uint64_t*)msg;
+            uint64_t *ids = len+1;
+            int j, msglen;
+
+            *len = numfds;
+            for (j = 0; j < numfds; j++) {
+                *ids++ = clientids[j];
+                *ids++ = slave_sockets.io.fdset.state[j];
+            }
+
+            /* Write the message to the parent. If we have no good slaves or
+             * we are unable to transfer the message to the parent, we exit
+             * with an error so that the parent will abort the replication
+             * process with all the childre that were waiting. */
+            msglen = sizeof(uint64_t)*(1+2*numfds);
+            if (*len == 0 ||
+                write(g_pserver->rdb_pipe_write_result_to_parent,msg,msglen)
+                != msglen)
+            {
+                retval = C_ERR;
+            }
+            zfree(msg);
+        }
+        zfree(clientids);
+        rioFreeFdset(&slave_sockets);
+        exitFromChild((retval == C_OK) ? 0 : 1);
+    } else {
+        /* Parent */
+        if (childpid == -1) {
+            serverLog(LL_WARNING,"Can't save in background: fork: %s",
+                strerror(errno));
+
+            /* Undo the state change. The caller will perform cleanup on
+             * all the slaves in BGSAVE_START state, but an early call to
+             * replicationSetupSlaveForFullResync() turned it into BGSAVE_END */
+            listRewind(g_pserver->slaves,&li);
+            while((ln = listNext(&li))) {
+                client *replica = (client*)ln->value;
+                int j;
+
+                for (j = 0; j < numfds; j++) {
+                    if (replica->id == clientids[j]) {
+                        replica->replstate = SLAVE_STATE_WAIT_BGSAVE_START;
+                        break;
+                    }
                 }
             }
+            close(pipefds[0]);
+            close(pipefds[1]);
+            closeChildInfoPipe();
+        } else {
+            g_pserver->stat_fork_time = ustime()-start;
+            g_pserver->stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / g_pserver->stat_fork_time / (1024*1024*1024); /* GB per second. */
+            latencyAddSampleIfNeeded("fork",g_pserver->stat_fork_time/1000);
+
+            serverLog(LL_NOTICE,"Background RDB transfer started by pid %d",
+                childpid);
+            g_pserver->rdb_save_time_start = time(NULL);
+            g_pserver->rdb_child_pid = childpid;
+            g_pserver->rdb_child_type = RDB_CHILD_TYPE_SOCKET;
+            updateDictResizePolicy();
         }
-        close(pipefds[0]);
-        close(pipefds[1]);
-        closeChildInfoPipe();
-        for (int idb = 0; idb < cserver.dbnum; ++idb)
-            g_pserver->db[idb].endSnapshot(args->rgpdb[idb]);
-        zfree(args->clientids);
-        zfree(args->fds);
-        zfree(args);
-        return C_ERR;
+        zfree(clientids);
+        zfree(fds);
+        return (childpid == -1) ? C_ERR : C_OK;
     }
-
-    g_pserver->stat_fork_time = ustime()-start;
-    g_pserver->stat_fork_rate = (double) zmalloc_used_memory() * 1000000 / g_pserver->stat_fork_time / (1024*1024*1024); /* GB per second. */
-    latencyAddSampleIfNeeded("fork",g_pserver->stat_fork_time/1000);
-
-    serverLog(LL_NOTICE,"Background RDB transfer started");
-    g_pserver->rdb_save_time_start = time(NULL);
-    g_pserver->rdbThreadVars.rdb_child_thread = child;
-    g_pserver->rdbThreadVars.fRdbThreadActive = true;
-    g_pserver->rdb_child_type = RDB_CHILD_TYPE_SOCKET;
-    updateDictResizePolicy();
     return C_OK; /* Unreached. */
 }
 
 void saveCommand(client *c) {
-    if (g_pserver->FRdbSaveInProgress()) {
+    if (g_pserver->rdb_child_pid != -1) {
         addReplyError(c,"Background save already in progress");
         return;
     }
     rdbSaveInfo rsi, *rsiptr;
     rsiptr = rdbPopulateSaveInfo(&rsi);
-    if (rdbSave(nullptr, rsiptr) == C_OK) {
+    if (rdbSave(rsiptr) == C_OK) {
         addReply(c,shared.ok);
     } else {
         addReply(c,shared.err);
@@ -2653,7 +2568,7 @@ void bgsaveCommand(client *c) {
     rdbSaveInfo rsi, *rsiptr;
     rsiptr = rdbPopulateSaveInfo(&rsi);
 
-    if (g_pserver->FRdbSaveInProgress()) {
+    if (g_pserver->rdb_child_pid != -1) {
         addReplyError(c,"Background save already in progress");
     } else if (g_pserver->aof_child_pid != -1) {
         if (schedule) {
@@ -2685,9 +2600,6 @@ rdbSaveInfo *rdbPopulateSaveInfo(rdbSaveInfo *rsi) {
     rdbSaveInfo rsi_init = RDB_SAVE_INFO_INIT;
     *rsi = rsi_init;
 
-    memcpy(rsi->repl_id, g_pserver->replid, sizeof(g_pserver->replid));
-    rsi->master_repl_offset = g_pserver->master_repl_offset;
-
     /* If the instance is a master, we can populate the replication info
      * only when repl_backlog is not NULL. If the repl_backlog is NULL,
      * it means that the instance isn't in any replication chains. In this

src/rdb.h

@@ -139,11 +139,11 @@ int rdbLoad(rdbSaveInfo *rsi);
 int rdbLoadFile(const char *filename, rdbSaveInfo *rsi);
 int rdbSaveBackground(rdbSaveInfo *rsi);
 int rdbSaveToSlavesSockets(rdbSaveInfo *rsi);
-void rdbRemoveTempFile(int childpid);
-int rdbSave(redisDbPersistentData **rgpdb, rdbSaveInfo *rsi);
-int rdbSaveFile(char *filename, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi);
-int rdbSaveFp(FILE *pf, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi);
-int rdbSaveS3(char *path, redisDbPersistentData **rgpdb, rdbSaveInfo *rsi);
+void rdbRemoveTempFile(pid_t childpid);
+int rdbSave(rdbSaveInfo *rsi);
+int rdbSaveFile(char *filename, rdbSaveInfo *rsi);
+int rdbSaveFp(FILE *pf, rdbSaveInfo *rsi);
+int rdbSaveS3(char *path, rdbSaveInfo *rsi);
 int rdbLoadS3(char *path, rdbSaveInfo *rsi);
 ssize_t rdbSaveObject(rio *rdb, robj_roptr o, robj *key);
 size_t rdbSavedObjectLen(robj *o);

src/redis-benchmark.cpp

@@ -173,6 +173,8 @@ typedef struct redisConfig {
     sds appendonly;
 } redisConfig;
 
+int g_fInCrash = false;
+
 /* Prototypes */
 static void writeHandler(aeEventLoop *el, int fd, void *privdata, int mask);
 static void createMissingClients(client c);

src/redis-cli.c

@@ -90,6 +90,8 @@ int spectrum_palette_mono[] = {0,233,234,235,237,239,241,243,245,247,249,251,253
 int *spectrum_palette;
 int spectrum_palette_size;
 
+int g_fInCrash = 0;
+
 /*------------------------------------------------------------------------------
  * Utility functions
  *--------------------------------------------------------------------------- */

src/replication.cpp

@@ -384,7 +384,11 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) {
 
         /* Don't feed slaves that are still waiting for BGSAVE to start */
         if (replica->replstate == SLAVE_STATE_WAIT_BGSAVE_START) continue;
-        std::unique_lock<decltype(replica->lock)> lock(replica->lock);
+        if (replica->flags & CLIENT_CLOSE_ASAP) continue;
+        std::unique_lock<decltype(replica->lock)> lock(replica->lock, std::defer_lock);
+		// When writing to clients on other threads the global lock is sufficient provided we only use AddReply*Async()
+		if (FCorrectThread(replica))
+			lock.lock();
         if (serverTL->current_client && FSameHost(serverTL->current_client, replica))
         {
             replica->reploff_skipped += g_pserver->master_repl_offset - master_repl_offset_start;
@@ -433,7 +437,9 @@ void replicationFeedSlavesFromMasterStream(list *slaves, char *buf, size_t bufle
 
     while((ln = listNext(&li))) {
         client *replica = (client*)ln->value;
-        std::lock_guard<decltype(replica->lock)> ulock(replica->lock);
+        std::unique_lock<decltype(replica->lock)> ulock(replica->lock, std::defer_lock);
+		if (FCorrectThread(replica))
+			ulock.lock();
         if (FMasterHost(replica))
             continue;   // Active Active case, don't feed back
 
@@ -482,7 +488,10 @@ void replicationFeedMonitors(client *c, list *monitors, int dictid, robj **argv,
     listRewind(monitors,&li);
     while((ln = listNext(&li))) {
         client *monitor = (client*)ln->value;
-        std::lock_guard<decltype(monitor->lock)> lock(monitor->lock);
+		std::unique_lock<decltype(monitor->lock)> lock(monitor->lock, std::defer_lock);
+		// When writing to clients on other threads the global lock is sufficient provided we only use AddReply*Async()
+		if (FCorrectThread(c))
+			lock.lock();
         addReplyAsync(monitor,cmdobj);
     }
     decrRefCount(cmdobj);
@@ -858,7 +867,7 @@ void syncCommand(client *c) {
     }
 
     /* CASE 1: BGSAVE is in progress, with disk target. */
-    if (g_pserver->FRdbSaveInProgress() &&
+    if (g_pserver->rdb_child_pid != -1 &&
         g_pserver->rdb_child_type == RDB_CHILD_TYPE_DISK)
     {
         /* Ok a background save is in progress. Let's check if it is a good
@@ -889,7 +898,7 @@ void syncCommand(client *c) {
         }
 
     /* CASE 2: BGSAVE is in progress, with socket target. */
-    } else if (g_pserver->FRdbSaveInProgress() &&
+    } else if (g_pserver->rdb_child_pid != -1 &&
                g_pserver->rdb_child_type == RDB_CHILD_TYPE_SOCKET)
     {
         /* There is an RDB child process but it is writing directly to
@@ -1205,7 +1214,21 @@ void updateSlavesWaitingBgsave(int bgsaveerr, int type)
                 }
                 else
                 {
-                    aePostFunction(g_pserver->rgthreadvar[replica->iel].el, [replica]{
+                    aePostFunction(g_pserver->rgthreadvar[replica->iel].el, [replica] {
+                        // Because the client could have been closed while the lambda waited to run we need to
+			// verify the replica is still connected
+                        listIter li;
+			listNode *ln;
+			listRewind(g_pserver->slaves,&li);
+			bool fFound = false;
+			while ((ln = listNext(&li))) {
+			    if (listNodeValue(ln) == replica) {
+			        fFound = true;
+				break;
+			     }
+			}
+			if (!fFound)
+			    return;
                         aeDeleteFileEvent(g_pserver->rgthreadvar[replica->iel].el,replica->fd,AE_WRITABLE);
                         if (aeCreateFileEvent(g_pserver->rgthreadvar[replica->iel].el, replica->fd, AE_WRITABLE, sendBulkToSlave, replica) == AE_ERR) {
                             freeClient(replica);
@@ -1510,12 +1533,13 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) {
         int aof_is_enabled = g_pserver->aof_state != AOF_OFF;
 
         /* Ensure background save doesn't overwrite synced data */
-        if (g_pserver->FRdbSaveInProgress()) {
+        if (g_pserver->rdb_child_pid != -1) {
             serverLog(LL_NOTICE,
                 "Replica is about to load the RDB file received from the "
                 "master, but there is a pending RDB child running. "
-                "Cancelling RDB the save and removing its temp file to avoid "
-                "any race");
+                "Killing process %ld and removing its temp file to avoid "
+                "any race",
+                    (long) g_pserver->rdb_child_pid);
             killRDBChild();
         }
 
@@ -3142,7 +3166,7 @@ void replicationCron(void) {
     * In case of diskless replication, we make sure to wait the specified
     * number of seconds (according to configuration) so that other slaves
     * have the time to arrive before we start streaming. */
-    if (!g_pserver->FRdbSaveInProgress() && g_pserver->aof_child_pid == -1) {
+    if (g_pserver->rdb_child_pid == -1 && g_pserver->aof_child_pid == -1) {
         time_t idle, max_idle = 0;
         int slaves_waiting = 0;
         int mincapa = -1;
@@ -3377,4 +3401,4 @@ void updateMasterAuth()
         if (cserver.default_masteruser)
             mi->masteruser = zstrdup(cserver.default_masteruser);
     }
-}
+}

src/rio.cpp

@@ -73,14 +73,6 @@ static size_t rioBufferRead(rio *r, void *buf, size_t len) {
     return 1;
 }
 
-static size_t rioConstBufferRead(rio *r, void *buf, size_t len) {
-     if (r->io.buffer.len-r->io.buffer.pos < (off_t)len)
-        return 0; /* not enough buffer to return len bytes. */
-    memcpy(buf,r->io.buffer.ptr+r->io.buffer.pos,len);
-    r->io.buffer.pos += len;
-    return 1;
-}
-
 /* Returns read/write position in buffer. */
 static off_t rioBufferTell(rio *r) {
     return r->io.buffer.pos;
@@ -105,32 +97,12 @@ static const rio rioBufferIO = {
     { { NULL, 0 } } /* union for io-specific vars */
 };
 
-static const rio rioConstBufferIO = {
-    rioConstBufferRead,
-    nullptr,
-    rioBufferTell,
-    rioBufferFlush,
-    NULL,           /* update_checksum */
-    0,              /* current checksum */
-    0,              /* bytes read or written */
-    0,              /* read/write chunk size */
-    { { NULL, 0 } } /* union for io-specific vars */
-};
-
 void rioInitWithBuffer(rio *r, sds s) {
     *r = rioBufferIO;
     r->io.buffer.ptr = s;
     r->io.buffer.pos = 0;
 }
 
-void rioInitWithConstBuffer(rio *r, const void *buf, size_t cb)
-{
-    *r = rioConstBufferIO;
-    r->io.buffer.ptr = (sds)buf;
-    r->io.buffer.pos = 0;
-    r->io.buffer.len = cb;
-}
-
 /* --------------------- Stdio file pointer implementation ------------------- */
 
 /* Returns 1 or 0 for success/failure. */

src/rio.h

@@ -70,7 +70,6 @@ struct _rio {
         struct {
             sds ptr;
             off_t pos;
-            off_t len;  // For const buffers only
         } buffer;
         /* Stdio file pointer target. */
         struct {
@@ -131,7 +130,6 @@ static inline int rioFlush(rio *r) {
 
 void rioInitWithFile(rio *r, FILE *fp);
 void rioInitWithBuffer(rio *r, sds s);
-void rioInitWithConstBuffer(rio *r, const void *rgch, size_t cch);
 void rioInitWithFdset(rio *r, int *fds, int numfds);
 
 void rioFreeFdset(rio *r);

src/semiorderedset.h

@@ -93,9 +93,9 @@ public:
         return end();
     }
 
-    setiter end() const
+    setiter end()
     {
-        setiter itr(const_cast<semiorderedset<T,T_KEY,MEMMOVE_SAFE>*>(this));
+        setiter itr(this);
         itr.idxPrimary = m_data.size();
         return itr;
     }

src/server.cpp

@@ -1179,21 +1179,6 @@ int dictSdsKeyCompare(void *privdata, const void *key1,
     return memcmp(key1, key2, l1) == 0;
 }
 
-void dictSdsNOPDestructor(void *, void *) {}
-
-void dictDbKeyDestructor(void *privdata, void *key)
-{
-    redisDbPersistentData *owner = (redisDbPersistentData*)privdata;
-    serverAssert(owner != nullptr);
-    if (owner->m_spdbSnapshot != nullptr)
-    {
-        dictEntry *deSnapshot = dictFind(owner->m_spdbSnapshot->m_pdict, key);
-        if (deSnapshot && (key == dictGetKey(deSnapshot)))
-            return; // don't free, it's now owned by the snapshot
-    }
-    sdsfree((sds)key);
-}
-
 /* A case insensitive version used for the command lookup table and other
  * places where case insensitive non binary-safe comparison is needed. */
 int dictSdsKeyCaseCompare(void *privdata, const void *key1,
@@ -1329,19 +1314,10 @@ dictType dbDictType = {
     NULL,                       /* key dup */
     NULL,                       /* val dup */
     dictSdsKeyCompare,          /* key compare */
-    dictDbKeyDestructor,          /* key destructor */
+    dictSdsDestructor,          /* key destructor */
     dictObjectDestructor   /* val destructor */
 };
 
-dictType dbSnapshotDictType = {
-    dictSdsHash,
-    NULL,
-    NULL,
-    dictSdsKeyCompare,
-    dictSdsNOPDestructor,
-    dictObjectDestructor,
-};
-
 /* g_pserver->lua_scripts sha (as sds string) -> scripts (as robj) cache. */
 dictType shaScriptObjectDictType = {
     dictSdsCaseHash,            /* hash function */
@@ -1463,7 +1439,8 @@ int htNeedsResize(dict *dict) {
 /* If the percentage of used slots in the HT reaches HASHTABLE_MIN_FILL
  * we resize the hash table to save memory */
 void tryResizeHashTables(int dbid) {
-    g_pserver->db[dbid].tryResize();
+    if (htNeedsResize(g_pserver->db[dbid].pdict))
+        dictResize(g_pserver->db[dbid].pdict);
 }
 
 /* Our hash table implementation performs rehashing incrementally while
@@ -1473,10 +1450,10 @@ void tryResizeHashTables(int dbid) {
  *
  * The function returns 1 if some rehashing was performed, otherwise 0
  * is returned. */
-int redisDbPersistentData::incrementallyRehash() {
+int incrementallyRehash(int dbid) {
     /* Keys dictionary */
-    if (dictIsRehashing(m_pdict)) {
-        dictRehashMilliseconds(m_pdict,1);
+    if (dictIsRehashing(g_pserver->db[dbid].pdict)) {
+        dictRehashMilliseconds(g_pserver->db[dbid].pdict,1);
         return 1; /* already used our millisecond for this loop... */
     }
     return 0;
@@ -1489,7 +1466,7 @@ int redisDbPersistentData::incrementallyRehash() {
  * for dict.c to resize the hash tables accordingly to the fact we have o not
  * running childs. */
 void updateDictResizePolicy(void) {
-    if (!g_pserver->FRdbSaveInProgress() && g_pserver->aof_child_pid == -1)
+    if (g_pserver->rdb_child_pid == -1 && g_pserver->aof_child_pid == -1)
         dictEnableResize();
     else
         dictDisableResize();
@@ -1715,6 +1692,9 @@ void clientsCron(int iel) {
             fastlock_unlock(&c->lock);
         }        
     }
+
+    /* Free any pending clients */
+    freeClientsInAsyncFreeQueue(iel);
 }
 
 /* This function handles 'background' operations we are required to do
@@ -1736,7 +1716,7 @@ void databasesCron(void) {
     /* Perform hash tables rehashing if needed, but only if there are no
      * other processes saving the DB on disk. Otherwise rehashing is bad
      * as will cause a lot of copy-on-write of memory pages. */
-    if (!g_pserver->FRdbSaveInProgress() && g_pserver->aof_child_pid == -1) {
+    if (g_pserver->rdb_child_pid == -1 && g_pserver->aof_child_pid == -1) {
         /* We use global counters so if we stop the computation at a given
          * DB we'll be able to start from the successive in the next
          * cron loop iteration. */
@@ -1757,7 +1737,7 @@ void databasesCron(void) {
         /* Rehash */
         if (g_pserver->activerehashing) {
             for (j = 0; j < dbs_per_call; j++) {
-                int work_done = g_pserver->db[rehash_db].incrementallyRehash();
+                int work_done = incrementallyRehash(rehash_db);
                 if (work_done) {
                     /* If the function did some work, stop here, we'll do
                      * more at the next cron loop. */
@@ -1835,6 +1815,9 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
     /* Update the time cache. */
     updateCachedTime();
 
+    /* Unpause clients if enough time has elapsed */
+    unpauseClientsIfNecessary();
+
     g_pserver->hz = g_pserver->config_hz;
     /* Adapt the g_pserver->hz value to the number of configured clients. If we have
      * many clients, we want to call serverCron() with an higher frequency. */
@@ -1842,7 +1825,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
         while (listLength(g_pserver->clients) / g_pserver->hz >
                MAX_CLIENTS_PER_CLOCK_TICK)
         {
-            g_pserver->hz *= 2;
+            g_pserver->hz += g_pserver->hz; // *= 2
             if (g_pserver->hz > CONFIG_MAX_HZ) {
                 g_pserver->hz = CONFIG_MAX_HZ;
                 break;
@@ -1916,9 +1899,9 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
         for (j = 0; j < cserver.dbnum; j++) {
             long long size, used, vkeys;
 
-            size = g_pserver->db[j].slots();
-            used = g_pserver->db[j].size();
-            vkeys = g_pserver->db[j].expireSize();
+            size = dictSlots(g_pserver->db[j].pdict);
+            used = dictSize(g_pserver->db[j].pdict);
+            vkeys = g_pserver->db[j].setexpire->size();
             if (used || vkeys) {
                 serverLog(LL_VERBOSE,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size);
                 /* dictPrintStats(g_pserver->dict); */
@@ -1945,37 +1928,20 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
 
     /* Start a scheduled AOF rewrite if this was requested by the user while
      * a BGSAVE was in progress. */
-    if (!g_pserver->FRdbSaveInProgress() && g_pserver->aof_child_pid == -1 &&
+    if (g_pserver->rdb_child_pid == -1 && g_pserver->aof_child_pid == -1 &&
         g_pserver->aof_rewrite_scheduled)
     {
         rewriteAppendOnlyFileBackground();
     }
 
     /* Check if a background saving or AOF rewrite in progress terminated. */
-    if (g_pserver->FRdbSaveInProgress() || g_pserver->aof_child_pid != -1 ||
+    if (g_pserver->rdb_child_pid != -1 || g_pserver->aof_child_pid != -1 ||
         ldbPendingChildren())
     {
         int statloc;
         pid_t pid;
 
-        if (g_pserver->FRdbSaveInProgress())
-        {
-            void *rval = nullptr;
-            if (pthread_tryjoin_np(g_pserver->rdbThreadVars.rdb_child_thread, &rval))
-            {
-                if (errno != EBUSY && errno != EAGAIN)
-                    serverLog(LL_WARNING, "Error joining the background RDB save thread: %s\n", strerror(errno));
-            }
-            else
-            {
-                int exitcode = (int)reinterpret_cast<ptrdiff_t>(rval);
-                backgroundSaveDoneHandler(exitcode,0);
-                if (exitcode == 0) receiveChildInfo();
-                updateDictResizePolicy();
-                closeChildInfoPipe();
-            }
-        }
-        else if ((pid = wait3(&statloc,WNOHANG,NULL)) != 0) {
+        if ((pid = wait3(&statloc,WNOHANG,NULL)) != 0) {
             int exitcode = WEXITSTATUS(statloc);
             int bysignal = 0;
 
@@ -1983,9 +1949,13 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
 
             if (pid == -1) {
                 serverLog(LL_WARNING,"wait3() returned an error: %s. "
-                    "aof_child_pid = %d",
+                    "rdb_child_pid = %d, aof_child_pid = %d",
                     strerror(errno),
+                    (int) g_pserver->rdb_child_pid,
                     (int) g_pserver->aof_child_pid);
+            } else if (pid == g_pserver->rdb_child_pid) {
+                backgroundSaveDoneHandler(exitcode,bysignal);
+                if (!bysignal && exitcode == 0) receiveChildInfo();
             } else if (pid == g_pserver->aof_child_pid) {
                 backgroundRewriteDoneHandler(exitcode,bysignal);
                 if (!bysignal && exitcode == 0) receiveChildInfo();
@@ -2026,7 +1996,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
 
         /* Trigger an AOF rewrite if needed. */
         if (g_pserver->aof_state == AOF_ON &&
-            !g_pserver->FRdbSaveInProgress() &&
+            g_pserver->rdb_child_pid == -1 &&
             g_pserver->aof_child_pid == -1 &&
             g_pserver->aof_rewrite_perc &&
             g_pserver->aof_current_size > g_pserver->aof_rewrite_min_size)
@@ -2055,9 +2025,6 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
             flushAppendOnlyFile(0);
     }
 
-    /* Clear the paused clients flag if needed. */
-    clientsArePaused(); /* Don't check return value, just use the side effect.*/
-
     /* Replication cron function -- used to reconnect to master,
      * detect transfer failures, start background RDB transfers and so forth. */
     run_with_period(1000) replicationCron();
@@ -2082,7 +2049,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
      * Note: this code must be after the replicationCron() call above so
      * make sure when refactoring this file to keep this order. This is useful
      * because we want to give priority to RDB savings for replication. */
-    if (!g_pserver->FRdbSaveInProgress() && g_pserver->aof_child_pid == -1 &&
+    if (g_pserver->rdb_child_pid == -1 && g_pserver->aof_child_pid == -1 &&
         g_pserver->rdb_bgsave_scheduled &&
         (g_pserver->unixtime-g_pserver->lastbgsave_try > CONFIG_BGSAVE_RETRY_DELAY ||
          g_pserver->lastbgsave_status == C_OK))
@@ -2114,6 +2081,9 @@ int serverCronLite(struct aeEventLoop *eventLoop, long long id, void *clientData
     {
         processUnblockedClients(iel);
     }
+
+    /* Unpause clients if enough time has elapsed */
+    unpauseClientsIfNecessary();
     
     ProcessPendingAsyncWrites();    // A bug but leave for now, events should clean up after themselves
     clientsCron(iel);
@@ -2907,6 +2877,7 @@ static void initServerThread(struct redisServerThreadVars *pvar, int fMain)
     pvar->cclients = 0;
     pvar->el = aeCreateEventLoop(g_pserver->maxclients+CONFIG_FDSET_INCR);
     pvar->current_client = nullptr;
+    pvar->clients_paused = 0;
     if (pvar->el == NULL) {
         serverLog(LL_WARNING,
             "Failed creating the event loop. Error message: '%s'",
@@ -2914,7 +2885,7 @@ static void initServerThread(struct redisServerThreadVars *pvar, int fMain)
         exit(1);
     }
 
-    fastlock_init(&pvar->lockPendingWrite);
+    fastlock_init(&pvar->lockPendingWrite, "lockPendingWrite");
 
     if (!fMain)
     {
@@ -2961,14 +2932,21 @@ void initServer(void) {
     signal(SIGPIPE, SIG_IGN);
     setupSignalHandlers();
 
-    fastlock_init(&g_pserver->flock);
-
     g_pserver->db = (redisDb*)zmalloc(sizeof(redisDb)*cserver.dbnum, MALLOC_LOCAL);
 
     /* Create the Redis databases, and initialize other internal state. */
     for (int j = 0; j < cserver.dbnum; j++) {
         new (&g_pserver->db[j]) redisDb;
-        g_pserver->db[j].initialize(j);
+        g_pserver->db[j].pdict = dictCreate(&dbDictType,NULL);
+        g_pserver->db[j].setexpire = new(MALLOC_LOCAL) expireset();
+        g_pserver->db[j].expireitr = g_pserver->db[j].setexpire->end();
+        g_pserver->db[j].blocking_keys = dictCreate(&keylistDictType,NULL);
+        g_pserver->db[j].ready_keys = dictCreate(&objectKeyPointerValueDictType,NULL);
+        g_pserver->db[j].watched_keys = dictCreate(&keylistDictType,NULL);
+        g_pserver->db[j].id = j;
+        g_pserver->db[j].avg_ttl = 0;
+        g_pserver->db[j].last_expire_set = 0;
+        g_pserver->db[j].defrag_later = listCreate();
     }
 
     /* Fixup Master Client Database */
@@ -2996,7 +2974,6 @@ void initServer(void) {
     g_pserver->ready_keys = listCreate();
     g_pserver->clients_waiting_acks = listCreate();
     g_pserver->get_ack_from_slaves = 0;
-    g_pserver->clients_paused = 0;
     cserver.system_memory_size = zmalloc_get_memory_size();
 
     createSharedObjects();
@@ -3008,7 +2985,7 @@ void initServer(void) {
     listSetFreeMethod(g_pserver->pubsub_patterns,freePubsubPattern);
     listSetMatchMethod(g_pserver->pubsub_patterns,listMatchPubsubPattern);
     g_pserver->cronloops = 0;
-    g_pserver->rdbThreadVars.fRdbThreadActive = false;
+    g_pserver->rdb_child_pid = -1;
     g_pserver->aof_child_pid = -1;
     g_pserver->rdb_child_type = RDB_CHILD_TYPE_NONE;
     g_pserver->rdb_bgsave_scheduled = 0;
@@ -3733,12 +3710,7 @@ int processCommand(client *c, int callFlags) {
         queueMultiCommand(c);
         addReply(c,shared.queued);
     } else {
-        std::unique_lock<decltype(c->db->lock)> ulock(c->db->lock);
-        for (int idb = 0; idb < cserver.dbnum; ++idb)
-            g_pserver->db[idb].trackChanges();
         call(c,callFlags);
-        for (int idb = 0; idb < cserver.dbnum; ++idb)
-            g_pserver->db[idb].processChanges();
         c->woff = g_pserver->master_repl_offset;
         if (listLength(g_pserver->ready_keys))
             handleClientsBlockedOnKeys();
@@ -3779,7 +3751,7 @@ int prepareForShutdown(int flags) {
     /* Kill the saving child if there is a background saving in progress.
        We want to avoid race conditions, for instance our saving child may
        overwrite the synchronous saving did by SHUTDOWN. */
-    if (g_pserver->FRdbSaveInProgress()) {
+    if (g_pserver->rdb_child_pid != -1) {
         serverLog(LL_WARNING,"There is a child saving an .rdb. Killing it!");
         killRDBChild();
     }
@@ -3810,7 +3782,7 @@ int prepareForShutdown(int flags) {
         /* Snapshotting. Perform a SYNC SAVE and exit */
         rdbSaveInfo rsi, *rsiptr;
         rsiptr = rdbPopulateSaveInfo(&rsi);
-        if (rdbSave(nullptr, rsiptr) != C_OK) {
+        if (rdbSave(rsiptr) != C_OK) {
             /* Ooops.. error saving! The best we can do is to continue
              * operating. Note that if there was a background saving process,
              * in the next cron() Redis will be notified that the background
@@ -4122,7 +4094,7 @@ sds genRedisInfoString(const char *section) {
             g_pserver->port,
             (intmax_t)uptime,
             (intmax_t)(uptime/(3600*24)),
-            g_pserver->hz,
+            g_pserver->hz.load(),
             g_pserver->config_hz,
             (unsigned long) lruclock,
             cserver.executable ? cserver.executable : "",
@@ -4293,12 +4265,12 @@ sds genRedisInfoString(const char *section) {
             "aof_last_cow_size:%zu\r\n",
             g_pserver->loading,
             g_pserver->dirty,
-            g_pserver->FRdbSaveInProgress(),
+            g_pserver->rdb_child_pid != -1,
             (intmax_t)g_pserver->lastsave,
             (g_pserver->lastbgsave_status == C_OK) ? "ok" : "err",
             (intmax_t)g_pserver->rdb_save_time_last,
-            (intmax_t)(g_pserver->FRdbSaveInProgress() ?
-                time(NULL)-g_pserver->rdb_save_time_start : -1),
+            (intmax_t)((g_pserver->rdb_child_pid == -1) ?
+                -1 : time(NULL)-g_pserver->rdb_save_time_start),
             g_pserver->stat_rdb_cow_bytes,
             g_pserver->aof_state != AOF_OFF,
             g_pserver->aof_child_pid != -1,
@@ -4614,8 +4586,8 @@ sds genRedisInfoString(const char *section) {
         for (j = 0; j < cserver.dbnum; j++) {
             long long keys, vkeys;
 
-            keys = g_pserver->db[j].size();
-            vkeys = g_pserver->db[j].expireSize();
+            keys = dictSize(g_pserver->db[j].pdict);
+            vkeys = g_pserver->db[j].setexpire->size();
 
             // Adjust TTL by the current time
             g_pserver->db[j].avg_ttl -= (g_pserver->mstime - g_pserver->db[j].last_expire_set);
@@ -4794,7 +4766,7 @@ static void sigShutdownHandler(int sig) {
      * on disk. */
     if (g_pserver->shutdown_asap && sig == SIGINT) {
         serverLogFromHandler(LL_WARNING, "You insist... exiting now.");
-        rdbRemoveTempFile(g_pserver->rdbThreadVars.tmpfileNum);
+        rdbRemoveTempFile(getpid());
         exit(1); /* Exit with an error since this was not a clean shutdown. */
     } else if (g_pserver->loading) {
         serverLogFromHandler(LL_WARNING, "Received shutdown signal during loading, exiting now.");

src/server.h

@@ -42,7 +42,6 @@
 #include <stdlib.h>
 #include <cmath>
 #include <string.h>
-#include <string>
 #include <time.h>
 #include <limits.h>
 #include <unistd.h>
@@ -55,7 +54,6 @@
 #include <vector>
 #include <algorithm>
 #include <memory>
-#include <set>
 #ifdef __cplusplus
 extern "C" {
 #include <lua.h>
@@ -91,7 +89,6 @@ typedef long long mstime_t; /* millisecond time type. */
 #include "sha1.h"
 #include "endianconv.h"
 #include "crc64.h"
-#include "IStorage.h"
 
 extern int g_fTestMode;
 
@@ -132,11 +129,6 @@ public:
         return m_ptr;
     }
 
-    const redisObject& operator*() const
-    {
-        return *m_ptr;
-    }
-
     bool operator!() const
     {
         return !m_ptr;
@@ -791,16 +783,6 @@ public:
             : when(when), spsubkey(subkey, sdsfree)
         {}
 
-        subexpireEntry(const subexpireEntry &e)
-            : when(e.when), spsubkey(nullptr, sdsfree)
-        {
-            if (e.spsubkey)
-                spsubkey = std::unique_ptr<const char, void(*)(const char*)>((const char*)sdsdup((sds)e.spsubkey.get()), sdsfree);
-        }
-
-        subexpireEntry(subexpireEntry &&e) = default;
-        subexpireEntry& operator=(subexpireEntry &&e) = default;
-
         bool operator<(long long when) const noexcept { return this->when < when; }
         bool operator<(const subexpireEntry &se) { return this->when < se.when; }
     };
@@ -813,10 +795,6 @@ public:
     expireEntryFat(sds keyPrimary)
         : m_keyPrimary(keyPrimary)
         {}
-        
-    expireEntryFat(const expireEntryFat &e) = default;
-    expireEntryFat(expireEntryFat &&e) = default;
-
     long long when() const noexcept { return m_vecexpireEntries.front().when; }
     const char *key() const noexcept { return m_keyPrimary; }
 
@@ -918,14 +896,6 @@ public:
         m_when = LLONG_MIN;
     }
 
-    expireEntry(const expireEntry &e)
-    {
-        u.m_key = e.u.m_key;
-        m_when = e.m_when;
-        if (e.FFat())
-            u.m_pfatentry = new (MALLOC_LOCAL) expireEntryFat(*e.u.m_pfatentry);
-    }
-
     expireEntry(expireEntry &&e)
     {
         u.m_key = e.u.m_key;
@@ -1069,183 +1039,17 @@ typedef struct clientReplyBlock {
 #endif
 } clientReplyBlock;
 
-struct dictEntry;
-class dict_const_iter
-{
-    friend struct redisDb;
-    friend class redisDbPersistentData;
-protected:
-    dictEntry *de;
-public:
-    explicit dict_const_iter(dictEntry *de)
-        : de(de)
-    {}
-
-    const char *key() const { return de ? (const char*)dictGetKey(de) : nullptr; }
-    robj_roptr val() const { return de ? (robj*)dictGetVal(de) : nullptr; }
-    const robj* operator->() const { return de ? (robj*)dictGetVal(de) : nullptr; }
-    operator robj_roptr() const { return de ? (robj*)dictGetVal(de) : nullptr; }
-
-    bool operator==(std::nullptr_t) const { return de == nullptr; }
-    bool operator!=(std::nullptr_t) const { return de != nullptr; }
-    bool operator==(const dict_const_iter &other) { return de == other.de; }
-};
-class dict_iter : public dict_const_iter
-{
-public:
-    explicit dict_iter(dictEntry *de)
-        : dict_const_iter(de)
-    {}
-    sds key() { return de ? (sds)dictGetKey(de) : nullptr; }
-    robj *val() { return de ? (robj*)dictGetVal(de) : nullptr; }
-    robj *operator->() { return de ? (robj*)dictGetVal(de) : nullptr; }
-    operator robj*() const { return de ? (robj*)dictGetVal(de) : nullptr; }
-};
-
-class redisDbPersistentData
-{
-    friend void dictDbKeyDestructor(void *privdata, void *key);
-public:
-    ~redisDbPersistentData();
-
-    static void swap(redisDbPersistentData *db1, redisDbPersistentData *db2);
-
-    size_t slots() const { return dictSlots(m_pdict); }
-    size_t size() const { return dictSize(m_pdict); }
-    void expand(uint64_t slots) { dictExpand(m_pdict, slots); }
-    
-    void trackkey(robj_roptr o)
-    {
-        trackkey(szFromObj(o));
-    }
-
-    void trackkey(const char *key)
-    {
-        if (m_fTrackingChanges && !m_fAllChanged)
-            m_setchanged.insert(std::string(key, sdslen(key)));
-    }
-
-    dict_iter find(const char *key) 
-    {
-        dictEntry *de = dictFind(m_pdict, key);
-        ensure(de);
-        return dict_iter(de);
-    }
-
-    dict_iter find(robj_roptr key)
-    {
-        return find(szFromObj(key));
-    }
-
-    dict_iter random()
-    {
-        dictEntry *de = dictGetRandomKey(m_pdict);
-        ensure(de);
-        return dict_iter(de);
-    }
-
-    const expireEntry &random_expire()
-    {
-        return m_setexpire->random_value();
-    }
-
-    auto findExpire(const char *key)
-    {
-        return m_setexpire->find(key);
-    }
-
-    dict_iter end()  { return dict_iter(nullptr); }
-    dict_const_iter end() const { return dict_const_iter(nullptr); }
-
-    void getStats(char *buf, size_t bufsize) { dictGetStats(buf, bufsize, m_pdict); }
-    void getExpireStats(char *buf, size_t bufsize) { m_setexpire->getstats(buf, bufsize); }
-
-    bool insert(char *k, robj *o);
-    void tryResize();
-    int incrementallyRehash();
-    void updateValue(dict_iter itr, robj *val);
-    bool syncDelete(robj *key);
-    bool asyncDelete(robj *key);
-    size_t expireSize() const { return m_setexpire->size(); }
-    int removeExpire(robj *key, dict_iter itr);
-    int removeSubkeyExpire(robj *key, robj *subkey);
-    void clear(void(callback)(void*));
-    void emptyDbAsync();
-    bool iterate(std::function<bool(const char*, robj*)> fn);
-    void setExpire(robj *key, robj *subkey, long long when);
-    void setExpire(expireEntry &&e);
-    expireEntry *getExpire(robj_roptr key);
-    void initialize();
-
-    void trackChanges() { m_fTrackingChanges++; }
-    void processChanges();
-
-    // This should only be used if you look at the key, we do not fixup
-    //  objects stored elsewhere
-    dict *dictUnsafeKeyOnly() { return m_pdict; }   
-
-    expireset *setexpireUnsafe() { return m_setexpire; }
-    const expireset *setexpire() { return m_setexpire; }
-
-    std::shared_ptr<redisDbPersistentData> createSnapshot();
-    void endSnapshot(const redisDbPersistentData *psnapshot);
-
-private:
-    void ensure(const char *key);
-    void ensure(dictEntry *de);
-    void storeDatabase();
-    void storeKey(const char *key, size_t cchKey, robj *o);
-
-    // Keyspace
-    dict *m_pdict = nullptr;                 /* The keyspace for this DB */
-    int m_fTrackingChanges = 0;     // Note: Stack based
-    bool m_fAllChanged = false;
-    std::set<std::string> m_setchanged;
-    IStorage *m_pstorage = nullptr;
-
-    // Expire
-    expireset *m_setexpire = nullptr;
-
-    std::shared_ptr<redisDbPersistentData> m_spdbSnapshot;
-};
-
 /* Redis database representation. There are multiple databases identified
  * by integers from 0 (the default database) up to the max configured
  * database. The database number is the 'id' field in the structure. */
-typedef struct redisDb : public redisDbPersistentData 
-{
-    // Legacy C API, Do not add more
-    friend void tryResizeHashTables(int);
-    friend int dbSyncDelete(redisDb *db, robj *key);
-    friend int dbAsyncDelete(redisDb *db, robj *key);
-    friend long long emptyDb(int dbnum, int flags, void(callback)(void*));
-    friend void scanGenericCommand(struct client *c, robj_roptr o, unsigned long cursor);
-    friend int dbSwapDatabases(int id1, int id2);
-    friend int removeExpire(redisDb *db, robj *key);
-    friend void setExpire(struct client *c, redisDb *db, robj *key, robj *subkey, long long when);
-    friend void setExpire(client *c, redisDb *db, robj *key, expireEntry &&e);
-    friend void evictionPoolPopulate(int dbid, redisDb *db, expireset *setexpire, struct evictionPoolEntry *pool);
-    friend void activeDefragCycle(void);
-    friend int freeMemoryIfNeeded(void);
-    friend void activeExpireCycle(int);
-    friend void expireSlaveKeys(void);
-
-    typedef ::dict_const_iter const_iter;
-    typedef ::dict_iter iter;
-
-    redisDb()
+typedef struct redisDb {
+    redisDb() 
         : expireitr(nullptr)
-    {}
-    void initialize(int id);
-
-    void dbOverwriteCore(redisDb::iter itr, robj *key, robj *val, bool fUpdateMvcc, bool fRemoveExpire);
-
-
-    bool FKeyExpires(const char *key);
-    size_t clear(bool fAsync, void(callback)(void*));
-
-public:
+    {};
+    dict *pdict;                 /* The keyspace for this DB */
+    expireset *setexpire;
     expireset::setiter expireitr;
+
     dict *blocking_keys;        /* Keys with clients waiting for data (BLPOP)*/
     dict *ready_keys;           /* Blocked keys that received a PUSH */
     dict *watched_keys;         /* WATCHED keys for MULTI/EXEC CAS */
@@ -1253,8 +1057,6 @@ public:
     long long last_expire_set;  /* when the last expire was set */
     double avg_ttl;             /* Average TTL, just for stats */
     list *defrag_later;         /* List of key names to attempt to defrag one by one, gradually. */
-
-    fastlock lock;
 } redisDb;
 
 /* Client MULTI/EXEC state */
@@ -1588,9 +1390,6 @@ typedef struct rdbSaveInfo {
     char repl_id[CONFIG_RUN_ID_SIZE+1];     /* Replication ID. */
     long long repl_offset;                  /* Replication offset. */
     int fForceSetKey;
-
-    /* Used In Save */
-    long long master_repl_offset;
 } rdbSaveInfo;
 
 #define RDB_SAVE_INFO_INIT {-1,0,"000000000000000000000000000000",-1, TRUE}
@@ -1627,6 +1426,7 @@ struct redisServerThreadVars {
     aeEventLoop *el;
     int ipfd[CONFIG_BINDADDR_MAX]; /* TCP socket file descriptors */
     int ipfd_count;             /* Used slots in ipfd[] */
+    int clients_paused;         /* True if clients are currently paused */
     std::vector<client*> clients_pending_write; /* There is to write or install handler. */
     list *unblocked_clients;     /* list of clients to unblock before next loop NOT THREADSAFE */
     list *clients_pending_asyncwrite;
@@ -1636,7 +1436,7 @@ struct redisServerThreadVars {
                                 client blocked on a module command needs
                                 to be processed. */
     client *lua_client = nullptr;   /* The "fake client" to query Redis from Lua */
-    struct fastlock lockPendingWrite;
+    struct fastlock lockPendingWrite { "thread pending write" };
     char neterr[ANET_ERR_LEN];   /* Error buffer for anet.c */
     long unsigned commandsExecuted = 0;
 };
@@ -1719,7 +1519,7 @@ struct redisServer {
     int config_hz;              /* Configured HZ value. May be different than
                                    the actual 'hz' field value if dynamic-hz
                                    is enabled. */
-    int hz;                     /* serverCron() calls frequency in hertz */
+    std::atomic<int> hz;                     /* serverCron() calls frequency in hertz */
     redisDb *db;
     dict *commands;             /* Command table */
     dict *orig_commands;        /* Command table before command renaming. */
@@ -1754,7 +1554,6 @@ struct redisServer {
     list *clients_to_close;     /* Clients to close asynchronously */
     list *slaves, *monitors;    /* List of slaves and MONITORs */
     rax *clients_index;         /* Active clients dictionary by client ID. */
-    int clients_paused;         /* True if clients are currently paused */
     mstime_t clients_pause_end_time; /* Time when we undo clients_paused */
     dict *migrate_cached_sockets;/* MIGRATE cached sockets */
     std::atomic<uint64_t> next_client_id; /* Next client unique ID. Incremental. */
@@ -1848,13 +1647,7 @@ struct redisServer {
     /* RDB persistence */
     long long dirty;                /* Changes to DB from the last save */
     long long dirty_before_bgsave;  /* Used to restore dirty on failed BGSAVE */
-    struct _rdbThreadVars
-    {
-        bool fRdbThreadActive = false;
-        volatile bool fRdbThreadCancel = false;
-        pthread_t rdb_child_thread;     /* PID of RDB saving child */
-        int tmpfileNum = 0;
-    } rdbThreadVars;
+    pid_t rdb_child_pid;            /* PID of RDB saving child */
     struct saveparam *saveparams;   /* Save points array for RDB */
     int saveparamslen;              /* Number of saving points */
     char *rdb_filename;             /* Name of RDB file */
@@ -2024,8 +1817,6 @@ struct redisServer {
 
     int fActiveReplica;                          /* Can this replica also be a master? */
 
-    struct fastlock flock;
-
     // Format:
     //  Lower 20 bits: a counter incrementing for each command executed in the same millisecond
     //  Upper 44 bits: mstime (least significant 44-bits) enough for ~500 years before rollover from date of addition
@@ -2033,8 +1824,6 @@ struct redisServer {
 
     /* System hardware info */
     size_t system_memory_size;  /* Total memory in system as reported by OS */
-
-    bool FRdbSaveInProgress() const { return rdbThreadVars.fRdbThreadActive; }
 };
 
 typedef struct pubsubPattern {
@@ -2140,11 +1929,9 @@ extern dictType zsetDictType;
 extern dictType clusterNodesDictType;
 extern dictType clusterNodesBlackListDictType;
 extern dictType dbDictType;
-extern dictType dbSnapshotDictType;
 extern dictType shaScriptObjectDictType;
 extern double R_Zero, R_PosInf, R_NegInf, R_Nan;
 extern dictType hashDictType;
-extern dictType keylistDictType;
 extern dictType replScriptCacheDictType;
 extern dictType keyptrDictType;
 extern dictType modulesDictType;
@@ -2255,6 +2042,7 @@ void disconnectSlavesExcept(unsigned char *uuid);
 int listenToPort(int port, int *fds, int *count, int fReusePort, int fFirstListen);
 void pauseClients(mstime_t duration);
 int clientsArePaused(void);
+void unpauseClientsIfNecessary();
 int processEventsWhileBlocked(int iel);
 int handleClientsWithPendingWrites(int iel);
 int clientHasPendingReplies(client *c);
@@ -2373,12 +2161,8 @@ const char *strEncoding(int encoding);
 int compareStringObjects(robj *a, robj *b);
 int collateStringObjects(robj *a, robj *b);
 int equalStringObjects(robj *a, robj *b);
-unsigned long long estimateObjectIdleTime(robj_roptr o);
+unsigned long long estimateObjectIdleTime(robj *o);
 void trimStringObjectIfNeeded(robj *o);
-
-robj *deserializeStoredObject(const void *data, size_t cb);
-sds serializeStoredObject(robj_roptr o);
-
 #define sdsEncodedObject(objptr) (objptr->encoding == OBJ_ENCODING_RAW || objptr->encoding == OBJ_ENCODING_EMBSTR)
 
 /* Synchronous I/O with timeout */
@@ -2431,7 +2215,7 @@ int writeCommandsDeniedByDiskError(void);
 
 /* RDB persistence */
 #include "rdb.h"
-int rdbSaveRio(rio *rdb, redisDbPersistentData **rgpdb, int *error, int flags, rdbSaveInfo *rsi);
+int rdbSaveRio(rio *rdb, int *error, int flags, rdbSaveInfo *rsi);
 void killRDBChild(void);
 
 /* AOF persistence */
@@ -2649,9 +2433,11 @@ int rewriteConfig(char *path);
 
 /* db.c -- Keyspace access API */
 int removeExpire(redisDb *db, robj *key);
+int removeExpireCore(redisDb *db, robj *key, dictEntry *de);
 int removeSubkeyExpire(redisDb *db, robj *key, robj *subkey);
 void propagateExpire(redisDb *db, robj *key, int lazy);
 int expireIfNeeded(redisDb *db, robj *key);
+expireEntry *getExpire(redisDb *db, robj_roptr key);
 void setExpire(client *c, redisDb *db, robj *key, robj *subkey, long long when);
 void setExpire(client *c, redisDb *db, robj *key, expireEntry &&entry);
 robj_roptr lookupKeyRead(redisDb *db, robj *key);
@@ -2659,8 +2445,8 @@ robj *lookupKeyWrite(redisDb *db, robj *key);
 robj_roptr lookupKeyReadOrReply(client *c, robj *key, robj *reply);
 robj *lookupKeyWriteOrReply(client *c, robj *key, robj *reply);
 robj_roptr lookupKeyReadWithFlags(redisDb *db, robj *key, int flags);
-robj_roptr objectCommandLookup(client *c, robj *key);
-robj_roptr objectCommandLookupOrReply(client *c, robj *key, robj *reply);
+robj *objectCommandLookup(client *c, robj *key);
+robj *objectCommandLookupOrReply(client *c, robj *key, robj *reply);
 void objectSetLRUOrLFU(robj *val, long long lfu_freq, long long lru_idle,
                        long long lru_clock);
 #define LOOKUP_NONE 0
@@ -2693,6 +2479,7 @@ void slotToKeyAdd(robj *key);
 void slotToKeyDel(robj *key);
 void slotToKeyFlush(void);
 int dbAsyncDelete(redisDb *db, robj *key);
+void emptyDbAsync(redisDb *db);
 void slotToKeyFlushAsync(void);
 size_t lazyfreeGetPendingObjectsCount(void);
 void freeObjAsync(robj *o);
@@ -2716,7 +2503,6 @@ void clusterPropagatePublish(robj *channel, robj *message);
 void migrateCloseTimedoutSockets(void);
 void clusterBeforeSleep(void);
 int clusterSendModuleMessageToTarget(const char *target, uint64_t module_id, uint8_t type, unsigned char *payload, uint32_t len);
-void createDumpPayload(rio *payload, robj_roptr o, robj *key);
 
 /* Sentinel */
 void initSentinelConfig(void);
@@ -3010,7 +2796,7 @@ void xorDigest(unsigned char *digest, const void *ptr, size_t len);
 int populateCommandTableParseFlags(struct redisCommand *c, const char *strflags);
 
 int moduleGILAcquiredByModule(void);
-extern bool g_fInCrash;
+extern int g_fInCrash;
 static inline int GlobalLocksAcquired(void)  // Used in asserts to verify all global locks are correctly acquired for a server-thread to operate
 {
     return aeThreadOwnsLock() || moduleGILAcquiredByModule() || g_fInCrash;

src/t_set.cpp

@@ -219,9 +219,9 @@ int setTypeRandomElement(robj *setobj, sds *sdsele, int64_t *llele) {
     return setobj->encoding;
 }
 
-int setTypeRandomElement(robj_roptr setobj, const char **sdsele, int64_t *llele)
+int setTypeRandomElement(robj_roptr setobj, const char **sdsele, const int64_t *llele)
 {
-    return setTypeRandomElement(setobj.unsafe_robjcast(), (sds*)sdsele, llele);
+    return setTypeRandomElement(setobj.unsafe_robjcast(), (sds*)sdsele, const_cast<int64_t*>(llele));
 }
 
 unsigned long setTypeSize(robj_roptr subject) {

tests/cluster/run.tcl

@@ -14,6 +14,8 @@ proc main {} {
     spawn_instance redis $::redis_base_port $::instances_count {
         "cluster-enabled yes"
         "appendonly yes"
+	"testmode yes"
+	"server-threads 3"
     }
     run_tests
     cleanup

tests/integration/logging.tcl

@@ -6,7 +6,7 @@ if {$system_name eq {linux} || $system_name eq {darwin}} {
         test "Server is able to generate a stack trace on selected systems" {
             r config set watchdog-period 200
             r debug sleep 1
-            set pattern "*debugCommand*"
+            set pattern "*watchdogSignalHandler*"
             set retry 10
             while {$retry} {
                 set result [exec tail -100 < [srv 0 stdout]]

tests/unit/bitops.tcl

@@ -238,7 +238,7 @@ start_server {tags {"bitops"}} {
         r set a "abcdefg"
         r bitop lshift x a 8
         r get x
-    } "\x00abcdefg"
+    } "\000abcdefg"
 
     test {BITOP lshift char} {
         r set a "\xAA"