init iteration

compile fix
somehow missed reference to m_setexpire
2023-08-22 22:48:26 -07:00 · 2023-08-22 13:18:14 -07:00 · 2023-08-22 13:03:03 -07:00 · 2023-08-22 12:48:28 -07:00 · 2023-06-29 21:50:05 -07:00 · 2023-06-28 17:44:36 -07:00
52 changed files with 373 additions and 1335 deletions
@@ -21,25 +21,13 @@ machamp:
        parent: make-build
        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
-        command: ./runtest --clients 4 --verbose --tls --skipunit unit/memefficiency
-      tests-with-flash:
-        type: cmd
-        parent: make-build
-        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
-        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
-        command: ./runtest --verbose --skipunit unit/memefficiency --flash
+        command: ./runtest --clients 4 --verbose 
      cluster-test:
        type: cmd
        parent: make-build
        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
-        command: ./runtest-cluster --tls
-      cluster-test-flash:
-        type: cmd
-        parent: make-build
-        # https://github.sc-corp.net/Snapchat/img/tree/master/keydb/ubuntu-20-04
-        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/keydb/keydb-ubuntu-20-04@sha256:cf869a3f5d1de1e1d976bb906689c37b7031938eb68661b844a38c532f27248c
-        command: ./runtest-cluster --tls --flash
+        command: ./runtest-cluster 
      sentinel-test:
        type: cmd
        parent: make-build
@@ -71,7 +59,7 @@ machamp:
      # to ensure a clearer docker build env
      code-checkout:
        type: cmd
-        command: echo checkout && git submodule init && git submodule update
+        command: echo checkout
        # default machamp builder image does not work for multi arch
        builder_image: us.gcr.io/snapchat-build-artifacts/prod/snapchat/img/ubuntu/ubuntu-23-04@sha256:bd43177a80e6ce1c3583e8ea959b88a9081c0f56b765ec9c5a157c27a637c23b
      docker:
@@ -26,16 +26,6 @@ on:
          # references a build defined in build.yaml
          build_name: keydb-docker-build
          arch_types: ["amd64", "arm64"]
-  # Doc: go/cool-guide
-  cool:
-    workflows:
-      - workflow_type: backend_workflow
-        build_name: keydb-build
-        arch_types: ["amd64", "arm64"]
-      - workflow_type: backend_workflow
-        # references a build defined in build.yaml
-        build_name: keydb-docker-build
-        arch_types: ["amd64", "arm64"]

 # below defines which branch is release branch / release tag
 machamp:
@@ -24,7 +24,7 @@ RUN set -eux; \
        gosu nobody true
 # build KeyDB
 ARG MAKE_JOBS=""
-ARG ENABLE_FLASH="yes"
+ARG ENABLE_FLASH=""
 COPY . /tmp/keydb-internal
 RUN set -eux; \
        cd /tmp/keydb-internal; \
@@ -73,25 +73,7 @@ RUN set -eux; \
               | xargs -r apt-mark manual \
        ; \
        apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
-        rm -rf /var/lib/apt/lists/*
-# build RedisBloom
-RUN set -eux; \
-        savedAptMark="$(apt-mark showmanual)"; \
-        apt-get update; \
-        DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confnew" install -qqy --no-install-recommends \
-                ca-certificates \
-                wget \
-                make \
-                pkg-config \
-                build-essential \
-                git; \
-        git clone --recursive https://github.com/RedisBloom/RedisBloom.git; \
-        cd RedisBloom; \
-        ./sbin/setup; \
-        bash -l; \
-        make; \
-        redisBloomPath=`make run -n | awk '{print $NF}'`; \
-        cp $redisBloomPath /usr/local/lib/
+        rm -rf /var/lib/apt/lists/*; \
 # create working directories and organize files
 RUN \
        mkdir /data && chown keydb:keydb /data; \
@@ -53,7 +53,7 @@ RUN set -eux; \
        grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \
        sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \
        grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \
-        make -j$(nproc) BUILD_TLS=yes ENABLE_FLASH=yes; \
+        make -j$(nproc) BUILD_TLS=yes; \
        cd src; \
        strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel keydb-server; \
        mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \
@@ -83,7 +83,6 @@ RUN \
        sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \
        sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \
        sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \
-        cd /usr/local/bin; \
        ln -s keydb-cli redis-cli; \
        cd /etc/keydb; \
        ln -s keydb.conf redis.conf; \
@@ -5,7 +5,7 @@ RUN mkdir -p /etc/keydb
 ARG BRANCH
 RUN set -eux; \
        \
-        apk add --no-cache su-exec tini; \
+        apk add --no-cache su-exec; \
        apk add --no-cache --virtual .build-deps \
                coreutils \
                gcc \
@@ -32,7 +32,7 @@ RUN set -eux; \
        grep -E '^ *createBoolConfig[(]"protected-mode",.*, *1 *,.*[)],$' ./src/config.cpp; \
        sed -ri 's!^( *createBoolConfig[(]"protected-mode",.*, *)1( *,.*[)],)$!\10\2!' ./src/config.cpp; \
        grep -E '^ *createBoolConfig[(]"protected-mode",.*, *0 *,.*[)],$' ./src/config.cpp; \
-        make -j$(nproc) BUILD_TLS=yes ENABLE_FLASH=yes; \
+        make -j$(nproc) BUILD_TLS=yes; \
        cd src; \
        strip keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel keydb-server; \
        mv keydb-server keydb-cli keydb-benchmark keydb-check-rdb keydb-check-aof keydb-diagnostic-tool keydb-sentinel /usr/local/bin/; \
@@ -54,7 +54,6 @@ RUN set -eux; \
        sed -i 's/^\(logfile .*\)$/# \1/' /etc/keydb/keydb.conf; \
        sed -i 's/protected-mode yes/protected-mode no/g' /etc/keydb/keydb.conf; \
        sed -i 's/^\(bind .*\)$/# \1/' /etc/keydb/keydb.conf; \
-        cd /usr/local/bin; \
        ln -s keydb-cli redis-cli; \
        cd /etc/keydb; \
        ln -s keydb.conf redis.conf; \
@@ -77,6 +76,6 @@ RUN set -eux; \
        chmod +x /usr/local/bin/docker-entrypoint.sh
 VOLUME /data
 WORKDIR /data
-ENTRYPOINT ["tini", "--", "docker-entrypoint.sh"]
+ENTRYPOINT ["docker-entrypoint.sh"]
 EXPOSE 6379
 CMD ["keydb-server", "/etc/keydb/keydb.conf"]
@@ -1,25 +1,10 @@
 #pragma once
 #include <functional>
-#include <unordered_set>
 #include "sds.h"
-#include "ae.h"
 #include <string>

 #define METADATA_DB_IDENTIFIER "c299fde0-6d42-4ec4-b939-34f680ffe39f"

-struct StorageToken {
-    enum class TokenType {
-        SingleRead,
-        SingleWrite,
-        Delete,
-        BatchWrite,
-    };
-    TokenType type;
-    std::unordered_set<struct client *> setc;
-    struct redisDbPersistentData *db;
-    virtual ~StorageToken() {}
-};
-
 class IStorageFactory
 {
 public:
@@ -51,12 +36,6 @@ public:
    virtual bool enumerate_hashslot(callback fn, unsigned int hashslot) const = 0;
    virtual size_t count() const = 0;

-    virtual StorageToken *begin_retrieve(struct aeEventLoop *, aePostFunctionTokenProc, sds *, size_t) {return nullptr;};
-    virtual void complete_retrieve(StorageToken * /*tok*/, callbackSingle /*fn*/) {};
-
-    virtual StorageToken* begin_endWriteBatch(struct aeEventLoop *, aePostFunctionTokenProc*) { return nullptr; } // NOP
-    virtual void complete_endWriteBatch(StorageToken * /*tok*/) {};
-
    virtual void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem) {
        beginWriteBatch();
        for (size_t ielem = 0; ielem < celem; ++ielem) {
@@ -65,7 +44,7 @@ public:
        endWriteBatch();
    }

-    virtual std::vector<std::string> getExpirationCandidates(unsigned int count) = 0;
+    virtual std::vector<std::string> getExpirationCandidates(unsigned int count, long long time) = 0;
    virtual std::vector<std::string> getEvictionCandidates(unsigned int count) = 0;
    virtual void setExpire(const char *key, size_t cchKey, long long expire) = 0;
    virtual void removeExpire(const char *key, size_t cchKey, long long expire) = 0;
@@ -73,10 +73,10 @@ ifneq ($(strip $(SANITIZE)),)
 endif

 ifeq ($(ENABLE_FLASH),yes)
-	FINAL_LIBS+= ../deps/rocksdb/librocksdb.a
 	FINAL_LIBS+= -lz -lcrypto -lbz2 -lzstd -llz4 -lsnappy
 	CXXFLAGS+= -I../deps/rocksdb/include/ -DENABLE_ROCKSDB
 	STORAGE_OBJ+= storage/rocksdb.o storage/rocksdbfactory.o
+	FINAL_LIBS+= ../deps/rocksdb/librocksdb.a 
 	DEPENDENCY_TARGETS+= rocksdb
 endif

@@ -136,33 +136,14 @@ void SnapshotPayloadParseState::flushQueuedKeys() {
    int idb = current_database;
    serverAssert(vecqueuedKeys.size() == vecqueuedVals.size());
    auto sizePrev = vecqueuedKeys.size();
+    (*insertsInFlight)++;
+    std::weak_ptr<std::atomic<int>> insertsInFlightTmp = insertsInFlight; // C++ GRRRRRRRRRRRRRRRR, we don't want to capute "this" because that's dangerous
    if (current_database < cserver.dbnum) {
-        if (g_pserver->m_pstorageFactory) {
-            (*insertsInFlight)++;
-            std::weak_ptr<std::atomic<int>> insertsInFlightTmp = insertsInFlight; // C++ GRRRRRRRRRRRRRRRR, we don't want to capute "this" because that's dangerous
-            g_pserver->asyncworkqueue->AddWorkFunction([idb, vecqueuedKeys = std::move(this->vecqueuedKeys), vecqueuedKeysCb = std::move(this->vecqueuedKeysCb), vecqueuedVals = std::move(this->vecqueuedVals), vecqueuedValsCb = std::move(this->vecqueuedValsCb), insertsInFlightTmp, pallocator = m_spallocator.release()]() mutable {
-                g_pserver->db[idb]->bulkDirectStorageInsert(vecqueuedKeys.data(), vecqueuedKeysCb.data(), vecqueuedVals.data(), vecqueuedValsCb.data(), vecqueuedKeys.size());
-                (*(insertsInFlightTmp.lock()))--;
-                delete pallocator;
-            });
-        } else {
-            for (size_t ival = 0; ival < vecqueuedKeys.size(); ++ival) {
-                size_t offset = 0;
-                auto spexpire = deserializeExpire(vecqueuedVals[ival], vecqueuedValsCb[ival], &offset);    
-                auto o = deserializeStoredObject(vecqueuedVals[ival] + offset, vecqueuedValsCb[ival] - offset);
-                sds sdsKey = sdsnewlen(vecqueuedKeys[ival], -static_cast<ssize_t>(vecqueuedKeysCb[ival]));
-                if (dbMerge(g_pserver->db[idb], sdsKey, o, false)) {
-                    if (spexpire != nullptr)
-                        g_pserver->db[idb]->setExpire(sdsKey, std::move(*spexpire));
-                } else {
-                    sdsfree(sdsKey);
-                }
-            }
-            vecqueuedKeys.clear();
-            vecqueuedKeysCb.clear();
-            vecqueuedVals.clear();
-            vecqueuedValsCb.clear();
-        }
+        g_pserver->asyncworkqueue->AddWorkFunction([idb, vecqueuedKeys = std::move(this->vecqueuedKeys), vecqueuedKeysCb = std::move(this->vecqueuedKeysCb), vecqueuedVals = std::move(this->vecqueuedVals), vecqueuedValsCb = std::move(this->vecqueuedValsCb), insertsInFlightTmp, pallocator = m_spallocator.release()]() mutable {
+            g_pserver->db[idb]->bulkDirectStorageInsert(vecqueuedKeys.data(), vecqueuedKeysCb.data(), vecqueuedVals.data(), vecqueuedValsCb.data(), vecqueuedKeys.size());
+            (*(insertsInFlightTmp.lock()))--;
+            delete pallocator;
+        });
    } else {
        // else drop the data
        vecqueuedKeys.clear();
@@ -233,6 +214,11 @@ void SnapshotPayloadParseState::trimState() {
    
    if (stackParse.empty()) {
        flushQueuedKeys();
+        while (*insertsInFlight > 0) {
+            // TODO: ProcessEventsWhileBlocked
+            aeReleaseLock();
+            aeAcquireLock();
+        }
    }
 }

@@ -63,5 +63,4 @@ public:
    void pushValue(const char *rgch, long long cch);
    void pushValue(long long value);
    bool shouldThrottle() const { return *insertsInFlight > (cserver.cthreads*4); }
-    bool hasIOInFlight() const { return *insertsInFlight > 0; }
 };
@@ -208,30 +208,6 @@ void StorageCache::retrieve(sds key, IStorage::callbackSingle fn) const
    m_spstorage->retrieve(key, sdslen(key), fn);
 }

-StorageToken *StorageCache::begin_retrieve(struct aeEventLoop *el, aePostFunctionTokenProc proc, sds *rgkey, size_t ckey) {
-    std::unique_lock<fastlock> ul(m_lock);
-    if (m_pdict != nullptr)
-    {
-        bool fAnyKey = false;
-        for (size_t ik = 0; ik < ckey; ++ik) {
-            uint64_t hash = dictSdsHash(rgkey[ik]);
-            dictEntry *de = dictFind(m_pdict, reinterpret_cast<void*>(hash));
-
-            if (de != nullptr)
-                fAnyKey = true;
-        }
-        if (!fAnyKey)
-            return nullptr; // All keys are missing - skip the io
-    }
-    ul.unlock();
-
-    return m_spstorage->begin_retrieve(el, proc, rgkey, ckey);
-}
-
-void StorageCache::complete_retrieve(StorageToken *tok, IStorage::callbackSingle fn) {
-    m_spstorage->complete_retrieve(tok, fn);
-}
-
 size_t StorageCache::count() const
 {
    std::unique_lock<fastlock> ul(m_lock, std::defer_lock);
@@ -257,5 +233,4 @@ void StorageCache::emergencyFreeCache() {
            dictRelease(d);
        });
    }
-}
-
+}
@@ -43,10 +43,6 @@ public:
    void insert(sds key, const void *data, size_t cbdata, bool fOverwrite);
    void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem);
    void retrieve(sds key, IStorage::callbackSingle fn) const;
-    StorageToken *begin_retrieve(struct aeEventLoop *el, aePostFunctionTokenProc proc, sds *rgkey, size_t ckey);
-    void complete_retrieve(StorageToken *tok, IStorage::callbackSingle fn);
-    StorageToken* begin_endWriteBatch(struct aeEventLoop *el, aePostFunctionTokenProc* proc) {return m_spstorage->begin_endWriteBatch(el,proc);} // NOP
-    void complete_endWriteBatch(StorageToken *tok) {m_spstorage->complete_endWriteBatch(tok);};
    bool erase(sds key);
    void emergencyFreeCache();
    bool keycacheIsEnabled() const { return m_pdict != nullptr; }
@@ -55,7 +51,7 @@ public:
    bool enumerate(IStorage::callback fn) const { return m_spstorage->enumerate(fn); }
    bool enumerate_hashslot(IStorage::callback fn, unsigned int hashslot) const { return m_spstorage->enumerate_hashslot(fn, hashslot); }

-    std::vector<std::string> getExpirationCandidates(unsigned int count) { return m_spstorage->getExpirationCandidates(count); }
+    std::vector<std::string> getExpirationCandidates(unsigned int count, long long time) { return m_spstorage->getExpirationCandidates(count, time); }
    std::vector<std::string> getEvictionCandidates(unsigned int count) { return m_spstorage->getEvictionCandidates(count); }
    void setExpire(const char *key, size_t cchKey, long long expire) { m_spstorage->setExpire(key, cchKey, expire); }
    void removeExpire(const char *key, size_t cchKey, long long expire) { m_spstorage->removeExpire(key, cchKey, expire); }
@@ -113,8 +113,6 @@ enum class AE_ASYNC_OP
    PostCppFunction,
    DeleteFileEvent,
    CreateFileEvent,
-    PostAsynDBFunction,
-
 };

 struct aeCommand
@@ -127,8 +125,6 @@ struct aeCommand
        aePostFunctionProc *proc;
        aeFileProc *fproc;
        std::function<void()> *pfn;
-        aePostFunctionTokenProc* tproc;
-
    };
    void *clientData;
 };
@@ -179,17 +175,6 @@ void aeProcessCmd(aeEventLoop *eventLoop, int fd, void *, int )
            (*cmd.pfn)();

            delete cmd.pfn;
-            break;
-        }
-        case AE_ASYNC_OP::PostAsynDBFunction:
-        {   //added to support async api IStorage
-           if (cmd.fLock && !ulock.owns_lock()) {
-                g_forkLock.releaseRead();
-                ulock.lock();
-                g_forkLock.acquireRead();
-            }  
-            ((aePostFunctionTokenProc*)cmd.tproc)(eventLoop,(StorageToken*)cmd.clientData);
-            break; 
        }
            break;
        }
@@ -271,20 +256,6 @@ int aePostFunction(aeEventLoop *eventLoop, aePostFunctionProc *proc, void *arg)
    return AE_OK;
 }

-int aePostFunction(aeEventLoop *eventLoop, aePostFunctionTokenProc *proc, StorageToken *token)
-{
-  //added to support async api IStorage
-    aeCommand cmd = {};
-    cmd.op = AE_ASYNC_OP::PostAsynDBFunction; 
-    cmd.tproc = proc;
-    cmd.clientData = (void*)token; 
-    cmd.fLock = false;
-    auto size = write(eventLoop->fdCmdWrite, &cmd, sizeof(cmd));
-    if (size != sizeof(cmd))
-        return AE_ERR;
-    return AE_OK;
-}
-
 int aePostFunction(aeEventLoop *eventLoop, std::function<void()> fn, bool fLock, bool fForceQueue)
 {
    if (eventLoop == g_eventLoopThisThread && !fForceQueue)
@@ -890,7 +861,7 @@ void aeReleaseForkLock()

 void aeForkLockInChild()
 {
-    g_forkLock.setMulti(false);
+    g_forkLock.setNotify(false);
 }

 int aeThreadOwnsLock()
@@ -72,15 +72,13 @@ extern "C" {
 #define AE_NOTUSED(V) ((void) V)

 struct aeEventLoop;
+
 /* Types and data structures */
 typedef void aeFileProc(struct aeEventLoop *eventLoop, int fd, void *clientData, int mask);
 typedef int aeTimeProc(struct aeEventLoop *eventLoop, long long id, void *clientData);
 typedef void aeEventFinalizerProc(struct aeEventLoop *eventLoop, void *clientData);
 typedef void aeBeforeSleepProc(struct aeEventLoop *eventLoop);
 typedef void aePostFunctionProc(void *pvArgs);
-//added to support async api IStorage
-struct StorageToken;
-typedef void aePostFunctionTokenProc(struct aeEventLoop *el, struct StorageToken *token);

 /* File event structure */
 typedef struct aeFileEvent {
@@ -136,8 +134,6 @@ int aePostFunction(aeEventLoop *eventLoop, aePostFunctionProc *proc, void *arg);
 #ifdef __cplusplus
 }   // EXTERN C
 int aePostFunction(aeEventLoop *eventLoop, std::function<void()> fn, bool fLock = true, bool fForceQueue = false);
-//added to support async api IStorage
-int aePostFunction(aeEventLoop *eventLoop, aePostFunctionTokenProc *proc, StorageToken *token);
 extern "C" {
 #endif
 void aeDeleteEventLoop(aeEventLoop *eventLoop);
@@ -92,8 +92,7 @@ void blockClient(client *c, int btype) {
    /* Master client should never be blocked unless pause or module */
    serverAssert(!(c->flags & CLIENT_MASTER &&
                   btype != BLOCKED_MODULE &&
-                   btype != BLOCKED_PAUSE &&
-                   btype != BLOCKED_STORAGE));
+                   btype != BLOCKED_PAUSE));

    c->flags |= CLIENT_BLOCKED;
    c->btype = btype;
@@ -208,8 +207,6 @@ void unblockClient(client *c) {
    } else if (c->btype == BLOCKED_PAUSE) {
        listDelNode(g_pserver->paused_clients,c->paused_list_node);
        c->paused_list_node = NULL;
-    } else if (c->btype == BLOCKED_STORAGE) {
-        serverTL->vecclientsProcess.push_back(c);
    } else {
        serverPanic("Unknown btype in unblockClient().");
    }
@@ -5991,9 +5991,7 @@ clusterNode *getNodeByQuery(client *c, struct redisCommand *cmd, robj **argv, in
                /* If it is not the first key, make sure it is exactly
                 * the same key as the first we saw. */
                if (!equalStringObjects(firstkey,thiskey)) {
-                    clusterNode* nThisKey = g_pserver->cluster->slots[thisslot];
-
-                    if ((slot != thisslot) && (nThisKey != n || migrating_slot || importing_slot || g_pserver->cluster->migrating_slots_to[slot] != nullptr || g_pserver->cluster->importing_slots_from[slot] != nullptr)) {
+                    if (slot != thisslot) {
                        /* Error: multiple keys from different slots. */
                        getKeysFreeResult(&result);
                        if (error_code)
@@ -361,7 +361,7 @@ bool initializeStorageProvider(const char **err)
            // Create The Storage Factory (if necessary)
            serverLog(LL_NOTICE, "Initializing FLASH storage provider (this may take a long time)");
            adjustOpenFilesLimit();
-            g_pserver->m_pstorageFactory = CreateRocksDBStorageFactory(g_sdsArgs, cserver.dbnum, cserver.storage_conf, cserver.storage_conf ? strlen(cserver.storage_conf) : 0, &g_pserver->asyncreadworkqueue, &g_pserver->asyncwriteworkqueue);
+            g_pserver->m_pstorageFactory = CreateRocksDBStorageFactory(g_sdsArgs, cserver.dbnum, cserver.storage_conf, cserver.storage_conf ? strlen(cserver.storage_conf) : 0);
 #else
            serverLog(LL_WARNING, "To use the flash storage provider please compile KeyDB with ENABLE_FLASH=yes");
            serverLog(LL_WARNING, "Exiting due to the use of an unsupported storage provider");
@@ -474,90 +474,6 @@ void initConfigValues() {
    }
 }

-struct in_addr maskFromBitsIPV4(int bits) {
-    struct in_addr mask;
-    memset(&mask, 0, sizeof(mask));
-    mask.s_addr = htonl(ULONG_MAX << (IPV4_BITS - bits));
-    return mask;
-}
-
-bool validateAndAddIPV4(sds string) {
-    struct in_addr ip, mask;
-    int subnetParts;
-    sds* subnetSplit = sdssplitlen(string, sdslen(string), "/", 1, &subnetParts);
-    TCleanup splitCleanup([&](){
-        sdsfreesplitres(subnetSplit, subnetParts);
-    });
-    if (subnetParts > 2) {
-        return false;
-    } else if (subnetParts == 2) {
-        char* endptr;
-        long bits = strtol(subnetSplit[1], &endptr, 10);
-        if (*endptr != '\0' || bits < 0 || bits > IPV4_BITS) {
-            return false;
-        }
-        mask = maskFromBitsIPV4(bits);
-    } else {
-        mask = maskFromBitsIPV4(IPV4_BITS);
-    }
-    if (subnetParts > 0) {
-        if (inet_pton(AF_INET, subnetSplit[0], &ip) == 0) {
-            return false;
-        }
-        g_pserver->overload_ignorelist.emplace(ip, mask);
-        return true;
-    } else {
-        return false;
-    }
-}
-
-struct in6_addr maskFromBitsIPV6(int bits) {
-    struct in6_addr mask;
-    memset(&mask, 0, sizeof(mask));
-    for (unsigned int i = 0; i < sizeof(struct in6_addr); i++) {
-        if (bits >= CHAR_BIT) {
-            mask.s6_addr[i] = UCHAR_MAX;
-            bits -= CHAR_BIT;
-        } else if (bits > 0) {
-            mask.s6_addr[i] = UCHAR_MAX << (CHAR_BIT - bits);
-            bits = 0;
-        } else {
-            break;
-        }
-    }
-    return mask;
-}
-
-bool validateAndAddIPV6(sds string) {
-    struct in6_addr ip, mask;
-    int subnetParts;
-    sds* subnetSplit = sdssplitlen(string, sdslen(string), "/", 1, &subnetParts);
-    TCleanup splitCleanup([&](){
-        sdsfreesplitres(subnetSplit, subnetParts);
-    });
-    if (subnetParts > 2) {
-        return false;
-    } else if (subnetParts == 2) {
-        char* endptr;
-        long bits = strtol(subnetSplit[1], &endptr, 10);
-        if (*endptr != '\0' || bits < 0 || bits > IPV6_BITS) {
-            return false;
-        }
-        mask = maskFromBitsIPV6(bits);
-    } else {
-        mask = maskFromBitsIPV6(IPV6_BITS);
-    }
-    if (subnetParts > 0) {
-        if (inet_pton(AF_INET6, subnetSplit[0], &ip) == 0) {
-            return false;
-        }
-        g_pserver->overload_ignorelist_ipv6.emplace(ip, mask);
-        return true;
-    } else {
-        return false;
-    }
-}
-
 void loadServerConfigFromString(char *config) {
    const char *err = NULL;
    int linenum = 0, totlines, i;
@@ -863,27 +779,6 @@ void loadServerConfigFromString(char *config) {
            }
            for (int i = 1; i < argc; i++)
                g_pserver->tls_auditlog_blocklist.emplace(argv[i], strlen(argv[i]));
-        } else if (!strcasecmp(argv[0], "tls-overload-ignorelist")) {
-            if (argc < 2) {
-                err = "must supply at least one element in the ignore list"; goto loaderr;
-            }
-            if (!g_pserver->tls_overload_ignorelist.empty()) {
-                err = "tls-overload-ignorelist may only be set once"; goto loaderr;
-            }
-            for (int i = 1; i < argc; i++)
-                g_pserver->tls_overload_ignorelist.emplace(argv[i], strlen(argv[i]));
-        } else if (!strcasecmp(argv[0], "overload-ignorelist")) {
-            if (argc < 2) {
-                err = "must supply at least one element in the ignore list"; goto loaderr;
-            }
-            if (!g_pserver->overload_ignorelist.empty() || !g_pserver->overload_ignorelist_ipv6.empty()) {
-                err = "overload-ignorelist may only be set once"; goto loaderr;
-            }
-            for (int i = 1; i < argc; i++) {
-                if (!validateAndAddIPV4(argv[i]) && !validateAndAddIPV6(argv[i])) {
-                    err = "overload-ignorelist must be a list of valid IP addresses or subnets"; goto loaderr;
-                }
-            }
        } else if (!strcasecmp(argv[0], "version-override") && argc == 2) {
            KEYDB_SET_VERSION = zstrdup(argv[1]);
            serverLog(LL_WARNING, "Warning version is overriden to: %s\n", KEYDB_SET_VERSION);
@@ -1013,8 +908,8 @@ void configSetCommand(client *c) {

    if (c->argc < 4 || c->argc > 4) {
        o = nullptr;
-        // Variadic set is only supported for tls-allowlist, tls-auditlog-blocklist, overload-ignorelist and tls-overload-ignorelist
-        if (strcasecmp(szFromObj(c->argv[2]), "tls-allowlist") && strcasecmp(szFromObj(c->argv[2]), "tls-auditlog-blocklist") && strcasecmp(szFromObj(c->argv[2]), "overload-ignorelist") && strcasecmp(szFromObj(c->argv[2]), "tls-overload-ignorelist") ) {
+        // Variadic set is only supported for tls-allowlist
+        if (strcasecmp(szFromObj(c->argv[2]), "tls-allowlist")) {
            addReplySubcommandSyntaxError(c);
            return;
        }
@@ -1189,28 +1084,6 @@ void configSetCommand(client *c) {
            robj *val = c->argv[i];
            g_pserver->tls_allowlist.emplace(szFromObj(val), sdslen(szFromObj(val)));
        }
-    } config_set_special_field("tls-auditlog-blocklist") {
-        g_pserver->tls_auditlog_blocklist.clear();
-        for (int i = 3; i < c->argc; ++i) {
-            robj *val = c->argv[i];
-            g_pserver->tls_auditlog_blocklist.emplace(szFromObj(val), sdslen(szFromObj(val)));
-        }
-    } config_set_special_field("tls-overload-ignorelist") {
-        g_pserver->tls_overload_ignorelist.clear();
-        for (int i = 3; i < c->argc; ++i) {
-            robj *val = c->argv[i];
-            g_pserver->tls_overload_ignorelist.emplace(szFromObj(val), sdslen(szFromObj(val)));
-        }
-    } config_set_special_field("overload-ignorelist") {
-        g_pserver->overload_ignorelist.clear();
-        g_pserver->overload_ignorelist_ipv6.clear();
-        for (int i = 3; i < c->argc; ++i) {
-            robj *val = c->argv[i];
-            if (!validateAndAddIPV4(szFromObj(val)) && !validateAndAddIPV6(szFromObj(val))) {
-                addReplyError(c, "overload-ignorelist must be a list of valid IP addresses or subnets");
-                return;
-            }
-        }
    /* Everything else is an error... */
    } config_set_else {
        addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s",
@@ -1422,37 +1295,6 @@ void configGetCommand(client *c) {
        }
        matches++;
    }
-    if (stringmatch(pattern, "tls-auditlog-blocklist", 1)) {
-        addReplyBulkCString(c,"tls-auditlog-blocklist");
-        addReplyArrayLen(c, (long)g_pserver->tls_auditlog_blocklist.size());
-        for (auto &elem : g_pserver->tls_auditlog_blocklist) {
-            addReplyBulkCBuffer(c, elem.get(), elem.size()); // addReplyBulkSds will free which we absolutely don't want
-        }
-        matches++;
-    }
-    if (stringmatch(pattern, "tls-overload-ignorelist", 1)) {
-        addReplyBulkCString(c,"tls-overload-ignorelist");
-        addReplyArrayLen(c, (long)g_pserver->tls_overload_ignorelist.size());
-        for (auto &elem : g_pserver->tls_overload_ignorelist) {
-            addReplyBulkCBuffer(c, elem.get(), elem.size()); // addReplyBulkSds will free which we absolutely don't want
-        }
-        matches++;
-    }
-    if (stringmatch(pattern, "overload-ignorelist", 1)) {
-        addReplyBulkCString(c,"overload-ignorelist");
-        addReplyArrayLen(c, (long)g_pserver->overload_ignorelist.size() + (long)g_pserver->overload_ignorelist_ipv6.size());
-        for (auto &elem : g_pserver->overload_ignorelist) {
-            sds elem_sds = elem.getString();
-            addReplyBulkCBuffer(c, elem_sds, sdslen(elem_sds));
-            sdsfree(elem_sds);
-        }
-        for (auto &elem : g_pserver->overload_ignorelist_ipv6) {
-            sds elem_sds = elem.getString();
-            addReplyBulkCBuffer(c, elem_sds, sdslen(elem_sds));
-            sdsfree(elem_sds);
-        }
-        matches++;
-    }

    setDeferredMapLen(c,replylen,matches);
 }
@@ -2136,56 +1978,6 @@ int rewriteConfig(char *path, int force_all) {
        rewriteConfigMarkAsProcessed(state, "tls-allowlist"); // ensure the line is removed if it existed
    }

-    if (!g_pserver->tls_overload_ignorelist.empty()) {
-        sds conf = sdsnew("tls-overload-ignorelist ");
-        for (auto &elem : g_pserver->tls_overload_ignorelist) {
-            conf = sdscatsds(conf, (sds)elem.get());
-            conf = sdscat(conf, " ");
-        }
-        // trim the trailing space
-        sdsrange(conf, 0, -1);
-        rewriteConfigRewriteLine(state,"tls-overload-ignorelist",conf,1 /*force*/);
-        // note: conf is owned by rewriteConfigRewriteLine - no need to free
-    } else {
-        rewriteConfigMarkAsProcessed(state, "tls-overload-ignorelist"); // ensure the line is removed if it existed
-    }
-
-    if (!g_pserver->tls_auditlog_blocklist.empty()) {
-        sds conf = sdsnew("tls-auditlog-blocklist ");
-        for (auto &elem : g_pserver->tls_auditlog_blocklist) {
-            conf = sdscatsds(conf, (sds)elem.get());
-            conf = sdscat(conf, " ");
-        }
-        // trim the trailing space
-        sdsrange(conf, 0, -1);
-        rewriteConfigRewriteLine(state,"tls-auditlog-blocklist",conf,1 /*force*/);
-        // note: conf is owned by rewriteConfigRewriteLine - no need to free
-    } else {
-        rewriteConfigMarkAsProcessed(state, "tls-auditlog-blocklist"); // ensure the line is removed if it existed
-    }
-
-    if (!g_pserver->overload_ignorelist.empty() || !g_pserver->overload_ignorelist_ipv6.empty()) {
-        sds conf = sdsnew("overload-ignorelist ");
-        for (auto &elem : g_pserver->overload_ignorelist) {
-            sds elem_sds = elem.getString();
-            conf = sdscatsds(conf, elem_sds);
-            sdsfree(elem_sds);
-            conf = sdscat(conf, " ");
-        }
-        for (auto &elem : g_pserver->overload_ignorelist_ipv6) {
-            sds elem_sds = elem.getString();
-            conf = sdscatsds(conf, elem_sds);
-            sdsfree(elem_sds);
-            conf = sdscat(conf, " ");
-        }
-        // trim the trailing space
-        sdsrange(conf, 0, -1);
-        rewriteConfigRewriteLine(state,"overload-ignorelist",conf,1 /*force*/);
-        // note: conf is owned by rewriteConfigRewriteLine - no need to free
-    } else {
-        rewriteConfigMarkAsProcessed(state, "overload-ignorelist"); // ensure the line is removed if it existed
-    }
-
    /* Rewrite Sentinel config if in Sentinel mode. */
    if (g_pserver->sentinel_mode) rewriteConfigSentinelOption(state);

@@ -2724,14 +2516,6 @@ static int isValidProcTitleTemplate(char *val, const char **err) {
    return 1;
 }

-static int isValidDisklessEnum(int e, const char **err) {
-    if (e == REPL_DISKLESS_LOAD_SWAPDB && g_pserver->m_pstorageFactory != nullptr) {
-        *err = "cannot use swapdb with a storage provider";
-        return 0;
-    }
-    return 1;
-}
-
 static int updateProcTitleTemplate(char *val, char *prev, const char **err) {
    UNUSED(val);
    UNUSED(prev);
@@ -3099,7 +2883,7 @@ standardConfig configs[] = {
    /* Enum Configs */
    createEnumConfig("supervised", NULL, IMMUTABLE_CONFIG, supervised_mode_enum, cserver.supervised_mode, SUPERVISED_NONE, NULL, NULL),
    createEnumConfig("syslog-facility", NULL, IMMUTABLE_CONFIG, syslog_facility_enum, g_pserver->syslog_facility, LOG_LOCAL0, NULL, NULL),
-    createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, g_pserver->repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED, isValidDisklessEnum, NULL),
+    createEnumConfig("repl-diskless-load", NULL, MODIFIABLE_CONFIG, repl_diskless_load_enum, g_pserver->repl_diskless_load, REPL_DISKLESS_LOAD_DISABLED, NULL, NULL),
    createEnumConfig("loglevel", NULL, MODIFIABLE_CONFIG, loglevel_enum, cserver.verbosity, LL_NOTICE, NULL, NULL),
    createEnumConfig("maxmemory-policy", NULL, MODIFIABLE_CONFIG, maxmemory_policy_enum, g_pserver->maxmemory_policy, MAXMEMORY_NO_EVICTION, NULL, NULL),
    createEnumConfig("appendfsync", NULL, MODIFIABLE_CONFIG, aof_fsync_enum, g_pserver->aof_fsync, AOF_FSYNC_EVERYSEC, NULL, NULL),
@@ -3162,11 +2946,10 @@ standardConfig configs[] = {
    createLongLongConfig("latency-monitor-threshold", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->latency_monitor_threshold, 0, INTEGER_CONFIG, NULL, NULL),
    createLongLongConfig("proto-max-bulk-len", NULL, MODIFIABLE_CONFIG, 1024*1024, LLONG_MAX, g_pserver->proto_max_bulk_len, 512ll*1024*1024, MEMORY_CONFIG, NULL, NULL), /* Bulk request max size */
    createLongLongConfig("stream-node-max-entries", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->stream_node_max_entries, 100, INTEGER_CONFIG, NULL, NULL),
-    createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, g_pserver->repl_backlog_config_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */
+    createLongLongConfig("repl-backlog-size", NULL, MODIFIABLE_CONFIG, 1, LLONG_MAX, g_pserver->repl_backlog_size, 1024*1024, MEMORY_CONFIG, NULL, updateReplBacklogSize), /* Default: 1mb */
    createLongLongConfig("repl-backlog-disk-reserve", NULL, IMMUTABLE_CONFIG, 0, LLONG_MAX, cserver.repl_backlog_disk_size, 0, MEMORY_CONFIG, NULL, NULL),
    createLongLongConfig("max-snapshot-slip", NULL, MODIFIABLE_CONFIG, 0, 5000, g_pserver->snapshot_slip, 400, 0, NULL, NULL),
    createLongLongConfig("max-rand-count", NULL, MODIFIABLE_CONFIG, 0, LONG_MAX/2, g_pserver->rand_total_threshold, LONG_MAX/2, 0, NULL, NULL),
-    createLongLongConfig("repl-backlog-max-writes-per-event", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->repl_backlog_max_writes_per_event, 8ll*1024*1024, MEMORY_CONFIG, NULL, NULL),

    /* Unsigned Long Long configs */
    createULongLongConfig("maxmemory", NULL, MODIFIABLE_CONFIG, 0, LLONG_MAX, g_pserver->maxmemory, 0, MEMORY_CONFIG, NULL, updateMaxmemory),
@@ -3198,10 +2981,7 @@ standardConfig configs[] = {
    createSizeTConfig("semi-ordered-set-bucket-size", NULL, MODIFIABLE_CONFIG, 0, 1024, g_semiOrderedSetTargetBucketSize, 0, INTEGER_CONFIG, NULL, NULL),
    createSDSConfig("availability-zone", NULL, MODIFIABLE_CONFIG, 0, g_pserver->sdsAvailabilityZone, "", NULL, NULL),
    createIntConfig("overload-protect-percent", NULL, MODIFIABLE_CONFIG, 0, 200, g_pserver->overload_protect_threshold, 0, INTEGER_CONFIG, NULL, NULL),
-    createIntConfig("overload-protect-tenacity", NULL, MODIFIABLE_CONFIG, 0, 100, g_pserver->overload_protect_tenacity, 10, INTEGER_CONFIG, NULL, NULL),
    createIntConfig("force-eviction-percent", NULL, MODIFIABLE_CONFIG, 0, 100, g_pserver->force_eviction_percent, 0, INTEGER_CONFIG, NULL, NULL),
-    createBoolConfig("enable-async-rehash", NULL, MODIFIABLE_CONFIG, g_pserver->enable_async_rehash, 1, NULL, NULL),
-    createBoolConfig("enable-keydb-fastsync", NULL, MODIFIABLE_CONFIG, g_pserver->fEnableFastSync, 1, NULL, NULL),

 #ifdef USE_OPENSSL
    createIntConfig("tls-port", NULL, MODIFIABLE_CONFIG, 0, 65535, g_pserver->tls_port, 0, INTEGER_CONFIG, NULL, updateTLSPort), /* TCP port. */
@@ -52,7 +52,6 @@ typedef enum {
 #define CONN_FLAG_READ_THREADSAFE        (1<<2)
 #define CONN_FLAG_WRITE_THREADSAFE       (1<<3)
 #define CONN_FLAG_AUDIT_LOGGING_REQUIRED (1<<4)
-#define CONN_FLAG_IGNORE_OVERLOAD (1<<5)

 #define CONN_TYPE_SOCKET            1
 #define CONN_TYPE_TLS               2
@@ -56,6 +56,7 @@ int expireIfNeeded(redisDb *db, robj *key, robj *o);
 void slotToKeyUpdateKeyCore(const char *key, size_t keylen, int add);

 std::unique_ptr<expireEntry> deserializeExpire(const char *str, size_t cch, size_t *poffset);
+sds serializeStoredObjectAndExpire(robj_roptr o);

 dictType dictChangeDescType {
    dictSdsHash,                /* hash function */
@@ -2528,12 +2529,6 @@ void slotToKeyFlush(int async) {
 * decrement the reference count to release the keys names. */
 unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count) {
    if (g_pserver->m_pstorageFactory != nullptr) {
-        // We must commit so the storage engine agrees on the number of items in the hash slot
-        if (g_pserver->db[0]->FTrackingChanges()) {
-            if (g_pserver->db[0]->processChanges(false))
-                g_pserver->db[0]->commitChanges();
-            g_pserver->db[0]->trackChanges(false);
-        }
        int j = 0;
        g_pserver->db[0]->getStorageCache()->enumerate_hashslot([&](const char *key, size_t cchKey, const void *, size_t )->bool {
            keys[j++] = createStringObject(key, cchKey);
@@ -2563,12 +2558,6 @@ unsigned int getKeysInSlot(unsigned int hashslot, robj **keys, unsigned int coun
 unsigned int delKeysInSlot(unsigned int hashslot) {
    serverAssert(GlobalLocksAcquired());
    if (g_pserver->m_pstorageFactory != nullptr) {
-        // We must commit so the storage engine agrees on the number of items in the hash slot
-        if (g_pserver->db[0]->FTrackingChanges()) {
-            if (g_pserver->db[0]->processChanges(false))
-                g_pserver->db[0]->commitChanges();
-            g_pserver->db[0]->trackChanges(false);
-        }
        int j = 0;
        g_pserver->db[0]->getStorageCache()->enumerate_hashslot([&](const char *key, size_t cchKey, const void *, size_t )->bool {
            robj *keyobj = createStringObject(key, cchKey);
@@ -2714,6 +2703,11 @@ void redisDbPersistentData::prepOverwriteForSnapshot(char *key)
        auto itr = m_pdbSnapshot->find_cached_threadsafe(key);
        if (itr.key() != nullptr)
        {
+            if (itr.val()->FExpires()) {
+                // Note: I'm sure we could handle this, but its too risky at the moment.
+                //  There are known bugs doing this with expires
+                return;
+            }
            sds keyNew = sdsdupshared(itr.key());
            if (dictAdd(m_pdictTombstone, keyNew, (void*)dictHashKey(m_pdict, key)) != DICT_OK)
                sdsfree(keyNew);
@@ -2751,13 +2745,6 @@ void redisDbPersistentData::clear(void(callback)(void*))
    if (m_spstorage != nullptr)
        m_spstorage->clear(callback);
    dictEmpty(m_pdictTombstone,callback);
-
-    // To avoid issues with async rehash we completly free the old dict and create a fresh one
-    dictRelease(m_pdict);
-    dictRelease(m_pdictTombstone);
-    m_pdict = dictCreate(&dbDictType, this);
-    m_pdictTombstone = dictCreate(&dbTombstoneDictType, this);
-
    m_pdbSnapshot = nullptr;
    m_numexpires = 0;
 }
@@ -2893,11 +2880,10 @@ LNotFound:
            {
                dictAdd(m_pdict, sdsNewKey, o);
                
-                o->SetFExpires(spexpire != nullptr);
                if (spexpire != nullptr) {
                    o->expire = std::move(*spexpire);
-                    ++m_numexpires;
                }
+                o->SetFExpires(spexpire != nullptr);
                g_pserver->stat_storage_provider_read_hits++;
            } else {
                sdsfree(sdsNewKey);
@@ -3001,7 +2987,6 @@ void redisDbPersistentData::processChangesAsync(std::atomic<int> &pendingJobs)
    dict *dictNew = dictCreate(&dbDictType, nullptr);
    std::swap(dictNew, m_pdict);
    m_cnewKeysPending = 0;
-    m_numexpires = 0;
    g_pserver->asyncworkqueue->AddWorkFunction([dictNew, this, &pendingJobs]{
        dictIterator *di = dictGetIterator(dictNew);
        dictEntry *de;
@@ -3057,18 +3042,7 @@ void redisDbPersistentData::commitChanges(const redisDbPersistentDataSnapshot **
        m_pdbSnapshotStorageFlush = nullptr;
    }
    if (m_spstorage != nullptr)
-    {
-#if 1
        m_spstorage->endWriteBatch();
-#else
-        auto *tok = m_spstorage->begin_endWriteBatch(serverTL->el, storageLoadCallback);
-        if (tok != nullptr)
-        {
-            tok->db = this;
-            tok->type = StorageToken::TokenType::BatchWrite;
-        }
-#endif
-    }
 }

 redisDbPersistentData::~redisDbPersistentData()
@@ -3140,14 +3114,7 @@ bool redisDbPersistentData::removeCachedValue(const char *key, dictEntry **ppde)
    // since we write ASAP the database already has a valid copy so safe to delete
    if (ppde != nullptr) {
        *ppde = dictUnlink(m_pdict, key);
-        robj *o = (robj*)dictGetVal(*ppde);
-        if (o->FExpires())
-            --m_numexpires;
    } else {
-        dictEntry *deT = dictFind(m_pdict, key);
-        robj *o = (robj*)dictGetVal(deT);
-        if (o->FExpires())
-            --m_numexpires;
        dictDelete(m_pdict, key);
    }

@@ -3167,11 +3134,8 @@ void redisDbPersistentData::trackChanges(bool fBulk, size_t sizeHint)
    if (fBulk)
        m_fAllChanged.fetch_add(1, std::memory_order_acq_rel);

-    if (sizeHint > 0) {
-        aeAcquireLock();
+    if (sizeHint > 0)
        dictExpand(m_dictChanged, sizeHint, false);
-        aeReleaseLock();
-    }
 }

 void redisDbPersistentData::removeAllCachedValues()
@@ -3194,7 +3158,6 @@ void redisDbPersistentData::removeAllCachedValues()
    } else {
        dictEmpty(m_pdict, nullptr);
    }
-    m_numexpires = 0;
 }

 void redisDbPersistentData::disableKeyCache()
@@ -3346,124 +3309,90 @@ void redisDbPersistentData::prefetchKeysAsync(client *c, parsed_command &command
                }
            }
        }
-#else
-        UNUSED(c);
-        UNUSED(command);
 #endif
        return;
    }
-}

-void redisDbPersistentData::prefetchKeysFlash(const std::unordered_set<client*> &setc)
-{
-    serverAssert(GlobalLocksAcquired());
-    std::vector<sds> veckeys;
-    std::unordered_set<client*> setcBlocked;
+    AeLocker lock;

-    for (client *c : setc) {
-        for (auto &command : c->vecqueuedcmd) {
-            getKeysResult result = GETKEYS_RESULT_INIT;
-            if (command.argc == 0)  // parse can do this it will be handled by processClient
-                break;
-            auto cmd = lookupCommand(szFromObj(command.argv[0]));
-            if (cmd == nullptr)
-                break; // Bad command? It's not for us to judge, just bail
-            
-            if (command.argc < std::abs(cmd->arity))
-                break; // Invalid number of args
-            
-            int numkeys = getKeysFromCommand(cmd, command.argv, command.argc, &result);
-            bool fQueued = false;
-            for (int ikey = 0; ikey < numkeys; ++ikey)
+    std::vector<robj*> veckeys;
+    lock.arm(c);
+    getKeysResult result = GETKEYS_RESULT_INIT;
+    auto cmd = lookupCommand(szFromObj(command.argv[0]));
+    if (cmd == nullptr)
+        return; // Bad command? It's not for us to judge, just bail
+    
+    if (command.argc < std::abs(cmd->arity))
+        return; // Invalid number of args
+    
+    int numkeys = getKeysFromCommand(cmd, command.argv, command.argc, &result);
+    for (int ikey = 0; ikey < numkeys; ++ikey)
+    {
+        robj *objKey = command.argv[result.keys[ikey]];
+        if (this->find_cached_threadsafe(szFromObj(objKey)) == nullptr)
+            veckeys.push_back(objKey);
+    }
+    lock.disarm();
+
+    getKeysFreeResult(&result);
+
+    std::vector<std::tuple<sds, robj*, std::unique_ptr<expireEntry>>> vecInserts;
+    for (robj *objKey : veckeys)
+    {
+        sds sharedKey = sdsdupshared((sds)szFromObj(objKey));
+        std::unique_ptr<expireEntry> spexpire;
+        robj *o = nullptr;
+        m_spstorage->retrieve((sds)szFromObj(objKey), [&](const char *, size_t, const void *data, size_t cb){
+                size_t offset = 0;
+                spexpire = deserializeExpire((const char*)data, cb, &offset);    
+                o = deserializeStoredObject(reinterpret_cast<const char*>(data) + offset, cb - offset);
+                serverAssert(o != nullptr);
+        });
+
+        if (o != nullptr) {
+            vecInserts.emplace_back(sharedKey, o, std::move(spexpire));
+        } else if (sharedKey != nullptr) {
+            sdsfree(sharedKey);
+        }
+    }
+
+    if (!vecInserts.empty()) {
+        lock.arm(c);
+        for (auto &tuple : vecInserts)
+        {
+            sds sharedKey = std::get<0>(tuple);
+            robj *o = std::get<1>(tuple);
+            std::unique_ptr<expireEntry> spexpire = std::move(std::get<2>(tuple));
+
+            if (o != nullptr)
            {
-                robj *objKey = command.argv[result.keys[ikey]];
-                if (this->find_cached_threadsafe(szFromObj(objKey)) == nullptr) {
-                    veckeys.push_back(szFromObj(objKey));
-                    fQueued = true;
+                if (this->find_cached_threadsafe(sharedKey) != nullptr)
+                {
+                    // While unlocked this was already ensured
+                    decrRefCount(o);
+                    sdsfree(sharedKey);
+                }
+                else
+                {
+                    if (spexpire != nullptr) {
+                        if (spexpire->when() < mstime()) {
+                            break;
+                        }
+                    }
+                    dictAdd(m_pdict, sharedKey, o);
+                    if (spexpire != nullptr)
+                        o->expire = std::move(*spexpire);
+                    o->SetFExpires(spexpire != nullptr);
                }
            }
-
-            if (fQueued) {
-                setcBlocked.insert(c);
-            }
-            getKeysFreeResult(&result);
-        }
-    }
-
-    if (veckeys.empty())
-        return;
-
-    StorageToken *tok = m_spstorage->begin_retrieve(serverTL->el, storageLoadCallback, veckeys.data(), veckeys.size());
-    if (tok != nullptr) {
-        for (client *c : setcBlocked) {
-            if (!(c->flags & CLIENT_BLOCKED)) {
-                blockClient(c, BLOCKED_STORAGE);
+            else
+            {
+                if (sharedKey != nullptr)
+                    sdsfree(sharedKey); // BUG but don't bother crashing
            }
        }
-        tok->setc = std::move(setcBlocked);
-        tok->type = StorageToken::TokenType::SingleRead;
-        tok->db = this;
+        lock.disarm();
    }
+
    return;
 }
-
-/*static*/ void redisDbPersistentData::storageLoadCallback(aeEventLoop *, StorageToken *tok) {
-    serverTL->setStorageTokensProcess.insert(tok);
-}
-
-void redisDbPersistentData::processStorageToken(StorageToken *tok) {
-    auto setc = std::move(tok->setc);
-    switch (tok->type)
-    {
-
-    case StorageToken::TokenType::SingleRead:
-    {
-        tok->db->m_spstorage->complete_retrieve(tok, [&](const char *szKey, size_t cbKey, const void *data, size_t cb) {
-            auto *db = tok->db;
-            size_t offset = 0;
-            sds key = sdsnewlen(szKey, -((ssize_t)cbKey));
-            auto spexpire = deserializeExpire((const char*)data, cb, &offset);    
-            robj *o = deserializeStoredObject(reinterpret_cast<const char*>(data) + offset, cb - offset);
-            serverAssert(o != nullptr);
-
-            if (db->find_cached_threadsafe(key) != nullptr) {
-            LUnneeded:
-                // While unlocked this was already ensured
-                decrRefCount(o);
-                sdsfree(key);
-            } else {
-                if (spexpire != nullptr) {
-                    if (spexpire->when() < mstime()) {
-                        goto LUnneeded;
-                    }
-                }
-                dictAdd(db->m_pdict, key, o);
-                o->SetFExpires(spexpire != nullptr);
-                if (spexpire != nullptr) {
-                    o->expire = std::move(*spexpire);
-                }
-            } 
-        });
-        break;
-    }
-    case StorageToken::TokenType::BatchWrite:
-    {
-        tok->db->m_spstorage->complete_endWriteBatch(tok);
-        break;
-    }
-    default:
-        serverAssert((tok->type == StorageToken::TokenType::SingleRead) || (tok->type == StorageToken::TokenType::BatchWrite));
-        break;
-    } //switch end
-
-    tok = nullptr; // Invalid past this point
-
-    for (client *c : setc)
-    {
-        std::unique_lock<fastlock> ul(c->lock);
-        if (c->flags & CLIENT_BLOCKED)
-            unblockClient(c);
-        
-        serverTL->vecclientsProcess.push_back(c);
-    }
-}
@@ -79,7 +79,6 @@ void printCrashReport(void);
 void bugReportEnd(int killViaSignal, int sig);
 void logStackTrace(void *eip, int uplevel);
 void getTempFileName(char tmpfile[], int tmpfileNum);
-bool initializeStorageProvider(const char **err);

 /* ================================= Debugging ============================== */

@@ -309,7 +308,7 @@ void computeDatasetDigest(unsigned char *final) {
        mixDigest(final,&aux,sizeof(aux));

        /* Iterate this DB writing every entry */
-        db->iterate_threadsafe([final](const char *key, robj_roptr o)->bool {
+        db->iterate_threadsafe([final, db](const char *key, robj_roptr o)->bool {
            unsigned char digest[20];
            robj *keyobj;

@@ -536,75 +535,30 @@ NULL
                return;
            }
        }
-        
-        if (g_pserver->m_pstorageFactory) {
-            protectClient(c);

-            for (int idb = 0; idb < cserver.dbnum; ++idb) {
-                if (g_pserver->db[idb]->processChanges(false))
-                    g_pserver->db[idb]->commitChanges();
-                g_pserver->db[idb]->storageProviderDelete();
-            }
-            delete g_pserver->metadataDb;
-
-            if (flush) emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
-
-            delete g_pserver->m_pstorageFactory;
-
-            const char *err;
-            if (!initializeStorageProvider(&err))
-            {
-                serverLog(LL_WARNING, "Failed to initialize storage provider: %s",err);
-                exit(EXIT_FAILURE);
-            }
-
-            g_pserver->metadataDb = g_pserver->m_pstorageFactory->createMetadataDb();
-            for (int idb = 0; idb < cserver.dbnum; ++idb)
-            {
-                int dbid = idb;
-                std::string dbid_key = "db-" + std::to_string(idb);
-                g_pserver->metadataDb->retrieve(dbid_key.c_str(), dbid_key.length(), [&](const char *, size_t, const void *data, size_t){
-                    dbid = *(int*)data;
-                });
-                delete g_pserver->db[idb];
-                g_pserver->db[idb] = new (MALLOC_LOCAL) redisDb();
-                g_pserver->db[idb]->initialize(dbid);
-            }
-
-            moduleFireServerEvent(REDISMODULE_EVENT_LOADING, REDISMODULE_SUBEVENT_LOADING_FLASH_START, NULL);
-            for (int idb = 0; idb < cserver.dbnum; ++idb)
-            {
-                g_pserver->db[idb]->storageProviderInitialize();
-                g_pserver->db[idb]->trackChanges(false);
-            }
-            moduleFireServerEvent(REDISMODULE_EVENT_LOADING, REDISMODULE_SUBEVENT_LOADING_ENDED, NULL);
-            
-            unprotectClient(c);
-        } else {
-            /* The default behavior is to save the RDB file before loading
-            * it back. */
-            if (save) {
-                rdbSaveInfo rsi, *rsiptr;
-                rsiptr = rdbPopulateSaveInfo(&rsi);
-                if (rdbSave(nullptr, rsiptr) != C_OK) {
-                    addReply(c,shared.err);
-                    return;
-                }
-            }
-
-            /* The default behavior is to remove the current dataset from
-            * memory before loading the RDB file, however when MERGE is
-            * used together with NOFLUSH, we are able to merge two datasets. */
-            if (flush) emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
-
-            protectClient(c);
-            int ret = rdbLoadFile(g_pserver->rdb_filename,NULL,flags);
-            unprotectClient(c);
-            if (ret != C_OK) {
-                addReplyError(c,"Error trying to load the RDB dump");
+        /* The default behavior is to save the RDB file before loading
+         * it back. */
+        if (save) {
+            rdbSaveInfo rsi, *rsiptr;
+            rsiptr = rdbPopulateSaveInfo(&rsi);
+            if (rdbSave(nullptr, rsiptr) != C_OK) {
+                addReply(c,shared.err);
                return;
            }
        }
+
+        /* The default behavior is to remove the current dataset from
+         * memory before loading the RDB file, however when MERGE is
+         * used together with NOFLUSH, we are able to merge two datasets. */
+        if (flush) emptyDb(-1,EMPTYDB_NO_FLAGS,NULL);
+
+        protectClient(c);
+        int ret = rdbLoadFile(g_pserver->rdb_filename,NULL,flags);
+        unprotectClient(c);
+        if (ret != C_OK) {
+            addReplyError(c,"Error trying to load the RDB dump");
+            return;
+        }
        serverLog(LL_WARNING,"DB reloaded by DEBUG RELOAD");
        addReply(c,shared.ok);
    } else if (!strcasecmp(szFromObj(c->argv[1]),"loadaof")) {
@@ -518,13 +518,11 @@ void pexpireMemberAtCommand(client *c)
            } while (sampled == 0 ||
                    (expired*100/sampled) > config_cycle_acceptable_stale);
        } else {
-            long prev_expired;
-            long long now = mstime();
+            long long now;
            size_t tried = 0;
            std::vector<std::string> keys;
            do {
-                prev_expired = total_expired;
-                keys = db->getStorageCache()->getExpirationCandidates(ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP);
+                keys = db->getStorageCache()->getExpirationCandidates(ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP, now);
                for (std::string key : keys) {
                    robj* keyobj = createStringObject(key.c_str(), key.size());
                    db->find(szFromObj(keyobj));
@@ -535,7 +533,7 @@ void pexpireMemberAtCommand(client *c)
                }
                total_sampled += keys.size();
                elapsed = ustime()-start;
-            } while (keys.size() > 0 && (elapsed < timelimit) && (total_expired - prev_expired) > 0);
+            } while (keys.size() > 0 && (elapsed < timelimit));

            if (ustime()-start > timelimit) {
                timelimit_exit = 1;
@@ -160,8 +160,6 @@ public:

    expireEntry &operator=(expireEntry &&e)
    {
-        if (FFat())
-            delete pfatentry();
        s = e.s;
        e.s.m_whenAndPtrUnion = 0;
        e.s.fFat = false;
@@ -169,8 +167,6 @@ public:
    }

    expireEntry &operator=(expireEntry &e) {
-        if (FFat())
-            delete pfatentry();
        if (e.FFat()) {
            s.m_whenAndPtrUnion = reinterpret_cast<long long>(new expireEntryFat(*e.pfatentry()));
            s.fFat = true;
@@ -690,9 +690,8 @@ void moduleHandlePropagationAfterCommandCallback(RedisModuleCtx *ctx) {
 }

 /* Free the context after the user function was called. */
-void moduleFreeContext(RedisModuleCtx *ctx, bool propogate) {
-    if (propogate)
-        moduleHandlePropagationAfterCommandCallback(ctx);
+void moduleFreeContext(RedisModuleCtx *ctx) {
+    moduleHandlePropagationAfterCommandCallback(ctx);
    autoMemoryCollect(ctx);
    poolAllocRelease(ctx);
    if (ctx->postponed_arrays) {
@@ -7058,7 +7057,7 @@ RedisModuleString *RM_DictPrev(RedisModuleCtx *ctx, RedisModuleDictIter *di, voi
 /* Compare the element currently pointed by the iterator to the specified
 * element given by key/keylen, according to the operator 'op' (the set of
 * valid operators are the same valid for RedisModule_DictIteratorStart).
- * If the comparison is successful the command returns REDISMODULE_OK
+ * If the comparision is successful the command returns REDISMODULE_OK
 * otherwise REDISMODULE_ERR is returned.
 *
 * This is useful when we want to just emit a lexicographical range, so
@@ -4,7 +4,7 @@ OBJECT_FILES := modmain.o
 MODSNAP_CXX_FLAGS := -std=gnu++14

 %.o: %.cpp
-	$(CXX) -o $@ -c $< $(MODULE_FLAGS) -I../../../deps/cpp-statsd-client/include $(MODSNAP_CXX_FLAGS) -g
+	$(CXX) -o $@ -c $< $(MODULE_FLAGS) -I../../../deps/cpp-statsd-client/include $(MODSNAP_CXX_FLAGS)

 modstatsd.so: $(OBJECT_FILES)
 	$(CXX) -shared $(OBJECT_FILES) -o modstatsd.so
@@ -67,6 +67,8 @@ public:

 /* constants */
 static time_t c_infoUpdateSeconds = 10;
+// the current Redis Cluster setup we configure replication factor as 2, each non-empty master node should have 2 replicas, given that there are 3 zones in each regions
+static const int EXPECTED_NUMBER_OF_REPLICAS = 2;

 StatsdClientWrapper *g_stats = nullptr;
 std::string m_strPrefix { "keydb" };
@@ -74,7 +76,7 @@ std::string m_strPrefix { "keydb" };
 const std::regex g_replica_or_db_info_regex { "^(slave|db)(\\d+)" };
 const char *g_string_counter_separator = "__";
 const uint64_t g_stats_buffer_size_bytes = 1600;
-std::string nodeName;
+utsname sysName;
 int unameResult;

 enum class StatsD_Type {
@@ -92,7 +94,6 @@ struct StatsRecord {

    /* Dynamic Values */
    long long prevVal = 0;
-    long long currVal = 0;
 };

 std::unordered_map<std::string, StatsRecord> g_mapInfoFields = {
@@ -100,8 +101,6 @@ std::unordered_map<std::string, StatsRecord> g_mapInfoFields = {
    { "used_memory", { StatsD_Type::STATSD_GAUGE_BYTES, false /* prefixOnly */}},
    { "used_memory_rss", { StatsD_Type::STATSD_GAUGE_BYTES }},
    { "maxmemory", { StatsD_Type::STATSD_GAUGE_BYTES, false /* prefixOnly */}},
-    { "maxstorage", { StatsD_Type::STATSD_GAUGE_BYTES, false /* prefixOnly */}},
-    { "storage_used", { StatsD_Type::STATSD_GAUGE_BYTES, false /* prefixOnly */}},
    { "used_memory_dataset_perc", { StatsD_Type::STATSD_GAUGE_FLOAT }},
    { "avg_lock_contention", { StatsD_Type::STATSD_GAUGE_LONGLONG }},
    { "repl_backlog_size", { StatsD_Type::STATSD_GAUGE_BYTES }},
@@ -254,7 +253,6 @@ std::unordered_map<std::string, StatsRecord> g_mapInfoFields = {
    { "cluster_size", { StatsD_Type::STATSD_GAUGE_LONGLONG }},
    { "storage_flash_available_bytes", { StatsD_Type::STATSD_GAUGE_BYTES }},
    { "storage_flash_total_bytes", { StatsD_Type::STATSD_GAUGE_BYTES }},
-    { "last_overload_cpu_reading", { StatsD_Type::STATSD_GAUGE_FLOAT }},
 };

 /* Globals */
@@ -299,8 +297,6 @@ void handleStatItem(struct RedisModuleCtx *ctx, std::string name, StatsRecord &r
    switch (record.type) {
        case StatsD_Type::STATSD_GAUGE_LONGLONG: {
            long long val = strtoll(pchValue, nullptr, 10);
-            record.prevVal = record.currVal;
-            record.currVal = val;
            g_stats->gauge(name, val, record.prefixOnly);
            RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_DEBUG, "Emitting metric \"%s\": %lld", name.c_str(), val);
            break;
@@ -322,10 +318,9 @@ void handleStatItem(struct RedisModuleCtx *ctx, std::string name, StatsRecord &r

        case StatsD_Type::STATSD_DELTA: {
            long long val = strtoll(pchValue, nullptr, 10);
-            record.prevVal = record.currVal;
-            record.currVal = val;
            g_stats->count(name, val - record.prevVal, record.prefixOnly);
            RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_DEBUG, "Emitting metric count for \"%s\": %lld", name.c_str() , val - record.prevVal);
+            record.prevVal = val;
            break;
        }

@@ -436,20 +431,6 @@ void handle_info_response(struct RedisModuleCtx *ctx, const char *szReply, size_
    }

    #undef SAFETY_CHECK_POINTER
-
-    // Emit keyspace hit rate metric which is computed from info values
-    auto hit = g_mapInfoFields.find("keyspace_hits");
-    auto miss = g_mapInfoFields.find("keyspace_misses");
-    if (hit != g_mapInfoFields.end() && miss != g_mapInfoFields.end()) {
-        long long hitValDelta = hit->second.currVal - hit->second.prevVal;
-        long long missValDelta = miss->second.currVal - miss->second.prevVal;
-        long long total = hitValDelta + missValDelta;
-        if (total > 0) {
-            double hitRate = (double)hitValDelta / total * 1000000;
-            g_stats->gauge("keyspace_hit_rate_per_million", hitRate, false /* prefixOnly */);
-            RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_DEBUG, "Emitting metric \"keyspace_hit_rate_per_million\": %f", hitRate);
-        }
-    }
 }

 void handle_cluster_nodes_response(struct RedisModuleCtx *ctx, const char *szReply, size_t len) {
@@ -457,8 +438,6 @@ void handle_cluster_nodes_response(struct RedisModuleCtx *ctx, const char *szRep
    const char *pchLineStart = szReply;
    long long primaries = 0;
    long long replicas = 0;
-    long long handshakes = 0;
-    long long errors = 0;
    while (SAFETY_CHECK_POINTER(pchLineStart) && *pchLineStart != '\0') {
        // Loop Each Line
        const char *pchLineEnd = pchLineStart;
@@ -471,10 +450,6 @@ void handle_cluster_nodes_response(struct RedisModuleCtx *ctx, const char *szRep
            ++primaries;
        } else if (std::string::npos != line.find("slave")) {
            ++replicas;
-        } else if (std::string::npos != line.find("handshake")) {
-            ++handshakes;
-        } else if (std::string::npos != line.find("fail?") || std::string::npos != line.find("fail") || std::string::npos != line.find("noaddr") || std::string::npos != line.find("nofailover") || std::string::npos != line.find("noflags") ) {
-            ++errors;
        } else {
            RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_WARNING, "Unexpected NODE format returned by \"CLUSTER NODES\" command: \"%s\"", line.c_str());
        }
@@ -506,10 +481,6 @@ void handle_cluster_nodes_response(struct RedisModuleCtx *ctx, const char *szRep
    RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_DEBUG, "Emitting metric \"primaries\": %llu", primaries);
    g_stats->gauge("replicas", replicas);
    RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_DEBUG, "Emitting metric \"replicas\": %llu", replicas);
-    g_stats->gauge("handshakes", handshakes);
-    RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_DEBUG, "Emitting metric \"handshakes\": %llu", handshakes);
-    g_stats->gauge("cluster_nodes_error_nodes", errors);
-    RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_DEBUG, "Emitting metric \"cluster_nodes_error_nodes\": %llu", errors);
    #undef SAFETY_CHECK_POINTER
 }

@@ -595,8 +566,10 @@ void emit_metrics_for_insufficient_replicas(struct RedisModuleCtx *ctx, long lon
    // check if the current node is a primary
    if (strncmp(role, "master", len) == 0) {
        RedisModuleCallReply *replicasReply = RedisModule_CallReplyArrayElement(reply, 2);
-        size_t numberOfActiveReplicas = RedisModule_CallReplyLength(replicasReply);
-        g_stats->gauge("numberOfActiveReplicas", numberOfActiveReplicas);
+        // check if there are less than 2 connected replicas
+        if (RedisModule_CallReplyLength(replicasReply) < EXPECTED_NUMBER_OF_REPLICAS) {
+            g_stats->increment("lessThanExpectedReplicas_error", 1);
+        }
    }
    RedisModule_FreeCallReply(reply);
 }
@@ -609,7 +582,7 @@ void event_cron_handler(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64
        size_t startTime = ustime();

 #ifdef __linux__
-	/* Log CPU Usage */
+        /* Log CPU Usage */
 	static long long s_mscpuLast = 0;
 	struct rusage self_ru;
 	getrusage(RUSAGE_SELF, &self_ru);
@@ -618,7 +591,7 @@ void event_cron_handler(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64
 			+ (self_ru.ru_stime.tv_sec * 1000) + (self_ru.ru_stime.tv_usec / 1000);


-	g_stats->gauge("cpu_load_perc", ((double)(mscpuCur - s_mscpuLast) / ((curTime - lastTime)*1000))*100, false /* prefixOnly */);   
+        g_stats->gauge("cpu_load_perc", (double)(mscpuCur - s_mscpuLast) / ((curTime - lastTime)*1000), true /* prefixOnly */);   
 	s_mscpuLast = mscpuCur;
 #endif

@@ -627,7 +600,7 @@ void event_cron_handler(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64

        /* node name */
        if (unameResult == 0) {
-            g_stats->increment("node_name" + std::string(g_string_counter_separator) + nodeName);
+            g_stats->increment("node_name" + std::string(g_string_counter_separator) + sysName.nodename);
        }

        /* Log INFO Fields */
@@ -680,7 +653,7 @@ void event_cron_handler(struct RedisModuleCtx *ctx, RedisModuleEvent eid, uint64
        reply = RedisModule_Call(ctx, "dbsize", "");
        long long keys = RedisModule_CallReplyInteger(reply);
        RedisModule_FreeCallReply(reply);
-        g_stats->gauge("keys", keys, false /* prefixOnly */);
+        g_stats->gauge("keys", keys);
        RedisModule_Log(ctx, REDISMODULE_LOGLEVEL_DEBUG, "Emitting metric \"keys\": %llu", keys);
        g_stats->timing("emit_keys_metric_time_taken_us", ustime() - commandStartTime);

@@ -699,20 +672,17 @@ extern "C" int RedisModule_OnLoad(RedisModuleCtx *ctx, RedisModuleString **argv,
    RedisModule_AutoMemory(ctx);
    /* Use pod name if available*/
    const char *podName = getenv("POD_NAME");
-    utsname sysName;
    unameResult = uname(&sysName);
-    if (unameResult == 0) {
-        nodeName = std::string(sysName.nodename);
-        std::replace(nodeName.begin(), nodeName.end(), '.', '-');
-    }
    if (podName != nullptr) {
        m_strPrefix = podName;
-        std::replace(m_strPrefix.begin(), m_strPrefix.end(), '.', '-');
    }
-    else if (unameResult == 0) {
-        m_strPrefix = nodeName;
-        unameResult = 1;
+    else {
+        if (unameResult == 0) {
+            m_strPrefix = std::string(sysName.nodename);
+            unameResult = 1;
+        }
    }
+    std::replace(m_strPrefix.begin(), m_strPrefix.end(), '.', '-');

    for (int iarg = 0; iarg < argc; ++iarg) {
        size_t len = 0;
@@ -38,7 +38,6 @@
 #include <vector>
 #include <mutex>
 #include "aelocker.h"
-#include <sys/socket.h>

 static void setProtocolError(const char *errstr, client *c);
 __thread int ProcessingEventsWhileBlocked = 0; /* See processEventsWhileBlocked(). */
@@ -164,6 +163,7 @@ client *createClient(connection *conn, int iel) {
    c->multibulklen = 0;
    c->bulklen = -1;
    c->sentlen = 0;
+    c->sentlenAsync = 0;
    c->flags = 0;
    c->fPendingAsyncWrite = FALSE;
    c->fPendingAsyncWriteHandler = FALSE;
@@ -1174,32 +1174,6 @@ int chooseBestThreadForAccept()
    return ielMinLoad;
 }

-bool checkOverloadIgnorelist(connection *conn) {
-    if (conn->flags & CONN_FLAG_IGNORE_OVERLOAD) {
-        return true;
-    }
-    struct sockaddr_storage sa;
-    socklen_t salen = sizeof(sa);
-    if (getpeername(conn->fd, (struct sockaddr *)&sa, &salen) == -1) {
-        return false;
-    }
-    if (sa.ss_family == AF_INET) {
-        struct sockaddr_in *s = (struct sockaddr_in *)&sa;
-        for (auto ignore : g_pserver->overload_ignorelist) {
-            if (ignore.match(s->sin_addr))
-                return true;
-        }
-    }
-    if (sa.ss_family == AF_INET6) {
-        struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa;
-        for (auto ignore : g_pserver->overload_ignorelist_ipv6) {
-            if (ignore.match(s->sin6_addr))
-                return true;
-        }
-    }
-    return false;
-}
-
 void clientAcceptHandler(connection *conn) {
    client *c = (client*)connGetPrivateData(conn);

@@ -1231,6 +1205,7 @@ void clientAcceptHandler(connection *conn) {
    {
        char cip[NET_IP_STR_LEN+1] = { 0 };
        connPeerToString(conn, cip, sizeof(cip)-1, NULL);
+
        if (strcmp(cip,"127.0.0.1") && strcmp(cip,"::1")) {
            const char *err =
                "-DENIED KeyDB is running in protected mode because protected "
@@ -1262,14 +1237,6 @@ void clientAcceptHandler(connection *conn) {
        }
    }

-    if (checkOverloadIgnorelist(conn))
-    {
-        c->flags |= CLIENT_IGNORE_OVERLOAD;
-        char cip[NET_IP_STR_LEN+1] = { 0 };
-        connPeerToString(conn, cip, sizeof(cip)-1, NULL);
-        serverLog(LL_NOTICE, "Ignoring client from %s for loadshedding", cip);
-    }
-
    g_pserver->stat_numconnections++;
    moduleFireServerEvent(REDISMODULE_EVENT_CLIENT_CHANGE,
                          REDISMODULE_SUBEVENT_CLIENT_CHANGE_CONNECTED,
@@ -1627,7 +1594,6 @@ void unlinkClient(client *c) {
    }

    serverTL->vecclientsProcess.erase(std::remove(serverTL->vecclientsProcess.begin(), serverTL->vecclientsProcess.end(), c), serverTL->vecclientsProcess.end());
-    serverTL->setclientsPrefetch.erase(c);

    /* Clear the tracking status. */
    if (c->flags & CLIENT_TRACKING) disableTracking(c);
@@ -1641,7 +1607,7 @@ bool freeClient(client *c) {

    /* If a client is protected, yet we need to free it right now, make sure
     * to at least use asynchronous freeing. */
-    if (c->flags & CLIENT_PROTECTED || c->casyncOpsPending || c->replstate == SLAVE_STATE_FASTSYNC_TX || c->btype == BLOCKED_STORAGE) {
+    if (c->flags & CLIENT_PROTECTED || c->casyncOpsPending || c->replstate == SLAVE_STATE_FASTSYNC_TX) {
        freeClientAsync(c);
        return false;
    }
@@ -1872,23 +1838,21 @@ int writeToClient(client *c, int handler_installed) {

        while (clientHasPendingReplies(c)) {
            long long repl_end_idx = getReplIndexFromOffset(c->repl_end_off);
-            long long repl_max_writes = g_pserver->repl_backlog_max_writes_per_event > 0 ? g_pserver->repl_backlog_max_writes_per_event : LLONG_MAX;
-
            serverAssert(c->repl_curr_off != -1);

-            if (c->repl_curr_off != c->repl_end_off) {
+            if (c->repl_curr_off != c->repl_end_off){
                long long repl_curr_idx = getReplIndexFromOffset(c->repl_curr_off); 
                long long nwritten2ndStage = 0; /* How much was written from the start of the replication backlog
                                                * in the event of a wrap around write */
                /* normal case with no wrap around */
-                if (repl_end_idx >= repl_curr_idx) {
-                    nwritten = connWrite(c->conn, g_pserver->repl_backlog + repl_curr_idx, std::min(repl_max_writes, repl_end_idx - repl_curr_idx));
+                if (repl_end_idx >= repl_curr_idx){
+                    nwritten = connWrite(c->conn, g_pserver->repl_backlog + repl_curr_idx, repl_end_idx - repl_curr_idx);
                /* wrap around case */
                } else {
-                    nwritten = connWrite(c->conn, g_pserver->repl_backlog + repl_curr_idx, std::min(repl_max_writes, g_pserver->repl_backlog_size - repl_curr_idx));
+                    nwritten = connWrite(c->conn, g_pserver->repl_backlog + repl_curr_idx, g_pserver->repl_backlog_size - repl_curr_idx);
                    /* only attempt wrapping if we write the correct number of bytes */
                    if (nwritten == g_pserver->repl_backlog_size - repl_curr_idx){
-                        nwritten2ndStage = connWrite(c->conn, g_pserver->repl_backlog, std::min(repl_max_writes - nwritten, repl_end_idx));
+                        nwritten2ndStage = connWrite(c->conn, g_pserver->repl_backlog, repl_end_idx);
                        if (nwritten2ndStage != -1)
                            nwritten += nwritten2ndStage;
                    }                
@@ -1905,8 +1869,6 @@ int writeToClient(client *c, int handler_installed) {
                if (nwritten2ndStage == -1) nwritten = -1;
                if (nwritten == -1)
                    break;
-
-                if (totwritten > g_pserver->repl_backlog_max_writes_per_event) break;
            } else {
                break;
            }
@@ -1977,7 +1939,7 @@ int writeToClient(client *c, int handler_installed) {
    if (nwritten == -1) {
        if (connGetState(c->conn) != CONN_STATE_CONNECTED) {
            serverLog(LL_VERBOSE,
-                "Error writing to client: %s (totwritten: %zd)", connGetLastError(c->conn), totwritten);
+                "Error writing to client: %s", connGetLastError(c->conn));
            freeClientAsync(c);
            
            return C_ERR;
@@ -2798,9 +2760,9 @@ void readQueryFromClient(connection *conn) {
        return;
    }

-    if (cserver.cthreads > 1 || g_pserver->m_pstorageFactory || g_pserver->is_overloaded) {
+    if (cserver.cthreads > 1 || g_pserver->m_pstorageFactory) {
        parseClientCommandBuffer(c);
-        if (g_pserver->enable_async_commands && !serverTL->disable_async_commands && listLength(g_pserver->monitors) == 0 && (aeLockContention() || serverTL->rgdbSnapshot[c->db->id] || g_fTestMode) && !serverTL->in_eval && !serverTL->in_exec && !g_pserver->is_overloaded) {
+        if (g_pserver->enable_async_commands && !serverTL->disable_async_commands && listLength(g_pserver->monitors) == 0 && (aeLockContention() || serverTL->rgdbSnapshot[c->db->id] || g_fTestMode) && !serverTL->in_eval && !serverTL->in_exec) {
            // Frequent writers aren't good candidates for this optimization, they cause us to renew the snapshot too often
            //  so we exclude them unless the snapshot we need already exists.
            // Note: In test mode we want to create snapshots as often as possibl to excercise them - we don't care about perf
@@ -2812,20 +2774,8 @@ void readQueryFromClient(connection *conn) {
                processInputBuffer(c, false, CMD_CALL_SLOWLOG | CMD_CALL_STATS | CMD_CALL_ASYNC);
            }
        }
-        if (!c->vecqueuedcmd.empty()) {
-            if (g_pserver->m_pstorageFactory != nullptr && g_pserver->prefetch_enabled && !(c->flags & CLIENT_BLOCKED)) {
-                serverTL->setclientsPrefetch.insert(c);
-            } else {
-                if (g_pserver->is_overloaded && !(c->flags & (CLIENT_MASTER | CLIENT_SLAVE | CLIENT_PENDING_WRITE | CLIENT_PUBSUB | CLIENT_BLOCKED | CLIENT_IGNORE_OVERLOAD)) && ((random() % 100) < g_pserver->overload_protect_strength)) {
-                    for (unsigned i = 0; i < c->vecqueuedcmd.size(); i++) {
-                        addReply(c, shared.overloaderr);
-                    }
-                    c->vecqueuedcmd.clear();
-                } else {
-                    serverTL->vecclientsProcess.push_back(c);
-                }
-            }
-        }
+        if (!c->vecqueuedcmd.empty())
+            serverTL->vecclientsProcess.push_back(c);
    } else {
        // If we're single threaded its actually better to just process the command here while the query is hot in the cache
        //  multithreaded lock contention dominates and batching is better
@@ -2839,9 +2789,9 @@ void processClients()
 {
    serverAssert(GlobalLocksAcquired());

-    // Note that this function is reentrant and vecclientsProcess may be modified by code called from processInputBuffer
+    // Note that this function is reentrant and vecclients may be modified by code called from processInputBuffer
    while (!serverTL->vecclientsProcess.empty()) {
-        client *c = *serverTL->vecclientsProcess.begin();
+        client *c = serverTL->vecclientsProcess.front();
        serverTL->vecclientsProcess.erase(serverTL->vecclientsProcess.begin());

        /* There is more data in the client input buffer, continue parsing it
@@ -46,7 +46,6 @@ robj *createObject(int type, void *ptr) {
    char *oB = (char*)zcalloc(sizeof(robj)+mvccExtraBytes, MALLOC_SHARED);
    robj *o = reinterpret_cast<robj*>(oB + mvccExtraBytes);
    
-    new (o) redisObject;
    o->type = type;
    o->encoding = OBJ_ENCODING_RAW;
    o->m_ptr = ptr;
@@ -419,7 +418,6 @@ void decrRefCount(robj_roptr o) {
        case OBJ_NESTEDHASH: freeNestedHashObject(o); break;
        default: serverPanic("Unknown object type"); break;
        }
-        o->~redisObject();
        if (g_pserver->fActiveReplica) {
            zfree(reinterpret_cast<redisObjectExtended*>(o.unsafe_robjcast())-1);
        } else {
@@ -2542,7 +2542,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb, sds key, int *error, uint64_t mvcc_ts
         * encoding version in the lower 10 bits of the module ID. */
        void *ptr = mt->rdb_load(&io,moduleid&1023);
        if (io.ctx) {
-            moduleFreeContext(io.ctx, false /* propogate */);
+            moduleFreeContext(io.ctx);
            zfree(io.ctx);
        }

@@ -2865,12 +2865,6 @@ public:

    size_t endWork() {
        if (!fLaunched) {
-            for (int idb = 0; idb < cserver.dbnum; ++idb) {
-                if (g_pserver->m_pstorageFactory) {
-                    g_pserver->db[idb]->processChangesAsync(cstorageWritesInFlight);
-                }
-            }
-            while (cstorageWritesInFlight > 0);
            return ckeysLoaded;
        }
        if (!vecbatch.empty()) {
@@ -8,46 +8,38 @@ class readWriteLock {
    int m_readCount = 0;
    int m_writeCount = 0;
    bool m_writeWaiting = false;
-    bool m_multi = true;
+    bool m_notify = true;
 public:
    readWriteLock(const char *name) : m_readLock(name), m_writeLock(name) {}

    void acquireRead() {
-        std::unique_lock<fastlock> rm(m_readLock, std::defer_lock);
-        if (m_multi) {
-            rm.lock();
-            while (m_writeCount > 0 || m_writeWaiting)
-                m_cv.wait(rm);
-        }
+        std::unique_lock<fastlock> rm(m_readLock);
+        while (m_writeCount > 0 || m_writeWaiting)
+            m_cv.wait(rm);
        m_readCount++;
    }
    
    bool tryAcquireRead() {
        std::unique_lock<fastlock> rm(m_readLock, std::defer_lock);
-        if (m_multi) {
-            if (!rm.try_lock())
-                return false;
-            if (m_writeCount > 0 || m_writeWaiting)
-                return false;
-        }
+        if (!rm.try_lock())
+            return false;
+        if (m_writeCount > 0 || m_writeWaiting)
+            return false;
        m_readCount++;
        return true;
    }

    void acquireWrite(bool exclusive = true) {
-        std::unique_lock<fastlock> rm(m_readLock, std::defer_lock);
-        if (m_multi) {
-            rm.lock();
-            m_writeWaiting = true;
-            while (m_readCount > 0)
+        std::unique_lock<fastlock> rm(m_readLock);
+        m_writeWaiting = true;
+        while (m_readCount > 0)
+            m_cv.wait(rm);
+        if (exclusive) {
+            /* Another thread might have the write lock while we have the read lock
+               but won't be able to release it until they can acquire the read lock
+               so release the read lock and try again instead of waiting to avoid deadlock */
+            while(!m_writeLock.try_lock())
                m_cv.wait(rm);
-            if (exclusive) {
-                /* Another thread might have the write lock while we have the read lock
-                but won't be able to release it until they can acquire the read lock
-                so release the read lock and try again instead of waiting to avoid deadlock */
-                while(!m_writeLock.try_lock())
-                    m_cv.wait(rm);
-            }
        }
        m_writeCount++;
        m_writeWaiting = false;
@@ -60,38 +52,32 @@ public:

    bool tryAcquireWrite(bool exclusive = true) {
        std::unique_lock<fastlock> rm(m_readLock, std::defer_lock);
-        if (m_multi) {
-            if (!rm.try_lock())
+        if (!rm.try_lock())
+            return false;
+        if (m_readCount > 0)
+            return false;
+        if (exclusive)
+            if (!m_writeLock.try_lock())
                return false;
-            if (m_readCount > 0)
-                return false;
-            if (exclusive)
-                if (!m_writeLock.try_lock())
-                    return false;
-        }
        m_writeCount++;
        return true;
    }

    void releaseRead() {
-        std::unique_lock<fastlock> rm(m_readLock, std::defer_lock);
-        if (m_multi) {
-            rm.lock();
-            m_cv.notify_all();
-        }
+        std::unique_lock<fastlock> rm(m_readLock);
        m_readCount--;
+        if (m_notify)
+            m_cv.notify_all();
    }

    void releaseWrite(bool exclusive = true) {
-        std::unique_lock<fastlock> rm(m_readLock, std::defer_lock);
+        std::unique_lock<fastlock> rm(m_readLock);
        serverAssert(m_writeCount > 0);
-        if (m_multi) {
-            rm.lock();
-            if (exclusive)
-                m_writeLock.unlock();
-            m_cv.notify_all();
-        }
+        if (exclusive)
+            m_writeLock.unlock();
        m_writeCount--;
+        if (m_notify)
+            m_cv.notify_all();
    }

    void downgradeWrite(bool exclusive = true) {
@@ -99,8 +85,8 @@ public:
        acquireRead();
    }

-    void setMulti(bool multi) {
-        m_multi = multi;
+    void setNotify(bool notify) {
+        m_notify = notify;
    }

    bool hasReader() {
@@ -2524,10 +2524,7 @@ cleanup:

 static int clusterManagerNodeConnect(clusterManagerNode *node) {
    if (node->context) redisFree(node->context);
-    struct timeval tv;
-    tv.tv_sec = config.cluster_manager_command.timeout / 1000;
-    tv.tv_usec = (config.cluster_manager_command.timeout % 1000) * 1000;
-    node->context = redisConnectWithTimeout(node->ip, node->port, tv);
+    node->context = redisConnect(node->ip, node->port);
    if (!node->context->err && config.tls) {
        const char *err = NULL;
        if (cliSecureConnection(node->context, config.sslconfig, &err) == REDIS_ERR && err) {
@@ -5629,10 +5626,7 @@ static int clusterManagerCommandImport(int argc, char **argv) {
    char *reply_err = NULL;
    redisReply *src_reply = NULL;
    // Connect to the source node.
-    struct timeval tv;
-    tv.tv_sec = config.cluster_manager_command.timeout / 1000;
-    tv.tv_usec = (config.cluster_manager_command.timeout % 1000) * 1000;
-    redisContext *src_ctx = redisConnectWithTimeout(src_ip, src_port, tv);
+    redisContext *src_ctx = redisConnect(src_ip, src_port);
    if (src_ctx->err) {
        success = 0;
        fprintf(stderr,"Could not connect to KeyDB at %s:%d: %s.\n", src_ip,
@@ -188,8 +188,6 @@ int bg_unlink(const char *filename) {
 /* ---------------------------------- MASTER -------------------------------- */

 bool createDiskBacklog() {
-    if (g_pserver->repl_backlog_disk != nullptr)
-        return true;    // already exists
    // Lets create some disk backed pages and add them here
    std::string path = "./repl-backlog-temp" + std::to_string(gettid());
 #if (defined __APPLE__ || defined __FreeBSD__)
@@ -283,7 +281,7 @@ void resizeReplicationBacklog(long long newsize) {
            
            if (cserver.repl_backlog_disk_size != 0) {
                if (newsize > g_pserver->repl_backlog_config_size || cserver.force_backlog_disk) {
-                    if (g_pserver->repl_backlog_disk == nullptr || g_pserver->repl_backlog == g_pserver->repl_backlog_disk)
+                    if (g_pserver->repl_backlog == g_pserver->repl_backlog_disk)
                        return; // Can't do anything more
                    serverLog(LL_NOTICE, "Switching to disk backed replication backlog due to exceeding memory limits");
                    backlog = g_pserver->repl_backlog_disk;
@@ -308,25 +306,10 @@ void resizeReplicationBacklog(long long newsize) {
                auto cbActiveBacklog = cbPhase1 + g_pserver->repl_backlog_idx;
                serverAssert(g_pserver->repl_backlog_histlen == cbActiveBacklog);
            }
-            if (g_pserver->repl_backlog != g_pserver->repl_backlog_disk) {
+            if (g_pserver->repl_backlog != g_pserver->repl_backlog_disk)
                zfree(g_pserver->repl_backlog);
-            } else {
+            else
                serverLog(LL_NOTICE, "Returning to memory backed replication backlog");
-                auto repl_backlog = g_pserver->repl_backlog_disk;
-                g_pserver->repl_backlog_disk = nullptr;
-                g_pserver->asyncworkqueue->AddWorkFunction([repl_backlog, size = cserver.repl_backlog_disk_size]{
-                    // The kernel doesn't make promises with how it will manage the memory but users really want to
-                    //  see the RSS go down.  So lets encourage that to happen.
-                    madvise(repl_backlog, size, MADV_DONTNEED); // NOTE: This will block until all pages are released
-                    aeAcquireLock();
-                    if (g_pserver->repl_backlog_disk == nullptr && cserver.repl_backlog_disk_size == size) {
-                        g_pserver->repl_backlog_disk = repl_backlog;
-                    } else {
-                        munmap(g_pserver->repl_backlog_disk, size);
-                    }
-                    aeReleaseLock();
-                }, true /*fHiPri*/);
-            }
            g_pserver->repl_backlog = backlog;
            g_pserver->repl_backlog_idx = g_pserver->repl_backlog_histlen;
            if (g_pserver->repl_batch_idxStart >= 0) {
@@ -346,17 +329,6 @@ void resizeReplicationBacklog(long long newsize) {
            /* Next byte we have is... the next since the buffer is empty. */
            g_pserver->repl_backlog_off = g_pserver->master_repl_offset+1;
            g_pserver->repl_backlog_start = g_pserver->master_repl_offset;
-            listIter li;
-            listNode *ln;
-            listRewind(g_pserver->slaves, &li);
-            while ((ln = listNext(&li))) {
-                client *replica = (client*)listNodeValue(ln);
-
-                std::unique_lock<fastlock> ul(replica->lock);
-
-                replica->repl_curr_off = g_pserver->master_repl_offset;
-                replica->repl_end_off = g_pserver->master_repl_offset;
-            }
        }
    }
    g_pserver->repl_backlog_size = newsize;
@@ -1045,6 +1017,8 @@ public:
    void addReplica(client *replica) {
        replicas.push_back(replica);
        replicationSetupSlaveForFullResync(replica,getPsyncInitialOffset());
+        // Optimize the socket for bulk transfer
+        //connDisableTcpNoDelay(replica->conn);
    }

    bool isActive() const { return !replicas.empty(); }
@@ -1059,7 +1033,7 @@ public:
            while (checkClientOutputBufferLimits(replica)
              && (replica->flags.load(std::memory_order_relaxed) & CLIENT_CLOSE_ASAP) == 0) {
                ul.unlock();
-                usleep(1000);   // give 1ms for the I/O before we poll again
+                usleep(0);
                ul.lock();
            }
        }
@@ -1117,7 +1091,7 @@ public:
        reply->used += size;
    }

-    void addLongLongWithPrefix(long long val, char prefix) {
+    void addLongWithPrefix(long val, char prefix) {
        char buf[128];
        int len;

@@ -1129,19 +1103,15 @@ public:
    }

    void addArrayLen(int len) {
-        addLongLongWithPrefix(len, '*');
+        addLongWithPrefix(len, '*');
    }

    void addLong(long val) {
-        addLongLongWithPrefix(val, ':');
-    }
-
-    void addLongLong(long long val) {
-        addLongLongWithPrefix(val, ':');
+        addLongWithPrefix(val, ':');
    }

    void addString(const char *s, unsigned long len) {
-        addLongLongWithPrefix(len, '$');
+        addLongWithPrefix(len, '$');
        addData(s, len);
        addData("\r\n", 2);
    }
@@ -1157,24 +1127,11 @@ public:
        }
    }

-    void putReplicasOnline() {
-        for (auto replica : replicas) {
-            std::unique_lock<fastlock> ul(replica->lock);
-            //  If we put the replica online before the output is drained then it will get immediately closed
-            while (checkClientOutputBufferLimits(replica)
-              && (replica->flags.load(std::memory_order_relaxed) & CLIENT_CLOSE_ASAP) == 0) {
-                ul.unlock();
-                usleep(1000);   // give 1ms for the I/O before we poll again
-                ul.lock();
-            }
-        }
-
-        aeAcquireLock();
+    void putSlavesOnline() {
        for (auto replica : replicas) {
            replica->replstate = SLAVE_STATE_FASTSYNC_DONE;
            replica->repl_put_online_on_ack = 1;
        }
-        aeReleaseLock();
    }

    void abort() {
@@ -1190,19 +1147,22 @@ int rdbSaveSnapshotForReplication(rdbSaveInfo *rsi) {
    // TODO: This needs to be on a background thread
    int retval = C_OK;
    serverAssert(GlobalLocksAcquired());
-    serverLog(LL_NOTICE, "Starting fast full sync with target: %s", "disk");
+    serverLog(LL_NOTICE, "Starting storage provider fast full sync with target: %s", "disk");

    std::shared_ptr<replicationBuffer> spreplBuf = std::make_shared<replicationBuffer>();
    listNode *ln;
    listIter li;
+    client *replica = nullptr;
    listRewind(g_pserver->slaves, &li);
-    while ((ln = listNext(&li))) {
+    while (replica == nullptr && (ln = listNext(&li))) {
        client *replicaCur = (client*)listNodeValue(ln);
-        if ((replicaCur->slave_capa & SLAVE_CAPA_KEYDB_FASTSYNC) && (replicaCur->replstate == SLAVE_STATE_WAIT_BGSAVE_START)) {
-            spreplBuf->addReplica(replicaCur);
+        if ((replicaCur->slave_capa & SLAVE_CAPA_ROCKSDB_SNAPSHOT) && (replicaCur->replstate == SLAVE_STATE_WAIT_BGSAVE_START)) {
+            replica = replicaCur;
+            spreplBuf->addReplica(replica);
            replicaCur->replstate = SLAVE_STATE_FASTSYNC_TX;
        }
    }
+    serverAssert(replica != nullptr);

    spreplBuf->addArrayLen(2);   // Two sections: Metadata and databases

@@ -1217,7 +1177,7 @@ int rdbSaveSnapshotForReplication(rdbSaveInfo *rsi) {
            spreplBuf->addString(rsi->repl_id, CONFIG_RUN_ID_SIZE);
        spreplBuf->addArrayLen(2);
            spreplBuf->addString("repl-offset", 11);
-            spreplBuf->addLongLong(rsi->master_repl_offset);
+            spreplBuf->addLong(rsi->master_repl_offset);

    if (dictSize(g_pserver->lua_scripts)) {
        dictEntry *de;
@@ -1233,13 +1193,13 @@ int rdbSaveSnapshotForReplication(rdbSaveInfo *rsi) {
        di = NULL; /* So that we don't release it again on error. */
    }

-    std::shared_ptr<std::vector<const redisDbPersistentDataSnapshot*>> spvecsnapshot = std::make_shared<std::vector<const redisDbPersistentDataSnapshot*>>();
+    std::shared_ptr<std::vector<std::unique_ptr<const StorageCache>>> spvecspsnapshot = std::make_shared<std::vector<std::unique_ptr<const StorageCache>>>();
    for (int idb = 0; idb < cserver.dbnum; ++idb) {
-        spvecsnapshot->emplace_back(g_pserver->db[idb]->createSnapshot(getMvccTstamp(), false));
+        spvecspsnapshot->emplace_back(g_pserver->db[idb]->CloneStorageCache());
    }
    aeReleaseLock();

-    g_pserver->asyncworkqueue->AddWorkFunction([spreplBuf = std::move(spreplBuf), spvecsnapshot = std::move(spvecsnapshot)]{
+    g_pserver->asyncworkqueue->AddWorkFunction([spreplBuf = std::move(spreplBuf), spvecspsnapshot = std::move(spvecspsnapshot)]{
        int retval = C_OK;
        auto timeStart = ustime();
        auto lastLogTime = timeStart;
@@ -1249,16 +1209,15 @@ int rdbSaveSnapshotForReplication(rdbSaveInfo *rsi) {
        // Databases
        replBuf.addArrayLen(cserver.dbnum);
        for (int idb = 0; idb < cserver.dbnum; ++idb) {
-            auto &spsnapshot = (*spvecsnapshot)[idb];
-            size_t snapshotDeclaredCount = spsnapshot->size();
+            auto &spsnapshot = (*spvecspsnapshot)[idb];
+            size_t snapshotDeclaredCount = spsnapshot->count();
            replBuf.addArrayLen(snapshotDeclaredCount);
            size_t count = 0;
-            bool result = spsnapshot->iterate_threadsafe([&replBuf, &count, &cbData, &lastLogTime, &cbLastUpdate](const char *strKey, robj_roptr o) -> bool{
+            bool result = spsnapshot->enumerate([&replBuf, &count, &cbData, &lastLogTime, &cbLastUpdate](const char *rgchKey, size_t cchKey, const void *rgchVal, size_t cchVal) -> bool{
                replBuf.addArrayLen(2);

-                replBuf.addString(strKey, sdslen(strKey));
-                sds strT = serializeStoredObjectAndExpire(o);
-                replBuf.addString(strT, sdslen(strT));
+                replBuf.addString(rgchKey, cchKey);
+                replBuf.addString((const char *)rgchVal, cchVal);
                ++count;
                if ((count % 8092) == 0) {
                    auto curTime = ustime();
@@ -1269,8 +1228,7 @@ int rdbSaveSnapshotForReplication(rdbSaveInfo *rsi) {
                        lastLogTime = ustime();
                    }
                }
-                cbData += sdslen(strKey) + sdslen(strT);
-                sdsfree(strT);
+                cbData += cchKey + cchVal;
                return replBuf.isActive();
            });

@@ -1295,7 +1253,9 @@ int rdbSaveSnapshotForReplication(rdbSaveInfo *rsi) {
        auto usec = ustime() - timeStart;
        serverLog(LL_NOTICE, "Transferred %zuMB total (%zuMB data) in %.2f seconds.  (%.2fGbit/s)", spreplBuf->cbWritten()/1024/1024, cbData/1024/1024, usec/1000000.0, (spreplBuf->cbWritten()*8.0)/(usec/1000000.0)/1000000000.0);
        if (retval == C_OK) {
-            replBuf.putReplicasOnline();
+            aeAcquireLock();
+            replBuf.putSlavesOnline();
+            aeReleaseLock();
        }
    });

@@ -1340,7 +1300,7 @@ int startBgsaveForReplication(int mincapa) {
    /* Only do rdbSave* when rsiptr is not NULL,
     * otherwise replica will miss repl-stream-db. */
    if (rsiptr) {
-        if (mincapa & SLAVE_CAPA_KEYDB_FASTSYNC && FFastSyncEnabled())
+        if (mincapa & SLAVE_CAPA_ROCKSDB_SNAPSHOT && g_pserver->m_pstorageFactory)
            retval = rdbSaveSnapshotForReplication(rsiptr);
        else if (socket_target)
            retval = rdbSaveToSlavesSockets(rsiptr);
@@ -1519,7 +1479,7 @@ void syncCommand(client *c) {
    }

    /* CASE 0: Fast Sync */
-    if (c->slave_capa & SLAVE_CAPA_KEYDB_FASTSYNC && FFastSyncEnabled()) {
+    if ((c->slave_capa & SLAVE_CAPA_ROCKSDB_SNAPSHOT) && g_pserver->m_pstorageFactory) {
        serverLog(LL_NOTICE,"Fast SYNC on next replication cycle");
    /* CASE 1: BGSAVE is in progress, with disk target. */
    } else if (g_pserver->FRdbSaveInProgress() &&
@@ -1710,8 +1670,8 @@ void replconfCommand(client *c) {
                c->slave_capa |= SLAVE_CAPA_PSYNC2;
            else if (!strcasecmp((const char*)ptrFromObj(c->argv[j+1]), "activeExpire"))
                c->slave_capa |= SLAVE_CAPA_ACTIVE_EXPIRE;
-            else if (!strcasecmp((const char*)ptrFromObj(c->argv[j+1]), "keydb-fastsync"))
-                c->slave_capa |= SLAVE_CAPA_KEYDB_FASTSYNC;
+            else if (!strcasecmp((const char*)ptrFromObj(c->argv[j+1]), "rocksdb-snapshot-load"))
+                c->slave_capa |= SLAVE_CAPA_ROCKSDB_SNAPSHOT;

            fCapaCommand = true;
        } else if (!strcasecmp((const char*)ptrFromObj(c->argv[j]),"ack")) {
@@ -1776,14 +1736,10 @@ void replconfCommand(client *c) {

    if (fCapaCommand) {
        sds reply = sdsnew("+OK");
-        if (g_pserver->fActiveReplica) {
+        if (g_pserver->fActiveReplica)
            reply = sdscat(reply, " active-replica");
-        }
-        if ((c->slave_capa & SLAVE_CAPA_KEYDB_FASTSYNC) && FFastSyncEnabled()) {
-            reply = sdscat(reply, " keydb-fastsync-save");
-        } else {
-            c->slave_capa = (c->slave_capa & (~SLAVE_CAPA_KEYDB_FASTSYNC)); // never try to fast sync for this as they won't expect it
-        }
+        if (g_pserver->m_pstorageFactory && (c->slave_capa & SLAVE_CAPA_ROCKSDB_SNAPSHOT) && !g_pserver->fActiveReplica)
+            reply = sdscat(reply, " rocksdb-snapshot-save");
        reply = sdscat(reply, "\r\n");
        addReplySds(c, reply);
    } else {
@@ -2532,7 +2488,7 @@ size_t parseCount(const char *rgch, size_t cch, long long *pvalue) {
    return cchNumeral + 3;
 }

-bool readFastSyncBulkPayload(connection *conn, redisMaster *mi, rdbSaveInfo &rsi) {
+bool readSnapshotBulkPayload(connection *conn, redisMaster *mi, rdbSaveInfo &rsi) {
    int fUpdate = g_pserver->fActiveReplica || g_pserver->enable_multimaster;
    serverAssert(GlobalLocksAcquired());
    serverAssert(mi->master == nullptr);
@@ -2557,11 +2513,6 @@ bool readFastSyncBulkPayload(connection *conn, redisMaster *mi, rdbSaveInfo &rsi
        }
    }

-    if (mi->repl_state == REPL_STATE_WAIT_STORAGE_IO) {
-        goto LWaitIO;
-    }
-
-    serverAssert(mi->parseState != nullptr);
    for (int iter = 0; iter < 10; ++iter) {
        if (mi->parseState->shouldThrottle())
            return false;
@@ -2572,16 +2523,11 @@ bool readFastSyncBulkPayload(connection *conn, redisMaster *mi, rdbSaveInfo &rsi

        auto nread = connRead(conn, mi->bulkreadBuffer+qblen, readlen);
        if (nread <= 0) {
-            if (connGetState(conn) != CONN_STATE_CONNECTED) {
-                serverLog(LL_WARNING,"I/O error trying to sync with MASTER: %s",
-                    (nread == -1) ? strerror(errno) : "connection lost");
+            if (connGetState(conn) != CONN_STATE_CONNECTED)
                cancelReplicationHandshake(mi, true);
-            }
            return false;
        }
-        g_pserver->stat_net_input_bytes += nread;
        mi->repl_transfer_lastio = g_pserver->unixtime;
-        mi->repl_transfer_read += nread;
        sdsIncrLen(mi->bulkreadBuffer,nread);

        size_t offset = 0;
@@ -2653,9 +2599,8 @@ bool readFastSyncBulkPayload(connection *conn, redisMaster *mi, rdbSaveInfo &rsi
            rsi.repl_stream_db = mi->parseState->getMetaDataLongLong("repl-stream-db");
            rsi.repl_offset = mi->parseState->getMetaDataLongLong("repl-offset");
            sds str = mi->parseState->getMetaDataStr("repl-id");
-            if (sdslen(str) == CONFIG_RUN_ID_SIZE) {
+            if (sdslen(str) == CONFIG_RUN_ID_SIZE+1) {
                memcpy(rsi.repl_id, str, CONFIG_RUN_ID_SIZE+1);
-                rsi.repl_id_is_set = 1;
            }

            fFinished = true;
@@ -2678,14 +2623,7 @@ bool readFastSyncBulkPayload(connection *conn, redisMaster *mi, rdbSaveInfo &rsi
    if (!fFinished)
        return false;

-LWaitIO:
-    if (mi->parseState->hasIOInFlight()) {
-        mi->repl_state = REPL_STATE_WAIT_STORAGE_IO;
-        return false;
-    }
-
    serverLog(LL_NOTICE, "Fast sync complete");
-    serverAssert(!mi->parseState->hasIOInFlight());
    delete mi->parseState;
    mi->parseState = nullptr;
    return true;
@@ -3061,8 +2999,8 @@ void readSyncBulkPayload(connection *conn) {
        return;
    }

-    if (mi->isKeydbFastsync) {
-        if (!readFastSyncBulkPayload(conn, mi, rsi))
+    if (mi->isRocksdbSnapshotRepl) {
+        if (!readSnapshotBulkPayload(conn, mi, rsi))
            return;
    } else {
        if (!readSyncBulkPayloadRdb(conn, mi, rsi, usemark))
@@ -3074,15 +3012,17 @@ void readSyncBulkPayload(connection *conn) {

    /* Final setup of the connected slave <- master link */
    replicationCreateMasterClient(mi,mi->repl_transfer_s,rsi.repl_stream_db);
-    if (mi->isKeydbFastsync) {
+    if (mi->isRocksdbSnapshotRepl) {
        /* We need to handle the case where the initial querybuf data was read by fast sync */
        /* This should match the work readQueryFromClient would do for a master client */
        mi->master->querybuf = sdscatsds(mi->master->querybuf, mi->bulkreadBuffer);
-        mi->master->pending_querybuf = sdscatsds(mi->master->pending_querybuf, mi->bulkreadBuffer);
-        mi->master->read_reploff += sdslen(mi->bulkreadBuffer);
-
        sdsfree(mi->bulkreadBuffer);
        mi->bulkreadBuffer = nullptr;
+
+        mi->master->pending_querybuf = sdscatlen(mi->master->pending_querybuf,
+            mi->master->querybuf,sdslen(mi->master->querybuf));
+
+        mi->master->read_reploff += sdslen(mi->master->querybuf);
    }
    mi->repl_transfer_s = nullptr;
    mi->repl_state = REPL_STATE_CONNECTED;
@@ -3121,13 +3061,13 @@ void readSyncBulkPayload(connection *conn) {
    }

    /* Send the initial ACK immediately to put this replica in online state. */
-    if (usemark || mi->isKeydbFastsync) replicationSendAck(mi);
+    if (usemark || mi->isRocksdbSnapshotRepl) replicationSendAck(mi);

    /* Restart the AOF subsystem now that we finished the sync. This
     * will trigger an AOF rewrite, and when done will start appending
     * to the new file. */
    if (g_pserver->aof_enabled) restartAOFAfterSYNC();
-    if (mi->isKeydbFastsync)
+    if (mi->isRocksdbSnapshotRepl)
        readQueryFromClient(conn); // There may be querybuf data we just appeneded
    return;
 }
@@ -3453,15 +3393,15 @@ void parseMasterCapa(redisMaster *mi, sds strcapa)
    char *pchEnd = szStart;

    mi->isActive = false;
-    mi->isKeydbFastsync = false;
+    mi->isRocksdbSnapshotRepl = false;
    for (;;)
    {
        if (*pchEnd == ' ' || *pchEnd == '\0') {
            // Parse the word
            if (strncmp(szStart, "active-replica", pchEnd - szStart) == 0) {
                mi->isActive = true;
-            } else if (strncmp(szStart, "keydb-fastsync-save", pchEnd - szStart) == 0) {
-                mi->isKeydbFastsync = true;
+            } else if (strncmp(szStart, "rocksdb-snapshot-save", pchEnd - szStart) == 0) {
+                mi->isRocksdbSnapshotRepl = true;
            }
            szStart = pchEnd + 1;
        }
@@ -3608,9 +3548,9 @@ retry_connect:
            "capa","psync2",
            "capa","activeExpire",
        };
-        if (FFastSyncEnabled() && g_pserver->repl_diskless_load != REPL_DISKLESS_LOAD_SWAPDB) {
+        if (g_pserver->m_pstorageFactory && !g_pserver->fActiveReplica && g_pserver->repl_diskless_load != REPL_DISKLESS_LOAD_SWAPDB) {
            veccapabilities.push_back("capa");
-            veccapabilities.push_back("keydb-fastsync");
+            veccapabilities.push_back("rocksdb-snapshot-load");
        }

        err = sendCommandArgv(conn, veccapabilities.size(), veccapabilities.data(), nullptr);
@@ -3793,7 +3733,7 @@ retry_connect:
    }

    /* Prepare a suitable temp file for bulk transfer */
-    if (!useDisklessLoad() && !mi->isKeydbFastsync) {
+    if (!useDisklessLoad()) {
        while(maxtries--) {
            auto dt = std::chrono::system_clock::now().time_since_epoch();
            auto dtMillisecond = std::chrono::duration_cast<std::chrono::milliseconds>(dt);
@@ -3922,10 +3862,6 @@ int cancelReplicationHandshake(redisMaster *mi, int reconnect) {
        delete mi->parseState;
        mi->parseState = nullptr;
    }
-    if (mi->bulkreadBuffer) {
-        sdsfree(mi->bulkreadBuffer);
-        mi->bulkreadBuffer = nullptr;
-    }

    if (mi->repl_state == REPL_STATE_TRANSFER) {
        replicationAbortSyncTransfer(mi);
@@ -4391,6 +4327,7 @@ void replicationCacheMaster(redisMaster *mi, client *c) {
    if (c->flags & CLIENT_MULTI) discardTransaction(c);
    listEmpty(c->reply);
    c->sentlen = 0;
+    c->sentlenAsync = 0;
    c->reply_bytes = 0;
    c->bufpos = 0;
    resetClient(c);
@@ -4829,10 +4766,6 @@ void replicationCron(void) {
    {
        redisMaster *mi = (redisMaster*)listNodeValue(lnMaster);

-        if (mi->repl_state == REPL_STATE_WAIT_STORAGE_IO && !mi->parseState->hasIOInFlight()) {
-            readSyncBulkPayload(mi->repl_transfer_s);
-        }
-
        std::unique_lock<decltype(mi->master->lock)> ulock;
        if (mi->master != nullptr)
            ulock = decltype(ulock)(mi->master->lock);
@@ -4849,8 +4782,7 @@ void replicationCron(void) {

        /* Bulk transfer I/O timeout? */
        if (mi->masterhost && mi->repl_state == REPL_STATE_TRANSFER &&
-            (time(NULL)-mi->repl_transfer_lastio) > g_pserver->repl_timeout &&
-            !(g_pserver->loading & LOADING_REPLICATION))
+            (time(NULL)-mi->repl_transfer_lastio) > g_pserver->repl_timeout)
        {
            serverLog(LL_WARNING,"Timeout receiving bulk data from MASTER... If the problem persists try to set the 'repl-timeout' parameter in keydb.conf to a larger value.");
            cancelReplicationHandshake(mi,true);
@@ -418,10 +418,7 @@ public:
    sdsstring &operator=(const sdsstring &other)
    {
        sdsfree(m_str);
-        if (other.m_str != nullptr)
-            m_str = sdsdup(other.m_str);
-        else
-            m_str = nullptr;
+        m_str = sdsdup(other.m_str);
        return *this;
    }

@@ -1961,6 +1961,15 @@ void getExpansiveClientsInfo(size_t *in_usage, size_t *out_usage) {
    *out_usage = o;
 }

+int closeClientOnOverload(client *c) {
+    if (g_pserver->overload_closed_clients > MAX_CLIENTS_SHED_PER_PERIOD) return false;
+    if (!g_pserver->is_overloaded) return false;
+    // Don't close masters, replicas, or pub/sub clients
+    if (c->flags & (CLIENT_MASTER | CLIENT_SLAVE | CLIENT_PENDING_WRITE | CLIENT_PUBSUB | CLIENT_BLOCKED)) return false;
+    freeClient(c);
+    ++g_pserver->overload_closed_clients;
+    return true;
+}

 /* This function is called by serverCron() and is used in order to perform
 * operations on clients that are important to perform constantly. For instance
@@ -2032,6 +2041,7 @@ void clientsCron(int iel) {
            if (clientsCronTrackExpansiveClients(c, curr_peak_mem_usage_slot)) goto LContinue;
            if (clientsCronTrackClientsMemUsage(c)) goto LContinue;
            if (closeClientOnOutputBufferLimitReached(c, 0)) continue; // Client also free'd
+            if (closeClientOnOverload(c)) continue;
        LContinue:
            fastlock_unlock(&c->lock);
        }        
@@ -2127,7 +2137,7 @@ void databasesCron(bool fMainThread) {
                ::dict *dict = g_pserver->db[rehash_db]->dictUnsafeKeyOnly();
                /* Are we async rehashing? And if so is it time to re-calibrate? */
                /* The recalibration limit is a prime number to ensure balancing across threads */
-                if (g_pserver->enable_async_rehash && rehashes_per_ms > 0 && async_rehashes < 131 && !cserver.active_defrag_enabled && cserver.cthreads > 1 && dictSize(dict) > 2048 && dictIsRehashing(dict) && !g_pserver->loading && aeLockContention() > 1) {
+                if (rehashes_per_ms > 0 && async_rehashes < 131 && !cserver.active_defrag_enabled && cserver.cthreads > 1 && dictSize(dict) > 2048 && dictIsRehashing(dict) && !g_pserver->loading) {
                    serverTL->rehashCtl = dictRehashAsyncStart(dict, rehashes_per_ms * ((1000 / g_pserver->hz) / 10));  // Estimate 10% CPU time spent in lock contention
                    if (serverTL->rehashCtl)
                        ++async_rehashes;
@@ -2591,13 +2601,8 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {

    /* Check for CPU Overload */
    run_with_period(10'000) {
-        if (g_pserver->is_overloaded) {
-            if (g_pserver->overload_protect_strength < 100)
-                g_pserver->overload_protect_strength *= 2;
-        } else {
-            g_pserver->overload_protect_strength = g_pserver->overload_protect_tenacity;
-        }
        g_pserver->is_overloaded = false;
+        g_pserver->overload_closed_clients = 0;
        static clock_t last = 0;
        if (g_pserver->overload_protect_threshold > 0) {
            clock_t cur = clock();
@@ -2605,7 +2610,6 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) {
            perc /= cserver.cthreads;
            perc *= 100.0;
            serverLog(LL_WARNING, "CPU Used: %.2f", perc);
-            g_pserver->last_overload_cpu_reading = static_cast<float>(perc);
            if (perc > g_pserver->overload_protect_threshold) {
                serverLog(LL_WARNING, "\tWARNING: CPU overload detected.");
                g_pserver->is_overloaded = true;
@@ -2838,20 +2842,6 @@ void beforeSleep(struct aeEventLoop *eventLoop) {

    locker.arm();

-    for (auto *tok : serverTL->setStorageTokensProcess) {
-        tok->db->processStorageToken(tok);
-    }
-    serverTL->setStorageTokensProcess.clear();
-
-    if (g_pserver->m_pstorageFactory != nullptr && !serverTL->setclientsPrefetch.empty()) {
-        g_pserver->db[0]->prefetchKeysFlash(serverTL->setclientsPrefetch);
-        for (client *c : serverTL->setclientsPrefetch) {
-            if (!(c->flags & CLIENT_BLOCKED))
-                serverTL->vecclientsProcess.push_back(c);
-        }
-        serverTL->setclientsPrefetch.clear();
-    }
-
    /* end any snapshots created by fast async commands */
    for (int idb = 0; idb < cserver.dbnum; ++idb) {
        if (serverTL->rgdbSnapshot[idb] != nullptr && serverTL->rgdbSnapshot[idb]->FStale()) {
@@ -3066,8 +3056,10 @@ void afterSleep(struct aeEventLoop *eventLoop) {
        serverAssert(serverTL->gcEpoch.isReset());
        serverTL->gcEpoch = g_pserver->garbageCollector.startEpoch();

+        aeAcquireLock();
        for (int idb = 0; idb < cserver.dbnum; ++idb)
            g_pserver->db[idb]->trackChanges(false);
+        aeReleaseLock();

        serverTL->disable_async_commands = false;
    }
@@ -3127,8 +3119,7 @@ void createSharedObjects(void) {
        "-NOREPLICAS Not enough good replicas to write.\r\n")));
    shared.busykeyerr = makeObjectShared(createObject(OBJ_STRING,sdsnew(
        "-BUSYKEY Target key name already exists.\r\n")));
-    shared.overloaderr = makeObjectShared(createObject(OBJ_STRING,sdsnew(
-        "-OVERLOAD KeyDB is overloaded.\r\n")));
+    

    /* The shared NULL depends on the protocol version. */
    shared.null[0] = NULL;
@@ -3262,7 +3253,6 @@ void initMasterInfo(redisMaster *master)
    master->repl_state = REPL_STATE_NONE;
    master->repl_down_since = 0; /* Never connected, repl is down since EVER. */
    master->mvccLastSync = 0;
-    master->repl_transfer_fd = -1;
 }

 void initServerConfig(void) {
@@ -4049,7 +4039,7 @@ void initServer(void) {
    g_pserver->cron_malloc_stats.allocator_active = 0;
    g_pserver->cron_malloc_stats.allocator_resident = 0;
    g_pserver->cron_malloc_stats.sys_available = 0;
-    g_pserver->cron_malloc_stats.sys_total = getMemTotal();
+    g_pserver->cron_malloc_stats.sys_total = g_pserver->force_eviction_percent ? getMemTotal() : 0;
    g_pserver->lastbgsave_status = C_OK;
    g_pserver->aof_last_write_status = C_OK;
    g_pserver->aof_last_write_errno = 0;
@@ -4160,16 +4150,6 @@ void InitServerLast() {

    g_pserver->asyncworkqueue = new (MALLOC_LOCAL) AsyncWorkQueue(cserver.cthreads);

-    if (g_pserver->m_pstorageFactory != nullptr) {
-        g_pserver->asyncreadworkqueue = new (MALLOC_LOCAL) AsyncWorkQueue(cserver.cthreads);
-
-        //Process one write/commit at a time to ensure consistency
-        g_pserver->asyncwriteworkqueue = new (MALLOC_LOCAL) AsyncWorkQueue(1);
-    } else {
-        g_pserver->asyncreadworkqueue = nullptr;
-        g_pserver->asyncwriteworkqueue = nullptr;
-    }
-
    // Allocate the repl backlog
    
 }
@@ -5122,8 +5102,8 @@ int processCommand(client *c, int callFlags) {

            sds str = sdsempty();
            for (int j = 0; j < numkeys; j++) {
-                str = sdscatsds(str, (sds)ptrFromObj(c->argv[keyindex[j]]));
-                str = sdscat(str, " ");
+                sdscatsds(str, (sds)ptrFromObj(c->argv[keyindex[j]]));
+                sdscat(str, " ");
            }
        
            if (numkeys > 0)
@@ -5683,8 +5663,7 @@ sds genRedisInfoString(const char *section) {
            "lru_clock:%u\r\n"
            "executable:%s\r\n"
            "config_file:%s\r\n"
-            "availability_zone:%s\r\n"
-            "features:%s\r\n",
+            "availability_zone:%s\r\n",
            KEYDB_SET_VERSION,
            redisGitSHA1(),
            strtol(redisGitDirty(),NULL,10) > 0,
@@ -5711,8 +5690,7 @@ sds genRedisInfoString(const char *section) {
            lruclock,
            cserver.executable ? cserver.executable : "",
            cserver.configfile ? cserver.configfile : "",
-            g_pserver->sdsAvailabilityZone,
-            "cluster_mget");
+            g_pserver->sdsAvailabilityZone);
    }

    /* Clients */
@@ -5827,9 +5805,7 @@ sds genRedisInfoString(const char *section) {
            "lazyfree_pending_objects:%zu\r\n"
            "lazyfreed_objects:%zu\r\n"
            "storage_provider:%s\r\n"
-            "available_system_memory:%s\r\n"
-            "maxstorage:%llu\r\n"
-            "storage_used:%lu\r\n",
+            "available_system_memory:%s\r\n",
            zmalloc_used,
            hmem,
            g_pserver->cron_malloc_stats.process_rss,
@@ -5875,9 +5851,7 @@ sds genRedisInfoString(const char *section) {
            lazyfreeGetPendingObjectsCount(),
            lazyfreeGetFreedObjectsCount(),
            g_pserver->m_pstorageFactory ? g_pserver->m_pstorageFactory->name() : "none",
-            available_system_mem,
-            g_pserver->maxstorage,
-            g_pserver->m_pstorageFactory ? g_pserver->m_pstorageFactory->totalDiskspaceUsed() : 0
+            available_system_mem
        );
        freeMemoryOverheadData(mh);
    }
@@ -6295,15 +6269,13 @@ sds genRedisInfoString(const char *section) {
        "used_cpu_sys_children:%ld.%06ld\r\n"
        "used_cpu_user_children:%ld.%06ld\r\n"
        "server_threads:%d\r\n"
-        "long_lock_waits:%" PRIu64 "\r\n"
-        "last_overload_cpu_reading:%.2f\r\n",
+        "long_lock_waits:%" PRIu64 "\r\n",
        (long)self_ru.ru_stime.tv_sec, (long)self_ru.ru_stime.tv_usec,
        (long)self_ru.ru_utime.tv_sec, (long)self_ru.ru_utime.tv_usec,
        (long)c_ru.ru_stime.tv_sec, (long)c_ru.ru_stime.tv_usec,
        (long)c_ru.ru_utime.tv_sec, (long)c_ru.ru_utime.tv_usec,
        cserver.cthreads,
-        fastlock_getlongwaitcount(),
-        g_pserver->last_overload_cpu_reading);
+        fastlock_getlongwaitcount());
 #ifdef RUSAGE_THREAD
        struct rusage m_ru;
        getrusage(RUSAGE_THREAD, &m_ru);
@@ -7439,12 +7411,7 @@ static void validateConfiguration()
        exit(EXIT_FAILURE);
    }

-    if (g_pserver->repl_diskless_load == REPL_DISKLESS_LOAD_SWAPDB && g_pserver->m_pstorageFactory) {
-        serverLog(LL_WARNING, "SWAPDB is not implemented when using a storage provider.");
-        exit(EXIT_FAILURE);
-    }
-
-    g_pserver->repl_backlog_size = g_pserver->repl_backlog_config_size; // this is normally set in the update logic, but not on initial config
+    g_pserver->repl_backlog_config_size = g_pserver->repl_backlog_size; // this is normally set in the update logic, but not on initial config
 }

 int iAmMaster(void) {
@@ -64,7 +64,6 @@
 #include <string>
 #include <mutex>
 #include <unordered_set>
-#include <arpa/inet.h>
 #ifdef __cplusplus
 extern "C" {
 #include <lua.h>
@@ -126,25 +125,9 @@ typedef long long ustime_t; /* microsecond time type. */
 #define OVERLOAD_PROTECT_PERIOD_MS 10'000 // 10 seconds
 #define MAX_CLIENTS_SHED_PER_PERIOD (OVERLOAD_PROTECT_PERIOD_MS / 10)  // Restrict to one client per 10ms

-#define IPV4_BITS 32
-#define IPV6_BITS 128
-
 extern int g_fTestMode;
 extern struct redisServer *g_pserver;

-class TCleanup {
-    std::function<void()> fn;
-
-public:
-    TCleanup(std::function<void()> fn)
-        : fn(fn)
-    {}
-
-    ~TCleanup() {
-        fn();
-    }
-};
-
 struct redisObject;
 class robj_roptr
 {
@@ -563,7 +546,6 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
                                          RDB without replication buffer. */
 #define CLIENT_FORCE_REPLY (1ULL<<44) /* Should addReply be forced to write the text? */
 #define CLIENT_AUDIT_LOGGING (1ULL<<45) /* Client commands required audit logging */
-#define CLIENT_IGNORE_OVERLOAD (1ULL<<46) /* Client that should not be disconnected by overload protection */

 /* Client block type (btype field in client structure)
 * if CLIENT_BLOCKED flag is set. */
@@ -575,8 +557,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT];
 #define BLOCKED_ZSET 5    /* BZPOP et al. */
 #define BLOCKED_PAUSE 6   /* Blocked by CLIENT PAUSE */
 #define BLOCKED_ASYNC 7
-#define BLOCKED_STORAGE 8
-#define BLOCKED_NUM 9     /* Number of blocked states. */
+#define BLOCKED_NUM 8     /* Number of blocked states. */

 /* Client request types */
 #define PROTO_REQ_INLINE 1
@@ -612,7 +593,6 @@ typedef enum {
    REPL_STATE_RECEIVE_PSYNC_REPLY, /* Wait for PSYNC reply */
    /* --- End of handshake states --- */
    REPL_STATE_TRANSFER,        /* Receiving .rdb from master */
-    REPL_STATE_WAIT_STORAGE_IO,
    REPL_STATE_CONNECTED,       /* Connected to master */
 } repl_state;

@@ -640,7 +620,7 @@ typedef enum {
 #define SLAVE_CAPA_EOF (1<<0)    /* Can parse the RDB EOF streaming format. */
 #define SLAVE_CAPA_PSYNC2 (1<<1) /* Supports PSYNC2 protocol. */
 #define SLAVE_CAPA_ACTIVE_EXPIRE (1<<2) /* Will the slave perform its own expirations? (Don't send delete) */
-#define SLAVE_CAPA_KEYDB_FASTSYNC (1<<3)
+#define SLAVE_CAPA_ROCKSDB_SNAPSHOT (1<<3)

 /* Synchronous read timeout - replica side */
 #define CONFIG_REPL_SYNCIO_TIMEOUT 5
@@ -986,7 +966,6 @@ struct redisObjectExtended {

 typedef struct redisObject {
    friend redisObject *createEmbeddedStringObject(const char *ptr, size_t len);
-    friend redisObject *createObject(int type, void *ptr);
 protected:
    redisObject() {}

@@ -1231,8 +1210,6 @@ public:
    bool keycacheIsEnabled();

    void prefetchKeysAsync(client *c, struct parsed_command &command);
-    void prefetchKeysFlash(const std::unordered_set<client*> &setc);
-    void processStorageToken(StorageToken *tok);

    bool FSnapshot() const { return m_spdbSnapshotHOLDER != nullptr; }

@@ -1242,7 +1219,6 @@ public:

    dict_iter find_cached_threadsafe(const char *key) const;

-    static void storageLoadCallback(struct aeEventLoop *el, struct StorageToken *token);
    static void activeExpireCycleCore(int type);

 protected:
@@ -1392,8 +1368,6 @@ struct redisDb : public redisDbPersistentDataSnapshot
    using redisDbPersistentData::keycacheIsEnabled;
    using redisDbPersistentData::dictUnsafeKeyOnly;
    using redisDbPersistentData::prefetchKeysAsync;
-    using redisDbPersistentData::prefetchKeysFlash;
-    using redisDbPersistentData::processStorageToken;
    using redisDbPersistentData::prepOverwriteForSnapshot;
    using redisDbPersistentData::FRehashing;
    using redisDbPersistentData::FTrackingChanges;
@@ -1635,6 +1609,7 @@ struct client {
    unsigned long long reply_bytes; /* Tot bytes of objects in reply list. */
    size_t sentlen;         /* Amount of bytes already sent in the current
                               buffer or object being sent. */
+    size_t sentlenAsync;    /* same as sentlen buf for async buffers (which are a different stream) */
    time_t ctime;           /* Client creation time. */
    long duration;          /* Current command duration. Used for measuring latency of blocking/non-blocking cmds */
    time_t lastinteraction; /* Time of the last interaction, used for timeout */
@@ -1769,7 +1744,7 @@ struct sharedObjectsStruct {
    *emptyarray, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr,
    *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr,
    *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr,
-    *busykeyerr, *oomerr, *overloaderr, *plus, *messagebulk, *pmessagebulk, *subscribebulk,
+    *busykeyerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk,
    *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *unlink,
    *rpop, *lpop, *lpush, *rpoplpush, *lmove, *blmove, *zpopmin, *zpopmax,
    *emptyscan, *multi, *exec, *left, *right, *hset, *srem, *xgroup, *xclaim,  
@@ -1895,7 +1870,7 @@ struct redisMaster {
    long long master_initial_offset;           /* Master PSYNC offset. */

    bool isActive = false;
-    bool isKeydbFastsync = false;
+    bool isRocksdbSnapshotRepl = false;
    int repl_state;          /* Replication status if the instance is a replica */
    off_t repl_transfer_size; /* Size of RDB to read from master during sync. */
    off_t repl_transfer_read; /* Amount of RDB read from master during sync. */
@@ -2232,8 +2207,6 @@ struct redisServerThreadVars {
    int propagate_in_transaction = 0;  /* Make sure we don't propagate nested MULTI/EXEC */
    int client_pause_in_transaction = 0; /* Was a client pause executed during this Exec? */
    std::vector<client*> vecclientsProcess;
-    std::unordered_set<client*> setclientsPrefetch;
-    std::unordered_set<StorageToken*> setStorageTokensProcess;
    dictAsyncRehashCtl *rehashCtl = nullptr;

    int getRdbKeySaveDelay();
@@ -2332,7 +2305,6 @@ struct redisServer {
    rax *errors;                /* Errors table */
    int activerehashing;        /* Incremental rehash in serverCron() */
    int active_defrag_running;  /* Active defragmentation running (holds current scan aggressiveness) */
-    int enable_async_rehash = 1;    /* Should we use the async rehash feature? */
    int cronloops;              /* Number of times the cron function run */
    char runid[CONFIG_RUN_ID_SIZE+1];  /* ID always different at every exec. */
    int sentinel_mode;          /* True if this instance is a Sentinel. */
@@ -2573,7 +2545,6 @@ struct redisServer {
    int repl_diskless_sync_delay;   /* Delay to start a diskless repl BGSAVE. */
    std::atomic <long long> repl_lowest_off; /* The lowest offset amongst all replicas
                                                -1 if there are no replicas */
-    long long repl_backlog_max_writes_per_event; /* Caps the output buffer sent to the replica at once */
    /* Replication (replica) */
    list *masters;
    int enable_multimaster; 
@@ -2715,7 +2686,6 @@ struct redisServer {

    int fActiveReplica;                          /* Can this replica also be a master? */
    int fWriteDuringActiveLoad;                  /* Can this active-replica write during an RDB load? */
-    int fEnableFastSync = false;

    // Format:
    //  Lower 20 bits: a counter incrementing for each command executed in the same millisecond
@@ -2723,8 +2693,6 @@ struct redisServer {
    uint64_t mvcc_tstamp;

    AsyncWorkQueue *asyncworkqueue;
-    AsyncWorkQueue *asyncreadworkqueue;
-    AsyncWorkQueue *asyncwriteworkqueue;

    /* System hardware info */
    size_t system_memory_size;  /* Total memory in system as reported by OS */
@@ -2743,95 +2711,7 @@ struct redisServer {
    int tls_rotation;

    std::set<sdsstring> tls_auditlog_blocklist; /* Certificates that can be excluded from audit logging */
-    std::set<sdsstring> tls_overload_ignorelist; /* Certificates that are be excluded load shedding */
    std::set<sdsstring> tls_allowlist;
-    class IPV4 {
-        struct in_addr m_ip;
-        struct in_addr m_mask;
-
-        int bitsFromMask() const {
-            uint32_t mask = ntohl(m_mask.s_addr);
-            int bits = 0;
-            while (mask > 0) {
-                bits += mask & 1;
-                mask >>= 1;
-            }
-            return bits;
-        }
-
-    public:
-        IPV4(struct in_addr ip, struct in_addr mask) : m_ip(ip), m_mask(mask) {};
-        bool match(struct in_addr ip) const
-        {
-            return (ip.s_addr & m_mask.s_addr) == m_ip.s_addr;
-        }
-
-        sds getString() const
-        {
-            sds result = sdsempty();
-            char buf[INET_ADDRSTRLEN];
-            inet_ntop(AF_INET, &m_ip, buf, sizeof(buf));
-            result = sdscat(result, buf);
-            int bits = bitsFromMask();
-            if (bits != IPV4_BITS) {
-                result = sdscat(result, "/");
-                result = sdscat(result, std::to_string(bits).c_str());
-            }
-            return result;
-        }
-
-        bool operator<(const IPV4& rhs) const
-        {
-            return memcmp(&m_ip, &rhs.m_ip, sizeof(m_ip)) < 0 || (memcmp(&m_ip, &rhs.m_ip, sizeof(m_ip)) == 0 && memcmp(&m_mask, &rhs.m_mask, sizeof(m_mask)) < 0);
-        }
-    };
-    class IPV6 {
-        struct in6_addr m_ip;
-        struct in6_addr m_mask;
-
-        int bitsFromMask() const {
-            int bits = 0;
-            for (unsigned int i = 0; i < sizeof(struct in6_addr); i++) {
-                uint8_t mask = m_mask.s6_addr[i];
-                while (mask > 0) {
-                    bits += mask & 1;
-                    mask >>= 1;
-                }
-            }
-            return bits;
-        }
-    public:
-        IPV6(struct in6_addr ip, struct in6_addr mask) : m_ip(ip), m_mask(mask) {};
-        bool match(struct in6_addr ip) const
-        {
-            for (unsigned int i = 0; i < sizeof(struct in6_addr); i++) {
-                if ((ip.s6_addr[i] & m_mask.s6_addr[i]) != m_ip.s6_addr[i])
-                    return false;
-            }
-            return true;
-        }
-
-        sds getString() const
-        {
-            sds result = sdsempty();
-            char buf[INET6_ADDRSTRLEN];
-            inet_ntop(AF_INET6, &m_ip, buf, sizeof(buf));
-            result = sdscat(result, buf);
-            int bits = bitsFromMask();
-            if (bits != IPV6_BITS) {
-                result = sdscat(result, "/");
-                result = sdscat(result, std::to_string(bits).c_str());
-            }
-            return result;
-        }
-
-        bool operator<(const IPV6& rhs) const
-        {
-            return memcmp(&m_ip, &rhs.m_ip, sizeof(m_ip)) < 0 || (memcmp(&m_ip, &rhs.m_ip, sizeof(m_ip)) == 0 && memcmp(&m_mask, &rhs.m_mask, sizeof(m_mask)) < 0);
-        }
-    };
-    std::set<IPV4> overload_ignorelist;
-    std::set<IPV6> overload_ignorelist_ipv6;
    redisTLSContextConfig tls_ctx_config;

    /* cpu affinity */
@@ -2874,14 +2754,12 @@ struct redisServer {

    sds sdsAvailabilityZone;
    int overload_protect_threshold = 0;
-    int overload_protect_tenacity = 0;
-    int overload_protect_strength = 0;
-    float last_overload_cpu_reading = 0.0f;
    int is_overloaded = 0;
+    int overload_closed_clients = 0;

-    int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a
-                                client blocked on a module command needs
-                                to be processed. */
+        int module_blocked_pipe[2]; /* Pipe used to awake the event loop if a
+                            client blocked on a module command needs
+                            to be processed. */

    bool FRdbSaveInProgress() const { return g_pserver->rdbThreadVars.fRdbThreadActive; }
 };
@@ -3039,7 +2917,7 @@ int moduleGetCommandKeysViaAPI(struct redisCommand *cmd, robj **argv, int argc,
 moduleType *moduleTypeLookupModuleByID(uint64_t id);
 void moduleTypeNameByID(char *name, uint64_t moduleid);
 const char *moduleTypeModuleName(moduleType *mt);
-void moduleFreeContext(struct RedisModuleCtx *ctx, bool propogate = true);
+void moduleFreeContext(struct RedisModuleCtx *ctx);
 void unblockClientFromModule(client *c);
 void moduleHandleBlockedClients(int iel);
 void moduleBlockedClientTimedOut(client *c);
@@ -3302,7 +3180,6 @@ void trimStringObjectIfNeeded(robj *o);
 robj *deserializeStoredObject(const void *data, size_t cb);
 std::unique_ptr<expireEntry> deserializeExpire(const char *str, size_t cch, size_t *poffset);
 sds serializeStoredObject(robj_roptr o, sds sdsPrefix = nullptr);
-sds serializeStoredObjectAndExpire(robj_roptr o);

 #define sdsEncodedObject(objptr) (objptr->encoding == OBJ_ENCODING_RAW || objptr->encoding == OBJ_ENCODING_EMBSTR)

@@ -4092,10 +3969,6 @@ inline int ielFromEventLoop(const aeEventLoop *eventLoop)
    return iel;
 }

-inline bool FFastSyncEnabled() {
-    return g_pserver->fEnableFastSync && !g_pserver->fActiveReplica;
-}
-
 inline int FCorrectThread(client *c)
 {
    return (c->conn == nullptr)
@@ -587,12 +587,8 @@ bool redisDbPersistentDataSnapshot::iterate_threadsafe_core(std::function<bool(c
                if (!fKeyOnly)
                {
                    size_t offset = 0;
-                    std::unique_ptr<expireEntry> spexpire = deserializeExpire((const char*)data, cbData, &offset);
+                    deserializeExpire((const char*)data, cbData, &offset);
                    o = deserializeStoredObject(reinterpret_cast<const char*>(data)+offset, cbData-offset);
-                    o->SetFExpires(spexpire != nullptr);
-                    if (spexpire != nullptr) {
-                        o->expire = std::move(*spexpire);
-                    }
                }
                fContinue = fn(sdsKey, o);
                if (o != nullptr)
@@ -25,7 +25,7 @@ bool FInternalKey(const char *key, size_t cch)
 std::string getPrefix(unsigned int hashslot)
 {
    char *hash_char = (char *)&hashslot;
-    return std::string(hash_char, 2);
+    return std::string(hash_char + (sizeof(unsigned int) - 2), 2);
 }

 std::string prefixKey(const char *key, size_t cchKey)
@@ -149,22 +149,11 @@ size_t RocksDBStorageProvider::clear()

    rocksdb::ColumnFamilyHandle *handle = nullptr;
    rocksdb::ColumnFamilyOptions cf_options(m_pfactory->RocksDbOptions());
-    cf_options.level_compaction_dynamic_level_bytes = true;
    m_spdb->CreateColumnFamily(cf_options, strName, &handle);
    m_spcolfamily = std::shared_ptr<rocksdb::ColumnFamilyHandle>(handle);

    if (!status.ok())
        throw status.ToString();
-   
-    status = m_spdb->DropColumnFamily(m_spexpirecolfamily.get());
-    strName = m_spexpirecolfamily->GetName();
-
-    m_spdb->CreateColumnFamily(cf_options, strName, &handle);
-    m_spexpirecolfamily = std::shared_ptr<rocksdb::ColumnFamilyHandle>(handle);
-
-    if (!status.ok())
-        throw status.ToString();
-
    m_count = 0;
    return celem;
 }
@@ -202,26 +191,23 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl
    std::string prefix = getPrefix(hashslot);
    std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(ReadOptions(), m_spcolfamily.get()));
    size_t count = 0;
-    serverAssert(prefix.size() >= 2);
-    bool full_iter = true;
-    for (it->Seek(rocksdb::Slice(prefix.data(), prefix.size())); it->Valid(); it->Next()) {
+    for (it->Seek(prefix.c_str()); it->Valid(); it->Next()) {
        if (FInternalKey(it->key().data(), it->key().size()))
            continue;
-        if (it->key().size() < 2 || memcmp(it->key().data(),prefix.data(),2) != 0)
+        if (strncmp(it->key().data(),prefix.c_str(),2) != 0)
            break;
        ++count;
        bool fContinue = fn(it->key().data()+2, it->key().size()-2, it->value().data(), it->value().size());
-        if (!fContinue) {
-            full_iter = false;
+        if (!fContinue)
            break;
-        }
    }
+    bool full_iter = !it->Valid() || (strncmp(it->key().data(),prefix.c_str(),2) != 0);
    if (full_iter && count != g_pserver->cluster->slots_keys_count[hashslot])
    {
-        serverLog(LL_WARNING, "WARNING: rocksdb hashslot %d count mismatch %zu vs expected %lu", hashslot, count, g_pserver->cluster->slots_keys_count[hashslot]);
+        printf("WARNING: rocksdb hashslot count mismatch");
    }
-    serverAssert(!full_iter || count == g_pserver->cluster->slots_keys_count[hashslot]);
-    serverAssert(it->status().ok()); // Check for any errors found during the scan
+    assert(!full_iter || count == g_pserver->cluster->slots_keys_count[hashslot]);
+    assert(it->status().ok()); // Check for any errors found during the scan
    return full_iter;
 }

@@ -256,14 +242,21 @@ void RocksDBStorageProvider::removeExpire(const char *key, size_t cchKey, long l
        throw status.ToString();
 }

-std::vector<std::string> RocksDBStorageProvider::getExpirationCandidates(unsigned int count)
+std::vector<std::string> RocksDBStorageProvider::getExpirationCandidates(unsigned int count, long long time)
 {
    std::vector<std::string> result;
    std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(ReadOptions(), m_spexpirecolfamily.get()));
-    for (it->SeekToFirst(); it->Valid() && result.size() < count; it->Next()) {
+    for (it->SeekToFirst(); it->Valid() && (*((long long *)it->key().data()) < time) && (result.size() < count); it->Next()) {
        if (FInternalKey(it->key().data(), it->key().size()))
            continue;
        result.emplace_back(it->value().data(), it->value().size());
+        rocksdb::Status status;
+        if (m_spbatch)
+            status = m_spbatch->Delete(m_spexpirecolfamily.get(), it->key());
+        else
+            status = m_spdb->Delete(WriteOptions(), m_spexpirecolfamily.get(), it->key());
+        if (!status.ok())
+            throw status.ToString();
    }
    return result;
 }
@@ -284,7 +277,7 @@ std::vector<std::string> RocksDBStorageProvider::getEvictionCandidates(unsigned
        }
    } else {
        std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(ReadOptions(), m_spexpirecolfamily.get()));
-        for (it->SeekToFirst(); it->Valid() && result.size() < count; it->Next()) {
+        for (it->SeekToFirst(); it->Valid() && result.size() < 16; it->Next()) {
            if (FInternalKey(it->key().data(), it->key().size()))
                continue;
            result.emplace_back(it->value().data(), it->value().size());
@@ -337,31 +330,6 @@ void RocksDBStorageProvider::endWriteBatch()
    m_lock.unlock();
 }

-struct BatchStorageToken : public StorageToken {
-    std::shared_ptr<rocksdb::DB> tspdb;    // Note: This must be first so it is deleted last
-    std::unique_ptr<rocksdb::WriteBatchWithIndex> tspbatch;
-};
-
-StorageToken* RocksDBStorageProvider::begin_endWriteBatch(struct aeEventLoop *el, aePostFunctionTokenProc* callback){
-    BatchStorageToken *tok = new BatchStorageToken();
-    tok->tspbatch = std::move(m_spbatch);
-    tok->tspdb = m_spdb;
-    m_spbatch = nullptr;
-    m_lock.unlock();
-    (*m_pfactory->m_wwqueue)->AddWorkFunction([this, el,callback,tok]{
-        tok->tspdb->Write(WriteOptions(),tok->tspbatch.get()->GetWriteBatch());
-        aePostFunction(el,callback,tok);
-    });
-
-    return tok;
-}
-
-void RocksDBStorageProvider::complete_endWriteBatch(StorageToken* tok){
-    delete tok;
-    tok = nullptr;
-}
-
-
 void RocksDBStorageProvider::batch_lock()
 {
    m_lock.lock();
@@ -386,63 +354,10 @@ bool RocksDBStorageProvider::FKeyExists(std::string& key) const
    return m_spdb->Get(ReadOptions(), m_spcolfamily.get(), rocksdb::Slice(key), &slice).ok();
 }

-struct RetrievalStorageToken : public StorageToken {
-    std::unordered_map<std::string, std::unique_ptr<rocksdb::PinnableSlice>> mapkeydata;
-};
-
-StorageToken *RocksDBStorageProvider::begin_retrieve(struct aeEventLoop *el, aePostFunctionTokenProc callback, sds *rgkey, size_t ckey) {
-    RetrievalStorageToken *tok = new RetrievalStorageToken();
-
-    for (size_t ikey = 0; ikey < ckey; ++ikey) {
-        tok->mapkeydata.insert(std::make_pair(std::string(rgkey[ikey], sdslen(rgkey[ikey])), nullptr));
-    }
-
-    auto opts = ReadOptions();
-    opts.async_io = true;
-    (*m_pfactory->m_rwqueue)->AddWorkFunction([this, el, callback, tok, opts]{
-        std::vector<std::string> veckeysStr;
-        std::vector<rocksdb::Slice> veckeys;
-        std::vector<rocksdb::PinnableSlice> vecvals;
-        std::vector<rocksdb::Status> vecstatus;
-        veckeys.reserve(tok->mapkeydata.size());
-        veckeysStr.reserve(tok->mapkeydata.size());
-        vecvals.resize(tok->mapkeydata.size());
-        vecstatus.resize(tok->mapkeydata.size());
-        for (auto &pair : tok->mapkeydata) {
-            veckeysStr.emplace_back(prefixKey(pair.first.data(), pair.first.size()));
-            veckeys.emplace_back(veckeysStr.back().data(), veckeysStr.back().size());
-        }
-
-        m_spdb->MultiGet(ReadOptions(),
-            m_spcolfamily.get(),
-            veckeys.size(), const_cast<const rocksdb::Slice*>(veckeys.data()),
-            vecvals.data(), vecstatus.data());
-
-        auto itrDst = tok->mapkeydata.begin();
-        for (size_t ires = 0; ires < veckeys.size(); ++ires) {
-            if (vecstatus[ires].ok()) {
-                itrDst->second = std::make_unique<rocksdb::PinnableSlice>(std::move(vecvals[ires]));
-            }
-            ++itrDst;
-        }
-        aePostFunction(el, callback, tok);
-    });
-    return tok;
-}
-
-void RocksDBStorageProvider::complete_retrieve(StorageToken *tok, callbackSingle fn) {
-    RetrievalStorageToken *rtok = reinterpret_cast<RetrievalStorageToken*>(tok);
-    for (auto &pair : rtok->mapkeydata) {
-        if (pair.second != nullptr) {
-            fn(pair.first.data(), pair.first.size(), pair.second->data(), pair.second->size());
-        }
-    }
-    delete rtok;
-}
 bool RocksDBStorageProvider::FExpireExists(std::string& key) const
 {
    rocksdb::PinnableSlice slice;
    if (m_spbatch)
        return m_spbatch->GetFromBatchAndDB(m_spdb.get(), ReadOptions(), m_spexpirecolfamily.get(), rocksdb::Slice(key), &slice).ok();
    return m_spdb->Get(ReadOptions(), m_spexpirecolfamily.get(), rocksdb::Slice(key), &slice).ok();
-}
+}
@@ -32,15 +32,11 @@ public:
    virtual void insert(const char *key, size_t cchKey, void *data, size_t cb, bool fOverwrite) override;
    virtual bool erase(const char *key, size_t cchKey) override;
    virtual void retrieve(const char *key, size_t cchKey, callbackSingle fn) const override;
-
-    virtual StorageToken *begin_retrieve(struct aeEventLoop *el, aePostFunctionTokenProc callback, sds *rgkey, size_t ckey);
-    virtual void complete_retrieve(StorageToken *tok, callbackSingle fn);
-
    virtual size_t clear() override;
    virtual bool enumerate(callback fn) const override;
    virtual bool enumerate_hashslot(callback fn, unsigned int hashslot) const override;

-    virtual std::vector<std::string> getExpirationCandidates(unsigned int count) override;
+    virtual std::vector<std::string> getExpirationCandidates(unsigned int count, long long time) override;
    virtual std::vector<std::string> getEvictionCandidates(unsigned int count) override;
    virtual void setExpire(const char *key, size_t cchKey, long long expire) override;
    virtual void removeExpire(const char *key, size_t cchKey, long long expire) override;
@@ -49,8 +45,6 @@ public:

    virtual void beginWriteBatch() override;
    virtual void endWriteBatch() override;
-    virtual StorageToken* begin_endWriteBatch(struct aeEventLoop *el, aePostFunctionTokenProc* proc);
-    virtual void complete_endWriteBatch(StorageToken *tok);

    virtual void bulkInsert(char **rgkeys, size_t *rgcbkeys, char **rgvals, size_t *rgcbvals, size_t celem) override;

@@ -1,6 +1,5 @@
 #pragma once
 #include "rocksdb.h"
-#include "../AsyncWorkQueue.h"

 class RocksDBStorageFactory : public IStorageFactory
 {
@@ -12,10 +11,7 @@ class RocksDBStorageFactory : public IStorageFactory
    bool m_fCreatedTempFolder = false;

 public:
-    AsyncWorkQueue **m_rwqueue;
-    AsyncWorkQueue **m_wwqueue;
-
-    RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig, AsyncWorkQueue **rwqueue, AsyncWorkQueue **wwqueue);
+    RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig);
    ~RocksDBStorageFactory();

    virtual IStorage *create(int db, key_load_iterator iter, void *privdata) override;
@@ -35,9 +35,9 @@ rocksdb::Options DefaultRocksDBOptions() {
    return options;
 }

-IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const char *rgchConfig, size_t cchConfig, AsyncWorkQueue **rwqueue, AsyncWorkQueue **wwqueue)
+IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const char *rgchConfig, size_t cchConfig)
 {
-    return new RocksDBStorageFactory(path, dbnum, rgchConfig, cchConfig, rwqueue, wwqueue);
+    return new RocksDBStorageFactory(path, dbnum, rgchConfig, cchConfig);
 }

 rocksdb::Options RocksDBStorageFactory::RocksDbOptions()
@@ -52,8 +52,8 @@ rocksdb::Options RocksDBStorageFactory::RocksDbOptions()
    return options;
 }

-RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig, AsyncWorkQueue **rwqueue, AsyncWorkQueue **wwqueue)
-    : m_path(dbfile), m_rwqueue(rwqueue), m_wwqueue(wwqueue)
+RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, const char *rgchConfig, size_t cchConfig)
+    : m_path(dbfile)
 {
    dbnum++; // create an extra db for metadata
    // Get the count of column families in the actual database
@@ -1,3 +1,3 @@
 #pragma once

-class IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const char *rgchConfig, size_t cchConfig, AsyncWorkQueue **rwqueue, AsyncWorkQueue **wwqueue);
+class IStorageFactory *CreateRocksDBStorageFactory(const char *path, int dbnum, const char *rgchConfig, size_t cchConfig);
@@ -29,7 +29,7 @@ public:
    virtual bool enumerate_hashslot(callback fn, unsigned int hashslot) const override;
    virtual size_t count() const override;

-    virtual std::vector<std::string> getExpirationCandidates(unsigned int) override { return std::vector<std::string>(); }
+    virtual std::vector<std::string> getExpirationCandidates(unsigned int, long long) override { return std::vector<std::string>(); }
    virtual std::vector<std::string> getEvictionCandidates(unsigned int) override { return std::vector<std::string>(); }
    virtual void setExpire(const char *, size_t, long long) override {}
    virtual void removeExpire(const char *, size_t, long long) override {}
@@ -555,6 +555,19 @@ void tlsSetCertificateFingerprint(tls_connection* conn, X509 * cert) {
 #define ASN1_STRING_get0_data ASN1_STRING_data 
 #endif 

+class TCleanup {
+    std::function<void()> fn;
+
+public:
+    TCleanup(std::function<void()> fn)
+        : fn(fn)
+    {}
+
+    ~TCleanup() {
+        fn();
+    }
+};
+
 bool tlsCheckCertificateAgainstAllowlist(tls_connection* conn, std::set<sdsstring> allowlist, const char** commonName){
    if (allowlist.empty()){
        // An empty list implies acceptance of all
@@ -627,18 +640,6 @@ bool tlsCheckCertificateAgainstAllowlist(tls_connection* conn, std::set<sdsstrin
    return false;
 }

-bool tlsCertificateIgnoreLoadShedding(tls_connection* conn) {
-    const char* cn = "";
-    if (tlsCheckCertificateAgainstAllowlist(conn, g_pserver->tls_overload_ignorelist, &cn)) {
-        // Certificate is in exclusion list, no need to audit log
-        serverLog(LL_NOTICE, "Loadshedding: disabled for %s", conn->c.fprint);
-        return true;
-    } else {
-        serverLog(LL_NOTICE, "Loadshedding: enabled for %s", conn->c.fprint);
-        return false;
-    }
-}
-
 bool tlsCertificateRequiresAuditLogging(tls_connection* conn){
    const char* cn = "";
    if (tlsCheckCertificateAgainstAllowlist(conn, g_pserver->tls_auditlog_blocklist, &cn)) {
@@ -891,9 +892,6 @@ void tlsHandleEvent(tls_connection *conn, int mask) {
                    if (tlsCertificateRequiresAuditLogging(conn)){
                        conn->c.flags |= CONN_FLAG_AUDIT_LOGGING_REQUIRED;
                    }
-                    if (tlsCertificateIgnoreLoadShedding(conn)){
-                        conn->c.flags |= CONN_FLAG_IGNORE_OVERLOAD;
-                    }
                }
            }

@@ -1071,12 +1069,6 @@ static int connTLSWrite(connection *conn_, const void *data, size_t data_len) {

    if (conn->c.state != CONN_STATE_CONNECTED) return -1;
    ERR_clear_error();
-
-     if (data_len > std::numeric_limits<int>::max()) {
-        // OpenSSL expects length to be 32-bit int
-        data_len = std::numeric_limits<int>::max();
-    }
-
    ret = SSL_write(conn->ssl, data, data_len);

    if (ret <= 0) {
@@ -10,16 +10,6 @@ source ../../support/cluster.tcl ; # Redis Cluster client.
 set ::instances_count 20 ; # How many instances we use at max.
 set ::tlsdir "../../tls"

-# Check if we compiled with flash
-set status [catch {exec ../../../src/keydb-server --is-flash-enabled}]
-if {$status == 0} {
-    puts "KeyDB was built with FLASH, including FLASH tests"
-    set ::flash_enabled 1
-} else {
-    puts "KeyDB was not built with FLASH.  Excluding FLASH tests"
-    set ::flash_enabled 0
-}
-
 proc main {} {
    parse_options
    spawn_instance redis $::redis_base_port $::instances_count {
@@ -14,7 +14,6 @@ test "Cluster is writable" {
    cluster_write_test 0
 }

-if {!$::flash_enabled} {
 test "Right to restore backups when fail to diskless load " {
    set master [Rn 0]
    set replica [Rn 1]
@@ -85,4 +84,3 @@ test "Right to restore backups when fail to diskless load " {
    assert_equal {1} [$replica get $slot0_key]
    assert_equal $slot0_key [$replica CLUSTER GETKEYSINSLOT 0 1] "POST RUN"
 }
-}
@@ -10,7 +10,7 @@
 # checked for consistency.
 if {$::flash_enabled} {
    proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
-        start_server [list tags {"repl"} overrides [list storage-provider {flash ./rocks.db.m} repl-backlog-size 64m]] {
+        start_server [list tags {"repl"} overrides [list storage-provider {flash .rocks.db.m} repl-backlog-size 64m]] {
            start_server [list tags {flash} overrides [list storage-provider {flash ./rocks.db} delete-on-evict no storage-flush-period 10]] {

                set master [srv -1 client]
@@ -114,10 +114,6 @@ if {$::flash_enabled} {

    foreach mdl {no yes} {
        foreach sdl {disabled swapdb} {
-            if {$::flash_enabled && $sdl == "swapdb"} {
-                continue
-            }
-
            test_psync {no reconnection, just sync} 6 1000000 3600 0 {
            } $mdl $sdl 0

@@ -128,10 +128,6 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reco

 foreach mdl {no yes} {
    foreach sdl {disabled swapdb} {
-        if {$::flash_enabled && $sdl == "swapdb"} {
-            continue
-        }
-
        test_psync {no reconnection, just sync} 6 1000000 3600 0 {
        } $mdl $sdl 0

@@ -119,10 +119,6 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reco

 foreach mdl {no yes} {
    foreach sdl {disabled swapdb} {
-        if {$::flash_enabled && $sdl == "swapdb"} {
-            continue
-        }
-
        test_psync {no reconnection, just sync} 6 1000000 3600 0 {
        } $mdl $sdl 0

@@ -252,10 +252,6 @@ start_server {tags {"repl"}} {

 foreach mdl {no yes} {
    foreach sdl {disabled swapdb} {
-        if {$::flash_enabled && $sdl == "swapdb"} {
-            # swapdb not compatible with flash
-            continue
-        }
        start_server {tags {"repl"}} {
            set master [srv 0 client]
            $master config set repl-diskless-sync $mdl
@@ -387,7 +383,6 @@ start_server {tags {"repl"}} {
    }
 }

-if {!$::flash_enabled} {
 test {slave fails full sync and diskless load swapdb recovers it} {
    start_server {tags {"repl"}} {
        set slave [srv 0 client]
@@ -552,7 +547,6 @@ test {diskless loading short read} {
        }
    }
 }
-}

 # get current stime and utime metrics for a thread (since it's creation)
 proc get_cpu_metrics { statfile } {
@@ -584,13 +578,12 @@ proc compute_cpu_usage {start end} {
    return [ list $pucpu $pscpu ]
 }

-if {!$::flash_enabled} {
+
 # test diskless rdb pipe with multiple replicas, which may drop half way
 start_server {tags {"repl"}} {
    set master [srv 0 client]
    $master config set repl-diskless-sync yes
    $master config set repl-diskless-sync-delay 1
-    $master config set enable-keydb-fastsync no
    set master_host [srv 0 host]
    set master_port [srv 0 port]
    set master_pid [srv 0 pid]
@@ -734,8 +727,6 @@ start_server {tags {"repl"}} {
            }
        }
    }
-
-    $master config set enable-keydb-fastsync yes
 }

 if 0 {
@@ -820,7 +811,6 @@ test "diskless replication read pipe cleanup" {
    }
 }
 }
-}

 test {replicaof right after disconnection} {
    # this is a rare race condition that was reproduced sporadically by the psync2 unit.
@@ -877,10 +867,8 @@ test {replicaof right after disconnection} {
 test {Kill rdb child process if its dumping RDB is not useful} {
    start_server {tags {"repl"}} {
        set slave1 [srv 0 client]
-        $slave1 config set enable-keydb-fastsync no
        start_server {} {
            set slave2 [srv 0 client]
-            $slave2 config set enable-keydb-fastsync no
            start_server {} {
                set master [srv 0 client]
                set master_host [srv 0 host]
@@ -81,6 +81,21 @@ if {$::flash_enabled} {
            }
        }

+        test { EXPIRE reduces used disk space } {
+            r flushall
+            r set testkey foo ex 10
+            r debug flush-storage
+            set used [r debug get-storage-usage]
+            assert_equal {1} [r dbsize]
+            wait_for_condition 50 1000 {
+                [r dbsize] == 0
+            } else {
+                fail "key is not expired"
+            }
+            set after [r debug get-storage-usage]
+            assert {$after < $used}
+        }
+
        test { SUBKEY EXPIRE persists after cache flush } {
            r flushall
            r sadd testkey foo bar baz
@@ -162,18 +162,15 @@ start_server {tags {"introspection"}} {
            aof_rewrite_cpulist
            time-thread-priority
            bgsave_cpulist
-            storage-cache-mode
-            storage-provider-options
-            use-fork
+	    storage-cache-mode
+	    storage-provider-options
+	    use-fork
            multi-master
            active-replica
            bind
            set-proc-title
            repl-backlog-disk-reserve
-            tls-allowlist
-            tls-auditlog-blocklist
-            tls-overload-ignorelist
-            overload-ignorelist
+	    tls-allowlist
            db-s3-object
        }

@@ -182,8 +182,6 @@ proc test_slave_buffers {test_name cmd_count payload_len limit_memory pipeline}
                fail "Replication not started."
            }

-            after 100
-
            # measure used memory after the slave connected and set maxmemory
            set orig_used [s -1 used_memory]
            set orig_client_buf [s -1 mem_clients_normal]
@@ -1,7 +1,7 @@
 set testmodule [file normalize tests/modules/load.so]

 if {$::flash_enabled} {
-    start_server [list tags [list "modules"] overrides [list storage-provider {flash ./rocks.db.master.load.test} loadmodule $testmodule]] {
+    start_server [list tags [list "modules"] overrides [list storage-provider {flash ./rocks.db.master.load.test} databases 256 loadmodule $testmodule]] {
        test "Module is notified of keys loaded from flash" {
            r flushall
            r set foo bar
@@ -51,7 +51,7 @@ if {$::tls} {
        }
    }

-    test {TLS Audit Log: Able to connect while with SAN having a comprehensive list} {
+    test {TLS: Able to connect while with SAN having a comprehensive list} {
        start_server {tags {"tls"} overrides {tls-auditlog-blocklist {dummy.keydb.dev san2.keydb.dev other.keydb.dev}}} {
            catch {r PING} e
            assert_match {PONG} $e
@@ -65,7 +65,7 @@ if {$::tls} {
        }
    }

-    test {TLS Audit Log: Able to connect while with SAN having a comprehensive list with wildcards} {
+    test {TLS Audit LogTLS: Able to connect while with SAN having a comprehensive list with wildcards} {
        start_server {tags {"tls"} overrides {tls-auditlog-blocklist {dummy.* san*.dev other.*}}} {
            catch {r PING} e
            assert_match {PONG} $e
Author	SHA1	Message	Date
Malavan Sotheeswaran	717805e191	init iteration	2023-08-22 22:48:26 -07:00
Malavan Sotheeswaran	8f8b61f211	compile fix	2023-08-22 13:18:14 -07:00
Malavan Sotheeswaran	fca3c77823	somehow missed reference to m_setexpire	2023-08-22 13:03:03 -07:00
Malavan Sotheeswaran	22a0ad9a23	Merge remote-tracking branch 'internal/main' into flash_expiration	2023-08-22 12:48:28 -07:00
Malavan Sotheeswaran	747eb8067e	disk space test	2023-06-29 21:50:05 -07:00
Malavan Sotheeswaran	1d547f89e1	1 second between each check	2023-06-28 17:44:36 -07:00
Malavan Sotheeswaran	9281b728d5	use dedicated db for load test to avoid loading other keys	2023-06-28 15:50:24 -07:00
Malavan Sotheeswaran	04cc2c01c6	change expire test to wait_for_condition	2023-06-28 00:27:51 -07:00
Malavan Sotheeswaran	f681ca1a9f	Merge remote-tracking branch 'internal/main' into flash_expiration	2023-06-27 15:25:47 -07:00
Malavan Sotheeswaran	336202eba5	strict <	2023-06-27 11:43:13 -07:00
Malavan Sotheeswaran	597dd12498	flash eviction test working	2023-06-27 00:06:32 -07:00
Malavan Sotheeswaran	7f86f4f876	WIP flash eviction tests	2023-06-26 22:06:38 -07:00
Malavan Sotheeswaran	ea7838eb46	fix test	2023-06-26 21:35:09 -07:00
Malavan Sotheeswaran	eb2018a423	reject versions before 6.3.4	2023-06-23 20:33:45 -07:00
Malavan Sotheeswaran	0eb118731e	fix subkey exprires	2023-06-23 18:31:19 -07:00
Malavan Sotheeswaran	a0a264a11c	figured out the bug!	2023-06-23 18:20:36 -07:00
Malavan Sotheeswaran	79b3490f49	fix warnings	2023-06-23 16:52:46 -07:00
Malavan Sotheeswaran	178df4ddc6	respect sample count	2023-06-23 16:32:50 -07:00
Malavan Sotheeswaran	6504e7965a	bake expire modifications into storage cache insert and erase	2023-06-23 12:21:47 -07:00
Malavan Sotheeswaran	7a9d4bbc31	some refactoring to avoid duplicate code	2023-06-23 12:03:08 -07:00
Malavan Sotheeswaran	0c0f8581f5	brackets to avoid ambiguos else	2023-06-21 19:15:11 -07:00
Malavan Sotheeswaran	967bd78658	only delete if exists	2023-06-21 19:12:38 -07:00
Malavan Sotheeswaran	605f369d65	remove expires from storage provider when removed from memory	2023-06-21 18:34:49 -07:00
Malavan Sotheeswaran	c6871a3179	run tests with flash	2023-06-21 16:36:47 -07:00
Malavan Sotheeswaran	f0d9466561	missing nullptr check	2023-06-14 15:11:27 -07:00
Malavan Sotheeswaran	ec3b1eb5c4	handle all keys case	2023-06-14 15:07:50 -07:00
Malavan Sotheeswaran	e248fec27f	for all keys we need to remove the prefix	2023-06-14 14:20:17 -07:00
Malavan Sotheeswaran	eea8932aa7	unit test doesn't make sense	2023-06-13 23:34:59 -07:00
Malavan Sotheeswaran	98eaebd4c8	accurate config update message	2023-06-13 22:52:26 -07:00
Malavan Sotheeswaran	8e41dc3ed2	fix spacing	2023-06-13 19:19:48 -07:00
Malavan Sotheeswaran	e1afce8a4a	remove extra blank line	2023-06-13 19:19:04 -07:00
Malavan Sotheeswaran	63dedf054f	add unit test for flash eviction	2023-06-13 19:12:26 -07:00
Malavan Sotheeswaran	5f1884fc36	include vector	2023-06-12 22:55:36 -07:00
Malavan Sotheeswaran	e54fd922d9	update expire column during fast sync	2023-06-12 21:51:40 -07:00
Malavan Sotheeswaran	b81ba8bfc9	Merge branch 'main' into flash_expiration	2023-06-12 15:09:37 -07:00
Malavan Sotheeswaran	f737084170	mstime not ustime	2023-05-26 03:05:29 -07:00
Malavan Sotheeswaran	49f48d808d	double check expire time	2023-05-26 03:00:15 -07:00
Malavan Sotheeswaran	71045949c2	fix include and crash	2023-05-26 02:42:19 -07:00
Malavan Sotheeswaran	ef1483e1d4	fix crash	2023-05-25 14:01:19 -07:00
Malavan Sotheeswaran	708b8d941b	actually add expires to rocksdb	2023-05-25 00:17:12 -07:00
Malavan Sotheeswaran	12fd46da0b	add ability to set expires in storge provider	2023-05-24 21:57:12 -07:00
Malavan Sotheeswaran	5aff96c577	fixes	2023-05-24 15:07:15 -07:00
Malavan Sotheeswaran	5cb2ead650	refactor for time limited expire cycle	2023-05-24 14:55:41 -07:00
Malavan Sotheeswaran	6853b15d50	add random hashslot impl	2023-05-23 21:26:01 -07:00
Malavan Sotheeswaran	f701161d0a	more progress	2023-05-23 21:10:40 -07:00
Malavan Sotheeswaran	6a32023cfc	use correct key variable	2023-05-23 21:05:23 -07:00
Malavan Sotheeswaran	6eb595d6b4	remove expire from memory when evicting from cache	2023-05-23 20:50:58 -07:00
Malavan Sotheeswaran	edae100767	refactor towards new design	2023-05-23 20:47:15 -07:00
Malavan Sotheeswaran	90fc43b849	add get expire and get eviction candidate impls for rocksdb	2023-05-18 10:19:49 -07:00
Malavan Sotheeswaran	d4857720f2	adding KeyDB side of flash expiration/eviction	2023-05-17 13:46:43 -07:00
Malavan Sotheeswaran	cdfca6a42b	add machamp script	2023-05-11 21:20:26 -07:00
Malavan Sotheeswaran	bae5c0dad1	switch machamp to main branch	2023-05-11 21:06:31 -07:00
Malavan Sotheeswaran	8b5fced206	add machamp	2023-05-11 21:05:59 -07:00
Malavan Sotheeswaran	e6c0e14cdd	add fixed length prefix extractor	2023-05-11 12:46:47 -07:00
Malavan Sotheeswaran	e47624dc8e	add option to show logs of 1 server at a time	2023-05-10 22:21:27 -07:00
Malavan Sotheeswaran	76e103b9bf	fix warnings	2023-05-10 15:25:39 -07:00
Malavan Sotheeswaran	ac1bd8eec4	add assert to ensure correctness	2023-05-10 14:19:17 -07:00
Malavan Sotheeswaran	9bfb89bfd8	forgot ;	2023-05-10 14:16:18 -07:00
Malavan Sotheeswaran	eb565ae8f7	check for count mismatch	2023-05-10 14:12:19 -07:00
Malavan Sotheeswaran	bc37934f6f	forgot return statement in getKeysInSlot	2023-05-09 21:10:35 -07:00
Malavan Sotheeswaran	e0007d9687	add --flash option for tests	2023-05-09 21:07:54 -07:00
Malavan Sotheeswaran	e52792fe2a	add hashslot enum to test storage provider	2023-05-09 13:52:37 -07:00
Malavan Sotheeswaran	880ca46788	fix compile	2023-05-09 13:49:19 -07:00
Malavan Sotheeswaran	6e1d6146e0	implement slot to keys with rocksdb	2023-05-09 12:36:48 -07:00
Malavan Sotheeswaran	ce248d7b66	don't prefix internal keys	2023-05-09 11:30:08 -07:00
Malavan Sotheeswaran	4bdd2ed5f2	remove hashslot prefix when loading from rocksdb	2023-05-08 21:53:56 -07:00
Malavan Sotheeswaran	e7ba8452bd	Merge remote-tracking branch 'origin/main' into flash_slot_to_key	2023-05-08 13:03:48 -07:00
Malavan Sotheeswaran	b9f049a25d	don't need to offset internal keys	2023-05-05 15:45:09 -07:00
Malavan Sotheeswaran	e69bfd5da6	add hash prefixing to rocksdb	2023-05-05 12:22:29 -07:00