Fix TSAN warnings on the repl backlog

.github/workflows/ci.yml

@@ -5,14 +5,17 @@ on: [push, pull_request]
 jobs:
 
   test-ubuntu-latest:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v1
+      with:
+        submodules: true
     - name: make
       run: |
           sudo apt-get update
-          sudo apt-get -y install uuid-dev libcurl4-openssl-dev
-          make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' BUILD_TLS=yes -j2
+          sudo apt-get -y remove libzstd || true
+          sudo apt-get -y install uuid-dev libcurl4-openssl-dev libbz2-dev zlib1g-dev libsnappy-dev liblz4-dev libzstd-dev libgflags-dev
+          make BUILD_TLS=yes -j2 KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror'
     - name: gen-cert
       run: ./utils/gen-test-certs.sh
     - name: test-tls
@@ -28,6 +31,9 @@ jobs:
     - name: module tests
       run: |
           ./runtest-moduleapi
+    - name: rotation test
+      run: |
+          ./runtest-rotation
         
   build-ubuntu-old:
     runs-on: ubuntu-18.04
@@ -38,6 +44,7 @@ jobs:
         sudo apt-get update
         sudo apt-get -y install uuid-dev libcurl4-openssl-dev
         make -j2
+
   build-macos-latest:
     runs-on: macos-latest
     steps:
@@ -49,18 +56,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: make
-      run: |
-          sudo apt-get update
-          sudo apt-get -y install uuid-dev libcurl4-openssl-dev
-          make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' MALLOC=libc -j2
-  build-ubuntu-32bit:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-    - name: make
-      run: |
-          sudo dpkg --add-architecture i386
-          sudo apt-get update
-          sudo apt-get -y install gcc-multilib g++-multilib libc6-dev-i386 lib32z1 uuid-dev:i386 libcurl4-openssl-dev:i386
-          make KEYDB_CFLAGS='-Werror' KEYDB_CXXFLAGS='-Werror' 32bit -j2
+    - name: make

README.md

@@ -99,7 +99,7 @@ Avoid forwarding RREPLAY messages to other masters? WARNING: This setting is dan
 
 
 ```
-    db-s3-object /path/to/bucket
+    scratch-file-path /path
 ```
 If you would like KeyDB to dump and load directly to AWS S3 this option specifies the bucket.  Using this option with the traditional RDB options will result in KeyDB backing up twice to both locations.  If both are specified KeyDB will first attempt to load from the local dump file and if that fails load from S3.  This requires the AWS CLI tools to be installed and configured which are used under the hood to transfer the data.
 

src/db.cpp

@@ -2669,11 +2669,6 @@ void redisDbPersistentData::prepOverwriteForSnapshot(char *key)
         auto itr = m_pdbSnapshot->find_cached_threadsafe(key);
         if (itr.key() != nullptr)
         {
-            if (itr.val()->FExpires()) {
-                // Note: I'm sure we could handle this, but its too risky at the moment.
-                //  There are known bugs doing this with expires
-                return;
-            }
             sds keyNew = sdsdupshared(itr.key());
             if (dictAdd(m_pdictTombstone, keyNew, (void*)dictHashKey(m_pdict, key)) != DICT_OK)
                 sdsfree(keyNew);

src/gc.h

@@ -2,7 +2,6 @@
 #include <vector>
 #include <assert.h>
 #include <unordered_set>
-#include <list>
 
 struct ICollectable
 {
@@ -46,14 +45,14 @@ public:
     void shutdown()
     {
         std::unique_lock<fastlock> lock(m_lock);
-        m_listepochs.clear();
+        m_vecepochs.clear();
         m_setepochOutstanding.clear();
     }
 
     bool empty() const
     {
         std::unique_lock<fastlock> lock(m_lock);
-        return m_listepochs.empty();
+        return m_vecepochs.empty();
     }
 
     void endEpoch(uint64_t epoch, bool fNoFree = false)
@@ -64,12 +63,12 @@ public:
         m_setepochOutstanding.erase(epoch);
         if (fNoFree)
             return;
-        std::list<EpochHolder> listclean;
+        std::vector<EpochHolder> vecclean;
         
         // No outstanding epochs?
         if (m_setepochOutstanding.empty())
         {
-            listclean = std::move(m_listepochs);  // Everything goes!
+            vecclean = std::move(m_vecepochs);  // Everything goes!
         }
         else
         {
@@ -78,20 +77,18 @@ public:
                 return; // No available epochs to free
 
             // Clean any epochs available (after the lock)
-            for (auto itr = m_listepochs.begin(); itr != m_listepochs.end(); /* itr incremented in loop*/)
+            for (size_t iepoch = 0; iepoch < m_vecepochs.size(); ++iepoch)
             {
-                auto &e = *itr;
-                auto itrNext = itr;
-                ++itrNext;
+                auto &e = m_vecepochs[iepoch];
                 if (e < minepoch)
                 {
-                    listclean.emplace_back(std::move(e));
-                    m_listepochs.erase(itr);
+                    vecclean.emplace_back(std::move(e));
+                    m_vecepochs.erase(m_vecepochs.begin() + iepoch);
+                    --iepoch;
                 }
-                itr = itrNext;
             }
 
-            assert(listclean.empty() || fMinElement);
+            assert(vecclean.empty() || fMinElement);
         }
 
         lock.unlock();  // don't hold it for the potentially long delete of vecclean
@@ -103,13 +100,13 @@ public:
         serverAssert(m_setepochOutstanding.find(epoch) != m_setepochOutstanding.end());
         serverAssert(sp->FWillFreeChildDebug() == false);
 
-        auto itr = std::find(m_listepochs.begin(), m_listepochs.end(), m_epochNext+1);
-        if (itr == m_listepochs.end())
+        auto itr = std::find(m_vecepochs.begin(), m_vecepochs.end(), m_epochNext+1);
+        if (itr == m_vecepochs.end())
         {
             EpochHolder e;
             e.tstamp = m_epochNext+1;
             e.m_vecObjs.push_back(std::move(sp));
-            m_listepochs.emplace_back(std::move(e));
+            m_vecepochs.emplace_back(std::move(e));
         }
         else
         {
@@ -120,7 +117,7 @@ public:
 private:
     mutable fastlock m_lock { "Garbage Collector"};
 
-    std::list<EpochHolder> m_listepochs;
+    std::vector<EpochHolder> m_vecepochs;
     std::unordered_set<uint64_t> m_setepochOutstanding;
     uint64_t m_epochNext = 0;
 };

src/networking.cpp

@@ -2743,10 +2743,9 @@ void readQueryFromClient(connection *conn) {
         parseClientCommandBuffer(c);
         if (g_pserver->enable_async_commands && !serverTL->disable_async_commands && listLength(g_pserver->monitors) == 0 && (aeLockContention() || serverTL->rgdbSnapshot[c->db->id] || g_fTestMode)) {
             // Frequent writers aren't good candidates for this optimization, they cause us to renew the snapshot too often
-            //  so we exclude them unless the snapshot we need already exists.
-            // Note: In test mode we want to create snapshots as often as possibl to excercise them - we don't care about perf
+            //  so we exclude them unless the snapshot we need already exists
             bool fSnapshotExists = c->db->mvccLastSnapshot >= c->mvccCheckpoint;
-            bool fWriteTooRecent = !g_fTestMode && (((getMvccTstamp() - c->mvccCheckpoint) >> MVCC_MS_SHIFT) < static_cast<uint64_t>(g_pserver->snapshot_slip)/2);
+            bool fWriteTooRecent = (((getMvccTstamp() - c->mvccCheckpoint) >> MVCC_MS_SHIFT) < static_cast<uint64_t>(g_pserver->snapshot_slip)/2);
 
             // The check below avoids running async commands if this is a frequent writer unless a snapshot is already there to service it
             if (!fWriteTooRecent || fSnapshotExists) {

src/rdb.cpp

@@ -1657,18 +1657,13 @@ int launchRdbSaveThread(pthread_t &child, rdbSaveInfo *rsi)
 
         g_pserver->rdbThreadVars.tmpfileNum++;
         g_pserver->rdbThreadVars.fRdbThreadCancel = false;
-        pthread_attr_t tattr;
-        pthread_attr_init(&tattr);
-        pthread_attr_setstacksize(&tattr, 1 << 23); // 8 MB
-        if (pthread_create(&child, &tattr, rdbSaveThread, args)) {
-            pthread_attr_destroy(&tattr);
+        if (pthread_create(&child, NULL, rdbSaveThread, args)) {
             for (int idb = 0; idb < cserver.dbnum; ++idb)
                 g_pserver->db[idb]->endSnapshot(args->rgpdb[idb]);
             args->~rdbSaveThreadArgs();
             zfree(args);
             return C_ERR;
         }
-        pthread_attr_destroy(&tattr);
         g_pserver->child_type = CHILD_TYPE_RDB;
     }
     return C_OK;
@@ -3839,11 +3834,7 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
 
     g_pserver->rdbThreadVars.tmpfileNum++;
     g_pserver->rdbThreadVars.fRdbThreadCancel = false;
-    pthread_attr_t tattr;
-    pthread_attr_init(&tattr);
-    pthread_attr_setstacksize(&tattr, 1 << 23); // 8 MB
-    if (pthread_create(&child, &tattr, rdbSaveToSlavesSocketsThread, args)) {
-        pthread_attr_destroy(&tattr);
+    if (pthread_create(&child, nullptr, rdbSaveToSlavesSocketsThread, args)) {
         serverLog(LL_WARNING,"Can't save in background: fork: %s",
             strerror(errno));
 
@@ -3869,7 +3860,6 @@ int rdbSaveToSlavesSockets(rdbSaveInfo *rsi) {
         closeChildInfoPipe();
         return C_ERR;
     }
-    pthread_attr_destroy(&tattr);
     g_pserver->child_type = CHILD_TYPE_RDB;
 
     serverLog(LL_NOTICE,"Background RDB transfer started");

utils/redis-copy.rb

@@ -1,27 +1,27 @@
-# redis-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo
+# keydb-copy.rb - Copyright (C) 2009-2010 Salvatore Sanfilippo
 # BSD license, See the COPYING file for more information.
 #
-# Copy the whole dataset from one KeyDB/Redis instance to another one
+# Copy the whole dataset from one Redis instance to another one
 #
 # WARNING: this utility is deprecated and serves as a legacy adapter
-#          for the more-robust redis-copy gem.
+#          for the more-robust keydb-copy gem.
 
 require 'shellwords'
 
 def redisCopy(opts={})
   src = "#{opts[:srchost]}:#{opts[:srcport]}"
   dst = "#{opts[:dsthost]}:#{opts[:dstport]}"
-  `redis-copy #{src.shellescape} #{dst.shellescape}`
+  `keydb-copy #{src.shellescape} #{dst.shellescape}`
 rescue Errno::ENOENT
-  $stderr.puts 'This utility requires the redis-copy executable',
-               'from the redis-copy gem on https://rubygems.org',
-               'To install it, run `gem install redis-copy`.'
+  $stderr.puts 'This utility requires the keydb-copy executable',
+               'from the keydb-copy gem on https://rubygems.org',
+               'To install it, run `gem install keydb-copy`.'
   exit 1
 end
 
-$stderr.puts "This utility is deprecated. Use the redis-copy gem instead."
+$stderr.puts "This utility is deprecated. Use the keydb-copy gem instead."
 if ARGV.length != 4
-    puts "Usage: redis-copy.rb <srchost> <srcport> <dsthost> <dstport>"
+    puts "Usage: keydb-copy.rb <srchost> <srcport> <dsthost> <dstport>"
     exit 1
 end
 puts "WARNING: it's up to you to FLUSHDB the destination host before to continue, press any key when ready."