Merge remote-tracking branch 'redis/6.0' into redis_merge

Former-commit-id: ef9a3cadcf

utils/create-cluster/README

@@ -15,8 +15,13 @@ To create a cluster, follow these steps:
 1. Edit create-cluster and change the start / end port, depending on the
 number of instances you want to create.
 2. Use "./create-cluster start" in order to run the instances.
+<<<<<<< HEAD
 3. Use "./create-cluster create" in order to execute keydb-cli --cluster create, so that
 an actual Redis cluster will be created.
+=======
+3. Use "./create-cluster create" in order to execute redis-cli --cluster create, so that
+an actual Redis cluster will be created. (If you're accessing your setup via a local container, ensure that the CLUSTER_HOST value is changed to your local IP)
+>>>>>>> redis/6.0
 4. Now you are ready to play with the cluster. AOF files and logs for each instances are created in the current directory.
 
 In order to stop a cluster:

utils/create-cluster/create-cluster

@@ -1,10 +1,13 @@
 #!/bin/bash
 
 # Settings
+CLUSTER_HOST=127.0.0.1
 PORT=30000
 TIMEOUT=2000
 NODES=6
 REPLICAS=1
+PROTECTED_MODE=yes
+ADDITIONAL_OPTIONS=""
 
 # You may want to put the above config parameters into config.sh in order to
 # override the defaults without modifying this script.
@@ -22,7 +25,11 @@ then
     while [ $((PORT < ENDPORT)) != "0" ]; do
         PORT=$((PORT+1))
         echo "Starting $PORT"
+<<<<<<< HEAD
         ../../src/keydb-server --port $PORT --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes
+=======
+        ../../src/redis-server --port $PORT  --protected-mode $PROTECTED_MODE --cluster-enabled yes --cluster-config-file nodes-${PORT}.conf --cluster-node-timeout $TIMEOUT --appendonly yes --appendfilename appendonly-${PORT}.aof --dbfilename dump-${PORT}.rdb --logfile ${PORT}.log --daemonize yes ${ADDITIONAL_OPTIONS}
+>>>>>>> redis/6.0
     done
     exit 0
 fi
@@ -32,7 +39,7 @@ then
     HOSTS=""
     while [ $((PORT < ENDPORT)) != "0" ]; do
         PORT=$((PORT+1))
-        HOSTS="$HOSTS 127.0.0.1:$PORT"
+        HOSTS="$HOSTS $CLUSTER_HOST:$PORT"
     done
     ../../src/keydb-cli --cluster create $HOSTS --cluster-replicas $REPLICAS
     exit 0
@@ -68,6 +75,12 @@ then
     exit 0
 fi
 
+if [ "$1" == "tailall" ]
+then
+    tail -f *.log
+    exit 0
+fi
+
 if [ "$1" == "call" ]
 then
     while [ $((PORT < ENDPORT)) != "0" ]; do
@@ -98,5 +111,6 @@ echo "create      -- Create a cluster using keydb-cli --cluster create."
 echo "stop        -- Stop Redis Cluster instances."
 echo "watch       -- Show CLUSTER NODES output (first 30 lines) of first node."
 echo "tail <id>   -- Run tail -f of instance at base port + ID."
+echo "tailall     -- Run tail -f for all the log files at once."
 echo "clean       -- Remove all instances data, logs, configs."
 echo "clean-logs  -- Remove just instances logs."

utils/gen-test-certs.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+mkdir -p tests/tls
+openssl genrsa -out tests/tls/ca.key 4096
+openssl req \
+    -x509 -new -nodes -sha256 \
+    -key tests/tls/ca.key \
+    -days 3650 \
+    -subj '/O=Redis Test/CN=Certificate Authority' \
+    -out tests/tls/ca.crt
+openssl genrsa -out tests/tls/redis.key 2048
+openssl req \
+    -new -sha256 \
+    -key tests/tls/redis.key \
+    -subj '/O=Redis Test/CN=Server' | \
+    openssl x509 \
+        -req -sha256 \
+        -CA tests/tls/ca.crt \
+        -CAkey tests/tls/ca.key \
+        -CAserial tests/tls/ca.txt \
+        -CAcreateserial \
+        -days 365 \
+        -out tests/tls/redis.crt
+openssl dhparam -out tests/tls/redis.dh 2048

utils/install_server.sh

@@ -73,6 +73,16 @@ if [ "$(id -u)" -ne 0 ] ; then
 	exit 1
 fi
 
+#bail if this system is managed by systemd
+_pid_1_exe="$(readlink -f /proc/1/exe)"
+if [ "${_pid_1_exe##*/}" = systemd ]
+then
+	echo "This systems seems to use systemd."
+	echo "Please take a look at the provided example service unit files in this directory, and adapt and install them. Sorry!"
+	exit 1
+fi
+unset _pid_1_exe
+
 if ! echo $REDIS_PORT | egrep -q '^[0-9]+$' ; then
 	_MANUAL_EXECUTION=true
 	#Read the redis port

utils/systemd-redis_multiple_servers@.service

@@ -0,0 +1,37 @@
+# example systemd template service unit file for multiple redis-servers
+#
+# You can use this file as a blueprint for your actual template service unit
+# file, if you intend to run multiple independent redis-server instances in
+# parallel using systemd's "template unit files" feature. If you do, you will
+# want to choose a better basename for your service unit by renaming this file
+# when copying it.
+#
+# Please take a look at the provided "systemd-redis_server.service" example
+# service unit file, too, if you choose to use this approach at managing
+# multiple redis-server instances via systemd.
+
+[Unit]
+Description=Redis data structure server - instance %i
+Documentation=https://redis.io/documentation
+# This template unit assumes your redis-server configuration file(s)
+# to live at /etc/redis/redis_server_<INSTANCE_NAME>.conf
+AssertPathExists=/etc/redis/redis_server_%i.conf
+#Before=your_application.service another_example_application.service
+#AssertPathExists=/var/lib/redis
+
+[Service]
+ExecStart=/usr/local/bin/redis-server /etc/redis/redis_server_%i.conf
+LimitNOFILE=10032
+NoNewPrivileges=yes
+#OOMScoreAdjust=-900
+#PrivateTmp=yes
+Type=notify
+TimeoutStartSec=infinity
+TimeoutStopSec=infinity
+UMask=0077
+#User=redis
+#Group=redis
+#WorkingDirectory=/var/lib/redis
+
+[Install]
+WantedBy=multi-user.target

utils/systemd-redis_server.service

@@ -0,0 +1,41 @@
+# example systemd service unit file for redis-server
+#
+# In order to use this as a template for providing a redis service in your
+# environment, _at the very least_ make sure to adapt the redis configuration
+# file you intend to use as needed (make sure to set "supervised systemd"), and
+# to set sane TimeoutStartSec and TimeoutStopSec property values in the unit's
+# "[Service]" section to fit your needs.
+#
+# Some properties, such as User= and Group=, are highly desirable for virtually
+# all deployments of redis, but cannot be provided in a manner that fits all
+# expectable environments. Some of these properties have been commented out in
+# this example service unit file, but you are highly encouraged to set them to
+# fit your needs.
+#
+# Please refer to systemd.unit(5), systemd.service(5), and systemd.exec(5) for
+# more information.
+
+[Unit]
+Description=Redis data structure server
+Documentation=https://redis.io/documentation
+#Before=your_application.service another_example_application.service
+#AssertPathExists=/var/lib/redis
+
+[Service]
+ExecStart=/usr/local/bin/redis-server --supervised systemd --daemonize no
+## Alternatively, have redis-server load a configuration file:
+#ExecStart=/usr/local/bin/redis-server /path/to/your/redis.conf
+LimitNOFILE=10032
+NoNewPrivileges=yes
+#OOMScoreAdjust=-900
+#PrivateTmp=yes
+Type=notify
+TimeoutStartSec=infinity
+TimeoutStopSec=infinity
+UMask=0077
+#User=redis
+#Group=redis
+#WorkingDirectory=/var/lib/redis
+
+[Install]
+WantedBy=multi-user.target

utils/tracking_collisions.c

@@ -0,0 +1,76 @@
+/* This is a small program used in order to understand the collison rate
+ * of CRC64 (ISO version) VS other stronger hashing functions in the context
+ * of hashing keys for the Redis "tracking" feature (client side caching
+ * assisted by the server).
+ *
+ * The program attempts to hash keys with common names in the form of
+ *
+ *  prefix:<counter>
+ *
+ * And counts the resulting collisons generated in the 24 bits of output
+ * needed for the tracking feature invalidation table (16 millions + entries)
+ *
+ * Compile with:
+ *
+ *  cc -O2 ./tracking_collisions.c ../src/crc64.c ../src/sha1.c
+ *  ./a.out
+ *
+ * --------------------------------------------------------------------------
+ *
+ * Copyright (C) 2019 Salvatore Sanfilippo
+ * This code is released under the BSD 2 clause license.
+ */
+
+#include <stdlib.h>
+#include <stdint.h>
+#include <string.h>
+#include <stdio.h>
+#include "../src/crc64.h"
+#include "../src/sha1.h"
+
+#define TABLE_SIZE (1<<24)
+int Table[TABLE_SIZE];
+
+uint64_t crc64Hash(char *key, size_t len) {
+    return crc64(0,(unsigned char*)key,len);
+}
+
+uint64_t sha1Hash(char *key, size_t len) {
+    SHA1_CTX ctx;
+    unsigned char hash[20];
+
+    SHA1Init(&ctx);
+    SHA1Update(&ctx,(unsigned char*)key,len);
+    SHA1Final(hash,&ctx);
+    uint64_t hash64;
+    memcpy(&hash64,hash,sizeof(hash64));
+    return hash64;
+}
+
+/* Test the hashing function provided as callback and return the
+ * number of collisions found. */
+unsigned long testHashingFunction(uint64_t (*hash)(char *, size_t)) {
+    unsigned long collisions = 0;
+    memset(Table,0,sizeof(Table));
+    char *prefixes[] = {"object", "message", "user", NULL};
+    for (int i = 0; prefixes[i] != NULL; i++) {
+        for (int j = 0; j < TABLE_SIZE/2; j++) {
+            char keyname[128];
+            size_t keylen = snprintf(keyname,sizeof(keyname),"%s:%d",
+                                     prefixes[i],j);
+            uint64_t bucket = hash(keyname,keylen) % TABLE_SIZE;
+            if (Table[bucket]) {
+                collisions++;
+            } else {
+                Table[bucket] = 1;
+            }
+        }
+    }
+    return collisions;
+}
+
+int main(void) {
+    printf("SHA1 : %lu\n", testHashingFunction(sha1Hash));
+    printf("CRC64: %lu\n", testHashingFunction(crc64Hash));
+    return 0;
+}