Merge branch 'unstable' into RELEASE_5

README.md

@@ -4,6 +4,8 @@
 
 ##### Need Help? Check out our extensive [documentation](https://docs.keydb.dev).
 
+##### Have feedback?  Take our quick survey: https://www.surveymonkey.com/r/Y9XNS93
+
 What is KeyDB?
 --------------
 

src/Makefile

@@ -105,7 +105,7 @@ endif
 FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS)
 FINAL_CXXFLAGS=$(CXX_STD) $(WARN) $(OPT) $(DEBUG) $(CXXFLAGS) $(REDIS_CFLAGS)
 FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG)
-FINAL_LIBS=-lm
+FINAL_LIBS=-lm -lcurl
 DEBUG=-g -ggdb
 
 ifeq ($(uname_S),SunOS)
@@ -213,9 +213,9 @@ endif
 
 REDIS_SERVER_NAME=keydb-server
 REDIS_SENTINEL_NAME=keydb-sentinel
-REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o $(ASM_OBJ)
+REDIS_SERVER_OBJ=adlist.o quicklist.o ae.o anet.o dict.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o redis-check-rdb.o redis-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o acl.o storage.o rdb-s3.o fastlock.o new.o tracking.o cron.o motd.o $(ASM_OBJ)
 REDIS_CLI_NAME=keydb-cli
-REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o new.o $(ASM_OBJ)
+REDIS_CLI_OBJ=anet.o adlist.o dict.o redis-cli.o redis-cli-cpphelper.o zmalloc.o release.o anet.o ae.o crc64.o siphash.o crc16.o storage-lite.o fastlock.o new.o motd.o $(ASM_OBJ)
 REDIS_BENCHMARK_NAME=keydb-benchmark
 REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o adlist.o dict.o zmalloc.o siphash.o redis-benchmark.o storage-lite.o fastlock.o new.o $(ASM_OBJ)
 REDIS_CHECK_RDB_NAME=keydb-check-rdb
@@ -285,7 +285,7 @@ $(REDIS_CHECK_AOF_NAME): $(REDIS_SERVER_NAME)
 
 # keydb-cli
 $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ)
-	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(FINAL_LIBS) -lcurl
+	$(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(FINAL_LIBS)
 
 # keydb-benchmark
 $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ)

src/asciilogo.h

@@ -36,7 +36,6 @@ const char *ascii_logo =
 "                                        Port: %d\n"
 "                                        PID: %ld\n"
 "                                                                     \n"
-"                     Like KeyDB? Star us on GitHub!                  \n"
+"                     %s\n"
 "                                                                     \n"
-"                     https://github.com/JohnSully/KeyDB              \n"
-"                                                                     \n\n";
+"                                                                     \n";

src/networking.cpp

@@ -177,6 +177,7 @@ client *createClient(int fd, int iel) {
     c->bufposAsync = 0;
     c->client_tracking_redirection = 0;
     c->casyncOpsPending = 0;
+    c->master_error = 0;
     memset(c->uuid, 0, UUID_BINARY_LEN);
 
     listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid);
@@ -432,6 +433,34 @@ void addReplyProtoAsync(client *c, const char *s, size_t len) {
     addReplyProtoCore(c, s, len, true);
 }
 
+std::string escapeString(sds str)
+{
+    std::string newstr;
+    size_t len = sdslen(str);
+    for (size_t ich = 0; ich < len; ++ich)
+    {
+        char ch = str[ich];
+        switch (ch)
+        {
+        case '\n':
+            newstr += "\\n";
+            break;
+
+        case '\t':
+            newstr += "\\t";
+            break;
+
+        case '\r':
+            newstr += "\\r";
+            break;
+
+        default:
+            newstr += ch;
+        }
+    }
+    return newstr;
+}
+
 /* Low level function called by the addReplyError...() functions.
  * It emits the protocol for a Redis error, in the form:
  *
@@ -464,6 +493,12 @@ void addReplyErrorLengthCore(client *c, const char *s, size_t len, bool fAsync)
         serverLog(LL_WARNING,"== CRITICAL == This %s is sending an error "
                              "to its %s: '%s' after processing the command "
                              "'%s'", from, to, s, cmdname);
+
+        if (c->querybuf && sdslen(c->querybuf)) {
+            std::string str = escapeString(c->querybuf);
+            serverLog(LL_WARNING, "\tquerybuf: %s", str.c_str());
+        }
+        c->master_error = 1;
     }
 }
 

src/redis-cli.c

@@ -57,6 +57,7 @@
 #include "anet.h"
 #include "ae.h"
 #include "storage.h"
+#include "motd.h"
 
 #include "redis-cli.h"
 
@@ -6536,118 +6537,6 @@ static void intrinsicLatencyMode(void) {
     }
 }
 
-/*------------------------------------------------------------------------------
- * Message of the day
- *--------------------------------------------------------------------------- */
-#ifdef MOTD
-#include <curl/curl.h> 
-
-static const char *szMotdCachePath()
-{
-    static sds sdsMotdCachePath = NULL;
-    if (sdsMotdCachePath != NULL)
-        return sdsMotdCachePath;
-
-    struct passwd *pw = getpwuid(getuid());
-    if (pw == NULL)
-        return "";
-    const char *homedir = pw->pw_dir;
-    sdsMotdCachePath = sdsnew(homedir);
-    sdsMotdCachePath = sdscat(sdsMotdCachePath, "/.keydb-cli-motd");
-    return sdsMotdCachePath;
-}
-static size_t motd_write_callback(void *ptr, size_t size, size_t nmemb, sds *str)
-{
-    *str = sdscatlen(*str, ptr, size*nmemb);
-    return (size*nmemb);
-}
-
-static char *fetchMOTDFromCache()
-{
-    struct stat attrib;
-    if (stat(szMotdCachePath(), &attrib) != 0)
-        return NULL;
-    time_t t = attrib.st_mtim.tv_sec;
-    time_t now = time(NULL);
-    if ((now - t) < 14400)
-    {
-        // If our cache was updated no more than 4 hours ago use it instead of fetching the MOTD
-        FILE *pf = fopen(szMotdCachePath(), "rb");
-        if (pf == NULL)
-            return NULL;
-        fseek(pf, 0L, SEEK_END);
-        long cb = ftell(pf);
-        fseek(pf, 0L, SEEK_SET);    // rewind
-        sds str = sdsnewlen(NULL, cb);
-        size_t cbRead = fread(str, 1, cb, pf);
-        fclose(pf);
-        if ((long)cbRead != cb)
-        {
-            sdsfree(str);
-            return NULL;
-        }
-        return str;
-    }
-    return NULL;
-}
-
-static void setMOTDCache(const char *sz)
-{
-    FILE *pf = fopen(szMotdCachePath(), "wb");
-    if (pf == NULL)
-        return;
-    size_t celem = fwrite(sz, strlen(sz), 1, pf);
-    (void)celem;    // best effort
-    fclose(pf);
-}
-
-static char *fetchMOTD()
-{
-    sds str;
-    CURL *curl;
-    CURLcode res;
-
-    /* First try and get the string from the cache */
-    str = fetchMOTDFromCache();
-    if (str != NULL)
-        return str;
-
-    str = sdsnew("");
-    curl = curl_easy_init();
-    if(curl) {
-        curl_easy_setopt(curl, CURLOPT_URL, "http://api.keydb.dev/motd/motd.txt");
-        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // follow redirects
-        curl_easy_setopt(curl, CURLOPT_TIMEOUT, 2); // take no more than two seconds
-        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, motd_write_callback);
-        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &str);
-
-        /* Perform the request, res will get the return code */ 
-        res = curl_easy_perform(curl);
-        /* Check for errors */ 
-        if(res != CURLE_OK)
-        {
-            sdsfree(str);
-            str = NULL;
-        }
-
-        /* always cleanup */ 
-        curl_easy_cleanup(curl);
-
-        if (str != NULL)
-            setMOTDCache(str);
-    }
-    return str;
-}
-
-#else
-
-static char *fetchMOTD()
-{
-    return NULL;
-}
-
-#endif
-
 /*------------------------------------------------------------------------------
  * Program main()
  *--------------------------------------------------------------------------- */
@@ -6818,7 +6707,7 @@ int main(int argc, char **argv) {
     if (argc == 0 && !config.eval) {
         /* Show the message of the day if we are interactive */
         if (config.output == OUTPUT_STANDARD) {
-            char *szMotd = fetchMOTD();
+            char *szMotd = fetchMOTD(1 /* cache */);
             if (szMotd != NULL) {
                 printf("Message of the day:\n  %s\n", szMotd);
                 sdsfree(szMotd);

src/replication.cpp

@@ -3482,6 +3482,8 @@ void replicaReplayCommand(client *c)
     cFake->flags &= ~(CLIENT_MASTER | CLIENT_PREVENT_REPL_PROP);
     bool fExec = ccmdPrev != serverTL->commandsExecuted;
     cFake->lock.unlock();
+    if (cFake->master_error)
+        addReplyError(c, "Error in rreplay command, please check logs");
     if (fExec || cFake->flags & CLIENT_MULTI)
     {
         addReply(c, shared.ok);

src/server.cpp

@@ -62,6 +62,7 @@
 #include <uuid/uuid.h>
 #include <mutex>
 #include "aelocker.h"
+#include "motd.h"
 
 int g_fTestMode = false;
 
@@ -4749,14 +4750,18 @@ void redisAsciiArt(void) {
             mode, g_pserver->port
         );
     } else {
+        sds motd = fetchMOTD(true);
         snprintf(buf,1024*16,ascii_logo,
             KEYDB_REAL_VERSION,
             redisGitSHA1(),
             strtol(redisGitDirty(),NULL,10) > 0,
             (sizeof(long) == 8) ? "64" : "32",
             mode, g_pserver->port,
-            (long) getpid()
+            (long) getpid(),
+            motd ? motd : ""
         );
+        if (motd)
+            sdsfree(motd);
         serverLogRaw(LL_NOTICE|LL_RAW,buf);
     }
     zfree(buf);

src/server.h

@@ -1337,6 +1337,7 @@ typedef struct client {
 
     int iel; /* the event loop index we're registered with */
     struct fastlock lock;
+    int master_error;
 } client;
 
 struct saveparam {