From 7aaff451bafb4b43e1626b329e59a4c9aa7fc31d Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Tue, 4 Jun 2024 19:58:41 +0200 Subject: Merging upstream version 5:7.2.5. Signed-off-by: Daniel Baumann --- 00-RELEASENOTES | 21 +++++++ src/aof.c | 4 +- src/blocked.c | 8 ++- src/cluster.c | 9 ++- src/module.c | 9 +-- src/quicklist.c | 82 +++++++++++++++++++++------ src/rdb.h | 5 +- src/redis-check-aof.c | 10 +++- src/redis-check-rdb.c | 2 + src/redis-cli.c | 3 +- src/script_lua.c | 13 ++++- src/server.c | 10 +++- src/server.h | 1 + src/t_zset.c | 5 +- src/version.h | 4 +- tests/integration/aof.tcl | 25 +++++++++ tests/support/cluster_util.tcl | 27 +++++++++ tests/test_helper.tcl | 2 + tests/unit/cluster/announced-endpoints.tcl | 18 ++++-- tests/unit/cluster/failure-marking.tcl | 53 ++++++++++++++++++ tests/unit/oom-score-adj.tcl | 15 ++++- tests/unit/scripting.tcl | 8 +++ tests/unit/type/list.tcl | 89 +++++++++++++++++++++++++++++- tests/unit/type/stream-cgroups.tcl | 30 +++++++++- tests/unit/type/zset.tcl | 34 ++++++++++++ 25 files changed, 437 insertions(+), 50 deletions(-) create mode 100644 tests/unit/cluster/failure-marking.tcl diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 40efaba..c286821 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -12,6 +12,27 @@ SECURITY: There are security fixes in the release. -------------------------------------------------------------------------------- +================================================================================ +Redis 7.2.5 Released Thu 16 May 2024 12:00:00 IST +================================================================================ + +Upgrade urgency MODERATE: Program an upgrade of the server, but it's not urgent. + +Bug fixes +========= + +* A single shard cluster leaves failed replicas in CLUSTER SLOTS instead of removing them (#12824) +* Crash in LSET command when replacing small items and exceeding 4GB (#12955) +* Blocking commands timeout is reset due to re-processing command (#13004) +* Conversion of numbers in Lua args to redis args can fail. Bug introduced in 7.2.0 (#13115) + +Bug fixes in CLI tools +====================== + +* redis-cli: --count (for --scan, --bigkeys, etc) was ignored unless --pattern was also used (#13092) +* redis-check-aof: incorrectly considering data in manifest format as MP-AOF (#12958) + + ================================================================================ Redis 7.2.4 Released Tue 09 Jan 2024 10:45:52 IST ================================================================================ diff --git a/src/aof.c b/src/aof.c index a89142b..6da6e1a 100644 --- a/src/aof.c +++ b/src/aof.c @@ -117,7 +117,9 @@ aofInfo *aofInfoDup(aofInfo *orig) { return ai; } -/* Format aofInfo as a string and it will be a line in the manifest. */ +/* Format aofInfo as a string and it will be a line in the manifest. + * + * When update this format, make sure to update redis-check-aof as well. */ sds aofInfoFormat(sds buf, aofInfo *ai) { sds filename_repr = NULL; diff --git a/src/blocked.c b/src/blocked.c index 6ad4667..7b48fca 100644 --- a/src/blocked.c +++ b/src/blocked.c @@ -370,7 +370,12 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo list *l; int j; - c->bstate.timeout = timeout; + if (!(c->flags & CLIENT_REPROCESSING_COMMAND)) { + /* If the client is re-processing the command, we do not set the timeout + * because we need to retain the client's original timeout. */ + c->bstate.timeout = timeout; + } + for (j = 0; j < numkeys; j++) { /* If the key already exists in the dictionary ignore it. */ if (!(client_blocked_entry = dictAddRaw(c->bstate.keys,keys[j],NULL))) { @@ -392,7 +397,6 @@ void blockForKeys(client *c, int btype, robj **keys, int numkeys, mstime_t timeo listAddNodeTail(l,c); dictSetVal(c->bstate.keys,client_blocked_entry,listLast(l)); - /* We need to add the key to blocking_keys_unblock_on_nokey, if the client * wants to be awakened if key is deleted (like XREADGROUP) */ if (unblock_on_nokey) { diff --git a/src/cluster.c b/src/cluster.c index c985d0b..637837c 100644 --- a/src/cluster.c +++ b/src/cluster.c @@ -988,7 +988,7 @@ void clusterInit(void) { server.cluster->myself = NULL; server.cluster->currentEpoch = 0; server.cluster->state = CLUSTER_FAIL; - server.cluster->size = 1; + server.cluster->size = 0; server.cluster->todo_before_sleep = 0; server.cluster->nodes = dictCreate(&clusterNodesDictType); server.cluster->shards = dictCreate(&clusterSdsToListType); @@ -4771,10 +4771,13 @@ void clusterCron(void) { /* Timeout reached. Set the node as possibly failing if it is * not already in this state. */ if (!(node->flags & (CLUSTER_NODE_PFAIL|CLUSTER_NODE_FAIL))) { - serverLog(LL_DEBUG,"*** NODE %.40s possibly failing", - node->name); node->flags |= CLUSTER_NODE_PFAIL; update_state = 1; + if (myself->flags & CLUSTER_NODE_MASTER && server.cluster->size == 1) { + markNodeAsFailingIfNeeded(node); + } else { + serverLog(LL_DEBUG,"*** NODE %.40s possibly failing", node->name); + } } } } diff --git a/src/module.c b/src/module.c index ac6cbbb..1cb418c 100644 --- a/src/module.c +++ b/src/module.c @@ -7706,15 +7706,15 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF bc->background_timer = 0; bc->background_duration = 0; - c->bstate.timeout = 0; + mstime_t timeout = 0; if (timeout_ms) { mstime_t now = mstime(); - if (timeout_ms > LLONG_MAX - now) { + if (timeout_ms > LLONG_MAX - now) { c->bstate.module_blocked_handle = NULL; addReplyError(c, "timeout is out of range"); /* 'timeout_ms+now' would overflow */ return bc; } - c->bstate.timeout = timeout_ms + now; + timeout = timeout_ms + now; } if (islua || ismulti) { @@ -7730,8 +7730,9 @@ RedisModuleBlockedClient *moduleBlockClient(RedisModuleCtx *ctx, RedisModuleCmdF addReplyError(c, "Clients undergoing module based authentication can only be blocked on auth"); } else { if (keys) { - blockForKeys(c,BLOCKED_MODULE,keys,numkeys,c->bstate.timeout,flags&REDISMODULE_BLOCK_UNBLOCK_DELETED); + blockForKeys(c,BLOCKED_MODULE,keys,numkeys,timeout,flags&REDISMODULE_BLOCK_UNBLOCK_DELETED); } else { + c->bstate.timeout = timeout; blockClient(c,BLOCKED_MODULE); } } diff --git a/src/quicklist.c b/src/quicklist.c index 301a216..3b11878 100644 --- a/src/quicklist.c +++ b/src/quicklist.c @@ -104,6 +104,9 @@ quicklistBookmark *_quicklistBookmarkFindByName(quicklist *ql, const char *name) quicklistBookmark *_quicklistBookmarkFindByNode(quicklist *ql, quicklistNode *node); void _quicklistBookmarkDelete(quicklist *ql, quicklistBookmark *bm); +quicklistNode *_quicklistSplitNode(quicklistNode *node, int offset, int after); +quicklistNode *_quicklistMergeNodes(quicklist *quicklist, quicklistNode *center); + /* Simple way to give quicklistEntry structs default values with one call. */ #define initEntry(e) \ do { \ @@ -378,6 +381,15 @@ REDIS_STATIC void __quicklistCompress(const quicklist *quicklist, quicklistCompressNode(reverse); } +/* This macro is used to compress a node. + * + * If the 'recompress' flag of the node is true, we compress it directly without + * checking whether it is within the range of compress depth. + * However, it's important to ensure that the 'recompress' flag of head and tail + * is always false, as we always assume that head and tail are not compressed. + * + * If the 'recompress' flag of the node is false, we check whether the node is + * within the range of compress depth before compressing it. */ #define quicklistCompress(_ql, _node) \ do { \ if ((_node)->recompress) \ @@ -529,19 +541,25 @@ REDIS_STATIC int _quicklistNodeAllowMerge(const quicklistNode *a, (node)->sz = lpBytes((node)->entry); \ } while (0) -static quicklistNode* __quicklistCreatePlainNode(void *value, size_t sz) { +static quicklistNode* __quicklistCreateNode(int container, void *value, size_t sz) { quicklistNode *new_node = quicklistCreateNode(); - new_node->entry = zmalloc(sz); - new_node->container = QUICKLIST_NODE_CONTAINER_PLAIN; - memcpy(new_node->entry, value, sz); + new_node->container = container; + if (container == QUICKLIST_NODE_CONTAINER_PLAIN) { + new_node->entry = zmalloc(sz); + memcpy(new_node->entry, value, sz); + } else { + new_node->entry = lpPrepend(lpNew(0), value, sz); + } new_node->sz = sz; new_node->count++; return new_node; } static void __quicklistInsertPlainNode(quicklist *quicklist, quicklistNode *old_node, - void *value, size_t sz, int after) { - __quicklistInsertNode(quicklist, old_node, __quicklistCreatePlainNode(value, sz), after); + void *value, size_t sz, int after) +{ + quicklistNode *new_node = __quicklistCreateNode(QUICKLIST_NODE_CONTAINER_PLAIN, value, sz); + __quicklistInsertNode(quicklist, old_node, new_node, after); quicklist->count++; } @@ -741,9 +759,13 @@ void quicklistReplaceEntry(quicklistIter *iter, quicklistEntry *entry, void *data, size_t sz) { quicklist* quicklist = iter->quicklist; + quicklistNode *node = entry->node; + unsigned char *newentry; - if (likely(!QL_NODE_IS_PLAIN(entry->node) && !isLargeElement(sz))) { - entry->node->entry = lpReplace(entry->node->entry, &entry->zi, data, sz); + if (likely(!QL_NODE_IS_PLAIN(entry->node) && !isLargeElement(sz) && + (newentry = lpReplace(entry->node->entry, &entry->zi, data, sz)) != NULL)) + { + entry->node->entry = newentry; quicklistNodeUpdateSz(entry->node); /* quicklistNext() and quicklistGetIteratorEntryAtIdx() provide an uncompressed node */ quicklistCompress(quicklist, entry->node); @@ -758,17 +780,37 @@ void quicklistReplaceEntry(quicklistIter *iter, quicklistEntry *entry, quicklistInsertAfter(iter, entry, data, sz); __quicklistDelNode(quicklist, entry->node); } - } else { - entry->node->dont_compress = 1; /* Prevent compression in quicklistInsertAfter() */ - quicklistInsertAfter(iter, entry, data, sz); + } else { /* The node is full or data is a large element */ + quicklistNode *split_node = NULL, *new_node; + node->dont_compress = 1; /* Prevent compression in __quicklistInsertNode() */ + + /* If the entry is not at the tail, split the node at the entry's offset. */ + if (entry->offset != node->count - 1 && entry->offset != -1) + split_node = _quicklistSplitNode(node, entry->offset, 1); + + /* Create a new node and insert it after the original node. + * If the original node was split, insert the split node after the new node. */ + new_node = __quicklistCreateNode(isLargeElement(sz) ? + QUICKLIST_NODE_CONTAINER_PLAIN : QUICKLIST_NODE_CONTAINER_PACKED, data, sz); + __quicklistInsertNode(quicklist, node, new_node, 1); + if (split_node) __quicklistInsertNode(quicklist, new_node, split_node, 1); + quicklist->count++; + + /* Delete the replaced element. */ if (entry->node->count == 1) { __quicklistDelNode(quicklist, entry->node); } else { unsigned char *p = lpSeek(entry->node->entry, -1); quicklistDelIndex(quicklist, entry->node, &p); entry->node->dont_compress = 0; /* Re-enable compression */ - quicklistCompress(quicklist, entry->node); - quicklistCompress(quicklist, entry->node->next); + new_node = _quicklistMergeNodes(quicklist, new_node); + /* We can't know if the current node and its sibling nodes are correctly compressed, + * and we don't know if they are within the range of compress depth, so we need to + * use quicklistCompress() for compression, which checks if node is within compress + * depth before compressing. */ + quicklistCompress(quicklist, new_node); + quicklistCompress(quicklist, new_node->prev); + if (new_node->next) quicklistCompress(quicklist, new_node->next); } } @@ -826,6 +868,8 @@ REDIS_STATIC quicklistNode *_quicklistListpackMerge(quicklist *quicklist, } keep->count = lpLength(keep->entry); quicklistNodeUpdateSz(keep); + keep->recompress = 0; /* Prevent 'keep' from being recompressed if + * it becomes head or tail after merging. */ nokeep->count = 0; __quicklistDelNode(quicklist, nokeep); @@ -844,9 +888,10 @@ REDIS_STATIC quicklistNode *_quicklistListpackMerge(quicklist *quicklist, * - (center->next, center->next->next) * - (center->prev, center) * - (center, center->next) + * + * Returns the new 'center' after merging. */ -REDIS_STATIC void _quicklistMergeNodes(quicklist *quicklist, - quicklistNode *center) { +REDIS_STATIC quicklistNode *_quicklistMergeNodes(quicklist *quicklist, quicklistNode *center) { int fill = quicklist->fill; quicklistNode *prev, *prev_prev, *next, *next_next, *target; prev = prev_prev = next = next_next = target = NULL; @@ -886,8 +931,9 @@ REDIS_STATIC void _quicklistMergeNodes(quicklist *quicklist, /* Use result of center merge (or original) to merge with next node. */ if (_quicklistNodeAllowMerge(target, target->next, fill)) { - _quicklistListpackMerge(quicklist, target, target->next); + target = _quicklistListpackMerge(quicklist, target, target->next); } + return target; } /* Split 'node' into two parts, parameterized by 'offset' and 'after'. @@ -1002,7 +1048,7 @@ REDIS_STATIC void _quicklistInsert(quicklistIter *iter, quicklistEntry *entry, } else { quicklistDecompressNodeForUse(node); new_node = _quicklistSplitNode(node, entry->offset, after); - quicklistNode *entry_node = __quicklistCreatePlainNode(value, sz); + quicklistNode *entry_node = __quicklistCreateNode(QUICKLIST_NODE_CONTAINER_PLAIN, value, sz); __quicklistInsertNode(quicklist, node, entry_node, after); __quicklistInsertNode(quicklist, entry_node, new_node, after); quicklist->count++; @@ -3224,7 +3270,7 @@ int quicklistTest(int argc, char *argv[], int flags) { memcpy(s, "helloworld", 10); memcpy(s + sz - 10, "1234567890", 10); - quicklistNode *node = __quicklistCreatePlainNode(s, sz); + quicklistNode *node = __quicklistCreateNode(QUICKLIST_NODE_CONTAINER_PLAIN, s, sz); /* Just to avoid triggering the assertion in __quicklistCompressNode(), * it disables the passing of quicklist head or tail node. */ diff --git a/src/rdb.h b/src/rdb.h index 234bde2..a47ea2b 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -81,9 +81,6 @@ #define RDB_TYPE_MODULE_PRE_GA 6 /* Used in 4.0 release candidates */ #define RDB_TYPE_MODULE_2 7 /* Module value with annotations for parsing without the generating module being loaded. */ -/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ - -/* Object types for encoded objects. */ #define RDB_TYPE_HASH_ZIPMAP 9 #define RDB_TYPE_LIST_ZIPLIST 10 #define RDB_TYPE_SET_INTSET 11 @@ -97,7 +94,7 @@ #define RDB_TYPE_STREAM_LISTPACKS_2 19 #define RDB_TYPE_SET_LISTPACK 20 #define RDB_TYPE_STREAM_LISTPACKS_3 21 -/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ +/* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType(), and rdb_type_string[] */ /* Test if a type is an object type. */ #define rdbIsObjectType(t) (((t) >= 0 && (t) <= 7) || ((t) >= 9 && (t) <= 21)) diff --git a/src/redis-check-aof.c b/src/redis-check-aof.c index 616177a..e28126d 100644 --- a/src/redis-check-aof.c +++ b/src/redis-check-aof.c @@ -233,6 +233,7 @@ int checkSingleAof(char *aof_filename, char *aof_filepath, int last_file, int fi struct redis_stat sb; if (redis_fstat(fileno(fp),&sb) == -1) { printf("Cannot stat file: %s, aborting...\n", aof_filename); + fclose(fp); exit(1); } @@ -343,6 +344,7 @@ int fileIsRDB(char *filepath) { struct redis_stat sb; if (redis_fstat(fileno(fp), &sb) == -1) { printf("Cannot stat file: %s\n", filepath); + fclose(fp); exit(1); } @@ -379,6 +381,7 @@ int fileIsManifest(char *filepath) { struct redis_stat sb; if (redis_fstat(fileno(fp), &sb) == -1) { printf("Cannot stat file: %s\n", filepath); + fclose(fp); exit(1); } @@ -395,15 +398,20 @@ int fileIsManifest(char *filepath) { break; } else { printf("Cannot read file: %s\n", filepath); + fclose(fp); exit(1); } } - /* Skip comments lines */ + /* We will skip comments lines. + * At present, the manifest format is fixed, see aofInfoFormat. + * We will break directly as long as it encounters other items. */ if (buf[0] == '#') { continue; } else if (!memcmp(buf, "file", strlen("file"))) { is_manifest = 1; + } else { + break; } } diff --git a/src/redis-check-rdb.c b/src/redis-check-rdb.c index 682135e..ffa05e8 100644 --- a/src/redis-check-rdb.c +++ b/src/redis-check-rdb.c @@ -98,7 +98,9 @@ char *rdb_type_string[] = { "hash-listpack", "zset-listpack", "quicklist-v2", + "stream-v2", "set-listpack", + "stream-v3", }; /* Show a few stats collected into 'rdbstate' */ diff --git a/src/redis-cli.c b/src/redis-cli.c index 3854701..b5c2736 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -8839,7 +8839,8 @@ static redisReply *sendScan(unsigned long long *it) { reply = redisCommand(context, "SCAN %llu MATCH %b COUNT %d", *it, config.pattern, sdslen(config.pattern), config.count); else - reply = redisCommand(context,"SCAN %llu",*it); + reply = redisCommand(context, "SCAN %llu COUNT %d", + *it, config.count); /* Handle any error conditions */ if(reply == NULL) { diff --git a/src/script_lua.c b/src/script_lua.c index 8cdd805..2c92638 100644 --- a/src/script_lua.c +++ b/src/script_lua.c @@ -818,8 +818,17 @@ static robj **luaArgsToRedisArgv(lua_State *lua, int *argc, int *argv_len) { /* We can't use lua_tolstring() for number -> string conversion * since Lua uses a format specifier that loses precision. */ lua_Number num = lua_tonumber(lua,j+1); - obj_len = fpconv_dtoa((double)num, dbuf); - dbuf[obj_len] = '\0'; + /* Integer printing function is much faster, check if we can safely use it. + * Since lua_Number is not explicitly an integer or a double, we need to make an effort + * to convert it as an integer when that's possible, since the string could later be used + * in a context that doesn't support scientific notation (e.g. 1e9 instead of 100000000). */ + long long lvalue; + if (double2ll((double)num, &lvalue)) + obj_len = ll2string(dbuf, sizeof(dbuf), lvalue); + else { + obj_len = fpconv_dtoa((double)num, dbuf); + dbuf[obj_len] = '\0'; + } obj_s = dbuf; } else { obj_s = (char*)lua_tolstring(lua,j+1,&obj_len); diff --git a/src/server.c b/src/server.c index 438325f..4d47b5e 100644 --- a/src/server.c +++ b/src/server.c @@ -3512,12 +3512,20 @@ void call(client *c, int flags) { * re-processing and unblock the client.*/ c->flags |= CLIENT_EXECUTING_COMMAND; + /* Setting the CLIENT_REPROCESSING_COMMAND flag so that during the actual + * processing of the command proc, the client is aware that it is being + * re-processed. */ + if (reprocessing_command) c->flags |= CLIENT_REPROCESSING_COMMAND; + monotime monotonic_start = 0; if (monotonicGetType() == MONOTONIC_CLOCK_HW) monotonic_start = getMonotonicUs(); c->cmd->proc(c); + /* Clear the CLIENT_REPROCESSING_COMMAND flag after the proc is executed. */ + if (reprocessing_command) c->flags &= ~CLIENT_REPROCESSING_COMMAND; + exitExecutionUnit(); /* In case client is blocked after trying to execute the command, @@ -3575,7 +3583,7 @@ void call(client *c, int flags) { /* Send the command to clients in MONITOR mode if applicable, * since some administrative commands are considered too dangerous to be shown. - * Other exceptions is a client which is unblocked and retring to process the command + * Other exceptions is a client which is unblocked and retrying to process the command * or we are currently in the process of loading AOF. */ if (update_command_stats && !reprocessing_command && !(c->cmd->flags & (CMD_SKIP_MONITOR|CMD_ADMIN))) { diff --git a/src/server.h b/src/server.h index cb55503..b1fa542 100644 --- a/src/server.h +++ b/src/server.h @@ -400,6 +400,7 @@ extern int configOOMScoreAdjValuesDefaults[CONFIG_OOM_COUNT]; auth had been authenticated from the Module. */ #define CLIENT_MODULE_PREVENT_AOF_PROP (1ULL<<48) /* Module client do not want to propagate to AOF */ #define CLIENT_MODULE_PREVENT_REPL_PROP (1ULL<<49) /* Module client do not want to propagate to replica */ +#define CLIENT_REPROCESSING_COMMAND (1ULL<<50) /* The client is re-processing the command. */ /* Client block type (btype field in client structure) * if CLIENT_BLOCKED flag is set. */ diff --git a/src/t_zset.c b/src/t_zset.c index 7717a4a..1b267e0 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1172,7 +1172,8 @@ unsigned long zsetLength(const robj *zobj) { * and the value len hint indicates the approximate individual size of the added elements, * they are used to determine the initial representation. * - * If the hints are not known, and underestimation or 0 is suitable. */ + * If the hints are not known, and underestimation or 0 is suitable. + * We should never pass a negative value because it will convert to a very large unsigned number. */ robj *zsetTypeCreate(size_t size_hint, size_t val_len_hint) { if (size_hint <= server.zset_max_listpack_entries && val_len_hint <= server.zset_max_listpack_value) @@ -3001,7 +3002,7 @@ static void zrangeResultFinalizeClient(zrange_result_handler *handler, /* Result handler methods for storing the ZRANGESTORE to a zset. */ static void zrangeResultBeginStore(zrange_result_handler *handler, long length) { - handler->dstobj = zsetTypeCreate(length, 0); + handler->dstobj = zsetTypeCreate(length >= 0 ? length : 0, 0); } static void zrangeResultEmitCBufferForStore(zrange_result_handler *handler, diff --git a/src/version.h b/src/version.h index 7c6eea6..0b22cd0 100644 --- a/src/version.h +++ b/src/version.h @@ -1,2 +1,2 @@ -#define REDIS_VERSION "7.2.4" -#define REDIS_VERSION_NUM 0x00070204 +#define REDIS_VERSION "7.2.5" +#define REDIS_VERSION_NUM 0x00070205 diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index 1f73fc3..6f2eb05 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -529,6 +529,18 @@ tags {"aof external:skip"} { assert_match "*Start checking Old-Style AOF*is valid*" $result } + test {Test redis-check-aof for old style resp AOF - has data in the same format as manifest} { + create_aof $aof_dirpath $aof_file { + append_to_aof [formatCommand set file file] + append_to_aof [formatCommand set "file appendonly.aof.2.base.rdb seq 2 type b" "file appendonly.aof.2.base.rdb seq 2 type b"] + } + + catch { + exec src/redis-check-aof $aof_file + } result + assert_match "*Start checking Old-Style AOF*is valid*" $result + } + test {Test redis-check-aof for old style rdb-preamble AOF} { catch { exec src/redis-check-aof tests/assets/rdb-preamble.aof @@ -577,6 +589,19 @@ tags {"aof external:skip"} { assert_match "*Start checking Multi Part AOF*Start to check BASE AOF (RDB format)*DB preamble is OK, proceeding with AOF tail*BASE AOF*is valid*Start to check INCR files*INCR AOF*is valid*All AOF files and manifest are valid*" $result } + test {Test redis-check-aof for Multi Part AOF contains a format error} { + create_aof_manifest $aof_dirpath $aof_manifest_file { + append_to_manifest "file appendonly.aof.1.base.aof seq 1 type b\n" + append_to_manifest "file appendonly.aof.1.incr.aof seq 1 type i\n" + append_to_manifest "!!!\n" + } + + catch { + exec src/redis-check-aof $aof_manifest_file + } result + assert_match "*Invalid AOF manifest file format*" $result + } + test {Test redis-check-aof only truncates the last file for Multi Part AOF in fix mode} { create_aof $aof_dirpath $aof_base_file { append_to_aof [formatCommand set foo hello] diff --git a/tests/support/cluster_util.tcl b/tests/support/cluster_util.tcl index 2e3611e..5160466 100644 --- a/tests/support/cluster_util.tcl +++ b/tests/support/cluster_util.tcl @@ -199,3 +199,30 @@ proc are_hostnames_propagated {match_string} { } return 1 } + +proc wait_node_marked_fail {ref_node_index instance_id_to_check} { + wait_for_condition 1000 50 { + [check_cluster_node_mark fail $ref_node_index $instance_id_to_check] + } else { + fail "Replica node never marked as FAIL ('fail')" + } +} + +proc wait_node_marked_pfail {ref_node_index instance_id_to_check} { + wait_for_condition 1000 50 { + [check_cluster_node_mark fail\? $ref_node_index $instance_id_to_check] + } else { + fail "Replica node never marked as PFAIL ('fail?')" + } +} + +proc check_cluster_node_mark {flag ref_node_index instance_id_to_check} { + set nodes [get_cluster_nodes $ref_node_index] + + foreach n $nodes { + if {[dict get $n id] eq $instance_id_to_check} { + return [cluster_has_flag $n $flag] + } + } + fail "Unable to find instance id in cluster nodes. ID: $instance_id_to_check" +} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 21fa35d..64671d7 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -94,6 +94,7 @@ set ::all_tests { unit/client-eviction unit/violations unit/replybufsize + unit/cluster/announced-endpoints unit/cluster/misc unit/cluster/cli unit/cluster/scripting @@ -103,6 +104,7 @@ set ::all_tests { unit/cluster/slot-ownership unit/cluster/links unit/cluster/cluster-response-tls + unit/cluster/failure-marking } # Index to the next test to run in the ::all_tests list. set ::next_test 0 diff --git a/tests/unit/cluster/announced-endpoints.tcl b/tests/unit/cluster/announced-endpoints.tcl index 941a8e0..becba22 100644 --- a/tests/unit/cluster/announced-endpoints.tcl +++ b/tests/unit/cluster/announced-endpoints.tcl @@ -1,8 +1,12 @@ start_cluster 2 2 {tags {external:skip cluster}} { test "Test change cluster-announce-port and cluster-announce-tls-port at runtime" { - set baseport [lindex [R 0 config get port] 1] - set count [expr [llength $::servers] +1 ] + if {$::tls} { + set baseport [lindex [R 0 config get tls-port] 1] + } else { + set baseport [lindex [R 0 config get port] 1] + } + set count [expr [llength $::servers] + 1] set used_port [find_available_port $baseport $count] R 0 config set cluster-announce-tls-port $used_port @@ -17,12 +21,16 @@ start_cluster 2 2 {tags {external:skip cluster}} { R 0 config set cluster-announce-tls-port 0 R 0 config set cluster-announce-port 0 - assert_match "*:$baseport@*" [R 0 CLUSTER NODES] + assert_match "*:$baseport@*" [R 0 CLUSTER NODES] } test "Test change cluster-announce-bus-port at runtime" { - set baseport [lindex [R 0 config get port] 1] - set count [expr [llength $::servers] +1 ] + if {$::tls} { + set baseport [lindex [R 0 config get tls-port] 1] + } else { + set baseport [lindex [R 0 config get port] 1] + } + set count [expr [llength $::servers] + 1] set used_port [find_available_port $baseport $count] # Verify config set cluster-announce-bus-port diff --git a/tests/unit/cluster/failure-marking.tcl b/tests/unit/cluster/failure-marking.tcl new file mode 100644 index 0000000..c4746c8 --- /dev/null +++ b/tests/unit/cluster/failure-marking.tcl @@ -0,0 +1,53 @@ +# Test a single primary can mark replica as `fail` +start_cluster 1 1 {tags {external:skip cluster}} { + + test "Verify that single primary marks replica as failed" { + set primary [srv -0 client] + + set replica1 [srv -1 client] + set replica1_pid [srv -1 pid] + set replica1_instance_id [dict get [cluster_get_myself 1] id] + + assert {[lindex [$primary role] 0] eq {master}} + assert {[lindex [$replica1 role] 0] eq {slave}} + + wait_for_sync $replica1 + + pause_process $replica1_pid + + wait_node_marked_fail 0 $replica1_instance_id + } +} + +# Test multiple primaries wait for a quorum and then mark a replica as `fail` +start_cluster 2 1 {tags {external:skip cluster}} { + + test "Verify that multiple primaries mark replica as failed" { + set primary1 [srv -0 client] + + set primary2 [srv -1 client] + set primary2_pid [srv -1 pid] + + set replica1 [srv -2 client] + set replica1_pid [srv -2 pid] + set replica1_instance_id [dict get [cluster_get_myself 2] id] + + assert {[lindex [$primary1 role] 0] eq {master}} + assert {[lindex [$primary2 role] 0] eq {master}} + assert {[lindex [$replica1 role] 0] eq {slave}} + + wait_for_sync $replica1 + + pause_process $replica1_pid + + # Pause other primary to allow time for pfail flag to appear + pause_process $primary2_pid + + wait_node_marked_pfail 0 $replica1_instance_id + + # Resume other primary and wait for to show replica as failed + resume_process $primary2_pid + + wait_node_marked_fail 0 $replica1_instance_id + } +} diff --git a/tests/unit/oom-score-adj.tcl b/tests/unit/oom-score-adj.tcl index 6c7b713..77c4c38 100644 --- a/tests/unit/oom-score-adj.tcl +++ b/tests/unit/oom-score-adj.tcl @@ -1,5 +1,4 @@ set system_name [string tolower [exec uname -s]] -set user_id [exec id -u] if {$system_name eq {linux}} { start_server {tags {"oom-score-adj external:skip"}} { @@ -56,8 +55,15 @@ if {$system_name eq {linux}} { } } + # Determine whether the current user is unprivileged + set original_value [exec cat /proc/self/oom_score_adj] + catch { + set fd [open "/proc/self/oom_score_adj" "w"] + puts $fd -1000 + close $fd + } e # Failed oom-score-adj tests can only run unprivileged - if {$user_id != 0} { + if {[string match "*permission denied*" $e]} { test {CONFIG SET oom-score-adj handles configuration failures} { # Bad config r config set oom-score-adj no @@ -81,6 +87,11 @@ if {$system_name eq {linux}} { # Make sure previous values remain assert {[r config get oom-score-adj-values] == {oom-score-adj-values {0 100 100}}} } + } else { + # Restore the original oom_score_adj value + set fd [open "/proc/self/oom_score_adj" "w"] + puts $fd $original_value + close $fd } test {CONFIG SET oom-score-adj-values doesn't touch proc when disabled} { diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index c2f79a7..18066a1 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -146,6 +146,14 @@ start_server {tags {"scripting"}} { } 1 x } {number 1} + test {EVAL - Lua number -> Redis integer conversion} { + r del hash + run_script { + local foo = redis.pcall('hincrby','hash','field',200000000) + return {type(foo),foo} + } 0 + } {number 200000000} + test {EVAL - Redis bulk -> Lua type conversion} { r set mykey myval run_script { diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 993b6d1..7c0fa87 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -220,6 +220,7 @@ start_server [list overrides [list save ""] ] { # checking LSET in case ziplist needs to be split test {Test LSET with packed is split in the middle} { + set original_config [config_get_set list-max-listpack-size 4] r flushdb r debug quicklist-packed-threshold 5b r RPUSH lst "aa" @@ -227,6 +228,7 @@ start_server [list overrides [list save ""] ] { r RPUSH lst "cc" r RPUSH lst "dd" r RPUSH lst "ee" + assert_encoding quicklist lst r lset lst 2 [string repeat e 10] assert_equal [r lpop lst] "aa" assert_equal [r lpop lst] "bb" @@ -234,6 +236,7 @@ start_server [list overrides [list save ""] ] { assert_equal [r lpop lst] "dd" assert_equal [r lpop lst] "ee" r debug quicklist-packed-threshold 0 + r config set list-max-listpack-size $original_config } {OK} {needs:debug} @@ -381,7 +384,63 @@ if {[lindex [r config get proto-max-bulk-len] 1] == 10000000000} { assert_equal [read_big_bulk {r rpop lst}] $str_length } {} {large-memory} - test {Test LMOVE on plain nodes over 4GB} { + test {Test LSET on plain nodes with large elements under packed_threshold over 4GB} { + r flushdb + r rpush lst a b c d e + for {set i 0} {$i < 5} {incr i} { + r write "*4\r\n\$4\r\nlset\r\n\$3\r\nlst\r\n\$1\r\n$i\r\n" + write_big_bulk 1000000000 + } + r ping + } {PONG} {large-memory} + + test {Test LSET splits a quicklist node, and then merge} { + # Test when a quicklist node can't be inserted and is split, the split + # node merges with the node before it and the `before` node is kept. + r flushdb + r rpush lst [string repeat "x" 4096] + r lpush lst a b c d e f g + r lpush lst [string repeat "y" 4096] + # now: [y...] [g f e d c b a x...] + # (node0) (node1) + # Keep inserting elements into node1 until node1 is split into two + # nodes([g] [...]), eventually node0 will merge with the [g] node. + # Since node0 is larger, after the merge node0 will be kept and + # the [g] node will be deleted. + for {set i 7} {$i >= 3} {incr i -1} { + r write "*4\r\n\$4\r\nlset\r\n\$3\r\nlst\r\n\$1\r\n$i\r\n" + write_big_bulk 1000000000 + } + assert_equal "g" [r lindex lst 1] + r ping + } {PONG} {large-memory} + + test {Test LSET splits a LZF compressed quicklist node, and then merge} { + # Test when a LZF compressed quicklist node can't be inserted and is split, + # the split node merges with the node before it and the split node is kept. + r flushdb + r config set list-compress-depth 1 + r lpush lst [string repeat "x" 2000] + r rpush lst [string repeat "y" 7000] + r rpush lst a b c d e f g + r rpush lst [string repeat "z" 8000] + r lset lst 0 h + # now: [h] [y... a b c d e f g] [z...] + # node0 node1(LZF) + # Keep inserting elements into node1 until node1 is split into two + # nodes([y...] [...]), eventually node0 will merge with the [y...] node. + # Since [y...] node is larger, after the merge node0 will be deleted and + # the [y...] node will be kept. + for {set i 7} {$i >= 3} {incr i -1} { + r write "*4\r\n\$4\r\nlset\r\n\$3\r\nlst\r\n\$1\r\n$i\r\n" + write_big_bulk 1000000000 + } + assert_equal "h" [r lindex lst 0] + r config set list-compress-depth 0 + r ping + } {PONG} {large-memory} + + test {Test LMOVE on plain nodes over 4GB} { r flushdb r RPUSH lst2{t} "aa" r RPUSH lst2{t} "bb" @@ -1186,6 +1245,34 @@ foreach {pop} {BLPOP BLMPOP_LEFT} { r select 9 } {OK} {singledb:skip needs:debug} + test {BLPOP unblock but the key is expired and then block again - reprocessing command} { + r flushall + r debug set-active-expire 0 + set rd [redis_deferring_client] + + set start [clock milliseconds] + $rd blpop mylist 1 + wait_for_blocked_clients_count 1 + + # The exec will try to awake the blocked client, but the key is expired, + # so the client will be blocked again during the command reprocessing. + r multi + r rpush mylist a + r pexpire mylist 100 + r debug sleep 0.2 + r exec + + assert_equal {} [$rd read] + set end [clock milliseconds] + + # Before the fix in #13004, this time would have been 1200+ (i.e. more than 1200ms), + # now it should be 1000, but in order to avoid timing issues, we increase the range a bit. + assert_range [expr $end-$start] 1000 1150 + + r debug set-active-expire 1 + $rd close + } {0} {needs:debug} + foreach {pop} {BLPOP BLMPOP_LEFT} { test "$pop when new key is moved into place" { set rd [redis_deferring_client] diff --git a/tests/unit/type/stream-cgroups.tcl b/tests/unit/type/stream-cgroups.tcl index a6cc5da..46e0b05 100644 --- a/tests/unit/type/stream-cgroups.tcl +++ b/tests/unit/type/stream-cgroups.tcl @@ -475,7 +475,7 @@ start_server { $rd close } - test {Blocking XREADGROUP for stream key that has clients blocked on list - avoid endless loop} { + test {Blocking XREADGROUP for stream key that has clients blocked on stream - avoid endless loop} { r DEL mystream r XGROUP CREATE mystream mygroup $ MKSTREAM @@ -498,6 +498,34 @@ start_server { assert_equal [r ping] {PONG} } + test {Blocking XREADGROUP for stream key that has clients blocked on stream - reprocessing command} { + r DEL mystream + r XGROUP CREATE mystream mygroup $ MKSTREAM + + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + $rd1 xreadgroup GROUP mygroup myuser BLOCK 0 STREAMS mystream > + wait_for_blocked_clients_count 1 + + set start [clock milliseconds] + $rd2 xreadgroup GROUP mygroup myuser BLOCK 1000 STREAMS mystream > + wait_for_blocked_clients_count 2 + + # After a while call xadd and let rd2 re-process the command. + after 200 + r xadd mystream * field value + assert_equal {} [$rd2 read] + set end [clock milliseconds] + + # Before the fix in #13004, this time would have been 1200+ (i.e. more than 1200ms), + # now it should be 1000, but in order to avoid timing issues, we increase the range a bit. + assert_range [expr $end-$start] 1000 1150 + + $rd1 close + $rd2 close + } + test {XGROUP DESTROY should unblock XREADGROUP with -NOGROUP} { r config resetstat r del mystream diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index 33427d8..0a42784 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -1942,6 +1942,34 @@ start_server {tags {"zset"}} { } } + test {BZPOPMIN unblock but the key is expired and then block again - reprocessing command} { + r flushall + r debug set-active-expire 0 + set rd [redis_deferring_client] + + set start [clock milliseconds] + $rd bzpopmin zset{t} 1 + wait_for_blocked_clients_count 1 + + # The exec will try to awake the blocked client, but the key is expired, + # so the client will be blocked again during the command reprocessing. + r multi + r zadd zset{t} 1 one + r pexpire zset{t} 100 + r debug sleep 0.2 + r exec + + assert_equal {} [$rd read] + set end [clock milliseconds] + + # Before the fix in #13004, this time would have been 1200+ (i.e. more than 1200ms), + # now it should be 1000, but in order to avoid timing issues, we increase the range a bit. + assert_range [expr $end-$start] 1000 1150 + + r debug set-active-expire 1 + $rd close + } {0} {needs:debug} + test "BZPOPMIN with same key multiple times should work" { set rd [redis_deferring_client] r del z1{t} z2{t} @@ -2211,12 +2239,18 @@ start_server {tags {"zset"}} { } {b 2 c 3} test {ZRANGESTORE BYLEX} { + set res [r zrangestore z3{t} z1{t} \[b \[c BYLEX] + assert_equal $res 2 + assert_encoding listpack z3{t} set res [r zrangestore z2{t} z1{t} \[b \[c BYLEX] assert_equal $res 2 r zrange z2{t} 0 -1 withscores } {b 2 c 3} test {ZRANGESTORE BYSCORE} { + set res [r zrangestore z4{t} z1{t} 1 2 BYSCORE] + assert_equal $res 2 + assert_encoding listpack z4{t} set res [r zrangestore z2{t} z1{t} 1 2 BYSCORE] assert_equal $res 2 r zrange z2{t} 0 -1 withscores -- cgit v1.2.3