From 9d584ea4c9f27ba36f1a1ff0d9f5862b51911d24 Mon Sep 17 00:00:00 2001 From: Moti Cohen Date: Mon, 17 Jun 2024 07:59:10 +0300 Subject: [PATCH] Add support to hash-field-expiration (7.4) (#47) * RDB_TYPE_HASH_METADATA - While RDB_TYPE_HASH keep key-value pairs to rdb file, this one keep key-value-expiry tuples such that the value of expiry cannot be bigger than 2^48-1. * RDB_TYPE_HASH_LISTPACK_EX - While RDB_TYPE_HASH_LISTPACK_EX keep in listpack pairs of key-value, this one keep key-value-expiry tuples such that the value of expiry cannot be bigger than 2^48-1. --- api/librdb-api.h | 8 +- src/ext/handlersFilter.c | 16 +++- src/ext/handlersToJson.c | 14 +-- src/ext/handlersToResp.c | 30 +++++-- src/ext/readerResp.c | 52 ++++++++--- src/lib/parser.c | 83 +++++++++++++----- src/lib/parser.h | 8 +- src/lib/parserRaw.c | 69 ++++++++++----- test/dumps/hash_data.json | 8 ++ test/dumps/hash_ex_v12_data.json | 8 ++ test/dumps/hash_lp_ex_v12_data.json | 9 ++ test/dumps/hash_lp_with_hexpire_v12.rdb | Bin 0 -> 169 bytes .../{plain_hash_raw.json => hash_raw.json} | 0 test/dumps/hash_struct.json | 8 ++ test/dumps/{plain_hash_v3.rdb => hash_v3.rdb} | Bin test/dumps/hash_with_expire_v12.rdb | Bin 0 -> 169 bytes test/test_common.c | 72 ++++++++------- test/test_rdb_to_json.c | 36 ++++++-- test/test_rdb_to_redis.c | 79 +++++++++++++++-- test/test_rdb_to_resp.c | 21 ++++- test/test_resp_reader.c | 14 +++ 21 files changed, 410 insertions(+), 125 deletions(-) create mode 100644 test/dumps/hash_data.json create mode 100644 test/dumps/hash_ex_v12_data.json create mode 100644 test/dumps/hash_lp_ex_v12_data.json create mode 100644 test/dumps/hash_lp_with_hexpire_v12.rdb rename test/dumps/{plain_hash_raw.json => hash_raw.json} (100%) create mode 100644 test/dumps/hash_struct.json rename test/dumps/{plain_hash_v3.rdb => hash_v3.rdb} (100%) create mode 100644 test/dumps/hash_with_expire_v12.rdb diff --git a/api/librdb-api.h b/api/librdb-api.h index 41f2386..17acc57 100644 --- a/api/librdb-api.h +++ b/api/librdb-api.h @@ -257,12 +257,14 @@ typedef struct RdbHandlersStructCallbacks { /* Callback to handle a listpack-based list value */ RdbRes (*handleListLP)(RdbParser *p, void *userData, RdbBulk listpack); - /* Callback to handle a field-value pair within a plain-hash */ - RdbRes (*handleHashPlain)(RdbParser *p, void *userData, RdbBulk field, RdbBulk value); + /* Callback to handle a field-value pair within a plain-hash. expireAt -1 if not set. */ + RdbRes (*handleHashPlain)(RdbParser *p, void *userData, RdbBulk field, RdbBulk value, int64_t expireAt); /* Callback to handle a ziplist-based hash value */ RdbRes (*handleHashZL)(RdbParser *p, void *userData, RdbBulk ziplist); /* Callback to handle a listpack-based hash value */ RdbRes (*handleHashLP)(RdbParser *p, void *userData, RdbBulk listpack); + /* Callback to handle a listpackex-based hash (with expiry on fields) */ + RdbRes (*handleHashLPEx)(RdbParser *p, void *userData, RdbBulk listpackEx); /* Callback to handle a zipmap-based hash value */ RdbRes (*handleHashZM)(RdbParser *p, void *userData, RdbBulk zipmap); @@ -305,7 +307,7 @@ typedef struct RdbHandlersDataCallbacks { /* Callback to handle an item from a list */ RdbRes (*handleListItem)(RdbParser *p, void *userData, RdbBulk item); /* Callback to handle a field-value pair within a hash */ - RdbRes (*handleHashField)(RdbParser *p, void *userData, RdbBulk field, RdbBulk value); + RdbRes (*handleHashField)(RdbParser *p, void *userData, RdbBulk field, RdbBulk value, int64_t expireAt); /* Callback to handle a member within a set */ RdbRes (*handleSetMember)(RdbParser *p, void *userData, RdbBulk member); /* Callback to handle a member within a sorted set along with its score */ diff --git a/src/ext/handlersFilter.c b/src/ext/handlersFilter.c index 64a1647..4fbec4f 100644 --- a/src/ext/handlersFilter.c +++ b/src/ext/handlersFilter.c @@ -43,9 +43,11 @@ static void initOpcodeToType(RdbxFilter *ctx) { ctx->opToType[RDB_TYPE_ZSET_LISTPACK] = RDB_DATA_TYPE_ZSET; /*hash*/ ctx->opToType[RDB_TYPE_HASH] = RDB_DATA_TYPE_HASH; + ctx->opToType[RDB_TYPE_HASH_METADATA] = RDB_DATA_TYPE_HASH; ctx->opToType[RDB_TYPE_HASH_ZIPMAP] = RDB_DATA_TYPE_HASH; ctx->opToType[RDB_TYPE_HASH_ZIPLIST] = RDB_DATA_TYPE_HASH; ctx->opToType[RDB_TYPE_HASH_LISTPACK] = RDB_DATA_TYPE_HASH; + ctx->opToType[RDB_TYPE_HASH_LISTPACK_EX] = RDB_DATA_TYPE_HASH; /*module*/ ctx->opToType[RDB_TYPE_MODULE_2] = RDB_DATA_TYPE_MODULE; ctx->opToType[RDB_OPCODE_MODULE_AUX] = RDB_DATA_TYPE_MODULE; @@ -120,8 +122,8 @@ static RdbRes filterList(RdbParser *p, void *userData, RdbBulk item) { return ((RdbxFilter *) userData)->cbReturnValue; } -static RdbRes filterHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { - UNUSED(p, field, value); +static RdbRes filterHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk value, int64_t expireAt) { + UNUSED(p, field, value, expireAt); return ((RdbxFilter *) userData)->cbReturnValue; } @@ -177,6 +179,11 @@ static RdbRes filterHashLP(RdbParser *p, void *userData, RdbBulk listpack) { return ((RdbxFilter *) userData)->cbReturnValue; } +static RdbRes filterHashLPEx(RdbParser *p, void *userData, RdbBulk listpackEx) { + UNUSED(p, listpackEx); + return ((RdbxFilter *) userData)->cbReturnValue; +} + static RdbRes filterHashZM(RdbParser *p, void *userData, RdbBulk zipmap) { UNUSED(p, zipmap); return ((RdbxFilter *) userData)->cbReturnValue; @@ -187,8 +194,8 @@ static RdbRes filterHashZL(RdbParser *p, void *userData, RdbBulk ziplist) { return ((RdbxFilter *) userData)->cbReturnValue; } -static RdbRes filterHashPlain(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { - UNUSED(p, field, value); +static RdbRes filterHashPlain(RdbParser *p, void *userData, RdbBulk field, RdbBulk value, int64_t expireAt) { + UNUSED(p, field, value, expireAt); return ((RdbxFilter *) userData)->cbReturnValue; } @@ -310,6 +317,7 @@ static void defaultFilterStructCb(RdbHandlersStructCallbacks *structCb) { filterHashPlain, /*handleHashPlain*/ filterHashZL, /*handleHashZL*/ filterHashLP, /*handleHashLP*/ + filterHashLPEx, /*handleHashLPEx*/ filterHashZM, /*handleHashZM*/ filterSetPlain, /*handleSetPlain*/ filterSetIS, /*handleSetIS*/ diff --git a/src/ext/handlersToJson.c b/src/ext/handlersToJson.c index 49762ef..04a0e48 100644 --- a/src/ext/handlersToJson.c +++ b/src/ext/handlersToJson.c @@ -467,7 +467,10 @@ static RdbRes toJsonZset(RdbParser *p, void *userData, RdbBulk member, double sc return RDB_OK; } -static RdbRes toJsonHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { +static RdbRes toJsonHash(RdbParser *p, void *userData, RdbBulk field, + RdbBulk value, int64_t expireAt) +{ + UNUSED(expireAt); RdbxToJson *ctx = userData; if (ctx->state == R2J_IN_KEY) { @@ -775,10 +778,11 @@ RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxTo toJsonStruct, /* handleListZL*/ toJsonStruct, /* handleListLP*/ /*hash*/ - toJsonHash, - toJsonStruct, /* handleHashZL*/ - toJsonStruct, /* handleHashLP*/ - toJsonStruct, /* handleHashZM*/ + toJsonHash, /*handleHashPlain*/ + toJsonStruct, /*handleHashZL*/ + toJsonStruct, /*handleHashLP*/ + toJsonStruct, /*handleHashLPEx*/ + toJsonStruct, /*handleHashZM*/ /*set*/ toJsonSet, toJsonStruct, /* handleSetIS*/ diff --git a/src/ext/handlersToResp.c b/src/ext/handlersToResp.c index ceee209..3a24953 100644 --- a/src/ext/handlersToResp.c +++ b/src/ext/handlersToResp.c @@ -489,7 +489,8 @@ static RdbRes toRespList(RdbParser *p, void *userData, RdbBulk item) { return writevWrap(ctx, iov, 6, &startCmd, 1); } -static RdbRes toRespHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { +static RdbRes toRespHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk value, int64_t expireAt) { + char expireTimeStr[32], expireTimeLenStr[32]; struct iovec iov[10]; RdbxToResp *ctx = userData; @@ -499,9 +500,9 @@ static RdbRes toRespHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk va int fieldLen = RDB_bulkLen(p, field); int valueLen = RDB_bulkLen(p, value); - RdbxRespWriterStartCmd startCmd; - startCmd.cmd = "HSET"; - startCmd.key = ctx->keyCtx.key; + RdbxRespWriterStartCmd hsetCmd; + hsetCmd.cmd = "HSET"; + hsetCmd.key = ctx->keyCtx.key; /* write RPUSH */ IOV_CONST(&iov[0], "*4\r\n$4\r\nHSET"); @@ -515,7 +516,26 @@ static RdbRes toRespHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk va IOV_LENGTH(&iov[5], valueLen, valueLenStr); IOV_STRING(&iov[6], value, valueLen); IOV_CONST(&iov[7], "\r\n"); - return writevWrap(ctx, iov, 8, &startCmd, 1); + IF_NOT_OK_RETURN(writevWrap(ctx, iov, 8, &hsetCmd, 1)); + + if (expireAt == -1) return RDB_OK; + + RdbxRespWriterStartCmd hpexpireatCmd; + hpexpireatCmd.cmd = "HPEXPIREAT"; + hpexpireatCmd.key = ctx->keyCtx.key; + /* write HPEXPIREAT */ + IOV_CONST(&iov[0], "*6\r\n$10\r\nHPEXPIREAT"); + /* write key */ + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); + /* write expiration-time in msec */ + IOV_LEN_AND_VAL(&iov[3], expireAt, expireTimeLenStr, expireTimeStr); + IOV_CONST(&iov[5], "$6\r\nFIELDS\r\n$1\r\n1"); + /* write field */ + IOV_LENGTH(&iov[6], fieldLen, fieldLenStr); + IOV_STRING(&iov[7], field, fieldLen); + IOV_CONST(&iov[8], "\r\n"); + return writevWrap(ctx, iov, 9, &hpexpireatCmd, 1); } static RdbRes toRespSet(RdbParser *p, void *userData, RdbBulk member) { diff --git a/src/ext/readerResp.c b/src/ext/readerResp.c index fb7063f..d2438cb 100644 --- a/src/ext/readerResp.c +++ b/src/ext/readerResp.c @@ -101,6 +101,7 @@ static RespRes readRespReplyLine(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { ++(buffInfo->at); /* fall-thru */ case PROC_LINE_END: + ctx->typeState = 0; return RESP_REPLY_OK; } } @@ -141,14 +142,15 @@ RespRes readRespReplyBulk(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { char ch; UNUSED(buffInfo); + /* Parsing : $\r\n\r\n */ enum ProcessBulkReadStates { PROC_BULK_READ_INIT = 0, - PROC_BULK_READ_LEN, - PROC_BULK_READ_LEN_CR, - PROC_BULK_READ_LEN_NL, - PROC_BULK_READ, - PROC_BULK_READ_CR, - PROC_BULK_READ_NL, + PROC_BULK_READ_LEN, /* Read bulk length */ + PROC_BULK_READ_LEN_CR, /* Read CR */ + PROC_BULK_READ_LEN_NL, /* Read NL */ + PROC_BULK_READ, /* Read data */ + PROC_BULK_READ_CR, /* Read CR */ + PROC_BULK_READ_NL, /* Read NL */ PROC_BULK_READ_END, }; @@ -255,6 +257,7 @@ static RespRes readRespReplyBulkArray(RespReaderCtx *ctx, RespReplyBuff *buffInf READ_NUM_BULKS_NL, READ_NEXT_BULK_HDR, READ_NEXT_BULK, + READ_NEXT_LINE, /* int, double, null, bool, bignum */ READ_END, }; @@ -311,12 +314,26 @@ static RespRes readRespReplyBulkArray(RespReaderCtx *ctx, RespReplyBuff *buffInf break; case READ_NEXT_BULK_HDR: - if (buffInfo->buff[buffInfo->at++] != '$') { - snprintf(ctx->errorMsg, sizeof(ctx->errorMsg), - "Invalid Multi-Bulk response. Failed to read Bulk header."); - return RESP_REPLY_ERR; + if (buffInfo->buff[buffInfo->at] == '$') { + buffInfo->at++; + ctx->typeArrayState = READ_NEXT_BULK; + break; + } + + if ((buffInfo->buff[buffInfo->at] == ':') || /*int*/ + (buffInfo->buff[buffInfo->at] == ',') || /*double*/ + (buffInfo->buff[buffInfo->at] == '_') || /*null*/ + (buffInfo->buff[buffInfo->at] == '#') || /*bool*/ + (buffInfo->buff[buffInfo->at] == '(')) /*bignum*/ + { + buffInfo->at++; + ctx->typeArrayState = READ_NEXT_LINE; + break; } - ctx->typeArrayState = READ_NEXT_BULK; /* fall-thru */ + + snprintf(ctx->errorMsg, sizeof(ctx->errorMsg), + "Invalid Multi-Bulk response. Failed to read Bulk header."); + return RESP_REPLY_ERR; case READ_NEXT_BULK: if ( (res = readRespReplyBulk(ctx, buffInfo)) != RESP_REPLY_OK) @@ -326,7 +343,18 @@ static RespRes readRespReplyBulkArray(RespReaderCtx *ctx, RespReplyBuff *buffInf ctx->typeArrayState = READ_NEXT_BULK_HDR; break; } - ctx->typeArrayState = READ_END; /* fall-through */ + ctx->typeArrayState = READ_END; + break; + + case READ_NEXT_LINE: + if ( (res = readRespReplyLine(ctx, buffInfo)) != RESP_REPLY_OK) + return res; + + if (--ctx->numBulksArray) { + ctx->typeArrayState = READ_NEXT_BULK_HDR; + break; + } + ctx->typeArrayState = READ_END; break; } diff --git a/src/lib/parser.c b/src/lib/parser.c index 3e9c606..b01cc1e 100644 --- a/src/lib/parser.c +++ b/src/lib/parser.c @@ -62,8 +62,10 @@ struct ParsingElementInfo peInfo[PE_MAX] = { [PE_LIST_ZL] = {elementListZL, "elementListZL", "Parsing Ziplist"}, /* hash */ [PE_HASH] = {elementHash, "elementHash", "Parsing Hash"}, + [PE_HASH_META] = {elementHash, "elementHashMeta", "Parsing Hash with expiry on fields"}, [PE_HASH_ZL] = {elementHashZL, "elementHashZL", "Parsing hash Ziplist"}, [PE_HASH_LP] = {elementHashLP, "elementHashLP", "Parsing hash Listpack"}, + [PE_HASH_LP_EX] = {elementHashLPEx, "elementHashLPEx", "Parsing hash ListpackEx (with expiry)"}, [PE_HASH_ZM] = {elementHashZM, "elementHashZM", "Parsing hash Zipmap"}, /* set */ [PE_SET] = {elementSet, "elementSet", "Parsing set"}, @@ -95,8 +97,10 @@ struct ParsingElementInfo peInfo[PE_MAX] = { [PE_RAW_LIST_ZL] = {elementRawListZL, "elementRawListZL", "Parsing raw list ZL (zip list)"}, /* hash */ [PE_RAW_HASH] = {elementRawHash, "elementRawHash", "Parsing raw Hash"}, + [PE_RAW_HASH_META] = {elementRawHash, "elementRawHashMeta", "Parsing raw Hash with expiry"}, [PE_RAW_HASH_ZL] = {elementRawHashZL, "elementRawHashZL", "Parsing raw hash Ziplist"}, [PE_RAW_HASH_LP] = {elementRawHashLP, "elementRawHashLP", "Parsing raw hash Listpack"}, + [PE_RAW_HASH_LP_EX] = {elementRawHashLPEx, "elementRawHashLPEx", "Parsing raw hash ListpackEx"}, [PE_RAW_HASH_ZM] = {elementRawHashZM, "elementRawHashZM", "Parsing raw hash Zipmap"}, /* set */ [PE_RAW_SET] = {elementRawSet, "elementRawSet", "Parsing raw set"}, @@ -557,9 +561,11 @@ _LIBRDB_API int RDB_handleByLevel(RdbParser *p, RdbDataType type, RdbHandlersLev break; case RDB_DATA_TYPE_HASH: p->handleTypeObjByLevel[RDB_TYPE_HASH] = lvl; + p->handleTypeObjByLevel[RDB_TYPE_HASH_METADATA] = lvl; p->handleTypeObjByLevel[RDB_TYPE_HASH_ZIPMAP] = lvl; p->handleTypeObjByLevel[RDB_TYPE_HASH_ZIPLIST] = lvl; p->handleTypeObjByLevel[RDB_TYPE_HASH_LISTPACK] = lvl; + p->handleTypeObjByLevel[RDB_TYPE_HASH_LISTPACK_EX] = lvl; break; case RDB_DATA_TYPE_MODULE: p->handleTypeObjByLevel[RDB_TYPE_MODULE_2] = lvl; @@ -701,12 +707,12 @@ static inline RdbStatus nextParsingElementKeyValue(RdbParser *p, ParsingElementType peRawValue, ParsingElementType peValue) { p->elmCtx.key.handleByLevel = p->handleTypeObjByLevel[p->currOpcode]; - + /* Which level should parse this element (raw or struct/data) */ if (p->handleTypeObjByLevel[p->currOpcode] == RDB_LEVEL_RAW) { - p->elmCtx.key.valueType = peRawValue; + p->elmCtx.key.parsingElemType = peRawValue; return nextParsingElement(p, PE_RAW_NEW_KEY); } else { - p->elmCtx.key.valueType = peValue; + p->elmCtx.key.parsingElemType = peValue; return nextParsingElement(p, PE_NEW_KEY); } } @@ -1041,12 +1047,13 @@ static RdbStatus hashZiplist(RdbParser *p, BulkInfo *ziplistBulk) { RDB_LEVEL_DATA, rdbData.handleHashField, embBulk1.binfo.ref, - embBulk2.binfo.ref); + embBulk2.binfo.ref, + -1 /*no expiry*/); } return RDB_STATUS_OK; } -static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) { +static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk, int withExpiry) { size_t items = 0; if (unlikely(0 == lpValidateIntegrity(lpBulk->ref, lpBulk->len, p->deepIntegCheck, counterCallback, &items))) { @@ -1055,21 +1062,27 @@ static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) { return RDB_STATUS_ERROR; } - if (unlikely((items & 1))) { + /* [field][value][expiry] vs [field][value] */ + int tupleSize = (withExpiry) ? 3 : 2; + if (unlikely((items % tupleSize) != 0)) { RDB_reportError(p, RDB_ERR_HASH_LP_INTEG_CHECK, - "hashListPack(): Listpack integrity check failed. Uneven number of items."); + "hashListPack(): listpack has unexpected number of items."); return RDB_STATUS_ERROR; } if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) { registerAppBulkForNextCb(p, lpBulk); - CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleHashLP, lpBulk->ref); + if (withExpiry) + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleHashLPEx, lpBulk->ref); + else + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleHashLP, lpBulk->ref); return RDB_STATUS_OK; } p->elmCtx.key.numItemsHint = items; unsigned char *iterLP = lpFirst(lpBulk->ref); while (iterLP) { + int64_t expiryVal = -1; unsigned char *field, *value; unsigned int fieldLen, valueLen; long long fieldVal, valueVal; @@ -1079,6 +1092,16 @@ static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) { iterLP = lpNext(lpBulk->ref, iterLP); value = lpGetValue(iterLP, &valueLen, &valueVal); iterLP = lpNext(lpBulk->ref, iterLP); + if (withExpiry) { + if (lpGet(iterLP, &expiryVal, NULL) != NULL) { + RDB_reportError(p, RDB_ERR_HASH_LP_INTEG_CHECK, + "hashListPack(): integrity check failed. Expiry is a string instead of value."); + return RDB_STATUS_ERROR; + } + + if (expiryVal == 0) expiryVal = -1; /* If expiry not set */ + iterLP = lpNext(lpBulk->ref, iterLP); + } if (!allocEmbeddedBulk(p, field, fieldLen, fieldVal, &embBulk1)) return RDB_STATUS_ERROR; @@ -1096,7 +1119,8 @@ static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) { RDB_LEVEL_DATA, rdbData.handleHashField, embBulk1.binfo.ref, - embBulk2.binfo.ref); + embBulk2.binfo.ref, + expiryVal); } return RDB_STATUS_OK; } @@ -1139,7 +1163,8 @@ static RdbStatus hashZipMap(RdbParser *p, BulkInfo *zpBulk) { RDB_LEVEL_DATA, rdbData.handleHashField, embBulk1.binfo.ref, - embBulk2.binfo.ref); + embBulk2.binfo.ref, + -1 /*no expiry*/); } return RDB_STATUS_OK; } @@ -1373,7 +1398,7 @@ RdbStatus elementNewKey(RdbParser *p) { p->elmCtx.key.numItemsHint = -1; /* Read value */ - return nextParsingElement(p, p->elmCtx.key.valueType); + return nextParsingElement(p, p->elmCtx.key.parsingElemType); } /* load an expire-time, associated with the next key to load. */ @@ -1442,8 +1467,10 @@ RdbStatus elementNextRdbType(RdbParser *p) { case RDB_TYPE_LIST_ZIPLIST: return nextParsingElementKeyValue(p, PE_RAW_LIST_ZL, PE_LIST_ZL); /* hash */ case RDB_TYPE_HASH: return nextParsingElementKeyValue(p, PE_RAW_HASH, PE_HASH); + case RDB_TYPE_HASH_METADATA: return nextParsingElementKeyValue(p, PE_RAW_HASH_META, PE_HASH_META); case RDB_TYPE_HASH_ZIPLIST: return nextParsingElementKeyValue(p, PE_RAW_HASH_ZL, PE_HASH_ZL); case RDB_TYPE_HASH_LISTPACK: return nextParsingElementKeyValue(p, PE_RAW_HASH_LP, PE_HASH_LP); + case RDB_TYPE_HASH_LISTPACK_EX: return nextParsingElementKeyValue(p, PE_RAW_HASH_LP_EX, PE_HASH_LP_EX); case RDB_TYPE_HASH_ZIPMAP: return nextParsingElementKeyValue(p, PE_RAW_HASH_ZM, PE_HASH_ZM); /* set */ case RDB_TYPE_SET: return nextParsingElementKeyValue(p, PE_RAW_SET, PE_SET); @@ -1454,9 +1481,12 @@ RdbStatus elementNextRdbType(RdbParser *p) { case RDB_TYPE_ZSET_2: return nextParsingElementKeyValue(p, PE_RAW_ZSET, PE_ZSET); case RDB_TYPE_ZSET_ZIPLIST: return nextParsingElementKeyValue(p, PE_RAW_ZSET_ZL, PE_ZSET_ZL); case RDB_TYPE_ZSET_LISTPACK: return nextParsingElementKeyValue(p, PE_RAW_ZSET_LP, PE_ZSET_LP); - /* module */ case RDB_TYPE_MODULE_2: return nextParsingElementKeyValue(p, PE_RAW_MODULE, PE_MODULE); + /* stream */ + case RDB_TYPE_STREAM_LISTPACKS: + case RDB_TYPE_STREAM_LISTPACKS_2: + case RDB_TYPE_STREAM_LISTPACKS_3: return nextParsingElementKeyValue(p, PE_RAW_STREAM_LP, PE_STREAM_LP); case RDB_OPCODE_MODULE_AUX: if (p->handleTypeObjByLevel[RDB_OPCODE_MODULE_AUX] == RDB_LEVEL_RAW) return nextParsingElement(p, PE_RAW_MODULE_AUX); @@ -1467,11 +1497,6 @@ RdbStatus elementNextRdbType(RdbParser *p) { case RDB_OPCODE_EOF: return nextParsingElement(p, PE_END_OF_FILE); - /* stream */ - case RDB_TYPE_STREAM_LISTPACKS: - case RDB_TYPE_STREAM_LISTPACKS_2: - case RDB_TYPE_STREAM_LISTPACKS_3: return nextParsingElementKeyValue(p, PE_RAW_STREAM_LP, PE_STREAM_LP); - case RDB_OPCODE_FUNCTION: RDB_reportError(p, RDB_ERR_PRERELEASE_FUNC_FORMAT_NOT_SUPPORTED, "Pre-release function format not supported."); @@ -1680,6 +1705,9 @@ RdbStatus elementHash(RdbParser *p) { BulkInfo *binfoField, *binfoValue; while(ctx->hash.visitingField < ctx->hash.numFields) { + uint64_t expireAt = 0; + if (p->parsingElement == PE_HASH_META) + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &expireAt, NULL, NULL)); IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoField)); IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoValue)); @@ -1690,12 +1718,14 @@ RdbStatus elementHash(RdbParser *p) { if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) { CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleHashPlain, binfoField->ref, - binfoValue->ref); + binfoValue->ref, + (expireAt) ? (int64_t) expireAt : -1); } else { CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleHashField, binfoField->ref, - binfoValue->ref); + binfoValue->ref, + (expireAt) ? (int64_t) expireAt : -1); } ++ctx->hash.visitingField; @@ -1724,6 +1754,19 @@ RdbStatus elementHashZL(RdbParser *p) { return nextParsingElement(p, PE_END_KEY); } +RdbStatus elementHashLPEx(RdbParser *p) { + BulkInfo *listpackBulk; + + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &listpackBulk)); + + /*** ENTER SAFE STATE ***/ + + if (RDB_STATUS_ERROR == hashListPack(p, listpackBulk, 1 /*withExpiry*/)) + return RDB_STATUS_ERROR; + + return nextParsingElement(p, PE_END_KEY); +} + RdbStatus elementHashLP(RdbParser *p) { BulkInfo *listpackBulk; @@ -1731,7 +1774,7 @@ RdbStatus elementHashLP(RdbParser *p) { /*** ENTER SAFE STATE ***/ - if (RDB_STATUS_ERROR == hashListPack(p, listpackBulk)) + if (RDB_STATUS_ERROR == hashListPack(p, listpackBulk, 0 /*withExpiry*/)) return RDB_STATUS_ERROR; return nextParsingElement(p, PE_END_KEY); diff --git a/src/lib/parser.h b/src/lib/parser.h index 1059dde..b656cad 100644 --- a/src/lib/parser.h +++ b/src/lib/parser.h @@ -140,8 +140,10 @@ typedef enum ParsingElementType { PE_QUICKLIST, PE_LIST_ZL, PE_HASH, + PE_HASH_META, PE_HASH_ZL, PE_HASH_LP, + PE_HASH_LP_EX, PE_HASH_ZM, PE_SET, PE_SET_IS, @@ -163,8 +165,10 @@ typedef enum ParsingElementType { PE_RAW_QUICKLIST, PE_RAW_LIST_ZL, PE_RAW_HASH, + PE_RAW_HASH_META, PE_RAW_HASH_ZL, PE_RAW_HASH_LP, + PE_RAW_HASH_LP_EX, PE_RAW_HASH_ZM, PE_RAW_SET, PE_RAW_SET_IS, @@ -228,7 +232,7 @@ typedef struct { typedef struct { RdbKeyInfo info; - ParsingElementType valueType; + ParsingElementType parsingElemType; RdbHandlersLevel handleByLevel; int64_t numItemsHint; /* hint for the total number of items in the current parsed key. -1 if unknown */ } ElementKeyCtx; @@ -532,6 +536,7 @@ RdbStatus elementListZL(RdbParser *p); RdbStatus elementHash(RdbParser *p); RdbStatus elementHashZL(RdbParser *p); RdbStatus elementHashLP(RdbParser *p); +RdbStatus elementHashLPEx(RdbParser *p); RdbStatus elementHashZM(RdbParser *p); RdbStatus elementSet(RdbParser *p); RdbStatus elementSetIS(RdbParser *p); @@ -553,6 +558,7 @@ RdbStatus elementRawListZL(RdbParser *p); RdbStatus elementRawHash(RdbParser *p); RdbStatus elementRawHashZL(RdbParser *p); RdbStatus elementRawHashLP(RdbParser *p); +RdbStatus elementRawHashLPEx(RdbParser *p); RdbStatus elementRawHashZM(RdbParser *p); RdbStatus elementRawSet(RdbParser *p); RdbStatus elementRawSetIS(RdbParser *p); diff --git a/src/lib/parserRaw.c b/src/lib/parserRaw.c index b4ce62f..a304eea 100644 --- a/src/lib/parserRaw.c +++ b/src/lib/parserRaw.c @@ -436,14 +436,18 @@ RdbStatus elementRawListZL(RdbParser *p) { } RdbStatus elementRawHash(RdbParser *p) { + uint64_t expireAt; + int numDigits; size_t len; unsigned char *unusedData; enum RAW_HASH_STATES { ST_RAW_HASH_HEADER=0, /* Retrieve number of nodes */ - ST_RAW_HASH_READ_NEXT_FIELD_STR, /* Call PE_RAW_STRING as sub-element (read field) */ - ST_RAW_HASH_READ_NEXT_VALUE_STR, /* Return from sub-element. - * Call PE_RAW_STRING as sub-element (read value) */ + ST_RAW_HASH_READ_NEXT_EXPIRE, /* Read hash-field expiry (if PE_RAW_HASH_META) */ + ST_RAW_HASH_READ_NEXT_FIELD_STR, /* Call sub-element to read field */ + ST_RAW_HASH_READ_NEXT_VALUE_STR, /* Returned from sub-element. Done reading field. + * Call sub-element to read value */ + ST_RAW_HASH_DONE_READ_VALUE_STR /* Return from sub-element. Done reading value */ } ; ElementRawHashCtx *hashCtx = &p->elmCtx.rawHash; @@ -451,45 +455,60 @@ RdbStatus elementRawHash(RdbParser *p) { switch (p->elmCtx.state) { - case ST_RAW_HASH_HEADER: { - int headerLen = 0; - + case ST_RAW_HASH_HEADER: + numDigits = 0; aggMakeRoom(p, 10); /* worse case 9 bytes for len */ - IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &hashCtx->numFields, (unsigned char *) rawCtx->at, &headerLen)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &hashCtx->numFields, + (unsigned char *) rawCtx->at, &numDigits)); /*** ENTER SAFE STATE ***/ hashCtx->visitField = 0; IF_NOT_OK_RETURN(cbHandleBegin(p, DATA_SIZE_UNKNOWN_AHEAD)); - IF_NOT_OK_RETURN(aggUpdateWritten(p, headerLen)); - } - updateElementState(p, ST_RAW_HASH_READ_NEXT_FIELD_STR, 0); /* fall-thru */ + IF_NOT_OK_RETURN(aggUpdateWritten(p, numDigits)); - case ST_RAW_HASH_READ_NEXT_FIELD_STR: - /*** ENTER SAFE STATE ***/ + if (hashCtx->numFields == 0) + return nextParsingElement(p, PE_RAW_END_KEY); /* empty-key */ - /* if reached this state from ST_RAW_HASH_READ_NEXT_VALUE_STR */ - if (hashCtx->visitField > 0) { - /* return from sub-element string parsing */ - subElementCallEnd(p, (char **) &unusedData, &len); - } + updateElementState(p, ST_RAW_HASH_READ_NEXT_EXPIRE, 0); /* fall-thru */ - if (hashCtx->visitField++ == hashCtx->numFields) - return nextParsingElement(p, PE_RAW_END_KEY); /* done */ + case ST_RAW_HASH_READ_NEXT_EXPIRE: + if (p->parsingElement == PE_RAW_HASH_META) { + numDigits = 0; + aggMakeRoom(p, 32); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &expireAt, + (unsigned char *) rawCtx->at, + &numDigits)); + /*** ENTER SAFE STATE ***/ + IF_NOT_OK_RETURN(aggUpdateWritten(p, numDigits)); + } + updateElementState(p, ST_RAW_HASH_READ_NEXT_FIELD_STR, 0); /* fall-thru */ + case ST_RAW_HASH_READ_NEXT_FIELD_STR: + /*** ENTER SAFE STATE (no rdb read)***/ return subElementCall(p, PE_RAW_STRING, ST_RAW_HASH_READ_NEXT_VALUE_STR); - case ST_RAW_HASH_READ_NEXT_VALUE_STR: { - + case ST_RAW_HASH_READ_NEXT_VALUE_STR: /*** ENTER SAFE STATE (no rdb read)***/ + /* returned from sub-element. Done reading field. */ + subElementCallEnd(p, (char **) &unusedData, &len); + return subElementCall(p, PE_RAW_STRING, ST_RAW_HASH_DONE_READ_VALUE_STR); + case ST_RAW_HASH_DONE_READ_VALUE_STR: + /*** ENTER SAFE STATE (no rdb read)***/ /* return from sub-element string parsing */ subElementCallEnd(p, (char **) &unusedData, &len); - return subElementCall(p, PE_RAW_STRING, ST_RAW_HASH_READ_NEXT_FIELD_STR); - } + if (++hashCtx->visitField == hashCtx->numFields) + return nextParsingElement(p, PE_RAW_END_KEY); /* done */ + + /* More fields to read. Distinct between meta and plain hash */ + if (p->parsingElement == PE_RAW_HASH_META) + return updateElementState(p, ST_RAW_HASH_READ_NEXT_EXPIRE, 0); + else + return updateElementState(p, ST_RAW_HASH_READ_NEXT_FIELD_STR, 0); default: RDB_reportError(p, RDB_ERR_PLAIN_HASH_INVALID_STATE, @@ -506,6 +525,10 @@ RdbStatus elementRawHashLP(RdbParser *p) { return singleStringTypeHandling(p, listpackValidateIntegrityCb, "elementRawHashLP"); } +RdbStatus elementRawHashLPEx(RdbParser *p) { + return singleStringTypeHandling(p, listpackValidateIntegrityCb, "elementRawHashLPEx"); +} + RdbStatus elementRawHashZM(RdbParser *p) { return singleStringTypeHandling(p, zipmapValidateIntegrityCb, "elementRawHashZM"); } diff --git a/test/dumps/hash_data.json b/test/dumps/hash_data.json new file mode 100644 index 0000000..7236adf --- /dev/null +++ b/test/dumps/hash_data.json @@ -0,0 +1,8 @@ +"__aux__" : { + "redis-ver":"3.2.1", + "redis-bits":"64", + "ctime":"1689605183", + "used-mem":"821904" +}, + +"myhash1":{"5":"5","6":"6","10":"10","1":"1","2":"2","8":"8","4":"4","9":"9","3":"3","7":"7","11":"11"} diff --git a/test/dumps/hash_ex_v12_data.json b/test/dumps/hash_ex_v12_data.json new file mode 100644 index 0000000..6afd22a --- /dev/null +++ b/test/dumps/hash_ex_v12_data.json @@ -0,0 +1,8 @@ +"__aux__" : { +"redis-ver":"255.255.255", +"redis-bits":"64", +"ctime":"1718101998", +"used-mem":"1168904", +"aof-base":"0" +}, +"myhash":{"field1":"value1","field3":"value3","field2":"value2"} diff --git a/test/dumps/hash_lp_ex_v12_data.json b/test/dumps/hash_lp_ex_v12_data.json new file mode 100644 index 0000000..aa015e3 --- /dev/null +++ b/test/dumps/hash_lp_ex_v12_data.json @@ -0,0 +1,9 @@ +"__aux__" : { + "redis-ver":"255.255.255", + "redis-bits":"64", + "ctime":"1718036120", + "used-mem":"1191512", + "aof-base":"0" +}, + +"myhash":{"field2":"value2","field1":"value1","field3":"value3"} diff --git a/test/dumps/hash_lp_with_hexpire_v12.rdb b/test/dumps/hash_lp_with_hexpire_v12.rdb new file mode 100644 index 0000000000000000000000000000000000000000..03b0fc2b5cf288a437db54545ad162fae8108d77 GIT binary patch literal 169 zcmWG?b@2=~FfcUw#aWb^l3A=P;l-K50g)B_Cv7=AM{h_mHZW+WD89JX@smGNa@VBlnE zOUq2nNikw?D@)8NO*LZw(*N$?|Nr(1QVo&}h6^N_4cWi^cVKwHD9K>FK$69noq_TH O|DE2`&fWgQ@&Ew8Mm_WZ literal 0 HcmV?d00001 diff --git a/test/dumps/plain_hash_raw.json b/test/dumps/hash_raw.json similarity index 100% rename from test/dumps/plain_hash_raw.json rename to test/dumps/hash_raw.json diff --git a/test/dumps/hash_struct.json b/test/dumps/hash_struct.json new file mode 100644 index 0000000..7236adf --- /dev/null +++ b/test/dumps/hash_struct.json @@ -0,0 +1,8 @@ +"__aux__" : { + "redis-ver":"3.2.1", + "redis-bits":"64", + "ctime":"1689605183", + "used-mem":"821904" +}, + +"myhash1":{"5":"5","6":"6","10":"10","1":"1","2":"2","8":"8","4":"4","9":"9","3":"3","7":"7","11":"11"} diff --git a/test/dumps/plain_hash_v3.rdb b/test/dumps/hash_v3.rdb similarity index 100% rename from test/dumps/plain_hash_v3.rdb rename to test/dumps/hash_v3.rdb diff --git a/test/dumps/hash_with_expire_v12.rdb b/test/dumps/hash_with_expire_v12.rdb new file mode 100644 index 0000000000000000000000000000000000000000..a1c6ec028b73b12819209bbb390a618030aa3896 GIT binary patch literal 169 zcmWG?b@2=~FfcUw#aWb^l3A=}79Etg9x=D$}sRtPTG5ls^5M#@&%t$QGU~Xh!u>b%6|C8lx rX_=`xDTZuii8-aIh71s{F@$RjRrSapB5VW^Hv0dMiI-Dtt)U74h6O(p literal 0 HcmV?d00001 diff --git a/test/test_common.c b/test/test_common.c index 5754aca..2ff13f8 100644 --- a/test/test_common.c +++ b/test/test_common.c @@ -180,7 +180,7 @@ void assert_file_payload(const char *filename, char *expData, int expLen, MatchT printf("%s\n---- file [%s] ----\n", errMsg, filename); printHexDump(filedata, filelen, buf, (int) sizeof(buf)); printf("%s", buf); - printf("\n---- Expected %s %s ----\n", matchTypeName, (expMatch) ? "" : "not to match"); + printf("\n---- Expected %s %s ----\n", matchTypeName, (expMatch) ? "" : "NOT to match"); printHexDump(expData, expLen, buf, (int) sizeof(buf)); printf("%s", buf); printf("\n------------\n"); @@ -281,12 +281,15 @@ char *sendRedisCmd(char *cmd, int expRetType, char *expRsp) { size_t written = serializeRedisReply(reply, rspbuf, sizeof(rspbuf)-1); + /* Check if the response contains the expected substring */ if (expRsp) { - /* For complex responses, check `expRsp` is a substring. Otherwise, exact match */ - if (expRetType != REDIS_REPLY_ARRAY) + if (expRetType == REDIS_REPLY_INTEGER) { + char str[21]; + sprintf(str, "%lld", reply->integer); + assert_string_equal(str, expRsp); + } else if (expRetType != REDIS_REPLY_ARRAY) { assert_string_equal(reply->str, expRsp); - else { - + } else { if (NULL == substring(rspbuf, written, expRsp)) { printf("Error: Response does not contain expected substring.\n"); printf("Actual Response: %s\n", rspbuf); @@ -341,16 +344,14 @@ void get_redis_version(redisContext *c, int *majorptr, int *minorptr) { exit(1); } +#define MAX_ARGS 50 void setupRedisServer(const char *extraArgs) { - - /* If redis not installed return gracefully */ if (!redisInstallFolder) return; - /* execl() not accept empty string */ const char *_extraArgs = (extraArgs) ? extraArgs : "--loglevel verbose"; pid_t pid = fork(); - assert_int_not_equal (pid, -1); + assert(pid != -1); int port = findFreePort(6500, 6600); @@ -361,29 +362,38 @@ void setupRedisServer(const char *extraArgs) { snprintf(testrdbModulePath, sizeof(testrdbModulePath), "%s/../tests/modules/testrdb.so", redisInstallFolder); snprintf(redisPortStr, sizeof(redisPortStr), "%d", port); - /* if module testrdb.so exists (ci.yaml takes care to build testrdb), part - * of redis repo testing, then load it for test_rdb_to_redis_module. The - * test will run only if testrdb appear in the server "MODULE LIST", - * otherwise skipped gracefully. */ + // Tokenize extraArgs and build the arguments list + char *args[MAX_ARGS]; + int argIndex = 0; + + args[argIndex++] = fullpath; + args[argIndex++] = "--port"; + args[argIndex++] = redisPortStr; + args[argIndex++] = "--dir"; + args[argIndex++] = "./test/tmp/"; + args[argIndex++] = "--logfile"; + args[argIndex++] = "./redis.log"; + + // Add module loading arguments if the module exists if (access(testrdbModulePath, F_OK) != -1) { - execl(fullpath, fullpath, - "--port", redisPortStr, - "--dir", "./test/tmp/", - "--logfile", "./redis.log", - "--loadmodule", testrdbModulePath, "4", - _extraArgs, - (char *) NULL); - } else { - execl(fullpath, fullpath, - "--port", redisPortStr, - "--dir", "./test/tmp/", - "--logfile", "./redis.log", - _extraArgs, - (char *) NULL); - } - - /* If execl returns, an error occurred! */ - perror("execl"); + args[argIndex++] = "--loadmodule"; + args[argIndex++] = testrdbModulePath; + args[argIndex++] = "4"; + } + + /* Tokenize extraArgs and add to the arguments list */ + char *extraArgsCopy = strdup(_extraArgs); + char *token = strtok(extraArgsCopy, " "); + while (token && argIndex < MAX_ARGS - 1) { + args[argIndex++] = token; + token = strtok(NULL, " "); + } + args[argIndex] = NULL; + + execvp(fullpath, args); + + /* If execvp returns, an error occurred */ + perror("execvp"); exit(1); } else { /* parent */ int retryCount = 3; diff --git a/test/test_rdb_to_json.c b/test/test_rdb_to_json.c index a5ffd8e..bb57452 100644 --- a/test/test_rdb_to_json.c +++ b/test/test_rdb_to_json.c @@ -156,22 +156,29 @@ static void test_r2j_plain_list_raw (void **state) { testRdbToJsonCommon(DUMP_FOLDER("plain_list_v6.rdb"), DUMP_FOLDER("plain_list_v6_raw.json"), &r2jConf); } -static void test_r2j_plain_hash_data(void **state) { +static void test_r2j_hash_data(void **state) { UNUSED(state); RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA); - testRdbToJsonCommon(DUMP_FOLDER("plain_hash_v3.rdb"), DUMP_FOLDER("plain_hash_data.json"), &r2jConf); + testRdbToJsonCommon(DUMP_FOLDER("hash_v3.rdb"), DUMP_FOLDER("hash_data.json"), &r2jConf); } -static void test_r2j_plain_hash_struct(void **state) { +static void test_r2j_hash_struct(void **state) { UNUSED(state); RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_STRUCT); - testRdbToJsonCommon(DUMP_FOLDER("plain_hash_v3.rdb"), DUMP_FOLDER("plain_hash_struct.json"), &r2jConf); + testRdbToJsonCommon(DUMP_FOLDER("hash_v3.rdb"), DUMP_FOLDER("hash_struct.json"), &r2jConf); } -static void test_r2j_plain_hash_raw (void **state) { +static void test_r2j_hash_raw (void **state) { UNUSED(state); RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_RAW); - testRdbToJsonCommon(DUMP_FOLDER("plain_hash_v3.rdb"), DUMP_FOLDER("plain_hash_raw.json"), &r2jConf); + testRdbToJsonCommon(DUMP_FOLDER("hash_v3.rdb"), DUMP_FOLDER("hash_raw.json"), &r2jConf); +} + +/* hash with expiry on fields */ +static void test_r2j_hash_ex_data(void **state) { + UNUSED(state); + RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA); + testRdbToJsonCommon(DUMP_FOLDER("hash_with_expire_v12.rdb"), DUMP_FOLDER("hash_ex_v12_data.json"), &r2jConf); } static void test_r2j_hash_zl_data(void **state) { @@ -198,6 +205,13 @@ static void test_r2j_hash_lp_data(void **state) { testRdbToJsonCommon(DUMP_FOLDER("hash_lp_v11.rdb"), DUMP_FOLDER("hash_lp_v11_data.json"), &r2jConf); } +/* lp with expiry on fields */ +static void test_r2j_hash_lp_ex_data(void **state) { + UNUSED(state); + RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA); + testRdbToJsonCommon(DUMP_FOLDER("hash_lp_with_hexpire_v12.rdb"), DUMP_FOLDER("hash_lp_ex_v12_data.json"), &r2jConf); +} + static void test_r2j_hash_lp_struct(void **state) { UNUSED(state); RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_STRUCT); @@ -511,9 +525,11 @@ int group_rdb_to_json(void) { cmocka_unit_test(test_r2j_plain_list_raw), /* hash */ - cmocka_unit_test(test_r2j_plain_hash_data), - cmocka_unit_test(test_r2j_plain_hash_struct), - cmocka_unit_test(test_r2j_plain_hash_raw), + cmocka_unit_test(test_r2j_hash_data), + cmocka_unit_test(test_r2j_hash_struct), + cmocka_unit_test(test_r2j_hash_raw), + + cmocka_unit_test(test_r2j_hash_ex_data), cmocka_unit_test(test_r2j_hash_zl_data), cmocka_unit_test(test_r2j_hash_zl_struct), @@ -523,6 +539,8 @@ int group_rdb_to_json(void) { cmocka_unit_test(test_r2j_hash_lp_struct), cmocka_unit_test(test_r2j_hash_lp_raw), + cmocka_unit_test(test_r2j_hash_lp_ex_data), + cmocka_unit_test(test_r2j_hash_zm_data), cmocka_unit_test(test_r2j_hash_zm_struct), cmocka_unit_test(test_r2j_hash_zm_raw), diff --git a/test/test_rdb_to_redis.c b/test/test_rdb_to_redis.c index ad79ef4..65c1fde 100644 --- a/test/test_rdb_to_redis.c +++ b/test/test_rdb_to_redis.c @@ -6,7 +6,7 @@ #include #include "test_common.h" -int majorVersion; +int serverMajorVer, serverMinorVer; void dummyLogger(RdbLogLevel l, const char *msg) { UNUSED(l, msg); } @@ -16,7 +16,7 @@ static int setupTest(void **state) { sendRedisCmd("SAVE", REDIS_REPLY_STATUS, NULL); /* FUNCTION FLUSH if redis version is 7.0 or higher */ - if (majorVersion >= 7) sendRedisCmd("FUNCTION FLUSH", REDIS_REPLY_STATUS, NULL); + if (serverMajorVer >= 7) sendRedisCmd("FUNCTION FLUSH", REDIS_REPLY_STATUS, NULL); return 0; } @@ -67,6 +67,34 @@ static void rdb_to_json(const char *rdbfile, const char *outfile) { RDB_deleteParser(parser); } +/* Is saving RDB, and librdb reload generates same digest + * + * isDigest - if set, compare DB digest before and after reload + * isRestore - if set, use RESTORE command after reload. Otherwise, plain commands + */ +static void rdb_save_librdb_reload_eq(int isRestore, char *serverRdbFile) { + char *res; + const char *rdbfile = TMP_FOLDER("reload.rdb"); + char expectedSha[100]; + + /* Calculate DB isDigest */ + res = sendRedisCmd("DEBUG DIGEST", REDIS_REPLY_STATUS, NULL); + memcpy(expectedSha, res, strlen(res) + 1); + + /* Keep aside rdb file */ + sendRedisCmd("SAVE", REDIS_REPLY_STATUS, NULL); + runSystemCmd("rm %s || true", rdbfile); + runSystemCmd("cp %s %s > /dev/null", serverRdbFile, rdbfile); + + /* Flush Redis */ + sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL); + + /* Reload the RDB file */ + rdb_to_tcp(rdbfile, 1, isRestore, NULL); + + sendRedisCmd("DEBUG DIGEST", REDIS_REPLY_STATUS, expectedSha); +} + /* * Testing RESP against live server: * 1. Convert RDB to Json (out1.json) @@ -90,7 +118,7 @@ static void test_rdb_to_redis_common(const char *rdbfile, int ignoreListOrder, c sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL); /* FUNCTION FLUSH */ - if (majorVersion >= 7) + if (serverMajorVer >= 7) sendRedisCmd("FUNCTION FLUSH", REDIS_REPLY_STATUS, NULL); /* 1. Convert RDB to Json (out1.json) */ @@ -154,9 +182,41 @@ static void test_rdb_to_redis_single_ziplist(void **state) { test_rdb_to_redis_common(DUMP_FOLDER("ziplist_v3.rdb"), 0, "$5\r\nRPUSH", NULL); } -static void test_rdb_to_redis_plain_hash(void **state) { +static void test_rdb_to_redis_hash(void **state) { UNUSED(state); - test_rdb_to_redis_common(DUMP_FOLDER("plain_hash_v3.rdb"), 0, "$4\r\nHSET", NULL); + test_rdb_to_redis_common(DUMP_FOLDER("hash_v3.rdb"), 0, "$4\r\nHSET", NULL); +} + +static void test_rdb_to_redis_hash_with_expire(void **state) { + UNUSED(state); + + /* hash-field-expiration available since 7.4 */ + if ((serverMajorVer<7) || ((serverMajorVer==7) && (serverMinorVer<4))) + skip(); + + setupRedisServer("--enable-debug-command yes --dbfilename expire.rdb"); + + /* listpack */ + sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL); + sendRedisCmd("CONFIG SET HASH-MAX-LISTPACK-ENTRIES 512", REDIS_REPLY_STATUS, NULL); + sendRedisCmd("HSET myhash f4 v1 f5 v2 f6 v3", REDIS_REPLY_INTEGER, "3"); + sendRedisCmd("HPEXPIREAT myhash 70368744177663 FIELDS 2 f4 f5", REDIS_REPLY_ARRAY, "1 1"); + rdb_save_librdb_reload_eq(0 /*restore*/, TMP_FOLDER("expire.rdb")); + rdb_save_librdb_reload_eq(1 /*restore*/, TMP_FOLDER("expire.rdb")); + sendRedisCmd("HPEXPIRETIME myhash FIELDS 3 f4 f5 f6", REDIS_REPLY_ARRAY, + "70368744177663 70368744177663 -1"); /* verify expected output */ + + /* dict (max-lp-entries=0) */ + sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL); + sendRedisCmd("CONFIG SET HASH-MAX-LISTPACK-ENTRIES 0", REDIS_REPLY_STATUS, NULL); + sendRedisCmd("HSET myhash f4 v1 f5 v2 f6 v3", REDIS_REPLY_INTEGER, "3"); + sendRedisCmd("HPEXPIREAT myhash 70368744177663 FIELDS 2 f4 f5", REDIS_REPLY_ARRAY, "1 1"); + rdb_save_librdb_reload_eq(0 /*restore*/, TMP_FOLDER("expire.rdb")); + rdb_save_librdb_reload_eq(1 /*restore*/, TMP_FOLDER("expire.rdb")); + sendRedisCmd("HPEXPIRETIME myhash FIELDS 3 f4 f5 f6", REDIS_REPLY_ARRAY, + "70368744177663 70368744177663 -1"); /* verify expected output */ + + teardownRedisServer(); } static void test_rdb_to_redis_hash_zl(void **state) { @@ -241,7 +301,7 @@ static void test_rdb_to_redis_policy_lru(void **state) { static void test_rdb_to_redis_function(void **state) { UNUSED(state); /* function available since 7.0 */ - if (majorVersion < 7) + if (serverMajorVer < 7) skip(); test_rdb_to_redis_common(DUMP_FOLDER("function.rdb"), 1, NULL, NULL); @@ -344,7 +404,7 @@ static void test_rdb_to_redis_func_lib_replace_if_exist(void **state) { int expectedStatus[] = {RDB_STATUS_OK, RDB_STATUS_ERROR, RDB_STATUS_OK}; /* function available since 7.0 */ - if (majorVersion < 7) + if (serverMajorVer < 7) skip(); for (int i = 0 ; i < 3 ; i++) { @@ -396,7 +456,7 @@ int group_rdb_to_redis(void) { return 0; } - getTargetRedisVersion(&majorVersion, NULL); + getTargetRedisVersion(&serverMajorVer, &serverMinorVer); const struct CMUnitTest tests[] = { /* string */ @@ -407,7 +467,8 @@ int group_rdb_to_redis(void) { cmocka_unit_test_setup(test_rdb_to_redis_quicklist, setupTest), cmocka_unit_test_setup(test_rdb_to_redis_single_ziplist, setupTest), /* hash */ - cmocka_unit_test_setup(test_rdb_to_redis_plain_hash, setupTest), + cmocka_unit_test_setup(test_rdb_to_redis_hash, setupTest), + cmocka_unit_test_setup(test_rdb_to_redis_hash_with_expire, setupTest), cmocka_unit_test_setup(test_rdb_to_redis_hash_zl, setupTest), cmocka_unit_test_setup(test_rdb_to_redis_hash_lp, setupTest), cmocka_unit_test_setup(test_rdb_to_redis_hash_zm, setupTest), diff --git a/test/test_rdb_to_resp.c b/test/test_rdb_to_resp.c index 5e35c5f..e4fbd43 100644 --- a/test/test_rdb_to_resp.c +++ b/test/test_rdb_to_resp.c @@ -187,9 +187,9 @@ static void test_r2r_multiple_lists_and_strings(void **state) { runWithAndWithoutRestore("multiple_lists_strings.rdb"); } -static void test_r2r_plain_hash(void **state) { +static void test_r2r_hash(void **state) { UNUSED(state); - runWithAndWithoutRestore("plain_hash_v3.rdb"); + runWithAndWithoutRestore("hash_v3.rdb"); } static void test_r2r_hash_zl(void **state) { @@ -252,6 +252,19 @@ static void test_r2r_misc_with_stream(void **state) { runWithAndWithoutRestore("misc_with_stream.rdb"); } +/* Test cases for hash with expire */ +static void test_r2r_hash_ex(void **state) { + UNUSED(state); + runWithAndWithoutRestore("hash_with_expire_v12.rdb"); +} + +/* Test cases for hash-listpack with expire. Supported since 7.4 */ +static void test_r2r_hash_lp_ex(void **state) { + UNUSED(state); + runWithAndWithoutRestore("hash_lp_with_hexpire_v12.rdb"); +} + + static void test_r2r_module(void **state) { UNUSED(state); unsigned char expRespData[] = { @@ -317,10 +330,12 @@ int group_rdb_to_resp(void) { cmocka_unit_test(test_r2r_quicklist2_list), cmocka_unit_test(test_r2r_list_ziplist), /* hash */ - cmocka_unit_test(test_r2r_plain_hash), + cmocka_unit_test(test_r2r_hash), cmocka_unit_test(test_r2r_hash_zl), cmocka_unit_test(test_r2r_hash_lp), cmocka_unit_test(test_r2r_hash_zm), + cmocka_unit_test(test_r2r_hash_ex), + cmocka_unit_test(test_r2r_hash_lp_ex), /* set */ cmocka_unit_test(test_r2r_plain_set), cmocka_unit_test(test_r2r_set_is), diff --git a/test/test_resp_reader.c b/test/test_resp_reader.c index 692cf55..73c276c 100644 --- a/test/test_resp_reader.c +++ b/test/test_resp_reader.c @@ -149,6 +149,19 @@ static void test_mixture_and_fragmented(void **state) { } } +static void test_reply_array_misc_data_types (void **state) { + UNUSED(state); + RespReaderCtx ctx; + test_resp_reader_common(&ctx, STR_AND_SIZE("*6\r\n(34928903284"), + 1, RESP_REPLY_PARTIAL, 0); + test_resp_reader_common(&ctx, STR_AND_SIZE("0923850932485094385094"), + 0, RESP_REPLY_PARTIAL, 0); + test_resp_reader_common(&ctx, STR_AND_SIZE("3825024385\r\n#t\r"), + 0, RESP_REPLY_PARTIAL, 0); + test_resp_reader_common(&ctx, STR_AND_SIZE("\n_\r\n,1.23\r\n:1\r\n:1\r\n"), + 0, RESP_REPLY_OK, 1); +} + /*************************** group_test_resp_reader *******************************/ int group_test_resp_reader(void) { const struct CMUnitTest tests[] = { @@ -166,6 +179,7 @@ int group_test_resp_reader(void) { cmocka_unit_test(test_single_bulk), cmocka_unit_test(test_three_bulks), cmocka_unit_test(test_mixture_and_fragmented), + cmocka_unit_test(test_reply_array_misc_data_types), }; return cmocka_run_group_tests(tests, NULL, NULL);