diff --git a/README.md b/README.md index c929c62..752567e 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ Install and run CLI extension of this library. Parse RDB file to json: }] -Run CLI extension to generate RESP commands +Run CLI extension to generate RESP commands (this time read file from standard input): - % rdb-cli ./test/dumps/multiple_lists_strings.rdb resp + % gzip -dc multiple_lists_strings.rdb.gz | rdb-cli - resp *3 $3 SET @@ -190,8 +190,8 @@ core library vs. extension library ("RDB" vs "RDBX"). - Parsing RDB file with user callbacks: - RdbRes myHandleNewKey(RdbParser *parser, void *userData, RdbBulk key,...) { - printf("%s\n", key); + RdbRes myHandleNewKey(RdbParser *parser, void *userData, RdbBulk key, RdbKeyInfo *info) { + printf("KEY=%s\n", key); return RDB_OK; } @@ -207,7 +207,7 @@ core library vs. extension library ("RDB" vs "RDBX"). RdbParser *parser = RDB_createParserRdb(NULL); RDBX_createReaderFile(parser, "dump.rdb"); RDBX_createHandlersToJson(parser, "redis.json", NULL); - RDBX_createHandlersFilterKey(parser, "id_*", 0); + RDBX_createHandlersFilterKey(parser, "id_*", 0 /*exclude*/); RDB_parse(parser); RDB_deleteParser(parser); @@ -228,28 +228,39 @@ destruction, or when newer block replacing old one. Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {json|resp|redis} [FORMAT_OPTIONS] OPTIONS: - -k, --filter-key Filter keys using regular expressions - -l, --log-file Path to the log file (Default: './rdb-cli.log') + -l, --log-file Path to the log file (Default: './rdb-cli.log') - FORMAT_OPTIONS ('json'): - -w, --with-aux-values Include auxiliary values - -f, --flatten Print flatten json, without DBs Parenthesis - -o, --output Specify the output file. If not specified, output goes to stdout + Multiple filters combination of keys/types/dbs can be specified: + -k, --key Include only keys that match REGEX + -K --no-key Exclude keys that match REGEX + -t, --type Include only selected TYPE {str|list|set|zset|hash|module|func} + -T, --no-type Exclude TYPE {str|list|set|zset|hash|module|func} + -d, --dbnum Include only selected db number + -D, --no-dbnum Exclude DB number - FORMAT_OPTIONS ('resp'): - -r, --support-restore Use the RESTORE command when possible - -t, --target-redis-ver Specify the target Redis version. Helps determine which commands can - be applied. Particularly crucial if support-restore being used - as RESTORE is closely tied to specific RDB versions. If versions not - aligned the parser will generate higher-level commands instead. - -o, --output Specify the output file. If not specified, output goes to stdout + FORMAT_OPTIONS ('json'): + -i, --include To include: {aux-val|func} + -f, --flatten Print flatten json, without DBs Parenthesis + -o, --output Specify the output file. If not specified, output to stdout FORMAT_OPTIONS ('redis'): - -r, --support-restore Use the RESTORE command when possible - -t, --target-redis-ver Specify the target Redis version - -h, --hostname Specify the server hostname (default: 127.0.0.1) - -p, --port Specify the server port (default: 6379) - -l, --pipeline-depth Number of pending commands before blocking for responses + -h, --hostname Specify the server hostname (default: 127.0.0.1) + -p, --port Specify the server port (default: 6379) + -l, --pipeline-depth Number of pending commands before blocking for responses + -u, --user Redis username for authentication + -P, --password Redis password for authentication + -a, --auth N [ARG1 ... ARGN] An alternative authentication command. Given as vector of arguments + + FORMAT_OPTIONS ('redis'|'resp'): + -r, --support-restore Use the RESTORE command when possible + -t, --target-redis-ver Specify the target Redis version. Helps determine which commands can + be applied. Particularly crucial if support-restore being used + as RESTORE is closely tied to specific RDB versions. If versions not + aligned the parser will generate higher-level commands instead. + -o, --output Specify the output file (For 'resp' only: if not specified, output to stdout) + -s, --start-cmd-num Start writing redis from command number + -e, --enum-commands Command enumeration and tracing by preceding each generated RESP command + with debug command of type: `SET _RDB_CLI_CMD_ID_ ` ## Advanced @@ -303,12 +314,12 @@ to give this indication and register it as well. ### Pause parser and resume At times, the application may need to execute additional tasks during parsing intervals, -such as updating a progress bar or performing other computations. To facilitate this, the -parser can be configured with a pause interval that specifies the number of bytes to be -read from RDB source before pausing. This means that each time the parser is invoked, it -will continue parsing until it has read a number of bytes equal to or greater than the -configured interval, at which point it will automatically pause and return -'RDB_STATUS_PAUSED' in order to allow the application to perform other tasks. Example: +such as updating a progress bar or verifying that used memory remains within limit. To +facilitate this, the parser can be configured with a pause interval that specifies the +number of bytes to be read from RDB source before pausing. This means that each time the +parser is invoked, it will continue parsing until it has read a number of bytes equal to +or greater than the configured interval, at which point it will automatically pause and +return 'RDB_STATUS_PAUSED' in order to allow the application to perform other tasks. Example: size_t intervalBytes = 1048576; RdbParser *parser = RDB_createParserRdb(memAlloc); diff --git a/api/librdb-ext-api.h b/api/librdb-ext-api.h index b8d0f16..0a2049a 100644 --- a/api/librdb-ext-api.h +++ b/api/librdb-ext-api.h @@ -12,7 +12,7 @@ extern "C" { typedef struct RdbxRespToFileWriter RdbxRespToFileWriter; typedef struct RdbxReaderFile RdbxReaderFile; typedef struct RdbxReaderFileDesc RdbxReaderFileDesc; -typedef struct RdbxFilterKey RdbxFilterKey; +typedef struct RdbxFilter RdbxFilter; typedef struct RdbxToJson RdbxToJson; typedef struct RdbxToResp RdbxToResp; typedef struct RdbxRespToRedisLoader RdbxRespToRedisLoader; @@ -31,6 +31,7 @@ typedef enum { /* HandlersFilterKey errors */ RDBX_ERR_FILTER_FAILED_COMPILE_REGEX, + RDBX_ERR_FAILED_CREATE_FILTER, /* rdb2resp errors */ @@ -79,9 +80,17 @@ _LIBRDB_API RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, * Create Filter Handlers ****************************************************************/ -_LIBRDB_API RdbxFilterKey *RDBX_createHandlersFilterKey(RdbParser *p, +_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterKey(RdbParser *p, const char *keyRegex, - uint32_t flags); + uint32_t exclude); + +_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterType(RdbParser *p, + RdbDataType type, + uint32_t exclude); + +_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterDbNum(RdbParser *p, + int dbnum, + uint32_t exclude); /**************************************************************** * Create RDB to RESP Handlers @@ -122,11 +131,11 @@ typedef struct RdbxToRespConf { _LIBRDB_API RdbxToResp *RDBX_createHandlersToResp(RdbParser *, RdbxToRespConf *); /**************************************************************** - * Attach RESP writer + * RESP writer * * Create instance for writing RDB to RESP stream. * - * Used by: RDBX_createRespToRedisTcp + * Imp by: RDBX_createRespToRedisTcp * RDBX_createRespToRedisFd * RDBX_createRespToFileWriter * @@ -159,17 +168,59 @@ _LIBRDB_API RdbxRespToFileWriter *RDBX_createRespToFileWriter(RdbParser *p, * Can configure pipeline depth of transmitted RESP commands. Set * to 0 to use default. ****************************************************************/ +typedef struct RdbxRedisAuth { + const char *pwd; + const char *user; + + /* alternative auth-cmd. Args must remain valid throughout the parser's lifetime. */ + struct { + int argc; + char **argv; + } cmd; +} RdbxRedisAuth; + _LIBRDB_API RdbxRespToRedisLoader *RDBX_createRespToRedisTcp(RdbParser *p, - RdbxToResp *rdbToResp, - const char *hostname, - int port); + RdbxToResp *rdbToResp, + RdbxRedisAuth *auth, /*opt*/ + const char *hostname, + int port); _LIBRDB_API RdbxRespToRedisLoader *RDBX_createRespToRedisFd(RdbParser *p, - RdbxToResp *rdbToResp, - int fd); + RdbxToResp *rdbToResp, + RdbxRedisAuth *auth, /*opt*/ + int fd); _LIBRDB_API void RDBX_setPipelineDepth(RdbxRespToRedisLoader *r2r, int depth); +/**************************************************************** + * Debugging RESP to Redis + * + * This section provides debugging assistance for analyzing Redis server failures + * when attempting to stream multiple RESP commands. This analysis can be particularly + * challenging in the following scenarios: + * + * - When using pipeline mode, which involves multiple concurrent pending commands + * at any given moment. + * - When not using the `delKeyBeforeWrite` flag and Redis server is not empty. + * - In a production environments with real-world loads. + * + * The following two debug functions are designed to help with the analysis of a given + * RDB file: + * + * RDBX_enumerateCmds + * Enumerates commands by preceding any RESP command with an additional trivial + * RESP command of the type 'echo '. This can be especially useful since + * the RESP-to-Redis instance prints the command number in case of a failure. + * + * RDBX_writeFromCmdNumber + * Writing commands starting from specified command-number and onward as part + * of reproducing effort. Once the problem was resolved, it might be also useful + * to continue uploading the redis server from the point it got failed. + ****************************************************************/ +_LIBRDB_API void RDBX_enumerateCmds(RdbxToResp *rdbToResp); + +_LIBRDB_API void RDBX_writeFromCmdNumber(RdbxToResp *rdbToResp, size_t cmdNum); + #ifdef __cplusplus } #endif diff --git a/examples/example1.c b/examples/example1.c index e4710ce..b6d98a5 100644 --- a/examples/example1.c +++ b/examples/example1.c @@ -31,7 +31,7 @@ int main(void) { RdbParser *parser; RdbxReaderFile *reader; RdbxToJson *rdbToJson; - RdbxFilterKey *filterKey; + RdbxFilter *filterKey; RdbRes err=RDB_OK; const char *infile = "./dumps/multiple_lists_strings.rdb"; @@ -53,7 +53,7 @@ int main(void) { if (!rdbToJson) goto PARSER_ERROR; /* Filter keys that starts with the word `mylist` */ - filterKey = RDBX_createHandlersFilterKey(parser, "mylist.*", 0 /*flags*/); + filterKey = RDBX_createHandlersFilterKey(parser, "mylist.*", 0 /*exclude*/); if (!filterKey) goto PARSER_ERROR; /* Run the parser */ diff --git a/runtests b/runtests index f315a8b..691d465 100755 --- a/runtests +++ b/runtests @@ -2,6 +2,8 @@ VALGRIND=0 REDIS_FOLDER="" +SPECIFIC_TEST="" +SPECIFIC_TEST_GROUP="" while [[ $# -gt 0 ]]; do case $1 in @@ -11,8 +13,14 @@ while [[ $# -gt 0 ]]; do -f|--redis-folder) REDIS_FOLDER=" --redis-folder $2 " shift 2 ;; + -t|--test) + SPECIFIC_TEST=" --test $2 " + shift 2 ;; + -g|--test-group) + SPECIFIC_TEST_GROUP=" --test-group $2 " + shift 2 ;; *|-h) - echo "Usage: $(basename $0) [-v|--valgrind] [-f|--folder REDIS_FOLDER]" + echo "Usage: $(basename $0) [-v|--valgrind] [-f|--folder REDIS_FOLDER] [-t|--test TEST] [-g|--test-group GROUP]" exit 1 ;; esac done @@ -26,12 +34,12 @@ if [[ ${VALGRIND} -ne 0 ]]; then --leak-resolution=high \ --error-exitcode=1 \ --log-file=test/log/valgrind.log \ - ./test/test_static_lib $REDIS_FOLDER && exit 0 + ./test/test_static_lib $REDIS_FOLDER $SPECIFIC_TEST $SPECIFIC_TEST_GROUP && exit 0 sed -n -e '/SUMMARY:/,$p' ./test/log/valgrind.log | tail -n 20 echo -en "\n(Entire log available at: ./test/log/valgrind.log)\n" exit 1 else - ./test/test_lib $REDIS_FOLDER + ./test/test_lib $REDIS_FOLDER $SPECIFIC_TEST $SPECIFIC_TEST_GROUP fi diff --git a/src/cli/rdb-cli.c b/src/cli/rdb-cli.c index b10fd94..3c4be2b 100644 --- a/src/cli/rdb-cli.c +++ b/src/cli/rdb-cli.c @@ -2,6 +2,7 @@ #include #include #include +#include /* Rely only on API (and not internal parser headers) */ #include "../../api/librdb-api.h" @@ -12,8 +13,14 @@ FILE* logfile = NULL; #define LOG_FILE_PATH_DEF "./rdb-cli.log" -static int getOptArg(int argc, char* argv[], int *at, char *abbrvOpt, char *opt, char *token, int *flag, const char **arg) { - if ((strcmp(token, abbrvOpt) == 0) || (strcmp(token, opt) == 0)) { +/* common options to all FORMATTERS */ +typedef struct Options { + const char *logfilePath; + RdbRes (*formatFunc)(RdbParser *p, int argc, char **argv); +} Options; + +static int getOptArg(int argc, char* argv[], int *at, char *abbrvOpt, char *opt, int *flag, const char **arg) { + if ((strcmp(argv[*at], abbrvOpt) == 0) || (strcmp(argv[*at], opt) == 0)) { if (arg) { if ((*at) + 1 == argc) { fprintf(stderr, "%s (%s) requires one argument.", opt, abbrvOpt); @@ -28,12 +35,28 @@ static int getOptArg(int argc, char* argv[], int *at, char *abbrvOpt, char *opt } } +static int getOptArgVal(int argc, char* argv[], int *at, char *abbrvOpt, char *opt, int *flag, int *val, int min, int max) { + const char *valStr; + if (getOptArg(argc, argv, at, abbrvOpt, opt, flag, &valStr)) { + + *val = atoi(valStr); + + /* check boundaries. Condition support also the limits INT_MAX and INT_MIN. */ + if (!((*val>=min) && (*val <=max))) { + fprintf(stderr, "Value of %s (%s) must be a integer between %d and %d", opt, abbrvOpt, min, max); + exit(RDB_ERR_GENERAL); + } + return 1; + } + return 0; +} + static void logger(RdbLogLevel l, const char *msg) { static char *logLevelStr[] = { - [RDB_LOG_ERR] = ":: ERROR ::", - [RDB_LOG_WRN] = ":: WARN ::", - [RDB_LOG_INF] = ":: INFO ::", - [RDB_LOG_DBG] = ":: DEBUG ::", + [RDB_LOG_ERR] = "ERROR :", + [RDB_LOG_WRN] = "WARN :", + [RDB_LOG_INF] = "INFO :", + [RDB_LOG_DBG] = "DEBUG :", }; if (logfile != NULL) @@ -43,7 +66,7 @@ static void logger(RdbLogLevel l, const char *msg) { printf("%s %s\n", logLevelStr[l], msg); } -static void loggerWrap(RdbLogLevel l, const char *msg, ...) { +void loggerWrap(RdbLogLevel l, const char *msg, ...) { char tmp[1024]; va_list args; va_start(args, msg); @@ -52,35 +75,51 @@ static void loggerWrap(RdbLogLevel l, const char *msg, ...) { logger(l, tmp); } -static void printUsage(void) { +static void printUsage(int shortUsage) { + if (shortUsage) { + printf("Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {json|resp|redis} [FORMAT_OPTIONS]\n"); + printf("For detailed usage, run command without arguments\n"); + return; + } printf("[v%s] ", RDB_getLibVersion(NULL,NULL,NULL)); printf("Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {json|resp|redis} [FORMAT_OPTIONS]\n"); printf("OPTIONS:\n"); - printf("\t-k, --filter-key Filter keys using regular expressions\n"); printf("\t-l, --log-file Path to the log file (Default: './rdb-cli.log')\n\n"); + printf("\tMultiple filters combination of keys/types/dbs can be specified:\n"); + printf("\t-k, --key Include only keys that match REGEX\n"); + printf("\t-K --no-key Exclude keys that match REGEX\n"); + printf("\t-t, --type Include only selected TYPE {str|list|set|zset|hash|module|func}\n"); + printf("\t-T, --no-type Exclude TYPE {str|list|set|zset|hash|module|func}\n"); + printf("\t-d, --dbnum Include only selected db number\n"); + printf("\t-D, --no-dbnum Exclude DB number\n\n"); printf("FORMAT_OPTIONS ('json'):\n"); printf("\t-i, --include To include: {aux-val|func}\n"); printf("\t-f, --flatten Print flatten json, without DBs Parenthesis\n"); - printf("\t-o, --output Specify the output file. If not specified, output goes to stdout\n\n"); + printf("\t-o, --output Specify the output file. If not specified, output to stdout\n\n"); - printf("FORMAT_OPTIONS ('resp'):\n"); + printf("FORMAT_OPTIONS ('redis'):\n"); + printf("\t-h, --hostname Specify the server hostname (default: 127.0.0.1)\n"); + printf("\t-p, --port Specify the server port (default: 6379)\n"); + printf("\t-l, --pipeline-depth Number of pending commands before blocking for responses\n"); + printf("\t-u, --user Redis username for authentication\n"); + printf("\t-P, --password Redis password for authentication\n"); + printf("\t-a, --auth N [ARG1 ... ARGN] An alternative authentication command. Given as vector of arguments\n\n"); + + printf("FORMAT_OPTIONS ('redis'|'resp'):\n"); printf("\t-r, --support-restore Use the RESTORE command when possible\n"); printf("\t-t, --target-redis-ver Specify the target Redis version. Helps determine which commands can\n"); printf("\t be applied. Particularly crucial if support-restore being used \n"); printf("\t as RESTORE is closely tied to specific RDB versions. If versions not\n"); printf("\t aligned the parser will generate higher-level commands instead.\n"); - printf("\t-o, --output Specify the output file. If not specified, output goes to stdout\n\n"); + printf("\t-o, --output Specify the output file (For 'resp' only: if not specified, output to stdout)\n"); + printf("\t-s, --start-cmd-num Start writing redis from command number\n"); + printf("\t-e, --enum-commands Command enumeration and tracing by preceding each generated RESP command\n"); + printf("\t with debug command of type: `SET _RDB_CLI_CMD_ID_ `\n"); - printf("FORMAT_OPTIONS ('redis'):\n"); - printf("\t-r, --support-restore Use the RESTORE command when possible\n"); - printf("\t-t, --target-redis-ver Specify the target Redis version\n"); - printf("\t-h, --hostname Specify the server hostname (default: 127.0.0.1)\n"); - printf("\t-p, --port Specify the server port (default: 6379)\n"); - printf("\t-l, --pipeline-depth Number of pending commands before blocking for responses\n"); } -static RdbRes formatJson(RdbParser *parser, char *input, int argc, char **argv) { +static RdbRes formatJson(RdbParser *parser, int argc, char **argv) { const char *includeArg; const char *output = NULL;/*default:stdout*/ int includeFunc=0, includeAuxField=0, flatten=0; /*without*/ @@ -88,18 +127,18 @@ static RdbRes formatJson(RdbParser *parser, char *input, int argc, char **argv) /* parse specific command options */ for (int at = 1; at < argc; ++at) { char *opt = argv[at]; - if (getOptArg(argc, argv, &at, "-o", "--output", opt, NULL, &output)) continue; - if (getOptArg(argc, argv, &at, "-f", "--flatten", opt, &flatten, NULL)) continue; + if (getOptArg(argc, argv, &at, "-o", "--output", NULL, &output)) continue; + if (getOptArg(argc, argv, &at, "-f", "--flatten", &flatten, NULL)) continue; - if (getOptArg(argc, argv, &at, "-i", "--include", opt, NULL, &includeArg)) { + if (getOptArg(argc, argv, &at, "-i", "--include", NULL, &includeArg)) { if (strcmp(includeArg, "aux-val") == 0) { includeAuxField = 1; continue; } if (strcmp(includeArg, "func") == 0) { includeFunc = 1; continue; } fprintf(stderr, "Invalid argument for '--include': %s\n", includeArg); return RDB_ERR_GENERAL; } - fprintf(stderr, "Invalid JSON [FORMAT_OPTIONS] argument: %s\n", opt); - printUsage(); + fprintf(stderr, "Invalid 'json' [FORMAT_OPTIONS] argument: %s\n", opt); + printUsage(1); return RDB_ERR_GENERAL; } @@ -111,139 +150,230 @@ static RdbRes formatJson(RdbParser *parser, char *input, int argc, char **argv) .includeFunc = includeFunc, }; - if (RDBX_createReaderFile(parser, input) == NULL) - return RDB_ERR_GENERAL; - if (RDBX_createHandlersToJson(parser, output, &conf) == NULL) return RDB_ERR_GENERAL; return RDB_OK; } -static RdbRes formatRedis(RdbParser *parser, char *input, int argc, char **argv) { - int port = 6379; - int pipeDepthVal=0; - RdbxToResp *rdbToResp; - const char *hostname = "127.0.0.1"; - const char *portStr=NULL; - const char *pipelineDepth=NULL; - +static RdbRes formatRedis(RdbParser *parser, int argc, char **argv) { + const char *output = NULL; + RdbxRedisAuth auth = {0}; RdbxToRespConf conf = { 0 }; + int commandEnum=0, startCmdNum=0, pipeDepthVal=0, port = 6379; + RdbxRespToRedisLoader *respToRedis; + RdbxToResp *rdbToResp, *rdbToResp2; + const char *hostname = "127.0.0.1"; /* parse specific command options */ for (int at = 1; at < argc; ++at) { char *opt = argv[at]; - if (getOptArg(argc, argv, &at, "-h", "--hostname", opt, NULL, &hostname)) continue; - if (getOptArg(argc, argv, &at, "-p", "--port", opt, NULL, &portStr)) continue; - if (getOptArg(argc, argv, &at, "-r", "--support-restore", opt, &(conf.supportRestore), NULL)) continue; - if (getOptArg(argc, argv, &at, "-t", "--target-redis-ver", opt, NULL, &(conf.dstRedisVersion))) continue; - if (getOptArg(argc, argv, &at, "-l", "--pipeline-depth", opt, NULL, &pipelineDepth)) continue; - - fprintf(stderr, "Invalid REDIS [FORMAT_OPTIONS] argument: %s\n", opt); - printUsage(); - return RDB_ERR_GENERAL; - } + if (getOptArg(argc, argv, &at, "-o", "--output", NULL, &output)) continue; + if (getOptArg(argc, argv, &at, "-h", "--hostname", NULL, &hostname)) continue; + if (getOptArgVal(argc, argv, &at, "-p", "--port", NULL, &port, 1, 65535)) continue; + if (getOptArg(argc, argv, &at, "-r", "--support-restore", &(conf.supportRestore), NULL)) continue; + if (getOptArg(argc, argv, &at, "-t", "--target-redis-ver", NULL, &(conf.dstRedisVersion))) continue; + if (getOptArgVal(argc, argv, &at, "-l", "--pipeline-depth", NULL, &pipeDepthVal, 1, 1000)) continue; + if (getOptArgVal(argc, argv, &at, "-n", "--start-cmd-num", NULL, &startCmdNum, 1, INT_MAX)) continue; + if (getOptArg(argc, argv, &at, "-u", "--user", NULL, &auth.user)) continue; + if (getOptArg(argc, argv, &at, "-P", "--password", NULL, &auth.pwd)) continue; + if (getOptArg(argc, argv, &at, "-e", "--enum-commands", &commandEnum, NULL)) continue; + if (getOptArgVal(argc, argv, &at, "-a", "--auth", NULL, &(auth.cmd.argc), 1, INT_MAX)) { + auth.cmd.argv = argv + at + 1; + if ((1 + at + auth.cmd.argc) >= argc) { + fprintf(stderr, "Insufficient number of arguments to option --auth\n"); + printUsage(1); + return RDB_ERR_GENERAL; + } + at += auth.cmd.argc; + continue; + } - if ((pipelineDepth) && ((pipeDepthVal = atoi(pipelineDepth)) == 0)) { - logger(RDB_LOG_ERR, "Value of '--pipeline-depth' ('-l') must be positive integer, bigger than 0"); + fprintf(stderr, "Invalid 'redis' [FORMAT_OPTIONS] argument: %s\n", opt); + printUsage(1); return RDB_ERR_GENERAL; } - if (portStr) { - port = atoi(portStr); - if (port == 0) { - loggerWrap(RDB_LOG_ERR, "Invalid port: %s\n", portStr); - return RDB_ERR_GENERAL; - } - } - - if (RDBX_createReaderFile(parser, input) == NULL) + if (((auth.user) || (auth.pwd)) && (auth.cmd.argc > 0)) { + fprintf(stderr, "Invalid AUTH arguments. --auth(-a) is mutually exclusive with --password(-P) and --user(-u)\n"); return RDB_ERR_GENERAL; + } if ((rdbToResp = RDBX_createHandlersToResp(parser, &conf)) == NULL) return RDB_ERR_GENERAL; - if (RDBX_createRespToRedisTcp(parser, rdbToResp, hostname, port) == NULL) + if (startCmdNum) + RDBX_writeFromCmdNumber(rdbToResp, startCmdNum); + + if (commandEnum) + RDBX_enumerateCmds(rdbToResp); + + if ((respToRedis = RDBX_createRespToRedisTcp(parser, rdbToResp, &auth, hostname, port)) == NULL) return RDB_ERR_GENERAL; + /* if in addition requested to generate a dump to a file (of RESP protocol) */ + if (output) { + if ((rdbToResp2 = RDBX_createHandlersToResp(parser, &conf)) == NULL) + return RDB_ERR_GENERAL; + + if (startCmdNum) + RDBX_writeFromCmdNumber(rdbToResp2, startCmdNum); + + if (commandEnum) + RDBX_enumerateCmds(rdbToResp2); + + if (RDBX_createRespToFileWriter(parser, rdbToResp2, output) == NULL) + return RDB_ERR_GENERAL; + } + + if (pipeDepthVal) + RDBX_setPipelineDepth(respToRedis, pipeDepthVal); + return RDB_OK; } -static RdbRes formatResp(RdbParser *parser, char *input, int argc, char **argv) { +static RdbRes formatResp(RdbParser *parser, int argc, char **argv) { RdbxToResp *rdbToResp; const char *output = NULL;/*default:stdout*/ + int commandEnum = 0, startCmdNum=0; RdbxToRespConf conf = { 0 }; /* parse specific command options */ for (int at = 1; at < argc; ++at) { char *opt = argv[at]; - if (getOptArg(argc, argv, &at, "-o", "--output", opt, NULL, &output)) continue; - if (getOptArg(argc, argv, &at, "-r", "--support-restore", opt, &(conf.supportRestore), NULL)) continue; - if (getOptArg(argc, argv, &at, "-t", "--target-redis-ver", opt, NULL, &(conf.dstRedisVersion))) continue; - - fprintf(stderr, "Invalid RESP [FORMAT_OPTIONS] argument: %s\n", opt); - printUsage(); + if (getOptArg(argc, argv, &at, "-o", "--output", NULL, &output)) continue; + if (getOptArg(argc, argv, &at, "-r", "--support-restore", &(conf.supportRestore), NULL)) continue; + if (getOptArg(argc, argv, &at, "-t", "--target-redis-ver", NULL, &(conf.dstRedisVersion))) continue; + if (getOptArg(argc, argv, &at, "-e", "--enum-commands", &commandEnum, NULL)) continue; + if (getOptArgVal(argc, argv, &at, "-n", "--start-cmd-num", NULL, &startCmdNum, 1, INT_MAX)) continue; + + fprintf(stderr, "Invalid 'resp' [FORMAT_OPTIONS] argument: %s\n", opt); + printUsage(1); return RDB_ERR_GENERAL; } - if (RDBX_createReaderFile(parser, input) == NULL) - return RDB_ERR_GENERAL; - if ((rdbToResp = RDBX_createHandlersToResp(parser, &conf)) == NULL) return RDB_ERR_GENERAL; + if (startCmdNum) + RDBX_writeFromCmdNumber(rdbToResp, startCmdNum); if (RDBX_createRespToFileWriter(parser, rdbToResp, output) == NULL) return RDB_ERR_GENERAL; + if (commandEnum) + RDBX_enumerateCmds(rdbToResp); + return RDB_OK; } -int main(int argc, char **argv) -{ - RdbStatus status; - const char *logfilePath = LOG_FILE_PATH_DEF; - const char *filterKey = NULL; - int at; - RdbRes res; - RdbRes (*formatFunc)(RdbParser *p, char *input, int argc, char **argv) = formatJson; +int matchRdbDataType(const char *dataTypeStr) { + if (!strcmp(dataTypeStr, "str")) return RDB_DATA_TYPE_STRING; + if (!strcmp(dataTypeStr, "list")) return RDB_DATA_TYPE_LIST; + if (!strcmp(dataTypeStr, "set")) return RDB_DATA_TYPE_SET; + if (!strcmp(dataTypeStr, "zset")) return RDB_DATA_TYPE_ZSET; + if (!strcmp(dataTypeStr, "hash")) return RDB_DATA_TYPE_HASH; + if (!strcmp(dataTypeStr, "module")) return RDB_DATA_TYPE_MODULE; + if (!strcmp(dataTypeStr, "stream")) return RDB_DATA_TYPE_STREAM; + if (!strcmp(dataTypeStr, "func")) return RDB_DATA_TYPE_FUNCTION; + + fprintf(stderr, "Invalid TYPE argument (%s). Valid values: str, list, set, zset, hash, module, stream, func", + dataTypeStr); + exit(RDB_ERR_GENERAL); +} - if (argc < 2) { - printUsage(); - return 1; - } +int readCommonOptions(RdbParser *p, int argc, char* argv[], Options *options, int applyFilters) { + const char *typeFilter, *keyFilter; + int dbNumFilter; + int at; - /* first argument is input file */ - char *input = argv[1]; + /* default */ + options->logfilePath = LOG_FILE_PATH_DEF; + options->formatFunc = formatJson; /* parse common options until FORMAT (json/resp/redis) specified */ for (at = 2; at < argc; ++at) { char *opt = argv[at]; - if (getOptArg(argc, argv, &at, "-l", "--log-file", opt, NULL, &logfilePath)) + if (getOptArg(argc, argv, &at, "-l", "--log-file", NULL, &(options->logfilePath))) continue; - if (getOptArg(argc, argv, &at, "-k", "--filter-key", opt, NULL, &filterKey)) + if (getOptArg(argc, argv, &at, "-k", "--key", NULL, &keyFilter)) { + if (applyFilters && (!RDBX_createHandlersFilterKey(p, keyFilter, 0))) + exit(RDBX_ERR_FAILED_CREATE_FILTER); continue; + } - if (strcmp(opt, "json") == 0) { formatFunc = formatJson; break; } - else if (strcmp(opt, "resp") == 0) { formatFunc = formatResp; break; } - else if (strcmp(opt, "redis") == 0) { formatFunc = formatRedis; break; } + if (getOptArg(argc, argv, &at, "-K", "--no-key", NULL, &keyFilter)) { + if (applyFilters && (!RDBX_createHandlersFilterKey(p, keyFilter, 1))) + exit(RDBX_ERR_FAILED_CREATE_FILTER); + continue; + } - fprintf(stderr, "Invalid [OPTIONS] argument: %s\n", opt); - printUsage(); - return RDB_ERR_GENERAL; + if (getOptArg(argc, argv, &at, "-t", "--type", NULL, &typeFilter)) { + if ((applyFilters) && (!RDBX_createHandlersFilterType(p, matchRdbDataType(typeFilter), 0))) + exit(RDBX_ERR_FAILED_CREATE_FILTER); + continue; + } + + if (getOptArg(argc, argv, &at, "-T", "--no-type", NULL, &typeFilter)) { + if ((applyFilters) && (!RDBX_createHandlersFilterType(p, matchRdbDataType(typeFilter), 1))) + exit(RDBX_ERR_FAILED_CREATE_FILTER); + continue; + } + + if (getOptArgVal(argc, argv, &at, "-d", "--dbnum", NULL, &dbNumFilter, 0, INT_MAX)) { + if ((applyFilters) && (!RDBX_createHandlersFilterDbNum(p, dbNumFilter, 0))) + exit(RDBX_ERR_FAILED_CREATE_FILTER); + continue; + } + + if (getOptArgVal(argc, argv, &at, "-D", "--no-dbnum", NULL, &dbNumFilter, 0, INT_MAX)) { + if ((applyFilters) && (!RDBX_createHandlersFilterDbNum(p, dbNumFilter, 1))) + exit(RDBX_ERR_FAILED_CREATE_FILTER); + continue; + } + + if (strcmp(opt, "json") == 0) { options->formatFunc = formatJson; break; } + else if (strcmp(opt, "resp") == 0) { options->formatFunc = formatResp; break; } + else if (strcmp(opt, "redis") == 0) { options->formatFunc = formatRedis; break; } + + fprintf(stderr, "At argv[%d], unexpected OPTIONS argument: %s\n", at, opt); + printUsage(1); + exit(RDB_ERR_GENERAL); + } + return at; +} + +int main(int argc, char **argv) +{ + Options options; + RdbStatus status; + int at; + RdbRes res; + + if (argc < 2) { + printUsage(0); + return 1; } + /* first argument is expected to be input file */ + char *input = argv[1]; + + /* Initially, read common options that are applicable to all formatters. To + * make it effective, apply filters later (applyFilters), ensuring that they + * are registered only after the FORMATTER registers its own handlers. */ + at = readCommonOptions(NULL, argc, argv, &options, 0); + if (at == argc) { logger(RDB_LOG_ERR, "Missing value."); - printUsage(); + printUsage(1); return RDB_ERR_GENERAL; } - if ((logfile = fopen(logfilePath, "w")) == NULL) { - printf("Error opening log file for writing: %s \n", logfilePath); + if ((logfile = fopen(options.logfilePath, "w")) == NULL) { + printf("Error opening log file for writing: %s \n", options.logfilePath); return RDB_ERR_GENERAL; } @@ -252,14 +382,22 @@ int main(int argc, char **argv) RDB_setLogLevel(parser, RDB_LOG_INF); RDB_setLogger(parser, logger); - if (RDB_OK != (res = formatFunc(parser, input, argc - at, argv + at))) + if (strcmp(input, "-") == 0) { + if (RDBX_createReaderFileDesc(parser, 0 /*stdin*/, 0) == NULL) + return RDB_ERR_GENERAL; + } else { + if (RDBX_createReaderFile(parser, input /*file*/) == NULL) + return RDB_ERR_GENERAL; + } + + if (RDB_OK != (res = options.formatFunc(parser, argc - at, argv + at))) return res; if (RDB_OK != RDB_getErrorCode(parser)) return RDB_getErrorCode(parser); - if (filterKey) - RDBX_createHandlersFilterKey(parser, filterKey, 0); + /* now that the formatter got registered, attach filters */ + readCommonOptions(parser, argc, argv, &options, 1); while ((status = RDB_parse(parser)) == RDB_STATUS_WAIT_MORE_DATA); diff --git a/src/ext/common.h b/src/ext/common.h index dbdf8b6..7a6f8e3 100644 --- a/src/ext/common.h +++ b/src/ext/common.h @@ -19,11 +19,20 @@ static inline void unused(void *dummy, ...) { (void)(dummy);} #define unlikely(x) (x) #endif -typedef union CallbacksUnion { - struct { HANDLERS_COMMON_CALLBACKS } common; - RdbHandlersRawCallbacks rawCb; - RdbHandlersStructCallbacks structCb; - RdbHandlersDataCallbacks dataCb; -} CallbacksUnion; +/*** IOVEC manipulation ***/ +#define IOV_CONST(iov, str) iov_plain(iov, str, sizeof(str)-1) +#define IOV_STRING(iov, str, len) iov_plain(iov, str, len) +#define IOV_VALUE(iov, val, ar) iov_value(iov, val, ar, sizeof(ar)) +#define IOV_LEN_AND_VALUE(iov, val, ar1, ar2) \ + do {\ + int l = IOV_VALUE((iov)+1, val, ar2); \ + IOV_VALUE( (iov), l, ar1); \ + } while (0); + +int iov_value(struct iovec *iov, long long count, char *buf, int bufsize); +inline void iov_plain(struct iovec *iov, const char *s, size_t l) { + iov->iov_base = (void *) s; + iov->iov_len = l; +} #endif /*define RDBX_COMMON_H*/ diff --git a/src/ext/handlersFilter.c b/src/ext/handlersFilter.c new file mode 100644 index 0000000..6763f4b --- /dev/null +++ b/src/ext/handlersFilter.c @@ -0,0 +1,352 @@ +#include +#include +#include "../lib/defines.h" /* valid include since it brings only RDB_* defines */ +#include "common.h" + +struct RdbxFilter { + regex_t regex_compiled; + int exclude; + RdbRes cbReturnValue; + RdbDataType opToType[256]; /* Mapping opcode to type. init only in case of filter types being used */ + + int regexInitialized; /* for filter keys */ + RdbDataType type; /* for filter types */ + int dbnum; /* for filter db */ +}; + +static void deleteFilterCtx(RdbParser *p, void *data) { + RdbxFilter *ctx = (RdbxFilter *) data; + if (ctx->regexInitialized) { + regfree(&ctx->regex_compiled); + } + RDB_free(p, ctx); +} + +/* mapping opcode to type */ +static void initOpcodeToType(RdbxFilter *ctx) { + memset(ctx->opToType, 0, sizeof(ctx->opToType)); + /*string*/ + ctx->opToType[RDB_TYPE_STRING] = RDB_DATA_TYPE_STRING; + /*list*/ + ctx->opToType[RDB_TYPE_LIST] = RDB_DATA_TYPE_LIST; + ctx->opToType[RDB_TYPE_LIST_ZIPLIST] = RDB_DATA_TYPE_LIST; + ctx->opToType[RDB_TYPE_LIST_QUICKLIST] = RDB_DATA_TYPE_LIST; + ctx->opToType[RDB_TYPE_LIST_QUICKLIST_2] = RDB_DATA_TYPE_LIST; + /*set*/ + ctx->opToType[RDB_TYPE_SET] = RDB_DATA_TYPE_SET; + ctx->opToType[RDB_TYPE_SET_INTSET] = RDB_DATA_TYPE_SET; + ctx->opToType[RDB_TYPE_SET_LISTPACK] = RDB_DATA_TYPE_SET; + /*zset*/ + ctx->opToType[RDB_TYPE_ZSET] = RDB_DATA_TYPE_ZSET; + ctx->opToType[RDB_TYPE_ZSET_2] = RDB_DATA_TYPE_ZSET; + ctx->opToType[RDB_TYPE_ZSET_ZIPLIST] = RDB_DATA_TYPE_ZSET; + ctx->opToType[RDB_TYPE_ZSET_LISTPACK] = RDB_DATA_TYPE_ZSET; + /*hash*/ + ctx->opToType[RDB_TYPE_HASH] = RDB_DATA_TYPE_HASH; + ctx->opToType[RDB_TYPE_HASH_ZIPMAP] = RDB_DATA_TYPE_HASH; + ctx->opToType[RDB_TYPE_HASH_ZIPLIST] = RDB_DATA_TYPE_HASH; + ctx->opToType[RDB_TYPE_HASH_LISTPACK] = RDB_DATA_TYPE_HASH; + /*module*/ + ctx->opToType[RDB_TYPE_MODULE_2] = RDB_DATA_TYPE_MODULE; + ctx->opToType[RDB_OPCODE_MODULE_AUX] = RDB_DATA_TYPE_MODULE; + /*stream*/ + ctx->opToType[RDB_TYPE_STREAM_LISTPACKS] = RDB_DATA_TYPE_STREAM; + ctx->opToType[RDB_TYPE_STREAM_LISTPACKS_2] = RDB_DATA_TYPE_STREAM; + ctx->opToType[RDB_TYPE_STREAM_LISTPACKS_3] = RDB_DATA_TYPE_STREAM; + /*func*/ + ctx->opToType[RDB_OPCODE_FUNCTION2] = RDB_DATA_TYPE_FUNCTION; +} + +/*** filtering BY key, type or dbnum ***/ + +static RdbRes filterNewKeyByRegex(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) { + UNUSED(p, info); + RdbxFilter *ctx = userData; + if (regexec(&ctx->regex_compiled, key, 0, NULL, 0) == 0) /* if match */ + return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK_DONT_PROPAGATE : RDB_OK; + else + return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK : RDB_OK_DONT_PROPAGATE; +} + +static RdbRes filterNewKeyByType(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) { + UNUSED(p, key); + RdbxFilter *ctx = userData; + if (ctx->opToType[info->opcode] == ctx->type) /* if match */ + return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK_DONT_PROPAGATE : RDB_OK; + else + return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK : RDB_OK_DONT_PROPAGATE; +} + +static RdbRes filterNewDbByNumber(RdbParser *p, void *userData, int dbnum) { + UNUSED(p); + RdbxFilter *ctx = userData; + if (dbnum == ctx->dbnum) /* if match */ + return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK_DONT_PROPAGATE : RDB_OK; + else + return ctx->cbReturnValue = (ctx->exclude) ? RDB_OK : RDB_OK_DONT_PROPAGATE; +} + +/*** Handling common ***/ + +static RdbRes filterNewKey(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) { + UNUSED(p, key, info); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterEndKey(RdbParser *p, void *userData) { + UNUSED(p); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterNewDb(RdbParser *p, void *userData, int dbnum) { + UNUSED(p, dbnum); + return ((RdbxFilter *) userData)->cbReturnValue = RDB_OK; /* clean possible leftovers */ +} + +static RdbRes filterDbSize(RdbParser *p, void *userData, uint64_t db_size, uint64_t exp_size) { + UNUSED(p, db_size, exp_size); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +/*** Handling data ***/ + +static RdbRes filterString(RdbParser *p, void *userData, RdbBulk str) { + UNUSED(p, str); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterList(RdbParser *p, void *userData, RdbBulk item) { + UNUSED(p, item); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { + UNUSED(p, field, value); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +/*** Handling struct ***/ + +static RdbRes filterListLP(RdbParser *p, void *userData, RdbBulk listpack) { + UNUSED(p, listpack); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterListZL(RdbParser *p, void *userData, RdbBulk ziplist) { + UNUSED(p, ziplist); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterListPlain(RdbParser *p, void *userData, RdbBulk listNode) { + UNUSED(p, listNode); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterHashLP(RdbParser *p, void *userData, RdbBulk listpack) { + UNUSED(p, listpack); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterHashZM(RdbParser *p, void *userData, RdbBulk zipmap) { + UNUSED(p, zipmap); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterHashZL(RdbParser *p, void *userData, RdbBulk ziplist) { + UNUSED(p, ziplist); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterHashPlain(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { + UNUSED(p, field, value); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterSetMember(RdbParser *p, void *userData, RdbBulk member) { + UNUSED(p, member); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterSetPlain(RdbParser *p, void *userData, RdbBulk item) { + UNUSED(p, item); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterSetIS(RdbParser *p, void *userData, RdbBulk intset) { + UNUSED(p, intset); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterSetLP(RdbParser *p, void *userData, RdbBulk listpack) { + UNUSED(p, listpack); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterFunction(RdbParser *p, void *userData, RdbBulk func) { + UNUSED(p, func); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterModule(RdbParser *p, void *userData, RdbBulk moduleName, size_t serializedSize) { + UNUSED(p, moduleName, serializedSize); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +/*** Handling raw ***/ + +static RdbRes filterFrag(RdbParser *p, void *userData, RdbBulk frag) { + UNUSED(p, frag); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterRawBegin(RdbParser *p, void *userData, size_t size) { + UNUSED(p, size); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterRawEnd(RdbParser *p, void *userData) { + UNUSED(p); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +/*** common init ***/ + +static void defaultFilterDataCb(RdbHandlersDataCallbacks *dataCb) { + memset(dataCb, 0, sizeof(*dataCb)); + dataCb->handleNewKey = filterNewKey; + dataCb->handleEndKey = filterEndKey; + dataCb->handleNewDb = filterNewDb; + dataCb->handleDbSize = filterDbSize; + + dataCb->handleStringValue = filterString; + dataCb->handleListItem = filterList; + dataCb->handleHashField = filterHash; + dataCb->handleSetMember = filterSetMember; + dataCb->handleFunction = filterFunction; + dataCb->handleModule = filterModule; +} + +static void defaultFilterStructCb(RdbHandlersStructCallbacks *structCb) { + memset(structCb, 0, sizeof(*structCb)); + /* common */ + structCb->handleNewKey = filterNewKey; + structCb->handleEndKey = filterEndKey; + structCb->handleNewDb = filterNewDb; + structCb->handleDbSize = filterDbSize; + + /* string */ + structCb->handleString = filterString; + /* list */ + structCb->handleListLP = filterListLP; + structCb->handleListZL = filterListZL; + structCb->handleListPlain = filterListPlain; + /* hash */ + structCb->handleHashPlain = filterHashPlain; + structCb->handleHashZL = filterHashZL; + structCb->handleHashLP = filterHashLP; + structCb->handleHashZM = filterHashZM; + + /* set */ + structCb->handleSetPlain = filterSetPlain; + structCb->handleHashZM = filterSetIS; + structCb->handleSetLP = filterSetLP; + + /* func */ + structCb->handleFunction = filterFunction; + /* module */ + structCb->handleModule = filterModule; +} + +static void defaultFilterRawCb(RdbHandlersRawCallbacks *rawCb) { + memset(rawCb, 0, sizeof(*rawCb)); + /* common */ + rawCb->handleNewKey = filterNewKey; + rawCb->handleEndKey = filterEndKey; + rawCb->handleNewDb = filterNewDb; + rawCb->handleDbSize = filterDbSize; + + //callbacks.rawCb.handleBeginModuleAux /* not part of keyspace */ + rawCb->handleBegin = filterRawBegin; + rawCb->handleFrag = filterFrag; + rawCb->handleEnd = filterRawEnd; +} + +static RdbxFilter *createHandlersFilterCommon(RdbParser *p, + const char *keyRegex, + RdbDataType *type, + int *dbnum, + uint32_t exclude) { + RdbRes (*handleNewKey)(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) = filterNewKey; + RdbRes (*handleNewDb)(RdbParser *p, void *userData, int dbnum) = filterNewDb; + RdbxFilter *ctx; + + if ( (ctx = RDB_alloc(p, sizeof(RdbxFilter))) == NULL) + return NULL; + + ctx->regexInitialized = 0; + + /* specific if-else init to filter regex/type/dbnum */ + if (keyRegex) { /* filter keys by regex */ + int rc; + /* compile the regular expression */ + if ( (rc = regcomp(&ctx->regex_compiled, keyRegex, REG_EXTENDED)) != 0) { + char buff[1024]; + regerror(rc, &ctx->regex_compiled, buff, sizeof(buff)); + RDB_reportError(p, (RdbRes) RDBX_ERR_FILTER_FAILED_COMPILE_REGEX, + "FilterKey: Error compiling regular expression: %s", buff); + deleteFilterCtx(p, ctx); + return NULL; + } + ctx->regexInitialized = 1; + handleNewKey = filterNewKeyByRegex; + } else if (type) { /* filter keys by type */ + ctx->type = *type; + handleNewKey = filterNewKeyByType; + initOpcodeToType(ctx); + } else { /* filter by dbnum */ + ctx->dbnum = *dbnum; + handleNewDb = filterNewDbByNumber; + } + + ctx->exclude = exclude; + ctx->cbReturnValue = RDB_OK; + + if (RDB_getNumHandlers(p, RDB_LEVEL_DATA)>0) { + RdbHandlersDataCallbacks dataCb; + defaultFilterDataCb(&dataCb); + dataCb.handleNewKey = handleNewKey; + dataCb.handleNewDb = handleNewDb; + RDB_createHandlersData(p, &dataCb, ctx, deleteFilterCtx); + } + + if (RDB_getNumHandlers(p, RDB_LEVEL_STRUCT)>0) { + RdbHandlersStructCallbacks structCb; + defaultFilterStructCb(&structCb); + structCb.handleNewKey = handleNewKey; + structCb.handleNewDb = handleNewDb; + RDB_createHandlersStruct(p, &structCb, ctx, deleteFilterCtx); + } + + if (RDB_getNumHandlers(p, RDB_LEVEL_RAW)>0) { + RdbHandlersRawCallbacks rawCb; + defaultFilterRawCb(&rawCb); + rawCb.handleNewKey = handleNewKey; + rawCb.handleNewDb = handleNewDb; + RDB_createHandlersRaw(p, &rawCb, ctx, deleteFilterCtx); + } + return ctx; +} + +/*** API ***/ + +_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterKey(RdbParser *p, const char *keyRegex, uint32_t exclude) { + return createHandlersFilterCommon(p, keyRegex, NULL, NULL, exclude); +} + +_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterType(RdbParser *p, RdbDataType type, uint32_t exclude) { + return createHandlersFilterCommon(p, NULL, &type, NULL, exclude); +} + +_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterDbNum(RdbParser *p, int dbnum, uint32_t exclude) { + return createHandlersFilterCommon(p, NULL, NULL, &dbnum, exclude); +} diff --git a/src/ext/handlersFilterKey.c b/src/ext/handlersFilterKey.c deleted file mode 100644 index f8ae43b..0000000 --- a/src/ext/handlersFilterKey.c +++ /dev/null @@ -1,163 +0,0 @@ -#include -#include -#include "common.h" - -struct RdbxFilterKey { - regex_t regex_compiled; - int regexInitialized; - int regex_cflags; - int filteroutKey; - RdbRes cbReturnValue; -}; - -static void deleteFilterKeyCtx(RdbParser *p, void *data) { - RdbxFilterKey *ctx = (RdbxFilterKey *) data; - if (ctx->regexInitialized) { - regfree(&ctx->regex_compiled); - } - RDB_free(p, ctx); -} - -/*** Handling common ***/ - -static RdbRes filterNewKey(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) { - UNUSED(p, info); - RdbxFilterKey *ctx = userData; - ctx->cbReturnValue = (regexec(&ctx->regex_compiled, key, 0, NULL, 0)) ? RDB_OK_DONT_PROPAGATE : RDB_OK; - return ctx->cbReturnValue; -} - -static RdbRes filterEndKey(RdbParser *p, void *userData) { - UNUSED(p); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -/*** Handling data ***/ - -static RdbRes filterString(RdbParser *p, void *userData, RdbBulk str) { - UNUSED(p, str); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterList(RdbParser *p, void *userData, RdbBulk item) { - UNUSED(p, item); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { - UNUSED(p, field, value); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -/*** Handling struct ***/ - -static RdbRes filterListLP(RdbParser *p, void *userData, RdbBulk listpack) { - UNUSED(p, listpack); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterListZL(RdbParser *p, void *userData, RdbBulk ziplist) { - UNUSED(p, ziplist); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterListPlain(RdbParser *p, void *userData, RdbBulk listNode) { - UNUSED(p, listNode); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterHashLP(RdbParser *p, void *userData, RdbBulk listpack) { - UNUSED(p, listpack); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterHashZM(RdbParser *p, void *userData, RdbBulk zipmap) { - UNUSED(p, zipmap); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterHashZL(RdbParser *p, void *userData, RdbBulk ziplist) { - UNUSED(p, ziplist); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterHashPlain(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { - UNUSED(p, field, value); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -/*** Handling raw ***/ - -static RdbRes filterFrag(RdbParser *p, void *userData, RdbBulk frag) { - UNUSED(p, frag); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterRawBegin(RdbParser *p, void *userData, size_t size) { - UNUSED(p, size); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -static RdbRes filterRawEnd(RdbParser *p, void *userData) { - UNUSED(p); - return ((RdbxFilterKey *) userData)->cbReturnValue; -} - -RdbxFilterKey *RDBX_createHandlersFilterKey(RdbParser *p, - const char *keyRegex, - uint32_t flags) -{ - RdbxFilterKey *ctx; - UNUSED(flags); - - CallbacksUnion callbacks; - memset (&callbacks, 0, sizeof(callbacks)); - - if ( (ctx = RDB_alloc(p, sizeof(RdbxFilterKey))) == NULL) - return NULL; - - ctx->regexInitialized = 0; - - /* compile the regular expression */ - if (regcomp(&ctx->regex_compiled, keyRegex, REG_EXTENDED) != 0) { - RDB_reportError(p, (RdbRes) RDBX_ERR_FILTER_FAILED_COMPILE_REGEX, - "FilterKey: Error compiling regular expression"); - deleteFilterKeyCtx(p, ctx); - return NULL; - } else { - ctx->regexInitialized = 1; - } - - callbacks.common.handleNewKey = filterNewKey; - callbacks.common.handleEndKey = filterEndKey; - - if (RDB_getNumHandlers(p, RDB_LEVEL_DATA)>0) { - callbacks.dataCb.handleStringValue = filterString; - callbacks.dataCb.handleListItem = filterList; - callbacks.dataCb.handleHashField = filterHash; - RDB_createHandlersData(p, &callbacks.dataCb, ctx, deleteFilterKeyCtx); - } - - if (RDB_getNumHandlers(p, RDB_LEVEL_STRUCT)>0) { - /* string */ - callbacks.structCb.handleString = filterString; - /* list */ - callbacks.structCb.handleListLP = filterListLP; - callbacks.structCb.handleListZL = filterListZL; - callbacks.structCb.handleListPlain = filterListPlain; - /* hash */ - callbacks.structCb.handleHashPlain = filterHashPlain; - callbacks.structCb.handleHashZL = filterHashZL; - callbacks.structCb.handleHashLP = filterHashLP; - callbacks.structCb.handleHashZM = filterHashZM; - RDB_createHandlersStruct(p, &callbacks.structCb, ctx, deleteFilterKeyCtx); - } - - if (RDB_getNumHandlers(p, RDB_LEVEL_RAW)>0) { - callbacks.rawCb.handleFrag = filterFrag; - callbacks.rawCb.handleBegin = filterRawBegin; - callbacks.rawCb.handleEnd = filterRawEnd; - RDB_createHandlersRaw(p, &callbacks.rawCb, ctx, deleteFilterKeyCtx); - } - return ctx; -} diff --git a/src/ext/handlersToJson.c b/src/ext/handlersToJson.c index 01d910f..2642045 100644 --- a/src/ext/handlersToJson.c +++ b/src/ext/handlersToJson.c @@ -394,7 +394,7 @@ static RdbRes toJsonHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk va fprintf(ctx->outfile, ","); outputQuotedEscaping(ctx, field, RDB_bulkLen(p, field)); fprintf(ctx->outfile, ":"); - outputQuotedEscaping(ctx, field, RDB_bulkLen(p, field)); + outputQuotedEscaping(ctx, value, RDB_bulkLen(p, value)); } else { RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, @@ -453,65 +453,70 @@ static RdbRes toJsonRawEnd(RdbParser *p, void *userData) { return RDB_OK; } +#define COMMON_HANDLERS_INIT(commonPart, incAuxField) \ + if (incAuxField) \ + commonPart.handleAuxField = handlingAuxField; \ + commonPart.handleNewKey = toJsonNewKey; \ + commonPart.handleEndKey = toJsonEndKey; \ + commonPart.handleNewDb = toJsonNewDb; \ + commonPart.handleStartRdb = toJsonNewRdb; \ + commonPart.handleEndRdb = toJsonEndRdb; + RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxToJsonConf *conf) { RdbxToJson *ctx = initRdbToJsonCtx(p, filename, conf); if (ctx == NULL) return NULL; - CallbacksUnion callbacks; - memset (&callbacks, 0, sizeof(callbacks)); - - if (ctx->conf.includeAuxField) - callbacks.common.handleAuxField = handlingAuxField; - - callbacks.common.handleNewKey = toJsonNewKey; - callbacks.common.handleEndKey = toJsonEndKey; - callbacks.common.handleNewDb = toJsonNewDb; - callbacks.common.handleStartRdb = toJsonNewRdb; - callbacks.common.handleEndRdb = toJsonEndRdb; - if (ctx->conf.level == RDB_LEVEL_DATA) { - - callbacks.dataCb.handleStringValue = toJsonString; - callbacks.dataCb.handleListItem = toJsonList; - callbacks.dataCb.handleHashField = toJsonHash; - callbacks.dataCb.handleSetMember = toJsonSet; - callbacks.dataCb.handleZsetMember = toJsonZset; - callbacks.dataCb.handleFunction = (conf->includeFunc) ? toJsonFunction : NULL; - callbacks.dataCb.handleModule = toJsonModule; - RDB_createHandlersData(p, &callbacks.dataCb, ctx, deleteRdbToJsonCtx); + RdbHandlersDataCallbacks dataCb; + memset(&dataCb, 0 , sizeof(dataCb)); + COMMON_HANDLERS_INIT(dataCb, ctx->conf.includeAuxField); + + dataCb.handleStringValue = toJsonString; + dataCb.handleListItem = toJsonList; + dataCb.handleHashField = toJsonHash; + dataCb.handleSetMember = toJsonSet; + dataCb.handleZsetMember = toJsonZset; + dataCb.handleFunction = (ctx->conf.includeFunc) ? toJsonFunction : NULL; + dataCb.handleModule = toJsonModule; + RDB_createHandlersData(p, &dataCb, ctx, deleteRdbToJsonCtx); } else if (ctx->conf.level == RDB_LEVEL_STRUCT) { + RdbHandlersStructCallbacks structCb; + memset(&structCb, 0 , sizeof(structCb)); + COMMON_HANDLERS_INIT(structCb, ctx->conf.includeAuxField); - callbacks.structCb.handleString = toJsonString; + structCb.handleString = toJsonString; /* list */ - callbacks.structCb.handleListPlain = toJsonList; - callbacks.structCb.handleListLP = toJsonStruct; - callbacks.structCb.handleListZL = toJsonStruct; + structCb.handleListPlain = toJsonList; + structCb.handleListLP = toJsonStruct; + structCb.handleListZL = toJsonStruct; /* hash */ - callbacks.structCb.handleHashPlain = toJsonHash; - callbacks.structCb.handleHashZL = toJsonStruct; - callbacks.structCb.handleHashLP = toJsonStruct; - callbacks.structCb.handleHashZM = toJsonStruct; + structCb.handleHashPlain = toJsonHash; + structCb.handleHashZL = toJsonStruct; + structCb.handleHashLP = toJsonStruct; + structCb.handleHashZM = toJsonStruct; /* set */ - callbacks.structCb.handleSetPlain = toJsonSet; - callbacks.structCb.handleSetIS = toJsonStruct; - callbacks.structCb.handleSetLP = toJsonStruct; + structCb.handleSetPlain = toJsonSet; + structCb.handleSetIS = toJsonStruct; + structCb.handleSetLP = toJsonStruct; /* zset */ - callbacks.structCb.handleZsetPlain = toJsonZset; - callbacks.structCb.handleZsetZL = toJsonStruct; - callbacks.structCb.handleZsetLP = toJsonStruct; + structCb.handleZsetPlain = toJsonZset; + structCb.handleZsetZL = toJsonStruct; + structCb.handleZsetLP = toJsonStruct; /* function */ - callbacks.structCb.handleFunction = (conf->includeFunc) ? toJsonFunction : NULL; + structCb.handleFunction = (ctx->conf.includeFunc) ? toJsonFunction : NULL; /* module */ - callbacks.structCb.handleModule = toJsonModule; - RDB_createHandlersStruct(p, &callbacks.structCb, ctx, deleteRdbToJsonCtx); + structCb.handleModule = toJsonModule; + RDB_createHandlersStruct(p, &structCb, ctx, deleteRdbToJsonCtx); } else if (ctx->conf.level == RDB_LEVEL_RAW) { - - callbacks.rawCb.handleFrag = toJsonFrag; - callbacks.rawCb.handleBegin = toJsonRawBegin; - callbacks.rawCb.handleEnd = toJsonRawEnd; - RDB_createHandlersRaw(p, &callbacks.rawCb, ctx, deleteRdbToJsonCtx); + RdbHandlersRawCallbacks rawCb; + memset(&rawCb, 0 , sizeof(rawCb)); + COMMON_HANDLERS_INIT(rawCb, ctx->conf.includeAuxField); + rawCb.handleFrag = toJsonFrag; + rawCb.handleBegin = toJsonRawBegin; + rawCb.handleEnd = toJsonRawEnd; + RDB_createHandlersRaw(p, &rawCb, ctx, deleteRdbToJsonCtx); } return ctx; diff --git a/src/ext/handlersToResp.c b/src/ext/handlersToResp.c index f21b45d..8367e2c 100644 --- a/src/ext/handlersToResp.c +++ b/src/ext/handlersToResp.c @@ -2,7 +2,6 @@ #include #include #include "common.h" -#include "utils.h" #include "../../deps/redis/crc64.h" #include "../../deps/redis/util.h" @@ -12,6 +11,8 @@ #define _REDISMODULE_AUX_BEFORE_RDB (1<<0) #define VER_VAL(major,minor) (((unsigned int)(major)<<8) | (unsigned int)(minor)) +#define KEY_CMD_ID_DBG "_RDB_CLI_CMD_ID_" + typedef struct RedisToRdbVersion { unsigned int redis; unsigned char rdb; @@ -33,19 +34,23 @@ typedef enum DelKeyBeforeWrite { DEL_KEY_BEFORE_BY_RESTORE_REPLACE, /* RESTORE supported */ } DelKeyBeforeWrite; -#define IOV_CONST(iov, str) iov_plain(iov, str, sizeof(str)-1) -#define IOV_STRING(iov, str, len) iov_plain(iov, str, len) -#define IOV_VALUE(iov, val, ar) iov_value(iov, val, ar, sizeof(ar)) -#define IOV_LEN_AND_VALUE(iov, val, ar1, ar2) \ - do {\ - int l = IOV_VALUE((iov)+1, val, ar2); \ - IOV_VALUE( (iov), l, ar1); \ - } while (0); - struct RdbxToResp { RdbxToRespConf conf; + struct RdbxToRespDebug { + + size_t cmdNum; + + /* configuration */ +#define RFLAG_ENUM_CMD_ID (1<<0) /* Enumerate and trace commands by pushing debug command + * of type "SET _RDB_CLI_CMD_ID_ " before each + * RESP command */ +#define RFLAG_WRITE_FROM_CMD_ID (1<<1) /* Flag for writing commands from a specific command-id */ + int flags; + size_t writeFromCmdNum; + } debug; + /* Init to 3. Attempted to be released three times on termination */ int refcount; @@ -68,7 +73,7 @@ struct RdbxToResp { int isModuleAux; uint64_t crc; struct { - char cmdPrefix[100]; + char cmdPrefix[128]; int cmdlen; } moduleAux; } rawCtx; @@ -95,12 +100,7 @@ static void deleteRdbToRespCtx(RdbParser *p, void *context) { RDB_free(p, ctx); } -static inline void iov_plain(struct iovec *iov, const char *s, size_t l) { - iov->iov_base = (void *) s; - iov->iov_len = l; -} - -static int iov_value(struct iovec *iov, long long count, char *buf, int bufsize) { +int iov_value(struct iovec *iov, long long count, char *buf, int bufsize) { int len = 0; len = ll2string(buf, bufsize, count); @@ -163,11 +163,49 @@ static void resolveSupportRestore(RdbParser *p, RdbxToResp *ctx, int srcRdbVer) RDB_handleByLevel(p, RDB_DATA_TYPE_MODULE, RDB_LEVEL_RAW, 0); } +static inline RdbRes onWriteNewCmdDbg(RdbxToResp *ctx) { + RdbxRespWriter *writer = &ctx->respWriter; + size_t currCmdNum = ctx->debug.cmdNum++; + + /* Write only commands starting from given command number */ + if ((ctx->debug.flags & RFLAG_WRITE_FROM_CMD_ID) && + (currCmdNum < ctx->debug.writeFromCmdNum)) + return RDB_OK; + + /* enumerate and trace cmd-id by preceding each cmd with "SET _RDB_CLI_CMD_ID_ " */ + if (ctx->debug.flags & RFLAG_ENUM_CMD_ID) { + char keyLenStr[32], cmdIdLenStr[32], cmdIdStr[32]; + + struct iovec iov[7]; + /* write SET */ + IOV_CONST(&iov[0], "*3\r\n$3\r\nSET\r\n$"); + /* write key */ + IOV_VALUE(&iov[1], sizeof(KEY_CMD_ID_DBG)-1, keyLenStr); + IOV_STRING(&iov[2], KEY_CMD_ID_DBG, sizeof(KEY_CMD_ID_DBG)-1); + /* write cmd-id */ + IOV_CONST(&iov[3], "\r\n$"); + IOV_LEN_AND_VALUE(&iov[4], currCmdNum, cmdIdLenStr, cmdIdStr); + if (unlikely(writer->writev(writer->ctx, iov, 6, 1, 1))) { + RdbRes errCode = RDB_getErrorCode(ctx->parser); + assert(errCode != RDB_OK); + return RDB_getErrorCode(ctx->parser); + } + } + return RDB_OK; +} + static inline RdbRes writevWrap(RdbxToResp *ctx, struct iovec *iov, int cnt, int startCmd, int endCmd) { + RdbRes res; RdbxRespWriter *writer = &ctx->respWriter; + + if (unlikely(ctx->debug.flags && startCmd)) { + if ((res = onWriteNewCmdDbg(ctx)) != RDB_OK) + return RDB_getErrorCode(ctx->parser); + } + if (unlikely(writer->writev(writer->ctx, iov, cnt, startCmd, endCmd))) { - RdbRes errCode = RDB_getErrorCode(ctx->parser); - assert(errCode != RDB_OK); + res = RDB_getErrorCode(ctx->parser); + assert(res != RDB_OK); return RDB_getErrorCode(ctx->parser); } @@ -643,3 +681,12 @@ _LIBRDB_API void RDBX_attachRespWriter(RdbxToResp *rdbToResp, RdbxRespWriter *wr rdbToResp->respWriter = *writer; rdbToResp->respWriterConfigured = 1; } + +_LIBRDB_API void RDBX_enumerateCmds(RdbxToResp *rdbToResp) { + rdbToResp->debug.flags |= RFLAG_ENUM_CMD_ID; +} + +_LIBRDB_API void RDBX_writeFromCmdNumber(RdbxToResp *rdbToResp, size_t cmdNum) { + rdbToResp->debug.flags |= RFLAG_WRITE_FROM_CMD_ID; + rdbToResp->debug.writeFromCmdNum = cmdNum; +} diff --git a/src/ext/respToRedisLoader.c b/src/ext/respToRedisLoader.c index 6ccdc54..60e6c49 100644 --- a/src/ext/respToRedisLoader.c +++ b/src/ext/respToRedisLoader.c @@ -1,7 +1,6 @@ #include -#include -#include #include +#include #include #include #include @@ -22,7 +21,7 @@ #define REPLY_BUFF_SIZE 4096 /* reply buffer size */ -#define MAX_EINTR_RETRY 3 +#define MAX_EINTR_RETRY 3 struct RdbxRespToRedisLoader { @@ -31,7 +30,6 @@ struct RdbxRespToRedisLoader { int num; int pipelineDepth; char cmdPrefix[NUM_RECORDED_CMDS][RECORDED_DATA_MAX_LEN]; - int cmdAt; } pendingCmds; RespReaderCtx respReader; @@ -195,43 +193,125 @@ static void redisLoaderDelete(void *context) { RDB_free(ctx->p, ctx); } +static RdbRes redisAuthCustomized(RdbxRespToRedisLoader *ctx, RdbxRedisAuth *auth) { + int i, iovs; + RdbRes res = RDB_OK; + + /* custom auth command - Need to break it into tokens based on spaces and + * tabs. And then translate it into RESP protocol */ + + char prefix[32]; + + /* allocate iovec (2 for header and trailer. 3 for each argument) */ + struct iovec *iov = (struct iovec *)malloc((auth->cmd.argc * 3 + 2) * sizeof(struct iovec)); + /* allocate temporary buffer to assist converting length to string of all args */ + char (*lenStr)[21] = (char (*)[21])malloc(auth->cmd.argc * 21 * sizeof(char)); + + if (iov == NULL || lenStr == NULL) { + RDB_reportError(ctx->p, RDB_ERR_FAIL_ALLOC, + "Failed to allocate for customized AUTH (tokens=%d)", auth->cmd.argc); + res = RDB_ERR_FAIL_ALLOC; // Return an error code + goto AuthEnd; + } + + /* set number of elements in the prefix of the RESP command */ + iov[0].iov_len = snprintf(prefix, sizeof(prefix)-1, "*%d", auth->cmd.argc); + iov[0].iov_base = prefix; + + for ( i = 0, iovs = 1 ; i < auth->cmd.argc ; ++i) + { + size_t tLen = strlen(auth->cmd.argv[i]); + IOV_CONST(&iov[iovs++], "\r\n$"); + IOV_VALUE(&iov[iovs++], tLen, lenStr[i]); + IOV_STRING(&iov[iovs++], auth->cmd.argv[i], tLen); + } + IOV_CONST(&iov[iovs++], "\r\n"); + redisLoaderWritev(ctx, iov, iovs, 1, 1); + +AuthEnd: + if (iov) free(iov); + if (lenStr) free(lenStr); + return res; +} + +static RdbRes redisAuth(RdbxRespToRedisLoader *ctx, RdbxRedisAuth *auth) { + int iovs; + char userLenStr[21], pwdLenStr[21]; + + if ((auth->pwd == NULL) && (auth->cmd.argc == 0)) + return RDB_OK; + + /* if customized auth command */ + if (auth->cmd.argv) + return redisAuthCustomized(ctx, auth); + + /* AUTH [username] password */ + struct iovec iov[10]; + if (auth->user) { + IOV_CONST(&iov[0], "*3\r\n$4\r\nauth\r\n$"); + /* write user */ + IOV_VALUE(&iov[1], strlen(auth->user), userLenStr); + IOV_STRING(&iov[2], auth->user, strlen(auth->user)); + IOV_CONST(&iov[3], "\r\n$"); + /* write pwd */ + IOV_VALUE(&iov[4], strlen(auth->pwd), pwdLenStr); + IOV_STRING(&iov[5], auth->pwd, strlen(auth->pwd)); + IOV_CONST(&iov[6], "\r\n"); + iovs = 7; + } else { + IOV_CONST(&iov[0], "*2\r\n$4\r\nauth\r\n$"); + /* write pwd */ + IOV_VALUE(&iov[1], strlen(auth->pwd), pwdLenStr); + IOV_STRING(&iov[2], auth->pwd, strlen(auth->pwd)); + IOV_CONST(&iov[3], "\r\n"); + iovs = 4; + } + + redisLoaderWritev(ctx, iov, iovs, 1, 1); + return RDB_OK; +} + +/*** LIB API functions ***/ + _LIBRDB_API void RDBX_setPipelineDepth(RdbxRespToRedisLoader *r2r, int depth) { r2r->pendingCmds.pipelineDepth = (depth <= 0 || depth>PIPELINE_DEPTH_MAX) ? PIPELINE_DEPTH_DEF : depth; } _LIBRDB_API RdbxRespToRedisLoader *RDBX_createRespToRedisFd(RdbParser *p, RdbxToResp *rdbToResp, - int fd) -{ - RdbxRespToRedisLoader *ctx; - if ((ctx = RDB_alloc(p, sizeof(RdbxRespToRedisLoader))) == NULL) { - close(fd); - RDB_reportError(p, (RdbRes) RDBX_ERR_RESP_FAILED_ALLOC, - "Failed to allocate struct RdbxRespToRedisLoader"); - return NULL; - } + RdbxRedisAuth *auth, + int fd) { + RdbxRespToRedisLoader *ctx; + if ((ctx = RDB_alloc(p, sizeof(RdbxRespToRedisLoader))) == NULL) { + close(fd); + RDB_reportError(p, (RdbRes) RDBX_ERR_RESP_FAILED_ALLOC, + "Failed to allocate struct RdbxRespToRedisLoader"); + return NULL; + } + + /* init RdbxRespToRedisLoader context */ + memset(ctx, 0, sizeof(RdbxRespToRedisLoader)); + ctx->p = p; + ctx->fd = fd; + ctx->pendingCmds.num = 0; + ctx->pendingCmds.pipelineDepth = PIPELINE_DEPTH_DEF; + readRespInit(&ctx->respReader); - /* init RdbxRespToRedisLoader context */ - memset(ctx, 0, sizeof(RdbxRespToRedisLoader)); - ctx->p = p; - ctx->fd = fd; - ctx->pendingCmds.num = 0; - ctx->pendingCmds.pipelineDepth = PIPELINE_DEPTH_DEF; - readRespInit(&ctx->respReader); - - /* Set 'this' writer to rdbToResp */ - RdbxRespWriter inst = {ctx, redisLoaderDelete, redisLoaderWritev, redisLoaderFlush}; - RDBX_attachRespWriter(rdbToResp, &inst); - return ctx; + if (auth && (redisAuth(ctx, auth) != RDB_OK)) + return NULL; + + /* Set 'this' writer to rdbToResp */ + RdbxRespWriter inst = {ctx, redisLoaderDelete, redisLoaderWritev, redisLoaderFlush}; + RDBX_attachRespWriter(rdbToResp, &inst); + return ctx; } _LIBRDB_API RdbxRespToRedisLoader *RDBX_createRespToRedisTcp(RdbParser *p, RdbxToResp *rdbToResp, + RdbxRedisAuth *auth, const char *hostname, int port) { - int sockfd; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); + int sockfd = socket(AF_INET, SOCK_STREAM, 0); if (sockfd == -1) { RDB_reportError(p, (RdbRes) RDBX_ERR_RESP2REDIS_CREATE_SOCKET, "Failed to create tcp socket"); return NULL; @@ -255,5 +335,5 @@ _LIBRDB_API RdbxRespToRedisLoader *RDBX_createRespToRedisTcp(RdbParser *p, return NULL; } - return RDBX_createRespToRedisFd(p, rdbToResp, sockfd); + return RDBX_createRespToRedisFd(p, rdbToResp, auth, sockfd); } diff --git a/src/ext/utils.c b/src/ext/utils.c deleted file mode 100644 index e19e163..0000000 --- a/src/ext/utils.c +++ /dev/null @@ -1,58 +0,0 @@ -#include -#include -#include "utils.h" - -/* printHexDump() Generates a formatted hexadecimal and ASCII representation of binary - * data. Given a memory address and its length, it produces a human-readable obuf, - * displaying byte offsets in hexadecimal and replacing non-printable characters with - * dots ('.'). - * - * Returns how many bytes written to obuf buffer. -1 Otherwise. - * - * Output example for input: "A123456789B123456789C123456789D123456789" - * 000000 41 31 32 33 34 35 36 37 38 39 42 31 32 33 34 35 A1234567 89B12345 - * 000010 36 37 38 39 43 31 32 33 34 35 36 37 38 39 44 31 6789C123 456789D1 - * 000020 32 33 34 35 36 37 38 39 23456789 - */ -int printHexDump(const char *input, size_t len, char *obuf, int obuflen) { - size_t i; - int iout=0, j, llen = 16; /* line len */ - unsigned char buff[llen + 10]; - - if (input == NULL || len <= 0 || obuf == NULL || obuflen < 200 || obuflen > 0xFFFFFF) - return -1; - - for (i = 0, j = 0; (i < len) && (iout + 100 < obuflen) ; i++) { - if ((i % llen) == 0) { - if (i > 0) { - buff[j] = '\0'; - iout += snprintf(obuf + iout, obuflen - iout, " %s\n", buff); - } - iout += snprintf(obuf + iout, obuflen - iout, "%06zx ", i); - j = 0; - } - - if (((int)i % llen) == (llen / 2)) { /* middle of the line */ - iout += snprintf(obuf + iout, obuflen - iout, " "); - buff[j++] = ' '; - buff[j++] = ' '; - } - - iout += snprintf(obuf + iout, obuflen - iout, " %02x", (unsigned char)input[i]); - buff[j++] = (isprint(input[i])) ? input[i] : '.'; - } - - /* pad the last line */ - for (; (i % llen) != 0; i++) { - iout += snprintf(obuf + iout, obuflen - iout, " "); - if (( (int)i % llen) == (llen / 2)) { - iout += snprintf(obuf + iout, obuflen - iout, " "); - } - } - - buff[j] = '\0'; - iout += snprintf(obuf + iout, obuflen - iout, " %s\n", buff); - if (i < len) - iout += snprintf(obuf + iout, obuflen - iout, "..."); - return iout; -} diff --git a/src/ext/utils.h b/src/ext/utils.h deleted file mode 100644 index 149d406..0000000 --- a/src/ext/utils.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef LIBRDB_UTILS_H -#define LIBRDB_UTILS_H - -int printHexDump(const char *addr, size_t len, char *obuf, int obuflen); - -#endif /*LIBRDB_UTILS_H*/ diff --git a/src/lib/bulkAlloc.c b/src/lib/bulkAlloc.c index b791482..7c6b996 100644 --- a/src/lib/bulkAlloc.c +++ b/src/lib/bulkAlloc.c @@ -180,7 +180,7 @@ BulkInfo *bulkPoolAlloc(RdbParser *p, size_t len, AllocTypeRq typeRq, char *refB /* if requested ref another memory but forced to allocate a new buffer since configured * RDB_BULK_ALLOC_EXTERN, then copy referenced data to the new allocated buffer */ - if (unlikely(typeRq == RQ_ALLOC_APP_BULK_REF) && (type != BULK_TYPE_REF)) + if ((typeRq == RQ_ALLOC_APP_BULK_REF) && (type != BULK_TYPE_REF)) memcpy(binfo->ref, refBuf, len); } else { @@ -473,6 +473,11 @@ static inline BulkType bulkUnmanagedResolveAllocType(RdbParser *p, AllocUnmngTyp void bulkUnmanagedAlloc(RdbParser *p, size_t len, AllocUnmngTypeRq rq, char *refBuf, BulkInfo *bi) { BulkType type = bulkUnmanagedResolveAllocType(p, rq); bulkPoolAllocNew(p, len, type, refBuf, bi); + + /* if requested ref another memory but forced to allocate a new buffer since configured + * RDB_BULK_ALLOC_EXTERN, then copy referenced data to the new allocated buffer */ + if ((rq == UNMNG_RQ_ALLOC_APP_BULK_REF) && (type != BULK_TYPE_REF)) + memcpy(bi->ref, refBuf, len); } void bulkUnmanagedFree(RdbParser *p, BulkInfo *binfo) { diff --git a/src/lib/bulkAlloc.h b/src/lib/bulkAlloc.h index b33e91f..eda3043 100644 --- a/src/lib/bulkAlloc.h +++ b/src/lib/bulkAlloc.h @@ -29,12 +29,10 @@ * the next item in the queue. * c) Flush - Clean the entire queue and deletes corresponding referenced buffers. * - * The bulk-pool utilizes three distinct types of allocators: - * a) Stack allocator - * b) Heap allocator - * c) External allocator + * The bulk-pool utilizes 3 distinct types of allocators: * - * The Stack Allocator is specifically designed to work in tandem with the BulkPool + * 1) STACK ALLOCATOR + * The Stack Allocator is specifically designed to work in tandem with the BulkPool * and supports the Allocate, Rollback, and Flush commands. When the pool * receives small allocation requests and the application has not restricted * allocation to a specific type, it prefers to allocate from the stack. If @@ -42,8 +40,10 @@ * to replay. If the parser reaches a new state, then the stack will be flushed * along with the pool. * + * 2) HEAP ALLOCATOR * The Heap Allocator allocates memory from the heap with refcount support. * + * 3) EXTERNAL ALLOCATOR * The External Allocator is not mandatory and can be provided by the application * client to allocate only the buffers that will be passed to the application's * callbacks. These buffers are referred to as RQ_ALLOC_APP_BULK within this API. diff --git a/src/lib/parser.c b/src/lib/parser.c index b074143..7d625fe 100644 --- a/src/lib/parser.c +++ b/src/lib/parser.c @@ -72,7 +72,9 @@ struct ParsingElementInfo peInfo[PE_MAX] = { [PE_ZSET_2] = {elementZset, "elementZset", "Parsing zset_2"}, [PE_ZSET_ZL] = {elementZsetZL, "elementZsetZL", "Parsing zset Ziplist"}, [PE_ZSET_LP] = {elementZsetLP, "elementZsetLP", "Parsing zset Listpack"}, + /* func */ [PE_FUNCTION] = {elementFunction, "elementFunction", "Parsing Function"}, + /* module */ [PE_MODULE] = {elementModule, "elementModule", "Parsing silently Module element"}, [PE_MODULE_AUX] = {elementModule, "elementModule", "Parsing silently Module Auxiliary data"}, @@ -115,12 +117,12 @@ struct ParsingElementInfo peInfo[PE_MAX] = { * the size of the allocation since in ZP/LP it is guaranteed to have * a terminating byte at the end, which makes it safe. * - * 2. Then allocate from cache `RQ_ALLOC_APP_BULK_REF` which will: + * 2. Then allocate from cache `UNMNG_RQ_ALLOC_APP_BULK_REF` which will: * - Set last character '\0' to terminate the value. * - And mark the value as a referenced bulk allocation - * (Note, iF app expects APP_BULK, then it is not possible to return + * (if app expects APP_BULK, then it is not possible to return * a reference and a new memory will be allocated instead with proper - * termination of '\0'). + * termination of '\0'. Inefficient but valid). * * First two steps are achieved by calling function allocEmbeddedBulk() * @@ -129,7 +131,7 @@ struct ParsingElementInfo peInfo[PE_MAX] = { * Otherwise ZP/LP will be left corrupted! */ typedef struct { - BulkInfo *binfo; + BulkInfo binfo; /* unmanaged bulk info */ unsigned char endCh; /* stores value of last char that was overriden with '\0' */ unsigned char *pEndCh; /* stores ref to last char that was overriden with '\0' */ } EmbeddedBulk; @@ -157,12 +159,12 @@ static void loggerCbDefault(RdbLogLevel l, const char *msg); static inline RdbStatus updateStateAfterParse(RdbParser *p, RdbStatus status); static void printParserState(RdbParser *p); -static inline void restoreEmbeddedBulk(EmbeddedBulk *embeddedBulk); -BulkInfo *allocEmbeddedBulk(RdbParser *p, - unsigned char *str, - unsigned int slen, - long long sval, - EmbeddedBulk *embeddedBulk); +static inline void restoreEmbeddedBulk(RdbParser *p, EmbeddedBulk *embeddedBulk); +static void *allocEmbeddedBulk(RdbParser *p, + unsigned char *str, + unsigned int slen, + long long sval, + EmbeddedBulk *embeddedBulk); /*** RDB Reader function ***/ static RdbStatus readRdbFromReader(RdbParser *p, size_t len, AllocTypeRq type, char *refBuf, BulkInfo **binfo); @@ -562,11 +564,11 @@ _LIBRDB_API const char *RDB_getLibVersion(int *major, int *minor, int *patch) { static const char *getStatusString(RdbStatus status) { switch ((int) status) { - case RDB_STATUS_OK: return "RDB_STATUS_OK"; - case RDB_STATUS_WAIT_MORE_DATA: return "RDB_STATUS_WAIT_MORE_DATA"; - case RDB_STATUS_PAUSED: return "RDB_STATUS_PAUSED"; - case RDB_STATUS_ERROR: return "RDB_STATUS_ERROR"; - case RDB_STATUS_ENDED: return "(RDB_STATUS_ENDED)"; /* internal state. (Not part of API) */ + case RDB_STATUS_OK: return "OK"; + case RDB_STATUS_WAIT_MORE_DATA: return "WAIT_MORE_DATA"; + case RDB_STATUS_PAUSED: return "PAUSED"; + case RDB_STATUS_ERROR: return "ERROR"; + case RDB_STATUS_ENDED: return "(ENDED)"; /* internal state. (Not part of API) */ default: assert(0); } } @@ -615,14 +617,31 @@ static RdbStatus parserMainLoop(RdbParser *p) { if (unlikely(p->debugData)) { while (1) { - RDB_log(p, RDB_LOG_DBG, "[Opcode=%d] %s(State=%d)", - p->currOpcode, - peInfo[p->parsingElement].funcname, - p->elmCtx.state); + char buff[512]; + + int isNewOpcode = (p->parsingElement == PE_NEXT_RDB_TYPE) ? 1 : 0; + int isNewDb = (p->parsingElement == PE_SELECT_DB) ? 1 : 0; + + size_t bytesReadBefore = p->bytesRead; + const char *funcName = peInfo[p->parsingElement].funcname; + int stateBefore = p->elmCtx.state; + status = peInfo[p->parsingElement].func(p); - RDB_log(p, RDB_LOG_DBG, "Return status=%s next %s(State=%d)\n", getStatusString(status), - peInfo[p->parsingElement].funcname, - p->elmCtx.state); + + int at = snprintf(buff, sizeof(buff)-1, "[0x%08zx] %s(State=%d)", bytesReadBefore, funcName, stateBefore); + + /* print important milestones in parsing */ + if (isNewOpcode) + at += snprintf(buff+at, sizeof(buff)-1-at, " [Opcode=%03d]", p->currOpcode); + if (isNewDb) + at += snprintf(buff+at, sizeof(buff)-1-at, " [SelectDB=%d]", p->selectedDb); + if(p->currKeyDbg[0] != '\0') + at += snprintf(buff+at, sizeof(buff)-1-at, " [key=%s]", p->currKeyDbg); + if (status != RDB_STATUS_OK) + at += snprintf(buff+at, sizeof(buff)-1-at, " >>> [Status=%s]", getStatusString(status)); + + RDB_log(p, RDB_LOG_DBG, buff); + if (status != RDB_STATUS_OK) break; /* if RDB_STATUS_OK then the parser completed a state and the cache is empty */ @@ -658,8 +677,13 @@ static inline RdbStatus nextParsingElementKeyValue(RdbParser *p, static RdbRes handleNewKeyPrintDbg(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) { UNUSED(p,userData,info); - UNUSED(key); - RDB_log(p, RDB_LOG_DBG, "Key=%s, ", key); + snprintf(p->currKeyDbg, sizeof(p->currKeyDbg)-1, "%s", key); + return RDB_OK; +} + +static RdbRes handleEndKeyPrintDbg(RdbParser *p, void *userData) { + UNUSED(p,userData); + p->currKeyDbg[0] = '\0'; return RDB_OK; } @@ -718,7 +742,7 @@ static RdbStatus finalizeConfig(RdbParser *p, int isParseFromBuff) { if ((p->debugData = getEnvVar(ENV_VAR_DEBUG_DATA, 0)) != 0) { RDB_setLogLevel(p, RDB_LOG_DBG); - RdbHandlersDataCallbacks cb = {.handleNewKey = handleNewKeyPrintDbg}; + RdbHandlersDataCallbacks cb = {.handleNewKey = handleNewKeyPrintDbg, .handleEndKey = handleEndKeyPrintDbg}; RDB_createHandlersData(p, &cb, NULL, NULL); } @@ -818,12 +842,12 @@ static inline RdbStatus unpackList(RdbParser *p, unsigned char *lp) { if (!allocEmbeddedBulk(p, item, itemLen, itemVal, &embBulk)) return RDB_STATUS_ERROR; - registerAppBulkForNextCb(p, embBulk.binfo); + registerAppBulkForNextCb(p, &embBulk.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(&embBulk), /*finalize*/ + restoreEmbeddedBulk(p, &embBulk), /*finalize*/ RDB_LEVEL_DATA, rdbData.handleListItem, - embBulk.binfo->ref); + embBulk.binfo.ref); eptr = lpNext( lp, eptr); } @@ -872,12 +896,12 @@ static RdbStatus listZiplistItem(RdbParser *p, BulkInfo *ziplistBulk) { if (!allocEmbeddedBulk(p, item, itemLen, itemVal, &embBulk)) return RDB_STATUS_ERROR; - registerAppBulkForNextCb(p, embBulk.binfo); + registerAppBulkForNextCb(p, &embBulk.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(&embBulk);, /*finalize*/ + restoreEmbeddedBulk(p, &embBulk);, /*finalize*/ RDB_LEVEL_DATA, rdbData.handleListItem, - embBulk.binfo->ref); + embBulk.binfo.ref); } return RDB_STATUS_OK; } @@ -890,35 +914,38 @@ static int counterCallback(unsigned char *ptr, unsigned int head_count, void *us return 1; } -static inline void restoreEmbeddedBulk(EmbeddedBulk *embeddedBulk) { +static inline void restoreEmbeddedBulk(RdbParser *p, EmbeddedBulk *embeddedBulk) { *(embeddedBulk->pEndCh) = embeddedBulk->endCh; + bulkUnmanagedFree(p, &embeddedBulk->binfo); } -BulkInfo *allocEmbeddedBulk(RdbParser *p, - unsigned char *str, - unsigned int slen, - long long sval, - EmbeddedBulk *embeddedBulk) +/* return 0 on failure */ +static void *allocEmbeddedBulk(RdbParser *p, + unsigned char *str, + unsigned int slen, + long long sval, + EmbeddedBulk *embeddedBulk) { - RdbStatus res; + /* The reason that a simplified unmanaged-bulk is used rather than allocating from + * pool, is because this feature is only used from a safe state after all the + * data prefetched (and for a momentary callback to the registered handlers) */ if (str) { unsigned char *strEnd = str + slen; embeddedBulk->endCh = *strEnd; embeddedBulk->pEndCh = strEnd; - res = allocFromCache(p, slen, RQ_ALLOC_APP_BULK_REF, (char *) str, &(embeddedBulk->binfo)); - if (unlikely(res!=RDB_STATUS_OK)) return NULL; + bulkUnmanagedAlloc(p, slen, UNMNG_RQ_ALLOC_APP_BULK_REF, (char *) str, &embeddedBulk->binfo); } else { static unsigned char dummy; embeddedBulk->pEndCh = &dummy; int buflen = 32; - res = allocFromCache(p, buflen, RQ_ALLOC_APP_BULK, NULL, &(embeddedBulk->binfo)); - if (unlikely(res!=RDB_STATUS_OK)) return NULL; - embeddedBulk->binfo->len = ll2string(embeddedBulk->binfo->ref, buflen, sval); + bulkUnmanagedAlloc(p, buflen, UNMNG_RQ_ALLOC_APP_BULK, (char *) str, &embeddedBulk->binfo); + if (!(embeddedBulk->binfo.ref)) return 0; + embeddedBulk->binfo.len = ll2string(embeddedBulk->binfo.ref, buflen, sval); } - return embeddedBulk->binfo; + return embeddedBulk->binfo.ref; } -RdbStatus hashZiplist(RdbParser *p, BulkInfo *ziplistBulk) { +static RdbStatus hashZiplist(RdbParser *p, BulkInfo *ziplistBulk) { size_t items = 0; if (unlikely(0 == ziplistValidateIntegrity(ziplistBulk->ref, ziplistBulk->len, p->deepIntegCheck, counterCallback, &items))) { @@ -958,19 +985,19 @@ RdbStatus hashZiplist(RdbParser *p, BulkInfo *ziplistBulk) { if (!allocEmbeddedBulk(p, value, valueLen, valueVal, &embBulk2)) return RDB_STATUS_ERROR; - registerAppBulkForNextCb(p, embBulk1.binfo); - registerAppBulkForNextCb(p, embBulk2.binfo); + registerAppBulkForNextCb(p, &embBulk1.binfo); + registerAppBulkForNextCb(p, &embBulk2.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(&embBulk1); restoreEmbeddedBulk(&embBulk2);, /*finalize*/ + restoreEmbeddedBulk(p, &embBulk1); restoreEmbeddedBulk(p, &embBulk2);, /*finalize*/ RDB_LEVEL_DATA, rdbData.handleHashField, - embBulk1.binfo->ref, - embBulk2.binfo->ref); + embBulk1.binfo.ref, + embBulk2.binfo.ref); } return RDB_STATUS_OK; } -RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) { +static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) { size_t items = 0; if (unlikely(0 == lpValidateIntegrity(lpBulk->ref, lpBulk->len, p->deepIntegCheck, counterCallback, &items))) { @@ -1010,19 +1037,19 @@ RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) { if (!allocEmbeddedBulk(p, value, valueLen, valueVal, &embBulk2)) return RDB_STATUS_ERROR; - registerAppBulkForNextCb(p, embBulk1.binfo); - registerAppBulkForNextCb(p, embBulk2.binfo); + registerAppBulkForNextCb(p, &embBulk1.binfo); + registerAppBulkForNextCb(p, &embBulk2.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(&embBulk1); restoreEmbeddedBulk(&embBulk2);, /*finalize*/ + restoreEmbeddedBulk(p, &embBulk1); restoreEmbeddedBulk(p, &embBulk2);, /*finalize*/ RDB_LEVEL_DATA, rdbData.handleHashField, - embBulk1.binfo->ref, - embBulk2.binfo->ref); + embBulk1.binfo.ref, + embBulk2.binfo.ref); } return RDB_STATUS_OK; } -RdbStatus hashZipMap(RdbParser *p, BulkInfo *zpBulk) { +static RdbStatus hashZipMap(RdbParser *p, BulkInfo *zpBulk) { unsigned char *field, *value; unsigned int fieldLen, valueLen; @@ -1050,14 +1077,14 @@ RdbStatus hashZipMap(RdbParser *p, BulkInfo *zpBulk) { if (!allocEmbeddedBulk(p, value, valueLen, 0, &embBulk2)) return RDB_STATUS_ERROR; - registerAppBulkForNextCb(p, embBulk1.binfo); - registerAppBulkForNextCb(p, embBulk2.binfo); + registerAppBulkForNextCb(p, &embBulk1.binfo); + registerAppBulkForNextCb(p, &embBulk2.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(&embBulk1); restoreEmbeddedBulk(&embBulk2);, /*finalize*/ + restoreEmbeddedBulk(p, &embBulk1); restoreEmbeddedBulk(p, &embBulk2);, /*finalize*/ RDB_LEVEL_DATA, rdbData.handleHashField, - embBulk1.binfo->ref, - embBulk2.binfo->ref); + embBulk1.binfo.ref, + embBulk2.binfo.ref); } return RDB_STATUS_OK; } @@ -1133,12 +1160,12 @@ static RdbStatus zsetZiplistItem(RdbParser *p, BulkInfo *ziplistBulk) { if (!allocEmbeddedBulk(p, item1, item1Len, item1Val, &embBulk)) return RDB_STATUS_ERROR; - registerAppBulkForNextCb(p, embBulk.binfo); + registerAppBulkForNextCb(p, &embBulk.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(&embBulk);, /*finalize*/ + restoreEmbeddedBulk(p, &embBulk);, /*finalize*/ RDB_LEVEL_DATA, rdbData.handleZsetMember, - embBulk.binfo->ref, + embBulk.binfo.ref, score); } return RDB_STATUS_OK; @@ -1199,6 +1226,7 @@ RdbStatus elementSelectDb(RdbParser *p) { IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dbid, NULL, NULL)); /*** ENTER SAFE STATE ***/ + p->selectedDb = (int) dbid; CALL_COMMON_HANDLERS_CB(p, handleNewDb, ((int) dbid)); return nextParsingElement(p, PE_NEXT_RDB_TYPE); @@ -1324,7 +1352,7 @@ RdbStatus elementNextRdbType(RdbParser *p) { case RDB_OPCODE_EOF: return nextParsingElement(p, PE_END_OF_FILE); - /* zset (TBD) */ + /* zset */ case RDB_TYPE_ZSET: return nextParsingElementKeyValue(p, PE_RAW_ZSET, PE_ZSET); case RDB_TYPE_ZSET_2: return nextParsingElementKeyValue(p, PE_RAW_ZSET, PE_ZSET); case RDB_TYPE_ZSET_ZIPLIST: return nextParsingElementKeyValue(p, PE_RAW_ZSET_ZL, PE_ZSET_ZL); @@ -1575,8 +1603,7 @@ RdbStatus elementHashZL(RdbParser *p) { /*** ENTER SAFE STATE ***/ - if (RDB_STATUS_ERROR == hashZiplist(p, ziplistBulk)) - return RDB_STATUS_ERROR; + IF_NOT_OK_RETURN(hashZiplist(p, ziplistBulk)); return nextParsingElement(p, PE_END_KEY); } @@ -1588,8 +1615,7 @@ RdbStatus elementHashLP(RdbParser *p) { /*** ENTER SAFE STATE ***/ - if (RDB_STATUS_ERROR == hashListPack(p, listpackBulk)) - return RDB_STATUS_ERROR; + IF_NOT_OK_RETURN(hashListPack(p, listpackBulk)); return nextParsingElement(p, PE_END_KEY); } @@ -1601,8 +1627,7 @@ RdbStatus elementHashZM(RdbParser *p) { /*** ENTER SAFE STATE ***/ - if (RDB_STATUS_ERROR == hashZipMap(p, zipmapBulk)) - return RDB_STATUS_ERROR; + IF_NOT_OK_RETURN(hashZipMap(p, zipmapBulk)); return nextParsingElement(p, PE_END_KEY); } @@ -1709,12 +1734,12 @@ RdbStatus elementSetLP(RdbParser *p) { if (!allocEmbeddedBulk(p, item, itemLen, itemVal, &embBulk)) return RDB_STATUS_ERROR; - registerAppBulkForNextCb(p, embBulk.binfo); + registerAppBulkForNextCb(p, &embBulk.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(&embBulk);, /*finalize*/ + restoreEmbeddedBulk(p, &embBulk), /*finalize*/ RDB_LEVEL_DATA, rdbData.handleSetMember, - embBulk.binfo->ref); + embBulk.binfo.ref); iterator = lpNext(listpackBulk->ref, iterator); } @@ -1817,12 +1842,12 @@ RdbStatus elementZsetLP(RdbParser *p) { if (!allocEmbeddedBulk(p, item1, item1Len, item1Val, &embBulk)) return RDB_STATUS_ERROR; - registerAppBulkForNextCb(p, embBulk.binfo); + registerAppBulkForNextCb(p, &embBulk.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(&embBulk);, /*finalize*/ + restoreEmbeddedBulk(p, &embBulk);, /*finalize*/ RDB_LEVEL_DATA, rdbData.handleZsetMember, - embBulk.binfo->ref, + embBulk.binfo.ref, score); iterator = lpNext(listpackBulk->ref, iterator); @@ -2068,7 +2093,7 @@ RdbStatus rdbLoadInteger(RdbParser *p, int enctype, AllocTypeRq type, char *refB if (enctype == RDB_ENC_INT8) { IF_NOT_OK_RETURN(rdbLoad(p, 1, RQ_ALLOC, NULL, binfo)); - val = ((unsigned char *) (*binfo)->ref)[0]; + val = ((signed char *) (*binfo)->ref)[0]; } else if (enctype == RDB_ENC_INT16) { uint16_t v; IF_NOT_OK_RETURN(rdbLoad(p, 2, RQ_ALLOC, NULL, binfo)); diff --git a/src/lib/parser.h b/src/lib/parser.h index b4b9617..511f5bd 100644 --- a/src/lib/parser.h +++ b/src/lib/parser.h @@ -339,6 +339,7 @@ struct RdbParser { ElementCtx elmCtx; /* parsing-element context */ AppCallbackCtx appCbCtx; /* Trace bulks that will be given to next app cb. Cleared after each cb */ RawContext rawCtx; + int selectedDb; /*** caching ***/ BulkPool *cache; /* Cleared after each parsing-element state change */ @@ -372,7 +373,10 @@ struct RdbParser { /*** misc ***/ int rdbversion; /* keep aside RDB version */ uint64_t checksum; - int debugData; /* if envvar LIBRDB_DEBUG_DATA=1 then print state machine transitions */ + + /*** debug (if envvar LIBRDB_DEBUG_DATA=1) ***/ + int debugData; /* print state machine transitions */ + char currKeyDbg[128]; /* Copy of current visited key by the parser for printing */ }; /* reader base struct */ diff --git a/src/lib/parserRaw.c b/src/lib/parserRaw.c index f980048..73f3280 100644 --- a/src/lib/parserRaw.c +++ b/src/lib/parserRaw.c @@ -3,7 +3,7 @@ * Whereas parsing-depth of file parser.c is LEVEL1 (RDB data-structures) and LEVEL2 * (Redis data-types), this file parses LEVEL0 of raw data. The incentive to this * separation is that the similarity between LEVEL1 and LEVEL2 is higher which - * expects structured data, whereas LEVEL0 is a dump of data that its main purpose + * expects structured data, whereas LEVEL0 is a dump of raw data that its main purpose * is to use it along with RESTORE command to play it against live Redis server. */ diff --git a/test/dumps/hash_lp_v11.rdb b/test/dumps/hash_lp_v11.rdb index 550b190..f1c717d 100644 Binary files a/test/dumps/hash_lp_v11.rdb and b/test/dumps/hash_lp_v11.rdb differ diff --git a/test/dumps/hash_lp_v11_data.json b/test/dumps/hash_lp_v11_data.json index 672b30e..f54235f 100644 --- a/test/dumps/hash_lp_v11_data.json +++ b/test/dumps/hash_lp_v11_data.json @@ -1,6 +1,7 @@ "redis-ver":"255.255.255", "redis-bits":"64", -"ctime":"1690555090", -"used-mem":"1003424", +"ctime":"1695280472", +"used-mem":"1062024", "aof-base":"0", -"myhash":{"abc1":"abc1","3":"3","1.1":"1.1","1":"1"} +"hash2":{"field7":"value7","field8":"value8","9":"value9","field10":"value10","field11":"11","12":"12.0"}, +"hash1":{"field2":"2","field3":"3","field4":"value4.0","field5":"5.0","6":"6"} diff --git a/test/dumps/hash_lp_v11_raw.json b/test/dumps/hash_lp_v11_raw.json index e042be3..1915638 100644 --- a/test/dumps/hash_lp_v11_raw.json +++ b/test/dumps/hash_lp_v11_raw.json @@ -1,6 +1,7 @@ "redis-ver":"255.255.255", "redis-bits":"64", -"ctime":"1690555090", -"used-mem":"1003424", +"ctime":"1695280472", +"used-mem":"1062024", "aof-base":"0", -"myhash":"\x10%%\x00\x00\x00\b\x00\x84abc1\x05\x84abc1\x05\x03\x01\x04\x01\x831.1\x04\x831.1\x04\x01\x01\x02\x01\xff" +"hash2":"\x10\xc3@C@V\x13V\x00\x00\x00\f\x00\x86field7\x07\x86value \x07`\x0f\x008\xa0\x0f\x038\x07\t\x01\x80\x19\x029\x07\x87`\x19\x0310\b\x87`*@\b\x80\x11\f1\b\x0b\x01\f\x01\x8412.0\x05\xff", +"hash1":"\x10\xc35>\x0f>\x00\x00\x00\n\x00\x86field2\x07\x02\x01\x80\t\x023\x07\x03\xa0\t\x0b4\x07\x88value4.0\t\x80\x1b\x0b5\x07\x835.0\x04\x06\x01\x06\x01\xff" diff --git a/test/dumps/hash_lp_v11_struct.json b/test/dumps/hash_lp_v11_struct.json index b81b5be..473155b 100644 --- a/test/dumps/hash_lp_v11_struct.json +++ b/test/dumps/hash_lp_v11_struct.json @@ -1,6 +1,7 @@ "redis-ver":"255.255.255", "redis-bits":"64", -"ctime":"1690555090", -"used-mem":"1003424", +"ctime":"1695280472", +"used-mem":"1062024", "aof-base":"0", -"myhash":["%\x00\x00\x00\b\x00\x84abc1\x05\x84abc1\x05\x03\x01\x04\x01\x831.1\x04\x831.1\x04\x01\x01\x02\x01\xff"] +"hash2":["V\x00\x00\x00\f\x00\x86field7\x07\x86value7\x07\x86field8\x07\x86value8\x07\t\x01\x86value9\x07\x87field10\b\x87value10\b\x87field11\b\x0b\x01\f\x01\x8412.0\x05\xff"], +"hash1":[">\x00\x00\x00\n\x00\x86field2\x07\x02\x01\x86field3\x07\x03\x01\x86field4\x07\x88value4.0\t\x86field5\x07\x835.0\x04\x06\x01\x06\x01\xff"] diff --git a/test/dumps/hash_zl_v6_data.json b/test/dumps/hash_zl_v6_data.json index a6b0afe..83cd75a 100644 --- a/test/dumps/hash_zl_v6_data.json +++ b/test/dumps/hash_zl_v6_data.json @@ -3,4 +3,4 @@ "ctime":"1690533464", "used-mem":"865216", "aof-preamble":"0", -"myhash":{"1":"2","3":"3","5":"5","7.0":"7.0","str1":"str1","str3":"str3"} +"myhash":{"1":"2","3":"4","5":"6","7.0":"8.0","str1":"str2","str3":"str4"} diff --git a/test/dumps/string_int_encoded.json b/test/dumps/string_int_encoded.json new file mode 100644 index 0000000..89ecaa6 --- /dev/null +++ b/test/dumps/string_int_encoded.json @@ -0,0 +1,57 @@ +"str::-4294967295":"-4294967295", +"str::255":"255", +"str::-16777217":"-16777217", +"str::16777218":"16777218", +"str::-4294967297":"-4294967297", +"str::257":"257", +"str::2147483649":"2147483649", +"str::-16777215":"-16777215", +"str::3":"3", +"str::-281474976710656":"-281474976710656", +"str::-16777214":"-16777214", +"str::-65534":"-65534", +"str::256":"256", +"str::65535":"65535", +"str::-258":"-258", +"str::-256":"-256", +"str::254":"254", +"str::-4294967294":"-4294967294", +"str::16777217":"16777217", +"str::-281474976710655":"-281474976710655", +"str::16777216":"16777216", +"str::-255":"-255", +"str::-4294967298":"-4294967298", +"str::-1":"-1", +"str::281474976710655":"281474976710655", +"str::1":"1", +"str::0":"0", +"str::-281474976710654":"-281474976710654", +"str::-65538":"-65538", +"str::-254":"-254", +"str::-16777216":"-16777216", +"str::281474976710654":"281474976710654", +"str::281474976710657":"281474976710657", +"str::-281474976710657":"-281474976710657", +"str::258":"258", +"str::9223372036854780000":"9223372036854780000", +"str::16777214":"16777214", +"str::-257":"-257", +"str::2":"2", +"str::2147483650":"2147483650", +"str::16777215":"16777215", +"str::-65537":"-65537", +"str::65537":"65537", +"str::-65536":"-65536", +"str::-16777218":"-16777218", +"str::65534":"65534", +"str::-4294967296":"-4294967296", +"str::65538":"65538", +"str::2147483646":"2147483646", +"str::281474976710658":"281474976710658", +"str::2147483648":"2147483648", +"str::281474976710656":"281474976710656", +"str::-9223372036854780000":"-9223372036854780000", +"str::-65535":"-65535", +"str::65536":"65536", +"str::2147483647":"2147483647", +"str::-281474976710658":"-281474976710658" diff --git a/test/dumps/string_int_encoded.rdb b/test/dumps/string_int_encoded.rdb new file mode 100644 index 0000000..a2e2d01 Binary files /dev/null and b/test/dumps/string_int_encoded.rdb differ diff --git a/test/test_common.c b/test/test_common.c index dfb8614..bf0465e 100644 --- a/test/test_common.c +++ b/test/test_common.c @@ -2,6 +2,7 @@ #define _POSIX_C_SOURCE 200809L #include +#include #include #include #include @@ -13,12 +14,14 @@ #include #include "../deps/hiredis/hiredis.h" #include "test_common.h" -#include "../src/ext/utils.c" /* for printHexDump() */ /* Live Redis server for some of the tests (Optional) */ -redisContext *redisConnContext = NULL; -int redisPort=0; -pid_t redisPID = 0; +#define MAX_NUM_REDIS_INST 2 +int currRedisInst = -1; +redisContext *redisServersStack[MAX_NUM_REDIS_INST] = {0}; +int redisPort[MAX_NUM_REDIS_INST]= {0}; +pid_t redisPID[MAX_NUM_REDIS_INST] = {0}; +const char *redisInstallFolder = NULL; void runSystemCmd(const char *cmdFormat, ...) { char cmd[1024]; @@ -130,7 +133,7 @@ void assert_file_payload(const char *filename, char *expData, int expLen, MatchT } if (((result != 0) && (expMatch)) || ((result == 0) && (!expMatch))) { - char buf[10000]; + char buf[8192]; printf("%s\n---- file [%s] ----\n", errMsg, filename); printHexDump(filedata, filelen, buf, (int) sizeof(buf)); printf("%s", buf); @@ -180,8 +183,11 @@ int findFreePort(int startPort, int endPort) { } void cleanupRedisServer(void) { - if (redisPID) - kill(redisPID, SIGTERM); + for (int i=0 ; i <=currRedisInst ; ++i ) { + if (redisPID[i]) + kill(redisPID[i], SIGTERM); + } + } size_t serializeRedisReply(const redisReply *reply, char *buffer, size_t bsize) { @@ -216,11 +222,11 @@ size_t serializeRedisReply(const redisReply *reply, char *buffer, size_t bsize) * Return the response serialized */ char *sendRedisCmd(char *cmd, int expRetType, char *expRsp) { - static char rspbuf[1000]; + static char rspbuf[1024]; - assert_non_null(redisConnContext); + assert_int_not_equal(currRedisInst, -1); - redisReply *reply = redisCommand(redisConnContext, cmd); + redisReply *reply = redisCommand(redisServersStack[currRedisInst], cmd); //printf ("Command:%s\n", cmd); @@ -241,20 +247,29 @@ char *sendRedisCmd(char *cmd, int expRetType, char *expRsp) { return rspbuf; } -void setupRedisServer(const char *installFolder) { +void setRedisInstallFolder(const char *path) { + redisInstallFolder = path; +} + +void setupRedisServer(const char *extraArgs) { + + /* If redis not installed return gracefully */ + if (!redisInstallFolder) return; + + /* execl() not accept empty string */ + const char *_extraArgs = (extraArgs) ? extraArgs : "--loglevel verbose"; + pid_t pid = fork(); assert_int_not_equal (pid, -1); - redisPort = findFreePort(6500, 6600); + int port = findFreePort(6500, 6600); if (pid == 0) { /* child */ char redisPortStr[10], fullpath[256], testrdbModulePath[256]; - printf("Found free port to run Redis: %d\n", redisPort); - - snprintf(fullpath, sizeof(fullpath), "%s/redis-server", installFolder); - snprintf(testrdbModulePath, sizeof(testrdbModulePath), "%s/../tests/modules/testrdb.so", installFolder); - snprintf(redisPortStr, sizeof(redisPortStr), "%d", redisPort); + snprintf(fullpath, sizeof(fullpath), "%s/redis-server", redisInstallFolder); + snprintf(testrdbModulePath, sizeof(testrdbModulePath), "%s/../tests/modules/testrdb.so", redisInstallFolder); + snprintf(redisPortStr, sizeof(redisPortStr), "%d", port); /* if module testrdb.so exists (ci.yaml takes care to build testrdb), part * of redis repo testing, then load it for test_rdb_to_redis_module. The @@ -266,12 +281,14 @@ void setupRedisServer(const char *installFolder) { "--dir", "./test/tmp/", "--logfile", "./redis.log", "--loadmodule", testrdbModulePath, "4", + _extraArgs, (char *) NULL); } else { execl(fullpath, fullpath, "--port", redisPortStr, "--dir", "./test/tmp/", "--logfile", "./redis.log", + _extraArgs, (char *) NULL); } @@ -281,7 +298,7 @@ void setupRedisServer(const char *installFolder) { } else { /* parent */ int retryCount = 3; - redisConnContext = redisConnect("localhost", redisPort); + redisContext *redisConnContext = redisConnect("localhost", port); while ((!redisConnContext) || (redisConnContext->err)) { if (redisConnContext) redisFree(redisConnContext); @@ -295,10 +312,14 @@ void setupRedisServer(const char *installFolder) { struct timespec req = {0, 50000*1000}, rem; nanosleep(&req, &rem); - redisConnContext = redisConnect("localhost", redisPort); + redisConnContext = redisConnect("localhost", port); } - redisPID = pid; + assert_true(++currRedisInst> Redis Server(%d) started on port %d with PID %d\n", currRedisInst, port, pid); /* Close any subprocess in case of exit due to error flow */ atexit(cleanupRedisServer); @@ -306,23 +327,28 @@ void setupRedisServer(const char *installFolder) { } void teardownRedisServer(void) { - if (redisConnContext) { - assert_non_null(redisConnContext); - assert_null(redisCommand(redisConnContext, "SHUTDOWN")); - redisFree(redisConnContext); - redisConnContext = NULL; + if (currRedisInst>=0) { + redisContext *ctx = redisServersStack[currRedisInst]; + assert_non_null(ctx); + assert_null(redisCommand(ctx, "SHUTDOWN")); + redisFree(ctx); + --currRedisInst; wait(NULL); } } int isSetRedisServer(void) { - return (redisConnContext != NULL); + return (currRedisInst>=0); } +int getRedisPort(void) { + assert_true(currRedisInst>=0); + return redisPort[currRedisInst]; +} /* Redis OSS does not support restoring module auxiliary data. This feature * is currently available only in Redis Enterprise. There are plans to bring * this functionality to Redis OSS in the near future. */ -int isSupportRestoreModuleAux(void) { +int isSupportRestoreModuleAux() { static int supported = -1; /* -1=UNINIT, 0=NO, 1=YES */ if (supported == -1) { char *res = sendRedisCmd("RESTOREMODAUX", REDIS_REPLY_ERROR, NULL); @@ -413,11 +439,12 @@ static unsigned char xorstr(const char *str) { /* sanitize, sort, and compare */ #define MAX_LINE_LENGTH 4096 +#define MAX_LINES 4096 void assert_json_equal(const char* filename1, const char* filename2, int ignoreListOrder) { char line1[MAX_LINE_LENGTH]; char line2[MAX_LINE_LENGTH]; - char* lines1[MAX_LINE_LENGTH]; - char* lines2[MAX_LINE_LENGTH]; + char* lines1[MAX_LINES]; + char* lines2[MAX_LINES]; int lineCount1 = 0; int lineCount2 = 0; int res = -1; @@ -482,3 +509,59 @@ void assert_json_equal(const char* filename1, const char* filename2, int ignoreL assert_true(0); } + + +/* printHexDump() Generates a formatted hexadecimal and ASCII representation of binary + * data. Given a memory address and its length, it produces a human-readable obuf, + * displaying byte offsets in hexadecimal and replacing non-printable characters with + * dots ('.'). + * + * Returns how many bytes written to obuf buffer. -1 Otherwise. + * + * Output example for input: "A123456789B123456789C123456789D123456789" + * 000000 41 31 32 33 34 35 36 37 38 39 42 31 32 33 34 35 A1234567 89B12345 + * 000010 36 37 38 39 43 31 32 33 34 35 36 37 38 39 44 31 6789C123 456789D1 + * 000020 32 33 34 35 36 37 38 39 23456789 + */ +int printHexDump(const char *input, size_t len, char *obuf, int obuflen) { + size_t i; + int iout=0, j, llen = 16; /* line len */ + unsigned char buff[llen + 10]; + + if (input == NULL || len <= 0 || obuf == NULL || obuflen < 200 || obuflen > 0xFFFFFF) + return -1; + + for (i = 0, j = 0; (i < len) && (iout + 100 < obuflen) ; i++) { + if ((i % llen) == 0) { + if (i > 0) { + buff[j] = '\0'; + iout += snprintf(obuf + iout, obuflen - iout, " %s\n", buff); + } + iout += snprintf(obuf + iout, obuflen - iout, "%06zx ", i); + j = 0; + } + + if (((int)i % llen) == (llen / 2)) { /* middle of the line */ + iout += snprintf(obuf + iout, obuflen - iout, " "); + buff[j++] = ' '; + buff[j++] = ' '; + } + + iout += snprintf(obuf + iout, obuflen - iout, " %02x", (unsigned char)input[i]); + buff[j++] = (isprint(input[i])) ? input[i] : '.'; + } + + /* pad the last line */ + for (; (i % llen) != 0; i++) { + iout += snprintf(obuf + iout, obuflen - iout, " "); + if (( (int)i % llen) == (llen / 2)) { + iout += snprintf(obuf + iout, obuflen - iout, " "); + } + } + + buff[j] = '\0'; + iout += snprintf(obuf + iout, obuflen - iout, " %s\n", buff); + if (i < len) + iout += snprintf(obuf + iout, obuflen - iout, "..."); + return iout; +} diff --git a/test/test_common.h b/test/test_common.h index 5d2f8bf..d8c7ab6 100644 --- a/test/test_common.h +++ b/test/test_common.h @@ -31,8 +31,9 @@ void runSystemCmd(const char *cmdFormat, ...); void assert_json_equal(const char *f1, const char *f2, int ignoreListOrder); /* Test against Redis Server */ -extern int redisPort; -void setupRedisServer(const char *installFolder); +void setRedisInstallFolder(const char *path); +int getRedisPort(void); +void setupRedisServer(const char *extraArgs); void teardownRedisServer(void); int isSetRedisServer(void); char *sendRedisCmd(char *cmd, int expRetType, char *expRsp); @@ -42,6 +43,7 @@ int isSupportRestoreModuleAux(void); int group_rdb_to_redis(void); int group_test_rdb_cli(void); int group_rdb_to_resp(void); +int group_examples(void); int group_main(void); int group_rdb_to_json(void); int group_mem_management(void); @@ -60,3 +62,5 @@ void cleanTmpFolder(void); void setEnvVar (const char *name, const char *val); char *substring(char *str, size_t len, char *substr); void assert_file_payload(const char *filename, char *expData, int expLen, MatchType matchType, int expMatch); + +int printHexDump(const char *addr, size_t len, char *obuf, int obuflen); diff --git a/test/test_main.c b/test/test_main.c index 5d16539..ede23be 100644 --- a/test/test_main.c +++ b/test/test_main.c @@ -51,33 +51,6 @@ static void test_empty_rdb(void **state) { RDB_deleteParser(parser); } -static void test_createHandlersRdbToJson_and_2_FilterKey(void **state) { - UNUSED(state); - - const char *rdbfile = DUMP_FOLDER("multiple_lists_strings.rdb"); - const char *jsonfile = TMP_FOLDER("multiple_lists_strings.json"); - const char *expJsonFile = DUMP_FOLDER("multiple_lists_strings_2filters.json"); - - RdbStatus status; - RdbParser *parser = RDB_createParserRdb(NULL); - RDB_setLogLevel(parser, RDB_LOG_ERR); - assert_non_null(RDBX_createReaderFile(parser, rdbfile)); - RdbxToJsonConf r2jConf = {RDB_LEVEL_DATA, RDBX_CONV_JSON_ENC_PLAIN, 0, 0, 1}; - assert_non_null(RDBX_createHandlersToJson(parser, - jsonfile, - &r2jConf)); - - assert_non_null(RDBX_createHandlersFilterKey(parser, ".*i.*", 0)); - assert_non_null(RDBX_createHandlersFilterKey(parser, "mylist.*", 0)); - - - while ((status = RDB_parse(parser)) == RDB_STATUS_WAIT_MORE_DATA); - assert_int_equal( status, RDB_STATUS_OK); - - RDB_deleteParser(parser); - assert_json_equal(jsonfile, expJsonFile, 0); -} - static void test_mixed_levels_registration(void **state) { UNUSED(state); const char *rdbfile = DUMP_FOLDER("multiple_lists_strings.rdb"); @@ -107,6 +80,11 @@ static void test_mixed_levels_registration(void **state) { assert_json_equal(jsonfileData, DUMP_FOLDER("multiple_lists_strings_subset_list.json"), 1); } +static void test_examples(void **state) { + UNUSED(state); + runSystemCmd("make example > /dev/null "); +} + static void printResPicture(int result) { if (result) printf(" x_x\n" @@ -128,13 +106,26 @@ static void printResPicture(int result) { result |= grp(); \ } +/*************************** group_examples *************************** + * Test the examples in the './examples' directory. These examples + * do not necessarily support asynchronous events 'WAIT_MORE_DATA.' + * Therefore, 'group_examples()' should not be called when the environment + * variable 'LIBRDB_SIM_WAIT_MORE_DATA' is set to '1'. + *********************************************************************/ +int group_examples(void) { + /* Insert here your test functions */ + const struct CMUnitTest tests[] = { + cmocka_unit_test(test_examples), + }; + return cmocka_run_group_tests(tests, NULL, NULL); +} + /*************************** group_main *******************************/ int group_main(void) { /* Insert here your test functions */ const struct CMUnitTest tests[] = { cmocka_unit_test(test_createReader_missingFile), cmocka_unit_test(test_empty_rdb), - cmocka_unit_test(test_createHandlersRdbToJson_and_2_FilterKey), cmocka_unit_test(test_mixed_levels_registration), }; return cmocka_run_group_tests(tests, NULL, NULL); @@ -148,7 +139,7 @@ int main(int argc, char *argv[]) { char *redisInstallFolder = getenv("LIBRDB_REDIS_FOLDER"); - const char *USAGE =" [-h|--help] [f|--redis-folder ] [-g|--run-group ] [-t|--test-filter ]"; + const char *USAGE =" [-h|--help] [f|--redis-folder ] [-g|--test-group ] [-t|--test ]"; /* Parse command-line arguments */ for (int i = 1; i < argc; i++) { if ((strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0)) { @@ -156,9 +147,9 @@ int main(int argc, char *argv[]) { exit(EXIT_SUCCESS); } else if ((strcmp(argv[i], "-f") == 0 || strcmp(argv[i], "--redis-folder") == 0) && i+1 < argc) { redisInstallFolder = argv[++i]; - } else if ((strcmp(argv[i], "-g") == 0 || strcmp(argv[i], "--run-group") == 0) && i+1 < argc) { + } else if ((strcmp(argv[i], "-g") == 0 || strcmp(argv[i], "--test-group") == 0) && i+1 < argc) { runGroupPrefix = argv[++i]; - } else if ((strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--test-filter") == 0) && i+1 < argc) { + } else if ((strcmp(argv[i], "-t") == 0 || strcmp(argv[i], "--test") == 0) && i+1 < argc) { testFilter = argv[++i]; } else { printf("Invalid argument: %s\n%s\n", argv[i], USAGE); @@ -173,13 +164,14 @@ int main(int argc, char *argv[]) { cleanTmpFolder(); /* Setup redis if configured */ - if (redisInstallFolder) - setupRedisServer(redisInstallFolder); + setRedisInstallFolder(redisInstallFolder); + setupRedisServer(NULL); //setenv("LIBRDB_DEBUG_DATA", "1", 1); /* << to see parser states printouts */ printf("\n*************** START TESTING *******************\n"); setEnvVar("LIBRDB_SIM_WAIT_MORE_DATA", "0"); + RUN_TEST_GROUP(group_examples); RUN_TEST_GROUP(group_test_resp_reader); RUN_TEST_GROUP(group_rdb_to_resp); RUN_TEST_GROUP(group_main); diff --git a/test/test_rdb_cli.c b/test/test_rdb_cli.c index e8ce754..0a5b205 100644 --- a/test/test_rdb_cli.c +++ b/test/test_rdb_cli.c @@ -39,7 +39,7 @@ static void test_rdb_cli_resp_common(const char *rdbfile) { RDB_deleteParser(parser); /* rdb-cli RDB to RESP and stream toward Redis Server */ - runSystemCmd("./bin/rdb-cli %s redis -h %s -p %d", rdbfile, "127.0.0.1", redisPort); + runSystemCmd("./bin/rdb-cli %s redis -h %s -p %d", rdbfile, "127.0.0.1", getRedisPort()); /* DUMP-RDB from Redis */ sendRedisCmd("SAVE", REDIS_REPLY_STATUS, NULL); @@ -70,6 +70,86 @@ static void test_rdb_cli_resp_to_redis(void **state) { test_rdb_cli_resp_common(DUMP_FOLDER("multiple_lists_strings.rdb")); } +static void test_rdb_cli_filter_db(void **state) { + UNUSED(state); + /* -d/--dbnum 0 (found x but not y or z) */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_dbs.rdb -d 0 json -f | grep x > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_dbs.rdb --dbnum 0 json -f | grep y && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_dbs.rdb --dbnum 0 json -f | grep z && exit 1 || exit 0 > /dev/null "); + /* -D/--no-dbnum 0 (found y and z but not x) */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_dbs.rdb -D 0 json -f | grep x && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_dbs.rdb --no-dbnum 0 json -f | grep y > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_dbs.rdb --no-dbnum 0 json -f | grep z > /dev/null "); +} + +static void test_rdb_cli_filter_key(void **state) { + UNUSED(state); + /* -k/--key (found string2 but not mylist1 or lzf_compressed) */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -k string2 json -f | grep string2 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -k string2 json -f | grep mylist1 && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -k string2 json -f | grep lzf_compressed && exit 1 || exit 0 > /dev/null "); + /* -K/--no-key (found mylist1 or lzf_compressed but not string2) */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -K string2 json -f | grep string2 && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -K string2 json -f | grep mylist1 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -K string2 json -f | grep lzf_compressed > /dev/null "); +} + +static void test_rdb_cli_filter_invalid_input(void **state) { + UNUSED(state); + /* invalid regex */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/single_key.rdb -k \"[*x\" json | grep \"Unmatched \\[\" > /dev/null"); +} + +static void test_rdb_cli_filter_type(void **state) { + UNUSED(state); + /* -t/--type */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb --type str json -f | grep string2 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb --type str json -f | grep lzf_compressed > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb --type str json -f | grep string1 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -t str json -f | grep mylist1 && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -t str json -f | grep mylist2 && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -t str json -f | grep mylist3 && exit 1 || exit 0 > /dev/null "); + /* -T/--no-type */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb --no-type str json -f | grep mylist1 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb --no-type str json -f | grep mylist2 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb --no-type str json -f | grep mylist3 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -T str json -f | grep string2 && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -T str json -f | grep lzf_compressed && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -T str json -f | grep string1 && exit 1 || exit 0 > /dev/null "); +} + +static void test_rdb_cli_filter_mix(void **state) { + UNUSED(state); + /* Combine 'type' and 'key' filters */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb --type str --key string json -f | grep string2 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb --type str --key string json -f | grep string1 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -t str -k string json -f | grep lzf_compressed && exit 1 || exit 0 > /dev/null"); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -t str -k string json -f | grep list1 && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -t str -k string json -f | grep list2 && exit 1 || exit 0 > /dev/null "); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/multiple_lists_strings.rdb -t str -k string json -f | grep list3 && exit 1 || exit 0 > /dev/null "); +} + +static void test_rdb_cli_redis_auth(void **state) { + UNUSED(state); + /* check password authentication */ + setupRedisServer("--requirepass abc"); + + /* auth custom command */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/single_key.rdb redis -a 2 auth abc -p %d > /dev/null ", getRedisPort()); + + /* auth pwd */ + sendRedisCmd("FLUSHALL", REDIS_REPLY_ERROR, NULL); /* expected to fail */ + sendRedisCmd("AUTH abc", REDIS_REPLY_STATUS, NULL); /* now expected to succeed */ + runSystemCmd(" ./bin/rdb-cli ./test/dumps/single_key.rdb redis --password abc -p %d > /dev/null ", getRedisPort()); + + /* auth user */ + sendRedisCmd("ACL SETUSER newuser on >newpwd +@all ~*", REDIS_REPLY_STATUS, NULL); + sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL); + runSystemCmd(" ./bin/rdb-cli ./test/dumps/single_key.rdb redis -P newpwd -u newuser -p %d > /dev/null ", getRedisPort()); + + teardownRedisServer(); +} + /*************************** group_test_rdb_cli *******************************/ int group_test_rdb_cli(void) { @@ -81,6 +161,12 @@ int group_test_rdb_cli(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_rdb_cli_json), cmocka_unit_test_setup(test_rdb_cli_resp_to_redis, setupTest), + cmocka_unit_test_setup(test_rdb_cli_filter_db, setupTest), + cmocka_unit_test_setup(test_rdb_cli_filter_key, setupTest), + cmocka_unit_test_setup(test_rdb_cli_filter_invalid_input, setupTest), + cmocka_unit_test_setup(test_rdb_cli_filter_type, setupTest), + cmocka_unit_test_setup(test_rdb_cli_filter_mix, setupTest), + cmocka_unit_test_setup(test_rdb_cli_redis_auth, setupTest), }; int res = cmocka_run_group_tests(tests, NULL, NULL); diff --git a/test/test_rdb_to_json.c b/test/test_rdb_to_json.c index 9e82f1a..1a13490 100644 --- a/test/test_rdb_to_json.c +++ b/test/test_rdb_to_json.c @@ -469,6 +469,12 @@ static void test_r2j_module_aux_data(void **state) { testRdbToJsonCommon(DUMP_FOLDER("module_aux.rdb"), DUMP_FOLDER("module_aux_data.json"), &r2jConf); } +static void test_r2j_string_int_encoded(void **state) { + UNUSED(state); + RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA); + r2jConf.includeAuxField = 0; + testRdbToJsonCommon(DUMP_FOLDER("string_int_encoded.rdb"), DUMP_FOLDER("string_int_encoded.json"), &r2jConf); +} /*************************** group_rdb_to_json *******************************/ int group_rdb_to_json(void) { const struct CMUnitTest tests[] = { @@ -548,6 +554,7 @@ int group_rdb_to_json(void) { cmocka_unit_test(test_r2j_multiple_lists_and_strings_struct), cmocka_unit_test(test_r2j_multiple_lists_and_strings_raw), cmocka_unit_test(test_r2j_multiple_dbs), + cmocka_unit_test(test_r2j_string_int_encoded), }; return cmocka_run_group_tests(tests, NULL, NULL); } diff --git a/test/test_rdb_to_redis.c b/test/test_rdb_to_redis.c index 26618df..04575bf 100644 --- a/test/test_rdb_to_redis.c +++ b/test/test_rdb_to_redis.c @@ -30,7 +30,7 @@ void rdb_to_tcp(const char *rdbfile, int pipelineDepth, int isRestore, char *res RDB_setLogLevel(parser, RDB_LOG_ERR); assert_non_null(RDBX_createReaderFile(parser, rdbfile)); assert_non_null(rdbToResp1 = RDBX_createHandlersToResp(parser, &rdb2respConf)); - assert_non_null(r2r = RDBX_createRespToRedisTcp(parser, rdbToResp1, "127.0.0.1", redisPort)); + assert_non_null(r2r = RDBX_createRespToRedisTcp(parser, rdbToResp1, NULL, "127.0.0.1", getRedisPort())); if (respFileName) { assert_non_null(rdbToResp2 = RDBX_createHandlersToResp(parser, &rdb2respConf)); assert_non_null(RDBX_createRespToFileWriter(parser, rdbToResp2, respFileName)); @@ -295,8 +295,9 @@ static void test_rdb_to_redis_del_before_write(void **state) { assert_non_null(RDBX_createRespToRedisTcp(parser, rdbToResp, + NULL, "127.0.0.1", - redisPort)); + getRedisPort())); while ((status = RDB_parse(parser)) == RDB_STATUS_WAIT_MORE_DATA); diff --git a/test/test_rdb_to_resp.c b/test/test_rdb_to_resp.c index fc419ac..e92cf3c 100644 --- a/test/test_rdb_to_resp.c +++ b/test/test_rdb_to_resp.c @@ -31,8 +31,8 @@ static void testRdbToRespCommon(const char *rdbfilename, int expMatch) { static int outputs = 0; - static char rdbfile[100]; - static char respfile[100]; + static char rdbfile[1024]; + static char respfile[1024]; /* build file path of input (rdb) file and output (resp) file */ snprintf(rdbfile, sizeof(rdbfile), "./test/dumps/%s", rdbfilename);