Filters; debug utils; AUTH; fix json hash and signed-int String (#19)

README.md

@@ -46,9 +46,9 @@ Install and run CLI extension of this library. Parse RDB file to json:
     }]
 
 
-Run CLI extension to generate RESP commands
+Run CLI extension to generate RESP commands (this time read file from standard input):
 
-    % rdb-cli ./test/dumps/multiple_lists_strings.rdb resp
+    % gzip -dc multiple_lists_strings.rdb.gz | rdb-cli - resp
     *3
     $3
     SET
@@ -190,8 +190,8 @@ core library vs. extension library ("RDB" vs "RDBX").
 
 - Parsing RDB file with user callbacks:
 
-      RdbRes myHandleNewKey(RdbParser *parser, void *userData,  RdbBulk key,...) { 
-          printf("%s\n", key);
+      RdbRes myHandleNewKey(RdbParser *parser, void *userData,  RdbBulk key, RdbKeyInfo *info) { 
+          printf("KEY=%s\n", key);
           return RDB_OK;
       } 
 
@@ -207,7 +207,7 @@ core library vs. extension library ("RDB" vs "RDBX").
       RdbParser *parser = RDB_createParserRdb(NULL);
       RDBX_createReaderFile(parser, "dump.rdb");
       RDBX_createHandlersToJson(parser, "redis.json", NULL);
-      RDBX_createHandlersFilterKey(parser, "id_*", 0);
+      RDBX_createHandlersFilterKey(parser, "id_*", 0 /*exclude*/);
       RDB_parse(parser);
       RDB_deleteParser(parser);
 
@@ -228,28 +228,39 @@ destruction, or when newer block replacing old one.
 
     Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {json|resp|redis} [FORMAT_OPTIONS]
     OPTIONS:
-    -k, --filter-key <REGEX>      Filter keys using regular expressions
-    -l, --log-file <PATH>         Path to the log file (Default: './rdb-cli.log')
+            -l, --log-file <PATH>         Path to the log file (Default: './rdb-cli.log')
 
-    FORMAT_OPTIONS ('json'):
-    -w, --with-aux-values         Include auxiliary values
-    -f, --flatten                 Print flatten json, without DBs Parenthesis
-    -o, --output <FILE>           Specify the output file. If not specified, output goes to stdout
+    Multiple filters combination of keys/types/dbs can be specified:
+            -k, --key <REGEX>             Include only keys that match REGEX
+            -K  --no-key <REGEX>          Exclude keys that match REGEX
+            -t, --type <TYPE>             Include only selected TYPE {str|list|set|zset|hash|module|func}
+            -T, --no-type <TYPE>          Exclude TYPE {str|list|set|zset|hash|module|func}
+            -d, --dbnum <DBNUM>           Include only selected db number
+            -D, --no-dbnum <DBNUM>        Exclude DB number
 
-    FORMAT_OPTIONS ('resp'):
-    -r, --support-restore         Use the RESTORE command when possible
-    -t, --target-redis-ver <VER>  Specify the target Redis version. Helps determine which commands can
-                                  be applied. Particularly crucial if support-restore being used
-                                  as RESTORE is closely tied to specific RDB versions. If versions not
-                                  aligned the parser will generate higher-level commands instead.
-    -o, --output <FILE>           Specify the output file. If not specified, output goes to stdout
+    FORMAT_OPTIONS ('json'):
+            -i, --include <EXTRAS>        To include: {aux-val|func}
+            -f, --flatten                 Print flatten json, without DBs Parenthesis
+            -o, --output <FILE>           Specify the output file. If not specified, output to stdout
 
     FORMAT_OPTIONS ('redis'):
-    -r, --support-restore         Use the RESTORE command when possible
-    -t, --target-redis-ver <VER>  Specify the target Redis version
-    -h, --hostname <HOSTNAME>     Specify the server hostname (default: 127.0.0.1)
-    -p, --port <PORT>             Specify the server port (default: 6379)
-    -l, --pipeline-depth <VALUE>  Number of pending commands before blocking for responses
+            -h, --hostname <HOSTNAME>     Specify the server hostname (default: 127.0.0.1)
+            -p, --port <PORT>             Specify the server port (default: 6379)
+            -l, --pipeline-depth <VALUE>  Number of pending commands before blocking for responses
+            -u, --user <USER>             Redis username for authentication
+            -P, --password <PWD>          Redis password for authentication
+            -a, --auth N [ARG1 ... ARGN]  An alternative authentication command. Given as vector of arguments
+
+    FORMAT_OPTIONS ('redis'|'resp'):
+            -r, --support-restore         Use the RESTORE command when possible
+            -t, --target-redis-ver <VER>  Specify the target Redis version. Helps determine which commands can
+                                          be applied. Particularly crucial if support-restore being used
+                                          as RESTORE is closely tied to specific RDB versions. If versions not
+                                          aligned the parser will generate higher-level commands instead.
+            -o, --output <FILE>           Specify the output file (For 'resp' only: if not specified, output to stdout)
+            -s, --start-cmd-num <NUM>     Start writing redis from command number
+            -e, --enum-commands           Command enumeration and tracing by preceding each generated RESP command
+                                          with debug command of type: `SET _RDB_CLI_CMD_ID_ <CMD-ID>`
 
 <a name="Advanced"></a>
 ## Advanced
@@ -303,12 +314,12 @@ to give this indication and register it as well.
 
 ### Pause parser and resume
 At times, the application may need to execute additional tasks during parsing intervals,
-such as updating a progress bar or performing other computations. To facilitate this, the
-parser can be configured with a pause interval that specifies the number of bytes to be
-read from RDB source before pausing. This means that each time the parser is invoked, it
-will continue parsing until it has read a number of bytes equal to or greater than the
-configured interval, at which point it will automatically pause and return
-'RDB_STATUS_PAUSED' in order to allow the application to perform other tasks. Example:
+such as updating a progress bar or verifying that used memory remains within limit. To 
+facilitate this, the parser can be configured with a pause interval that specifies the 
+number of bytes to be read from RDB source before pausing. This means that each time the 
+parser is invoked, it will continue parsing until it has read a number of bytes equal to 
+or greater than the configured interval, at which point it will automatically pause and 
+return 'RDB_STATUS_PAUSED' in order to allow the application to perform other tasks. Example:
 
       size_t intervalBytes = 1048576;  
       RdbParser *parser = RDB_createParserRdb(memAlloc);

api/librdb-ext-api.h

@@ -12,7 +12,7 @@ extern "C" {
 typedef struct RdbxRespToFileWriter RdbxRespToFileWriter;
 typedef struct RdbxReaderFile RdbxReaderFile;
 typedef struct RdbxReaderFileDesc RdbxReaderFileDesc;
-typedef struct RdbxFilterKey RdbxFilterKey;
+typedef struct RdbxFilter RdbxFilter;
 typedef struct RdbxToJson RdbxToJson;
 typedef struct RdbxToResp RdbxToResp;
 typedef struct RdbxRespToRedisLoader RdbxRespToRedisLoader;
@@ -31,6 +31,7 @@ typedef enum {
 
     /* HandlersFilterKey errors */
     RDBX_ERR_FILTER_FAILED_COMPILE_REGEX,
+    RDBX_ERR_FAILED_CREATE_FILTER,
 
     /* rdb2resp errors */
 
@@ -79,9 +80,17 @@ _LIBRDB_API RdbxToJson *RDBX_createHandlersToJson(RdbParser *p,
  * Create Filter Handlers
  ****************************************************************/
 
-_LIBRDB_API RdbxFilterKey *RDBX_createHandlersFilterKey(RdbParser *p,
+_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterKey(RdbParser *p,
                                                         const char *keyRegex,
-                                                        uint32_t flags);
+                                                        uint32_t exclude);
+
+_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterType(RdbParser *p,
+                                                      RdbDataType type,
+                                                      uint32_t exclude);
+
+_LIBRDB_API RdbxFilter *RDBX_createHandlersFilterDbNum(RdbParser *p,
+                                                       int dbnum,
+                                                       uint32_t exclude);
 
 /****************************************************************
  * Create RDB to RESP Handlers
@@ -122,11 +131,11 @@ typedef struct RdbxToRespConf {
 _LIBRDB_API RdbxToResp *RDBX_createHandlersToResp(RdbParser *, RdbxToRespConf *);
 
 /****************************************************************
- * Attach RESP writer
+ * RESP writer
  *
  * Create instance for writing RDB to RESP stream.
  *
- * Used by:  RDBX_createRespToRedisTcp
+ * Imp by:   RDBX_createRespToRedisTcp
  *           RDBX_createRespToRedisFd
  *           RDBX_createRespToFileWriter
  *           <user-defined-writer>
@@ -159,17 +168,59 @@ _LIBRDB_API RdbxRespToFileWriter *RDBX_createRespToFileWriter(RdbParser *p,
  * Can configure pipeline depth of transmitted RESP commands. Set
  * to 0 to use default.
  ****************************************************************/
+typedef struct RdbxRedisAuth {
+    const char *pwd;
+    const char *user;
+
+    /* alternative auth-cmd. Args must remain valid throughout the parser's lifetime. */
+    struct {
+        int argc;
+        char **argv;
+    } cmd;
+} RdbxRedisAuth;
+
 _LIBRDB_API RdbxRespToRedisLoader *RDBX_createRespToRedisTcp(RdbParser *p,
-                                                            RdbxToResp *rdbToResp,
-                                                            const char *hostname,
-                                                            int port);
+                                                             RdbxToResp *rdbToResp,
+                                                             RdbxRedisAuth *auth, /*opt*/
+                                                             const char *hostname,
+                                                             int port);
 
 _LIBRDB_API RdbxRespToRedisLoader *RDBX_createRespToRedisFd(RdbParser *p,
-                                                          RdbxToResp *rdbToResp,
-                                                          int fd);
+                                                            RdbxToResp *rdbToResp,
+                                                            RdbxRedisAuth *auth, /*opt*/
+                                                            int fd);
 
 _LIBRDB_API void RDBX_setPipelineDepth(RdbxRespToRedisLoader *r2r, int depth);
 
+/****************************************************************
+ * Debugging RESP to Redis
+ *
+ * This section provides debugging assistance for analyzing Redis server failures
+ * when attempting to stream multiple RESP commands. This analysis can be particularly
+ * challenging in the following scenarios:
+ *
+ * - When using pipeline mode, which involves multiple concurrent pending commands
+ *   at any given moment.
+ * - When not using the `delKeyBeforeWrite` flag and Redis server is not empty.
+ * - In a production environments with real-world loads.
+ *
+ * The following two debug functions are designed to help with the analysis of a given
+ * RDB file:
+ *
+ * RDBX_enumerateCmds
+ *   Enumerates commands by preceding any RESP command with an additional trivial
+ *   RESP command of the type 'echo <cmd-number>'. This can be especially useful since
+ *   the RESP-to-Redis instance prints the command number in case of a failure.
+ *
+ * RDBX_writeFromCmdNumber
+ *   Writing commands starting from specified command-number and onward as part
+ *   of reproducing effort. Once the problem was resolved, it might be also useful
+ *   to continue uploading the redis server from the point it got failed.
+ ****************************************************************/
+_LIBRDB_API void RDBX_enumerateCmds(RdbxToResp *rdbToResp);
+
+_LIBRDB_API void RDBX_writeFromCmdNumber(RdbxToResp *rdbToResp, size_t cmdNum);
+
 #ifdef __cplusplus
 }
 #endif

examples/example1.c

@@ -31,7 +31,7 @@ int main(void) {
     RdbParser *parser;
     RdbxReaderFile *reader;
     RdbxToJson *rdbToJson;
-    RdbxFilterKey *filterKey;
+    RdbxFilter *filterKey;
     RdbRes err=RDB_OK;
 
     const char *infile = "./dumps/multiple_lists_strings.rdb";
@@ -53,7 +53,7 @@ int main(void) {
     if (!rdbToJson) goto PARSER_ERROR;
 
     /* Filter keys that starts with the word `mylist` */
-    filterKey = RDBX_createHandlersFilterKey(parser, "mylist.*", 0 /*flags*/);
+    filterKey = RDBX_createHandlersFilterKey(parser, "mylist.*", 0 /*exclude*/);
     if (!filterKey) goto PARSER_ERROR;
 
     /* Run the parser */

runtests

@@ -2,6 +2,8 @@
 
 VALGRIND=0
 REDIS_FOLDER=""
+SPECIFIC_TEST=""
+SPECIFIC_TEST_GROUP=""
 
 while [[ $# -gt 0 ]]; do
   case $1 in
@@ -11,8 +13,14 @@ while [[ $# -gt 0 ]]; do
     -f|--redis-folder)
       REDIS_FOLDER=" --redis-folder $2 "
       shift 2 ;;
+    -t|--test)
+      SPECIFIC_TEST=" --test $2 "
+      shift 2 ;;
+    -g|--test-group)
+      SPECIFIC_TEST_GROUP=" --test-group $2 "
+      shift 2 ;;
     *|-h)
-      echo "Usage: $(basename $0) [-v|--valgrind] [-f|--folder REDIS_FOLDER]"
+      echo "Usage: $(basename $0) [-v|--valgrind] [-f|--folder REDIS_FOLDER] [-t|--test TEST] [-g|--test-group GROUP]"
       exit 1 ;;
   esac
 done
@@ -26,12 +34,12 @@ if [[ ${VALGRIND} -ne 0 ]]; then
   		--leak-resolution=high \
   		--error-exitcode=1 \
   		--log-file=test/log/valgrind.log \
-  		./test/test_static_lib $REDIS_FOLDER && exit 0
+  		./test/test_static_lib $REDIS_FOLDER $SPECIFIC_TEST $SPECIFIC_TEST_GROUP && exit 0
 
       sed -n -e '/SUMMARY:/,$p' ./test/log/valgrind.log | tail -n 20
   		echo  -en "\n(Entire log available at: ./test/log/valgrind.log)\n"
   		exit 1
 
 else
-  ./test/test_lib $REDIS_FOLDER
+  ./test/test_lib $REDIS_FOLDER $SPECIFIC_TEST $SPECIFIC_TEST_GROUP
 fi