diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30120b7..156c769 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: with: submodules: 'recursive' - - name: Install Prerequisites + - name: Install Prerequisites and clone Redis run: | sudo apt-get update sudo apt-get install -y cmake libssl-dev valgrind git @@ -26,20 +26,30 @@ jobs: make ctest sudo make install - # Install redis for tests against external Redis server - - # Build Redis to run it against librdb tests git clone https://github.com/redis/redis.git ~/redis - make -j 4 -C ~/redis - make -C ~/redis/tests/modules - + shell: bash - - name: Build and run project + - name: Test librdb vs. redis-5.0 + run: | + pushd ~/redis + git checkout 5.0 + make distclean + make -j 4 -C ~/redis + popd + export LIBRDB_REDIS_FOLDER=~/redis/src + make test + working-directory: ${{github.workspace}} + + - name: Test librdb vs. redis-unstable run: | + pushd ~/redis + git checkout unstable + make -j 4 -C ~/redis + make -C ~/redis/tests/modules + popd export LIBRDB_REDIS_FOLDER=~/redis/src - export LD_LIBRARY_PATH=/usr/local/lib/ - make all valgrind + make all example valgrind working-directory: ${{github.workspace}} build-clang: @@ -52,10 +62,10 @@ jobs: with: submodules: 'recursive' - - name: Install Prerequisites + - name: Install Prerequisites and clone Redis run: | sudo apt-get update - sudo apt-get install -y cmake libssl-dev git + sudo apt-get install -y cmake libssl-dev git clang git clone https://git.cryptomilk.org/projects/cmocka.git cd cmocka mkdir build @@ -63,20 +73,30 @@ jobs: cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local make sudo make install - # Install redis for tests against external Redis server - - # Build Redis to run it against librdb tests + git clone https://github.com/redis/redis.git ~/redis - make -j 4 -C ~/redis - make -C ~/redis/tests/modules - shell: bash - - name: Build and run project + - name: Test librdb vs. redis-6.0 + run: | + pushd ~/redis + git checkout 6.0 + make distclean + make -j 4 -C ~/redis + popd + export LIBRDB_REDIS_FOLDER=~/redis/src + make test + working-directory: ${{github.workspace}} + + - name: Test librdb vs. redis-6.2 run: | # clang is more strict to shared-obj versioning. Satisfy its needs. - (cd ./deps/hiredis ; ln -s ./libhiredis.so ./libhiredis.so.1.2.1-dev) - export LD_LIBRARY_PATH=/usr/local/lib/ + pushd ~/redis + git checkout 6.2 + make -j 4 -C ~/redis + make -C ~/redis/tests/modules + popd export LIBRDB_REDIS_FOLDER=~/redis/src - make all test + make clean debug test working-directory: ${{github.workspace}} + diff --git a/Makefile b/Makefile index 692d1d3..786e9a1 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ PREFIX?=/usr/local + DESTDIR?= INSTALL = /usr/bin/install -c BINDIR=$(DESTDIR)$(PREFIX)/bin @@ -30,6 +31,11 @@ distclean: clean example: all cd examples && export LD_LIBRARY_PATH=../lib && ./example1 +# ------------------------- DEBUG ------------------------------------- + +debug: + OPTIMIZATION="-O0" LIBRDB_DEBUG=1 $(MAKE) + # ------------------------- TEST -------------------------------------- build_test: all @@ -45,16 +51,16 @@ valgrind: build_test install: all $(INSTALL) -d $(BINDIR) $(INSTALL) -m 755 bin/rdb-cli $(BINDIR)/rdb-cli-$(VERSION) - ln -fs $(BINDIR)/rdb-cli-$(VERSION) $(BINDIR)/rdb-cli + ln -fsr $(BINDIR)/rdb-cli-$(VERSION) $(BINDIR)/rdb-cli $(INSTALL) -d $(LIBDIR) $(INSTALL) -m 755 lib/librdb.so $(LIBDIR)/librdb.so.$(VERSION) - ln -fs $(LIBDIR)/librdb.so.$(VERSION) $(LIBDIR)/librdb.so + ln -fsr $(LIBDIR)/librdb.so.$(VERSION) $(LIBDIR)/librdb.so $(INSTALL) -m 755 lib/librdb-ext.so $(LIBDIR)/librdb-ext.so.$(VERSION) - ln -fs $(LIBDIR)/librdb-ext.so.$(VERSION) $(LIBDIR)/librdb-ext.so + ln -fsr $(LIBDIR)/librdb-ext.so.$(VERSION) $(LIBDIR)/librdb-ext.so $(INSTALL) -m 755 lib/librdb.a $(LIBDIR)/librdb.a.$(VERSION) - ln -fs $(LIBDIR)/librdb.a.$(VERSION) $(LIBDIR)/librdb.a + ln -fsr $(LIBDIR)/librdb.a.$(VERSION) $(LIBDIR)/librdb.a $(INSTALL) -m 755 lib/librdb-ext.a $(LIBDIR)/librdb-ext.a.$(VERSION) - ln -fs $(LIBDIR)/librdb-ext.a.$(VERSION) $(LIBDIR)/librdb-ext.a + ln -fsr $(LIBDIR)/librdb-ext.a.$(VERSION) $(LIBDIR)/librdb-ext.a $(INSTALL) -d $(INCDIR) $(INSTALL) -m 644 api/librdb-api.h $(INCDIR) $(INSTALL) -m 644 api/librdb-ext-api.h $(INCDIR) @@ -78,14 +84,16 @@ uninstall: help: @echo "librdb (v$(VERSION)) target rules:" @echo " all - Build parser libraries, tests, and run tests" + @echo " debug - Build without compiler optimization and with assert() enabled" @echo " test - Run tests with shared lib" @echo " valgrind - Run tests with static lib and valgrind" @echo " example - Run the example" @echo " clean - Clean without deps folders" @echo " distclean - Clean including deps folders" - @echo " install - Build parser libraries and copy to (DESTDIR)\(PREFIX)" - @echo " uninstall - Remove libraries from (DESTDIR)\(PREFIX)" + @echo " install - install to (DESTDIR)/(PREFIX)/bin and (DESTDIR)/(PREFIX)/lib" + @echo " By default PREFIX=/usr/local" + @echo " uninstall - Remove from (DESTDIR)\(PREFIX)/bin and (DESTDIR)/(PREFIX)/lib" @echo " help - Prints this message" -.PHONY: all test valgrind example clean distclean install uninstall build_test help version \ No newline at end of file +.PHONY: all debug test valgrind example clean distclean install uninstall build_test help \ No newline at end of file diff --git a/README.md b/README.md index 752567e..ee3da7c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# librdb - DRAFT +# librdb -This is a C library for parsing RDB files. +C library for parsing RDB files. The Parser is implemented in the spirit of SAX parser. It fires off a series of events as it reads the RDB file from beginning to end, and callbacks to handlers registered on @@ -9,59 +9,58 @@ selected types of data. The primary objective of this project is to offer an efficient and robust C library for parsing Redis RDB files. It also provides an extension library for parsing to JSON and RESP -protocols, enabling consumption by various writers. Additionally, a command-line interface -(CLI) is available for utilizing these functionalities. - -## Current status -The project is currently in its early phase and is considered to be a draft. At present, -the parser is only capable of handling STRING, LIST, HASH and SET data types. We are -actively seeking feedback on the design, API, and implementation to refine the project -before proceeding with further development. Community contributions are welcome, yet -please note that the codebase is still undergoing significant changes and may evolve in -the future. +protocols. ## Getting Started -If you just wish to get a basic understanding of the library's functionality, without -running tests (To see parser internal state printouts, execute the command -`export LIBRDB_DEBUG_DATA=1` beforehand): +If you just wish to get a basic understanding of the library's functionality: + + % make example - % make all example +To see cool internal state printouts of the parser, set env-var `LIBRDB_DEBUG_DATA` beforehand: + + % export LIBRDB_DEBUG_DATA=1 + % make example To build and run tests, you need to have cmocka unit testing framework installed: % make test -Install and run CLI extension of this library. Parse RDB file to json: +To install into /usr/local/: % make install - % rdb-cli multiple_lists_strings.rdb json +To run CLI extension of this library and let it parse RDB file to json: + + % rdb-cli mixed_data_types.rdb json [{ - "string2":"Hi there!", - "mylist1":["v1"], - "mylist3":["v3","v2","v1"], - "lzf_compressed":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "string1":"blaa", - "mylist2":["v2","v1"] + "my_key":"Hello, Redis!", + "my_set":["member1","member2"], + "my_zset":{"Bob":"5","Alice":"10","Charlie":"15"}, + "my_hash":{"field1":"value1","field2":"field2"}, + "my_list":["item1","item2", "item3"], + "my_stream":{ + "entries":[ + { "id":"1695649068107-0", "values":{"message":"Message1"} }, + { "id":"1695649068110-0", "values":{"message":"Message2"} }, + { "id":"1695893015933-0", "values":{"field1":"value1", "field2":"value2", "field3":"value3"} } + ]} }] -Run CLI extension to generate RESP commands (this time read file from standard input): +To generate RESP commands: - % gzip -dc multiple_lists_strings.rdb.gz | rdb-cli - resp - *3 - $3 - SET - $7 - string2 + % rdb-cli multiple_lists_strings.rdb resp + *2 + $6 + SELECT + $1 ... -Run against live Redis server and upload RDB file (example assumes Redis is installed locally): +To run against live Redis server and upload RDB file, assuming Redis is installed as well: % redis-server --port 6379 & % rdb-cli multiple_lists_strings.rdb redis -h 127.0.0.1 -p 6379 % redis-cli keys "*" - 1) "string2" 2) "mylist3" 3) "mylist2" @@ -81,27 +80,6 @@ issues, it is worthwhile to develop a new parser with a modern architecture, tha can also challenge the current integrated RDB parser of Redis and even replace it in the future. -### Replacing the integrated RDB parser of Redis? -It is necessary to address first the reasons and missing features in the available parser, -making replacing it a feasible option: - -1. The current parser is not designed to extend and customize it. -2. Lacks of a unit testing framework. -3. Does not support asynchronous parsing - That is, today reading of RDB sources is - possible only in blocking IO mode, without the option to let Redis thread to carry on to - other tasks and notify it asynchronously once a read operation is done. -4. Doesn’t support pause and resume capabilities - A parsing of RDB is being made from - start till completion as a single operation, without the option to indicate the parser - to pause and save its state, in order to do other tasks in between. -5. Once we decide to write RDB parser library, it is better to maintain a single parser - rather than two. - -Although it is challenging to develop a reusable and extensible parser that can match in -performance, according to our initial evaluation as long as the new parser will avoid -redundant copies and allocation of data, it is expected that it will show similar performance, -or minor degradation at most, and yet we will gain all the advantages mentioned above. Having -said that, our primary focus is to develop an “independent” RDB parser library. - ## Main building blocks The RDB library parser composed of 3 main building blocks: @@ -110,15 +88,12 @@ The RDB library parser composed of 3 main building blocks: +--------+ +--------+ +----------+ ### Reader -The **Reader** gives interface to the parser to access the RDB source. As first phase we will support: - * Reading from a file (Status: Done) - * Reading from a socket (Status: Todo) - * User defined reader (Status: Done) - -Possible extensions might be reading from S3, gz file, or a live redis instance. +The **Reader** gives interface to the parser to access the RDB source. It can be either +reading from a file, a socket or user defined reader. Possible extensions might be reading +from S3, gz file, or a live redis instance. -This block is optional. As an alternative, the parser can be fed with chunks of data that -hold RDB payload. +This block is optional. As an alternative, the parser can be fed with prefetched chunks of +data. ### Parser The **Parser** is the core engine. It will parse RDB file and trigger registered handlers. @@ -228,7 +203,7 @@ destruction, or when newer block replacing old one. Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {json|resp|redis} [FORMAT_OPTIONS] OPTIONS: - -l, --log-file Path to the log file (Default: './rdb-cli.log') + -l, --log-file {|-} Path to the log file or stdout (Default: './rdb-cli.log') Multiple filters combination of keys/types/dbs can be specified: -k, --key Include only keys that match REGEX @@ -253,6 +228,7 @@ destruction, or when newer block replacing old one. FORMAT_OPTIONS ('redis'|'resp'): -r, --support-restore Use the RESTORE command when possible + -d, --del-before-write Delete each key before writing. Relevant for non-empty db -t, --target-redis-ver Specify the target Redis version. Helps determine which commands can be applied. Particularly crucial if support-restore being used as RESTORE is closely tied to specific RDB versions. If versions not @@ -370,10 +346,9 @@ of callbacks, it is the duty of the application to configure for each RDB object what level it is needed to get handled by calling `RDB_handleByLevel()`. Otherwise, the parser will resolve it by parsing and calling handlers that are registered at lowest level. -As for the common callbacks to all levels (which includes `handleStartRdb`, `handleNewDb`, -`handleEndRdb`, `handleDbSize` and `handleAuxField`) if registered at different -levels then all of them will be called, one by one, starting from handlers that are -registered at the lowest level. +As for the common callbacks to all levels, such as `handleStartRdb` or `handleNewDb`, +if registered at different levels then all of them will be called, one by one, starting +from handlers that are registered at the lowest level. ## Implementation notes The Redis RDB file format consists of a series of opcodes followed by the actual data that diff --git a/api/librdb-api.h b/api/librdb-api.h index 9c59746..704a4ed 100644 --- a/api/librdb-api.h +++ b/api/librdb-api.h @@ -8,17 +8,15 @@ extern "C" { #endif #ifndef _LIBRDB_API -#define _LIBRDB_API +#define _LIBRDB_API __attribute__((visibility("default"))) #endif -#define MAX_RDB_VER_SUPPORT 11 +typedef char *RdbBulk; +typedef char *RdbBulkCopy; /**************************************************************** * Incomplete structures for compiler checks but opaque access ****************************************************************/ -typedef char *RdbBulk; -typedef char *RdbBulkCopy; - typedef struct RdbReader RdbReader; typedef struct RdbParser RdbParser; typedef struct RdbHandlers RdbHandlers; @@ -51,7 +49,6 @@ typedef enum RdbRes { RDB_ERR_FAILED_CREATE_PARSER, RDB_ERR_FAILED_OPEN_LOG_FILE, RDB_ERR_FAILED_READ_RDB_FILE, - RDB_ERR_NO_MEMORY, RDB_ERR_FAILED_OPEN_RDB_FILE, RDB_ERR_WRONG_FILE_SIGNATURE, RDB_ERR_UNSUPPORTED_RDB_VERSION, @@ -72,6 +69,7 @@ typedef enum RdbRes { RDB_ERR_ZSET_LP_INTEG_CHECK, RDB_ERR_HASH_LP_INTEG_CHECK, RDB_ERR_HASH_ZM_INTEG_CHECK, + RDB_ERR_STREAM_LP_INTEG_CHECK, RDB_ERR_SSTYPE_INTEG_CHECK, RDB_ERR_STRING_INVALID_STATE, RDB_ERR_PLAIN_HASH_INVALID_STATE, @@ -81,6 +79,7 @@ typedef enum RdbRes { RDB_ERR_QUICK_LIST_INVALID_STATE, RDB_ERR_SSTYPE_INVALID_STATE, RDB_ERR_MODULE_INVALID_STATE, + RDB_ERR_STREAM_INVALID_STATE, RDB_ERR_INVALID_BULK_ALLOC_TYPE, RDB_ERR_INVALID_BULK_CLONE_REQUEST, RDB_ERR_INVALID_BULK_LENGTH_REQUEST, @@ -147,26 +146,34 @@ typedef struct RdbKeyInfo { int opcode; } RdbKeyInfo; +typedef struct RdbSlotInfo { + uint64_t slot_id; + uint64_t slot_size; + uint64_t expires_slot_size; +} RdbSlotInfo; + typedef struct RdbStreamID { - uint64_t ms; - uint64_t seq; + uint64_t ms; /* Unix time in milliseconds. */ + uint64_t seq; /* sequence number */ } RdbStreamID; typedef struct RdbStreamMeta { - uint64_t length; /* Current number of elements inside this stream. */ - uint64_t entriesAdded; /* All time count of elements added. */ - RdbStreamID *firstID; - RdbStreamID *lastID; - RdbStreamID *maxDeletedEntryID; + uint64_t length; /* Current number of elements inside this stream. */ + uint64_t entriesAdded; /* All time count of elements added. */ + RdbStreamID firstID; + RdbStreamID lastID; + RdbStreamID maxDelEntryID; /* maximum deleted entry id */ } RdbStreamMeta; typedef struct RdbStreamPendingEntry { - long long deliveryTime; + RdbStreamID id; + uint64_t deliveryTime; uint64_t deliveryCount; } RdbStreamPendingEntry; typedef struct RdbStreamGroupMeta { RdbStreamID lastId; + int64_t entriesRead; } RdbStreamGroupMeta; typedef struct RdbStreamConsumerMeta { @@ -197,6 +204,8 @@ typedef void (*RdbLoggerCB) (RdbLogLevel l, const char *msg); RdbRes (*handleNewDb)(RdbParser *p, void *userData, int dbnum); \ /* Callback per db before the keys, with the key count and the total voletaile key count */ \ RdbRes (*handleDbSize)(RdbParser *p, void *userData, uint64_t db_size, uint64_t exp_size); \ + /* Callback per cluster slot with related info */ \ + RdbRes (*handleSlotInfo)(RdbParser *p, void *userData, RdbSlotInfo *info); \ /* Callback in the beginning of the RDB with various keys and values. exists since redis 3.2 (RDB v7) */ \ RdbRes (*handleAuxField)(RdbParser *p, void *userData, RdbBulk auxkey, RdbBulk auxval); \ /* Callback on each new key along with additional info, such as, expire-time, LRU, etc. */ \ @@ -271,10 +280,6 @@ typedef struct RdbHandlersStructCallbacks { /* Callback to handle module. Currently only reports about the name & size. */ RdbRes (*handleModule)(RdbParser *p, void *userData, RdbBulk moduleName, size_t serializedSize); - /*** TODO: RdbHandlersStructCallbacks: ***/ - - /*** TODO: RdbHandlersStructCallbacks: stream stuff ... ***/ - /* Callback to handle a stream key with listpack value */ RdbRes (*handleStreamLP)(RdbParser *p, void *userData, RdbBulk nodekey, RdbBulk streamLP); @@ -298,24 +303,16 @@ typedef struct RdbHandlersDataCallbacks { RdbRes (*handleHashField)(RdbParser *p, void *userData, RdbBulk field, RdbBulk value); /* Callback to handle a member within a set */ RdbRes (*handleSetMember)(RdbParser *p, void *userData, RdbBulk member); + /* Callback to handle a member within a sorted set along with its score */ + RdbRes (*handleZsetMember)(RdbParser *p, void *userData, RdbBulk member, double score); /* Callback to handle function code */ RdbRes (*handleFunction)(RdbParser *p, void *userData, RdbBulk func); /* Callback to handle module. Currently only reports about the name & size */ RdbRes (*handleModule)(RdbParser *p, void *userData, RdbBulk moduleName, size_t serializedSize); - - - - /*** TODO: RdbHandlersDataCallbacks: handleZsetElement ***/ - - /* Callback to handle a member within a sorted set along with its score */ - RdbRes (*handleZsetMember)(RdbParser *p, void *userData, RdbBulk member, double score); - - /*** TODO: RdbHandlersDataCallbacks: stream stuff ... ***/ - /* Callback to handle metadata associated with a stream */ RdbRes (*handleStreamMetadata)(RdbParser *p, void *userData, RdbStreamMeta *meta); /* Callback to handle an item within a stream along with its field and value */ - RdbRes (*handleStreamItem)(RdbParser *p, void *userData, RdbStreamID *id, RdbBulk field, RdbBulk value); + RdbRes (*handleStreamItem)(RdbParser *p, void *userData, RdbStreamID *id, RdbBulk field, RdbBulk value, int64_t itemsLeft); /* Callback to handle the creation of a new consumer group within a stream */ RdbRes (*handleStreamNewCGroup)(RdbParser *p, void *userData, RdbBulk grpName, RdbStreamGroupMeta *meta); /* Callback to handle a pending entry within a consumer group */ @@ -323,7 +320,7 @@ typedef struct RdbHandlersDataCallbacks { /* Callback to handle the creation of a new consumer within a stream */ RdbRes (*handleStreamNewConsumer)(RdbParser *p, void *userData, RdbBulk consName, RdbStreamConsumerMeta *meta); /* Callback to handle a pending entry within a consumer */ - RdbRes (*handleStreamConsumerPendingEntry)(RdbParser *p, void *userData, RdbStreamPendingEntry *pendingEntry); + RdbRes (*handleStreamConsumerPendingEntry)(RdbParser *p, void *userData, RdbStreamID *streamId); } RdbHandlersDataCallbacks; @@ -399,10 +396,19 @@ _LIBRDB_API void RDB_dontPropagate(RdbParser *p); * Parser setters & getters ****************************************************************/ +/* set deep integrity check */ _LIBRDB_API void RDB_setDeepIntegCheck(RdbParser *p, int deep); + +/* get number of bytes processed so far */ _LIBRDB_API size_t RDB_getBytesProcessed(RdbParser *p); + +/* get current state of the parser */ _LIBRDB_API RdbState RDB_getState(RdbParser *p); + +/* get number of handlers registered at given level */ _LIBRDB_API int RDB_getNumHandlers(RdbParser *p, RdbHandlersLevel lvl); + +/* set the parser to ignore checksum errors */ _LIBRDB_API void RDB_IgnoreChecksum(RdbParser *p); /* There could be relatively large strings stored within Redis, which are @@ -528,6 +534,11 @@ void RDB_free(RdbParser *p, void *ptr); * it actually allocated behind on stack, heap, reference another memory, or * externally allocated by user supplied RdbBulk allocation function. * + * You can print RdbBulk with printf() as there is an implicit \0 at the + * end of the string. However the string is binary safe and can contain + * \0 characters in the middle. The length is stored out of bound and can be + * queried by function RDB_bulkLen() only from the handlers callbacks. + * * In order to process the string behind the current call-stack, function * `RDB_bulkClone()` is the way to clone a string, within callback context only! * If bulk allocated on stack (default of bulkAllocType) or reference another @@ -583,8 +594,11 @@ typedef enum RdbDataType { RDB_DATA_TYPE_MAX } RdbDataType; -/* Can be called at any point along parsing (Useful after parsing source rdb version) */ -_LIBRDB_API void RDB_handleByLevel(RdbParser *p, RdbDataType t, RdbHandlersLevel lvl, unsigned int flags); +/* Return 0 on success, 1 otherwise. Can be called at any point along parsing + * (Useful after parsing source rdb version) */ +_LIBRDB_API int RDB_handleByLevel(RdbParser *p, + RdbDataType t, + RdbHandlersLevel lvl); /***************************************************************** * LIBRDB Versioning @@ -594,6 +608,8 @@ _LIBRDB_API void RDB_handleByLevel(RdbParser *p, RdbDataType t, RdbHandlersLevel _LIBRDB_API const char* RDB_getLibVersion(int* major, int* minor, int* patch); +_LIBRDB_API int RDB_getMaxSuppportRdbVersion(void); + #ifdef __cplusplus } #endif diff --git a/api/librdb-ext-api.h b/api/librdb-ext-api.h index 0a2049a..9bc24d3 100644 --- a/api/librdb-ext-api.h +++ b/api/librdb-ext-api.h @@ -28,17 +28,22 @@ typedef enum { /* rdb2json errors */ RDBX_ERR_FAILED_OPEN_FILE, RDBX_ERR_R2J_INVALID_STATE, + RDBX_ERR_R2J_INVALID_LEVEL, /* HandlersFilterKey errors */ RDBX_ERR_FILTER_FAILED_COMPILE_REGEX, RDBX_ERR_FAILED_CREATE_FILTER, /* rdb2resp errors */ + RDBX_ERR_STREAM_DUPLICATE_PEL, + RDBX_ERR_STREAM_INTEG_CHECK, /* resp writer/loader */ RDBX_ERR_RESP_WRITE, + RDBX_ERR_RESP_INVALID_TARGET_VERSION, RDBX_ERR_RESP_READ, RDBX_ERR_RESP2REDIS_CREATE_SOCKET, + RDBX_ERR_RESP2REDIS_CONF_NONBLOCK_SOCKET, RDBX_ERR_RESP2REDIS_INVALID_ADDRESS, RDBX_ERR_RESP2REDIS_FAILED_CONNECT, RDBX_ERR_RESP2REDIS_FAILED_READ, @@ -69,6 +74,7 @@ typedef struct RdbxToJsonConf { RdbxToJsonEnc encoding; /* Encoding format for the resulting JSON */ int includeAuxField; /* Set to include auxiliary fields in JSON output */ int includeFunc; /* Set to include functions in JSON output */ + int includeStreamMeta; /* Set to include Stream metadata in JSON output */ int flatten; /* Set to create a flattened JSON structure */ } RdbxToJsonConf; diff --git a/deps/README.md b/deps/README.md index b61c815..ef52b5d 100644 --- a/deps/README.md +++ b/deps/README.md @@ -1,11 +1,5 @@ # deps The purpose of this folder is to provide the necessary dependencies for the librdb parser. -Currently, it includes only redis subfolder, which contains a subset of files from the Redis -repository that have been slightly adapted for reuse by librdb. - -As the librdb library evolves, it might require additional dependencies apart from the redis -subfolder. In such cases, new subfolders may be added under the deps directory to -accommodate these dependencies. # librdb dependencies @@ -20,3 +14,6 @@ To upgrade, use as base reference specified version in version.h file, though it update so often (Otherwise, consider in the future having better methodology to consume and upgrade redis code). +## hiredis +This directory contains the 'hiredis' project as a submodule. It is exclusively utilized +by the tests to manipulate the Redis server. \ No newline at end of file diff --git a/deps/redis/Makefile b/deps/redis/Makefile index 7530f5b..a7b0f13 100644 --- a/deps/redis/Makefile +++ b/deps/redis/Makefile @@ -1,9 +1,11 @@ SOURCES = $(notdir $(basename $(wildcard *.c))) OBJECTS = $(patsubst %,%.o,$(SOURCES)) +OPTIMIZATION?=-O3 + STD = -std=c99 WARNS = -Wall -Wextra -pedantic -Werror -CFLAGS = -fPIC -O3 $(STD) $(WARNS) +CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(WARNS) -fvisibility=hidden DEBUG = -g3 -DDEBUG=1 LIBS = diff --git a/deps/redis/crc64.c b/deps/redis/crc64.c index 73e0391..1740840 100644 --- a/deps/redis/crc64.c +++ b/deps/redis/crc64.c @@ -25,7 +25,7 @@ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ - +#include #include "crc64.h" #include "crcspeed.h" static uint64_t crc64_table[8][256] = {{0}}; @@ -122,6 +122,20 @@ uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { return crcspeed64native(crc64_table, crc, (void *) s, l); } +/* Not sure if it is really required to use mutex here, but just in case */ +void crc64_init_thread_safe(void) { + static int crcInitalized = 0; + static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + if (!crcInitalized) { + pthread_mutex_lock(&mutex); + if (!crcInitalized) { + crc64_init(); + crcInitalized = 1; + } + pthread_mutex_unlock(&mutex); + } +} + /* Test main */ #ifdef REDIS_TEST #include diff --git a/deps/redis/crc64.h b/deps/redis/crc64.h index e0fccd9..9859a40 100644 --- a/deps/redis/crc64.h +++ b/deps/redis/crc64.h @@ -4,8 +4,10 @@ #include void crc64_init(void); +void crc64_init_thread_safe(void); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); + #ifdef REDIS_TEST int crc64Test(int argc, char *argv[], int flags); #endif diff --git a/deps/redis/rax.c b/deps/redis/rax.c new file mode 100644 index 0000000..9f64267 --- /dev/null +++ b/deps/redis/rax.c @@ -0,0 +1,1930 @@ +/* Rax -- A radix tree implementation. + * + * Version 1.2 -- 7 February 2019 + * + * Copyright (c) 2017-2019, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include "rax.h" + +#ifndef RAX_MALLOC_INCLUDE +#define RAX_MALLOC_INCLUDE "rax_malloc.h" +#endif + +#include RAX_MALLOC_INCLUDE + +/* This is a special pointer that is guaranteed to never have the same value + * of a radix tree node. It's used in order to report "not found" error without + * requiring the function to have multiple return values. */ +void *raxNotFound = (void*)"rax-not-found-pointer"; + +/* -------------------------------- Debugging ------------------------------ */ + +void raxDebugShowNode(const char *msg, raxNode *n); + +/* Turn debugging messages on/off by compiling with RAX_DEBUG_MSG macro on. + * When RAX_DEBUG_MSG is defined by default Rax operations will emit a lot + * of debugging info to the standard output, however you can still turn + * debugging on/off in order to enable it only when you suspect there is an + * operation causing a bug using the function raxSetDebugMsg(). */ +#ifdef RAX_DEBUG_MSG +#define debugf(...) \ + if (raxDebugMsg) { \ + printf("%s:%s:%d:\t", __FILE__, __func__, __LINE__); \ + printf(__VA_ARGS__); \ + fflush(stdout); \ + } + +#define debugnode(msg,n) raxDebugShowNode(msg,n) +#else +#define debugf(...) +#define debugnode(msg,n) +#endif + +/* By default log debug info if RAX_DEBUG_MSG is defined. */ +static int raxDebugMsg = 1; + +/* When debug messages are enabled, turn them on/off dynamically. By + * default they are enabled. Set the state to 0 to disable, and 1 to + * re-enable. */ +void raxSetDebugMsg(int onoff) { + raxDebugMsg = onoff; +} + +/* ------------------------- raxStack functions -------------------------- + * The raxStack is a simple stack of pointers that is capable of switching + * from using a stack-allocated array to dynamic heap once a given number of + * items are reached. It is used in order to retain the list of parent nodes + * while walking the radix tree in order to implement certain operations that + * need to navigate the tree upward. + * ------------------------------------------------------------------------- */ + +/* Initialize the stack. */ +static inline void raxStackInit(raxStack *ts) { + ts->stack = ts->static_items; + ts->items = 0; + ts->maxitems = RAX_STACK_STATIC_ITEMS; + ts->oom = 0; +} + +/* Push an item into the stack, returns 1 on success, 0 on out of memory. */ +static inline int raxStackPush(raxStack *ts, void *ptr) { + if (ts->items == ts->maxitems) { + if (ts->stack == ts->static_items) { + ts->stack = rax_malloc(sizeof(void*)*ts->maxitems*2); + if (ts->stack == NULL) { + ts->stack = ts->static_items; + ts->oom = 1; + errno = ENOMEM; + return 0; + } + memcpy(ts->stack,ts->static_items,sizeof(void*)*ts->maxitems); + } else { + void **newalloc = rax_realloc(ts->stack,sizeof(void*)*ts->maxitems*2); + if (newalloc == NULL) { + ts->oom = 1; + errno = ENOMEM; + return 0; + } + ts->stack = newalloc; + } + ts->maxitems *= 2; + } + ts->stack[ts->items] = ptr; + ts->items++; + return 1; +} + +/* Pop an item from the stack, the function returns NULL if there are no + * items to pop. */ +static inline void *raxStackPop(raxStack *ts) { + if (ts->items == 0) return NULL; + ts->items--; + return ts->stack[ts->items]; +} + +/* Return the stack item at the top of the stack without actually consuming + * it. */ +static inline void *raxStackPeek(raxStack *ts) { + if (ts->items == 0) return NULL; + return ts->stack[ts->items-1]; +} + +/* Free the stack in case we used heap allocation. */ +static inline void raxStackFree(raxStack *ts) { + if (ts->stack != ts->static_items) rax_free(ts->stack); +} + +/* ---------------------------------------------------------------------------- + * Radix tree implementation + * --------------------------------------------------------------------------*/ + +/* Return the padding needed in the characters section of a node having size + * 'nodesize'. The padding is needed to store the child pointers to aligned + * addresses. Note that we add 4 to the node size because the node has a four + * bytes header. */ +#define raxPadding(nodesize) ((sizeof(void*)-(((nodesize)+4) % sizeof(void*))) & (sizeof(void*)-1)) + +/* Return the pointer to the last child pointer in a node. For the compressed + * nodes this is the only child pointer. */ +#define raxNodeLastChildPtr(n) ((raxNode**) ( \ + ((char*)(n)) + \ + raxNodeCurrentLength(n) - \ + sizeof(raxNode*) - \ + (((n)->iskey && !(n)->isnull) ? sizeof(void*) : 0) \ +)) + +/* Return the pointer to the first child pointer. */ +#define raxNodeFirstChildPtr(n) ((raxNode**) ( \ + (n)->data + \ + (n)->size + \ + raxPadding((n)->size))) + +/* Return the current total size of the node. Note that the second line + * computes the padding after the string of characters, needed in order to + * save pointers to aligned addresses. */ +#define raxNodeCurrentLength(n) ( \ + sizeof(raxNode)+(n)->size+ \ + raxPadding((n)->size)+ \ + ((n)->iscompr ? sizeof(raxNode*) : sizeof(raxNode*)*(n)->size)+ \ + (((n)->iskey && !(n)->isnull)*sizeof(void*)) \ +) + +/* Allocate a new non compressed node with the specified number of children. + * If datafield is true, the allocation is made large enough to hold the + * associated data pointer. + * Returns the new node pointer. On out of memory NULL is returned. */ +raxNode *raxNewNode(size_t children, int datafield) { + size_t nodesize = sizeof(raxNode)+children+raxPadding(children)+ + sizeof(raxNode*)*children; + if (datafield) nodesize += sizeof(void*); + raxNode *node = rax_malloc(nodesize); + if (node == NULL) return NULL; + node->iskey = 0; + node->isnull = 0; + node->iscompr = 0; + node->size = children; + return node; +} + +/* Allocate a new rax and return its pointer. On out of memory the function + * returns NULL. */ +rax *raxNew(void) { + rax *rax = rax_malloc(sizeof(*rax)); + if (rax == NULL) return NULL; + rax->numele = 0; + rax->numnodes = 1; + rax->head = raxNewNode(0,0); + if (rax->head == NULL) { + rax_free(rax); + return NULL; + } else { + return rax; + } +} + +/* realloc the node to make room for auxiliary data in order + * to store an item in that node. On out of memory NULL is returned. */ +raxNode *raxReallocForData(raxNode *n, void *data) { + if (data == NULL) return n; /* No reallocation needed, setting isnull=1 */ + size_t curlen = raxNodeCurrentLength(n); + return rax_realloc(n,curlen+sizeof(void*)); +} + +/* Set the node auxiliary data to the specified pointer. */ +void raxSetData(raxNode *n, void *data) { + n->iskey = 1; + if (data != NULL) { + n->isnull = 0; + void **ndata = (void**) + ((char*)n+raxNodeCurrentLength(n)-sizeof(void*)); + memcpy(ndata,&data,sizeof(data)); + } else { + n->isnull = 1; + } +} + +/* Get the node auxiliary data. */ +void *raxGetData(raxNode *n) { + if (n->isnull) return NULL; + void **ndata =(void**)((char*)n+raxNodeCurrentLength(n)-sizeof(void*)); + void *data; + memcpy(&data,ndata,sizeof(data)); + return data; +} + +/* Add a new child to the node 'n' representing the character 'c' and return + * its new pointer, as well as the child pointer by reference. Additionally + * '***parentlink' is populated with the raxNode pointer-to-pointer of where + * the new child was stored, which is useful for the caller to replace the + * child pointer if it gets reallocated. + * + * On success the new parent node pointer is returned (it may change because + * of the realloc, so the caller should discard 'n' and use the new value). + * On out of memory NULL is returned, and the old node is still valid. */ +raxNode *raxAddChild(raxNode *n, unsigned char c, raxNode **childptr, raxNode ***parentlink) { + assert(n->iscompr == 0); + + size_t curlen = raxNodeCurrentLength(n); + n->size++; + size_t newlen = raxNodeCurrentLength(n); + n->size--; /* For now restore the original size. We'll update it only on + success at the end. */ + + /* Alloc the new child we will link to 'n'. */ + raxNode *child = raxNewNode(0,0); + if (child == NULL) return NULL; + + /* Make space in the original node. */ + raxNode *newn = rax_realloc(n,newlen); + if (newn == NULL) { + rax_free(child); + return NULL; + } + n = newn; + + /* After the reallocation, we have up to 8/16 (depending on the system + * pointer size, and the required node padding) bytes at the end, that is, + * the additional char in the 'data' section, plus one pointer to the new + * child, plus the padding needed in order to store addresses into aligned + * locations. + * + * So if we start with the following node, having "abde" edges. + * + * Note: + * - We assume 4 bytes pointer for simplicity. + * - Each space below corresponds to one byte + * + * [HDR*][abde][Aptr][Bptr][Dptr][Eptr]|AUXP| + * + * After the reallocation we need: 1 byte for the new edge character + * plus 4 bytes for a new child pointer (assuming 32 bit machine). + * However after adding 1 byte to the edge char, the header + the edge + * characters are no longer aligned, so we also need 3 bytes of padding. + * In total the reallocation will add 1+4+3 bytes = 8 bytes: + * + * (Blank bytes are represented by ".") + * + * [HDR*][abde][Aptr][Bptr][Dptr][Eptr]|AUXP|[....][....] + * + * Let's find where to insert the new child in order to make sure + * it is inserted in-place lexicographically. Assuming we are adding + * a child "c" in our case pos will be = 2 after the end of the following + * loop. */ + int pos; + for (pos = 0; pos < n->size; pos++) { + if (n->data[pos] > c) break; + } + + /* Now, if present, move auxiliary data pointer at the end + * so that we can mess with the other data without overwriting it. + * We will obtain something like that: + * + * [HDR*][abde][Aptr][Bptr][Dptr][Eptr][....][....]|AUXP| + */ + unsigned char *src, *dst; + if (n->iskey && !n->isnull) { + src = ((unsigned char*)n+curlen-sizeof(void*)); + dst = ((unsigned char*)n+newlen-sizeof(void*)); + memmove(dst,src,sizeof(void*)); + } + + /* Compute the "shift", that is, how many bytes we need to move the + * pointers section forward because of the addition of the new child + * byte in the string section. Note that if we had no padding, that + * would be always "1", since we are adding a single byte in the string + * section of the node (where now there is "abde" basically). + * + * However we have padding, so it could be zero, or up to 8. + * + * Another way to think at the shift is, how many bytes we need to + * move child pointers forward *other than* the obvious sizeof(void*) + * needed for the additional pointer itself. */ + size_t shift = newlen - curlen - sizeof(void*); + + /* We said we are adding a node with edge 'c'. The insertion + * point is between 'b' and 'd', so the 'pos' variable value is + * the index of the first child pointer that we need to move forward + * to make space for our new pointer. + * + * To start, move all the child pointers after the insertion point + * of shift+sizeof(pointer) bytes on the right, to obtain: + * + * [HDR*][abde][Aptr][Bptr][....][....][Dptr][Eptr]|AUXP| + */ + src = n->data+n->size+ + raxPadding(n->size)+ + sizeof(raxNode*)*pos; + memmove(src+shift+sizeof(raxNode*),src,sizeof(raxNode*)*(n->size-pos)); + + /* Move the pointers to the left of the insertion position as well. Often + * we don't need to do anything if there was already some padding to use. In + * that case the final destination of the pointers will be the same, however + * in our example there was no pre-existing padding, so we added one byte + * plus three bytes of padding. After the next memmove() things will look + * like that: + * + * [HDR*][abde][....][Aptr][Bptr][....][Dptr][Eptr]|AUXP| + */ + if (shift) { + src = (unsigned char*) raxNodeFirstChildPtr(n); + memmove(src+shift,src,sizeof(raxNode*)*pos); + } + + /* Now make the space for the additional char in the data section, + * but also move the pointers before the insertion point to the right + * by shift bytes, in order to obtain the following: + * + * [HDR*][ab.d][e...][Aptr][Bptr][....][Dptr][Eptr]|AUXP| + */ + src = n->data+pos; + memmove(src+1,src,n->size-pos); + + /* We can now set the character and its child node pointer to get: + * + * [HDR*][abcd][e...][Aptr][Bptr][....][Dptr][Eptr]|AUXP| + * [HDR*][abcd][e...][Aptr][Bptr][Cptr][Dptr][Eptr]|AUXP| + */ + n->data[pos] = c; + n->size++; + src = (unsigned char*) raxNodeFirstChildPtr(n); + raxNode **childfield = (raxNode**)(src+sizeof(raxNode*)*pos); + memcpy(childfield,&child,sizeof(child)); + *childptr = child; + *parentlink = childfield; + return n; +} + +/* Turn the node 'n', that must be a node without any children, into a + * compressed node representing a set of nodes linked one after the other + * and having exactly one child each. The node can be a key or not: this + * property and the associated value if any will be preserved. + * + * The function also returns a child node, since the last node of the + * compressed chain cannot be part of the chain: it has zero children while + * we can only compress inner nodes with exactly one child each. */ +raxNode *raxCompressNode(raxNode *n, unsigned char *s, size_t len, raxNode **child) { + assert(n->size == 0 && n->iscompr == 0); + void *data = NULL; /* Initialized only to avoid warnings. */ + size_t newsize; + + debugf("Compress node: %.*s\n", (int)len,s); + + /* Allocate the child to link to this node. */ + *child = raxNewNode(0,0); + if (*child == NULL) return NULL; + + /* Make space in the parent node. */ + newsize = sizeof(raxNode)+len+raxPadding(len)+sizeof(raxNode*); + if (n->iskey) { + data = raxGetData(n); /* To restore it later. */ + if (!n->isnull) newsize += sizeof(void*); + } + raxNode *newn = rax_realloc(n,newsize); + if (newn == NULL) { + rax_free(*child); + return NULL; + } + n = newn; + + n->iscompr = 1; + n->size = len; + memcpy(n->data,s,len); + if (n->iskey) raxSetData(n,data); + raxNode **childfield = raxNodeLastChildPtr(n); + memcpy(childfield,child,sizeof(*child)); + return n; +} + +/* Low level function that walks the tree looking for the string + * 's' of 'len' bytes. The function returns the number of characters + * of the key that was possible to process: if the returned integer + * is the same as 'len', then it means that the node corresponding to the + * string was found (however it may not be a key in case the node->iskey is + * zero or if simply we stopped in the middle of a compressed node, so that + * 'splitpos' is non zero). + * + * Otherwise if the returned integer is not the same as 'len', there was an + * early stop during the tree walk because of a character mismatch. + * + * The node where the search ended (because the full string was processed + * or because there was an early stop) is returned by reference as + * '*stopnode' if the passed pointer is not NULL. This node link in the + * parent's node is returned as '*plink' if not NULL. Finally, if the + * search stopped in a compressed node, '*splitpos' returns the index + * inside the compressed node where the search ended. This is useful to + * know where to split the node for insertion. + * + * Note that when we stop in the middle of a compressed node with + * a perfect match, this function will return a length equal to the + * 'len' argument (all the key matched), and will return a *splitpos which is + * always positive (that will represent the index of the character immediately + * *after* the last match in the current compressed node). + * + * When instead we stop at a compressed node and *splitpos is zero, it + * means that the current node represents the key (that is, none of the + * compressed node characters are needed to represent the key, just all + * its parents nodes). */ +static inline size_t raxLowWalk(rax *rax, unsigned char *s, size_t len, raxNode **stopnode, raxNode ***plink, int *splitpos, raxStack *ts) { + raxNode *h = rax->head; + raxNode **parentlink = &rax->head; + + size_t i = 0; /* Position in the string. */ + size_t j = 0; /* Position in the node children (or bytes if compressed).*/ + while(h->size && i < len) { + debugnode("Lookup current node",h); + unsigned char *v = h->data; + + if (h->iscompr) { + for (j = 0; j < h->size && i < len; j++, i++) { + if (v[j] != s[i]) break; + } + if (j != h->size) break; + } else { + /* Even when h->size is large, linear scan provides good + * performances compared to other approaches that are in theory + * more sounding, like performing a binary search. */ + for (j = 0; j < h->size; j++) { + if (v[j] == s[i]) break; + } + if (j == h->size) break; + i++; + } + + if (ts) raxStackPush(ts,h); /* Save stack of parent nodes. */ + raxNode **children = raxNodeFirstChildPtr(h); + if (h->iscompr) j = 0; /* Compressed node only child is at index 0. */ + memcpy(&h,children+j,sizeof(h)); + parentlink = children+j; + j = 0; /* If the new node is non compressed and we do not + iterate again (since i == len) set the split + position to 0 to signal this node represents + the searched key. */ + } + debugnode("Lookup stop node is",h); + if (stopnode) *stopnode = h; + if (plink) *plink = parentlink; + if (splitpos && h->iscompr) *splitpos = j; + return i; +} + +/* Insert the element 's' of size 'len', setting as auxiliary data + * the pointer 'data'. If the element is already present, the associated + * data is updated (only if 'overwrite' is set to 1), and 0 is returned, + * otherwise the element is inserted and 1 is returned. On out of memory the + * function returns 0 as well but sets errno to ENOMEM, otherwise errno will + * be set to 0. + */ +int raxGenericInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old, int overwrite) { + size_t i; + int j = 0; /* Split position. If raxLowWalk() stops in a compressed + node, the index 'j' represents the char we stopped within the + compressed node, that is, the position where to split the + node for insertion. */ + raxNode *h, **parentlink; + + debugf("### Insert %.*s with value %p\n", (int)len, s, data); + i = raxLowWalk(rax,s,len,&h,&parentlink,&j,NULL); + + /* If i == len we walked following the whole string. If we are not + * in the middle of a compressed node, the string is either already + * inserted or this middle node is currently not a key, but can represent + * our key. We have just to reallocate the node and make space for the + * data pointer. */ + if (i == len && (!h->iscompr || j == 0 /* not in the middle if j is 0 */)) { + debugf("### Insert: node representing key exists\n"); + /* Make space for the value pointer if needed. */ + if (!h->iskey || (h->isnull && overwrite)) { + h = raxReallocForData(h,data); + if (h) memcpy(parentlink,&h,sizeof(h)); + } + if (h == NULL) { + errno = ENOMEM; + return 0; + } + + /* Update the existing key if there is already one. */ + if (h->iskey) { + if (old) *old = raxGetData(h); + if (overwrite) raxSetData(h,data); + errno = 0; + return 0; /* Element already exists. */ + } + + /* Otherwise set the node as a key. Note that raxSetData() + * will set h->iskey. */ + raxSetData(h,data); + rax->numele++; + return 1; /* Element inserted. */ + } + + /* If the node we stopped at is a compressed node, we need to + * split it before to continue. + * + * Splitting a compressed node have a few possible cases. + * Imagine that the node 'h' we are currently at is a compressed + * node containing the string "ANNIBALE" (it means that it represents + * nodes A -> N -> N -> I -> B -> A -> L -> E with the only child + * pointer of this node pointing at the 'E' node, because remember that + * we have characters at the edges of the graph, not inside the nodes + * themselves. + * + * In order to show a real case imagine our node to also point to + * another compressed node, that finally points at the node without + * children, representing 'O': + * + * "ANNIBALE" -> "SCO" -> [] + * + * When inserting we may face the following cases. Note that all the cases + * require the insertion of a non compressed node with exactly two + * children, except for the last case which just requires splitting a + * compressed node. + * + * 1) Inserting "ANNIENTARE" + * + * |B| -> "ALE" -> "SCO" -> [] + * "ANNI" -> |-| + * |E| -> (... continue algo ...) "NTARE" -> [] + * + * 2) Inserting "ANNIBALI" + * + * |E| -> "SCO" -> [] + * "ANNIBAL" -> |-| + * |I| -> (... continue algo ...) [] + * + * 3) Inserting "AGO" (Like case 1, but set iscompr = 0 into original node) + * + * |N| -> "NIBALE" -> "SCO" -> [] + * |A| -> |-| + * |G| -> (... continue algo ...) |O| -> [] + * + * 4) Inserting "CIAO" + * + * |A| -> "NNIBALE" -> "SCO" -> [] + * |-| + * |C| -> (... continue algo ...) "IAO" -> [] + * + * 5) Inserting "ANNI" + * + * "ANNI" -> "BALE" -> "SCO" -> [] + * + * The final algorithm for insertion covering all the above cases is as + * follows. + * + * ============================= ALGO 1 ============================= + * + * For the above cases 1 to 4, that is, all cases where we stopped in + * the middle of a compressed node for a character mismatch, do: + * + * Let $SPLITPOS be the zero-based index at which, in the + * compressed node array of characters, we found the mismatching + * character. For example if the node contains "ANNIBALE" and we add + * "ANNIENTARE" the $SPLITPOS is 4, that is, the index at which the + * mismatching character is found. + * + * 1. Save the current compressed node $NEXT pointer (the pointer to the + * child element, that is always present in compressed nodes). + * + * 2. Create "split node" having as child the non common letter + * at the compressed node. The other non common letter (at the key) + * will be added later as we continue the normal insertion algorithm + * at step "6". + * + * 3a. IF $SPLITPOS == 0: + * Replace the old node with the split node, by copying the auxiliary + * data if any. Fix parent's reference. Free old node eventually + * (we still need its data for the next steps of the algorithm). + * + * 3b. IF $SPLITPOS != 0: + * Trim the compressed node (reallocating it as well) in order to + * contain $splitpos characters. Change child pointer in order to link + * to the split node. If new compressed node len is just 1, set + * iscompr to 0 (layout is the same). Fix parent's reference. + * + * 4a. IF the postfix len (the length of the remaining string of the + * original compressed node after the split character) is non zero, + * create a "postfix node". If the postfix node has just one character + * set iscompr to 0, otherwise iscompr to 1. Set the postfix node + * child pointer to $NEXT. + * + * 4b. IF the postfix len is zero, just use $NEXT as postfix pointer. + * + * 5. Set child[0] of split node to postfix node. + * + * 6. Set the split node as the current node, set current index at child[1] + * and continue insertion algorithm as usually. + * + * ============================= ALGO 2 ============================= + * + * For case 5, that is, if we stopped in the middle of a compressed + * node but no mismatch was found, do: + * + * Let $SPLITPOS be the zero-based index at which, in the + * compressed node array of characters, we stopped iterating because + * there were no more keys character to match. So in the example of + * the node "ANNIBALE", adding the string "ANNI", the $SPLITPOS is 4. + * + * 1. Save the current compressed node $NEXT pointer (the pointer to the + * child element, that is always present in compressed nodes). + * + * 2. Create a "postfix node" containing all the characters from $SPLITPOS + * to the end. Use $NEXT as the postfix node child pointer. + * If the postfix node length is 1, set iscompr to 0. + * Set the node as a key with the associated value of the new + * inserted key. + * + * 3. Trim the current node to contain the first $SPLITPOS characters. + * As usually if the new node length is just 1, set iscompr to 0. + * Take the iskey / associated value as it was in the original node. + * Fix the parent's reference. + * + * 4. Set the postfix node as the only child pointer of the trimmed + * node created at step 1. + */ + + /* ------------------------- ALGORITHM 1 --------------------------- */ + if (h->iscompr && i != len) { + debugf("ALGO 1: Stopped at compressed node %.*s (%p)\n", + h->size, h->data, (void*)h); + debugf("Still to insert: %.*s\n", (int)(len-i), s+i); + debugf("Splitting at %d: '%c'\n", j, ((char*)h->data)[j]); + debugf("Other (key) letter is '%c'\n", s[i]); + + /* 1: Save next pointer. */ + raxNode **childfield = raxNodeLastChildPtr(h); + raxNode *next; + memcpy(&next,childfield,sizeof(next)); + debugf("Next is %p\n", (void*)next); + debugf("iskey %d\n", h->iskey); + if (h->iskey) { + debugf("key value is %p\n", raxGetData(h)); + } + + /* Set the length of the additional nodes we will need. */ + size_t trimmedlen = j; + size_t postfixlen = h->size - j - 1; + int split_node_is_key = !trimmedlen && h->iskey && !h->isnull; + size_t nodesize; + + /* 2: Create the split node. Also allocate the other nodes we'll need + * ASAP, so that it will be simpler to handle OOM. */ + raxNode *splitnode = raxNewNode(1, split_node_is_key); + raxNode *trimmed = NULL; + raxNode *postfix = NULL; + + if (trimmedlen) { + nodesize = sizeof(raxNode)+trimmedlen+raxPadding(trimmedlen)+ + sizeof(raxNode*); + if (h->iskey && !h->isnull) nodesize += sizeof(void*); + trimmed = rax_malloc(nodesize); + } + + if (postfixlen) { + nodesize = sizeof(raxNode)+postfixlen+raxPadding(postfixlen)+ + sizeof(raxNode*); + postfix = rax_malloc(nodesize); + } + + /* OOM? Abort now that the tree is untouched. */ + if (splitnode == NULL || + (trimmedlen && trimmed == NULL) || + (postfixlen && postfix == NULL)) + { + rax_free(splitnode); + rax_free(trimmed); + rax_free(postfix); + errno = ENOMEM; + return 0; + } + splitnode->data[0] = h->data[j]; + + if (j == 0) { + /* 3a: Replace the old node with the split node. */ + if (h->iskey) { + void *ndata = raxGetData(h); + raxSetData(splitnode,ndata); + } + memcpy(parentlink,&splitnode,sizeof(splitnode)); + } else { + /* 3b: Trim the compressed node. */ + trimmed->size = j; + memcpy(trimmed->data,h->data,j); + trimmed->iscompr = j > 1 ? 1 : 0; + trimmed->iskey = h->iskey; + trimmed->isnull = h->isnull; + if (h->iskey && !h->isnull) { + void *ndata = raxGetData(h); + raxSetData(trimmed,ndata); + } + raxNode **cp = raxNodeLastChildPtr(trimmed); + memcpy(cp,&splitnode,sizeof(splitnode)); + memcpy(parentlink,&trimmed,sizeof(trimmed)); + parentlink = cp; /* Set parentlink to splitnode parent. */ + rax->numnodes++; + } + + /* 4: Create the postfix node: what remains of the original + * compressed node after the split. */ + if (postfixlen) { + /* 4a: create a postfix node. */ + postfix->iskey = 0; + postfix->isnull = 0; + postfix->size = postfixlen; + postfix->iscompr = postfixlen > 1; + memcpy(postfix->data,h->data+j+1,postfixlen); + raxNode **cp = raxNodeLastChildPtr(postfix); + memcpy(cp,&next,sizeof(next)); + rax->numnodes++; + } else { + /* 4b: just use next as postfix node. */ + postfix = next; + } + + /* 5: Set splitnode first child as the postfix node. */ + raxNode **splitchild = raxNodeLastChildPtr(splitnode); + memcpy(splitchild,&postfix,sizeof(postfix)); + + /* 6. Continue insertion: this will cause the splitnode to + * get a new child (the non common character at the currently + * inserted key). */ + rax_free(h); + h = splitnode; + } else if (h->iscompr && i == len) { + /* ------------------------- ALGORITHM 2 --------------------------- */ + debugf("ALGO 2: Stopped at compressed node %.*s (%p) j = %d\n", + h->size, h->data, (void*)h, j); + + /* Allocate postfix & trimmed nodes ASAP to fail for OOM gracefully. */ + size_t postfixlen = h->size - j; + size_t nodesize = sizeof(raxNode)+postfixlen+raxPadding(postfixlen)+ + sizeof(raxNode*); + if (data != NULL) nodesize += sizeof(void*); + raxNode *postfix = rax_malloc(nodesize); + + nodesize = sizeof(raxNode)+j+raxPadding(j)+sizeof(raxNode*); + if (h->iskey && !h->isnull) nodesize += sizeof(void*); + raxNode *trimmed = rax_malloc(nodesize); + + if (postfix == NULL || trimmed == NULL) { + rax_free(postfix); + rax_free(trimmed); + errno = ENOMEM; + return 0; + } + + /* 1: Save next pointer. */ + raxNode **childfield = raxNodeLastChildPtr(h); + raxNode *next; + memcpy(&next,childfield,sizeof(next)); + + /* 2: Create the postfix node. */ + postfix->size = postfixlen; + postfix->iscompr = postfixlen > 1; + postfix->iskey = 1; + postfix->isnull = 0; + memcpy(postfix->data,h->data+j,postfixlen); + raxSetData(postfix,data); + raxNode **cp = raxNodeLastChildPtr(postfix); + memcpy(cp,&next,sizeof(next)); + rax->numnodes++; + + /* 3: Trim the compressed node. */ + trimmed->size = j; + trimmed->iscompr = j > 1; + trimmed->iskey = 0; + trimmed->isnull = 0; + memcpy(trimmed->data,h->data,j); + memcpy(parentlink,&trimmed,sizeof(trimmed)); + if (h->iskey) { + void *aux = raxGetData(h); + raxSetData(trimmed,aux); + } + + /* Fix the trimmed node child pointer to point to + * the postfix node. */ + cp = raxNodeLastChildPtr(trimmed); + memcpy(cp,&postfix,sizeof(postfix)); + + /* Finish! We don't need to continue with the insertion + * algorithm for ALGO 2. The key is already inserted. */ + rax->numele++; + rax_free(h); + return 1; /* Key inserted. */ + } + + /* We walked the radix tree as far as we could, but still there are left + * chars in our string. We need to insert the missing nodes. */ + while(i < len) { + raxNode *child; + + /* If this node is going to have a single child, and there + * are other characters, so that that would result in a chain + * of single-childed nodes, turn it into a compressed node. */ + if (h->size == 0 && len-i > 1) { + debugf("Inserting compressed node\n"); + size_t comprsize = len-i; + if (comprsize > RAX_NODE_MAX_SIZE) + comprsize = RAX_NODE_MAX_SIZE; + raxNode *newh = raxCompressNode(h,s+i,comprsize,&child); + if (newh == NULL) goto oom; + h = newh; + memcpy(parentlink,&h,sizeof(h)); + parentlink = raxNodeLastChildPtr(h); + i += comprsize; + } else { + debugf("Inserting normal node\n"); + raxNode **new_parentlink; + raxNode *newh = raxAddChild(h,s[i],&child,&new_parentlink); + if (newh == NULL) goto oom; + h = newh; + memcpy(parentlink,&h,sizeof(h)); + parentlink = new_parentlink; + i++; + } + rax->numnodes++; + h = child; + } + raxNode *newh = raxReallocForData(h,data); + if (newh == NULL) goto oom; + h = newh; + if (!h->iskey) rax->numele++; + raxSetData(h,data); + memcpy(parentlink,&h,sizeof(h)); + return 1; /* Element inserted. */ + + oom: + /* This code path handles out of memory after part of the sub-tree was + * already modified. Set the node as a key, and then remove it. However we + * do that only if the node is a terminal node, otherwise if the OOM + * happened reallocating a node in the middle, we don't need to free + * anything. */ + if (h->size == 0) { + h->isnull = 1; + h->iskey = 1; + rax->numele++; /* Compensate the next remove. */ + assert(raxRemove(rax,s,i,NULL) != 0); + } + errno = ENOMEM; + return 0; +} + +/* Overwriting insert. Just a wrapper for raxGenericInsert() that will + * update the element if there is already one for the same key. */ +int raxInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old) { + return raxGenericInsert(rax,s,len,data,old,1); +} + +/* Non overwriting insert function: if an element with the same key + * exists, the value is not updated and the function returns 0. + * This is just a wrapper for raxGenericInsert(). */ +int raxTryInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old) { + return raxGenericInsert(rax,s,len,data,old,0); +} + +/* Find a key in the rax, returns raxNotFound special void pointer value + * if the item was not found, otherwise the value associated with the + * item is returned. */ +void *raxFind(rax *rax, unsigned char *s, size_t len) { + raxNode *h; + + debugf("### Lookup: %.*s\n", (int)len, s); + int splitpos = 0; + size_t i = raxLowWalk(rax,s,len,&h,NULL,&splitpos,NULL); + if (i != len || (h->iscompr && splitpos != 0) || !h->iskey) + return raxNotFound; + return raxGetData(h); +} + +/* Return the memory address where the 'parent' node stores the specified + * 'child' pointer, so that the caller can update the pointer with another + * one if needed. The function assumes it will find a match, otherwise the + * operation is an undefined behavior (it will continue scanning the + * memory without any bound checking). */ +raxNode **raxFindParentLink(raxNode *parent, raxNode *child) { + raxNode **cp = raxNodeFirstChildPtr(parent); + raxNode *c; + while(1) { + memcpy(&c,cp,sizeof(c)); + if (c == child) break; + cp++; + } + return cp; +} + +/* Low level child removal from node. The new node pointer (after the child + * removal) is returned. Note that this function does not fix the pointer + * of the parent node in its parent, so this task is up to the caller. + * The function never fails for out of memory. */ +raxNode *raxRemoveChild(raxNode *parent, raxNode *child) { + debugnode("raxRemoveChild before", parent); + /* If parent is a compressed node (having a single child, as for definition + * of the data structure), the removal of the child consists into turning + * it into a normal node without children. */ + if (parent->iscompr) { + void *data = NULL; + if (parent->iskey) data = raxGetData(parent); + parent->isnull = 0; + parent->iscompr = 0; + parent->size = 0; + if (parent->iskey) raxSetData(parent,data); + debugnode("raxRemoveChild after", parent); + return parent; + } + + /* Otherwise we need to scan for the child pointer and memmove() + * accordingly. + * + * 1. To start we seek the first element in both the children + * pointers and edge bytes in the node. */ + raxNode **cp = raxNodeFirstChildPtr(parent); + raxNode **c = cp; + unsigned char *e = parent->data; + + /* 2. Search the child pointer to remove inside the array of children + * pointers. */ + while(1) { + raxNode *aux; + memcpy(&aux,c,sizeof(aux)); + if (aux == child) break; + c++; + e++; + } + + /* 3. Remove the edge and the pointer by memmoving the remaining children + * pointer and edge bytes one position before. */ + int taillen = parent->size - (e - parent->data) - 1; + debugf("raxRemoveChild tail len: %d\n", taillen); + memmove(e,e+1,taillen); + + /* Compute the shift, that is the amount of bytes we should move our + * child pointers to the left, since the removal of one edge character + * and the corresponding padding change, may change the layout. + * We just check if in the old version of the node there was at the + * end just a single byte and all padding: in that case removing one char + * will remove a whole sizeof(void*) word. */ + size_t shift = ((parent->size+4) % sizeof(void*)) == 1 ? sizeof(void*) : 0; + + /* Move the children pointers before the deletion point. */ + if (shift) + memmove(((char*)cp)-shift,cp,(parent->size-taillen-1)*sizeof(raxNode**)); + + /* Move the remaining "tail" pointers at the right position as well. */ + size_t valuelen = (parent->iskey && !parent->isnull) ? sizeof(void*) : 0; + memmove(((char*)c)-shift,c+1,taillen*sizeof(raxNode**)+valuelen); + + /* 4. Update size. */ + parent->size--; + + /* realloc the node according to the theoretical memory usage, to free + * data if we are over-allocating right now. */ + raxNode *newnode = rax_realloc(parent,raxNodeCurrentLength(parent)); + if (newnode) { + debugnode("raxRemoveChild after", newnode); + } + /* Note: if rax_realloc() fails we just return the old address, which + * is valid. */ + return newnode ? newnode : parent; +} + +/* Remove the specified item. Returns 1 if the item was found and + * deleted, 0 otherwise. */ +int raxRemove(rax *rax, unsigned char *s, size_t len, void **old) { + raxNode *h; + raxStack ts; + + debugf("### Delete: %.*s\n", (int)len, s); + raxStackInit(&ts); + int splitpos = 0; + size_t i = raxLowWalk(rax,s,len,&h,NULL,&splitpos,&ts); + if (i != len || (h->iscompr && splitpos != 0) || !h->iskey) { + raxStackFree(&ts); + return 0; + } + if (old) *old = raxGetData(h); + h->iskey = 0; + rax->numele--; + + /* If this node has no children, the deletion needs to reclaim the + * no longer used nodes. This is an iterative process that needs to + * walk the three upward, deleting all the nodes with just one child + * that are not keys, until the head of the rax is reached or the first + * node with more than one child is found. */ + + int trycompress = 0; /* Will be set to 1 if we should try to optimize the + tree resulting from the deletion. */ + + if (h->size == 0) { + debugf("Key deleted in node without children. Cleanup needed.\n"); + raxNode *child = NULL; + while(h != rax->head) { + child = h; + debugf("Freeing child %p [%.*s] key:%d\n", (void*)child, + (int)child->size, (char*)child->data, child->iskey); + rax_free(child); + rax->numnodes--; + h = raxStackPop(&ts); + /* If this node has more then one child, or actually holds + * a key, stop here. */ + if (h->iskey || (!h->iscompr && h->size != 1)) break; + } + if (child) { + debugf("Unlinking child %p from parent %p\n", + (void*)child, (void*)h); + raxNode *new = raxRemoveChild(h,child); + if (new != h) { + raxNode *parent = raxStackPeek(&ts); + raxNode **parentlink; + if (parent == NULL) { + parentlink = &rax->head; + } else { + parentlink = raxFindParentLink(parent,h); + } + memcpy(parentlink,&new,sizeof(new)); + } + + /* If after the removal the node has just a single child + * and is not a key, we need to try to compress it. */ + if (new->size == 1 && new->iskey == 0) { + trycompress = 1; + h = new; + } + } + } else if (h->size == 1) { + /* If the node had just one child, after the removal of the key + * further compression with adjacent nodes is potentially possible. */ + trycompress = 1; + } + + /* Don't try node compression if our nodes pointers stack is not + * complete because of OOM while executing raxLowWalk() */ + if (trycompress && ts.oom) trycompress = 0; + + /* Recompression: if trycompress is true, 'h' points to a radix tree node + * that changed in a way that could allow to compress nodes in this + * sub-branch. Compressed nodes represent chains of nodes that are not + * keys and have a single child, so there are two deletion events that + * may alter the tree so that further compression is needed: + * + * 1) A node with a single child was a key and now no longer is a key. + * 2) A node with two children now has just one child. + * + * We try to navigate upward till there are other nodes that can be + * compressed, when we reach the upper node which is not a key and has + * a single child, we scan the chain of children to collect the + * compressible part of the tree, and replace the current node with the + * new one, fixing the child pointer to reference the first non + * compressible node. + * + * Example of case "1". A tree stores the keys "FOO" = 1 and + * "FOOBAR" = 2: + * + * + * "FOO" -> "BAR" -> [] (2) + * (1) + * + * After the removal of "FOO" the tree can be compressed as: + * + * "FOOBAR" -> [] (2) + * + * + * Example of case "2". A tree stores the keys "FOOBAR" = 1 and + * "FOOTER" = 2: + * + * |B| -> "AR" -> [] (1) + * "FOO" -> |-| + * |T| -> "ER" -> [] (2) + * + * After the removal of "FOOTER" the resulting tree is: + * + * "FOO" -> |B| -> "AR" -> [] (1) + * + * That can be compressed into: + * + * "FOOBAR" -> [] (1) + */ + if (trycompress) { + debugf("After removing %.*s:\n", (int)len, s); + debugnode("Compression may be needed",h); + debugf("Seek start node\n"); + + /* Try to reach the upper node that is compressible. + * At the end of the loop 'h' will point to the first node we + * can try to compress and 'parent' to its parent. */ + raxNode *parent; + while(1) { + parent = raxStackPop(&ts); + if (!parent || parent->iskey || + (!parent->iscompr && parent->size != 1)) break; + h = parent; + debugnode("Going up to",h); + } + raxNode *start = h; /* Compression starting node. */ + + /* Scan chain of nodes we can compress. */ + size_t comprsize = h->size; + int nodes = 1; + while(h->size != 0) { + raxNode **cp = raxNodeLastChildPtr(h); + memcpy(&h,cp,sizeof(h)); + if (h->iskey || (!h->iscompr && h->size != 1)) break; + /* Stop here if going to the next node would result into + * a compressed node larger than h->size can hold. */ + if (comprsize + h->size > RAX_NODE_MAX_SIZE) break; + nodes++; + comprsize += h->size; + } + if (nodes > 1) { + /* If we can compress, create the new node and populate it. */ + size_t nodesize = + sizeof(raxNode)+comprsize+raxPadding(comprsize)+sizeof(raxNode*); + raxNode *new = rax_malloc(nodesize); + /* An out of memory here just means we cannot optimize this + * node, but the tree is left in a consistent state. */ + if (new == NULL) { + raxStackFree(&ts); + return 1; + } + new->iskey = 0; + new->isnull = 0; + new->iscompr = 1; + new->size = comprsize; + rax->numnodes++; + + /* Scan again, this time to populate the new node content and + * to fix the new node child pointer. At the same time we free + * all the nodes that we'll no longer use. */ + comprsize = 0; + h = start; + while(h->size != 0) { + memcpy(new->data+comprsize,h->data,h->size); + comprsize += h->size; + raxNode **cp = raxNodeLastChildPtr(h); + raxNode *tofree = h; + memcpy(&h,cp,sizeof(h)); + rax_free(tofree); rax->numnodes--; + if (h->iskey || (!h->iscompr && h->size != 1)) break; + } + debugnode("New node",new); + + /* Now 'h' points to the first node that we still need to use, + * so our new node child pointer will point to it. */ + raxNode **cp = raxNodeLastChildPtr(new); + memcpy(cp,&h,sizeof(h)); + + /* Fix parent link. */ + if (parent) { + raxNode **parentlink = raxFindParentLink(parent,start); + memcpy(parentlink,&new,sizeof(new)); + } else { + rax->head = new; + } + + debugf("Compressed %d nodes, %d total bytes\n", + nodes, (int)comprsize); + } + } + raxStackFree(&ts); + return 1; +} + +/* This is the core of raxFree(): performs a depth-first scan of the + * tree and releases all the nodes found. */ +void raxRecursiveFree(rax *rax, raxNode *n, void (*free_callback)(void*)) { + debugnode("free traversing",n); + int numchildren = n->iscompr ? 1 : n->size; + raxNode **cp = raxNodeLastChildPtr(n); + while(numchildren--) { + raxNode *child; + memcpy(&child,cp,sizeof(child)); + raxRecursiveFree(rax,child,free_callback); + cp--; + } + debugnode("free depth-first",n); + if (free_callback && n->iskey && !n->isnull) + free_callback(raxGetData(n)); + rax_free(n); + rax->numnodes--; +} + +/* Free a whole radix tree, calling the specified callback in order to + * free the auxiliary data. */ +void raxFreeWithCallback(rax *rax, void (*free_callback)(void*)) { + raxRecursiveFree(rax,rax->head,free_callback); + assert(rax->numnodes == 0); + rax_free(rax); +} + +/* Free a whole radix tree. */ +void raxFree(rax *rax) { + raxFreeWithCallback(rax,NULL); +} + +/* ------------------------------- Iterator --------------------------------- */ + +/* Initialize a Rax iterator. This call should be performed a single time + * to initialize the iterator, and must be followed by a raxSeek() call, + * otherwise the raxPrev()/raxNext() functions will just return EOF. */ +void raxStart(raxIterator *it, rax *rt) { + it->flags = RAX_ITER_EOF; /* No crash if the iterator is not seeked. */ + it->rt = rt; + it->key_len = 0; + it->key = it->key_static_string; + it->key_max = RAX_ITER_STATIC_LEN; + it->data = NULL; + it->node_cb = NULL; + raxStackInit(&it->stack); +} + +/* Append characters at the current key string of the iterator 'it'. This + * is a low level function used to implement the iterator, not callable by + * the user. Returns 0 on out of memory, otherwise 1 is returned. */ +int raxIteratorAddChars(raxIterator *it, unsigned char *s, size_t len) { + if (len == 0) return 1; + if (it->key_max < it->key_len+len) { + unsigned char *old = (it->key == it->key_static_string) ? NULL : + it->key; + size_t new_max = (it->key_len+len)*2; + it->key = rax_realloc(old,new_max); + if (it->key == NULL) { + it->key = (!old) ? it->key_static_string : old; + errno = ENOMEM; + return 0; + } + if (old == NULL) memcpy(it->key,it->key_static_string,it->key_len); + it->key_max = new_max; + } + /* Use memmove since there could be an overlap between 's' and + * it->key when we use the current key in order to re-seek. */ + memmove(it->key+it->key_len,s,len); + it->key_len += len; + return 1; +} + +/* Remove the specified number of chars from the right of the current + * iterator key. */ +void raxIteratorDelChars(raxIterator *it, size_t count) { + it->key_len -= count; +} + +/* Do an iteration step towards the next element. At the end of the step the + * iterator key will represent the (new) current key. If it is not possible + * to step in the specified direction since there are no longer elements, the + * iterator is flagged with RAX_ITER_EOF. + * + * If 'noup' is true the function starts directly scanning for the next + * lexicographically smaller children, and the current node is already assumed + * to be the parent of the last key node, so the first operation to go back to + * the parent will be skipped. This option is used by raxSeek() when + * implementing seeking a non existing element with the ">" or "<" options: + * the starting node is not a key in that particular case, so we start the scan + * from a node that does not represent the key set. + * + * The function returns 1 on success or 0 on out of memory. */ +int raxIteratorNextStep(raxIterator *it, int noup) { + if (it->flags & RAX_ITER_EOF) { + return 1; + } else if (it->flags & RAX_ITER_JUST_SEEKED) { + it->flags &= ~RAX_ITER_JUST_SEEKED; + return 1; + } + + /* Save key len, stack items and the node where we are currently + * so that on iterator EOF we can restore the current key and state. */ + size_t orig_key_len = it->key_len; + size_t orig_stack_items = it->stack.items; + raxNode *orig_node = it->node; + + while(1) { + int children = it->node->iscompr ? 1 : it->node->size; + if (!noup && children) { + debugf("GO DEEPER\n"); + /* Seek the lexicographically smaller key in this subtree, which + * is the first one found always going towards the first child + * of every successive node. */ + if (!raxStackPush(&it->stack,it->node)) return 0; + raxNode **cp = raxNodeFirstChildPtr(it->node); + if (!raxIteratorAddChars(it,it->node->data, + it->node->iscompr ? it->node->size : 1)) return 0; + memcpy(&it->node,cp,sizeof(it->node)); + /* Call the node callback if any, and replace the node pointer + * if the callback returns true. */ + if (it->node_cb && it->node_cb(&it->node)) + memcpy(cp,&it->node,sizeof(it->node)); + /* For "next" step, stop every time we find a key along the + * way, since the key is lexicographically smaller compared to + * what follows in the sub-children. */ + if (it->node->iskey) { + it->data = raxGetData(it->node); + return 1; + } + } else { + /* If we finished exploring the previous sub-tree, switch to the + * new one: go upper until a node is found where there are + * children representing keys lexicographically greater than the + * current key. */ + while(1) { + int old_noup = noup; + + /* Already on head? Can't go up, iteration finished. */ + if (!noup && it->node == it->rt->head) { + it->flags |= RAX_ITER_EOF; + it->stack.items = orig_stack_items; + it->key_len = orig_key_len; + it->node = orig_node; + return 1; + } + /* If there are no children at the current node, try parent's + * next child. */ + unsigned char prevchild = it->key[it->key_len-1]; + if (!noup) { + it->node = raxStackPop(&it->stack); + } else { + noup = 0; + } + /* Adjust the current key to represent the node we are + * at. */ + int todel = it->node->iscompr ? it->node->size : 1; + raxIteratorDelChars(it,todel); + + /* Try visiting the next child if there was at least one + * additional child. */ + if (!it->node->iscompr && it->node->size > (old_noup ? 0 : 1)) { + raxNode **cp = raxNodeFirstChildPtr(it->node); + int i = 0; + while (i < it->node->size) { + debugf("SCAN NEXT %c\n", it->node->data[i]); + if (it->node->data[i] > prevchild) break; + i++; + cp++; + } + if (i != it->node->size) { + debugf("SCAN found a new node\n"); + raxIteratorAddChars(it,it->node->data+i,1); + if (!raxStackPush(&it->stack,it->node)) return 0; + memcpy(&it->node,cp,sizeof(it->node)); + /* Call the node callback if any, and replace the node + * pointer if the callback returns true. */ + if (it->node_cb && it->node_cb(&it->node)) + memcpy(cp,&it->node,sizeof(it->node)); + if (it->node->iskey) { + it->data = raxGetData(it->node); + return 1; + } + break; + } + } + } + } + } +} + +/* Seek the greatest key in the subtree at the current node. Return 0 on + * out of memory, otherwise 1. This is a helper function for different + * iteration functions below. */ +int raxSeekGreatest(raxIterator *it) { + while(it->node->size) { + if (it->node->iscompr) { + if (!raxIteratorAddChars(it,it->node->data, + it->node->size)) return 0; + } else { + if (!raxIteratorAddChars(it,it->node->data+it->node->size-1,1)) + return 0; + } + raxNode **cp = raxNodeLastChildPtr(it->node); + if (!raxStackPush(&it->stack,it->node)) return 0; + memcpy(&it->node,cp,sizeof(it->node)); + } + return 1; +} + +/* Like raxIteratorNextStep() but implements an iteration step moving + * to the lexicographically previous element. The 'noup' option has a similar + * effect to the one of raxIteratorNextStep(). */ +int raxIteratorPrevStep(raxIterator *it, int noup) { + if (it->flags & RAX_ITER_EOF) { + return 1; + } else if (it->flags & RAX_ITER_JUST_SEEKED) { + it->flags &= ~RAX_ITER_JUST_SEEKED; + return 1; + } + + /* Save key len, stack items and the node where we are currently + * so that on iterator EOF we can restore the current key and state. */ + size_t orig_key_len = it->key_len; + size_t orig_stack_items = it->stack.items; + raxNode *orig_node = it->node; + + while(1) { + int old_noup = noup; + + /* Already on head? Can't go up, iteration finished. */ + if (!noup && it->node == it->rt->head) { + it->flags |= RAX_ITER_EOF; + it->stack.items = orig_stack_items; + it->key_len = orig_key_len; + it->node = orig_node; + return 1; + } + + unsigned char prevchild = it->key[it->key_len-1]; + if (!noup) { + it->node = raxStackPop(&it->stack); + } else { + noup = 0; + } + + /* Adjust the current key to represent the node we are + * at. */ + int todel = it->node->iscompr ? it->node->size : 1; + raxIteratorDelChars(it,todel); + + /* Try visiting the prev child if there is at least one + * child. */ + if (!it->node->iscompr && it->node->size > (old_noup ? 0 : 1)) { + raxNode **cp = raxNodeLastChildPtr(it->node); + int i = it->node->size-1; + while (i >= 0) { + debugf("SCAN PREV %c\n", it->node->data[i]); + if (it->node->data[i] < prevchild) break; + i--; + cp--; + } + /* If we found a new subtree to explore in this node, + * go deeper following all the last children in order to + * find the key lexicographically greater. */ + if (i != -1) { + debugf("SCAN found a new node\n"); + /* Enter the node we just found. */ + if (!raxIteratorAddChars(it,it->node->data+i,1)) return 0; + if (!raxStackPush(&it->stack,it->node)) return 0; + memcpy(&it->node,cp,sizeof(it->node)); + /* Seek sub-tree max. */ + if (!raxSeekGreatest(it)) return 0; + } + } + + /* Return the key: this could be the key we found scanning a new + * subtree, or if we did not find a new subtree to explore here, + * before giving up with this node, check if it's a key itself. */ + if (it->node->iskey) { + it->data = raxGetData(it->node); + return 1; + } + } +} + +/* Seek an iterator at the specified element. + * Return 0 if the seek failed for syntax error or out of memory. Otherwise + * 1 is returned. When 0 is returned for out of memory, errno is set to + * the ENOMEM value. */ +int raxSeek(raxIterator *it, const char *op, unsigned char *ele, size_t len) { + int eq = 0, lt = 0, gt = 0, first = 0, last = 0; + + it->stack.items = 0; /* Just resetting. Initialized by raxStart(). */ + it->flags |= RAX_ITER_JUST_SEEKED; + it->flags &= ~RAX_ITER_EOF; + it->key_len = 0; + it->node = NULL; + + /* Set flags according to the operator used to perform the seek. */ + if (op[0] == '>') { + gt = 1; + if (op[1] == '=') eq = 1; + } else if (op[0] == '<') { + lt = 1; + if (op[1] == '=') eq = 1; + } else if (op[0] == '=') { + eq = 1; + } else if (op[0] == '^') { + first = 1; + } else if (op[0] == '$') { + last = 1; + } else { + errno = 0; + return 0; /* Error. */ + } + + /* If there are no elements, set the EOF condition immediately and + * return. */ + if (it->rt->numele == 0) { + it->flags |= RAX_ITER_EOF; + return 1; + } + + if (first) { + /* Seeking the first key greater or equal to the empty string + * is equivalent to seeking the smaller key available. */ + return raxSeek(it,">=",NULL,0); + } + + if (last) { + /* Find the greatest key taking always the last child till a + * final node is found. */ + it->node = it->rt->head; + if (!raxSeekGreatest(it)) return 0; + assert(it->node->iskey); + it->data = raxGetData(it->node); + return 1; + } + + /* We need to seek the specified key. What we do here is to actually + * perform a lookup, and later invoke the prev/next key code that + * we already use for iteration. */ + int splitpos = 0; + size_t i = raxLowWalk(it->rt,ele,len,&it->node,NULL,&splitpos,&it->stack); + + /* Return OOM on incomplete stack info. */ + if (it->stack.oom) return 0; + + if (eq && i == len && (!it->node->iscompr || splitpos == 0) && + it->node->iskey) + { + /* We found our node, since the key matches and we have an + * "equal" condition. */ + if (!raxIteratorAddChars(it,ele,len)) return 0; /* OOM. */ + it->data = raxGetData(it->node); + } else if (lt || gt) { + /* Exact key not found or eq flag not set. We have to set as current + * key the one represented by the node we stopped at, and perform + * a next/prev operation to seek. */ + raxIteratorAddChars(it, ele, i-splitpos); + + /* We need to set the iterator in the correct state to call next/prev + * step in order to seek the desired element. */ + debugf("After initial seek: i=%d len=%d key=%.*s\n", + (int)i, (int)len, (int)it->key_len, it->key); + if (i != len && !it->node->iscompr) { + /* If we stopped in the middle of a normal node because of a + * mismatch, add the mismatching character to the current key + * and call the iterator with the 'noup' flag so that it will try + * to seek the next/prev child in the current node directly based + * on the mismatching character. */ + if (!raxIteratorAddChars(it,ele+i,1)) return 0; + debugf("Seek normal node on mismatch: %.*s\n", + (int)it->key_len, (char*)it->key); + + it->flags &= ~RAX_ITER_JUST_SEEKED; + if (lt && !raxIteratorPrevStep(it,1)) return 0; + if (gt && !raxIteratorNextStep(it,1)) return 0; + it->flags |= RAX_ITER_JUST_SEEKED; /* Ignore next call. */ + } else if (i != len && it->node->iscompr) { + debugf("Compressed mismatch: %.*s\n", + (int)it->key_len, (char*)it->key); + /* In case of a mismatch within a compressed node. */ + int nodechar = it->node->data[splitpos]; + int keychar = ele[i]; + it->flags &= ~RAX_ITER_JUST_SEEKED; + if (gt) { + /* If the key the compressed node represents is greater + * than our seek element, continue forward, otherwise set the + * state in order to go back to the next sub-tree. */ + if (nodechar > keychar) { + if (!raxIteratorNextStep(it,0)) return 0; + } else { + if (!raxIteratorAddChars(it,it->node->data,it->node->size)) + return 0; + if (!raxIteratorNextStep(it,1)) return 0; + } + } + if (lt) { + /* If the key the compressed node represents is smaller + * than our seek element, seek the greater key in this + * subtree, otherwise set the state in order to go back to + * the previous sub-tree. */ + if (nodechar < keychar) { + if (!raxSeekGreatest(it)) return 0; + it->data = raxGetData(it->node); + } else { + if (!raxIteratorAddChars(it,it->node->data,it->node->size)) + return 0; + if (!raxIteratorPrevStep(it,1)) return 0; + } + } + it->flags |= RAX_ITER_JUST_SEEKED; /* Ignore next call. */ + } else { + debugf("No mismatch: %.*s\n", + (int)it->key_len, (char*)it->key); + /* If there was no mismatch we are into a node representing the + * key, (but which is not a key or the seek operator does not + * include 'eq'), or we stopped in the middle of a compressed node + * after processing all the key. Continue iterating as this was + * a legitimate key we stopped at. */ + it->flags &= ~RAX_ITER_JUST_SEEKED; + if (it->node->iscompr && it->node->iskey && splitpos && lt) { + /* If we stopped in the middle of a compressed node with + * perfect match, and the condition is to seek a key "<" than + * the specified one, then if this node is a key it already + * represents our match. For instance we may have nodes: + * + * "f" -> "oobar" = 1 -> "" = 2 + * + * Representing keys "f" = 1, "foobar" = 2. A seek for + * the key < "foo" will stop in the middle of the "oobar" + * node, but will be our match, representing the key "f". + * + * So in that case, we don't seek backward. */ + it->data = raxGetData(it->node); + } else { + if (gt && !raxIteratorNextStep(it,0)) return 0; + if (lt && !raxIteratorPrevStep(it,0)) return 0; + } + it->flags |= RAX_ITER_JUST_SEEKED; /* Ignore next call. */ + } + } else { + /* If we are here just eq was set but no match was found. */ + it->flags |= RAX_ITER_EOF; + return 1; + } + return 1; +} + +/* Go to the next element in the scope of the iterator 'it'. + * If EOF (or out of memory) is reached, 0 is returned, otherwise 1 is + * returned. In case 0 is returned because of OOM, errno is set to ENOMEM. */ +int raxNext(raxIterator *it) { + if (!raxIteratorNextStep(it,0)) { + errno = ENOMEM; + return 0; + } + if (it->flags & RAX_ITER_EOF) { + errno = 0; + return 0; + } + return 1; +} + +/* Go to the previous element in the scope of the iterator 'it'. + * If EOF (or out of memory) is reached, 0 is returned, otherwise 1 is + * returned. In case 0 is returned because of OOM, errno is set to ENOMEM. */ +int raxPrev(raxIterator *it) { + if (!raxIteratorPrevStep(it,0)) { + errno = ENOMEM; + return 0; + } + if (it->flags & RAX_ITER_EOF) { + errno = 0; + return 0; + } + return 1; +} + +// mask this specific unused function just because it needs yet another lib (math.h) +#if 0 +/* Perform a random walk starting in the current position of the iterator. + * Return 0 if the tree is empty or on out of memory. Otherwise 1 is returned + * and the iterator is set to the node reached after doing a random walk + * of 'steps' steps. If the 'steps' argument is 0, the random walk is performed + * using a random number of steps between 1 and two times the logarithm of + * the number of elements. + * + * NOTE: if you use this function to generate random elements from the radix + * tree, expect a disappointing distribution. A random walk produces good + * random elements if the tree is not sparse, however in the case of a radix + * tree certain keys will be reported much more often than others. At least + * this function should be able to explore every possible element eventually. */ +int raxRandomWalk(raxIterator *it, size_t steps) { + if (it->rt->numele == 0) { + it->flags |= RAX_ITER_EOF; + return 0; + } + + if (steps == 0) { + size_t fle = 1+floor(log(it->rt->numele)); + fle *= 2; + steps = 1 + rand() % fle; + } + + raxNode *n = it->node; + while(steps > 0 || !n->iskey) { + int numchildren = n->iscompr ? 1 : n->size; + int r = rand() % (numchildren+(n != it->rt->head)); + + if (r == numchildren) { + /* Go up to parent. */ + n = raxStackPop(&it->stack); + int todel = n->iscompr ? n->size : 1; + raxIteratorDelChars(it,todel); + } else { + /* Select a random child. */ + if (n->iscompr) { + if (!raxIteratorAddChars(it,n->data,n->size)) return 0; + } else { + if (!raxIteratorAddChars(it,n->data+r,1)) return 0; + } + raxNode **cp = raxNodeFirstChildPtr(n)+r; + if (!raxStackPush(&it->stack,n)) return 0; + memcpy(&n,cp,sizeof(n)); + } + if (n->iskey) steps--; + } + it->node = n; + it->data = raxGetData(it->node); + return 1; +} +#endif + +/* Compare the key currently pointed by the iterator to the specified + * key according to the specified operator. Returns 1 if the comparison is + * true, otherwise 0 is returned. */ +int raxCompare(raxIterator *iter, const char *op, unsigned char *key, size_t key_len) { + int eq = 0, lt = 0, gt = 0; + + if (op[0] == '=' || op[1] == '=') eq = 1; + if (op[0] == '>') gt = 1; + else if (op[0] == '<') lt = 1; + else if (op[1] != '=') return 0; /* Syntax error. */ + + size_t minlen = key_len < iter->key_len ? key_len : iter->key_len; + int cmp = memcmp(iter->key,key,minlen); + + /* Handle == */ + if (lt == 0 && gt == 0) return cmp == 0 && key_len == iter->key_len; + + /* Handle >, >=, <, <= */ + if (cmp == 0) { + /* Same prefix: longer wins. */ + if (eq && key_len == iter->key_len) return 1; + else if (lt) return iter->key_len < key_len; + else if (gt) return iter->key_len > key_len; + else return 0; /* Avoid warning, just 'eq' is handled before. */ + } else if (cmp > 0) { + return gt ? 1 : 0; + } else /* (cmp < 0) */ { + return lt ? 1 : 0; + } +} + +/* Free the iterator. */ +void raxStop(raxIterator *it) { + if (it->key != it->key_static_string) rax_free(it->key); + raxStackFree(&it->stack); +} + +/* Return if the iterator is in an EOF state. This happens when raxSeek() + * failed to seek an appropriate element, so that raxNext() or raxPrev() + * will return zero, or when an EOF condition was reached while iterating + * with raxNext() and raxPrev(). */ +int raxEOF(raxIterator *it) { + return it->flags & RAX_ITER_EOF; +} + +/* Return the number of elements inside the radix tree. */ +uint64_t raxSize(rax *rax) { + return rax->numele; +} + +/* ----------------------------- Introspection ------------------------------ */ + +/* This function is mostly used for debugging and learning purposes. + * It shows an ASCII representation of a tree on standard output, outline + * all the nodes and the contained keys. + * + * The representation is as follow: + * + * "foobar" (compressed node) + * [abc] (normal node with three children) + * [abc]=0x12345678 (node is a key, pointing to value 0x12345678) + * [] (a normal empty node) + * + * Children are represented in new indented lines, each children prefixed by + * the "`-(x)" string, where "x" is the edge byte. + * + * [abc] + * `-(a) "ladin" + * `-(b) [kj] + * `-(c) [] + * + * However when a node has a single child the following representation + * is used instead: + * + * [abc] -> "ladin" -> [] + */ + +/* The actual implementation of raxShow(). */ +void raxRecursiveShow(int level, int lpad, raxNode *n) { + char s = n->iscompr ? '"' : '['; + char e = n->iscompr ? '"' : ']'; + + int numchars = printf("%c%.*s%c", s, n->size, n->data, e); + if (n->iskey) { + numchars += printf("=%p",raxGetData(n)); + } + + int numchildren = n->iscompr ? 1 : n->size; + /* Note that 7 and 4 magic constants are the string length + * of " `-(x) " and " -> " respectively. */ + if (level) { + lpad += (numchildren > 1) ? 7 : 4; + if (numchildren == 1) lpad += numchars; + } + raxNode **cp = raxNodeFirstChildPtr(n); + for (int i = 0; i < numchildren; i++) { + char *branch = " `-(%c) "; + if (numchildren > 1) { + printf("\n"); + for (int j = 0; j < lpad; j++) putchar(' '); + printf(branch,n->data[i]); + } else { + printf(" -> "); + } + raxNode *child; + memcpy(&child,cp,sizeof(child)); + raxRecursiveShow(level+1,lpad,child); + cp++; + } +} + +/* Show a tree, as outlined in the comment above. */ +void raxShow(rax *rax) { + raxRecursiveShow(0,0,rax->head); + putchar('\n'); +} + +/* Used by debugnode() macro to show info about a given node. */ +void raxDebugShowNode(const char *msg, raxNode *n) { + if (raxDebugMsg == 0) return; + printf("%s: %p [%.*s] key:%u size:%u children:", + msg, (void*)n, (int)n->size, (char*)n->data, n->iskey, n->size); + int numcld = n->iscompr ? 1 : n->size; + raxNode **cldptr = raxNodeLastChildPtr(n) - (numcld-1); + while(numcld--) { + raxNode *child; + memcpy(&child,cldptr,sizeof(child)); + cldptr++; + printf("%p ", (void*)child); + } + printf("\n"); + fflush(stdout); +} + +/* Touch all the nodes of a tree returning a check sum. This is useful + * in order to make Valgrind detect if there is something wrong while + * reading the data structure. + * + * This function was used in order to identify Rax bugs after a big refactoring + * using this technique: + * + * 1. The rax-test is executed using Valgrind, adding a printf() so that for + * the fuzz tester we see what iteration in the loop we are in. + * 2. After every modification of the radix tree made by the fuzz tester + * in rax-test.c, we add a call to raxTouch(). + * 3. Now as soon as an operation will corrupt the tree, raxTouch() will + * detect it (via Valgrind) immediately. We can add more calls to narrow + * the state. + * 4. At this point a good idea is to enable Rax debugging messages immediately + * before the moment the tree is corrupted, to see what happens. + */ +unsigned long raxTouch(raxNode *n) { + debugf("Touching %p\n", (void*)n); + unsigned long sum = 0; + if (n->iskey) { + sum += (unsigned long)raxGetData(n); + } + + int numchildren = n->iscompr ? 1 : n->size; + raxNode **cp = raxNodeFirstChildPtr(n); + int count = 0; + for (int i = 0; i < numchildren; i++) { + if (numchildren > 1) { + sum += (long)n->data[i]; + } + raxNode *child; + memcpy(&child,cp,sizeof(child)); + if (child == (void*)0x65d1760) count++; + if (count > 1) exit(1); + sum += raxTouch(child); + cp++; + } + return sum; +} diff --git a/deps/redis/rax.h b/deps/redis/rax.h new file mode 100644 index 0000000..a836b84 --- /dev/null +++ b/deps/redis/rax.h @@ -0,0 +1,216 @@ +/* Rax -- A radix tree implementation. + * + * Copyright (c) 2017-2018, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef RAX_H +#define RAX_H + +#include + +/* Representation of a radix tree as implemented in this file, that contains + * the strings "foo", "foobar" and "footer" after the insertion of each + * word. When the node represents a key inside the radix tree, we write it + * between [], otherwise it is written between (). + * + * This is the vanilla representation: + * + * (f) "" + * \ + * (o) "f" + * \ + * (o) "fo" + * \ + * [t b] "foo" + * / \ + * "foot" (e) (a) "foob" + * / \ + * "foote" (r) (r) "fooba" + * / \ + * "footer" [] [] "foobar" + * + * However, this implementation implements a very common optimization where + * successive nodes having a single child are "compressed" into the node + * itself as a string of characters, each representing a next-level child, + * and only the link to the node representing the last character node is + * provided inside the representation. So the above representation is turned + * into: + * + * ["foo"] "" + * | + * [t b] "foo" + * / \ + * "foot" ("er") ("ar") "foob" + * / \ + * "footer" [] [] "foobar" + * + * However this optimization makes the implementation a bit more complex. + * For instance if a key "first" is added in the above radix tree, a + * "node splitting" operation is needed, since the "foo" prefix is no longer + * composed of nodes having a single child one after the other. This is the + * above tree and the resulting node splitting after this event happens: + * + * + * (f) "" + * / + * (i o) "f" + * / \ + * "firs" ("rst") (o) "fo" + * / \ + * "first" [] [t b] "foo" + * / \ + * "foot" ("er") ("ar") "foob" + * / \ + * "footer" [] [] "foobar" + * + * Similarly after deletion, if a new chain of nodes having a single child + * is created (the chain must also not include nodes that represent keys), + * it must be compressed back into a single node. + * + */ + +#define RAX_NODE_MAX_SIZE ((1<<29)-1) +typedef struct raxNode { + uint32_t iskey:1; /* Does this node contain a key? */ + uint32_t isnull:1; /* Associated value is NULL (don't store it). */ + uint32_t iscompr:1; /* Node is compressed. */ + uint32_t size:29; /* Number of children, or compressed string len. */ + /* Data layout is as follows: + * + * If node is not compressed we have 'size' bytes, one for each children + * character, and 'size' raxNode pointers, point to each child node. + * Note how the character is not stored in the children but in the + * edge of the parents: + * + * [header iscompr=0][abc][a-ptr][b-ptr][c-ptr](value-ptr?) + * + * if node is compressed (iscompr bit is 1) the node has 1 children. + * In that case the 'size' bytes of the string stored immediately at + * the start of the data section, represent a sequence of successive + * nodes linked one after the other, for which only the last one in + * the sequence is actually represented as a node, and pointed to by + * the current compressed node. + * + * [header iscompr=1][xyz][z-ptr](value-ptr?) + * + * Both compressed and not compressed nodes can represent a key + * with associated data in the radix tree at any level (not just terminal + * nodes). + * + * If the node has an associated key (iskey=1) and is not NULL + * (isnull=0), then after the raxNode pointers pointing to the + * children, an additional value pointer is present (as you can see + * in the representation above as "value-ptr" field). + */ + unsigned char data[]; +} raxNode; + +typedef struct rax { + raxNode *head; + uint64_t numele; + uint64_t numnodes; +} rax; + +/* Stack data structure used by raxLowWalk() in order to, optionally, return + * a list of parent nodes to the caller. The nodes do not have a "parent" + * field for space concerns, so we use the auxiliary stack when needed. */ +#define RAX_STACK_STATIC_ITEMS 32 +typedef struct raxStack { + void **stack; /* Points to static_items or an heap allocated array. */ + size_t items, maxitems; /* Number of items contained and total space. */ + /* Up to RAXSTACK_STACK_ITEMS items we avoid to allocate on the heap + * and use this static array of pointers instead. */ + void *static_items[RAX_STACK_STATIC_ITEMS]; + int oom; /* True if pushing into this stack failed for OOM at some point. */ +} raxStack; + +/* Optional callback used for iterators and be notified on each rax node, + * including nodes not representing keys. If the callback returns true + * the callback changed the node pointer in the iterator structure, and the + * iterator implementation will have to replace the pointer in the radix tree + * internals. This allows the callback to reallocate the node to perform + * very special operations, normally not needed by normal applications. + * + * This callback is used to perform very low level analysis of the radix tree + * structure, scanning each possible node (but the root node), or in order to + * reallocate the nodes to reduce the allocation fragmentation (this is the + * Redis application for this callback). + * + * This is currently only supported in forward iterations (raxNext) */ +typedef int (*raxNodeCallback)(raxNode **noderef); + +/* Radix tree iterator state is encapsulated into this data structure. */ +#define RAX_ITER_STATIC_LEN 128 +#define RAX_ITER_JUST_SEEKED (1<<0) /* Iterator was just seeked. Return current + element for the first iteration and + clear the flag. */ +#define RAX_ITER_EOF (1<<1) /* End of iteration reached. */ +#define RAX_ITER_SAFE (1<<2) /* Safe iterator, allows operations while + iterating. But it is slower. */ +typedef struct raxIterator { + int flags; + rax *rt; /* Radix tree we are iterating. */ + unsigned char *key; /* The current string. */ + void *data; /* Data associated to this key. */ + size_t key_len; /* Current key length. */ + size_t key_max; /* Max key len the current key buffer can hold. */ + unsigned char key_static_string[RAX_ITER_STATIC_LEN]; + raxNode *node; /* Current node. Only for unsafe iteration. */ + raxStack stack; /* Stack used for unsafe iteration. */ + raxNodeCallback node_cb; /* Optional node callback. Normally set to NULL. */ +} raxIterator; + +/* A special pointer returned for not found items. */ +extern void *raxNotFound; + +/* Exported API. */ +rax *raxNew(void); +int raxInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old); +int raxTryInsert(rax *rax, unsigned char *s, size_t len, void *data, void **old); +int raxRemove(rax *rax, unsigned char *s, size_t len, void **old); +void *raxFind(rax *rax, unsigned char *s, size_t len); +void raxFree(rax *rax); +void raxFreeWithCallback(rax *rax, void (*free_callback)(void*)); +void raxStart(raxIterator *it, rax *rt); +int raxSeek(raxIterator *it, const char *op, unsigned char *ele, size_t len); +int raxNext(raxIterator *it); +int raxPrev(raxIterator *it); +//int raxRandomWalk(raxIterator *it, size_t steps); +int raxCompare(raxIterator *iter, const char *op, unsigned char *key, size_t key_len); +void raxStop(raxIterator *it); +int raxEOF(raxIterator *it); +void raxShow(rax *rax); +uint64_t raxSize(rax *rax); +unsigned long raxTouch(raxNode *n); +void raxSetDebugMsg(int onoff); + +/* Internal API. May be used by the node callback in order to access rax nodes + * in a low level way, so this function is exported as well. */ +void raxSetData(raxNode *n, void *data); + +#endif diff --git a/deps/redis/rax_malloc.h b/deps/redis/rax_malloc.h new file mode 100644 index 0000000..3bbd2bf --- /dev/null +++ b/deps/redis/rax_malloc.h @@ -0,0 +1,45 @@ +/* Rax -- A radix tree implementation. + * + * Copyright (c) 2017, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/* Allocator selection. + * + * This file is used in order to change the Rax allocator at compile time. + * Just define the following defines to what you want to use. Also add + * the include of your alternate allocator if needed (not needed in order + * to use the default libc allocator). */ + +#ifndef RAX_ALLOC_H +#define RAX_ALLOC_H +//#include "zmalloc.h" +#include +#define rax_malloc malloc +#define rax_realloc realloc +#define rax_free free +#endif diff --git a/deps/redis/stream.h b/deps/redis/stream.h new file mode 100644 index 0000000..86002f7 --- /dev/null +++ b/deps/redis/stream.h @@ -0,0 +1,48 @@ +#ifndef STREAM_H +#define STREAM_H + +#include "listpack.h" + +/* Stream item ID: a 128 bit number composed of a milliseconds time and + * a sequence counter. IDs generated in the same millisecond (or in a past + * millisecond if the clock jumped backward) will use the millisecond time + * of the latest generated ID and an incremented sequence. */ +typedef struct streamID { + uint64_t ms; /* Unix time in milliseconds. */ + uint64_t seq; /* Sequence number. */ +} streamID; + +/* Copied from Redis repo (/src/stream.h) + * We define an iterator to iterate stream items in an abstract way, without + * caring about the radix tree + listpack representation. Technically speaking + * the iterator is only used inside streamReplyWithRange(), so could just + * be implemented inside the function, but practically there is the AOF + * rewriting code that also needs to iterate the stream to emit the XADD + * commands. */ +typedef struct streamIterator { + //stream *stream; /* The stream we are iterating. */ + streamID master_id; /* ID of the master entry at listpack head. */ + uint64_t master_fields_count; /* Master entries # of fields. */ + unsigned char *master_fields_start; /* Master entries start in listpack. */ + unsigned char *master_fields_ptr; /* Master field to emit next. */ + int entry_flags; /* Flags of entry we are emitting. */ + int rev; /* True if iterating end to start (reverse). */ + int skip_tombstones; /* True if not emitting tombstone entries. */ + uint64_t start_key[2]; /* Start key as 128 bit big endian. */ + uint64_t end_key[2]; /* End key as 128 bit big endian. */ +// raxIterator ri; /* Rax iterator. */ + unsigned char *lp; /* Current listpack. */ + unsigned char *lp_ele; /* Current listpack cursor. */ + unsigned char *lp_flags; /* Current entry flags pointer. */ + /* Buffers used to hold the string of lpGet() when the element is + * integer encoded, so that there is no string representation of the + * element inside the listpack itself. */ + unsigned char field_buf[LP_INTBUF_SIZE]; + unsigned char value_buf[LP_INTBUF_SIZE]; +} streamIterator; + +int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields); +void streamIteratorGetField(streamIterator *si, unsigned char **fieldptr, unsigned char **valueptr, int64_t *fieldlen, int64_t *valuelen); +void streamIteratorStart(streamIterator *si, char *raxnodekey, unsigned char *listpack); + +#endif //STREAM_H diff --git a/deps/redis/t_stream.c b/deps/redis/t_stream.c new file mode 100644 index 0000000..fd329c0 --- /dev/null +++ b/deps/redis/t_stream.c @@ -0,0 +1,233 @@ +#include +#include "stream.h" +#include "endianconv.h" +#include "util.h" +#include "assert.h" + + +#define lpGetInteger(ele) lpGetIntegerIfValid(ele, NULL) + +/* Every stream item inside the listpack, has a flags field that is used to + * mark the entry as deleted, or having the same field as the "master" + * entry at the start of the listpack> */ +#define STREAM_ITEM_FLAG_NONE 0 /* No special flags. */ +#define STREAM_ITEM_FLAG_DELETED (1<<0) /* Entry is deleted. Skip it. */ +#define STREAM_ITEM_FLAG_SAMEFIELDS (1<<1) /* Same fields as master entry. */ + +/* This is a wrapper function for lpGet() to directly get an integer value + * from the listpack (that may store numbers as a string), converting + * the string if needed. + * The 'valid" argument is an optional output parameter to get an indication + * if the record was valid, when this parameter is NULL, the function will + * fail with an assertion. */ +static inline int64_t lpGetIntegerIfValid(unsigned char *ele, int *valid) { + int64_t v; + unsigned char *e = lpGet(ele,&v,NULL); + if (e == NULL) { + if (valid) + *valid = 1; + return v; + } + /* The following code path should never be used for how listpacks work: + * they should always be able to store an int64_t value in integer + * encoded form. However the implementation may change. */ + long long ll; + int ret = string2ll((char*)e,v,&ll); + if (valid) + *valid = ret; + else + assert(ret != 0); + v = ll; + return v; +} + +/* Convert the specified stream entry ID as a 128 bit big endian number, so + * that the IDs can be sorted lexicographically. */ +static void streamEncodeID(void *buf, streamID *id) { + uint64_t e[2]; + e[0] = htonu64(id->ms); + e[1] = htonu64(id->seq); + memcpy(buf,e,sizeof(e)); +} + +/* This is the reverse of streamEncodeID(): the decoded ID will be stored + * in the 'id' structure passed by reference. The buffer 'buf' must point + * to a 128 bit big-endian encoded ID. */ +static void streamDecodeID(void *buf, streamID *id) { + uint64_t e[2]; + memcpy(e,buf,sizeof(e)); + id->ms = ntohu64(e[0]); + id->seq = ntohu64(e[1]); +} + +/* Copied from Redis repo (/src/t_stream.c) + * Get the field and value of the current item we are iterating. This should + * be called immediately after streamIteratorGetID(), and for each field + * according to the number of fields returned by streamIteratorGetID(). + * The function populates the field and value pointers and the corresponding + * lengths by reference, that are valid until the next iterator call, assuming + * no one touches the stream meanwhile. */ +void streamIteratorGetField(streamIterator *si, unsigned char **fieldptr, unsigned char **valueptr, int64_t *fieldlen, int64_t *valuelen) { + if (si->entry_flags & STREAM_ITEM_FLAG_SAMEFIELDS) { + *fieldptr = lpGet(si->master_fields_ptr,fieldlen,si->field_buf); + si->master_fields_ptr = lpNext(si->lp,si->master_fields_ptr); + } else { + *fieldptr = lpGet(si->lp_ele,fieldlen,si->field_buf); + si->lp_ele = lpNext(si->lp,si->lp_ele); + } + *valueptr = lpGet(si->lp_ele,valuelen,si->value_buf); + si->lp_ele = lpNext(si->lp,si->lp_ele); +} + +void streamIteratorStart(streamIterator *si, char *raxnodekey, unsigned char *listpack) { + si->start_key[0] = 0; + si->start_key[1] = 0; + si->end_key[0] = UINT64_MAX; + si->end_key[1] = UINT64_MAX; +// si->stream = NULL; + si->lp = listpack; /* There is no current listpack right now. */ + si->lp_ele = NULL; /* Current listpack cursor. */ + si->rev = 0; /* Direction, if non-zero reversed, from end to start. */ + si->skip_tombstones = 1; /* By default tombstones aren't emitted. */ + streamDecodeID(raxnodekey,&si->master_id); /* Get the master ID. */ +} + +/* Return 1 and store the current item ID at 'id' if there are still + * elements within the iteration range, otherwise return 0 in order to + * signal the iteration terminated. */ +int streamIteratorGetID(streamIterator *si, streamID *id, int64_t *numfields) { + while(1) { /* Will stop when element > stop_key or end of radix tree. */ + /* If the current listpack is set to NULL, this is the start of the + * iteration or the previous listpack was completely iterated. + * Go to the next node. */ + if (si->lp == NULL || si->lp_ele == NULL) { + if (!si->lp) /* iterating a single listpack, and got to it's end. */ + return 0; + si->lp_ele = lpFirst(si->lp); /* Seek items count */ + si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek deleted count. */ + si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek num fields. */ + si->master_fields_count = lpGetInteger(si->lp_ele); + si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek first field. */ + si->master_fields_start = si->lp_ele; + /* We are now pointing to the first field of the master entry. + * We need to seek either the first or the last entry depending + * on the direction of the iteration. */ + if (!si->rev) { + /* If we are iterating in normal order, skip the master fields + * to seek the first actual entry. */ + for (uint64_t i = 0; i < si->master_fields_count; i++) + si->lp_ele = lpNext(si->lp,si->lp_ele); + } else { + /* If we are iterating in reverse direction, just seek the + * last part of the last entry in the listpack (that is, the + * fields count). */ + si->lp_ele = lpLast(si->lp); + } + } else if (si->rev) { + /* If we are iterating in the reverse order, and this is not + * the first entry emitted for this listpack, then we already + * emitted the current entry, and have to go back to the previous + * one. */ + int64_t lp_count = lpGetInteger(si->lp_ele); + while(lp_count--) si->lp_ele = lpPrev(si->lp,si->lp_ele); + /* Seek lp-count of prev entry. */ + si->lp_ele = lpPrev(si->lp,si->lp_ele); + } + + /* For every radix tree node, iterate the corresponding listpack, + * returning elements when they are within range. */ + while(1) { + if (!si->rev) { + /* If we are going forward, skip the previous entry + * lp-count field (or in case of the master entry, the zero + * term field) */ + si->lp_ele = lpNext(si->lp,si->lp_ele); + if (si->lp_ele == NULL) { + si->lp = NULL; /* needed for iteration on single listpack. */ + break; + } + } else { + /* If we are going backward, read the number of elements this + * entry is composed of, and jump backward N times to seek + * its start. */ + int64_t lp_count = lpGetInteger(si->lp_ele); + if (lp_count == 0) { /* We reached the master entry. */ + si->lp = NULL; + si->lp_ele = NULL; + break; + } + while(lp_count--) si->lp_ele = lpPrev(si->lp,si->lp_ele); + } + + /* Get the flags entry. */ + si->lp_flags = si->lp_ele; + int64_t flags = lpGetInteger(si->lp_ele); + si->lp_ele = lpNext(si->lp,si->lp_ele); /* Seek ID. */ + + /* Get the ID: it is encoded as difference between the master + * ID and this entry ID. */ + *id = si->master_id; + id->ms += lpGetInteger(si->lp_ele); + si->lp_ele = lpNext(si->lp,si->lp_ele); + id->seq += lpGetInteger(si->lp_ele); + si->lp_ele = lpNext(si->lp,si->lp_ele); + unsigned char buf[sizeof(streamID)]; + streamEncodeID(buf,id); + + /* The number of entries is here or not depending on the flags. */ + if (flags & STREAM_ITEM_FLAG_SAMEFIELDS) { + *numfields = si->master_fields_count; + } else { + *numfields = lpGetInteger(si->lp_ele); + si->lp_ele = lpNext(si->lp,si->lp_ele); + } + assert(*numfields>=0); + + /* If current >= start, and the entry is not marked as + * deleted or tombstones are included, emit it. */ + if (!si->rev) { + if (memcmp(buf,si->start_key,sizeof(streamID)) >= 0 && + (!si->skip_tombstones || !(flags & STREAM_ITEM_FLAG_DELETED))) + { + if (memcmp(buf,si->end_key,sizeof(streamID)) > 0) + return 0; /* We are already out of range. */ + si->entry_flags = flags; + if (flags & STREAM_ITEM_FLAG_SAMEFIELDS) + si->master_fields_ptr = si->master_fields_start; + return 1; /* Valid item returned. */ + } + } else { + if (memcmp(buf,si->end_key,sizeof(streamID)) <= 0 && + (!si->skip_tombstones || !(flags & STREAM_ITEM_FLAG_DELETED))) + { + if (memcmp(buf,si->start_key,sizeof(streamID)) < 0) + return 0; /* We are already out of range. */ + si->entry_flags = flags; + if (flags & STREAM_ITEM_FLAG_SAMEFIELDS) + si->master_fields_ptr = si->master_fields_start; + return 1; /* Valid item returned. */ + } + } + + /* If we do not emit, we have to discard if we are going + * forward, or seek the previous entry if we are going + * backward. */ + if (!si->rev) { + int64_t to_discard = (flags & STREAM_ITEM_FLAG_SAMEFIELDS) ? + *numfields : *numfields*2; + for (int64_t i = 0; i < to_discard; i++) + si->lp_ele = lpNext(si->lp,si->lp_ele); + } else { + int64_t prev_times = 4; /* flag + id ms + id seq + one more to + go back to the previous entry "count" + field. */ + /* If the entry was not flagged SAMEFIELD we also read the + * number of fields, so go back one more. */ + if (!(flags & STREAM_ITEM_FLAG_SAMEFIELDS)) prev_times++; + while(prev_times--) si->lp_ele = lpPrev(si->lp,si->lp_ele); + } + } + + /* End of listpack reached. Try the next/prev radix tree node. */ + } +} diff --git a/deps/redis/util.c b/deps/redis/util.c index 3524577..c8b8df2 100644 --- a/deps/redis/util.c +++ b/deps/redis/util.c @@ -105,6 +105,81 @@ int ll2string(char *dst, size_t dstlen, long long svalue) { return length + negative; } +/* Convert a string into a long long. Returns REDIS_OK if the string could be + * parsed into a (non-overflowing) long long, REDIS_ERR otherwise. The value + * will be set to the parsed value when appropriate. + * + * Note that this function demands that the string strictly represents + * a long long: no spaces or other characters before or after the string + * representing the number are accepted, nor zeroes at the start if not + * for the string "0" representing the zero number. + * + * Because of its strictness, it is safe to use this function to check if + * you can convert a string into a long long, and obtain back the string + * from the number without any loss in the string representation. */ +int string2ll(const char *s, size_t slen, long long *value) { + const char *p = s; + size_t plen = 0; + int negative = 0; + unsigned long long v; + + if (plen == slen) + return 1; + + /* Special case: first and only digit is 0. */ + if (slen == 1 && p[0] == '0') { + if (value != NULL) *value = 0; + return 1; + } + + if (p[0] == '-') { + negative = 1; + p++; plen++; + + /* Abort on only a negative sign. */ + if (plen == slen) + return 1; + } + + /* First digit should be 1-9, otherwise the string should just be 0. */ + if (p[0] >= '1' && p[0] <= '9') { + v = p[0]-'0'; + p++; plen++; + } else if (p[0] == '0' && slen == 1) { + *value = 0; + return 0; + } else { + return 1; + } + + while (plen < slen && p[0] >= '0' && p[0] <= '9') { + if (v > (ULLONG_MAX / 10)) /* Overflow. */ + return 1; + v *= 10; + + if (v > (ULLONG_MAX - (p[0]-'0'))) /* Overflow. */ + return 1; + v += p[0]-'0'; + + p++; plen++; + } + + /* Return if not all bytes were used. */ + if (plen < slen) + return 1; + + if (negative) { + if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */ + return 1; + if (value != NULL) *value = -v; + } else { + if (v > LLONG_MAX) /* Overflow. */ + return 1; + if (value != NULL) *value = v; + } + return 0; +} + unsigned int getEnvVar(const char* varName, unsigned int defaultVal) { const char* envValueStr = getenv(varName); if (envValueStr == NULL) { diff --git a/deps/redis/util.h b/deps/redis/util.h index 39b8162..4c7e541 100644 --- a/deps/redis/util.h +++ b/deps/redis/util.h @@ -18,6 +18,7 @@ * Since it uses %g and not %f, some 40 chars should be enough. */ #define MAX_D2STRING_CHARS 128 +int string2ll(const char *s, size_t slen, long long *value); int ll2string(char *s, size_t len, long long value); int ull2string(char *s, size_t len, unsigned long long value); int lpStringToInt64(const char *s, unsigned long slen, int64_t *value); diff --git a/examples/Makefile b/examples/Makefile index fd4339a..ddcca45 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -12,10 +12,12 @@ SOURCES = $(notdir $(basename $(wildcard *.c))) OBJECTS = $(patsubst %,%.o,$(SOURCES)) TARGETS = $(basename $(SOURCES)) +OPTIMIZATION?=-O3 + STD = -std=c99 STACK = -fstack-protector-all -Wstack-protector WARNS = -Wall -Wextra -pedantic -Werror -CFLAGS = -fPIC -O3 $(STD) $(STACK) $(WARNS) +CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS) DEBUG = -g3 -DDEBUG=1 LIBS = -L /usr/lib -L $(LIB_DIR) -l $(LIB_NAME) -l $(LIB_NAME_EXT) diff --git a/runtests b/runtests index 691d465..63e0e41 100755 --- a/runtests +++ b/runtests @@ -25,7 +25,7 @@ while [[ $# -gt 0 ]]; do esac done -export LD_LIBRARY_PATH="`pwd`/deps/hiredis:`pwd`/lib"${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +export LD_LIBRARY_PATH="`pwd`/lib"${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}:/usr/local/lib if [[ ${VALGRIND} -ne 0 ]]; then valgrind \ diff --git a/src/cli/Makefile b/src/cli/Makefile index 4f010c4..a94ef0a 100644 --- a/src/cli/Makefile +++ b/src/cli/Makefile @@ -1,8 +1,8 @@ default: all -LIB_NAME = rdb LIB_DIR = ../../lib -LIB_NAME_EXT = $(LIB_NAME)-ext +LIB_NAME = librdb.a +LIB_NAME_EXT = librdb-ext.a # Artifacts: TARGET_APP = rdb-cli @@ -12,12 +12,14 @@ SOURCES = $(notdir $(basename $(wildcard *.c))) OBJECTS = $(patsubst %,%.o,$(SOURCES)) TARGETS = $(basename $(SOURCES)) +OPTIMIZATION?=-O3 + STD = -std=c99 STACK = -fstack-protector-all -Wstack-protector WARNS = -Wall -Wextra -pedantic -Werror -CFLAGS = -fPIC -O3 $(STD) $(STACK) $(WARNS) +CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS) DEBUG = -g3 -DDEBUG=1 -LIBS = -L /usr/lib -L $(LIB_DIR) -l $(LIB_NAME) -l $(LIB_NAME_EXT) +LIBS = -L /usr/lib -L $(LIB_DIR) -l:$(LIB_NAME_EXT) -l:$(LIB_NAME) ifeq ($(BUILD_TLS),yes) CFLAGS+=-DUSE_OPENSSL=1 diff --git a/src/cli/rdb-cli.c b/src/cli/rdb-cli.c index 3c4be2b..359d10b 100644 --- a/src/cli/rdb-cli.c +++ b/src/cli/rdb-cli.c @@ -2,6 +2,7 @@ #include #include #include +#include #include /* Rely only on API (and not internal parser headers) */ @@ -19,6 +20,8 @@ typedef struct Options { RdbRes (*formatFunc)(RdbParser *p, int argc, char **argv); } Options; +void loggerWrap(RdbLogLevel l, const char *msg, ...); + static int getOptArg(int argc, char* argv[], int *at, char *abbrvOpt, char *opt, int *flag, const char **arg) { if ((strcmp(argv[*at], abbrvOpt) == 0) || (strcmp(argv[*at], opt) == 0)) { if (arg) { @@ -43,7 +46,7 @@ static int getOptArgVal(int argc, char* argv[], int *at, char *abbrvOpt, char *o /* check boundaries. Condition support also the limits INT_MAX and INT_MIN. */ if (!((*val>=min) && (*val <=max))) { - fprintf(stderr, "Value of %s (%s) must be a integer between %d and %d", opt, abbrvOpt, min, max); + loggerWrap(RDB_LOG_ERR, "Value of %s (%s) must be a integer between %d and %d", opt, abbrvOpt, min, max); exit(RDB_ERR_GENERAL); } return 1; @@ -84,17 +87,17 @@ static void printUsage(int shortUsage) { printf("[v%s] ", RDB_getLibVersion(NULL,NULL,NULL)); printf("Usage: rdb-cli /path/to/dump.rdb [OPTIONS] {json|resp|redis} [FORMAT_OPTIONS]\n"); printf("OPTIONS:\n"); - printf("\t-l, --log-file Path to the log file (Default: './rdb-cli.log')\n\n"); + printf("\t-l, --log-file Path to the log file or stdout (Default: './rdb-cli.log')\n\n"); printf("\tMultiple filters combination of keys/types/dbs can be specified:\n"); printf("\t-k, --key Include only keys that match REGEX\n"); - printf("\t-K --no-key Exclude keys that match REGEX\n"); + printf("\t-K --no-key Exclude all keys that match REGEX\n"); printf("\t-t, --type Include only selected TYPE {str|list|set|zset|hash|module|func}\n"); printf("\t-T, --no-type Exclude TYPE {str|list|set|zset|hash|module|func}\n"); printf("\t-d, --dbnum Include only selected db number\n"); printf("\t-D, --no-dbnum Exclude DB number\n\n"); printf("FORMAT_OPTIONS ('json'):\n"); - printf("\t-i, --include To include: {aux-val|func}\n"); + printf("\t-i, --include To include: {aux-val|func|stream-meta}\n"); printf("\t-f, --flatten Print flatten json, without DBs Parenthesis\n"); printf("\t-o, --output Specify the output file. If not specified, output to stdout\n\n"); @@ -108,6 +111,7 @@ static void printUsage(int shortUsage) { printf("FORMAT_OPTIONS ('redis'|'resp'):\n"); printf("\t-r, --support-restore Use the RESTORE command when possible\n"); + printf("\t-d, --del-before-write Delete each key before writing. Relevant for non-empty db\n"); printf("\t-t, --target-redis-ver Specify the target Redis version. Helps determine which commands can\n"); printf("\t be applied. Particularly crucial if support-restore being used \n"); printf("\t as RESTORE is closely tied to specific RDB versions. If versions not\n"); @@ -122,7 +126,7 @@ static void printUsage(int shortUsage) { static RdbRes formatJson(RdbParser *parser, int argc, char **argv) { const char *includeArg; const char *output = NULL;/*default:stdout*/ - int includeFunc=0, includeAuxField=0, flatten=0; /*without*/ + int includeStreamMeta=0, includeFunc=0, includeAuxField=0, flatten=0; /*without*/ /* parse specific command options */ for (int at = 1; at < argc; ++at) { @@ -133,11 +137,12 @@ static RdbRes formatJson(RdbParser *parser, int argc, char **argv) { if (getOptArg(argc, argv, &at, "-i", "--include", NULL, &includeArg)) { if (strcmp(includeArg, "aux-val") == 0) { includeAuxField = 1; continue; } if (strcmp(includeArg, "func") == 0) { includeFunc = 1; continue; } - fprintf(stderr, "Invalid argument for '--include': %s\n", includeArg); + if (strcmp(includeArg, "stream-meta") == 0) { includeStreamMeta = 1; continue; } + loggerWrap(RDB_LOG_ERR, "Invalid argument for '--include': %s\n", includeArg); return RDB_ERR_GENERAL; } - fprintf(stderr, "Invalid 'json' [FORMAT_OPTIONS] argument: %s\n", opt); + loggerWrap(RDB_LOG_ERR, "Invalid 'json' [FORMAT_OPTIONS] argument: %s\n", opt); printUsage(1); return RDB_ERR_GENERAL; } @@ -148,6 +153,7 @@ static RdbRes formatJson(RdbParser *parser, int argc, char **argv) { .flatten = flatten, .includeAuxField = includeAuxField, .includeFunc = includeFunc, + .includeStreamMeta = includeStreamMeta, }; if (RDBX_createHandlersToJson(parser, output, &conf) == NULL) @@ -172,6 +178,7 @@ static RdbRes formatRedis(RdbParser *parser, int argc, char **argv) { if (getOptArg(argc, argv, &at, "-h", "--hostname", NULL, &hostname)) continue; if (getOptArgVal(argc, argv, &at, "-p", "--port", NULL, &port, 1, 65535)) continue; if (getOptArg(argc, argv, &at, "-r", "--support-restore", &(conf.supportRestore), NULL)) continue; + if (getOptArg(argc, argv, &at, "-d", "--del-before-write", &(conf.delKeyBeforeWrite), NULL)) continue; if (getOptArg(argc, argv, &at, "-t", "--target-redis-ver", NULL, &(conf.dstRedisVersion))) continue; if (getOptArgVal(argc, argv, &at, "-l", "--pipeline-depth", NULL, &pipeDepthVal, 1, 1000)) continue; if (getOptArgVal(argc, argv, &at, "-n", "--start-cmd-num", NULL, &startCmdNum, 1, INT_MAX)) continue; @@ -181,7 +188,7 @@ static RdbRes formatRedis(RdbParser *parser, int argc, char **argv) { if (getOptArgVal(argc, argv, &at, "-a", "--auth", NULL, &(auth.cmd.argc), 1, INT_MAX)) { auth.cmd.argv = argv + at + 1; if ((1 + at + auth.cmd.argc) >= argc) { - fprintf(stderr, "Insufficient number of arguments to option --auth\n"); + loggerWrap(RDB_LOG_ERR, "Insufficient number of arguments to option --auth\n"); printUsage(1); return RDB_ERR_GENERAL; } @@ -189,13 +196,13 @@ static RdbRes formatRedis(RdbParser *parser, int argc, char **argv) { continue; } - fprintf(stderr, "Invalid 'redis' [FORMAT_OPTIONS] argument: %s\n", opt); + loggerWrap(RDB_LOG_ERR, "Invalid 'redis' [FORMAT_OPTIONS] argument: %s\n", opt); printUsage(1); return RDB_ERR_GENERAL; } if (((auth.user) || (auth.pwd)) && (auth.cmd.argc > 0)) { - fprintf(stderr, "Invalid AUTH arguments. --auth(-a) is mutually exclusive with --password(-P) and --user(-u)\n"); + loggerWrap(RDB_LOG_ERR, "Invalid AUTH arguments. --auth(-a) is mutually exclusive with --password(-P) and --user(-u)\n"); return RDB_ERR_GENERAL; } @@ -244,11 +251,12 @@ static RdbRes formatResp(RdbParser *parser, int argc, char **argv) { char *opt = argv[at]; if (getOptArg(argc, argv, &at, "-o", "--output", NULL, &output)) continue; if (getOptArg(argc, argv, &at, "-r", "--support-restore", &(conf.supportRestore), NULL)) continue; + if (getOptArg(argc, argv, &at, "-d", "--del-before-write", &(conf.delKeyBeforeWrite), NULL)) continue; if (getOptArg(argc, argv, &at, "-t", "--target-redis-ver", NULL, &(conf.dstRedisVersion))) continue; if (getOptArg(argc, argv, &at, "-e", "--enum-commands", &commandEnum, NULL)) continue; if (getOptArgVal(argc, argv, &at, "-n", "--start-cmd-num", NULL, &startCmdNum, 1, INT_MAX)) continue; - fprintf(stderr, "Invalid 'resp' [FORMAT_OPTIONS] argument: %s\n", opt); + loggerWrap(RDB_LOG_ERR, "Invalid 'resp' [FORMAT_OPTIONS] argument: %s\n", opt); printUsage(1); return RDB_ERR_GENERAL; } @@ -278,8 +286,8 @@ int matchRdbDataType(const char *dataTypeStr) { if (!strcmp(dataTypeStr, "stream")) return RDB_DATA_TYPE_STREAM; if (!strcmp(dataTypeStr, "func")) return RDB_DATA_TYPE_FUNCTION; - fprintf(stderr, "Invalid TYPE argument (%s). Valid values: str, list, set, zset, hash, module, stream, func", - dataTypeStr); + loggerWrap(RDB_LOG_ERR, "Invalid TYPE argument (%s). Valid values: str, list, set, zset, hash, module, stream, func", + dataTypeStr); exit(RDB_ERR_GENERAL); } @@ -339,7 +347,7 @@ int readCommonOptions(RdbParser *p, int argc, char* argv[], Options *options, in else if (strcmp(opt, "resp") == 0) { options->formatFunc = formatResp; break; } else if (strcmp(opt, "redis") == 0) { options->formatFunc = formatRedis; break; } - fprintf(stderr, "At argv[%d], unexpected OPTIONS argument: %s\n", at, opt); + loggerWrap(RDB_LOG_ERR, "At argv[%d], unexpected OPTIONS argument: %s\n", at, opt); printUsage(1); exit(RDB_ERR_GENERAL); } @@ -373,7 +381,7 @@ int main(int argc, char **argv) } if ((logfile = fopen(options.logfilePath, "w")) == NULL) { - printf("Error opening log file for writing: %s \n", options.logfilePath); + printf("Error opening log file for writing `%s`: %s\n", options.logfilePath, strerror(errno)); return RDB_ERR_GENERAL; } @@ -405,6 +413,8 @@ int main(int argc, char **argv) return RDB_getErrorCode(parser); RDB_deleteParser(parser); + fclose(logfile); + return 0; } diff --git a/src/ext/Makefile b/src/ext/Makefile index 51e218d..7ebdcc2 100644 --- a/src/ext/Makefile +++ b/src/ext/Makefile @@ -13,11 +13,20 @@ TARGET_LIB_STATIC_EXT = $(LIB_DIR)/lib$(LIB_NAME_EXT).a SOURCES = $(notdir $(basename $(wildcard *.c))) OBJECTS = $(patsubst %,%.o,$(SOURCES)) +# Source files in deps/redis directory. For now, librdb.so and librdb-ext.so, +# each will have its own copy. Take care not to pass Redis structs from one lib +# to another! +REDIS_SOURCES = $(notdir $(basename $(wildcard ../../deps/redis/*.c))) +REDIS_OBJECTS = $(patsubst %,../../deps/redis/%.o,$(REDIS_SOURCES)) + +OPTIMIZATION?=-O3 +LIBRDB_DEBUG?=0 + STD = -std=c99 STACK = -fstack-protector-all -Wstack-protector WARNS = -Wall -Wextra -pedantic -Werror -CFLAGS = -fPIC -O3 $(STD) $(STACK) $(WARNS) $(EXTERN_RP_CONFIG) -DEBUG = -g3 -DDEBUG=1 +CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS) -fvisibility=hidden +DEBUG = -g3 -DLIBRDB_DEBUG=$(LIBRDB_DEBUG) LDFLAGS = LIBS = -L $(LIB_DIR) -l $(LIB_NAME) @@ -28,13 +37,14 @@ endif all: $(TARGET_LIB_EXT) $(TARGET_LIB_STATIC_EXT) @echo "Done."; -$(TARGET_LIB_EXT): $(OBJECTS) +$(TARGET_LIB_EXT): $(OBJECTS) $(REDIS_OBJECTS) $(CC) -o $@ -shared ${LDFLAGS} $^ $(LIBS) -$(TARGET_LIB_STATIC_EXT): $(OBJECTS) +$(TARGET_LIB_STATIC_EXT): $(OBJECTS) $(REDIS_OBJECTS) ar rcs $@ $^ --include $(OBJECTS:.o=.d) +# Include object file dependencies +-include $(OBJECTS:.o=.d) $(REDIS_OBJECTS:.o=.d) %.o: %.c $(CC) $(CFLAGS) -c $*.c -o $*.o $(DEBUG) diff --git a/src/ext/common.c b/src/ext/common.c new file mode 100644 index 0000000..9b86a12 --- /dev/null +++ b/src/ext/common.c @@ -0,0 +1,24 @@ +#include "common.h" +#include "../../deps/redis/util.h" + +/* Example:: Input: length=123 return: buf="\r\n$123\r\n" */ +void iov_length(struct iovec *iov, long long length, char *buf, int bufsize) { + int len = 0; + buf[0] = '\r'; + buf[1] = '\n'; + buf[2] = '$'; + len = ll2string(buf+3, bufsize-5, length); + buf[len + 3] = '\r'; + buf[len + 4] = '\n'; + iov_plain(iov, buf, len+5); +} + +/* For value 123 the function will return in buf: "123\r\n" */ +int iov_value(struct iovec *iov, long long value, char *buf, int bufsize) { + int len = 0; + len = ll2string(buf, bufsize-2, value); /* -2 for: 'r' and '\n' */ + buf[len] = '\r'; + buf[len+1] = '\n'; + iov_plain(iov, buf, len+2); + return len; +} diff --git a/src/ext/common.h b/src/ext/common.h index 7a6f8e3..a51bd28 100644 --- a/src/ext/common.h +++ b/src/ext/common.h @@ -19,18 +19,33 @@ static inline void unused(void *dummy, ...) { (void)(dummy);} #define unlikely(x) (x) #endif +#define IF_NOT_OK_RETURN(cmd) do {RdbRes s; s = cmd; if (unlikely(s!=RDB_OK)) return s;} while (0) + /*** IOVEC manipulation ***/ +// INPUT: str="ABC" OUTPUT: iov={ "ABC" , 3 } #define IOV_CONST(iov, str) iov_plain(iov, str, sizeof(str)-1) + +// INPUT: str="ABC", len=3 OUTPUT: iov={ "ABC" , 3 } #define IOV_STRING(iov, str, len) iov_plain(iov, str, len) + +// INPUT: len=45678 OUTPUT: iov={ "\r\n$45678\r\n" , 10 } +#define IOV_LENGTH(iov, len, ar) iov_length(iov, len, ar, sizeof(ar)) + +// INPUT: val=45678 OUTPUT: iov={ "45678\r\n" , 7 } #define IOV_VALUE(iov, val, ar) iov_value(iov, val, ar, sizeof(ar)) -#define IOV_LEN_AND_VALUE(iov, val, ar1, ar2) \ + +// INPUT: val=45678 OUTPUT: iov={{ "$5\r\n" , 4 } , { "45678\r\n" , 7 }} +#define IOV_LEN_AND_VAL(iov, val, ar1, ar2) \ do {\ int l = IOV_VALUE((iov)+1, val, ar2); \ - IOV_VALUE( (iov), l, ar1); \ + IOV_LENGTH( (iov), l, ar1); \ } while (0); int iov_value(struct iovec *iov, long long count, char *buf, int bufsize); -inline void iov_plain(struct iovec *iov, const char *s, size_t l) { + +void iov_length(struct iovec *iov, long long length, char *buf, int bufsize); + +static inline void iov_plain(struct iovec *iov, const char *s, size_t l) { iov->iov_base = (void *) s; iov->iov_len = l; } diff --git a/src/ext/handlersFilter.c b/src/ext/handlersFilter.c index 6763f4b..64a1647 100644 --- a/src/ext/handlersFilter.c +++ b/src/ext/handlersFilter.c @@ -125,6 +125,36 @@ static RdbRes filterHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk va return ((RdbxFilter *) userData)->cbReturnValue; } +static RdbRes filterStreamMetadata(RdbParser *p, void *userData, RdbStreamMeta *meta) { + UNUSED(p, meta); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterStreamItem(RdbParser *p, void *userData, RdbStreamID *id, RdbBulk field, RdbBulk value, int64_t itemsLeft) { + UNUSED(p, id, field, value, itemsLeft); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterStreamNewCGroup(RdbParser *p, void *userData, RdbBulk grpName, RdbStreamGroupMeta *meta) { + UNUSED(p, grpName, meta); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterStreamCGroupPendingEntry(RdbParser *p, void *userData, RdbStreamPendingEntry *pendingEntry) { + UNUSED(p, pendingEntry); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterStreamNewConsumer(RdbParser *p, void *userData, RdbBulk consName, RdbStreamConsumerMeta *meta) { + UNUSED(p, consName, meta); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterStreamConsumerPendingEntry(RdbParser *p, void *userData, RdbStreamID *streamId) { + UNUSED(p, streamId); + return ((RdbxFilter *) userData)->cbReturnValue; +} + /*** Handling struct ***/ static RdbRes filterListLP(RdbParser *p, void *userData, RdbBulk listpack) { @@ -167,6 +197,11 @@ static RdbRes filterSetMember(RdbParser *p, void *userData, RdbBulk member) { return ((RdbxFilter *) userData)->cbReturnValue; } +static RdbRes filterZsetMember(RdbParser *p, void *userData, RdbBulk member, double score) { + UNUSED(p, userData, member, score); + return ((RdbxFilter *) userData)->cbReturnValue; +} + static RdbRes filterSetPlain(RdbParser *p, void *userData, RdbBulk item) { UNUSED(p, item); return ((RdbxFilter *) userData)->cbReturnValue; @@ -182,6 +217,21 @@ static RdbRes filterSetLP(RdbParser *p, void *userData, RdbBulk listpack) { return ((RdbxFilter *) userData)->cbReturnValue; } +static RdbRes filterZsetPlain(RdbParser *p, void *userData, RdbBulk item, double score) { + UNUSED(p, item, score); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterZsetZL(RdbParser *p, void *userData, RdbBulk ziplist) { + UNUSED(p, ziplist); + return ((RdbxFilter *) userData)->cbReturnValue; +} + +static RdbRes filterZsetLP(RdbParser *p, void *userData, RdbBulk listpack) { + UNUSED(p, listpack); + return ((RdbxFilter *) userData)->cbReturnValue; +} + static RdbRes filterFunction(RdbParser *p, void *userData, RdbBulk func) { UNUSED(p, func); return ((RdbxFilter *) userData)->cbReturnValue; @@ -192,6 +242,11 @@ static RdbRes filterModule(RdbParser *p, void *userData, RdbBulk moduleName, siz return ((RdbxFilter *) userData)->cbReturnValue; } +static RdbRes filterStreamLP(RdbParser *p, void *userData, RdbBulk nodekey, RdbBulk streamLP) { + UNUSED(p, nodekey, streamLP); + return ((RdbxFilter *) userData)->cbReturnValue; +} + /*** Handling raw ***/ static RdbRes filterFrag(RdbParser *p, void *userData, RdbBulk frag) { @@ -212,63 +267,79 @@ static RdbRes filterRawEnd(RdbParser *p, void *userData) { /*** common init ***/ static void defaultFilterDataCb(RdbHandlersDataCallbacks *dataCb) { - memset(dataCb, 0, sizeof(*dataCb)); - dataCb->handleNewKey = filterNewKey; - dataCb->handleEndKey = filterEndKey; - dataCb->handleNewDb = filterNewDb; - dataCb->handleDbSize = filterDbSize; - - dataCb->handleStringValue = filterString; - dataCb->handleListItem = filterList; - dataCb->handleHashField = filterHash; - dataCb->handleSetMember = filterSetMember; - dataCb->handleFunction = filterFunction; - dataCb->handleModule = filterModule; + static const RdbHandlersDataCallbacks defDataCb = { + NULL, /*handleStartRdb*/ + NULL, /*handleEndRdb*/ + filterNewDb, /*handleNewDb*/ + filterDbSize, /*handleDbSize*/ + NULL, /*handleSlotInfo*/ + NULL, /*handleAuxField*/ + filterNewKey, /*handleNewKey*/ + filterEndKey, /*handleEndKey*/ + filterString, /*handleStringValue*/ + filterList, /*handleListItem*/ + filterHash, /*handleHashField*/ + filterSetMember, /*handleSetMember*/ + filterZsetMember, /*handleZsetMember*/ + filterFunction, /*handleFunction*/ + filterModule, /*handleModule*/ + filterStreamMetadata, /*handleStreamMetadata*/ + filterStreamItem, /*handleStreamItem*/ + filterStreamNewCGroup, /*handleStreamNewCGroup*/ + filterStreamCGroupPendingEntry, /*handleStreamCGroupPendingEntry*/ + filterStreamNewConsumer, /*handleStreamNewConsumer*/ + filterStreamConsumerPendingEntry, /*handleStreamConsumerPendingEntry*/ + }; + *dataCb = defDataCb; } static void defaultFilterStructCb(RdbHandlersStructCallbacks *structCb) { - memset(structCb, 0, sizeof(*structCb)); - /* common */ - structCb->handleNewKey = filterNewKey; - structCb->handleEndKey = filterEndKey; - structCb->handleNewDb = filterNewDb; - structCb->handleDbSize = filterDbSize; - - /* string */ - structCb->handleString = filterString; - /* list */ - structCb->handleListLP = filterListLP; - structCb->handleListZL = filterListZL; - structCb->handleListPlain = filterListPlain; - /* hash */ - structCb->handleHashPlain = filterHashPlain; - structCb->handleHashZL = filterHashZL; - structCb->handleHashLP = filterHashLP; - structCb->handleHashZM = filterHashZM; - - /* set */ - structCb->handleSetPlain = filterSetPlain; - structCb->handleHashZM = filterSetIS; - structCb->handleSetLP = filterSetLP; - - /* func */ - structCb->handleFunction = filterFunction; - /* module */ - structCb->handleModule = filterModule; + static const RdbHandlersStructCallbacks defStructCb = { + NULL, /*handleStartRdb*/ + NULL, /*handleEndRdb*/ + filterNewDb, /*handleNewDb*/ + filterDbSize, /*handleDbSize*/ + NULL, /*handleSlotInfo*/ + NULL, /*handleAuxField*/ + filterNewKey, /*handleNewKey*/ + filterEndKey, /*handleEndKey*/ + filterString, /*handleString*/ + filterListPlain, /*handleListPlain*/ + filterListZL, /*handleListZL*/ + filterListLP, /*handleListLP*/ + filterHashPlain, /*handleHashPlain*/ + filterHashZL, /*handleHashZL*/ + filterHashLP, /*handleHashLP*/ + filterHashZM, /*handleHashZM*/ + filterSetPlain, /*handleSetPlain*/ + filterSetIS, /*handleSetIS*/ + filterSetLP, /*handleSetLP*/ + filterZsetPlain, /*handleZsetPlain*/ + filterZsetZL, /*handleZsetZL*/ + filterZsetLP, /*handleZsetLP*/ + filterFunction, /*handleFunction*/ + filterModule, /*handleModule*/ + filterStreamLP, /*handleStreamLP*/ + }; + *structCb = defStructCb; } static void defaultFilterRawCb(RdbHandlersRawCallbacks *rawCb) { - memset(rawCb, 0, sizeof(*rawCb)); - /* common */ - rawCb->handleNewKey = filterNewKey; - rawCb->handleEndKey = filterEndKey; - rawCb->handleNewDb = filterNewDb; - rawCb->handleDbSize = filterDbSize; - - //callbacks.rawCb.handleBeginModuleAux /* not part of keyspace */ - rawCb->handleBegin = filterRawBegin; - rawCb->handleFrag = filterFrag; - rawCb->handleEnd = filterRawEnd; + static const RdbHandlersRawCallbacks defRawCb = { + NULL, /*handleStartRdb*/ + NULL, /*handleEndRdb*/ + filterNewDb, /*handleNewDb*/ + filterDbSize, /*handleDbSize*/ + NULL, /*handleSlotInfo*/ + NULL, /*handleAuxField*/ + filterNewKey, /*handleNewKey*/ + filterEndKey, /*handleEndKey*/ + NULL, /*handleBeginModuleAux (Not part of keyspace)*/ + filterRawBegin, /*handleBegin*/ + filterFrag, /*handleFrag*/ + filterRawEnd /*handleEnd*/ + }; + *rawCb = defRawCb; } static RdbxFilter *createHandlersFilterCommon(RdbParser *p, diff --git a/src/ext/handlersToJson.c b/src/ext/handlersToJson.c index 2642045..824ac86 100644 --- a/src/ext/handlersToJson.c +++ b/src/ext/handlersToJson.c @@ -3,12 +3,10 @@ #include #include #include "common.h" - #include "../../deps/redis/util.h" struct RdbxToJson; -#define _RDB_TYPE_MODULE_2 7 #define _STDOUT_STR "" typedef enum @@ -22,7 +20,17 @@ typedef enum R2J_IN_SET, R2J_IN_STRING, R2J_IN_HASH, - R2J_IN_ZSET + R2J_IN_ZSET, + + /* Possible states in R2J_IN_STREAM */ + R2J_IN_STREAM, + R2J_IN_STREAM_ENTRIES, + R2J_IN_STREAM_ENTRIES_PAIRS, + R2J_IN_STREAM_CG, + R2J_IN_STREAM_CG_PEL, + R2J_IN_STREAM_CG_CONSUMER, + R2J_IN_STREAM_CG_CONSUMER_PEL, + } RdbxToJsonState; struct RdbxToJson { @@ -56,9 +64,7 @@ static void outputPlainEscaping(RdbxToJson *ctx, char *p, size_t len) { case '\t': fprintf(ctx->outfile, "\\t"); break; case '\b': fprintf(ctx->outfile, "\\b"); break; default: - /* TODO: formalize rdb2json supported outputs */ - //fprintf(ctx->outfile, (*p >= 0 && *p <= 0x1f) ? "\\u%04x" : "%c",*p); - fprintf(ctx->outfile, (isprint(*p)) ? "%c" : "\\x%02x", (unsigned char)*p); + fprintf(ctx->outfile, (isprint(*p)) ? "%c" : "\\u%04x", (unsigned char)*p); } p++; } @@ -73,8 +79,7 @@ static void outputQuotedEscaping(RdbxToJson *ctx, char *data, size_t len) { static void deleteRdbToJsonCtx(RdbParser *p, void *data) { RdbxToJson *ctx = (RdbxToJson *) data; - if (ctx->keyCtx.key) - RDB_bulkCopyFree(p, ctx->keyCtx.key); + RDB_bulkCopyFree(p, ctx->keyCtx.key); RDB_log(p, RDB_LOG_DBG, "handlersToJson: Closing file %s", ctx->outfileName); @@ -114,6 +119,7 @@ static RdbxToJson *initRdbToJsonCtx(RdbParser *p, const char *outfilename, RdbxT ctx->conf.level = RDB_LEVEL_DATA; ctx->conf.includeAuxField = 0; ctx->conf.includeFunc = 0; + ctx->conf.includeStreamMeta = 0; /* override configuration if provided */ if (conf) ctx->conf = *conf; @@ -129,7 +135,7 @@ static RdbxToJson *initRdbToJsonCtx(RdbParser *p, const char *outfilename, RdbxT /*** Handling common ***/ -static RdbRes handlingAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBulk auxval) { +static RdbRes toJsonAuxField(RdbParser *p, void *userData, RdbBulk auxkey, RdbBulk auxval) { RdbxToJson *ctx = userData; UNUSED(p); @@ -147,6 +153,25 @@ static RdbRes toJsonEndKey(RdbParser *p, void *userData) { /* output json part */ switch(ctx->state) { + case R2J_IN_STREAM: + fprintf(ctx->outfile, "\n }"); + break; + case R2J_IN_STREAM_ENTRIES: + fprintf(ctx->outfile, "\n ]}"); + break; + case R2J_IN_STREAM_CG: + fprintf(ctx->outfile, "}]}"); + break; + case R2J_IN_STREAM_CG_PEL: + fprintf(ctx->outfile, "]}]}"); + break; + case R2J_IN_STREAM_CG_CONSUMER: + fprintf(ctx->outfile, "}]}]}"); + break; + case R2J_IN_STREAM_CG_CONSUMER_PEL: + fprintf(ctx->outfile, "]}]}]}"); + break; + case R2J_IN_LIST: case R2J_IN_SET: fprintf(ctx->outfile, "]"); @@ -414,10 +439,146 @@ static RdbRes toJsonFunction(RdbParser *p, void *userData, RdbBulk func) { return RDB_OK; } +static RdbRes toJsonStreamItem(RdbParser *p, void *userData, RdbStreamID *id, RdbBulk field, RdbBulk value, int64_t itemsLeft) { + RdbxToJson *ctx = userData; + + if ( (ctx->state == R2J_IN_KEY) || (ctx->state == R2J_IN_STREAM_ENTRIES)) { + /* start of stream array of entries */ + if (ctx->state == R2J_IN_KEY) + fprintf(ctx->outfile, "{\n \"entries\":["); + + /* output another stream entry */ + fprintf(ctx->outfile, "%c\n { \"id\":\"%lu-%lu\", ", + (ctx->state == R2J_IN_STREAM_ENTRIES) ? ',' : ' ', + id->ms, id->seq ); + fprintf(ctx->outfile, "\"items\":{"); + outputQuotedEscaping(ctx, field, RDB_bulkLen(p, field)); + fprintf(ctx->outfile, ":"); + outputQuotedEscaping(ctx, value, RDB_bulkLen(p, value)); + } else if (ctx->state == R2J_IN_STREAM_ENTRIES_PAIRS) { + outputQuotedEscaping(ctx, field, RDB_bulkLen(p, field)); + fprintf(ctx->outfile, ":"); + outputQuotedEscaping(ctx, value, RDB_bulkLen(p, value)); + } else { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonStreamItem(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + + if (itemsLeft) { + fprintf(ctx->outfile, ", "); + ctx->state = R2J_IN_STREAM_ENTRIES_PAIRS; + } else { + fprintf(ctx->outfile, "} }"); + ctx->state = R2J_IN_STREAM_ENTRIES; + } + return RDB_OK; +} + +static RdbRes toJsonStreamMetadata(RdbParser *p, void *userData, RdbStreamMeta *meta) { + RdbxToJson *ctx = userData; + if (ctx->state != R2J_IN_STREAM_ENTRIES) { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonStreamMetadata(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + ctx->state = R2J_IN_STREAM; + fprintf(ctx->outfile, "],\n \"length\": %lu, ", meta->length); + fprintf(ctx->outfile, "\n \"entriesAdded\": %lu, ", meta->entriesAdded); + fprintf(ctx->outfile, "\n \"firstID\": \"%lu-%lu\", ", meta->firstID.ms, meta->firstID.seq); + fprintf(ctx->outfile, "\n \"lastID\": \"%lu-%lu\", ", meta->lastID.ms, meta->lastID.seq); + fprintf(ctx->outfile, "\n \"maxDelEntryID\": \"%lu-%lu\",", meta->maxDelEntryID.ms, meta->maxDelEntryID.seq); + return RDB_OK; +} + +static RdbRes toJsonStreamNewCGroup(RdbParser *p, void *userData, RdbBulk grpName, RdbStreamGroupMeta *meta) { + RdbxToJson *ctx = userData; + char *prefix; + if (ctx->state == R2J_IN_STREAM) { + prefix = "\n \"groups\": [\n"; + } else if (ctx->state == R2J_IN_STREAM_CG) { + prefix = "},\n"; + } else if (ctx->state == R2J_IN_STREAM_CG_PEL) { + prefix = "]},\n"; + } else if (ctx->state == R2J_IN_STREAM_CG_CONSUMER_PEL) { + prefix = "]}]},\n"; + } else if (ctx->state == R2J_IN_STREAM_CG_CONSUMER) { + prefix = "}]},\n"; + } else { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonStreamNewCGroup(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + fprintf(ctx->outfile, "%s {\"name\": \"%s\", \"lastid\": \"%lu-%lu\", \"entriesRead\": %lu", + prefix, grpName, meta->lastId.ms, meta->lastId.seq, meta->entriesRead); + + ctx->state = R2J_IN_STREAM_CG; + return RDB_OK; +} + +static RdbRes toJsonStreamCGroupPendingEntry(RdbParser *p, void *userData, RdbStreamPendingEntry *pe) { + char *prefix; + RdbxToJson *ctx = userData; + if (ctx->state == R2J_IN_STREAM_CG) { + ctx->state = R2J_IN_STREAM_CG_PEL; + prefix = ",\n \"pending\": [ "; + } else if (ctx->state == R2J_IN_STREAM_CG_PEL) { + prefix = ", "; + } else { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonStreamCGroupPendingEntry(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + fprintf(ctx->outfile, "%s\n { \"sent\": %lu, \"id\":\"%lu-%lu\", \"count\": %lu }", + prefix, pe->deliveryTime, pe->id.ms, pe->id.seq, pe->deliveryCount); + return RDB_OK; +} + +static RdbRes toJsonStreamNewConsumer(RdbParser *p, void *userData, RdbBulk consName, RdbStreamConsumerMeta *meta) { + RdbxToJson *ctx = userData; + char *prefix =""; + + if (ctx->state == R2J_IN_STREAM_CG) { + prefix = ",\n \"consumers\""; + } else if (ctx->state == R2J_IN_STREAM_CG_PEL) { + /* close pending entries array */ + prefix = "],\n \"consumers\": ["; + } else if (ctx->state == R2J_IN_STREAM_CG_CONSUMER) { + prefix = "}, "; + } else if (ctx->state == R2J_IN_STREAM_CG_CONSUMER_PEL) { + prefix = "]}, "; /* take care to close previous cons + cons PEL */ + } else { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_STATE, + "toJsonStreamNewConsumer(): Invalid state value: %d", ctx->state); + return (RdbRes) RDBX_ERR_R2J_INVALID_STATE; + } + + ctx->state = R2J_IN_STREAM_CG_CONSUMER; + fprintf(ctx->outfile, "%s\n { \"name\": \"%s\", \"activeTime\": %llu, \"seenTime\": %llu", + prefix, consName, meta->activeTime, meta->seenTime); + + return RDB_OK; +} + +static RdbRes toJsonStreamConsumerPendingEntry(RdbParser *p, void *userData, RdbStreamID *streamId) { + UNUSED(p); + RdbxToJson *ctx = userData; + char *prefix; + if (ctx->state == R2J_IN_STREAM_CG_CONSUMER) { + prefix = ",\n \"pending\": ["; + + } if (ctx->state == R2J_IN_STREAM_CG_CONSUMER_PEL) { + prefix = ", "; + } + + ctx->state = R2J_IN_STREAM_CG_CONSUMER_PEL; + fprintf(ctx->outfile, "%s\n {\"id\":\"%lu-%lu\"}", prefix, streamId->ms, streamId->seq); + return RDB_OK; +} + /*** Handling struct ***/ static RdbRes toJsonStruct(RdbParser *p, void *userData, RdbBulk value) { - UNUSED(p); RdbxToJson *ctx = userData; /* output json part */ @@ -428,10 +589,22 @@ static RdbRes toJsonStruct(RdbParser *p, void *userData, RdbBulk value) { return RDB_OK; } +static RdbRes toJsonStreamLP(RdbParser *p, void *userData, RdbBulk nodekey, RdbBulk streamLP) { + RdbxToJson *ctx = userData; + + /* output json part */ + fprintf(ctx->outfile, "{"); + outputQuotedEscaping(ctx, nodekey, RDB_bulkLen(p, nodekey)); + fprintf(ctx->outfile, ":"); + outputQuotedEscaping(ctx, streamLP, RDB_bulkLen(p, streamLP)); + fprintf(ctx->outfile, "}"); + + return RDB_OK; +} + /*** Handling raw ***/ static RdbRes toJsonFrag(RdbParser *p, void *userData, RdbBulk frag) { - UNUSED(p); RdbxToJson *ctx = userData; /* output json part */ ctx->encfunc(ctx, frag, RDB_bulkLen(p, frag)); @@ -439,8 +612,7 @@ static RdbRes toJsonFrag(RdbParser *p, void *userData, RdbBulk frag) { } static RdbRes toJsonRawBegin(RdbParser *p, void *userData, size_t size) { - UNUSED(p); - UNUSED(size); + UNUSED(p, size); RdbxToJson *ctx = userData; fprintf(ctx->outfile, "\""); return RDB_OK; @@ -453,70 +625,121 @@ static RdbRes toJsonRawEnd(RdbParser *p, void *userData) { return RDB_OK; } -#define COMMON_HANDLERS_INIT(commonPart, incAuxField) \ - if (incAuxField) \ - commonPart.handleAuxField = handlingAuxField; \ - commonPart.handleNewKey = toJsonNewKey; \ - commonPart.handleEndKey = toJsonEndKey; \ - commonPart.handleNewDb = toJsonNewDb; \ - commonPart.handleStartRdb = toJsonNewRdb; \ - commonPart.handleEndRdb = toJsonEndRdb; - RdbxToJson *RDBX_createHandlersToJson(RdbParser *p, const char *filename, RdbxToJsonConf *conf) { RdbxToJson *ctx = initRdbToJsonCtx(p, filename, conf); if (ctx == NULL) return NULL; if (ctx->conf.level == RDB_LEVEL_DATA) { - RdbHandlersDataCallbacks dataCb; - memset(&dataCb, 0 , sizeof(dataCb)); - COMMON_HANDLERS_INIT(dataCb, ctx->conf.includeAuxField); - - dataCb.handleStringValue = toJsonString; - dataCb.handleListItem = toJsonList; - dataCb.handleHashField = toJsonHash; - dataCb.handleSetMember = toJsonSet; - dataCb.handleZsetMember = toJsonZset; - dataCb.handleFunction = (ctx->conf.includeFunc) ? toJsonFunction : NULL; - dataCb.handleModule = toJsonModule; + + RdbHandlersDataCallbacks dataCb = { + toJsonNewRdb, + toJsonEndRdb, + toJsonNewDb, + NULL, /* handleResizeDb */ + NULL, + NULL, + toJsonNewKey, + toJsonEndKey, + toJsonString, + toJsonList, + toJsonHash, + toJsonSet, + toJsonZset, + NULL, /* handleFunction */ + toJsonModule, + NULL, /*handleStreamMetadata*/ + toJsonStreamItem, + NULL, /* handleStreamNewCGroup */ + NULL, /* handleStreamCGroupPendingEntry */ + NULL, /* handleStreamNewConsumer */ + NULL, /* handleStreamConsumerPendingEntry */ + }; + + if (ctx->conf.includeAuxField) + dataCb.handleAuxField = toJsonAuxField; + + if (ctx->conf.includeFunc) + dataCb.handleFunction = toJsonFunction; + + if (ctx->conf.includeStreamMeta) { + dataCb.handleStreamMetadata = toJsonStreamMetadata; + dataCb.handleStreamNewCGroup = toJsonStreamNewCGroup; + dataCb.handleStreamCGroupPendingEntry = toJsonStreamCGroupPendingEntry; + dataCb.handleStreamNewConsumer = toJsonStreamNewConsumer; + dataCb.handleStreamConsumerPendingEntry = toJsonStreamConsumerPendingEntry; + } + RDB_createHandlersData(p, &dataCb, ctx, deleteRdbToJsonCtx); } else if (ctx->conf.level == RDB_LEVEL_STRUCT) { - RdbHandlersStructCallbacks structCb; - memset(&structCb, 0 , sizeof(structCb)); - COMMON_HANDLERS_INIT(structCb, ctx->conf.includeAuxField); - - structCb.handleString = toJsonString; - /* list */ - structCb.handleListPlain = toJsonList; - structCb.handleListLP = toJsonStruct; - structCb.handleListZL = toJsonStruct; - /* hash */ - structCb.handleHashPlain = toJsonHash; - structCb.handleHashZL = toJsonStruct; - structCb.handleHashLP = toJsonStruct; - structCb.handleHashZM = toJsonStruct; - /* set */ - structCb.handleSetPlain = toJsonSet; - structCb.handleSetIS = toJsonStruct; - structCb.handleSetLP = toJsonStruct; - /* zset */ - structCb.handleZsetPlain = toJsonZset; - structCb.handleZsetZL = toJsonStruct; - structCb.handleZsetLP = toJsonStruct; - /* function */ - structCb.handleFunction = (ctx->conf.includeFunc) ? toJsonFunction : NULL; - /* module */ - structCb.handleModule = toJsonModule; + RdbHandlersStructCallbacks structCb = { + toJsonNewRdb, + toJsonEndRdb, + toJsonNewDb, + NULL, /* handleResizeDb */ + NULL, + NULL, + toJsonNewKey, + toJsonEndKey, + toJsonString, + /*list*/ + toJsonList, + toJsonStruct, /* handleListZL*/ + toJsonStruct, /* handleListLP*/ + /*hash*/ + toJsonHash, + toJsonStruct, /* handleHashZL*/ + toJsonStruct, /* handleHashLP*/ + toJsonStruct, /* handleHashZM*/ + /*set*/ + toJsonSet, + toJsonStruct, /* handleSetIS*/ + toJsonStruct, /* handleSetLP*/ + /*zset*/ + toJsonZset, + toJsonStruct, /* handleZsetZL*/ + toJsonStruct, /* handleZsetLP*/ + /*function*/ + NULL, /* handleFunction */ + /*module*/ + toJsonModule, + /*stream*/ + toJsonStreamLP, + }; + + if (ctx->conf.includeAuxField) + structCb.handleAuxField = toJsonAuxField; + + if (ctx->conf.includeFunc) + structCb.handleFunction = toJsonFunction; + RDB_createHandlersStruct(p, &structCb, ctx, deleteRdbToJsonCtx); } else if (ctx->conf.level == RDB_LEVEL_RAW) { - RdbHandlersRawCallbacks rawCb; - memset(&rawCb, 0 , sizeof(rawCb)); - COMMON_HANDLERS_INIT(rawCb, ctx->conf.includeAuxField); - rawCb.handleFrag = toJsonFrag; - rawCb.handleBegin = toJsonRawBegin; - rawCb.handleEnd = toJsonRawEnd; + RdbHandlersRawCallbacks rawCb = { + toJsonNewRdb, + toJsonEndRdb, + toJsonNewDb, + NULL, /* handleResizeDb */ + NULL, + NULL, + toJsonNewKey, + toJsonEndKey, + NULL, /*handleBeginModuleAux*/ + toJsonRawBegin, + toJsonFrag, + toJsonRawEnd, + }; + + if (ctx->conf.includeAuxField) + rawCb.handleAuxField = toJsonAuxField; + RDB_createHandlersRaw(p, &rawCb, ctx, deleteRdbToJsonCtx); + + } else { + RDB_reportError(p, (RdbRes) RDBX_ERR_R2J_INVALID_LEVEL, + "RDBX_createHandlersToJson(): Invalid level value: %d", ctx->conf.level); + return NULL; } return ctx; diff --git a/src/ext/handlersToResp.c b/src/ext/handlersToResp.c index 8367e2c..68fa39d 100644 --- a/src/ext/handlersToResp.c +++ b/src/ext/handlersToResp.c @@ -6,26 +6,33 @@ #include "../../deps/redis/crc64.h" #include "../../deps/redis/util.h" #include "../../deps/redis/endianconv.h" +#include "../../deps/redis/rax.h" +/* RDB opcode defines */ +#define _RDB_TYPE_MODULE_2 7 #define _RDB_TYPE_STRING 0 +#define _RDB_TYPE_STREAM_LISTPACKS_2 19 #define _REDISMODULE_AUX_BEFORE_RDB (1<<0) + #define VER_VAL(major,minor) (((unsigned int)(major)<<8) | (unsigned int)(minor)) #define KEY_CMD_ID_DBG "_RDB_CLI_CMD_ID_" typedef struct RedisToRdbVersion { + const char *redisStr; unsigned int redis; unsigned char rdb; } RedisToRdbVersion; const RedisToRdbVersion redisToRdbVersion[] = { - {VER_VAL(7,2), 11}, - {VER_VAL(7,0), 10}, - {VER_VAL(5,0), 9}, //6 and 6.2 had v9 too - {VER_VAL(4,0), 8}, - {VER_VAL(3,2), 7}, - {VER_VAL(2,6), 6}, //2.8 had v6 too - {VER_VAL(2,4), 5}, + {"99.99", VER_VAL(99,99), 12}, // TODO: Update released version + {"7.2", VER_VAL(7,2), 11}, + {"7.0", VER_VAL(7,0), 10}, + {"5.0", VER_VAL(5,0), 9}, //6 and 6.2 had v9 too + {"4.0", VER_VAL(4,0), 8}, + {"3.2", VER_VAL(3,2), 7}, + {"2.6", VER_VAL(2,6), 6}, //2.8 had v6 too + {"2.4", VER_VAL(2,4), 5}, }; typedef enum DelKeyBeforeWrite { @@ -43,10 +50,10 @@ struct RdbxToResp { size_t cmdNum; /* configuration */ -#define RFLAG_ENUM_CMD_ID (1<<0) /* Enumerate and trace commands by pushing debug command +#define RFLAG_WRITE_FROM_CMD_ID (1<<0) /* Flag for writing commands from a specific command-id */ +#define RFLAG_ENUM_CMD_ID (1<<1) /* Enumerate and trace commands by pushing debug command * of type "SET _RDB_CLI_CMD_ID_ " before each * RESP command */ -#define RFLAG_WRITE_FROM_CMD_ID (1<<1) /* Flag for writing commands from a specific command-id */ int flags; size_t writeFromCmdNum; } debug; @@ -58,7 +65,7 @@ struct RdbxToResp { RdbxRespWriter respWriter; int respWriterConfigured; - unsigned int targetVerValue; /* major << 8 | minor */ + unsigned int targetRedisVerVal; /* major << 8 | minor */ struct { RdbBulkCopy key; @@ -69,18 +76,48 @@ struct RdbxToResp { struct { int sentFirstFrag; - size_t rawSize; + size_t restoreSize; int isModuleAux; uint64_t crc; struct { char cmdPrefix[128]; int cmdlen; } moduleAux; - } rawCtx; + } restoreCtx; int srcRdbVer; + int dstRdbVer; + + struct { + /* +1 on start of XADD. Another +1 on end of XADD. Reset by toRespStreamMetaData() */ + long long xaddStartEndCounter; + + /* The radix tree representing pending entries for a consumer group is rebuilt + * each time a new consumer group is visited (The tree is used exclusively + * to map consumer IDs within this group to their respective associated + * delivery times, which are necessary for reconstructing XCLAIM commands). */ + rax *groupPel; + + RdbBulkCopy grpName, consName; + int grpNameLen, consNameLen; + + } streamCtx; }; +static void deletePendingEntriesList(RdbParser *p, rax **pel) { + /* Free all entries in the Rax tree */ + raxIterator ri_cons; + raxStart(&ri_cons,*pel); + raxSeek(&ri_cons,"^",NULL,0); + while(raxNext(&ri_cons)) { + RdbStreamPendingEntry *entry = ri_cons.data; + RDB_free(p, entry); + } + /* Free the entire Rax tree */ + raxFree(*pel); + *pel = NULL; +} + static void deleteRdbToRespCtx(RdbParser *p, void *context) { RdbxToResp *ctx = (RdbxToResp *) context; @@ -89,9 +126,14 @@ static void deleteRdbToRespCtx(RdbParser *p, void *context) { /* ignore the first release attempt */ if (--ctx->refcount) return; - if (ctx->keyCtx.key != NULL) - RDB_bulkCopyFree(p, ctx->keyCtx.key); - ctx->keyCtx.key = NULL; + RDB_bulkCopyFree(p, ctx->streamCtx.grpName); + + if(ctx->streamCtx.groupPel) + deletePendingEntriesList(p, &ctx->streamCtx.groupPel); + + RDB_bulkCopyFree(p, ctx->keyCtx.key); + + RDB_bulkCopyFree(p, ctx->streamCtx.consName); /* delete respWriter */ if (ctx->respWriter.delete) @@ -100,51 +142,48 @@ static void deleteRdbToRespCtx(RdbParser *p, void *context) { RDB_free(p, ctx); } -int iov_value(struct iovec *iov, long long count, char *buf, int bufsize) { - int len = 0; - len = ll2string(buf, bufsize, count); - - int lenWithNewLine = len; - buf[lenWithNewLine++] = '\r'; - buf[lenWithNewLine++] = '\n'; +static int setRdbVerFromDestRedisVer(RdbxToResp *ctx) { - iov_plain(iov, buf, lenWithNewLine); - return len; -} - -static int rdbVerFromRedisVer(RdbxToResp *ctx) { const char *ver = ctx->conf.dstRedisVersion; - if (!ver) return 0; + if (!ver) { + RDB_log(ctx->parser, RDB_LOG_WRN, "Target Redis version is not configured! " + "Set it to Redis version: %s", redisToRdbVersion[0].redisStr); - int mjr=0, mnr, pch, pos1 = 0, pos2 = 0; + ctx->targetRedisVerVal = redisToRdbVersion[0].redis; + return redisToRdbVersion[0].rdb; + } + + int mjr = 0, mnr, pch, pos1 = 0, pos2 = 0; int scanned = sscanf(ver, "%d.%d%n.%d%n", &mjr, &mnr, &pos1, &pch, &pos2); if ((scanned == 3 && pos2 == (int) strlen(ver)) || (scanned == 2 && pos1 == (int) strlen(ver))) { - ctx->targetVerValue = VER_VAL(mjr, mnr); + ctx->targetRedisVerVal = VER_VAL(mjr, mnr); unsigned int i; for (i = 0; i < sizeof(redisToRdbVersion) / sizeof(redisToRdbVersion[0]); i++) - if (ctx->targetVerValue >= redisToRdbVersion[i].redis) return redisToRdbVersion[i].rdb; + if (ctx->targetRedisVerVal >= redisToRdbVersion[i].redis) { + return redisToRdbVersion[i].rdb; + } } + RDB_log(ctx->parser, RDB_LOG_WRN, "Configured Obsolete or invalid target version [%s]. Earliest is [2.4]", ver); return 0; } -static void resolveSupportRestore(RdbParser *p, RdbxToResp *ctx, int srcRdbVer) { - int dstRdbVer; - - ctx->srcRdbVer = srcRdbVer; +/* + * This function Resolves whether the parser can support RESTORE commands. That is, if + * requested via configuration to support RESTORE, yet it is required to verify that + * source version of RDB file is aligned with destination version of the Redis + * target. If not, then RESTORE configuration won't be honored. + */ +static RdbRes resolveSupportRestore(RdbParser *p, RdbxToResp *ctx) { ctx->keyCtx.delBeforeWrite = (ctx->conf.delKeyBeforeWrite) ? DEL_KEY_BEFORE_BY_DEL_CMD : DEL_KEY_BEFORE_NONE; - if (ctx->conf.supportRestore) { - /* if not configured destination RDB version, then resolve it from - * configured destination Redis version */ - dstRdbVer = rdbVerFromRedisVer(ctx); - if (dstRdbVer < srcRdbVer) { + if (ctx->dstRdbVer < ctx->srcRdbVer) { RDB_log(p, RDB_LOG_WRN, "Cannot support RESTORE. source RDB version (=%d) is higher than destination (=%d)", - srcRdbVer, dstRdbVer); + ctx->srcRdbVer, ctx->dstRdbVer); ctx->conf.supportRestore = 0; } else { if (ctx->conf.delKeyBeforeWrite) { @@ -154,13 +193,19 @@ static void resolveSupportRestore(RdbParser *p, RdbxToResp *ctx, int srcRdbVer) } } + /* Now that it is being decided whether to use RESTORE or not, configure accordingly + * parsing level of the parser */ RdbHandlersLevel lvl = (ctx->conf.supportRestore) ? RDB_LEVEL_RAW : RDB_LEVEL_DATA; for (int i = 0; i < RDB_DATA_TYPE_MAX; ++i) { - RDB_handleByLevel(p, (RdbDataType) i, lvl, 0); + /* No need to check return value of RDB_handleByLevel() since it is + * guaranteed to succeed */ + RDB_handleByLevel(p, (RdbDataType) i, lvl); } - /* librdb cannot parse a module object */ - RDB_handleByLevel(p, RDB_DATA_TYPE_MODULE, RDB_LEVEL_RAW, 0); + /* Enforce RESTORE for modules. librdb cannot really parse high-level module object */ + RDB_handleByLevel(p, RDB_DATA_TYPE_MODULE, RDB_LEVEL_RAW); + + return RDB_OK; } static inline RdbRes onWriteNewCmdDbg(RdbxToResp *ctx) { @@ -184,10 +229,14 @@ static inline RdbRes onWriteNewCmdDbg(RdbxToResp *ctx) { IOV_STRING(&iov[2], KEY_CMD_ID_DBG, sizeof(KEY_CMD_ID_DBG)-1); /* write cmd-id */ IOV_CONST(&iov[3], "\r\n$"); - IOV_LEN_AND_VALUE(&iov[4], currCmdNum, cmdIdLenStr, cmdIdStr); + IOV_LEN_AND_VAL(&iov[4], currCmdNum, cmdIdLenStr, cmdIdStr); if (unlikely(writer->writev(writer->ctx, iov, 6, 1, 1))) { RdbRes errCode = RDB_getErrorCode(ctx->parser); - assert(errCode != RDB_OK); + + /* If failed to write RESP writer but no error reported, then write some general error */ + if (errCode == RDB_OK) + RDB_reportError(ctx->parser, RDB_ERR_GENERAL, "Failed to writev() RESP"); + return RDB_getErrorCode(ctx->parser); } } @@ -205,13 +254,70 @@ static inline RdbRes writevWrap(RdbxToResp *ctx, struct iovec *iov, int cnt, int if (unlikely(writer->writev(writer->ctx, iov, cnt, startCmd, endCmd))) { res = RDB_getErrorCode(ctx->parser); - assert(res != RDB_OK); + + /* If failed to write RESP writer but no error reported, then write some general error */ + if (res == RDB_OK) + RDB_reportError(ctx->parser, RDB_ERR_GENERAL, "Failed to writev() RESP"); + return RDB_getErrorCode(ctx->parser); } return RDB_OK; } +static inline RdbRes sendFirstRestoreFrag(RdbxToResp *ctx, RdbBulk frag, size_t fragLen) { + long long expireTime = 0; + char expireTimeStr[32], expireTimeLenStr[32], keyLenStr[32], lenStr[32]; + struct iovec iov[10]; + int extra_args = 0, iovs = 0; + + /* this logic must be exactly the same as in toRespRestoreFragEnd() */ + if (ctx->targetRedisVerVal >= VER_VAL(5, 0)) + { + if (ctx->keyCtx.info.expiretime != -1) { + expireTime = ctx->keyCtx.info.expiretime; + extra_args++; /* ABSTTL */ + } + + if ((ctx->keyCtx.info.lfuFreq != -1) || (ctx->keyCtx.info.lruIdle != -1)) { + extra_args += 2; + } + } + + if (ctx->keyCtx.delBeforeWrite == DEL_KEY_BEFORE_BY_RESTORE_REPLACE) + extra_args++; + + char cmd[64]; + + int len = snprintf(cmd, sizeof(cmd), "*%d\r\n$7\r\nRESTORE", 4+extra_args); + + IOV_STRING(&iov[iovs++], cmd, len); /* RESTORE */ + IOV_LENGTH(&iov[iovs++], ctx->keyCtx.keyLen, keyLenStr); /* write key len */ + IOV_STRING(&iov[iovs++], ctx->keyCtx.key, ctx->keyCtx.keyLen); /* write key */ + + if (expireTime) { + IOV_LEN_AND_VAL(&iov[iovs], expireTime, expireTimeLenStr, expireTimeStr); + iovs += 2; + IOV_CONST(&iov[iovs++], "$"); + } else { + IOV_CONST(&iov[iovs++], "\r\n$1\r\n0\r\n$"); + } + + IOV_VALUE(&iov[iovs++], ctx->restoreCtx.restoreSize + 10, lenStr); /* write restore len + trailer */ + IOV_STRING(&iov[iovs++], frag, fragLen); /* write first frag */ + return writevWrap(ctx, iov, iovs, 1, 0); +} + +static inline RdbRes sendFirstRestoreFragModuleAux(RdbxToResp *ctx, RdbBulk frag, size_t fragLen) { + struct iovec iov[3]; + char lenStr[32]; + iov[0].iov_base = ctx->restoreCtx.moduleAux.cmdPrefix; + iov[0].iov_len = ctx->restoreCtx.moduleAux.cmdlen; + IOV_LENGTH(&iov[1], ctx->restoreCtx.restoreSize + 10, lenStr); /* write restore len + trailer */ + IOV_STRING(&iov[2], frag, fragLen); /* write first frag */ + return writevWrap(ctx, iov, 3, 1, 0); +} + /*** Handling common ***/ static RdbRes toRespNewDb(RdbParser *p, void *userData, int dbid) { @@ -224,8 +330,8 @@ static RdbRes toRespNewDb(RdbParser *p, void *userData, int dbid) { int cnt = ll2string(dbidStr, sizeof(dbidStr), dbid); - IOV_CONST(&iov[0], "*2\r\n$6\r\nSELECT\r\n$"); - IOV_VALUE(&iov[1], cnt, cntStr); + IOV_CONST(&iov[0], "*2\r\n$6\r\nSELECT"); + IOV_LENGTH(&iov[1], cnt, cntStr); IOV_STRING(&iov[2], dbidStr, cnt); IOV_CONST(&iov[3], "\r\n"); return writevWrap(ctx, iov, 4, 1, 1); @@ -238,9 +344,10 @@ static RdbRes toRespStartRdb(RdbParser *p, void *userData, int rdbVersion) { /* If not configured respWriter then output it to STDOUT */ assert (ctx->respWriterConfigured == 1); - resolveSupportRestore(p, ctx, rdbVersion); + ctx->srcRdbVer = rdbVersion; + ctx->dstRdbVer = setRdbVerFromDestRedisVer(ctx); - return RDB_OK; + return resolveSupportRestore(p, ctx); } static RdbRes toRespNewKey(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo *info) { @@ -258,8 +365,8 @@ static RdbRes toRespNewKey(RdbParser *p, void *userData, RdbBulk key, RdbKeyInfo if ((ctx->keyCtx.delBeforeWrite == DEL_KEY_BEFORE_BY_DEL_CMD) && (info->opcode != _RDB_TYPE_STRING)) { struct iovec iov[4]; char keyLenStr[32]; - IOV_CONST(&iov[0], "*2\r\n$3\r\nDEL\r\n$"); - IOV_VALUE(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_CONST(&iov[0], "*2\r\n$3\r\nDEL"); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); IOV_CONST(&iov[3], "\r\n"); return writevWrap(ctx, iov, 4, 1, 1); @@ -276,19 +383,17 @@ static RdbRes toRespEndKey(RdbParser *p, void *userData) { struct iovec iov[6]; char keyLenStr[32], expireLenStr[32], expireStr[32]; /* PEXPIREAT */ - IOV_CONST(&iov[0], "*3\r\n$9\r\nPEXPIREAT\r\n$"); + IOV_CONST(&iov[0], "*3\r\n$9\r\nPEXPIREAT"); /* KEY-LEN and KEY */ - IOV_VALUE(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); - IOV_CONST(&iov[3], "\r\n$"); - IOV_LEN_AND_VALUE(iov+4, ctx->keyCtx.info.expiretime, expireLenStr, expireStr); + IOV_LEN_AND_VAL(iov+3, ctx->keyCtx.info.expiretime, expireLenStr, expireStr); - return writevWrap(ctx, iov, 6, 1, 1); + return writevWrap(ctx, iov, 5, 1, 1); } - if (ctx->keyCtx.key != NULL) - RDB_bulkCopyFree(p, ctx->keyCtx.key); + RDB_bulkCopyFree(p, ctx->keyCtx.key); ctx->keyCtx.key = NULL; return RDB_OK; @@ -306,16 +411,15 @@ static RdbRes toRespString(RdbParser *p, void *userData, RdbBulk string) { struct iovec iov[7]; /* write SET */ - IOV_CONST(&iov[0], "*3\r\n$3\r\nSET\r\n$"); + IOV_CONST(&iov[0], "*3\r\n$3\r\nSET"); /* write key */ - IOV_VALUE(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); - IOV_CONST(&iov[3], "\r\n$"); /* write string */ - IOV_VALUE(&iov[4], valLen, valLenStr); - IOV_STRING(&iov[5], string, valLen); - IOV_CONST(&iov[6], "\r\n"); - return writevWrap(ctx, iov, 7, 1, 1); + IOV_LENGTH(&iov[3], valLen, valLenStr); + IOV_STRING(&iov[4], string, valLen); + IOV_CONST(&iov[5], "\r\n"); + return writevWrap(ctx, iov, 6, 1, 1); } static RdbRes toRespList(RdbParser *p, void *userData, RdbBulk item) { @@ -328,16 +432,15 @@ static RdbRes toRespList(RdbParser *p, void *userData, RdbBulk item) { struct iovec iov[7]; /* write RPUSH */ - IOV_CONST(&iov[0], "*3\r\n$5\r\nRPUSH\r\n$"); + IOV_CONST(&iov[0], "*3\r\n$5\r\nRPUSH"); /* write key */ - IOV_VALUE(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); - IOV_CONST(&iov[3], "\r\n$"); /* write item */ - IOV_VALUE(&iov[4], valLen, valLenStr); - IOV_STRING(&iov[5], item, valLen); - IOV_CONST(&iov[6], "\r\n"); - return writevWrap(ctx, iov, 7, 1, 1); + IOV_LENGTH(&iov[3], valLen, valLenStr); + IOV_STRING(&iov[4], item, valLen); + IOV_CONST(&iov[5], "\r\n"); + return writevWrap(ctx, iov, 6, 1, 1); } static RdbRes toRespHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk value) { @@ -351,20 +454,18 @@ static RdbRes toRespHash(RdbParser *p, void *userData, RdbBulk field, RdbBulk va struct iovec iov[10]; /* write RPUSH */ - IOV_CONST(&iov[0], "*4\r\n$4\r\nHSET\r\n$"); + IOV_CONST(&iov[0], "*4\r\n$4\r\nHSET"); /* write key */ - IOV_VALUE(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); - IOV_CONST(&iov[3], "\r\n$"); /* write field */ - IOV_VALUE(&iov[4], fieldLen, fieldLenStr); - IOV_STRING(&iov[5], field, fieldLen); - IOV_CONST(&iov[6], "\r\n$"); + IOV_LENGTH(&iov[3], fieldLen, fieldLenStr); + IOV_STRING(&iov[4], field, fieldLen); /* write value */ - IOV_VALUE(&iov[7], valueLen, valueLenStr); - IOV_STRING(&iov[8], value, valueLen); - IOV_CONST(&iov[9], "\r\n"); - return writevWrap(ctx, iov, sizeof(iov)/sizeof(iov[0]), 1, 1); + IOV_LENGTH(&iov[5], valueLen, valueLenStr); + IOV_STRING(&iov[6], value, valueLen); + IOV_CONST(&iov[7], "\r\n"); + return writevWrap(ctx, iov, 8, 1, 1); } static RdbRes toRespSet(RdbParser *p, void *userData, RdbBulk member) { @@ -375,16 +476,15 @@ static RdbRes toRespSet(RdbParser *p, void *userData, RdbBulk member) { struct iovec iov[7]; /* write RPUSH */ - IOV_CONST(&iov[0], "*3\r\n$4\r\nSADD\r\n$"); + IOV_CONST(&iov[0], "*3\r\n$4\r\nSADD"); /* write key */ - IOV_VALUE(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); - IOV_CONST(&iov[3], "\r\n$"); /* write member */ - IOV_VALUE(&iov[4], valLen, valLenStr); - IOV_STRING(&iov[5], member, valLen); - IOV_CONST(&iov[6], "\r\n"); - return writevWrap(ctx, iov, 7, 1, 1); + IOV_LENGTH(&iov[3], valLen, valLenStr); + IOV_STRING(&iov[4], member, valLen); + IOV_CONST(&iov[5], "\r\n"); + return writevWrap(ctx, iov, 6, 1, 1); } static RdbRes toRespZset(RdbParser *p, void *userData, RdbBulk member, double score) { @@ -395,25 +495,23 @@ static RdbRes toRespZset(RdbParser *p, void *userData, RdbBulk member, double sc struct iovec iov[10]; /* write ZADD */ - IOV_CONST(&iov[0], "*4\r\n$4\r\nZADD\r\n$"); + IOV_CONST(&iov[0], "*4\r\n$4\r\nZADD"); /* write key */ - IOV_VALUE(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); - IOV_CONST(&iov[3], "\r\n$"); /* write score */ char score_str[MAX_D2STRING_CHARS]; int len = d2string(score_str, sizeof(score_str), score); assert(len != 0); - IOV_VALUE(&iov[4], len, scoreLenStr); - IOV_STRING(&iov[5], score_str, strlen(score_str)); - IOV_CONST(&iov[6], "\r\n$"); + IOV_LENGTH(&iov[3], len, scoreLenStr); + IOV_STRING(&iov[4], score_str, strlen(score_str)); /* write member */ - IOV_VALUE(&iov[7], valLen, valLenStr); - IOV_STRING(&iov[8], member, valLen); - IOV_CONST(&iov[9], "\r\n"); - return writevWrap(ctx, iov, 10, 1, 1); + IOV_LENGTH(&iov[5], valLen, valLenStr); + IOV_STRING(&iov[6], member, valLen); + IOV_CONST(&iov[7], "\r\n"); + return writevWrap(ctx, iov, 8, 1, 1); } static RdbRes toRespEndRdb(RdbParser *p, void *userData) { @@ -438,160 +536,334 @@ static RdbRes toRespFunction(RdbParser *p, void *userData, RdbBulk func) { int funcLen = RDB_bulkLen(p, func); struct iovec iov[4]; - IOV_CONST(&iov[0], "*4\r\n$8\r\nFUNCTION\r\n$4\r\nLOAD\r\n$7\r\nREPLACE\r\n$"); + IOV_CONST(&iov[0], "*4\r\n$8\r\nFUNCTION\r\n$4\r\nLOAD\r\n$7\r\nREPLACE"); /* write member */ - IOV_VALUE(&iov[1], funcLen, funcLenStr); + IOV_LENGTH(&iov[1], funcLen, funcLenStr); IOV_STRING(&iov[2], func, funcLen); IOV_CONST(&iov[3], "\r\n"); return writevWrap( (RdbxToResp *) userData, iov, 4, 1, 1); } -/*** Handling raw ***/ -/* Callback on start of serializing module aux data (alternative to toRespRawBegin). - * Following this call, one or more calls will be made to toRespRawFrag() to - * stream fragments of the serialized data. And at the end toRespRawFragEnd() - * will be called */ -static RdbRes toRespRawBeginModuleAux(RdbParser *p, void *userData, RdbBulk name, int encver, int when, size_t rawSize) { - char encstr[10]; +static RdbRes toRespStreamMetaData(RdbParser *p, void *userData, RdbStreamMeta *meta) { + UNUSED(p); + char keyLenStr[32], idStr[100], idLenStr[32], maxDelEntryIdLenStr[64], maxDelEntryId[100], entriesLenStr[32], entriesStr[32]; + RdbxToResp *ctx = userData; + struct iovec iov[15]; + + if (ctx->streamCtx.xaddStartEndCounter == 0) { + /* Use the XGROUP CREATE MKSTREAM + DESTROY trick to generate an empty stream if + * the key we are serializing is an empty stream, which is possible + * for the Stream type. (We don't use the MAXLEN 0 trick from aof.c + * because of Redis Enterprise CRDT compatibility issues - Can't XSETID "back") */ + + IOV_CONST(&iov[0], "*6\r\n$6\r\nXGROUP\r\n$6\r\nCREATE"); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); + IOV_CONST(&iov[3], "$7\r\ndummyCG\r\n$1\r\n$\r\n$8\r\nMKSTREAM\r\n"); + IF_NOT_OK_RETURN(writevWrap( (RdbxToResp *) userData, iov, 4, 1, 1)); + + IOV_CONST(&iov[0], "*4\r\n$6\r\nXGROUP\r\n$7\r\nDESTROY"); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); + IOV_CONST(&iov[3], "$7\r\ndummyCG\r\n"); + IF_NOT_OK_RETURN(writevWrap( (RdbxToResp *) userData, iov, 4, 1, 1)); + } + + /* take care to reset it for next stream-item */ + ctx->streamCtx.xaddStartEndCounter = 0; + + int idLen = snprintf(idStr, sizeof(idStr), "%lu-%lu",meta->lastID.ms,meta->lastID.seq); + int maxDelEntryIdLen = snprintf(maxDelEntryId, sizeof(maxDelEntryId), "%lu-%lu", meta->maxDelEntryID.ms, meta->maxDelEntryID.seq); + + if ((ctx->keyCtx.info.opcode >= _RDB_TYPE_STREAM_LISTPACKS_2) && (ctx->targetRedisVerVal >= VER_VAL(7, 0))) { + IOV_CONST(&iov[0], "*7\r\n$6\r\nXSETID"); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); + IOV_LENGTH(&iov[3], idLen, idLenStr); + IOV_STRING(&iov[4], idStr, idLen); + IOV_CONST(&iov[5], "\r\n$12\r\nENTRIESADDED"); + IOV_LEN_AND_VAL(&iov[6], meta->entriesAdded, entriesLenStr, entriesStr); + IOV_CONST(&iov[8], "$12\r\nMAXDELETEDID"); + IOV_LENGTH(&iov[9], maxDelEntryIdLen, maxDelEntryIdLenStr); + IOV_STRING(&iov[10], maxDelEntryId, maxDelEntryIdLen); + IOV_CONST(&iov[11], "\r\n"); + return writevWrap( (RdbxToResp *) userData, iov, 12, 1, 1); + } else { + IOV_CONST(&iov[0], "*3\r\n$6\r\nXSETID"); + IOV_LENGTH(&iov[1], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[2], ctx->keyCtx.key, ctx->keyCtx.keyLen); + IOV_LENGTH(&iov[3], idLen, idLenStr); + IOV_STRING(&iov[4], idStr, idLen); + IOV_CONST(&iov[5], "\r\n"); + return writevWrap( (RdbxToResp *) userData, iov, 6, 1, 1); + } +} - /* reset rawCtx */ +static RdbRes toRespStreamItem(RdbParser *p, void *userData, RdbStreamID *id, RdbBulk field, RdbBulk val, int64_t itemsLeft) { + char cmd[64], idStr[100], idLenStr[64], keyLenStr[32], fieldLenStr[32], valLenStr[32]; + int iovs = 0, startCmd = 0 , endCmd = 0; + struct iovec iov[15]; RdbxToResp *ctx = userData; - ctx->rawCtx.rawSize = rawSize; - ctx->rawCtx.sentFirstFrag = 0; - ctx->rawCtx.isModuleAux = 1; - ctx->rawCtx.crc = 0; - /* if target doesn't support module-aux, then skip it */ - if (!ctx->conf.supportRestoreModuleAux) - return RDB_OK; + size_t fieldLen = RDB_bulkLen(p, field); + size_t valLen = RDB_bulkLen(p, val); + + /* Start of (another) stream item? */ + if ((ctx->streamCtx.xaddStartEndCounter % 2) == 0) { + int cmdLen = snprintf(cmd, sizeof(cmd), "*%lu\r\n$4\r\nXADD", 3 + (itemsLeft + 1) * 2); + IOV_STRING(&iov[iovs++], cmd, cmdLen); + IOV_LENGTH(&iov[iovs++], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[iovs++], ctx->keyCtx.key, ctx->keyCtx.keyLen); + int idLen = snprintf(idStr, sizeof(idStr), "%lu-%lu",id->ms,id->seq); + IOV_LENGTH(&iov[iovs++], idLen, idLenStr); + IOV_STRING(&iov[iovs++], idStr, idLen); + + startCmd = 1; + ++ctx->streamCtx.xaddStartEndCounter; + } - /* Build the cmd instead of keeping the values and build it later */ - size_t enclen = snprintf(encstr, sizeof(encstr), "%d", encver); - const char* whenstr = (when==_REDISMODULE_AUX_BEFORE_RDB) ? "before" :"after"; - ctx->rawCtx.moduleAux.cmdlen = snprintf(ctx->rawCtx.moduleAux.cmdPrefix, - sizeof(ctx->rawCtx.moduleAux.cmdPrefix), - "*5\r\n$13\r\nRESTOREMODAUX\r\n$%zu\r\n%s\r\n$%zu\r\n%s\r\n$%zu\r\n%s\r\n$", - strlen(name), name, enclen, encstr, strlen(whenstr), whenstr); - return RDB_OK; + IOV_LENGTH(&iov[iovs++], fieldLen, fieldLenStr); + IOV_STRING(&iov[iovs++], field, fieldLen); + IOV_LENGTH(&iov[iovs++], valLen, valLenStr); + IOV_STRING(&iov[iovs++], val, valLen); + + /* if end of variadic command */ + if (!itemsLeft) { + IOV_CONST(&iov[iovs++], "\r\n"); + endCmd = 1; + ++ctx->streamCtx.xaddStartEndCounter; + } + + return writevWrap( (RdbxToResp *) userData, iov, iovs, startCmd, endCmd); } -/* Callback on start of serializing value of a key. Following this call, one - * or more calls will be made to toRespRawFrag() to stream fragments of the - * serialized data. And at the end toRespRawFragEnd() will be called */ -static RdbRes toRespRawBegin(RdbParser *p, void *userData, size_t size) { - UNUSED(p); +/* Emit the XGROUP CREATE in order to create the group. */ +static RdbRes toRespStreamNewCGroup(RdbParser *p, void *userData, RdbBulk grpName, RdbStreamGroupMeta *meta) { + struct iovec iov[16]; + int iovs = 0; RdbxToResp *ctx = userData; + char keyLenStr[32], gNameLenStr[32], idStr[100], idLenStr[32], entriesReadStr[32], entriesReadLenStr[32]; + + /* (re)allocate mem to keep group name */ + RDB_bulkCopyFree(p, ctx->streamCtx.grpName); + ctx->streamCtx.grpNameLen = RDB_bulkLen(p, grpName); + if(!(ctx->streamCtx.grpName = RDB_bulkClone(p, grpName))) + return RDB_ERR_FAIL_ALLOC; + + /* (re)allocate rax tree for group pel */ + if(ctx->streamCtx.groupPel) + deletePendingEntriesList(p, &ctx->streamCtx.groupPel); + if(!(ctx->streamCtx.groupPel = raxNew())) + return RDB_ERR_FAIL_ALLOC; + + int idLen = snprintf(idStr, sizeof(idStr), "%lu-%lu",meta->lastId.ms,meta->lastId.seq); + + if ( (meta->entriesRead>=0) && (ctx->targetRedisVerVal >= VER_VAL(7, 0))) { + /* XGROUP CREATE */ + IOV_CONST(&iov[iovs++], "*7\r\n$6\r\nXGROUP\r\n$6\r\nCREATE"); + /* key */ + IOV_LENGTH(&iov[iovs++], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[iovs++], ctx->keyCtx.key, ctx->keyCtx.keyLen); + /* group name */ + IOV_LENGTH(&iov[iovs++], ctx->streamCtx.grpNameLen, gNameLenStr); + IOV_STRING(&iov[iovs++], ctx->streamCtx.grpName, ctx->streamCtx.grpNameLen); + /* last id */ + IOV_LENGTH(&iov[iovs++], idLen, idLenStr); + IOV_STRING(&iov[iovs++], idStr, idLen); + /* entries read */ + IOV_CONST(&iov[iovs++], "\r\n$11\r\nENTRIESREAD"); + IOV_LEN_AND_VAL(&iov[iovs], meta->entriesRead, entriesReadLenStr, entriesReadStr); + iovs += 2; + } else { + /* XGROUP CREATE */ + IOV_CONST(&iov[iovs++], "*5\r\n$6\r\nXGROUP\r\n$6\r\nCREATE"); + /* key */ + IOV_LENGTH(&iov[iovs++], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[iovs++], ctx->keyCtx.key, ctx->keyCtx.keyLen); + /* group name */ + IOV_LENGTH(&iov[iovs++], ctx->streamCtx.grpNameLen, gNameLenStr); + IOV_STRING(&iov[iovs++], ctx->streamCtx.grpName, ctx->streamCtx.grpNameLen); + /* last id */ + IOV_LENGTH(&iov[iovs++], idLen, idLenStr); + IOV_STRING(&iov[iovs++], idStr, idLen); + IOV_CONST(&iov[iovs++], "\r\n"); + } + return writevWrap(ctx, iov, iovs, 1, 1); +} + +static RdbRes toRespStreamCGroupPendingEntry(RdbParser *p, void *userData, RdbStreamPendingEntry *pendingEntry) { + RdbxToResp *ctx = userData; + RdbStreamPendingEntry *pe; + + /* Make a copy pending entry */ + if ((pe = RDB_alloc(p, sizeof(RdbStreamPendingEntry))) == NULL) + return RDB_ERR_FAIL_ALLOC; + + memcpy(pe, pendingEntry, sizeof(RdbStreamPendingEntry)); + + if (!raxTryInsert(ctx->streamCtx.groupPel, (unsigned char *) &(pe->id), sizeof(pe->id), pe, NULL)) + return (RdbRes) RDBX_ERR_STREAM_DUPLICATE_PEL; - /* reset rawCtx */ - ctx->rawCtx.rawSize = size; - ctx->rawCtx.sentFirstFrag = 0; - ctx->rawCtx.isModuleAux = 0; - ctx->rawCtx.crc = 0; return RDB_OK; } -static inline RdbRes sendFirstRawFrag(RdbxToResp *ctx, RdbBulk frag, size_t fragLen) { - long long expireTime = 0; - char expireTimeStr[32], expireTimeLenStr[32], keyLenStr[32], lenStr[32]; - struct iovec iov[10]; - int extra_args = 0, iovs = 0; +static RdbRes toRespStreamNewConsumer(RdbParser *p, void *userData, RdbBulk consName, RdbStreamConsumerMeta *meta) { + UNUSED(meta); + RdbxToResp *ctx = userData; - /* this logic must be exactly the same as in toRespRawFragEnd() */ - if (ctx->targetVerValue >= VER_VAL(5,0)) - { - if (ctx->keyCtx.info.expiretime != -1) { - expireTime = ctx->keyCtx.info.expiretime; - extra_args++; /* ABSTTL */ - } + /* (re)allocate mem to keep consumer name */ + RDB_bulkCopyFree(p, ctx->streamCtx.consName); - if ((ctx->keyCtx.info.lfuFreq != -1) || (ctx->keyCtx.info.lruIdle != -1)) { - extra_args += 2; - } + ctx->streamCtx.consNameLen = RDB_bulkLen(p, consName); + if(!(ctx->streamCtx.consName = RDB_bulkClone(p, consName))) + return RDB_ERR_FAIL_ALLOC; + + return RDB_OK; +} + +/* Callback to handle a pending entry within a consumer */ +static RdbRes toRespStreamConsumerPendingEntry(RdbParser *p, void *userData, RdbStreamID *streamId) { + RdbStreamPendingEntry *pe; + char cmdTrailer[256], idStr[100], keyLenStr[32], gNameLenStr[32], cNameLenStr[32], sentTime[32], sentCount[32]; + struct iovec iov[16]; + int iovs = 0; + RdbxToResp *ctx = userData; + + if ((pe = raxFind(ctx->streamCtx.groupPel, (unsigned char *)streamId, sizeof(*streamId))) == raxNotFound) { + RDB_reportError(p, (RdbRes) RDBX_ERR_STREAM_INTEG_CHECK, + "toRespStreamNewConsumer(): Cannot find consumer pending entry in group PEL"); + return (RdbRes) RDBX_ERR_STREAM_INTEG_CHECK; } - if (ctx->keyCtx.delBeforeWrite == DEL_KEY_BEFORE_BY_RESTORE_REPLACE) - extra_args++; + /* XCLAIM */ + IOV_CONST(&iov[iovs++], "*12\r\n$6\r\nXCLAIM"); + /* key */ + IOV_LENGTH(&iov[iovs++], ctx->keyCtx.keyLen, keyLenStr); + IOV_STRING(&iov[iovs++], ctx->keyCtx.key, ctx->keyCtx.keyLen); + /* group name */ + IOV_LENGTH(&iov[iovs++], ctx->streamCtx.grpNameLen, gNameLenStr); + IOV_STRING(&iov[iovs++], ctx->streamCtx.grpName, ctx->streamCtx.grpNameLen); + + /* consumer name */ + IOV_LENGTH(&iov[iovs++], ctx->streamCtx.consNameLen, cNameLenStr); + IOV_STRING(&iov[iovs++], ctx->streamCtx.consName, ctx->streamCtx.consNameLen); + /* trailer of the command */ + int idLen = snprintf(idStr, sizeof(idStr), "%lu-%lu",streamId->ms, streamId->seq); + int sentTimeLen = ll2string(sentTime, sizeof(sentTime), pe->deliveryTime); + int sentCountLen = ll2string(sentCount, sizeof(sentCount), pe->deliveryCount); + int cmdTrailerLen = snprintf(cmdTrailer, sizeof(cmdTrailer), + "\r\n$1\r\n0\r\n$%d\r\n%s\r\n$4\r\nTIME\r\n$%d\r\n%s\r\n$10\r\nRETRYCOUNT\r\n$%d\r\n%s\r\n$6\r\nJUSTID\r\n$5\r\nFORCE\r\n", + idLen, idStr, sentTimeLen, sentTime, sentCountLen, sentCount); + /* max: 2 + 2 + 1 + 3 + 21*2+1 + 2 + 4 + 3 + 21 + 2 + 10 + 3 +21 +2 + 6 + 2 +5 + 2*16 */ + IOV_STRING(&iov[iovs++], cmdTrailer, cmdTrailerLen); + return writevWrap(ctx, iov, iovs, 1, 1); +} - char cmd[64]; +/*** Handling raw (RESTORE) ***/ +/* Callback on start of serializing module aux data (alternative to toRespRestoreBegin). + * Following this call, one or more calls will be made to toRespRestoreFrag() to + * stream fragments of the serialized data. And at the end toRespRestoreFragEnd() + * will be called */ +static RdbRes toRespRestoreBeginModuleAux(RdbParser *p, void *userData, RdbBulk name, int encver, int when, size_t rawSize) { + char encstr[10]; + UNUSED(p); - int len = snprintf(cmd, sizeof(cmd), "*%d\r\n$7\r\nRESTORE\r\n$", 4+extra_args); + /* reset restoreCtx */ + RdbxToResp *ctx = userData; + ctx->restoreCtx.restoreSize = rawSize; + ctx->restoreCtx.sentFirstFrag = 0; + ctx->restoreCtx.isModuleAux = 1; + ctx->restoreCtx.crc = 0; - IOV_STRING(&iov[iovs++], cmd, len); /* RESTORE */ - IOV_VALUE(&iov[iovs++], ctx->keyCtx.keyLen, keyLenStr); /* write key len */ - IOV_STRING(&iov[iovs++], ctx->keyCtx.key, ctx->keyCtx.keyLen); /* write key */ + /* if target doesn't support module-aux, then skip it */ + if (!ctx->conf.supportRestoreModuleAux) + return RDB_OK; - if (expireTime) { - IOV_CONST(&iov[iovs++], "\r\n$"); - IOV_LEN_AND_VALUE(&iov[iovs], expireTime, expireTimeLenStr, expireTimeStr); - iovs += 2; - IOV_CONST(&iov[iovs++], "$"); - } else { - IOV_CONST(&iov[iovs++], "\r\n$1\r\n0\r\n$"); - } - IOV_VALUE(&iov[iovs++], ctx->rawCtx.rawSize + 10, lenStr); /* write raw len + trailer */ - IOV_STRING(&iov[iovs++], frag, fragLen); /* write first frag */ - return writevWrap(ctx, iov, iovs, 1, 0); + /* Build the cmd instead of keeping the values and build it later */ + size_t enclen = snprintf(encstr, sizeof(encstr), "%d", encver); + const char* whenstr = (when==_REDISMODULE_AUX_BEFORE_RDB) ? "before" :"after"; + ctx->restoreCtx.moduleAux.cmdlen = snprintf(ctx->restoreCtx.moduleAux.cmdPrefix, + sizeof(ctx->restoreCtx.moduleAux.cmdPrefix), + "*5\r\n$13\r\nRESTOREMODAUX\r\n$%zu\r\n%s\r\n$%zu\r\n%s\r\n$%zu\r\n%s", + strlen(name), name, enclen, encstr, strlen(whenstr), whenstr); + return RDB_OK; } -static inline RdbRes sendFirstRawFragModuleAux(RdbxToResp *ctx, RdbBulk frag, size_t fragLen) { - struct iovec iov[3]; - char lenStr[32]; - iov[0].iov_base = ctx->rawCtx.moduleAux.cmdPrefix; - iov[0].iov_len = ctx->rawCtx.moduleAux.cmdlen; - IOV_VALUE(&iov[1], ctx->rawCtx.rawSize + 10, lenStr); /* write raw len + trailer */ - IOV_STRING(&iov[2], frag, fragLen); /* write first frag */ - return writevWrap(ctx, iov, 3, 1, 0); +/* Callback on start of serializing value of a key. Following this call, one + * or more calls will be made to toRespRestoreFrag() to stream fragments of the + * serialized data. And at the end toRespRestoreFragEnd() will be called */ +static RdbRes toRespRestoreBegin(RdbParser *p, void *userData, size_t size) { + UNUSED(p); + RdbxToResp *ctx = userData; + + /* reset restoreCtx */ + ctx->restoreCtx.restoreSize = size; + ctx->restoreCtx.sentFirstFrag = 0; + ctx->restoreCtx.isModuleAux = 0; + ctx->restoreCtx.crc = 0; + return RDB_OK; } /* Callback for fragments of a serialized value associated with a new key or module - * auxiliary data. This callback is invoked after toRespRawBegin() or - * toRespRawBeginModuleAux(), and it may be called multiple times until the - * serialization is complete. Finally, toRespRawFragEnd() will be called to signal + * auxiliary data. This callback is invoked after toRespRestoreBegin() or + * toRespRestoreBeginModuleAux(), and it may be called multiple times until the + * serialization is complete. Finally, toRespRestoreFragEnd() will be called to signal * the registered handlers for the completion of the operation. */ -static RdbRes toRespRawFrag(RdbParser *p, void *userData, RdbBulk frag) { +static RdbRes toRespRestoreFrag(RdbParser *p, void *userData, RdbBulk frag) { UNUSED(p); RdbxToResp *ctx = userData; struct iovec iov[10]; int iovs = 0; /* if processing module-aux but target doesn't support, then skip it */ - if ( (ctx->rawCtx.isModuleAux) && (!ctx->conf.supportRestoreModuleAux)) + if ((ctx->restoreCtx.isModuleAux) && (!ctx->conf.supportRestoreModuleAux)) return RDB_OK; size_t fragLen = RDB_bulkLen(p, frag); - ctx->rawCtx.crc = crc64(ctx->rawCtx.crc, (unsigned char *) frag , fragLen); + ctx->restoreCtx.crc = crc64(ctx->restoreCtx.crc, (unsigned char *) frag , fragLen); /* if first frag, handled differently */ - if (likely(!(ctx->rawCtx.sentFirstFrag))) { - ctx->rawCtx.sentFirstFrag = 1; - if (ctx->rawCtx.isModuleAux) - return sendFirstRawFragModuleAux(ctx, frag, fragLen); + if (likely(!(ctx->restoreCtx.sentFirstFrag))) { + ctx->restoreCtx.sentFirstFrag = 1; + if (ctx->restoreCtx.isModuleAux) + return sendFirstRestoreFragModuleAux(ctx, frag, fragLen); else - return sendFirstRawFrag(ctx, frag, fragLen); + return sendFirstRestoreFrag(ctx, frag, fragLen); } IOV_STRING(&iov[iovs++], frag, fragLen); return writevWrap(ctx, iov, iovs, 1, 0); } -/* This call will be followed one or more calls to toRespRawFrag() which indicates +/* This call will be followed one or more calls to toRespRestoreFrag() which indicates * for completion of streaming of fragments of serialized value of a new key or * module-aux data. */ -RdbRes toRespRawFragEnd(RdbParser *p, void *userData) { +static RdbRes toRespRestoreFragEnd(RdbParser *p, void *userData) { UNUSED(p); char cmd[1024]; /* degenerate usage of iov. All copied strings are small */ RdbxToResp *ctx = userData; - uint64_t *crc = &(ctx->rawCtx.crc); + uint64_t *crc = &(ctx->restoreCtx.crc); /* if processing module-aux but target doesn't support, then skip it */ - if ( (ctx->rawCtx.isModuleAux) && (!ctx->conf.supportRestoreModuleAux)) + if ((ctx->restoreCtx.isModuleAux) && (!ctx->conf.supportRestoreModuleAux)) return RDB_OK; - /* Add RDB version 2 bytes */ - cmd[0] = ctx->srcRdbVer & 0xff; - cmd[1] = (ctx->srcRdbVer >> 8) & 0xff; + /* Add RDB version 2 bytes. If it is module */ + + if (unlikely(ctx->keyCtx.info.opcode == _RDB_TYPE_MODULE_2)) { + /* Module object cannot forwarded to destination as a set of Redis commands. + * (Function resolveSupportRestore() enforced the parser to use RESTORE in case + * of modules) In order to avoid failure on downgrade when using RESTORE + * command, we are using the target rdb version for it */ + int rdbVer = (ctx->srcRdbVer > ctx->dstRdbVer) ? ctx->dstRdbVer : ctx->srcRdbVer; + cmd[0] = rdbVer & 0xff; + cmd[1] = (rdbVer >> 8) & 0xff; + } else { + cmd[0] = ctx->srcRdbVer & 0xff; + cmd[1] = (ctx->srcRdbVer >> 8) & 0xff; + } /* Add CRC64 8 bytes */ *crc = crc64(*crc, (unsigned char *) cmd, 2); @@ -606,8 +878,8 @@ RdbRes toRespRawFragEnd(RdbParser *p, void *userData) { else len += snprintf(cmd+len, sizeof(cmd)-len, "\r\n"); - /* This logic must be exactly the same as in toRespRawFrag() */ - if (likely(ctx->targetVerValue >= VER_VAL(5,0))) { + /* This logic must be exactly the same as in toRespRestoreFrag() */ + if (likely(ctx->targetRedisVerVal >= VER_VAL(5, 0))) { /* Add ABSTTL */ if (ctx->keyCtx.info.expiretime != -1) { @@ -636,8 +908,10 @@ RdbRes toRespRawFragEnd(RdbParser *p, void *userData) { _LIBRDB_API RdbxToResp *RDBX_createHandlersToResp(RdbParser *p, RdbxToRespConf *conf) { RdbxToResp *ctx; - /* Verify table up-to-date and aligned */ - assert(redisToRdbVersion[0].rdb == MAX_RDB_VER_SUPPORT); + crc64_init_thread_safe(); + + /* Verify table is aligned with LIBRDB_SUPPORT_MAX_RDB_VER */ + assert(redisToRdbVersion[0].rdb == RDB_getMaxSuppportRdbVersion()); if ((ctx = RDB_alloc(p, sizeof(RdbxToResp))) == NULL) return NULL; @@ -646,31 +920,52 @@ _LIBRDB_API RdbxToResp *RDBX_createHandlersToResp(RdbParser *p, RdbxToRespConf * if (conf) ctx->conf = *conf; ctx->parser = p; ctx->refcount = 2; - - RdbHandlersDataCallbacks dataCb; - memset(&dataCb, 0, sizeof(RdbHandlersDataCallbacks)); - dataCb.handleStartRdb = toRespStartRdb; - dataCb.handleNewDb = toRespNewDb; - dataCb.handleNewKey = toRespNewKey; - dataCb.handleEndKey = toRespEndKey; - dataCb.handleStringValue = toRespString; - dataCb.handleListItem = toRespList; - dataCb.handleHashField = toRespHash; - dataCb.handleSetMember = toRespSet; - dataCb.handleZsetMember = toRespZset; - dataCb.handleEndRdb = toRespEndRdb; - dataCb.handleFunction = toRespFunction; + ctx->streamCtx.xaddStartEndCounter = 0; + ctx->streamCtx.grpName = NULL; + ctx->streamCtx.groupPel = NULL; + + static RdbHandlersDataCallbacks dataCb = { + toRespStartRdb, + toRespEndRdb, + toRespNewDb, + NULL, /*db-size*/ + NULL, /*slot-info*/ + NULL, /*aux-field*/ + toRespNewKey, + toRespEndKey, + toRespString, + toRespList, + toRespHash, + toRespSet, + toRespZset, + toRespFunction, + NULL, /*module*/ + toRespStreamMetaData, + toRespStreamItem, + toRespStreamNewCGroup, + toRespStreamCGroupPendingEntry, + toRespStreamNewConsumer, + toRespStreamConsumerPendingEntry + }; RDB_createHandlersData(p, &dataCb, ctx, deleteRdbToRespCtx); - RdbHandlersRawCallbacks rawCb; - memset(&rawCb, 0, sizeof(RdbHandlersRawCallbacks)); - rawCb.handleStartRdb = NULL; /* already registered to this common callback */ - rawCb.handleNewKey = toRespNewKey; - rawCb.handleEndKey = toRespEndKey; - rawCb.handleFrag = toRespRawFrag; - rawCb.handleBeginModuleAux = toRespRawBeginModuleAux; - rawCb.handleBegin = toRespRawBegin; - rawCb.handleEnd = toRespRawFragEnd; + + + static RdbHandlersRawCallbacks rawCb = { + /* no need to register (twice) common cb. Already registered by dataCb */ + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + toRespNewKey, + toRespEndKey, + toRespRestoreBeginModuleAux, + toRespRestoreBegin, + toRespRestoreFrag, + toRespRestoreFragEnd, + }; RDB_createHandlersRaw(p, &rawCb, ctx, deleteRdbToRespCtx); return ctx; diff --git a/src/ext/readerFile.c b/src/ext/readerFile.c index ba322f4..443f22c 100644 --- a/src/ext/readerFile.c +++ b/src/ext/readerFile.c @@ -54,7 +54,8 @@ RdbxReaderFile *RDBX_createReaderFile(RdbParser *p, const char *filename) { } if (!(f = fopen(filename, "rb"))) { - RDB_reportError(p, RDB_ERR_FAILED_OPEN_RDB_FILE, "Failed to open RDB file: %s", filename); + RDB_reportError(p, RDB_ERR_FAILED_OPEN_RDB_FILE, + "Failed to open RDB file `%s`: %s\n", filename, strerror(errno)); return NULL; } diff --git a/src/ext/readerResp.c b/src/ext/readerResp.c index 61f4f03..98f20b5 100644 --- a/src/ext/readerResp.c +++ b/src/ext/readerResp.c @@ -11,6 +11,9 @@ static inline void unused(void *dummy, ...) { (void)(dummy);} #define MAX_RSP_BULK_SIZE 1024*1024 +/* This limit doesn't really exist in redis but at client side (hiredis) */ +#define MAX_ARRAY_ELEMENTS ((1LL<<32) - 1) + typedef enum RespReplyType { RESP_REPLY_IDLE=0, RESP_REPLY_STRING, @@ -29,12 +32,6 @@ typedef enum RespReplyType { RESP_REPLY_VERB, } RespReplyType; -typedef struct RespReplyBuff { - const char *buff; - int len; - int at; -} RespReplyBuff; - /*** static functions (private) ***/ static size_t charToString(char *buf, size_t size, char byte) { @@ -163,8 +160,7 @@ RespRes readRespReplyBulk(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { case PROC_BULK_READ_INIT: ctx->bulkLen = 0; ctx->bulkAt = 0; - ctx->typeState = PROC_BULK_READ_LEN; - break; + ctx->typeState = PROC_BULK_READ_LEN; /* fall-thru */ case PROC_BULK_READ_LEN: ch = buffInfo->buff[(buffInfo->at)]; @@ -239,18 +235,101 @@ RespRes readRespReplyBulk(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { /* fall-through */ case PROC_BULK_READ_END: + ctx->typeState = 0; return RESP_REPLY_OK; } } } -static RespRes readRespReplyAggregate(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { +static RespRes readRespReplyBulkArray(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { + char ch; + RespRes res; UNUSED(buffInfo); - /* Currently there are no commands, which sent by respToTcpLoader.c, that will cause to - * get back aggregated replies. Might change in the future */ - snprintf(ctx->errorMsg,sizeof(ctx->errorMsg),"Unexpected aggregate reply"); - return RESP_REPLY_ERR; + enum ProcessBulkArrayReadStates { + READ_INIT = 0, + READ_NUM_BULKS, + READ_NUM_BULKS_CR, + READ_NUM_BULKS_NL, + READ_NEXT_BULK_HDR, + READ_NEXT_BULK, + READ_END, + }; + + while (1) { + if (buffInfo->at == buffInfo->len) + return RESP_REPLY_PARTIAL; + + switch (ctx->typeArrayState) { + case READ_INIT: + ctx->numBulksArray = 0; + ctx->typeArrayState = READ_NUM_BULKS; /* fall-thru */ + + case READ_NUM_BULKS: + ch = buffInfo->buff[(buffInfo->at)]; + while ((ch >= '0') && (ch <= '9')) { + ctx->numBulksArray = ctx->numBulksArray * 10 + (ch - '0'); + + if (ctx->numBulksArray > MAX_ARRAY_ELEMENTS) { + snprintf(ctx->errorMsg, sizeof(ctx->errorMsg), "Multi-bulk length out of range"); + return RESP_REPLY_ERR; + } + ch = buffInfo->buff[(++(buffInfo->at))]; + + if (buffInfo->at == buffInfo->len) + return RESP_REPLY_PARTIAL; + } + + ctx->typeArrayState = READ_NUM_BULKS_CR; + break; + + case READ_NUM_BULKS_CR: + if (buffInfo->buff[buffInfo->at++] != '\r') { + snprintf(ctx->errorMsg, sizeof(ctx->errorMsg), + "Invalid Multi-Bulk response. Failed to read number of bulks"); + return RESP_REPLY_ERR; + } + ctx->typeArrayState = READ_NUM_BULKS_NL; + break; + + case READ_NUM_BULKS_NL: + if (buffInfo->buff[buffInfo->at++] != '\n') { + snprintf(ctx->errorMsg, sizeof(ctx->errorMsg), + "Invalid Bulk response. Failed to read bulk length"); + return RESP_REPLY_ERR; + } + + if (ctx->numBulksArray == 0) { + snprintf(ctx->errorMsg, sizeof(ctx->errorMsg), "Bulk Array must be bigger than zero"); + return RESP_REPLY_ERR; + } + + ctx->typeArrayState = READ_NEXT_BULK_HDR; + break; + + case READ_NEXT_BULK_HDR: + if (buffInfo->buff[buffInfo->at++] != '$') { + snprintf(ctx->errorMsg, sizeof(ctx->errorMsg), + "Invalid Multi-Bulk response. Failed to read Bulk header."); + return RESP_REPLY_ERR; + } + ctx->typeArrayState = READ_NEXT_BULK; /* fall-thru */ + + case READ_NEXT_BULK: + if ( (res = readRespReplyBulk(ctx, buffInfo)) != RESP_REPLY_OK) + return res; + + if (--ctx->numBulksArray) { + ctx->typeArrayState = READ_NEXT_BULK_HDR; + break; + } + ctx->typeArrayState = READ_END; /* fall-through */ + + case READ_END: + ctx->typeArrayState = 0; + return RESP_REPLY_OK; + } + } } static RespRes readRespReply(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { @@ -305,6 +384,7 @@ static RespRes readRespReply(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { /* start read type */ ctx->typeState = 0; + ctx->typeArrayState = 0; ctx->errorMsgLen = 0; buffInfo->at++; } @@ -327,7 +407,7 @@ static RespRes readRespReply(RespReaderCtx *ctx, RespReplyBuff *buffInfo) { case RESP_REPLY_MAP: case RESP_REPLY_SET: case RESP_REPLY_PUSH: - return readRespReplyAggregate(ctx, buffInfo); + return readRespReplyBulkArray(ctx, buffInfo); default: assert(NULL); return RESP_REPLY_ERR; /* Avoid warning. */ diff --git a/src/ext/readerResp.h b/src/ext/readerResp.h index eb62826..a7fd404 100644 --- a/src/ext/readerResp.h +++ b/src/ext/readerResp.h @@ -8,6 +8,12 @@ typedef enum RespReplyRes { RESP_REPLY_ERR, } RespRes; +typedef struct RespReplyBuff { + const char *buff; + int len; + int at; +} RespReplyBuff; + typedef struct { /* PUBLIC: read-only */ @@ -18,11 +24,15 @@ typedef struct { /* PRIVATE: */ int type; int typeState; + int typeArrayState; /* private bulk response state */ unsigned int bulkLen; unsigned int bulkAt; + /* private bulk-array response state */ + long long numBulksArray; + } RespReaderCtx; void readRespInit(RespReaderCtx *ctx); diff --git a/src/ext/respToRedisLoader.c b/src/ext/respToRedisLoader.c index 60e6c49..b242b44 100644 --- a/src/ext/respToRedisLoader.c +++ b/src/ext/respToRedisLoader.c @@ -19,7 +19,7 @@ #define NUM_RECORDED_CMDS 400 /* Number of commands to backlog, in a cyclic array */ #define RECORDED_DATA_MAX_LEN 40 /* Maximum payload size from any command to record into cyclic array */ -#define REPLY_BUFF_SIZE 4096 /* reply buffer size */ +#define REPLY_BUFF_SIZE 1024 /* reply buffer size */ #define MAX_EINTR_RETRY 3 @@ -65,8 +65,7 @@ static void onReadRepliesError(RdbxRespToRedisLoader *ctx) { } } -/* Read 'numToRead' replies from the socket. - * Return 0 for success, 1 otherwise. */ +/* Read 'numToRead' replies from the socket. * Return 0 for success, 1 otherwise. */ static int readReplies(RdbxRespToRedisLoader *ctx, int numToRead) { char buff[REPLY_BUFF_SIZE]; @@ -88,7 +87,7 @@ static int readReplies(RdbxRespToRedisLoader *ctx, int numToRead) { RDB_reportError(ctx->p, (RdbRes) RDBX_ERR_RESP2REDIS_CONN_CLOSE, "Connection closed by the remote side"); return 1; } else { - RDB_reportError(ctx->p, (RdbRes) RDBX_ERR_RESP2REDIS_FAILED_READ, "Failed to recv() from Redis server. Exit."); + RDB_reportError(ctx->p, (RdbRes) RDBX_ERR_RESP2REDIS_FAILED_READ, "Failed to recv() from Redis server. (errno=%d)", errno); return 1; } } diff --git a/src/lib/Makefile b/src/lib/Makefile index 52b0e60..919bed6 100644 --- a/src/lib/Makefile +++ b/src/lib/Makefile @@ -11,19 +11,28 @@ TARGET_LIB_STATIC = $(LIB_DIR)/lib$(LIB_NAME).a SOURCES = $(notdir $(basename $(wildcard *.c))) OBJECTS = $(patsubst %,%.o,$(SOURCES)) -# Source files in deps/redis directory +# Source files in deps/redis directory. For now, librdb.so and librdb-ext.so, +# each will have its own copy. Take care not to pass Redis structs from one lib +# to another! REDIS_SOURCES = $(notdir $(basename $(wildcard ../../deps/redis/*.c))) REDIS_OBJECTS = $(patsubst %,../../deps/redis/%.o,$(REDIS_SOURCES)) -EXTERN_RP_CONFIG= +OPTIMIZATION?=-O3 +LIBRDB_DEBUG?=0 STD = -std=c99 STACK = -fstack-protector-all -Wstack-protector WARNS = -Wall -Wextra -pedantic -Werror -CFLAGS = -fPIC -O3 $(STD) $(STACK) $(WARNS) $(EXTERN_RP_CONFIG) -DEBUG = -g3 -DDEBUG=1 +CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS) -fvisibility=hidden +DEBUG = -g3 LIBS = +ifeq ($(LIBRDB_DEBUG), 1) + CFLAGS += -DLIBRDB_DEBUG=1 +else + CFLAGS += -DNDEBUG=1 +endif + ######################################### RULES ####################################### all: $(TARGET_LIB) $(TARGET_LIB_STATIC) @echo "Done."; diff --git a/src/lib/bulkAlloc.c b/src/lib/bulkAlloc.c index 7c6b996..8f58fce 100644 --- a/src/lib/bulkAlloc.c +++ b/src/lib/bulkAlloc.c @@ -51,6 +51,7 @@ static inline BulkType bulkUnmanagedResolveAllocType(RdbParser *p, AllocUnmngTyp /*** LIB API functions ***/ _LIBRDB_API void RDB_bulkCopyFree(RdbParser *p, RdbBulkCopy b) { + if (!b) return; switch (p->mem.bulkAllocType) { case RDB_BULK_ALLOC_STACK: /* fall through - Note that even when bulkAllocType is set to RDB_BULK_ALLOC_STACK, @@ -293,6 +294,7 @@ int bulkPoolIsNewNextAllocDbg(RdbParser *p) { void bulkPoolAssertFlushedDbg(RdbParser *p) { BulkPool *pool = p->cache; + UNUSED(pool); /* if assert() disabled */ assert(pool->qwrite == pool->queue); assert(pool->qread == pool->queue); } diff --git a/src/lib/defines.h b/src/lib/defines.h index 0a4dea2..515e1c1 100644 --- a/src/lib/defines.h +++ b/src/lib/defines.h @@ -36,6 +36,7 @@ /* NOTE: WHEN ADDING NEW RDB TYPE, UPDATE rdbIsObjectType() BELOW */ /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ +#define RDB_OPCODE_SLOT_INFO 244 /* Individual slot info, such as slot id and size (cluster mode only). */ #define RDB_OPCODE_FUNCTION2 245 /* function library data */ #define RDB_OPCODE_FUNCTION 246 /* old function library data for 7.0 rc1 and rc2 */ #define RDB_OPCODE_MODULE_AUX 247 /* Module auxiliary data. */ @@ -108,4 +109,5 @@ #define RDB_MODULE_OPCODE_DOUBLE 4 /* Double. */ #define RDB_MODULE_OPCODE_STRING 5 /* String. */ +#define UNINIT_STREAM_ENTRIES_READ (-2) #endif /*DEFINES_H*/ diff --git a/src/lib/parser.c b/src/lib/parser.c index 7d625fe..71ad5f2 100644 --- a/src/lib/parser.c +++ b/src/lib/parser.c @@ -24,13 +24,14 @@ #include "parser.h" #include "version.h" #include "defines.h" -#include "../../deps/redis/endianconv.h" #include "../../deps/redis/util.h" +#include "../../deps/redis/endianconv.h" #include "../../deps/redis/listpack.h" #include "../../deps/redis/ziplist.h" #include "../../deps/redis/zipmap.h" #include "../../deps/redis/intset.h" #include "../../deps/redis/lzf.h" +#include "../../deps/redis/stream.h" #include "../../deps/redis/t_zset.h" #define DONE_FILL_BULK SIZE_MAX @@ -39,6 +40,7 @@ struct ParsingElementInfo peInfo[PE_MAX] = { [PE_RDB_HEADER] = {elementRdbHeader, "elementRdbHeader", "Start parsing RDB header"}, [PE_NEXT_RDB_TYPE] = {elementNextRdbType, "elementNextRdbType", "Parsing next RDB type"}, [PE_AUX_FIELD] = {elementAuxField, "elementAuxField", "Parsing auxiliary field" }, + [PE_SLOT_INFO] = {elementSlotInfo, "elementSlotInfo", "Parse cluster slot info"}, [PE_SELECT_DB] = {elementSelectDb, "elementSelectDb", "Parsing select-db"}, [PE_RESIZE_DB] = {elementResizeDb, "elementResizeDb", "Parsing resize-db"}, [PE_EXPIRETIME] = {elementExpireTime, "elementExpireTime", "Parsing expire-time"}, @@ -72,11 +74,13 @@ struct ParsingElementInfo peInfo[PE_MAX] = { [PE_ZSET_2] = {elementZset, "elementZset", "Parsing zset_2"}, [PE_ZSET_ZL] = {elementZsetZL, "elementZsetZL", "Parsing zset Ziplist"}, [PE_ZSET_LP] = {elementZsetLP, "elementZsetLP", "Parsing zset Listpack"}, - /* func */ + /* function */ [PE_FUNCTION] = {elementFunction, "elementFunction", "Parsing Function"}, /* module */ [PE_MODULE] = {elementModule, "elementModule", "Parsing silently Module element"}, [PE_MODULE_AUX] = {elementModule, "elementModule", "Parsing silently Module Auxiliary data"}, + /* stream */ + [PE_STREAM_LP] = {elementStreamLP, "elementStreamLP", "Parsing stream Listpack"}, /*** parsing raw data (RDB_LEVEL_RAW) ***/ @@ -106,6 +110,8 @@ struct ParsingElementInfo peInfo[PE_MAX] = { /* module */ [PE_RAW_MODULE] = {elementRawModule, "elementRawModule", "Parsing raw Module element"}, [PE_RAW_MODULE_AUX] = {elementRawModule, "elementRawModule(aux)", "Parsing Module Auxiliary data"}, + /* stream */ + [PE_RAW_STREAM_LP] = {elementRawStreamLP, "elementRawStreamLP", "Parsing raw stream Listpack"}, }; /* Strings in ziplist/listpacks are embedded without '\0' termination. To avoid @@ -506,8 +512,7 @@ _LIBRDB_API RdbHandlers *RDB_createHandlersData(RdbParser *p, return hndl; } -_LIBRDB_API void RDB_handleByLevel(RdbParser *p, RdbDataType type, RdbHandlersLevel lvl, unsigned int flags) { - UNUSED(flags); +_LIBRDB_API int RDB_handleByLevel(RdbParser *p, RdbDataType type, RdbHandlersLevel lvl) { switch (type) { case RDB_DATA_TYPE_STRING: p->handleTypeObjByLevel[RDB_TYPE_STRING] = lvl; @@ -548,9 +553,9 @@ _LIBRDB_API void RDB_handleByLevel(RdbParser *p, RdbDataType type, RdbHandlersLe p->handleTypeObjByLevel[RDB_OPCODE_FUNCTION2] = lvl; break; default: - assert(0); + return 1; } - + return 0; } _LIBRDB_API const char *RDB_getLibVersion(int *major, int *minor, int *patch) { @@ -560,6 +565,10 @@ _LIBRDB_API const char *RDB_getLibVersion(int *major, int *minor, int *patch) { return LIBRDB_VERSION_STRING; } +_LIBRDB_API int RDB_getMaxSuppportRdbVersion(void) { + return LIBRDB_SUPPORT_MAX_RDB_VER; +} + /*** various functions ***/ static const char *getStatusString(RdbStatus status) { @@ -569,7 +578,9 @@ static const char *getStatusString(RdbStatus status) { case RDB_STATUS_PAUSED: return "PAUSED"; case RDB_STATUS_ERROR: return "ERROR"; case RDB_STATUS_ENDED: return "(ENDED)"; /* internal state. (Not part of API) */ - default: assert(0); + default: + assert(0); + return "INVALID_STATUS!"; } } @@ -648,10 +659,9 @@ static RdbStatus parserMainLoop(RdbParser *p) { bulkPoolAssertFlushedDbg(p); } } else { - /* If this loop become too much performance intensive, then we can optimize - * certain transitions by avoiding passing through the main loop. It can be - * done by flushing the cache with function bulkPoolFlush(), and then make - * direct call to next state */ + /* Certain state transitions doesn't pass through the main loop. It is done + * by flushing the cache via function updateElementState(), and then make + * direct call or simply pass-through to next state */ while ((status = peInfo[p->parsingElement].func(p)) == RDB_STATUS_OK); } return updateStateAfterParse(p, status); @@ -730,15 +740,11 @@ static void resolveMultipleLevelsRegistration(RdbParser *p) { } static RdbStatus finalizeConfig(RdbParser *p, int isParseFromBuff) { - static int is_crc_init = 0; assert(p->state == RDB_STATE_CONFIGURING); RDB_log(p, RDB_LOG_INF, "Finalizing parser configuration"); - if (!is_crc_init) { - crc64_init(); - is_crc_init = 1; - } + crc64_init_thread_safe(); if ((p->debugData = getEnvVar(ENV_VAR_DEBUG_DATA, 0)) != 0) { RDB_setLogLevel(p, RDB_LOG_DBG); @@ -774,10 +780,9 @@ static RdbStatus finalizeConfig(RdbParser *p, int isParseFromBuff) { static void printParserState(RdbParser *p) { RDB_log(p, RDB_LOG_ERR, "Parser error message: %s", RDB_getErrorMessage(p)); RDB_log(p, RDB_LOG_ERR, "Parser error code: %d", RDB_getErrorCode(p)); - RDB_log(p, RDB_LOG_ERR, "Parser element func name: %s", peInfo[p->parsingElement].funcname); - RDB_log(p, RDB_LOG_ERR, "Parser element func description: %s", peInfo[p->parsingElement].description); - RDB_log(p, RDB_LOG_ERR, "Parser element state:%d", p->elmCtx.state); - //bulkPoolPrintDbg(p); + RDB_log(p, RDB_LOG_ERR, "Parser element func name: %s(state=%d)", + peInfo[p->parsingElement].funcname, p->elmCtx.state); + RDB_log(p, RDB_LOG_ERR, "Parsed opcode: %d", p->currOpcode); } static void loggerCbDefault(RdbLogLevel l, const char *msg) { @@ -817,7 +822,7 @@ RdbStatus allocFromCache(RdbParser *p, *binfo = bulkPoolAlloc(p, len, type, refBuf); if (unlikely( (*binfo)->ref == NULL)) { - RDB_reportError(p, RDB_ERR_NO_MEMORY, + RDB_reportError(p, RDB_ERR_FAIL_ALLOC, "allocFromCache() failed allocating %llu bytes (allocation type=%d)", (unsigned long long)len, type); @@ -988,7 +993,10 @@ static RdbStatus hashZiplist(RdbParser *p, BulkInfo *ziplistBulk) { registerAppBulkForNextCb(p, &embBulk1.binfo); registerAppBulkForNextCb(p, &embBulk2.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(p, &embBulk1); restoreEmbeddedBulk(p, &embBulk2);, /*finalize*/ + { /* Finalization (on either success or failure) */ + restoreEmbeddedBulk(p, &embBulk1); + restoreEmbeddedBulk(p, &embBulk2); + }, RDB_LEVEL_DATA, rdbData.handleHashField, embBulk1.binfo.ref, @@ -1040,7 +1048,10 @@ static RdbStatus hashListPack(RdbParser *p, BulkInfo *lpBulk) { registerAppBulkForNextCb(p, &embBulk1.binfo); registerAppBulkForNextCb(p, &embBulk2.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(p, &embBulk1); restoreEmbeddedBulk(p, &embBulk2);, /*finalize*/ + { /* Finalization (on either success or failure) */ + restoreEmbeddedBulk(p, &embBulk1); + restoreEmbeddedBulk(p, &embBulk2); + }, RDB_LEVEL_DATA, rdbData.handleHashField, embBulk1.binfo.ref, @@ -1080,7 +1091,10 @@ static RdbStatus hashZipMap(RdbParser *p, BulkInfo *zpBulk) { registerAppBulkForNextCb(p, &embBulk1.binfo); registerAppBulkForNextCb(p, &embBulk2.binfo); CALL_HANDLERS_CB(p, - restoreEmbeddedBulk(p, &embBulk1); restoreEmbeddedBulk(p, &embBulk2);, /*finalize*/ + { /* Finalization (on either success or failure) */ + restoreEmbeddedBulk(p, &embBulk1); + restoreEmbeddedBulk(p, &embBulk2); + }, RDB_LEVEL_DATA, rdbData.handleHashField, embBulk1.binfo.ref, @@ -1171,6 +1185,45 @@ static RdbStatus zsetZiplistItem(RdbParser *p, BulkInfo *ziplistBulk) { return RDB_STATUS_OK; } +static RdbStatus unzipStreamListpack(RdbParser *p, char *nodekey, unsigned char *lp) { + UNUSED(nodekey); + streamIterator si; + streamIteratorStart(&si,nodekey,lp); + RdbStreamID id; + int64_t numfields; + + while(streamIteratorGetID(&si,(streamID *) &id,&numfields)) { + while(numfields--) { + + unsigned char *field, *value; + int64_t field_len, value_len; + streamIteratorGetField(&si,&field,&value,&field_len,&value_len); + + EmbeddedBulk embField, embValue; + if (!allocEmbeddedBulk(p, field, field_len, 0, &embField)) + return RDB_STATUS_ERROR; + + if (!allocEmbeddedBulk(p, value, value_len, 0, &embValue)) + return RDB_STATUS_ERROR; + + registerAppBulkForNextCb(p, &embField.binfo); + registerAppBulkForNextCb(p, &embValue.binfo); + CALL_HANDLERS_CB(p, + { /* Finalization (on either success or failure) */ + restoreEmbeddedBulk(p, &embField); + restoreEmbeddedBulk(p, &embValue); + }, + RDB_LEVEL_DATA, + rdbData.handleStreamItem, + &id, + embField.binfo.ref, + embValue.binfo.ref, + numfields); + } + } + return RDB_STATUS_OK; +} + /*** Parsing Common Elements ***/ RdbStatus elementRdbHeader(RdbParser *p) { @@ -1189,7 +1242,7 @@ RdbStatus elementRdbHeader(RdbParser *p) { /* read rdb version */ p->rdbversion = atoi(((char *) binfo->ref) + 5); - if (p->rdbversion < 1 || p->rdbversion > MAX_RDB_VER_SUPPORT) { + if (p->rdbversion < 1 || p->rdbversion > LIBRDB_SUPPORT_MAX_RDB_VER) { RDB_reportError(p, RDB_ERR_UNSUPPORTED_RDB_VERSION, "Can't handle RDB format version: %d", p->rdbversion); return RDB_STATUS_ERROR; @@ -1232,6 +1285,19 @@ RdbStatus elementSelectDb(RdbParser *p) { return nextParsingElement(p, PE_NEXT_RDB_TYPE); } +RdbStatus elementSlotInfo(RdbParser *p) { + RdbSlotInfo info; + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &info.slot_id, NULL, NULL)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &info.slot_size, NULL, NULL)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &info.expires_slot_size, NULL, NULL)); + + /*** ENTER SAFE STATE ***/ + + CALL_COMMON_HANDLERS_CB(p, handleSlotInfo, &info); + + return nextParsingElement(p, PE_NEXT_RDB_TYPE); +} + RdbStatus elementResizeDb(RdbParser *p) { /* RESIZEDB: Hint about the size of the keys in the currently * selected data base, in order to avoid useless rehashing. */ @@ -1320,6 +1386,7 @@ RdbStatus elementNextRdbType(RdbParser *p) { case RDB_OPCODE_EXPIRETIME_MS: return nextParsingElement(p, PE_EXPIRETIMEMSEC); case RDB_OPCODE_AUX: return nextParsingElement(p, PE_AUX_FIELD); case RDB_OPCODE_SELECTDB: return nextParsingElement(p, PE_SELECT_DB); + case RDB_OPCODE_SLOT_INFO: return nextParsingElement(p, PE_SLOT_INFO); case RDB_OPCODE_RESIZEDB: return nextParsingElement(p, PE_RESIZE_DB); case RDB_OPCODE_FREQ: return nextParsingElement(p, PE_FREQ); case RDB_OPCODE_IDLE: return nextParsingElement(p, PE_IDLE); @@ -1340,6 +1407,12 @@ RdbStatus elementNextRdbType(RdbParser *p) { case RDB_TYPE_SET: return nextParsingElementKeyValue(p, PE_RAW_SET, PE_SET); case RDB_TYPE_SET_LISTPACK: return nextParsingElementKeyValue(p, PE_RAW_SET_LP, PE_SET_LP); case RDB_TYPE_SET_INTSET: return nextParsingElementKeyValue(p, PE_RAW_SET_IS, PE_SET_IS); + /* zset */ + case RDB_TYPE_ZSET: return nextParsingElementKeyValue(p, PE_RAW_ZSET, PE_ZSET); + case RDB_TYPE_ZSET_2: return nextParsingElementKeyValue(p, PE_RAW_ZSET, PE_ZSET); + case RDB_TYPE_ZSET_ZIPLIST: return nextParsingElementKeyValue(p, PE_RAW_ZSET_ZL, PE_ZSET_ZL); + case RDB_TYPE_ZSET_LISTPACK: return nextParsingElementKeyValue(p, PE_RAW_ZSET_LP, PE_ZSET_LP); + /* module */ case RDB_TYPE_MODULE_2: return nextParsingElementKeyValue(p, PE_RAW_MODULE, PE_MODULE); @@ -1352,19 +1425,10 @@ RdbStatus elementNextRdbType(RdbParser *p) { case RDB_OPCODE_EOF: return nextParsingElement(p, PE_END_OF_FILE); - /* zset */ - case RDB_TYPE_ZSET: return nextParsingElementKeyValue(p, PE_RAW_ZSET, PE_ZSET); - case RDB_TYPE_ZSET_2: return nextParsingElementKeyValue(p, PE_RAW_ZSET, PE_ZSET); - case RDB_TYPE_ZSET_ZIPLIST: return nextParsingElementKeyValue(p, PE_RAW_ZSET_ZL, PE_ZSET_ZL); - case RDB_TYPE_ZSET_LISTPACK: return nextParsingElementKeyValue(p, PE_RAW_ZSET_LP, PE_ZSET_LP); - - /* stream (TBD) */ + /* stream */ case RDB_TYPE_STREAM_LISTPACKS: case RDB_TYPE_STREAM_LISTPACKS_2: - case RDB_TYPE_STREAM_LISTPACKS_3: - RDB_reportError(p, RDB_ERR_NOT_SUPPORTED_RDB_ENCODING_TYPE, - "Not supported RDB encoding type: %d", p->currOpcode); - return RDB_STATUS_ERROR; + case RDB_TYPE_STREAM_LISTPACKS_3: return nextParsingElementKeyValue(p, PE_RAW_STREAM_LP, PE_STREAM_LP); case RDB_OPCODE_FUNCTION: RDB_reportError(p, RDB_ERR_PRERELEASE_FUNC_FORMAT_NOT_SUPPORTED, @@ -1372,7 +1436,7 @@ RdbStatus elementNextRdbType(RdbParser *p) { return RDB_STATUS_ERROR; default: - RDB_reportError(p, RDB_ERR_UNKNOWN_RDB_ENCODING_TYPE, "Unknown RDB encoding type"); + RDB_reportError(p, RDB_ERR_UNKNOWN_RDB_ENCODING_TYPE, "Unknown RDB encoding type: %d", p->currOpcode); return RDB_STATUS_ERROR; } } @@ -1433,22 +1497,27 @@ RdbStatus elementList(RdbParser *p) { /*** ENTER SAFE STATE ***/ - updateElementState(p, ST_LIST_NEXT_NODE); /* fall-thru */ + updateElementState(p, ST_LIST_NEXT_NODE, 0); /* fall-thru */ - case ST_LIST_NEXT_NODE: { - BulkInfo *binfoNode; - IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoNode)); + case ST_LIST_NEXT_NODE: + while (ctx->list.numNodes) { + BulkInfo *binfoNode; + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoNode)); - /*** ENTER SAFE STATE ***/ + /*** ENTER SAFE STATE ***/ - registerAppBulkForNextCb(p, binfoNode); - if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) - CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleListPlain, binfoNode->ref); - else - CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleListItem, binfoNode->ref); + --ctx->list.numNodes; + + registerAppBulkForNextCb(p, binfoNode); + if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleListPlain, binfoNode->ref); + else + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleListItem, binfoNode->ref); + + updateElementState(p, ST_LIST_NEXT_NODE, 0); + } + return nextParsingElement(p, PE_END_KEY); - return (--ctx->list.numNodes) ? updateElementState(p, ST_LIST_NEXT_NODE) : nextParsingElement(p, PE_END_KEY); - } default: RDB_reportError(p, RDB_ERR_PLAIN_LIST_INVALID_STATE, "elementList() : invalid parsing element state: %d", ctx->state); @@ -1469,59 +1538,63 @@ RdbStatus elementQuickList(RdbParser *p) { /*** ENTER SAFE STATE ***/ - updateElementState(p, ST_LIST_NEXT_NODE); /* fall-thru */ + updateElementState(p, ST_LIST_NEXT_NODE, 0); /* fall-thru */ - case ST_LIST_NEXT_NODE: { - uint64_t container = QUICKLIST_NODE_CONTAINER_PACKED; - BulkInfo *binfoNode; + case ST_LIST_NEXT_NODE: + while (ctx->list.numNodes) { + uint64_t container = QUICKLIST_NODE_CONTAINER_PACKED; + BulkInfo *binfoNode; - if (p->currOpcode == RDB_TYPE_LIST_QUICKLIST_2) { - IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &container, NULL, NULL)); + if (p->currOpcode == RDB_TYPE_LIST_QUICKLIST_2) { + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &container, NULL, NULL)); - if (container != QUICKLIST_NODE_CONTAINER_PACKED && - container != QUICKLIST_NODE_CONTAINER_PLAIN) { - RDB_reportError(p, RDB_ERR_QUICK_LIST_INTEG_CHECK, - "elementQuickList(1): Quicklist integrity check failed"); - return RDB_STATUS_ERROR; + if (container != QUICKLIST_NODE_CONTAINER_PACKED && + container != QUICKLIST_NODE_CONTAINER_PLAIN) { + RDB_reportError(p, RDB_ERR_QUICK_LIST_INTEG_CHECK, + "elementQuickList(1): Quicklist integrity check failed"); + return RDB_STATUS_ERROR; + } } - } - IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoNode)); + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoNode)); - /* ****************************** ENTER SAFE STATE ********************************* - * STARTING FROM THIS POINT, UP-TO END OF STATE, WON'T BE ANY MORE READS FROM RDB, * - * SO IT IS SAFE NOW TO CALL HANDLERS CALLBACKS WITHOUT THE RISK OF ROLLBACK DUE * - * TO `RDB_STATUS_WAIT_MORE_DATA` (WE CAN ADD LOCK VERIFICATION BY NEED). * - ***********************************************************************************/ + /* ****************************** ENTER SAFE STATE ********************************* + * STARTING FROM THIS POINT, UP-TO END OF STATE, WON'T BE ANY MORE READS FROM RDB, * + * SO IT IS SAFE NOW TO CALL HANDLERS CALLBACKS WITHOUT THE RISK OF ROLLBACK DUE * + * TO `RDB_STATUS_WAIT_MORE_DATA` (WE CAN ADD LOCK VERIFICATION BY NEED). * + ***********************************************************************************/ - if (container == QUICKLIST_NODE_CONTAINER_PLAIN) { - RdbHandlersLevel lvl = p->elmCtx.key.handleByLevel; + ctx->list.numNodes--; - registerAppBulkForNextCb(p, binfoNode); - if (lvl == RDB_LEVEL_STRUCT) - CALL_HANDLERS_CB(p, NOP, lvl, rdbStruct.handleListPlain, binfoNode->ref); - else - CALL_HANDLERS_CB(p, NOP, lvl, rdbData.handleListItem, binfoNode->ref); + if (container == QUICKLIST_NODE_CONTAINER_PLAIN) { + RdbHandlersLevel lvl = p->elmCtx.key.handleByLevel; - } else { + registerAppBulkForNextCb(p, binfoNode); + if (lvl == RDB_LEVEL_STRUCT) + CALL_HANDLERS_CB(p, NOP, lvl, rdbStruct.handleListPlain, binfoNode->ref); + else + CALL_HANDLERS_CB(p, NOP, lvl, rdbData.handleListItem, binfoNode->ref); - unsigned char *lp = (unsigned char *) binfoNode->ref; + } else { - if (p->currOpcode == RDB_TYPE_LIST_QUICKLIST_2) { - if (!lpValidateIntegrity(lp, binfoNode->len, p->deepIntegCheck, NULL, NULL)) { - RDB_reportError(p, RDB_ERR_QUICK_LIST_INTEG_CHECK, - "elementQuickList(2): Quicklist integrity check failed"); - return RDB_STATUS_ERROR; + unsigned char *lp = (unsigned char *) binfoNode->ref; + + if (p->currOpcode == RDB_TYPE_LIST_QUICKLIST_2) { + if (!lpValidateIntegrity(lp, binfoNode->len, p->deepIntegCheck, NULL, NULL)) { + RDB_reportError(p, RDB_ERR_QUICK_LIST_INTEG_CHECK, + "elementQuickList(2): Quicklist integrity check failed"); + return RDB_STATUS_ERROR; + } + IF_NOT_OK_RETURN(listListpackItem(p, binfoNode)); + } else { + if (RDB_STATUS_ERROR == listZiplistItem(p, binfoNode)) + return RDB_STATUS_ERROR; } - IF_NOT_OK_RETURN(listListpackItem(p, binfoNode)); - } else { - if (RDB_STATUS_ERROR == listZiplistItem(p, binfoNode)) - return RDB_STATUS_ERROR; } - } - return (--ctx->list.numNodes) ? updateElementState(p, ST_LIST_NEXT_NODE) : nextParsingElement(p, PE_END_KEY); - } + updateElementState(p, ST_LIST_NEXT_NODE, 0); + } + return nextParsingElement(p, PE_END_KEY); default: RDB_reportError(p, RDB_ERR_QUICK_LIST_INVALID_STATE, @@ -1559,7 +1632,7 @@ RdbStatus elementHash(RdbParser *p) { /*** ENTER SAFE STATE ***/ - updateElementState(p, ST_HASH_NEXT); /* fall-thru */ + updateElementState(p, ST_HASH_NEXT, 0); /* fall-thru */ case ST_HASH_NEXT: { BulkInfo *binfoField, *binfoValue; @@ -1586,7 +1659,7 @@ RdbStatus elementHash(RdbParser *p) { } ++ctx->hash.visitingField; - return updateElementState(p, ST_HASH_NEXT); + return updateElementState(p, ST_HASH_NEXT, 0); } default: @@ -1603,7 +1676,8 @@ RdbStatus elementHashZL(RdbParser *p) { /*** ENTER SAFE STATE ***/ - IF_NOT_OK_RETURN(hashZiplist(p, ziplistBulk)); + if (RDB_STATUS_ERROR == hashZiplist(p, ziplistBulk)) + return RDB_STATUS_ERROR; return nextParsingElement(p, PE_END_KEY); } @@ -1615,7 +1689,8 @@ RdbStatus elementHashLP(RdbParser *p) { /*** ENTER SAFE STATE ***/ - IF_NOT_OK_RETURN(hashListPack(p, listpackBulk)); + if (RDB_STATUS_ERROR == hashListPack(p, listpackBulk)) + return RDB_STATUS_ERROR; return nextParsingElement(p, PE_END_KEY); } @@ -1627,7 +1702,8 @@ RdbStatus elementHashZM(RdbParser *p) { /*** ENTER SAFE STATE ***/ - IF_NOT_OK_RETURN(hashZipMap(p, zipmapBulk)); + if (RDB_STATUS_ERROR == hashZipMap(p, zipmapBulk)) + return RDB_STATUS_ERROR; return nextParsingElement(p, PE_END_KEY); } @@ -1646,22 +1722,27 @@ RdbStatus elementSet(RdbParser *p) { ctx->set.left = ctx->key.numItemsHint; - updateElementState(p, ST_SET_NEXT_ITEM); /* fall-thru */ + updateElementState(p, ST_SET_NEXT_ITEM, 0); /* fall-thru */ - case ST_SET_NEXT_ITEM: { - BulkInfo *binfoItem; - IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoItem)); + case ST_SET_NEXT_ITEM: + while(ctx->set.left) { + BulkInfo *binfoItem; + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoItem)); - /*** ENTER SAFE STATE ***/ + /*** ENTER SAFE STATE ***/ - registerAppBulkForNextCb(p, binfoItem); - if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) - CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleSetPlain, binfoItem->ref); - else - CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleSetMember, binfoItem->ref); + ctx->set.left--; + + registerAppBulkForNextCb(p, binfoItem); + if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleSetPlain, binfoItem->ref); + else + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleSetMember, binfoItem->ref); + + updateElementState(p, ST_SET_NEXT_ITEM, 0); + } + return nextParsingElement(p, PE_END_KEY); - return (--ctx->set.left) ? updateElementState(p, ST_SET_NEXT_ITEM) : nextParsingElement(p, PE_END_KEY); - } default: RDB_reportError(p, RDB_ERR_PLAIN_SET_INVALID_STATE, "elementSet() : invalid parsing element state: %d", ctx->state); @@ -1719,7 +1800,6 @@ RdbStatus elementSetLP(RdbParser *p) { return RDB_STATUS_ERROR; } - /* TODO: handle empty listpack */ p->elmCtx.key.numItemsHint = lpLength(listpackBulk->ref); if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) { @@ -1761,30 +1841,34 @@ RdbStatus elementZset(RdbParser *p) { ctx->zset.left = ctx->key.numItemsHint; - updateElementState(p, ST_ZSET_NEXT_ITEM); /* fall-thru */ + updateElementState(p, ST_ZSET_NEXT_ITEM, 0); /* fall-thru */ case ST_ZSET_NEXT_ITEM: { double score; BulkInfo *binfoItem; + while (1) { + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoItem)); - IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoItem)); - - if (p->currOpcode == RDB_TYPE_ZSET_2) { - IF_NOT_OK_RETURN(rdbLoadBinaryDoubleValue(p, &score)); - } else { - IF_NOT_OK_RETURN(rdbLoadDoubleValue(p, &score)); - } + if (p->currOpcode == RDB_TYPE_ZSET_2) { + IF_NOT_OK_RETURN(rdbLoadBinaryDoubleValue(p, &score)); + } else { + IF_NOT_OK_RETURN(rdbLoadDoubleValue(p, &score)); + } - /*** ENTER SAFE STATE ***/ + /*** ENTER SAFE STATE ***/ - registerAppBulkForNextCb(p, binfoItem); - if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) - CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleZsetPlain, binfoItem->ref, score); - else - CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleZsetMember, binfoItem->ref, score); + registerAppBulkForNextCb(p, binfoItem); + if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleZsetPlain, binfoItem->ref, score); + else + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleZsetMember, binfoItem->ref, score); - return (--ctx->zset.left) ? updateElementState(p, ST_ZSET_NEXT_ITEM) : nextParsingElement(p, PE_END_KEY); + if (--ctx->zset.left) + updateElementState(p, ST_ZSET_NEXT_ITEM, 0); + else + return nextParsingElement(p, PE_END_KEY); + } } default: RDB_reportError(p, RDB_ERR_PLAIN_ZSET_INVALID_STATE, @@ -1939,7 +2023,7 @@ RdbStatus elementModule(RdbParser *p) { } /*** ENTER SAFE STATE ***/ ctx->module.startBytesRead = p->bytesRead - hdrSize ; - updateElementState(p, ST_MODULE_NEXT_OPCODE); + updateElementState(p, ST_MODULE_NEXT_OPCODE, 0); break; } case ST_MODULE_OPCODE_SINT: @@ -1947,28 +2031,28 @@ RdbStatus elementModule(RdbParser *p) { uint64_t val; /*UNUSED*/ IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &val, NULL, NULL)); /*** ENTER SAFE STATE ***/ - updateElementState(p, ST_MODULE_NEXT_OPCODE); + updateElementState(p, ST_MODULE_NEXT_OPCODE, 0); break; } case ST_MODULE_OPCODE_FLOAT: { float val; /*UNUSED*/ IF_NOT_OK_RETURN(rdbLoadFloatValue(p, &val)); /*** ENTER SAFE STATE ***/ - updateElementState(p, ST_MODULE_NEXT_OPCODE); + updateElementState(p, ST_MODULE_NEXT_OPCODE, 0); break; } case ST_MODULE_OPCODE_DOUBLE: { double val; /*UNUSED*/ IF_NOT_OK_RETURN(rdbLoadBinaryDoubleValue(p, &val)); /*** ENTER SAFE STATE ***/ - updateElementState(p, ST_MODULE_NEXT_OPCODE); + updateElementState(p, ST_MODULE_NEXT_OPCODE, 0); break; } case ST_MODULE_OPCODE_STRING: { BulkInfo *bInfo; /*UNUSED*/ IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC, NULL, &bInfo)); /*** ENTER SAFE STATE ***/ - updateElementState(p, ST_MODULE_NEXT_OPCODE); + updateElementState(p, ST_MODULE_NEXT_OPCODE, 0); break; } case ST_MODULE_NEXT_OPCODE: { @@ -1979,7 +2063,7 @@ RdbStatus elementModule(RdbParser *p) { if ((int) opcode != RDB_MODULE_OPCODE_EOF) { /* Valid cast. Took care to align opcode with module states */ - updateElementState(p, (int) opcode); + updateElementState(p, (int) opcode, 0); break; } @@ -2013,8 +2097,272 @@ RdbStatus elementModule(RdbParser *p) { } } +/*** stream ***/ + +/* The decoded ID will be stored + * in the 'id' structure passed by reference. The buffer 'buf' must point + * to a 128 bit big-endian encoded ID. */ +void streamDecodeID(void *buf, RdbStreamID *id) { + uint64_t e[2]; + memcpy(e,buf,sizeof(e)); + id->ms = ntohu64(e[0]); + id->seq = ntohu64(e[1]); +} + + +RdbStatus elementStreamLP(RdbParser *p) { + + enum STREAM_STATES { /* STATES FLOW: (Indentation represent conceptual nested loop among states) */ + ST_READ_NUM_LP=0, /* Read number of LP (lpLeft) to load */ + ST_LOAD_ALL_LP, /* Load all listpack */ + ST_LOAD_METADATA, /* Load Stream metadata */ + ST_LOAD_NEXT_CONS_GROUP, /* While more consumer-groups to load */ + ST_LOAD_GLOBAL_PEL, /* Load the global PEL for this consumer group */ + ST_LOAD_NUM_CONSUMERS, /* Load the number of consumers */ + ST_LOAD_NEXT_CONSUMER, /* While more consumers to load */ + ST_LOAD_NEXT_CONSUMER_PEL, /* Load the PEL about entries owned by next consumer */ + }; + + ElementCtx *ctx = &p->elmCtx; + + while (1) { + switch (ctx->state) { + case ST_READ_NUM_LP: + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &ctx->stream.numListPacks, NULL, NULL)); + /*** ENTER SAFE STATE ***/ + return updateElementState(p, ST_LOAD_ALL_LP, 0); /* fall-thru */ + + case ST_LOAD_ALL_LP: + while (ctx->stream.numListPacks > 0) { + BulkInfo *binfoNodekey, *binfoListPack; + + /* Get the master ID, the one we'll use as key of the radix tree node: the + entries inside the listpack itself are delta-encoded relatively to this ID. */ + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoNodekey)); + + if (binfoNodekey->len != sizeof(RdbStreamID)) { + RDB_reportError(p, RDB_ERR_STREAM_LP_INTEG_CHECK, "elementStreamLP(): Invalid node key entry"); + return RDB_STATUS_ERROR; + } + + /* Load the listpack. */ + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoListPack)); + + /*** ENTER SAFE STATE ***/ + + if (unlikely(!lpValidateIntegrity(binfoListPack->ref, binfoListPack->len, + p->deepIntegCheck, NULL, 0))) + { + RDB_reportError(p, RDB_ERR_STREAM_LP_INTEG_CHECK, + "elementStreamLP(): LISTPACK integrity check failed"); + return RDB_STATUS_ERROR; + } + + if (unlikely(lpFirst(binfoListPack->ref) == NULL)) { + /* Serialized listpacks should never be empty, since on deletion we should + * remove the radix tree key if the resulting listpack is empty. */ + RDB_reportError(p, RDB_ERR_STREAM_LP_INTEG_CHECK, + "elementStreamLP(): Empty listpack inside stream"); + return RDB_STATUS_ERROR; + } + + if (p->elmCtx.key.handleByLevel == RDB_LEVEL_STRUCT) { + registerAppBulkForNextCb(p, binfoListPack); + registerAppBulkForNextCb(p, binfoNodekey); + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_STRUCT, rdbStruct.handleStreamLP, binfoNodekey->ref, + binfoListPack->ref); + } else { + IF_NOT_OK_RETURN(unzipStreamListpack(p, binfoNodekey->ref, binfoListPack->ref)); + } + + ctx->stream.numListPacks--; + + updateElementState(p, ST_LOAD_ALL_LP, 0); + } + updateElementState(p, ST_LOAD_METADATA, 1); /* fall-thru */ + + case ST_LOAD_METADATA: { + RdbStreamMeta *streamMeta = &ctx->stream.streamMeta; + /* Load total number of items inside the stream. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamMeta->length, NULL, NULL)); + + /* Load the last entry ID. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamMeta->lastID.ms, NULL, NULL)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamMeta->lastID.seq, NULL, NULL)); + + if (p->currOpcode >= RDB_TYPE_STREAM_LISTPACKS_2) { + /* Load the first entry ID. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamMeta->firstID.ms, NULL, NULL)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamMeta->firstID.seq, NULL, NULL)); + + /* Load the maximal deleted entry ID. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamMeta->maxDelEntryID.ms, NULL, NULL)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamMeta->maxDelEntryID.seq, NULL, NULL)); + + /* Load entries added */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamMeta->entriesAdded, NULL, NULL)); + + } else { + /* During migration the offset can be initialized to the stream's + * length. At this point, we also don't care about tombstones + * because CG offsets will be later initialized as well. */ + streamMeta->firstID.ms = 0; + streamMeta->firstID.seq = 0; + streamMeta->maxDelEntryID.ms = 0; + streamMeta->maxDelEntryID.seq = 0; + streamMeta->entriesAdded = streamMeta->length; + } + + /* Load total number of items inside the stream. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &ctx->stream.cgroupsLeft, NULL, NULL)); /* fall-thru */ + + /*** ENTER SAFE STATE ***/ + + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleStreamMetadata, streamMeta); + } + updateElementState(p, ST_LOAD_NEXT_CONS_GROUP, 0); /* fall-thru */ + + case ST_LOAD_NEXT_CONS_GROUP: { + /* Get the consumer group name and ID. We can then create the consumer + * group ASAP and populate its structure as we read more data. */ + RdbStreamGroupMeta cgMeta; + BulkInfo *binfoNameCG; + + /* if not more consumer groups */ + if (!(ctx->stream.cgroupsLeft)) + return nextParsingElement(p, PE_END_KEY); + + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &binfoNameCG)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &cgMeta.lastId.ms, NULL, NULL)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &cgMeta.lastId.seq, NULL, NULL)); + + /* Load group offset. */ + if (p->currOpcode >= RDB_TYPE_STREAM_LISTPACKS_2) { + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, (uint64_t *) &cgMeta.entriesRead, + NULL, NULL)); + } else { + cgMeta.entriesRead = UNINIT_STREAM_ENTRIES_READ; + } + + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &(p->elmCtx.stream.pelLeft), NULL, NULL)); + + /*** ENTER SAFE STATE ***/ + + registerAppBulkForNextCb(p, binfoNameCG); + CALL_HANDLERS_CB(p, NOP, RDB_LEVEL_DATA, rdbData.handleStreamNewCGroup, binfoNameCG->ref, &cgMeta); + + ctx->stream.cgroupsLeft--; + } + updateElementState(p, ST_LOAD_GLOBAL_PEL, 0); /* fall-thru */ + + case ST_LOAD_GLOBAL_PEL: + while (p->elmCtx.stream.pelLeft) { + BulkInfo *binfoStreamID; + RdbStreamPendingEntry *pel = &p->elmCtx.stream.pel; + IF_NOT_OK_RETURN(rdbLoad(p, sizeof(RdbStreamID), RQ_ALLOC, NULL, &binfoStreamID)); + IF_NOT_OK_RETURN(rdbLoadMillisecTime(p, (int64_t *) &pel->deliveryTime)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &pel->deliveryCount, NULL, NULL)); + + /*** ENTER SAFE STATE ***/ + + streamDecodeID(binfoStreamID->ref, &pel->id); + + CALL_HANDLERS_CB(p,NOP, RDB_LEVEL_DATA, rdbData.handleStreamCGroupPendingEntry, pel); + p->elmCtx.stream.pelLeft--; + updateElementState(p, ST_LOAD_GLOBAL_PEL, 0); + } + updateElementState(p, ST_LOAD_NUM_CONSUMERS, 0); /* fall-thru */ + + case ST_LOAD_NUM_CONSUMERS: + /* Now that we loaded our global PEL, we need to load the consumers and their local PELs. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &(p->elmCtx.stream.consumersLeft), NULL, NULL)); + /*** ENTER SAFE STATE ***/ + updateElementState(p, ST_LOAD_NEXT_CONSUMER, 0); /* fall-thru */ + + case ST_LOAD_NEXT_CONSUMER: + if (p->elmCtx.stream.consumersLeft == 0) { + updateElementState(p, ST_LOAD_NEXT_CONS_GROUP, 0); + break; + } + BulkInfo *bConsName; + RdbStreamConsumerMeta consMeta; + int64_t int64tmp; + + IF_NOT_OK_RETURN(rdbLoadString(p, RQ_ALLOC_APP_BULK, NULL, &bConsName)); + IF_NOT_OK_RETURN(rdbLoadMillisecTime(p, &int64tmp)); + consMeta.seenTime = (long long) int64tmp; + + if (p->currOpcode >= RDB_TYPE_STREAM_LISTPACKS_3) { + IF_NOT_OK_RETURN(rdbLoadMillisecTime(p, &int64tmp)); + consMeta.activeTime = (long long) int64tmp; + } else { + /* That's the best estimate we got */ + consMeta.activeTime = consMeta.seenTime; + } + + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &(p->elmCtx.stream.consumer.pelLeft), NULL, NULL)); + + /*** ENTER SAFE STATE ***/ + + registerAppBulkForNextCb(p, bConsName); + CALL_HANDLERS_CB(p,NOP, RDB_LEVEL_DATA, rdbData.handleStreamNewConsumer, bConsName->ref, &consMeta); + + p->elmCtx.stream.consumersLeft--; + + updateElementState(p, ST_LOAD_NEXT_CONSUMER_PEL, 0); /* fall-thru */ + + case ST_LOAD_NEXT_CONSUMER_PEL: + while (p->elmCtx.stream.consumer.pelLeft) { + BulkInfo *binfo; + IF_NOT_OK_RETURN(rdbLoad(p, sizeof(RdbStreamID), RQ_ALLOC, NULL, &binfo)); + + /*** ENTER SAFE STATE ***/ + + streamDecodeID(binfo->ref, binfo->ref); + + CALL_HANDLERS_CB(p,NOP, RDB_LEVEL_DATA, rdbData.handleStreamConsumerPendingEntry, (RdbStreamID *) binfo->ref); + p->elmCtx.stream.consumer.pelLeft--; + updateElementState(p, ST_LOAD_NEXT_CONSUMER_PEL, 0); + } + if (p->elmCtx.stream.consumersLeft) + updateElementState(p, ST_LOAD_NEXT_CONSUMER, 0); + else + updateElementState(p, ST_LOAD_NEXT_CONS_GROUP, 0); + break; + + default: + RDB_reportError(p, RDB_ERR_STREAM_INVALID_STATE, + "elementStreamLP() : Invalid parsing element state: %d.", ctx->state); + return RDB_STATUS_ERROR; + break; + } + } + return RDB_STATUS_OK; +} + /*** Loaders from RDB ***/ +/* This function loads a time from the RDB file. The reason for conditional + * conversion below is because before Redis 5 (RDB version 9), the function + * failed to convert data to/from little endian, so RDB files with keys having + * expires could not be shared between big endian and little endian systems + * (because the expire time will be totally wrong). The fix for this is just + * to call memrev64ifbe(), however if we fix this for all the RDB versions, + * this call will introduce an incompatibility for big endian systems: + * after upgrading to Redis version 5 they will no longer be able to load their + * own old RDB files. Because of that, we instead fix the function only for new + * RDB versions, and load older RDB versions as we used to do in the past, + * allowing big endian systems to load their own old RDB files. */ +RdbStatus rdbLoadMillisecTime(RdbParser *p, int64_t *val) { + BulkInfo *binfo; + IF_NOT_OK_RETURN(rdbLoad(p, 8, RQ_ALLOC, NULL, &binfo)); + /*** ENTER SAFE STATE ***/ + *val = *((int64_t *) binfo->ref); + if (p->rdbversion >= 9) /* see comment above */ + memrev64ifbe(val); + return RDB_STATUS_OK; +} + RdbStatus rdbLoadFloatValue(RdbParser *p, float *val) { BulkInfo *binfo; IF_NOT_OK_RETURN(rdbLoad(p, sizeof(*val), RQ_ALLOC, NULL, &binfo)); diff --git a/src/lib/parser.h b/src/lib/parser.h index 511f5bd..a0bf7dc 100644 --- a/src/lib/parser.h +++ b/src/lib/parser.h @@ -92,7 +92,7 @@ typedef struct BulkInfo { /* Allocation requests from the parser to BulkPool */ typedef enum { /* Allocate for internal use of the parser */ - RQ_ALLOC, + RQ_ALLOC, /* either alloc from internal stack or heap */ RQ_ALLOC_REF, /*placement-new*/ /* Allocate RdbBulk in order to pass it to app callbacks */ RQ_ALLOC_APP_BULK, @@ -117,6 +117,7 @@ typedef enum ParsingElementType { PE_NEXT_RDB_TYPE, PE_AUX_FIELD, PE_SELECT_DB, + PE_SLOT_INFO, PE_RESIZE_DB, PE_EXPIRETIME, PE_EXPIRETIMEMSEC, @@ -145,6 +146,7 @@ typedef enum ParsingElementType { PE_MODULE, PE_FUNCTION, PE_MODULE_AUX, + PE_STREAM_LP, /* parsing raw data types */ PE_RAW_NEW_KEY, @@ -166,6 +168,7 @@ typedef enum ParsingElementType { PE_RAW_ZSET_LP, PE_RAW_MODULE, PE_RAW_MODULE_AUX, + PE_RAW_STREAM_LP, PE_END_OF_FILE, PE_MAX @@ -204,6 +207,18 @@ typedef struct { size_t startBytesRead; /* to evaluate how many bytes in total of current module */ } ElementModuleCtx; +typedef struct { + uint64_t numListPacks; + RdbStreamMeta streamMeta; + uint64_t cgroupsLeft; + uint64_t pelLeft; /* pending entry list */ + RdbStreamPendingEntry pel; + uint64_t consumersLeft; + struct { + uint64_t pelLeft; + } consumer; /* current processed consumer */ +} ElementStreamCtx; + typedef struct { RdbKeyInfo info; ParsingElementType valueType; @@ -241,6 +256,13 @@ typedef struct { uint64_t when; } ElementRawModuleAux; +typedef struct { + uint64_t lpLeft; + uint64_t cgroupsLeft; + uint64_t globPelLeft; /* global pending entries left to read */ + uint64_t consumersLeft; +} ElementRawStreamCtx; + typedef struct ElementCtx { ElementKeyCtx key; ElementListCtx list; @@ -248,6 +270,7 @@ typedef struct ElementCtx { ElementZsetCtx zset; ElementHashCtx hash; ElementModuleCtx module; + ElementStreamCtx stream; /* raw elements context */ ElementRawStringCtx rawString; @@ -256,6 +279,7 @@ typedef struct ElementCtx { ElementRawZsetCtx rawZset; ElementRawHashCtx rawHash; ElementRawModuleAux rawModAux; + ElementRawStreamCtx rawStream; int state; /* parsing-element state */ } ElementCtx; @@ -420,8 +444,12 @@ static inline void registerAppBulkForNextCb(RdbParser *p, BulkInfo *binfo) { extern void bulkPoolFlush(RdbParser *p); /* avoid cyclic headers inclusion */ -static inline RdbStatus updateElementState(RdbParser *p, int newState) { - bulkPoolFlush(p); +static inline RdbStatus updateElementState(RdbParser *p, int newState, int noFlush) { + + /* if state completed without allocating anything then + * we can save few cpu cycles by avoid flushing the pool */ + if (!noFlush) bulkPoolFlush(p); + p->elmCtx.state = newState; return RDB_STATUS_OK; } @@ -446,6 +474,7 @@ RdbStatus subElementReturn(RdbParser *p, BulkInfo *bulkResult); void subElementCallEnd(RdbParser *p, RdbBulk *bulkResult, size_t *len); /*** Loaders from RDB ***/ +RdbStatus rdbLoadMillisecTime(RdbParser *p, int64_t *val); RdbStatus rdbLoadFloatValue(RdbParser *p, float *val); RdbStatus rdbLoadBinaryDoubleValue(RdbParser *p, double *val); RdbStatus rdbLoadDoubleValue(RdbParser *p, double *val); @@ -479,6 +508,7 @@ RdbStatus elementRdbHeader(RdbParser *p); RdbStatus elementNextRdbType(RdbParser *p); RdbStatus elementAuxField(RdbParser *p); RdbStatus elementSelectDb(RdbParser *p); +RdbStatus elementSlotInfo(RdbParser *p); RdbStatus elementResizeDb(RdbParser *p); RdbStatus elementExpireTime(RdbParser *p); RdbStatus elementExpireTimeMsec(RdbParser *p); @@ -502,6 +532,7 @@ RdbStatus elementZsetZL(RdbParser *p); RdbStatus elementZsetLP(RdbParser *p); RdbStatus elementFunction(RdbParser *p); RdbStatus elementModule(RdbParser *p); +RdbStatus elementStreamLP(RdbParser *p); /*** Raw Parsing Elements ***/ RdbStatus elementRawNewKey(RdbParser *p); @@ -521,5 +552,6 @@ RdbStatus elementRawZset(RdbParser *p); RdbStatus elementRawZsetLP(RdbParser *p); RdbStatus elementRawZsetZL(RdbParser *p); RdbStatus elementRawModule(RdbParser *p); +RdbStatus elementRawStreamLP(RdbParser *p); #endif /*LIBRDB_PARSER_H*/ diff --git a/src/lib/parserRaw.c b/src/lib/parserRaw.c index 73f3280..e276cb2 100644 --- a/src/lib/parserRaw.c +++ b/src/lib/parserRaw.c @@ -162,7 +162,7 @@ RdbStatus elementRawList(RdbParser *p) { } - updateElementState(p, ST_RAW_LIST_NEXT_NODE_CALL_STR); /* fall-thru */ + updateElementState(p, ST_RAW_LIST_NEXT_NODE_CALL_STR, 0); /* fall-thru */ case ST_RAW_LIST_NEXT_NODE_CALL_STR: return subElementCall(p, PE_RAW_STRING, ST_RAW_LIST_NEXT_NODE_STR_RETURN); @@ -180,7 +180,7 @@ RdbStatus elementRawList(RdbParser *p) { if (--listCtx->numNodes == 0) return nextParsingElement(p, PE_RAW_END_KEY); /* done */ - return updateElementState(p, ST_RAW_LIST_NEXT_NODE_CALL_STR); + return updateElementState(p, ST_RAW_LIST_NEXT_NODE_CALL_STR, 1); } default: @@ -219,7 +219,7 @@ RdbStatus elementRawQuickList(RdbParser *p) { } - updateElementState(p, ST_RAW_LIST_NEXT_NODE_CALL_STR); /* fall-thru */ + updateElementState(p, ST_RAW_LIST_NEXT_NODE_CALL_STR, 0); /* fall-thru */ case ST_RAW_LIST_NEXT_NODE_CALL_STR: { listCtx->container = QUICKLIST_NODE_CONTAINER_PACKED; @@ -270,7 +270,7 @@ RdbStatus elementRawQuickList(RdbParser *p) { if (--listCtx->numNodes == 0) return nextParsingElement(p, PE_RAW_END_KEY); /* done */ - return updateElementState(p, ST_RAW_LIST_NEXT_NODE_CALL_STR); + return updateElementState(p, ST_RAW_LIST_NEXT_NODE_CALL_STR, 0); } default: @@ -328,10 +328,10 @@ RdbStatus elementRawString(RdbParser *p) { IF_NOT_OK_RETURN(aggUpdateWritten(p, headerlen)); if (p->callSubElm.callerElm != PE_MAX) - return updateElementState(p, ST_RAW_STRING_PASS_AND_REPLY_CALLER); + return updateElementState(p, ST_RAW_STRING_PASS_AND_REPLY_CALLER, 0); } - updateElementState(p, ST_RAW_STRING_PASS_CHUNKS); /* fall-thru */ + updateElementState(p, ST_RAW_STRING_PASS_CHUNKS, 0); /* fall-thru */ case ST_RAW_STRING_PASS_CHUNKS: { @@ -360,7 +360,7 @@ RdbStatus elementRawString(RdbParser *p) { if (!(strCtx->len)) /* stop condition */ return nextParsingElement(p, PE_RAW_END_KEY); - updateElementState(p, ST_RAW_STRING_PASS_CHUNKS); + updateElementState(p, ST_RAW_STRING_PASS_CHUNKS, 0); } } @@ -466,7 +466,7 @@ RdbStatus elementRawHash(RdbParser *p) { IF_NOT_OK_RETURN(cbHandleBegin(p, DATA_SIZE_UNKNOWN_AHEAD)); IF_NOT_OK_RETURN(aggUpdateWritten(p, headerLen)); } - updateElementState(p, ST_RAW_HASH_READ_NEXT_FIELD_STR); /* fall-thru */ + updateElementState(p, ST_RAW_HASH_READ_NEXT_FIELD_STR, 0); /* fall-thru */ case ST_RAW_HASH_READ_NEXT_FIELD_STR: /*** ENTER SAFE STATE ***/ @@ -545,7 +545,7 @@ RdbStatus elementRawSet(RdbParser *p) { IF_NOT_OK_RETURN(cbHandleBegin(p, DATA_SIZE_UNKNOWN_AHEAD)); IF_NOT_OK_RETURN(aggUpdateWritten(p, headerLen)); } - updateElementState(p, ST_RAW_SET_NEXT_ITEM_CALL_STR); /* fall-thru */ + updateElementState(p, ST_RAW_SET_NEXT_ITEM_CALL_STR, 0); /* fall-thru */ case ST_RAW_SET_NEXT_ITEM_CALL_STR: return subElementCall(p, PE_RAW_STRING, ST_RAW_SET_NEXT_ITEM_STR_RETURN); @@ -562,7 +562,7 @@ RdbStatus elementRawSet(RdbParser *p) { if (--setCtx->numItems == 0) return nextParsingElement(p, PE_RAW_END_KEY); /* done */ - return updateElementState(p, ST_RAW_SET_NEXT_ITEM_CALL_STR); + return updateElementState(p, ST_RAW_SET_NEXT_ITEM_CALL_STR, 1); } default: @@ -605,7 +605,7 @@ RdbStatus elementRawZset(RdbParser *p) { IF_NOT_OK_RETURN(cbHandleBegin(p, DATA_SIZE_UNKNOWN_AHEAD)); IF_NOT_OK_RETURN(aggUpdateWritten(p, headerLen)); } - updateElementState(p, ST_RAW_ZSET_READ_MEMBER); /* fall-thru */ + updateElementState(p, ST_RAW_ZSET_READ_MEMBER, 0); /* fall-thru */ case ST_RAW_ZSET_READ_MEMBER: return subElementCall(p, PE_RAW_STRING, ST_RAW_ZSET_READ_SCORE); @@ -635,7 +635,7 @@ RdbStatus elementRawZset(RdbParser *p) { if (--zsetCtx->numItems == 0) return nextParsingElement(p, PE_RAW_END_KEY); /* done */ - return updateElementState(p, ST_RAW_ZSET_READ_MEMBER); + return updateElementState(p, ST_RAW_ZSET_READ_MEMBER, 0); } default: RDB_reportError(p, RDB_ERR_PLAIN_ZSET_INVALID_STATE, @@ -648,17 +648,17 @@ RdbStatus elementRawModule(RdbParser *p) { typedef enum RAW_MODULE_STATES { /* Start handling module or module-aux */ - ST_RAW_MODULE_START=0, + ST_START=0, /* Following enums are aligned to module-opcodes */ - ST_RAW_MODULE_OPCODE_SINT=RDB_MODULE_OPCODE_SINT, - ST_RAW_MODULE_OPCODE_UINT=RDB_MODULE_OPCODE_UINT, - ST_RAW_MODULE_OPCODE_FLOAT=RDB_MODULE_OPCODE_FLOAT, - ST_RAW_MODULE_OPCODE_DOUBLE=RDB_MODULE_OPCODE_DOUBLE, - ST_RAW_MODULE_OPCODE_STRING_CALL_STR=RDB_MODULE_OPCODE_STRING, - - ST_RAW_MODULE_OPCODE_STRING_STR_RETURN, /* return from sub-element string parsing */ - ST_RAW_MODULE_NEXT_OPCODE, - ST_RAW_MODULE_AUX_START, + ST_OPCODE_SINT=RDB_MODULE_OPCODE_SINT, + ST_OPCODE_UINT=RDB_MODULE_OPCODE_UINT, + ST_OPCODE_FLOAT=RDB_MODULE_OPCODE_FLOAT, + ST_OPCODE_DOUBLE=RDB_MODULE_OPCODE_DOUBLE, + ST_OPCODE_STRING_CALL_STR=RDB_MODULE_OPCODE_STRING, + + ST_OPCODE_STRING_STR_RETURN, /* return from sub-element string parsing */ + ST_NEXT_OPCODE, + ST_AUX_START, } RAW_MODULE_STATES; RawContext *rawCtx = &p->rawCtx; @@ -666,7 +666,7 @@ RdbStatus elementRawModule(RdbParser *p) { while (1) { switch (p->elmCtx.state) { - case ST_RAW_MODULE_START: { + case ST_START: { int len = 0; if (p->currOpcode == RDB_TYPE_MODULE_2) { @@ -676,17 +676,17 @@ RdbStatus elementRawModule(RdbParser *p) { /*** ENTER SAFE STATE ***/ IF_NOT_OK_RETURN(cbHandleBegin(p, DATA_SIZE_UNKNOWN_AHEAD)); IF_NOT_OK_RETURN(aggUpdateWritten(p, len)); - updateElementState(p, ST_RAW_MODULE_NEXT_OPCODE); + updateElementState(p, ST_NEXT_OPCODE, 0); break; } /* No new-key precedes module aux. Init Aggregator of bulks here. * Note that the call is made from a safe state */ aggAllocFirstBulk(p); - updateElementState(p, ST_RAW_MODULE_AUX_START); + updateElementState(p, ST_AUX_START, 0); } /* fall-thru */ - case ST_RAW_MODULE_AUX_START: { + case ST_AUX_START: { int len = 0; ElementRawModuleAux *ma = &p->elmCtx.rawModAux; IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &ma->moduleid, NULL, &len)); @@ -700,12 +700,12 @@ RdbStatus elementRawModule(RdbParser *p) { /*** ENTER SAFE STATE ***/ IF_NOT_OK_RETURN(cbHandleBegin(p, DATA_SIZE_UNKNOWN_AHEAD)); IF_NOT_OK_RETURN(aggUpdateWritten(p, len)); - updateElementState(p, ST_RAW_MODULE_NEXT_OPCODE); + updateElementState(p, ST_NEXT_OPCODE, 0); break; } - case ST_RAW_MODULE_OPCODE_SINT: - case ST_RAW_MODULE_OPCODE_UINT: { + case ST_OPCODE_SINT: + case ST_OPCODE_UINT: { uint64_t val = 0; int len = 0; @@ -713,43 +713,43 @@ RdbStatus elementRawModule(RdbParser *p) { IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &val, (unsigned char *) rawCtx->at, &len)); /*** ENTER SAFE STATE ***/ IF_NOT_OK_RETURN(aggUpdateWritten(p, len)); - updateElementState(p, ST_RAW_MODULE_NEXT_OPCODE); + updateElementState(p, ST_NEXT_OPCODE, 0); break; } - case ST_RAW_MODULE_OPCODE_FLOAT: { + case ST_OPCODE_FLOAT: { IF_NOT_OK_RETURN(aggMakeRoom(p, sizeof(float))); IF_NOT_OK_RETURN(rdbLoadFloatValue(p, (float *) rawCtx->at)); /*** ENTER SAFE STATE ***/ IF_NOT_OK_RETURN(aggUpdateWritten(p, sizeof(float))); - updateElementState(p, ST_RAW_MODULE_NEXT_OPCODE); + updateElementState(p, ST_NEXT_OPCODE, 0); break; } - case ST_RAW_MODULE_OPCODE_DOUBLE: { + case ST_OPCODE_DOUBLE: { IF_NOT_OK_RETURN(aggMakeRoom(p, sizeof(double))); IF_NOT_OK_RETURN(rdbLoadBinaryDoubleValue(p, (double *) rawCtx->at)); /*** ENTER SAFE STATE ***/ IF_NOT_OK_RETURN(aggUpdateWritten(p, sizeof(double))); - updateElementState(p, ST_RAW_MODULE_NEXT_OPCODE); + updateElementState(p, ST_NEXT_OPCODE, 0); break; } - case ST_RAW_MODULE_OPCODE_STRING_CALL_STR: { + case ST_OPCODE_STRING_CALL_STR: { /* call raw string as subelement */ - return subElementCall(p, PE_RAW_STRING, ST_RAW_MODULE_OPCODE_STRING_STR_RETURN); + return subElementCall(p, PE_RAW_STRING, ST_OPCODE_STRING_STR_RETURN); } - case ST_RAW_MODULE_OPCODE_STRING_STR_RETURN: { + case ST_OPCODE_STRING_STR_RETURN: { /*** ENTER SAFE STATE (no rdb read)***/ size_t len; char *dataRet; subElementCallEnd(p, &dataRet, &len); - updateElementState(p, ST_RAW_MODULE_NEXT_OPCODE); + updateElementState(p, ST_NEXT_OPCODE, 1); break; } - case ST_RAW_MODULE_NEXT_OPCODE: { + case ST_NEXT_OPCODE: { int len = 0; uint64_t opcode = 0; @@ -760,7 +760,7 @@ RdbStatus elementRawModule(RdbParser *p) { if ((int) opcode != RDB_MODULE_OPCODE_EOF) { /* Valid cast. Took care to align opcode with module states */ - updateElementState(p, (RAW_MODULE_STATES) opcode); + updateElementState(p, (RAW_MODULE_STATES) opcode, 0); break; } @@ -784,6 +784,231 @@ RdbStatus elementRawModule(RdbParser *p) { } } +RdbStatus elementRawStreamLP(RdbParser *p) { + + enum RAW_STREAM_STATES { /* STATES FLOW: (Indentation represent conceptual nested loop among states) */ + ST_READ_NUM_LP=0, /* Read number of LP (lpLeft) to load */ + ST_LOAD_NEXT_LP_IS_MORE, /* While more LP to load */ + ST_LOAD_NEXT_LP_STR1_RETURN, /* Complete loading node-key of next LP */ + ST_LOAD_NEXT_LP_STR2_RETURN, /* Complete loading next LP */ + ST_LOAD_METADATA, /* Load Stream metadata */ + ST_LOAD_NEXT_CG_IS_MORE, /* While more CG (Consumer groups) to load */ + ST_LOAD_NEXT_CG_STR_RETURN, /* Complete loading CG name */ + ST_LOAD_GLOBAL_PEL, /* Load CG's global PEL (Pending Entries List) */ + ST_LOAD_NUM_CONSUMERS, /* Load number of consumers of current CG */ + ST_LOAD_NEXT_CONSUMER, /* Load next consumer */ + ST_LOAD_NEXT_CONSUMER_STR_RETURN, /* Complete loading consumer name. Load consumer PEL. */ + } ; + + ElementRawStreamCtx *streamCtx = &p->elmCtx.rawStream; + RawContext *rawCtx = &p->rawCtx; + + switch (p->elmCtx.state) { + + case ST_READ_NUM_LP: { + int headerLen = 0; + aggMakeRoom(p, 10); /* worse case 9 bytes for written */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamCtx->lpLeft, + (unsigned char *) rawCtx->at, &headerLen)); + /*** ENTER SAFE STATE ***/ + IF_NOT_OK_RETURN(cbHandleBegin(p, DATA_SIZE_UNKNOWN_AHEAD)); + IF_NOT_OK_RETURN(aggUpdateWritten(p, headerLen)); + } + updateElementState(p, ST_LOAD_NEXT_LP_IS_MORE, 0); /* fall-thru */ + + case ST_LOAD_NEXT_LP_IS_MORE: + /*** ENTER SAFE STATE ***/ + if (unlikely(!(streamCtx->lpLeft))) { + /* if no more listpacks to load, jump to load stream metadata */ + return updateElementState(p, ST_LOAD_METADATA, 1); + } + streamCtx->lpLeft--; + /* call raw string as sub-element to read nodekey */ + return subElementCall(p, PE_RAW_STRING, ST_LOAD_NEXT_LP_STR1_RETURN); + + case ST_LOAD_NEXT_LP_STR1_RETURN: { + size_t nodekeySize; + unsigned char *nodekeyUnused; + subElementCallEnd(p, (char **) &nodekeyUnused, &nodekeySize); + /* call raw string as sub-element to read listpack */ + return subElementCall(p, PE_RAW_STRING, ST_LOAD_NEXT_LP_STR2_RETURN); + } + + case ST_LOAD_NEXT_LP_STR2_RETURN: { + size_t lpSize; + unsigned char *lpUnused; + /*** ENTER SAFE STATE (no rdb read)***/ + subElementCallEnd(p, (char **) &lpUnused, &lpSize); + if (!lpValidateIntegrity(lpUnused, lpSize, p->deepIntegCheck, NULL, 0)) { + RDB_reportError(p, RDB_ERR_STREAM_LP_INTEG_CHECK, + "elementRawStreamLP(): LISTPACK integ check failed"); + return RDB_STATUS_ERROR; + } + } + updateElementState(p, ST_LOAD_NEXT_LP_IS_MORE, 1); /* fall-thru */ + + case ST_LOAD_METADATA: { + uint64_t dummyVal; + + aggMakeRoom(p, LONG_STR_SIZE * 9); + + /* Load total number of items inside the stream. */ + int written = 0; + IF_NOT_OK_RETURN( rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), + &written)); /* meta->length */ + + /* Load the last entry ID. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + + if (p->currOpcode >= RDB_TYPE_STREAM_LISTPACKS_2) { + /* Load the first entry ID. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + + /* Load the maximal deleted entry ID. */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + + /* Load entries added */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + } + + /* Load total number of items inside the stream */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &streamCtx->cgroupsLeft, + (unsigned char *) (rawCtx->at + written), &written)); + + /*** ENTER SAFE STATE ***/ + + IF_NOT_OK_RETURN(aggUpdateWritten(p, written)); + } + updateElementState(p, ST_LOAD_NEXT_CG_IS_MORE, 0); /* fall-thru */ + + case ST_LOAD_NEXT_CG_IS_MORE: + /*** ENTER SAFE STATE (no rdb read)***/ + if (unlikely(!(streamCtx->cgroupsLeft))) { + /* if no more consumer-groups to load, then reached end of key */ + return nextParsingElement(p, PE_RAW_END_KEY); /* done */ + } + streamCtx->cgroupsLeft--; + /* call raw string as sub-element to read consumer-group name */ + return subElementCall(p, PE_RAW_STRING, ST_LOAD_NEXT_CG_STR_RETURN); + + case ST_LOAD_NEXT_CG_STR_RETURN: { + int written = 0; + size_t cgNameLen; + uint64_t dummyVal; + unsigned char *cgName; + + /* return from sub-element string parsing */ + subElementCallEnd(p, (char **) &cgName, &cgNameLen); + aggMakeRoom(p, LONG_STR_SIZE * 4); + + /* read consumer-group lastid */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + /* Load group offset. */ + if (p->currOpcode >= RDB_TYPE_STREAM_LISTPACKS_2) { + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + written), &written)); + } + IF_NOT_OK_RETURN( + rdbLoadLen(p, NULL, &streamCtx->globPelLeft, (unsigned char *) (rawCtx->at + written), &written)); + + /*** ENTER SAFE STATE ***/ + + IF_NOT_OK_RETURN(aggUpdateWritten(p, written)); + + if (!(streamCtx->globPelLeft)) + return updateElementState(p, ST_LOAD_NUM_CONSUMERS, 0); + } + + updateElementState(p, ST_LOAD_GLOBAL_PEL, 0); /* fall-thru */ + + case ST_LOAD_GLOBAL_PEL: + while (streamCtx->globPelLeft) { + uint64_t dummyVal; + BulkInfo *binfo; + int pelLen = 0; + IF_NOT_OK_RETURN(aggMakeRoom(p, LONG_STR_SIZE * 2 + sizeof(RdbStreamID))); + /* load streamid */ + IF_NOT_OK_RETURN(rdbLoad(p, sizeof(RdbStreamID), RQ_ALLOC_REF, rawCtx->at, &binfo)); + pelLen += sizeof(RdbStreamID); + /* load millisec */ + IF_NOT_OK_RETURN(rdbLoad(p, 8, RQ_ALLOC_REF, rawCtx->at + pelLen, &binfo)); + pelLen += 8; + /* load delivery count */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &dummyVal, (unsigned char *) (rawCtx->at + pelLen), &pelLen)); + + /*** ENTER SAFE STATE ***/ + + IF_NOT_OK_RETURN(aggUpdateWritten(p, pelLen)); + + streamCtx->globPelLeft--; + } + updateElementState(p, ST_LOAD_NUM_CONSUMERS, 0); /* fall-thru */ + + case ST_LOAD_NUM_CONSUMERS: { + int written = 0; + IF_NOT_OK_RETURN( + rdbLoadLen(p, NULL, &streamCtx->consumersLeft, (unsigned char *) rawCtx->at, &written)); + + /*** ENTER SAFE STATE ***/ + + IF_NOT_OK_RETURN(aggUpdateWritten(p, written)); /* fall-thru */ + } + updateElementState(p, ST_LOAD_NEXT_CONSUMER, 0); /* fall-thru */ + + case ST_LOAD_NEXT_CONSUMER: + /*** ENTER SAFE STATE (no rdb read) ***/ + if (streamCtx->consumersLeft) { + streamCtx->consumersLeft--; + /* call raw string as sub-element to read consumer name */ + return subElementCall(p, PE_RAW_STRING, ST_LOAD_NEXT_CONSUMER_STR_RETURN); + } else { + return updateElementState(p, ST_LOAD_NEXT_CG_IS_MORE, 0); + } + + case ST_LOAD_NEXT_CONSUMER_STR_RETURN: { + size_t consNameLen; + unsigned char *consName; + uint64_t consPelSize, consLen = 0; + BulkInfo *binfo; + int written = 0; + + subElementCallEnd(p, (char **) &consName, &consNameLen); + + IF_NOT_OK_RETURN(aggMakeRoom(p, 8*2 + LONG_STR_SIZE)); + + /* load millisec */ + IF_NOT_OK_RETURN(rdbLoad(p, 8, RQ_ALLOC_REF, rawCtx->at + consLen, &binfo)); + consLen += 8; + if (p->currOpcode >= RDB_TYPE_STREAM_LISTPACKS_3) { + IF_NOT_OK_RETURN(rdbLoad(p, 8, RQ_ALLOC_REF, rawCtx->at + consLen, &binfo)); + consLen += 8; + } + /* load consumer PEL consNameLen */ + IF_NOT_OK_RETURN(rdbLoadLen(p, NULL, &consPelSize, (unsigned char *) rawCtx->at + consLen, &written)); + consLen += written; + + IF_NOT_OK_RETURN(aggMakeRoom(p, consPelSize * sizeof(RdbStreamID))); + + /* load consumer PELs */ + IF_NOT_OK_RETURN(rdbLoad(p, consPelSize * sizeof(RdbStreamID), RQ_ALLOC_REF, rawCtx->at + consLen, &binfo)); + consLen += consPelSize * sizeof(RdbStreamID); + + /*** ENTER SAFE STATE (no rdb read) ***/ + + IF_NOT_OK_RETURN(aggUpdateWritten(p, consLen)); + + return updateElementState(p, ST_LOAD_NEXT_CONSUMER, 0); + } + + default: + RDB_reportError(p, RDB_ERR_STREAM_INVALID_STATE, + "elementRawStreamLP() : Invalid parsing element state: %d.", p->elmCtx.state); + return RDB_STATUS_ERROR; + } +} /*** various functions ***/ @@ -873,7 +1098,7 @@ static int intsetValidateIntegrityCb(unsigned char* str, size_t size, RdbParser static RdbStatus singleStringTypeHandling(RdbParser *p, singleStringTypeValidateCb validateCb, char *callerName) { - enum RAW_LIST_STATES { + enum RAW_SINGLE_STRING_TYPE_STATES { ST_RAW_SSTYPE_START=0, ST_RAW_SSTYPE_CALL_STR, /* Call PE_RAW_STRING as sub-element */ ST_RAW_SSTYPE_RET_FROM_STR, /* integ check of the returned string from PE_RAW_STRING */ diff --git a/src/lib/version.h b/src/lib/version.h index 2cfe3a4..3c70f23 100644 --- a/src/lib/version.h +++ b/src/lib/version.h @@ -4,3 +4,6 @@ /* Keep direct value for external readers */ #define LIBRDB_VERSION_STRING "0.0.1" + +/* Update Maximum supported RDB version */ +#define LIBRDB_SUPPORT_MAX_RDB_VER 12 diff --git a/test/Makefile b/test/Makefile index 62db4e1..7f72f53 100644 --- a/test/Makefile +++ b/test/Makefile @@ -3,6 +3,7 @@ default: all LIB_NAME = rdb LIB_DIR = ../lib LIB_NAME_EXT = $(LIB_NAME)-ext +LIBHIREDIS=../deps/hiredis/libhiredis.a # Artifacts: TARGET_TEST = test_lib @@ -16,13 +17,12 @@ STD = -std=c99 STACK = -fstack-protector-all -Wstack-protector WARNS = -Wall -Wextra -pedantic -Werror -Wno-unused-function -CFLAGS = -fPIC -O3 $(STD) $(STACK) $(WARNS) -DEBUG = -g3 -DDEBUG=1 -LIBS = -l cmocka -L /usr/lib -L $(LIB_DIR) -l $(LIB_NAME) -l $(LIB_NAME_EXT) -L../deps/hiredis -lhiredis -LIBS_STATIC = -l cmocka -L /usr/lib -L $(LIB_DIR) -l:lib$(LIB_NAME_EXT).a -L../deps/hiredis -lhiredis - -LIBHIREDIS=../deps/hiredis/libhiredis.a +OPTIMIZATION?=-O3 +CFLAGS = -fPIC $(OPTIMIZATION) $(STD) $(STACK) $(WARNS) +DEBUG = -g3 -DDEBUG=1 +LIBS = -l cmocka -L /usr/lib -L $(LIB_DIR) -l $(LIB_NAME) -l $(LIB_NAME_EXT) -l:$(LIBHIREDIS) +LIBS_STATIC = -l cmocka -L /usr/lib -L $(LIB_DIR) -l:lib$(LIB_NAME_EXT).a -l:$(LIBHIREDIS) ifeq ($(BUILD_TLS),yes) CFLAGS+=-DUSE_OPENSSL=1 @@ -34,10 +34,10 @@ all: $(TARGET_TEST) $(TARGET_TEST_STATIC) @echo "Done."; $(TARGET_TEST): $(OBJECTS) - $(CC) $(OBJECTS) -o $@ $(LIBHIREDIS) $(DEBUG) $(CFLAGS) $(LIBS) + $(CC) $(OBJECTS) -o $@ $(DEBUG) $(CFLAGS) $(LIBS) $(TARGET_TEST_STATIC): $(OBJECTS) - $(CC) $(OBJECTS) -o $@ $(LIBHIREDIS) $(DEBUG) $(CFLAGS) $(LIBS) $(LIBS_STATIC) + $(CC) $(OBJECTS) -o $@ $(DEBUG) $(CFLAGS) $(LIBS) $(LIBS_STATIC) -include $(OBJECTS:.o=.d) diff --git a/test/dumps/function.json b/test/dumps/function.json index 01487fb..4866553 100644 --- a/test/dumps/function.json +++ b/test/dumps/function.json @@ -3,4 +3,4 @@ "ctime":"1693250800", "used-mem":"866344", "aof-base":"0", -"Function_1":"#!lua name=mylib \n redis.register_function('myfunc', function(keys, args) return args[1] end)", +"Function_1":"#!lua name=mylib \n redis.register_function('myfunc', function(keys, args) return args[1] end)" diff --git a/test/dumps/hash_lp_v11_raw.json b/test/dumps/hash_lp_v11_raw.json index 1915638..cfcdb23 100644 --- a/test/dumps/hash_lp_v11_raw.json +++ b/test/dumps/hash_lp_v11_raw.json @@ -3,5 +3,5 @@ "ctime":"1695280472", "used-mem":"1062024", "aof-base":"0", -"hash2":"\x10\xc3@C@V\x13V\x00\x00\x00\f\x00\x86field7\x07\x86value \x07`\x0f\x008\xa0\x0f\x038\x07\t\x01\x80\x19\x029\x07\x87`\x19\x0310\b\x87`*@\b\x80\x11\f1\b\x0b\x01\f\x01\x8412.0\x05\xff", -"hash1":"\x10\xc35>\x0f>\x00\x00\x00\n\x00\x86field2\x07\x02\x01\x80\t\x023\x07\x03\xa0\t\x0b4\x07\x88value4.0\t\x80\x1b\x0b5\x07\x835.0\x04\x06\x01\x06\x01\xff" +"hash2":"\u0010\u00c3@C@V\u0013V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value \u0007`\u000f\u00008\u00a0\u000f\u00038\u0007\t\u0001\u0080\u0019\u00029\u0007\u0087`\u0019\u000310\b\u0087`*@\b\u0080\u0011\f1\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff", +"hash1":"\u0010\u00c35>\u000f>\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0080\t\u00023\u0007\u0003\u00a0\t\u000b4\u0007\u0088value4.0\t\u0080\u001b\u000b5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff" diff --git a/test/dumps/hash_lp_v11_struct.json b/test/dumps/hash_lp_v11_struct.json index 473155b..240cd14 100644 --- a/test/dumps/hash_lp_v11_struct.json +++ b/test/dumps/hash_lp_v11_struct.json @@ -3,5 +3,5 @@ "ctime":"1695280472", "used-mem":"1062024", "aof-base":"0", -"hash2":["V\x00\x00\x00\f\x00\x86field7\x07\x86value7\x07\x86field8\x07\x86value8\x07\t\x01\x86value9\x07\x87field10\b\x87value10\b\x87field11\b\x0b\x01\f\x01\x8412.0\x05\xff"], -"hash1":[">\x00\x00\x00\n\x00\x86field2\x07\x02\x01\x86field3\x07\x03\x01\x86field4\x07\x88value4.0\t\x86field5\x07\x835.0\x04\x06\x01\x06\x01\xff"] +"hash2":["V\u0000\u0000\u0000\f\u0000\u0086field7\u0007\u0086value7\u0007\u0086field8\u0007\u0086value8\u0007\t\u0001\u0086value9\u0007\u0087field10\b\u0087value10\b\u0087field11\b\u000b\u0001\f\u0001\u008412.0\u0005\u00ff"], +"hash1":[">\u0000\u0000\u0000\n\u0000\u0086field2\u0007\u0002\u0001\u0086field3\u0007\u0003\u0001\u0086field4\u0007\u0088value4.0\t\u0086field5\u0007\u00835.0\u0004\u0006\u0001\u0006\u0001\u00ff"] diff --git a/test/dumps/hash_zl_v6_raw.json b/test/dumps/hash_zl_v6_raw.json index 16124f0..a58ea14 100644 --- a/test/dumps/hash_zl_v6_raw.json +++ b/test/dumps/hash_zl_v6_raw.json @@ -3,4 +3,4 @@ "ctime":"1690533464", "used-mem":"865216", "aof-preamble":"0", -"myhash":"\r99\x00\x00\x002\x00\x00\x00\f\x00\x00\xf2\x02\xf3\x02\xf4\x02\xf5\x02\xf6\x02\xf7\x02\x037.0\x05\x038.0\x05\x04str1\x06\x04str2\x06\x04str3\x06\x04str4\xff" \ No newline at end of file +"myhash":"\r99\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff" \ No newline at end of file diff --git a/test/dumps/hash_zl_v6_struct.json b/test/dumps/hash_zl_v6_struct.json index bff6830..18b29cf 100644 --- a/test/dumps/hash_zl_v6_struct.json +++ b/test/dumps/hash_zl_v6_struct.json @@ -3,4 +3,4 @@ "ctime":"1690533464", "used-mem":"865216", "aof-preamble":"0", -"myhash":["9\x00\x00\x002\x00\x00\x00\f\x00\x00\xf2\x02\xf3\x02\xf4\x02\xf5\x02\xf6\x02\xf7\x02\x037.0\x05\x038.0\x05\x04str1\x06\x04str2\x06\x04str3\x06\x04str4\xff"] +"myhash":["9\u0000\u0000\u00002\u0000\u0000\u0000\f\u0000\u0000\u00f2\u0002\u00f3\u0002\u00f4\u0002\u00f5\u0002\u00f6\u0002\u00f7\u0002\u00037.0\u0005\u00038.0\u0005\u0004str1\u0006\u0004str2\u0006\u0004str3\u0006\u0004str4\u00ff"] diff --git a/test/dumps/hash_zm_v2_raw.json b/test/dumps/hash_zm_v2_raw.json index b7d919f..7ded1a1 100644 --- a/test/dumps/hash_zm_v2_raw.json +++ b/test/dumps/hash_zm_v2_raw.json @@ -1 +1 @@ -"myhash":"\t\xc3@C@M\x15\t\x011\x01\x001\x012\x01\x002\x013\x01\x003\x031.1\x03\x00@\x04\x021.2@\b\x002 \b\x003@\b\x073\x04aaa1\x04\x00`\x05 \x04\x002`\n\x002@\n\x003`\n\x013\xff" +"myhash":"\t\u00c3@C@M\u0015\t\u00011\u0001\u00001\u00012\u0001\u00002\u00013\u0001\u00003\u00031.1\u0003\u0000@\u0004\u00021.2@\b\u00002 \b\u00003@\b\u00073\u0004aaa1\u0004\u0000`\u0005 \u0004\u00002`\n\u00002@\n\u00003`\n\u00013\u00ff" diff --git a/test/dumps/hash_zm_v2_struct.json b/test/dumps/hash_zm_v2_struct.json index 2bcab0f..5e43c6a 100644 --- a/test/dumps/hash_zm_v2_struct.json +++ b/test/dumps/hash_zm_v2_struct.json @@ -1 +1 @@ -"myhash":["\t\x011\x01\x001\x012\x01\x002\x013\x01\x003\x031.1\x03\x001.1\x031.2\x03\x001.2\x031.3\x03\x001.3\x04aaa1\x04\x00aaa1\x04aaa2\x04\x00aaa2\x04aaa3\x04\x00aaa3\xff"] +"myhash":["\t\u00011\u0001\u00001\u00012\u0001\u00002\u00013\u0001\u00003\u00031.1\u0003\u00001.1\u00031.2\u0003\u00001.2\u00031.3\u0003\u00001.3\u0004aaa1\u0004\u0000aaa1\u0004aaa2\u0004\u0000aaa2\u0004aaa3\u0004\u0000aaa3\u00ff"] diff --git a/test/dumps/module_raw.json b/test/dumps/module_raw.json index a4339e1..2e2d8ec 100644 --- a/test/dumps/module_raw.json +++ b/test/dumps/module_raw.json @@ -1 +1 @@ -"key1":"\x07\x81\xb5\xeb-\xff\xfa\xddl\x01\x05\x06value1\x00" +"key1":"\u0007\u0081\u00b5\u00eb-\u00ff\u00fa\u00ddl\u0001\u0005\u0006value1\u0000" diff --git a/test/dumps/multiple_lists_strings_raw.json b/test/dumps/multiple_lists_strings_raw.json index 37c35ad..6a9615a 100644 --- a/test/dumps/multiple_lists_strings_raw.json +++ b/test/dumps/multiple_lists_strings_raw.json @@ -3,9 +3,9 @@ "ctime":"1677580558", "used-mem":"937464", "aof-base":"0", -"string2":"\x00\tHi there!", -"mylist1":"\x12\x01\x02\x0b\x0b\x00\x00\x00\x01\x00\x82v1\x03\xff", -"mylist3":"\x12\x01\x02\x13\x13\x00\x00\x00\x03\x00\x82v3\x03\x82v2\x03\x82v1\x03\xff", -"lzf_compressed":"\x00\xc3\t@v\x01cc\xe0i\x00\x01cc", -"string1":"\x00\x04blaa", -"mylist2":"\x12\x01\x02\x0f\x0f\x00\x00\x00\x02\x00\x82v2\x03\x82v1\x03\xff" +"string2":"\u0000\tHi there!", +"mylist1":"\u0012\u0001\u0002\u000b\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff", +"mylist3":"\u0012\u0001\u0002\u0013\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff", +"lzf_compressed":"\u0000\u00c3\t@v\u0001cc\u00e0i\u0000\u0001cc", +"string1":"\u0000\u0004blaa", +"mylist2":"\u0012\u0001\u0002\u000f\u000f\u0000\u0000\u0000\u0002\u0000\u0082v2\u0003\u0082v1\u0003\u00ff" diff --git a/test/dumps/multiple_lists_strings_struct.json b/test/dumps/multiple_lists_strings_struct.json index 100970e..83ae876 100644 --- a/test/dumps/multiple_lists_strings_struct.json +++ b/test/dumps/multiple_lists_strings_struct.json @@ -4,8 +4,8 @@ "used-mem":"937464", "aof-base":"0", "string2":"Hi there!", -"mylist1":["\x0b\x00\x00\x00\x01\x00\x82v1\x03\xff"], -"mylist3":["\x13\x00\x00\x00\x03\x00\x82v3\x03\x82v2\x03\x82v1\x03\xff"], +"mylist1":["\u000b\u0000\u0000\u0000\u0001\u0000\u0082v1\u0003\u00ff"], +"mylist3":["\u0013\u0000\u0000\u0000\u0003\u0000\u0082v3\u0003\u0082v2\u0003\u0082v1\u0003\u00ff"], "lzf_compressed":"cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "string1":"blaa", -"mylist2":["\x0f\x00\x00\x00\x02\x00\x82v2\x03\x82v1\x03\xff"] +"mylist2":["\u000f\u0000\u0000\u0000\u0002\u0000\u0082v2\u0003\u0082v1\u0003\u00ff"] diff --git a/test/dumps/multiple_lists_strings_subset_list.json b/test/dumps/multiple_lists_strings_subset_list.json index 200f003..e2e284d 100644 --- a/test/dumps/multiple_lists_strings_subset_list.json +++ b/test/dumps/multiple_lists_strings_subset_list.json @@ -1,3 +1,3 @@ "mylist1":["v1"], "mylist3":["v3","v2","v1"], -"mylist2":["v2","v1"], \ No newline at end of file +"mylist2":["v2","v1"] \ No newline at end of file diff --git a/test/dumps/multiple_lists_strings_subset_str.json b/test/dumps/multiple_lists_strings_subset_str.json index 59c4631..888b1f3 100644 --- a/test/dumps/multiple_lists_strings_subset_str.json +++ b/test/dumps/multiple_lists_strings_subset_str.json @@ -1,3 +1,3 @@ -"string2":"\x00\tHithere!", -"lzf_compressed":"\x00\xc3\t@v\x01cc\xe0i\x00\x01cc", -"string1":"\x00\x04blaa", +"string2":"\u0000\tHi there!", +"lzf_compressed":"\u0000\u00c3\t@v\u0001cc\u00e0i\u0000\u0001cc", +"string1":"\u0000\u0004blaa" diff --git a/test/dumps/plain_hash_raw.json b/test/dumps/plain_hash_raw.json index 36cba36..4b03a40 100644 --- a/test/dumps/plain_hash_raw.json +++ b/test/dumps/plain_hash_raw.json @@ -2,4 +2,4 @@ "redis-bits":"64", "ctime":"1689605183", "used-mem":"821904", -"myhash1":"\x04\x0b\xc0\x05\xc0\x05\xc0\x06\xc0\x06\xc0\n\xc0\n\xc0\x01\xc0\x01\xc0\x02\xc0\x02\xc0\b\xc0\b\xc0\x04\xc0\x04\xc0\t\xc0\t\xc0\x03\xc0\x03\xc0\x07\xc0\x07\xc0\x0b\xc0\x0b" \ No newline at end of file +"myhash1":"\u0004\u000b\u00c0\u0005\u00c0\u0005\u00c0\u0006\u00c0\u0006\u00c0\n\u00c0\n\u00c0\u0001\u00c0\u0001\u00c0\u0002\u00c0\u0002\u00c0\b\u00c0\b\u00c0\u0004\u00c0\u0004\u00c0\t\u00c0\t\u00c0\u0003\u00c0\u0003\u00c0\u0007\u00c0\u0007\u00c0\u000b\u00c0\u000b" \ No newline at end of file diff --git a/test/dumps/plain_list_v6_raw.json b/test/dumps/plain_list_v6_raw.json index 91ec68f..29a9b97 100644 --- a/test/dumps/plain_list_v6_raw.json +++ b/test/dumps/plain_list_v6_raw.json @@ -1 +1 @@ - "ll":"\x01B\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01\xc0\x01" + "ll":"\u0001B\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001\u00c0\u0001" diff --git a/test/dumps/plain_set_v6_raw.json b/test/dumps/plain_set_v6_raw.json index 1a24aae..1365121 100644 --- a/test/dumps/plain_set_v6_raw.json +++ b/test/dumps/plain_set_v6_raw.json @@ -1 +1 @@ -"myset":"\x02\n\x02a1\x031.3\x02b2\xc0\x02\xc0\x03\x031.1\x031.2\xc3\t@Q\x01xx\xe0D\x00\x01xx\x02c3\xc0\x01" +"myset":"\u0002\n\u0002a1\u00031.3\u0002b2\u00c0\u0002\u00c0\u0003\u00031.1\u00031.2\u00c3\t@Q\u0001xx\u00e0D\u0000\u0001xx\u0002c3\u00c0\u0001" diff --git a/test/dumps/plain_zset_2_v11_raw.json b/test/dumps/plain_zset_2_v11_raw.json index a00f2bd..1b259aa 100644 --- a/test/dumps/plain_zset_2_v11_raw.json +++ b/test/dumps/plain_zset_2_v11_raw.json @@ -3,4 +3,4 @@ "ctime":"1695705520", "used-mem":"967352", "aof-base":"0", -"myzset":"\x05\x18\x02a9\x00\x00\x00\x00\x00\x00\xf0\x7f\x02a8\x00\x00\x00\x00\x00\x00\xf0\x7f\x03a23\xbc\t\x7f\xf3M\x8b E\x03a11\x00\x00\x00\x00\x00\x00@C\x03a21\x00\x00\x00\x00\x00\x00\x10C\x03a17\x00\x00\x00\x00\x00\x00\xf0@\x03a15\x00\x00\x00\x00\x00\xe0o@\x03a13\x1b\x9d\xf3S\x1c\xc7!@\x02a7\x9a\x99\x99\x99\x99\x99\x01@\x02a6\x00\x00\x00\x00\x00\x00\x00\x00\x02a5\x00\x00\x00\x00\x00\x00\x00\x00\x02a4\x00\x00\x00\x00\x00\x00\x00\x80\x02a3\x00\x00\x00\x00\x00\x00\x00\x80\x02a2\x00\x00\x00\x00\x00\x00\x00\x00\x02a1\x00\x00\x00\x00\x00\x00\x00\x00\x03a19\x00\x00\x00\x00\x00\x00\xf0\xbf\x03a20\x9a\x99\x99\x99\x99\x99\xf1\xbf\x03a14r\xa7t\xb0\xfe\xff#\xc0\x03a16\x00\x00\x00\x00\x00\xe0o\xc0\x03a18\x00\x00\x00\x00\x00\x00\xf0\xc0\x03a22\x00\x00\x00\x00\x00\x00\x10\xc3\x03a12\x00\x00\x00\x00\x00\x00@\xc3\x03a24\xc0\xe1\xa2\xca\x14|\xe1\xc5\x03a10\x00\x00\x00\x00\x00\x00\xf0\xff" +"myzset":"\u0005\u0018\u0002a9\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u007f\u0002a8\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u007f\u0003a23\u00bc\t\u007f\u00f3M\u008b E\u0003a11\u0000\u0000\u0000\u0000\u0000\u0000@C\u0003a21\u0000\u0000\u0000\u0000\u0000\u0000\u0010C\u0003a17\u0000\u0000\u0000\u0000\u0000\u0000\u00f0@\u0003a15\u0000\u0000\u0000\u0000\u0000\u00e0o@\u0003a13\u001b\u009d\u00f3S\u001c\u00c7!@\u0002a7\u009a\u0099\u0099\u0099\u0099\u0099\u0001@\u0002a6\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002a5\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002a4\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0080\u0002a3\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0080\u0002a2\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0002a1\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0003a19\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u00bf\u0003a20\u009a\u0099\u0099\u0099\u0099\u0099\u00f1\u00bf\u0003a14r\u00a7t\u00b0\u00fe\u00ff#\u00c0\u0003a16\u0000\u0000\u0000\u0000\u0000\u00e0o\u00c0\u0003a18\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u00c0\u0003a22\u0000\u0000\u0000\u0000\u0000\u0000\u0010\u00c3\u0003a12\u0000\u0000\u0000\u0000\u0000\u0000@\u00c3\u0003a24\u00c0\u00e1\u00a2\u00ca\u0014|\u00e1\u00c5\u0003a10\u0000\u0000\u0000\u0000\u0000\u0000\u00f0\u00ff" diff --git a/test/dumps/plain_zset_v6_raw.json b/test/dumps/plain_zset_v6_raw.json index f026b50..dcbb100 100644 --- a/test/dumps/plain_zset_v6_raw.json +++ b/test/dumps/plain_zset_v6_raw.json @@ -1 +1 @@ -"myzset":"\x03\x18\x03a23\f1.000033e+25\x03a19\x02-1\x02a9\xfe\x03a24\x17-4.3290001231231309e+28\x03a12\x11-9007199254740992\x02a7\x122.2000000000000002\x03a13\x128.8888879999999997\x03a17\x0565536\x03a21\x101125899906842624\x03a20\x13-1.1000000000000001\x03a22\x11-1125899906842624\x02a3\x02-0\x02a2\x010\x03a15\x03255\x02a8\xfe\x03a11\x109007199254740992\x03a10\xff\x02a5\x010\x03a14\x13-9.9999900000000004\x02a6\x010\x03a16\x04-255\x02a4\x02-0\x02a1\x010\x03a18\x06-65536" +"myzset":"\u0003\u0018\u0003a23\f1.000033e+25\u0003a19\u0002-1\u0002a9\u00fe\u0003a24\u0017-4.3290001231231309e+28\u0003a12\u0011-9007199254740992\u0002a7\u00122.2000000000000002\u0003a13\u00128.8888879999999997\u0003a17\u000565536\u0003a21\u00101125899906842624\u0003a20\u0013-1.1000000000000001\u0003a22\u0011-1125899906842624\u0002a3\u0002-0\u0002a2\u00010\u0003a15\u0003255\u0002a8\u00fe\u0003a11\u00109007199254740992\u0003a10\u00ff\u0002a5\u00010\u0003a14\u0013-9.9999900000000004\u0002a6\u00010\u0003a16\u0004-255\u0002a4\u0002-0\u0002a1\u00010\u0003a18\u0006-65536" diff --git a/test/dumps/quicklist_raw.json b/test/dumps/quicklist_raw.json index 12fa03f..d214f35 100644 --- a/test/dumps/quicklist_raw.json +++ b/test/dumps/quicklist_raw.json @@ -3,5 +3,5 @@ "ctime":"1629882015", "used-mem":"496256", "aof-preamble":"0", -"list":"\x0e\x01\r\r\x00\x00\x00\n\x00\x00\x00\x01\x00\x00\xf8\xff", -"x":"\x00\xc0\x07" +"list":"\u000e\u0001\r\r\u0000\u0000\u0000\n\u0000\u0000\u0000\u0001\u0000\u0000\u00f8\u00ff", +"x":"\u0000\u00c0\u0007" diff --git a/test/dumps/quicklist_struct.json b/test/dumps/quicklist_struct.json index ac1bc59..ae1d4e4 100644 --- a/test/dumps/quicklist_struct.json +++ b/test/dumps/quicklist_struct.json @@ -3,5 +3,5 @@ "ctime":"1629882015", "used-mem":"496256", "aof-preamble":"0", -"list":["\r\x00\x00\x00\n\x00\x00\x00\x01\x00\x00\xf8\xff"], +"list":["\r\u0000\u0000\u0000\n\u0000\u0000\u0000\u0001\u0000\u0000\u00f8\u00ff"], "x":"7" diff --git a/test/dumps/set_is_v11_raw.json b/test/dumps/set_is_v11_raw.json index ccf9fb9..c7cf27d 100644 --- a/test/dumps/set_is_v11_raw.json +++ b/test/dumps/set_is_v11_raw.json @@ -3,4 +3,4 @@ "ctime":"1691304897", "used-mem":"978520", "aof-base":"0", -"myintest":["\x04\x00\x00\x00\x06\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xd2\x02\x96I"] +"myintest":["\u0004\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0004\u0000\u0000\u0000\u0005\u0000\u0000\u0000\u00d2\u0002\u0096I"] diff --git a/test/dumps/set_is_v11_struct.json b/test/dumps/set_is_v11_struct.json index ccf9fb9..c7cf27d 100644 --- a/test/dumps/set_is_v11_struct.json +++ b/test/dumps/set_is_v11_struct.json @@ -3,4 +3,4 @@ "ctime":"1691304897", "used-mem":"978520", "aof-base":"0", -"myintest":["\x04\x00\x00\x00\x06\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00\x04\x00\x00\x00\x05\x00\x00\x00\xd2\x02\x96I"] +"myintest":["\u0004\u0000\u0000\u0000\u0006\u0000\u0000\u0000\u0001\u0000\u0000\u0000\u0002\u0000\u0000\u0000\u0003\u0000\u0000\u0000\u0004\u0000\u0000\u0000\u0005\u0000\u0000\u0000\u00d2\u0002\u0096I"] diff --git a/test/dumps/set_lp_v11_raw.json b/test/dumps/set_lp_v11_raw.json index 4dc8ab4..18a6797 100644 --- a/test/dumps/set_lp_v11_raw.json +++ b/test/dumps/set_lp_v11_raw.json @@ -3,4 +3,4 @@ "ctime":"1691481599", "used-mem":"866040", "aof-base":"0", -"myset":["%\x00\x00\x00\t\x00\x01\x01\x02\x01\x03\x01\x831.1\x04\x831.2\x04\x831.3\x04\x81a\x02\x81b\x02\x81c\x02\xff"] +"myset":["%\u0000\u0000\u0000\t\u0000\u0001\u0001\u0002\u0001\u0003\u0001\u00831.1\u0004\u00831.2\u0004\u00831.3\u0004\u0081a\u0002\u0081b\u0002\u0081c\u0002\u00ff"] diff --git a/test/dumps/set_lp_v11_struct.json b/test/dumps/set_lp_v11_struct.json index 3cff256..a67d1a0 100644 --- a/test/dumps/set_lp_v11_struct.json +++ b/test/dumps/set_lp_v11_struct.json @@ -3,4 +3,4 @@ "ctime":"1691481599", "used-mem":"866040", "aof-base":"0", -"myset":["%\x00\x00\x00\t\x00\x01\x01\x02\x01\x03\x01\x831.1\x04\x831.2\x04\x831.3\x04\x81a\x02\x81b\x02\x81c\x02\xff"] \ No newline at end of file +"myset":["%\u0000\u0000\u0000\t\u0000\u0001\u0001\u0002\u0001\u0003\u0001\u00831.1\u0004\u00831.2\u0004\u00831.3\u0004\u0081a\u0002\u0081b\u0002\u0081c\u0002\u00ff"] \ No newline at end of file diff --git a/test/dumps/single_key_raw.json b/test/dumps/single_key_raw.json index 332ec1e..73e14f5 100644 --- a/test/dumps/single_key_raw.json +++ b/test/dumps/single_key_raw.json @@ -6,4 +6,4 @@ "repl-id":"67ebe8f627f436e2630eef8661a697fa33563a8f", "repl-offset":"162341903", "aof-base":"0", -"xxx":"\x00\xc0o" +"xxx":"\u0000\u00c0o" diff --git a/test/dumps/single_list_raw.json b/test/dumps/single_list_raw.json index 416e1f5..c654977 100644 --- a/test/dumps/single_list_raw.json +++ b/test/dumps/single_list_raw.json @@ -6,4 +6,4 @@ "repl-id":"f42ea6b158e5926941d08bde6b9ecb6ae88dcd45", "repl-offset":"162395634", "aof-base":"0", -"mylist":"\x12\x01\x02\x19\x19\x00\x00\x00\x03\x00\x84val3\x05\x84val2\x05\x84val1\x05\xff" +"mylist":"\u0012\u0001\u0002\u0019\u0019\u0000\u0000\u0000\u0003\u0000\u0084val3\u0005\u0084val2\u0005\u0084val1\u0005\u00ff" \ No newline at end of file diff --git a/test/dumps/single_list_struct.json b/test/dumps/single_list_struct.json index c861a22..ad3b44d 100644 --- a/test/dumps/single_list_struct.json +++ b/test/dumps/single_list_struct.json @@ -6,4 +6,4 @@ "repl-id":"f42ea6b158e5926941d08bde6b9ecb6ae88dcd45", "repl-offset":"162395634", "aof-base":"0", -"mylist":["\x19\x00\x00\x00\x03\x00\x84val3\x05\x84val2\x05\x84val1\x05\xff"] +"mylist":["\u0019\u0000\u0000\u0000\u0003\u0000\u0084val3\u0005\u0084val2\u0005\u0084val1\u0005\u00ff"] diff --git a/test/dumps/stream_data.json b/test/dumps/stream_data.json new file mode 100644 index 0000000..b2d6ee3 --- /dev/null +++ b/test/dumps/stream_data.json @@ -0,0 +1,9 @@ +"mystream":{ +"entries":[ +{ "id":"1695649068107-0", "items":{"message":"Message1"} }, +{ "id":"1695649068110-0", "items":{"message":"Message2"} }, +{ "id":"1695649069139-0", "items":{"message":"Message3"} }, +{ "id":"1695649446276-0", "items":{"message":"Message4"} }, +{ "id":"1695649456516-0", "items":{"message":"Message5"} }, +{ "id":"1695893015933-0", "items":{"field1":"value1", "field2":"value2", "field3":"value3"} } +]} diff --git a/test/dumps/stream_data_with_meta.json b/test/dumps/stream_data_with_meta.json new file mode 100644 index 0000000..e2d351d --- /dev/null +++ b/test/dumps/stream_data_with_meta.json @@ -0,0 +1,29 @@ + "mystream":{ + "entries":[ + { "id":"1695649068107-0", "items":{"message":"Message1"} }, + { "id":"1695649068110-0", "items":{"message":"Message2"} }, + { "id":"1695649069139-0", "items":{"message":"Message3"} }, + { "id":"1695649446276-0", "items":{"message":"Message4"} }, + { "id":"1695649456516-0", "items":{"message":"Message5"} }, + { "id":"1695893015933-0", "items":{"field1":"value1", "field2":"value2", "field3":"value3"} }], + "length": 6, + "entriesAdded": 6, + "firstID": "1695649068107-0", + "lastID": "1695893015933-0", + "maxDelEntryID": "0-0", + "groups": [ + {"name": "groupA", "lastid": "1695649446276-0", "entriesRead": 4, + "pending": [ + { "sent": 1695649446276, "id":"1695649446276-0", "count": 1 }], + "consumers": [ + { "name": "consumerA1", "activeTime": 1696679585023, "seenTime": 1696679585023}, + { "name": "consumerA2", "activeTime": 1696679585024, "seenTime": 1696679585024, + "pending": [ + {"id":"1695649446276-0"}]}]}, + {"name": "groupB", "lastid": "1695649069139-0", "entriesRead": 3, + "pending": [ + { "sent": 1695649069139, "id":"1695649069139-0", "count": 1 }], + "consumers": [ + { "name": "consumerB1", "activeTime": 1696679585026, "seenTime": 1696679585026, + "pending": [ + {"id":"1695649069139-0"}]}]}]} diff --git a/test/dumps/stream_v11.rdb b/test/dumps/stream_v11.rdb new file mode 100644 index 0000000..3e36f59 Binary files /dev/null and b/test/dumps/stream_v11.rdb differ diff --git a/test/dumps/stream_v11_target_ver_6.2.resp b/test/dumps/stream_v11_target_ver_6.2.resp new file mode 100644 index 0000000..1ba7db7 --- /dev/null +++ b/test/dumps/stream_v11_target_ver_6.2.resp @@ -0,0 +1,153 @@ +*5 +$4 +XADD +$8 +mystream +$15 +1695649068107-0 +$7 +message +$8 +Message1 +*5 +$4 +XADD +$8 +mystream +$15 +1695649068110-0 +$7 +message +$8 +Message2 +*5 +$4 +XADD +$8 +mystream +$15 +1695649069139-0 +$7 +message +$8 +Message3 +*5 +$4 +XADD +$8 +mystream +$15 +1695649446276-0 +$7 +message +$8 +Message4 +*5 +$4 +XADD +$8 +mystream +$15 +1695649456516-0 +$7 +message +$8 +Message5 +*9 +$4 +XADD +$8 +mystream +$15 +1695893015933-0 +$6 +field1 +$6 +value1 +$6 +field2 +$6 +value2 +$6 +field3 +$6 +value3 +*3 +$6 +XSETID +$8 +mystream +$15 +1695893015933-0 +*5 +$6 +XGROUP +$6 +CREATE +$8 +mystream +$6 +groupA +$15 +1695649446276-0 +*12 +$6 +XCLAIM +$8 +mystream +$6 +groupA +$10 +consumerA2 +$1 +0 +$15 +1695649446276-0 +$4 +TIME +$13 +1695649446276 +$10 +RETRYCOUNT +$1 +1 +$6 +JUSTID +$5 +FORCE +*5 +$6 +XGROUP +$6 +CREATE +$8 +mystream +$6 +groupB +$15 +1695649069139-0 +*12 +$6 +XCLAIM +$8 +mystream +$6 +groupB +$10 +consumerB1 +$1 +0 +$15 +1695649069139-0 +$4 +TIME +$13 +1695649069139 +$10 +RETRYCOUNT +$1 +1 +$6 +JUSTID +$5 +FORCE diff --git a/test/dumps/stream_v11_target_ver_7.2.resp b/test/dumps/stream_v11_target_ver_7.2.resp new file mode 100644 index 0000000..cba9d1f --- /dev/null +++ b/test/dumps/stream_v11_target_ver_7.2.resp @@ -0,0 +1,169 @@ +*5 +$4 +XADD +$8 +mystream +$15 +1695649068107-0 +$7 +message +$8 +Message1 +*5 +$4 +XADD +$8 +mystream +$15 +1695649068110-0 +$7 +message +$8 +Message2 +*5 +$4 +XADD +$8 +mystream +$15 +1695649069139-0 +$7 +message +$8 +Message3 +*5 +$4 +XADD +$8 +mystream +$15 +1695649446276-0 +$7 +message +$8 +Message4 +*5 +$4 +XADD +$8 +mystream +$15 +1695649456516-0 +$7 +message +$8 +Message5 +*9 +$4 +XADD +$8 +mystream +$15 +1695893015933-0 +$6 +field1 +$6 +value1 +$6 +field2 +$6 +value2 +$6 +field3 +$6 +value3 +*7 +$6 +XSETID +$8 +mystream +$15 +1695893015933-0 +$12 +ENTRIESADDED +$1 +6 +$12 +MAXDELETEDID +$3 +0-0 +*7 +$6 +XGROUP +$6 +CREATE +$8 +mystream +$6 +groupA +$15 +1695649446276-0 +$11 +ENTRIESREAD +$1 +4 +*12 +$6 +XCLAIM +$8 +mystream +$6 +groupA +$10 +consumerA2 +$1 +0 +$15 +1695649446276-0 +$4 +TIME +$13 +1695649446276 +$10 +RETRYCOUNT +$1 +1 +$6 +JUSTID +$5 +FORCE +*7 +$6 +XGROUP +$6 +CREATE +$8 +mystream +$6 +groupB +$15 +1695649069139-0 +$11 +ENTRIESREAD +$1 +3 +*12 +$6 +XCLAIM +$8 +mystream +$6 +groupB +$10 +consumerB1 +$1 +0 +$15 +1695649069139-0 +$4 +TIME +$13 +1695649069139 +$10 +RETRYCOUNT +$1 +1 +$6 +JUSTID +$5 +FORCE diff --git a/test/dumps/validate_all_json_files.sh b/test/dumps/validate_all_json_files.sh new file mode 100755 index 0000000..18505ae --- /dev/null +++ b/test/dumps/validate_all_json_files.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +for unwrap_json_file in *.json; do + tmp_file=wrapped_$unwrap_json_file + # Wrap the unwrapped JSON file with curly braces to make it valid JSON + echo '{' > $tmp_file + cat "$unwrap_json_file" >> $tmp_file + echo '}' >> $tmp_file + + # Use jq to check if the JSON is valid + if jq empty $tmp_file &> /dev/null; then + echo "Validation succeeded for $unwrap_json_file" + else + echo "Validation failed for $unwrap_json_file" + exit 1 + fi + + # Clean up the temporary wrapped file + rm $tmp_file +done +exit 0 \ No newline at end of file diff --git a/test/dumps/ziplist_raw.json b/test/dumps/ziplist_raw.json index b65cf57..37a8a92 100644 --- a/test/dumps/ziplist_raw.json +++ b/test/dumps/ziplist_raw.json @@ -1 +1 @@ -"ziplist_compresses_easily":"\n\xc3<@\x95\x04\x95\x00\x00\x00n \x03\x00\x06 \x02\x00a`\x00\x01\b\f`\x06\xa0\x00\x01\x0e\x12\xa0\b\xe0\x02\x00\x01\x14\x18\xe0\x02\f\xe0\x04\x00\x01\x1a\x1e\xe0\x04\x0e\xe0\b\x00\x01 $\xe0\b\x12\xe0\n\x00\x00\xff" \ No newline at end of file +"ziplist_compresses_easily":"\n\u00c3<@\u0095\u0004\u0095\u0000\u0000\u0000n \u0003\u0000\u0006 \u0002\u0000a`\u0000\u0001\b\f`\u0006\u00a0\u0000\u0001\u000e\u0012\u00a0\b\u00e0\u0002\u0000\u0001\u0014\u0018\u00e0\u0002\f\u00e0\u0004\u0000\u0001\u001a\u001e\u00e0\u0004\u000e\u00e0\b\u0000\u0001 $\u00e0\b\u0012\u00e0\n\u0000\u0000\u00ff" \ No newline at end of file diff --git a/test/dumps/ziplist_struct.json b/test/dumps/ziplist_struct.json index 3e362ef..78b70b9 100644 --- a/test/dumps/ziplist_struct.json +++ b/test/dumps/ziplist_struct.json @@ -1 +1 @@ -"ziplist_compresses_easily":["\x95\x00\x00\x00n\x00\x00\x00\x06\x00\x00\x06aaaaaa\b\faaaaaaaaaaaa\x0e\x12aaaaaaaaaaaaaaaaaa\x14\x18aaaaaaaaaaaaaaaaaaaaaaaa\x1a\x1eaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa $aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\xff"] \ No newline at end of file +"ziplist_compresses_easily":["\u0095\u0000\u0000\u0000n\u0000\u0000\u0000\u0006\u0000\u0000\u0006aaaaaa\b\faaaaaaaaaaaa\u000e\u0012aaaaaaaaaaaaaaaaaa\u0014\u0018aaaaaaaaaaaaaaaaaaaaaaaa\u001a\u001eaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa $aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\u00ff"] \ No newline at end of file diff --git a/test/dumps/zset_lp_v11_raw.json b/test/dumps/zset_lp_v11_raw.json index 90e13c1..d50bb23 100644 --- a/test/dumps/zset_lp_v11_raw.json +++ b/test/dumps/zset_lp_v11_raw.json @@ -3,4 +3,4 @@ "ctime":"1695704816", "used-mem":"969032", "aof-base":"0", -"myzset":"\x11\xc3@\xe6A\x12\x1f\x12\x01\x00\x000\x00\x83a10\x04\x84-inf\x05\x83a24\x04\x96-4.329000\x02123@\x02\x0631e+28\x17 '\x032\x04\xf4\x00`\x00\x02\xe0\xff\t +\xe0\x00\x0e\x00\xfc@\x0e\x0718\x04\xf2\x00\x00\xff\x04 \t\x046\x04\xdf\x01\x02 \x07\x064\x04\x88-9.9@\x00@/@f\x021.1 f\x0e19\x04\xdf\xff\x02\x82a1\x03\x00\x01\x82a2`\x05\x003`\x05\x004`\x05\x005`\x05\x006`\x05\x057\x03\x832.2@V\x053\x04\x888.8`\x00 N\x0415\x04\xc0\xff@e\x007@w\x00\x01 \x0121\xc0\x9f\x01\x04\x00@ \xe0\x00\x0e\x00 @\x0e\x0523\x04\x8c1. \xd6\x02033 \xd0\x065\r\x82a8\x03\x83 \xf6\x03\x04\x82a9`\b\x01\x04\xff" +"myzset":"\u0011\u00c3@\u00e6A\u0012\u001f\u0012\u0001\u0000\u00000\u0000\u0083a10\u0004\u0084-inf\u0005\u0083a24\u0004\u0096-4.329000\u0002123@\u0002\u000631e+28\u0017 '\u00032\u0004\u00f4\u0000`\u0000\u0002\u00e0\u00ff\t +\u00e0\u0000\u000e\u0000\u00fc@\u000e\u000718\u0004\u00f2\u0000\u0000\u00ff\u0004 \t\u00046\u0004\u00df\u0001\u0002 \u0007\u00064\u0004\u0088-9.9@\u0000@/@f\u00021.1 f\u000e19\u0004\u00df\u00ff\u0002\u0082a1\u0003\u0000\u0001\u0082a2`\u0005\u00003`\u0005\u00004`\u0005\u00005`\u0005\u00006`\u0005\u00057\u0003\u00832.2@V\u00053\u0004\u00888.8`\u0000 N\u000415\u0004\u00c0\u00ff@e\u00007@w\u0000\u0001 \u000121\u00c0\u009f\u0001\u0004\u0000@ \u00e0\u0000\u000e\u0000 @\u000e\u000523\u0004\u008c1. \u00d6\u0002033 \u00d0\u00065\r\u0082a8\u0003\u0083 \u00f6\u0003\u0004\u0082a9`\b\u0001\u0004\u00ff" diff --git a/test/dumps/zset_lp_v11_struct.json b/test/dumps/zset_lp_v11_struct.json index c05c7a9..debe4b0 100644 --- a/test/dumps/zset_lp_v11_struct.json +++ b/test/dumps/zset_lp_v11_struct.json @@ -3,4 +3,4 @@ "ctime":"1695704816", "used-mem":"969032", "aof-base":"0", -"myzset":["\x12\x01\x00\x000\x00\x83a10\x04\x84-inf\x05\x83a24\x04\x96-4.329000123123131e+28\x17\x83a12\x04\xf4\x00\x00\x00\x00\x00\x00\xe0\xff\t\x83a22\x04\xf4\x00\x00\x00\x00\x00\x00\xfc\xff\t\x83a18\x04\xf2\x00\x00\xff\x04\x83a16\x04\xdf\x01\x02\x83a14\x04\x88-9.99999\t\x83a20\x04\x84-1.1\x05\x83a19\x04\xdf\xff\x02\x82a1\x03\x00\x01\x82a2\x03\x00\x01\x82a3\x03\x00\x01\x82a4\x03\x00\x01\x82a5\x03\x00\x01\x82a6\x03\x00\x01\x82a7\x03\x832.2\x04\x83a13\x04\x888.888888\t\x83a15\x04\xc0\xff\x02\x83a17\x04\xf2\x00\x00\x01\x04\x83a21\x04\xf4\x00\x00\x00\x00\x00\x00\x04\x00\t\x83a11\x04\xf4\x00\x00\x00\x00\x00\x00 \x00\t\x83a23\x04\x8c1.000033e+25\r\x82a8\x03\x83inf\x04\x82a9\x03\x83inf\x04\xff"] +"myzset":["\u0012\u0001\u0000\u00000\u0000\u0083a10\u0004\u0084-inf\u0005\u0083a24\u0004\u0096-4.329000123123131e+28\u0017\u0083a12\u0004\u00f4\u0000\u0000\u0000\u0000\u0000\u0000\u00e0\u00ff\t\u0083a22\u0004\u00f4\u0000\u0000\u0000\u0000\u0000\u0000\u00fc\u00ff\t\u0083a18\u0004\u00f2\u0000\u0000\u00ff\u0004\u0083a16\u0004\u00df\u0001\u0002\u0083a14\u0004\u0088-9.99999\t\u0083a20\u0004\u0084-1.1\u0005\u0083a19\u0004\u00df\u00ff\u0002\u0082a1\u0003\u0000\u0001\u0082a2\u0003\u0000\u0001\u0082a3\u0003\u0000\u0001\u0082a4\u0003\u0000\u0001\u0082a5\u0003\u0000\u0001\u0082a6\u0003\u0000\u0001\u0082a7\u0003\u00832.2\u0004\u0083a13\u0004\u00888.888888\t\u0083a15\u0004\u00c0\u00ff\u0002\u0083a17\u0004\u00f2\u0000\u0000\u0001\u0004\u0083a21\u0004\u00f4\u0000\u0000\u0000\u0000\u0000\u0000\u0004\u0000\t\u0083a11\u0004\u00f4\u0000\u0000\u0000\u0000\u0000\u0000 \u0000\t\u0083a23\u0004\u008c1.000033e+25\r\u0082a8\u0003\u0083inf\u0004\u0082a9\u0003\u0083inf\u0004\u00ff"] diff --git a/test/dumps/zset_zl_v6_raw.json b/test/dumps/zset_zl_v6_raw.json index 27d70ad..da99547 100644 --- a/test/dumps/zset_zl_v6_raw.json +++ b/test/dumps/zset_zl_v6_raw.json @@ -1 +1 @@ -"myzset":"\f\xc3A\x0fAP\x04P\x01\x00\x00J \x03\x1f0\x00\x00\x03a10\x05\x04-inf\x06\x03a24\x05\x17-4.329000123@\x02\x07309e+28\x19 (\x032\x05\xe0\x00`\x00\x02\xe0\xff\n ,\xe0\x00\x0e\x00\xfc@\x0e\x0718\x05\xf0\x00\x00\xff\x05 \t\x056\x05\xc0\x01\xff\x04 \b\x064\x05\x13-9.9@\x00 P\xa0\x00\x014\x15 ;\x000 \x19\x021.1\xa0\x12\xa0\x00\x001 \x19\x0e19\x05\xfe\xff\x03\x02a1\x04\xf1\x02\x02a2`\x05\b3\x04\x02-0\x04\x02a4\xa0\x07\x005`\x15\x006`\x05\x057\x04\x122.2\xa0@\xa0\x00\x012\x14 G\x053\x05\x128.8@\x00\x007@\x7f`\x00\x007@\x18\x045\x05\xc0\xff\x00@\x9d\x007@\xb0\x00\x01 \xb0\x0121\xc0\xd8\x01\x04\x00 \xd8\x001\xe0\x00\x0e\x00 @\x0e\x0523\x05\f1.@Z\x0133!\t\x065\x0e\x02a8\x04\x03!0\x03\x05\x02a9@\b\x01f\xff" +"myzset":"\f\u00c3A\u000fAP\u0004P\u0001\u0000\u0000J \u0003\u001f0\u0000\u0000\u0003a10\u0005\u0004-inf\u0006\u0003a24\u0005\u0017-4.329000123@\u0002\u0007309e+28\u0019 (\u00032\u0005\u00e0\u0000`\u0000\u0002\u00e0\u00ff\n ,\u00e0\u0000\u000e\u0000\u00fc@\u000e\u000718\u0005\u00f0\u0000\u0000\u00ff\u0005 \t\u00056\u0005\u00c0\u0001\u00ff\u0004 \b\u00064\u0005\u0013-9.9@\u0000 P\u00a0\u0000\u00014\u0015 ;\u00000 \u0019\u00021.1\u00a0\u0012\u00a0\u0000\u00001 \u0019\u000e19\u0005\u00fe\u00ff\u0003\u0002a1\u0004\u00f1\u0002\u0002a2`\u0005\b3\u0004\u0002-0\u0004\u0002a4\u00a0\u0007\u00005`\u0015\u00006`\u0005\u00057\u0004\u00122.2\u00a0@\u00a0\u0000\u00012\u0014 G\u00053\u0005\u00128.8@\u0000\u00007@\u007f`\u0000\u00007@\u0018\u00045\u0005\u00c0\u00ff\u0000@\u009d\u00007@\u00b0\u0000\u0001 \u00b0\u000121\u00c0\u00d8\u0001\u0004\u0000 \u00d8\u00001\u00e0\u0000\u000e\u0000 @\u000e\u000523\u0005\f1.@Z\u000133!\t\u00065\u000e\u0002a8\u0004\u0003!0\u0003\u0005\u0002a9@\b\u0001f\u00ff" diff --git a/test/dumps/zset_zl_v6_struct.json b/test/dumps/zset_zl_v6_struct.json index 8dd8a52..87cd015 100644 --- a/test/dumps/zset_zl_v6_struct.json +++ b/test/dumps/zset_zl_v6_struct.json @@ -1 +1 @@ -"myzset":["P\x01\x00\x00J\x01\x00\x000\x00\x00\x03a10\x05\x04-inf\x06\x03a24\x05\x17-4.3290001231231309e+28\x19\x03a12\x05\xe0\x00\x00\x00\x00\x00\x00\xe0\xff\n\x03a22\x05\xe0\x00\x00\x00\x00\x00\x00\xfc\xff\n\x03a18\x05\xf0\x00\x00\xff\x05\x03a16\x05\xc0\x01\xff\x04\x03a14\x05\x13-9.9999900000000004\x15\x03a20\x05\x13-1.1000000000000001\x15\x03a19\x05\xfe\xff\x03\x02a1\x04\xf1\x02\x02a2\x04\xf1\x02\x02a3\x04\x02-0\x04\x02a4\x04\x02-0\x04\x02a5\x04\xf1\x02\x02a6\x04\xf1\x02\x02a7\x04\x122.2000000000000002\x14\x03a13\x05\x128.8888879999999997\x14\x03a15\x05\xc0\xff\x00\x04\x03a17\x05\xf0\x00\x00\x01\x05\x03a21\x05\xe0\x00\x00\x00\x00\x00\x00\x04\x00\n\x03a11\x05\xe0\x00\x00\x00\x00\x00\x00 \x00\n\x03a23\x05\f1.000033e+25\x0e\x02a8\x04\x03inf\x05\x02a9\x04\x03inf\xff"] \ No newline at end of file +"myzset":["P\u0001\u0000\u0000J\u0001\u0000\u00000\u0000\u0000\u0003a10\u0005\u0004-inf\u0006\u0003a24\u0005\u0017-4.3290001231231309e+28\u0019\u0003a12\u0005\u00e0\u0000\u0000\u0000\u0000\u0000\u0000\u00e0\u00ff\n\u0003a22\u0005\u00e0\u0000\u0000\u0000\u0000\u0000\u0000\u00fc\u00ff\n\u0003a18\u0005\u00f0\u0000\u0000\u00ff\u0005\u0003a16\u0005\u00c0\u0001\u00ff\u0004\u0003a14\u0005\u0013-9.9999900000000004\u0015\u0003a20\u0005\u0013-1.1000000000000001\u0015\u0003a19\u0005\u00fe\u00ff\u0003\u0002a1\u0004\u00f1\u0002\u0002a2\u0004\u00f1\u0002\u0002a3\u0004\u0002-0\u0004\u0002a4\u0004\u0002-0\u0004\u0002a5\u0004\u00f1\u0002\u0002a6\u0004\u00f1\u0002\u0002a7\u0004\u00122.2000000000000002\u0014\u0003a13\u0005\u00128.8888879999999997\u0014\u0003a15\u0005\u00c0\u00ff\u0000\u0004\u0003a17\u0005\u00f0\u0000\u0000\u0001\u0005\u0003a21\u0005\u00e0\u0000\u0000\u0000\u0000\u0000\u0000\u0004\u0000\n\u0003a11\u0005\u00e0\u0000\u0000\u0000\u0000\u0000\u0000 \u0000\n\u0003a23\u0005\f1.000033e+25\u000e\u0002a8\u0004\u0003inf\u0005\u0002a9\u0004\u0003inf\u00ff"] \ No newline at end of file diff --git a/test/test_common.c b/test/test_common.c index bf0465e..a0439af 100644 --- a/test/test_common.c +++ b/test/test_common.c @@ -2,6 +2,7 @@ #define _POSIX_C_SOURCE 200809L #include +#include #include #include #include @@ -22,6 +23,16 @@ redisContext *redisServersStack[MAX_NUM_REDIS_INST] = {0}; int redisPort[MAX_NUM_REDIS_INST]= {0}; pid_t redisPID[MAX_NUM_REDIS_INST] = {0}; const char *redisInstallFolder = NULL; +char redisVer[10]; +int redisVersionInit, redisVerMajor, redisVerMinor; + +const char *getTargetRedisVersion(int *major, int *minor) { + /* must be called only after setupRedisServer() */ + assert(redisVersionInit == 1); + if (major) *major = redisVerMajor; + if (minor) *minor = redisVerMinor; + return redisVer; +} void runSystemCmd(const char *cmdFormat, ...) { char cmd[1024]; @@ -38,10 +49,10 @@ void runSystemCmd(const char *cmdFormat, ...) { } } -char *readFile(const char *filename, size_t *length) { +char *readFile(const char *filename, size_t *length, char *ignoredCh) { FILE* fp; char* str; - size_t size; + size_t size, i = 0; assert_non_null(fp = fopen(filename, "r")); @@ -50,14 +61,19 @@ char *readFile(const char *filename, size_t *length) { fseek (fp, 0, SEEK_SET); assert_non_null(str = (char*) malloc(size + 1)); - size_t readBytes = fread(str, 1, size, fp); - assert_int_equal(readBytes, size); + char ch; + while (fread(&ch, 1, 1, fp) == 1) { + int incr = 1; + str[i] = ch; + for (int j = 0 ; (ignoredCh) && (ignoredCh[j] != '\0') && (incr) ; ++j) + incr = (ignoredCh[j] == ch) ? 0 : 1; + i += incr; + } - str[size] = '\0'; + str[i] = '\0'; fclose(fp); - if (length) *length = size; - + if (length) *length = i; return str; } @@ -103,7 +119,7 @@ char *substring(char *str, size_t len, char *substr) { void assert_file_payload(const char *filename, char *expData, int expLen, MatchType matchType, int expMatch) { const char *matchTypeName, *errMsg; size_t filelen; - char *filedata = readFile(filename, &filelen); + char *filedata = readFile(filename, &filelen, NULL); int result=1; switch (matchType) { @@ -137,7 +153,7 @@ void assert_file_payload(const char *filename, char *expData, int expLen, MatchT printf("%s\n---- file [%s] ----\n", errMsg, filename); printHexDump(filedata, filelen, buf, (int) sizeof(buf)); printf("%s", buf); - printf("\n---- Expected %s ----\n", matchTypeName); + printf("\n---- Expected %s %s ----\n", matchTypeName, (expMatch) ? "" : "not to match"); printHexDump(expData, expLen, buf, (int) sizeof(buf)); printf("%s", buf); printf("\n------------\n"); @@ -251,6 +267,43 @@ void setRedisInstallFolder(const char *path) { redisInstallFolder = path; } +/* Extract Redis version. Aborts on any failure. */ +void get_redis_version(redisContext *c, int *majorptr, int *minorptr) { +#define REDIS_VERSION_FIELD "redis_version:" + redisReply *reply; + char *eptr, *s, *e; + int major, minor; + + reply = redisCommand(c, "INFO"); + if (reply == NULL || c->err || reply->type != REDIS_REPLY_STRING) + goto abort; + if ((s = strstr(reply->str, REDIS_VERSION_FIELD)) == NULL) + goto abort; + + s += strlen(REDIS_VERSION_FIELD); + + /* We need a field terminator and at least 'x.y.z' (5) bytes of data */ + if ((e = strstr(s, "\r\n")) == NULL || (e - s) < 5) + goto abort; + + /* Extract version info */ + major = strtol(s, &eptr, 10); + if (*eptr != '.') goto abort; + minor = strtol(eptr+1, NULL, 10); + + /* Push info the caller wants */ + if (majorptr) *majorptr = major; + if (minorptr) *minorptr = minor; + + freeReplyObject(reply); + return; + + abort: + freeReplyObject(reply); + fprintf(stderr, "Error: Cannot determine Redis version, aborting\n"); + exit(1); +} + void setupRedisServer(const char *extraArgs) { /* If redis not installed return gracefully */ @@ -297,6 +350,7 @@ void setupRedisServer(const char *extraArgs) { exit(1); } else { /* parent */ int retryCount = 3; + static char *prefixVer = ""; redisContext *redisConnContext = redisConnect("localhost", port); while ((!redisConnContext) || (redisConnContext->err)) { @@ -319,7 +373,20 @@ void setupRedisServer(const char *extraArgs) { redisPort[currRedisInst] = port; redisServersStack[currRedisInst] = redisConnContext; redisPID[currRedisInst] = pid; - printf(">> Redis Server(%d) started on port %d with PID %d\n", currRedisInst, port, pid); + + if (!redisVersionInit) { + get_redis_version(redisConnContext, &redisVerMajor, &redisVerMinor); + snprintf(redisVer, sizeof(redisVer), "%d.%d", redisVerMajor, redisVerMinor); + if ((redisVerMajor == 255) && (redisVerMinor == 255)) {/* unstable version? */ + snprintf(redisVer, sizeof(redisVer), + MAX_SUPPORTED_REDIS_VERSION); + prefixVer = "Unresolved Version. Assumed "; + } + redisVersionInit = 1; + + } + printf(">> Redis Server(%d) started on port %d with PID %d (%sVersion=%s)\n", + currRedisInst, port, pid, prefixVer, redisVer); /* Close any subprocess in case of exit due to error flow */ atexit(cleanupRedisServer); @@ -450,10 +517,10 @@ void assert_json_equal(const char* filename1, const char* filename2, int ignoreL int res = -1; FILE* file1 = fopen(filename1, "r"); - assert_non_null(file1); + ASSERT_TRUE(file1, "Failed to open file: %s", filename1); FILE* file2 = fopen(filename2, "r"); - assert_non_null(file2); + ASSERT_TRUE(file2, "Failed to open file: %s", filename2); while (fgets(line1, MAX_LINE_LENGTH, file1)) { sanitize_json_line(line1); @@ -497,12 +564,12 @@ void assert_json_equal(const char* filename1, const char* filename2, int ignoreL printf("Json files not equal.\n"); printf("---- %s ----\n", filename1); - char *f1 = readFile(filename1, NULL); + char *f1 = readFile(filename1, NULL, NULL); printf ("%s", f1); free(f1); printf("\n---- %s ----\n", filename2); - char *f2 = readFile(filename2, NULL); + char *f2 = readFile(filename2, NULL, NULL); printf ("%s", f2); free(f2); printf("\n------------\n"); diff --git a/test/test_common.h b/test/test_common.h index d8c7ab6..440c68d 100644 --- a/test/test_common.h +++ b/test/test_common.h @@ -9,6 +9,8 @@ #include "../api/librdb-api.h" /* RDB library header */ #include "../api/librdb-ext-api.h" /* RDB library extension header */ +#define MAX_SUPPORTED_REDIS_VERSION "7.2" + #define UNUSED(...) unused( (void *) NULL, __VA_ARGS__); static inline void unused(void *dummy, ...) { (void)(dummy);} @@ -17,6 +19,15 @@ static inline void unused(void *dummy, ...) { (void)(dummy);} #define STR_AND_SIZE(str) str, (sizeof(str)-1) +#define ASSERT_TRUE(exp, format, ...) \ + do { \ + if (!(exp)) { \ + fprintf(stderr, "Assertion failed: %s\n", #exp); \ + fprintf(stderr, "Error: " format "\n", __VA_ARGS__); \ + assert_true(0); \ + } \ + } while (0) + typedef enum MatchType { M_PREFIX, M_ENTIRE, @@ -34,6 +45,7 @@ void assert_json_equal(const char *f1, const char *f2, int ignoreListOrder); void setRedisInstallFolder(const char *path); int getRedisPort(void); void setupRedisServer(const char *extraArgs); +const char *getTargetRedisVersion(int *major, int *minor); /* call only after setupRedisServer() */ void teardownRedisServer(void); int isSetRedisServer(void); char *sendRedisCmd(char *cmd, int expRetType, char *expRsp); @@ -57,7 +69,7 @@ void *xclone(void *str, size_t len); void xfree(void *ptr); void *xrealloc(void *ptr, size_t size); -char *readFile(const char *filename, size_t *len); +char *readFile(const char *filename, size_t *len, char *ignoredCh); void cleanTmpFolder(void); void setEnvVar (const char *name, const char *val); char *substring(char *str, size_t len, char *substr); diff --git a/test/test_main.c b/test/test_main.c index ede23be..37fa1ad 100644 --- a/test/test_main.c +++ b/test/test_main.c @@ -18,8 +18,8 @@ static void test_createReader_missingFile(void **state) { assert_int_equal(err, RDB_ERR_FAILED_OPEN_RDB_FILE); /* verify returned error string */ - assert_string_equal(RDB_getErrorMessage(parser), "Failed to open RDB file: ./test/dumps/non_exist_file.rdb"); - + assert_string_equal(RDB_getErrorMessage(parser), + "Failed to open RDB file `./test/dumps/non_exist_file.rdb`: No such file or directory\n"); RDB_deleteParser(parser); } @@ -38,6 +38,7 @@ static void test_empty_rdb(void **state) { .encoding = RDBX_CONV_JSON_ENC_PLAIN, .includeAuxField = 1, .includeFunc = 0, + .includeStreamMeta = 0, .flatten = 1, }; @@ -61,15 +62,15 @@ static void test_mixed_levels_registration(void **state) { RdbParser *parser = RDB_createParserRdb(NULL); RDB_setLogLevel(parser, RDB_LOG_ERR); assert_non_null(RDBX_createReaderFile(parser, rdbfile)); - RdbxToJsonConf conf1 = {RDB_LEVEL_DATA, RDBX_CONV_JSON_ENC_PLAIN, 0, 0, 1}; + RdbxToJsonConf conf1 = {RDB_LEVEL_DATA, RDBX_CONV_JSON_ENC_PLAIN, 0, 0, 0, 1}; assert_non_null(RDBX_createHandlersToJson(parser, jsonfileData, &conf1)); - RdbxToJsonConf conf2 = {RDB_LEVEL_RAW, RDBX_CONV_JSON_ENC_PLAIN, 0, 0, 1}; + RdbxToJsonConf conf2 = {RDB_LEVEL_RAW, RDBX_CONV_JSON_ENC_PLAIN, 0, 0, 0, 1}; assert_non_null(RDBX_createHandlersToJson(parser, jsonfileRaw, &conf2)); /* configure at what level of the parser each obj type should be handled and callback */ - RDB_handleByLevel(parser, RDB_DATA_TYPE_STRING, RDB_LEVEL_RAW, 0); - RDB_handleByLevel(parser, RDB_DATA_TYPE_LIST, RDB_LEVEL_DATA, 0); + RDB_handleByLevel(parser, RDB_DATA_TYPE_STRING, RDB_LEVEL_RAW); + RDB_handleByLevel(parser, RDB_DATA_TYPE_LIST, RDB_LEVEL_DATA); while ((status = RDB_parse(parser)) == RDB_STATUS_WAIT_MORE_DATA); assert_int_equal( status, RDB_STATUS_OK); @@ -139,7 +140,15 @@ int main(int argc, char *argv[]) { char *redisInstallFolder = getenv("LIBRDB_REDIS_FOLDER"); - const char *USAGE =" [-h|--help] [f|--redis-folder ] [-g|--test-group ] [-t|--test ]"; + const char *USAGE = "Usage: [OPTIONS]\n" + "Options:\n" + " -h, --help Show this help message\n" + " -f, --redis-folder Specify the Redis folder to use for the tests\n" + " -g, --test-group Selected test group to run\n" + " -t, --test Selected test to run"; + + + /* Parse command-line arguments */ for (int i = 1; i < argc; i++) { if ((strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0)) { @@ -182,7 +191,6 @@ int main(int argc, char *argv[]) { RUN_TEST_GROUP(group_rdb_to_redis); /*external*/ RUN_TEST_GROUP(group_test_rdb_cli); /*external*/ - printf("\n*************** SIMULATING WAIT_MORE_DATA *******************\n"); setEnvVar("LIBRDB_SIM_WAIT_MORE_DATA", "1"); RUN_TEST_GROUP(group_main); diff --git a/test/test_malloc.c b/test/test_malloc.c index 5cba917..46da20e 100644 --- a/test/test_malloc.c +++ b/test/test_malloc.c @@ -43,6 +43,7 @@ static void test_extern_alloc(void **state) { .encoding = RDBX_CONV_JSON_ENC_PLAIN, .includeAuxField = 1, .includeFunc = 0, + .includeStreamMeta = 0, .flatten = 1, }; assert_non_null(RDBX_createHandlersToJson(parser, diff --git a/test/test_rdb_cli.c b/test/test_rdb_cli.c index 0a5b205..611ddc5 100644 --- a/test/test_rdb_cli.c +++ b/test/test_rdb_cli.c @@ -26,6 +26,7 @@ static void test_rdb_cli_resp_common(const char *rdbfile) { .encoding = RDBX_CONV_JSON_ENC_PLAIN, .includeAuxField = 0, .includeFunc = 0, + .includeStreamMeta = 0, .flatten = 1, }; @@ -143,9 +144,17 @@ static void test_rdb_cli_redis_auth(void **state) { runSystemCmd(" ./bin/rdb-cli ./test/dumps/single_key.rdb redis --password abc -p %d > /dev/null ", getRedisPort()); /* auth user */ - sendRedisCmd("ACL SETUSER newuser on >newpwd +@all ~*", REDIS_REPLY_STATUS, NULL); - sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL); - runSystemCmd(" ./bin/rdb-cli ./test/dumps/single_key.rdb redis -P newpwd -u newuser -p %d > /dev/null ", getRedisPort()); + int major; + getTargetRedisVersion(&major, NULL); + /* ACL available since 6.0 */ + if (major>=6) { + sendRedisCmd("ACL SETUSER newuser on >newpwd +@all ~*", + REDIS_REPLY_STATUS, NULL); + sendRedisCmd("FLUSHALL", REDIS_REPLY_STATUS, NULL); + runSystemCmd( + " ./bin/rdb-cli ./test/dumps/single_key.rdb redis -P newpwd -u newuser -p %d > /dev/null ", + getRedisPort()); + } teardownRedisServer(); } diff --git a/test/test_rdb_to_json.c b/test/test_rdb_to_json.c index 1a13490..0868349 100644 --- a/test/test_rdb_to_json.c +++ b/test/test_rdb_to_json.c @@ -11,6 +11,7 @@ .includeAuxField = 1, \ .includeFunc = 0, \ .flatten = 1, \ + .includeStreamMeta = 0, \ }; /* Test different use cases to convert given rdb file to json: @@ -37,7 +38,7 @@ void testRdbToJsonCommon(const char *rdbfile, RdbMemAlloc *pMemAlloc = (type != RDB_BULK_ALLOC_MAX) ? &memAlloc : NULL; /* read file to buffer for testing RDB_parseBuff() */ - buffer = (unsigned char *) readFile(rdbfile, &bufLen); + buffer = (unsigned char *) readFile(rdbfile, &bufLen, NULL); /*** 1. RDB_parse - parse with RDB reader ***/ remove(jsonfile); @@ -440,25 +441,15 @@ static void test_r2j_function (void **state) { static void test_r2j_module_raw(void **state) { UNUSED(state); - RdbxToJsonConf r2jConf = { - .level = RDB_LEVEL_RAW, - .encoding = RDBX_CONV_JSON_ENC_PLAIN, - .includeAuxField = 0, - .includeFunc = 0, - .flatten = 1, - }; + RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_RAW); + r2jConf.includeAuxField = 0; testRdbToJsonCommon(DUMP_FOLDER("module.rdb"), DUMP_FOLDER("module_raw.json"), &r2jConf); } static void test_r2j_module_data(void **state) { UNUSED(state); - RdbxToJsonConf r2jConf = { - .level = RDB_LEVEL_DATA, - .encoding = RDBX_CONV_JSON_ENC_PLAIN, - .includeAuxField = 0, - .includeFunc = 0, - .flatten = 1, - }; + RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA); + r2jConf.includeAuxField = 0; testRdbToJsonCommon(DUMP_FOLDER("module.rdb"), DUMP_FOLDER("module_data.json"), &r2jConf); } @@ -475,6 +466,17 @@ static void test_r2j_string_int_encoded(void **state) { r2jConf.includeAuxField = 0; testRdbToJsonCommon(DUMP_FOLDER("string_int_encoded.rdb"), DUMP_FOLDER("string_int_encoded.json"), &r2jConf); } + +static void test_r2j_stream_data(void **state) { + UNUSED(state); + RdbxToJsonConf r2jConf = DEF_CONF(RDB_LEVEL_DATA); + r2jConf.includeAuxField = 0; + r2jConf.flatten = 1; + testRdbToJsonCommon(DUMP_FOLDER("stream_v11.rdb"), DUMP_FOLDER("stream_data.json"), &r2jConf); + r2jConf.includeStreamMeta = 1; + testRdbToJsonCommon(DUMP_FOLDER("stream_v11.rdb"), DUMP_FOLDER("stream_data_with_meta.json"), &r2jConf); +} + /*************************** group_rdb_to_json *******************************/ int group_rdb_to_json(void) { const struct CMUnitTest tests[] = { @@ -549,6 +551,9 @@ int group_rdb_to_json(void) { cmocka_unit_test(test_r2j_module_raw), cmocka_unit_test(test_r2j_module_aux_data), + /* stream */ + cmocka_unit_test(test_r2j_stream_data), + /* misc */ cmocka_unit_test(test_r2j_multiple_lists_and_strings_data), cmocka_unit_test(test_r2j_multiple_lists_and_strings_struct), diff --git a/test/test_rdb_to_redis.c b/test/test_rdb_to_redis.c index 04575bf..3e454c7 100644 --- a/test/test_rdb_to_redis.c +++ b/test/test_rdb_to_redis.c @@ -22,7 +22,7 @@ void rdb_to_tcp(const char *rdbfile, int pipelineDepth, int isRestore, char *res RdbxToRespConf rdb2respConf = { .supportRestore = isRestore, - .dstRedisVersion = "45.67.89", + .dstRedisVersion = getTargetRedisVersion(NULL, NULL), .supportRestoreModuleAux = isSupportRestoreModuleAux() }; @@ -50,6 +50,7 @@ static void rdb_to_json(const char *rdbfile, const char *outfile) { .includeAuxField = 0, .includeFunc = 0, .flatten = 1, + .includeStreamMeta = 0, }; RdbParser *parser = RDB_createParserRdb(NULL); @@ -91,10 +92,12 @@ static void test_rdb_to_redis_common(const char *rdbfile, int ignoreListOrder, c rdb_to_tcp(rdbfile, 1, isRestore, TMP_FOLDER("cmd.resp")); sendRedisCmd("SAVE", REDIS_REPLY_STATUS, NULL); - if (expRespCmd) { - /* Verify corresponding RESP commands includes `expRespCmd` or `RESTORE` */ - char *exp = (isRestore) ? "RESTORE" : expRespCmd; - assert_file_payload(TMP_FOLDER("cmd.resp"), exp, strlen(exp), M_SUBSTR, 1); + if (expRespCmd && !isRestore) { + /* Verify corresponding RESP commands includes `expRespCmd` */ + assert_file_payload(TMP_FOLDER("cmd.resp"), + expRespCmd, + strlen(expRespCmd), + M_SUBSTR, 1); } /* 3. From DUMP-RDB generate Json (out2.json) */ @@ -229,6 +232,11 @@ static void test_rdb_to_redis_policy_lru(void **state) { static void test_rdb_to_redis_function(void **state) { UNUSED(state); + int major; + getTargetRedisVersion(&major, NULL); + /* function available since 7.0 */ + if (major < 7) + skip(); test_rdb_to_redis_common(DUMP_FOLDER("function.rdb"), 1, NULL, NULL); } @@ -265,6 +273,10 @@ void test_rdb_to_redis_module(void **state) { sendRedisCmd("testrdb.get.key 123456", REDIS_REPLY_STRING, "7890"); } +void test_rdb_to_redis_stream(void **state) { + UNUSED(state); + test_rdb_to_redis_common(DUMP_FOLDER("stream_v11.rdb"), 1, NULL, NULL); +} /* iff 'delKeyBeforeWrite' is not set, then the parser will return an error on * loading 100_lists.rdb ("mylist1 mylist2 ... mylist100") on key 'mylist62' @@ -277,7 +289,7 @@ static void test_rdb_to_redis_del_before_write(void **state) { RdbxToRespConf rdb2respConf = { .delKeyBeforeWrite = delKeyBeforeWrite, .supportRestore = 1, - .dstRedisVersion = "45.67.89" + .dstRedisVersion = getTargetRedisVersion(NULL, NULL), }; /* create key `mylist62` that goanna appear as well in the RDB file */ @@ -346,6 +358,12 @@ int group_rdb_to_redis(void) { cmocka_unit_test_setup(test_rdb_to_redis_zset_lp, setupTest), cmocka_unit_test_setup(test_rdb_to_redis_zset_zl, setupTest), + /* module */ + cmocka_unit_test_setup(test_rdb_to_redis_module, setupTest), + + /* stream */ + cmocka_unit_test_setup(test_rdb_to_redis_stream, setupTest), + /* expired keys */ cmocka_unit_test_setup(test_rdb_to_redis_set_expired, setupTest), cmocka_unit_test_setup(test_rdb_to_redis_set_not_expired, setupTest), @@ -360,7 +378,6 @@ int group_rdb_to_redis(void) { cmocka_unit_test_setup(test_rdb_to_redis_del_before_write, setupTest), cmocka_unit_test_setup(test_rdb_to_redis_multiple_dbs, setupTest), cmocka_unit_test_setup(test_rdb_to_redis_function, setupTest), - cmocka_unit_test_setup(test_rdb_to_redis_module, setupTest), }; int res = cmocka_run_group_tests(tests, NULL, NULL); diff --git a/test/test_rdb_to_resp.c b/test/test_rdb_to_resp.c index e92cf3c..5575435 100644 --- a/test/test_rdb_to_resp.c +++ b/test/test_rdb_to_resp.c @@ -1,4 +1,7 @@ +#include #include +#include + #include "test_common.h" /* To enhance the clarity of our tests and keep expected outputs concise, a @@ -71,7 +74,7 @@ static void runWithAndWithoutRestore(const char *rdbfile) { r2rConf.delKeyBeforeWrite = 0; /* expect not use RESTORE */ - r2rConf.dstRedisVersion = "0.0.1"; + r2rConf.dstRedisVersion = "0.1"; testRdbToRespCommon(rdbfile, &r2rConf, (char*)restorePrefix, sizeof(restorePrefix), M_PREFIX, 0); /* expect use RESTORE */ @@ -239,6 +242,11 @@ static void test_r2r_zset_zl(void **state) { runWithAndWithoutRestore("zset_zl_v6.rdb"); } +static void test_r2r_stream(void **state) { + UNUSED(state); + runWithAndWithoutRestore("stream_v11.rdb"); +} + static void test_r2r_module(void **state) { UNUSED(state); unsigned char expRespData[] = { @@ -260,21 +268,33 @@ static void test_r2r_module(void **state) { static void test_r2r_module_aux(void **state) { UNUSED(state); unsigned char expRespData[] = { - 0x2a, 0x34, 0x0d, 0x0a, 0x24, 0x37, 0x0d, 0x0a, 0x52, 0x45, 0x53, 0x54, 0x4f, 0x52, 0x45, 0x0d, // *4..$7.. RESTORE. - 0x0a, 0x24, 0x34, 0x0d, 0x0a, 0x6b, 0x65, 0x79, 0x31, 0x0d, 0x0a, 0x24, 0x31, 0x0d, 0x0a, 0x30, // .$4..key 1..$1..0 - 0x0d, 0x0a, 0x24, 0x32, 0x39, 0x0d, 0x0a, 0x07, 0x81, 0xb5, 0xeb, 0x2d, 0xff, 0xfa, 0xdd, 0x6c, // ..$29... ...-...l - 0x01, 0x05, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x31, 0x00, 0x0b, 0x00, 0x03, 0xb6, 0x8b, 0x8a, // ...value 1....... - 0x58, 0x44, 0xb9, 0xff, 0x0d, 0x0a, // XD.... + 0x2a, 0x34, 0x0d, 0x0a, 0x24, 0x37, 0x0d, 0x0a, 0x52, 0x45, 0x53, 0x54, 0x4f, 0x52, 0x45, 0x0d, // *4..$7.. RESTORE. + 0x0a, 0x24, 0x31, 0x0d, 0x0a, 0x78, 0x0d, 0x0a, 0x24, 0x31, 0x0d, 0x0a, 0x30, 0x0d, 0x0a, 0x24, // .$1..x.. $1..0..$ + 0x31, 0x33, 0x0d, 0x0a, 0x00, 0xc0, 0x01, 0x0b, 0x00, 0x4f, 0xa7, 0x5a, 0xc5, 0x2c, 0x9e, 0xf8, // 13...... .O.Z.,.. + 0x75, 0x0d, 0x0a, 0x00, 0x0b, 0x00, 0xa6, 0xe6, 0xfb, 0x24, 0xce, 0x1a, 0x8c, 0x25, 0x0d, 0x0a, // u....... .$...%.. }; RdbxToRespConf r2rConf; - memset(&r2rConf, 0, sizeof(r2rConf)); r2rConf.supportRestore = 1; r2rConf.dstRedisVersion = "7.2"; /* resolved to rdb version 11 */ testRdbToRespCommon("module_aux.rdb", &r2rConf, (char *) expRespData, sizeof(expRespData), M_ENTIRE, 1); } +static void test_r2r_stream_with_target_62_and_72(void **state) { + size_t fileLen; + UNUSED(state); + RdbxToRespConf r2rConf1 = { .delKeyBeforeWrite=0, .supportRestore=0, .dstRedisVersion="6.2",}; + char *f1 = readFile(DUMP_FOLDER("stream_v11_target_ver_6.2.resp"), &fileLen, NULL); + testRdbToRespCommon("stream_v11.rdb", &r2rConf1, f1, fileLen, M_ENTIRE, 1); + free(f1); + + RdbxToRespConf r2rConf = { .delKeyBeforeWrite=0, .supportRestore=0, .dstRedisVersion="7.2",}; + f1 = readFile(DUMP_FOLDER("stream_v11_target_ver_7.2.resp"), &fileLen, NULL); + testRdbToRespCommon("stream_v11.rdb", &r2rConf, f1, fileLen, M_ENTIRE, 1); + free(f1); +} + /*************************** group_rdb_to_resp *******************************/ int group_rdb_to_resp(void) { const struct CMUnitTest tests[] = { @@ -305,18 +325,18 @@ int group_rdb_to_resp(void) { cmocka_unit_test(test_r2r_plain_zset_2), cmocka_unit_test(test_r2r_zset_lp), cmocka_unit_test(test_r2r_zset_zl), - - /* misc */ - cmocka_unit_test(test_r2r_multiple_lists_and_strings), - cmocka_unit_test(test_r2r_del_before_write_restore_replace), - /* mem policy */ cmocka_unit_test(test_r2r_policy_lfu), cmocka_unit_test(test_r2r_policy_lru), - /* module*/ cmocka_unit_test(test_r2r_module), -// cmocka_unit_test(test_r2r_module_aux), + cmocka_unit_test(test_r2r_module_aux), + /* stream */ + cmocka_unit_test(test_r2r_stream), + cmocka_unit_test(test_r2r_stream_with_target_62_and_72), + /* misc */ + cmocka_unit_test(test_r2r_multiple_lists_and_strings), + cmocka_unit_test(test_r2r_del_before_write_restore_replace), }; return cmocka_run_group_tests(tests, NULL, NULL); } diff --git a/test/test_resp_reader.c b/test/test_resp_reader.c index d6f5caa..44da150 100644 --- a/test/test_resp_reader.c +++ b/test/test_resp_reader.c @@ -12,6 +12,8 @@ static void test_resp_reader_common(RespReaderCtx *ctx, RespRes expRes, int expReplies) { + RespReaderCtx alterCtx; + if (!ctx) ctx = &alterCtx; if (initCtx) readRespInit(ctx); RespRes res = readRespReplies(ctx, payload, payloadSize); @@ -21,22 +23,31 @@ static void test_resp_reader_common(RespReaderCtx *ctx, static void test_single_status(void **state) { UNUSED(state); - RespReaderCtx ctx; - test_resp_reader_common(&ctx, STR_AND_SIZE("+OK\r\n"), + test_resp_reader_common(NULL, STR_AND_SIZE("+OK\r\n"), 1, RESP_REPLY_OK, 1); } static void test_single_int(void **state) { UNUSED(state); - RespReaderCtx ctx; - test_resp_reader_common(&ctx, STR_AND_SIZE(":1\r\n"), + test_resp_reader_common(NULL, STR_AND_SIZE(":1\r\n"), + 1, RESP_REPLY_OK, 1); +} + +static void test_array_3_bulks(void **state) { + UNUSED(state); + test_resp_reader_common(NULL, STR_AND_SIZE("*3\r\n$2\r\n12\r\n$1\r\nA\r\n$3\r\nABC\r\n"), + 1, RESP_REPLY_OK, 1); +} + +void test_array_single_bulk(void **state) { + UNUSED(state); + test_resp_reader_common(NULL, STR_AND_SIZE("*1\r\n$15\r\n1695649446276-0\r\n"), 1, RESP_REPLY_OK, 1); } static void test_two_statuses_and_partial_reply(void **state) { UNUSED(state); - RespReaderCtx ctx; - test_resp_reader_common(&ctx, STR_AND_SIZE("+OK\r\n+OK\r\n+OK\r"), + test_resp_reader_common(NULL, STR_AND_SIZE("+OK\r\n+OK\r\n+OK\r"), 1, RESP_REPLY_PARTIAL, 2); } @@ -71,8 +82,7 @@ static void test_single_bulk(void **state) { static void test_three_bulks(void **state) { UNUSED(state); - RespReaderCtx ctx; - test_resp_reader_common(&ctx, STR_AND_SIZE("$5\r\nmylib\r\n$4\r\nm\rib\r\n$8\r\nm123ylib\r\n"), 1, RESP_REPLY_OK, 3); + test_resp_reader_common(NULL, STR_AND_SIZE("$5\r\nmylib\r\n$4\r\nm\rib\r\n$8\r\nm123ylib\r\n"), 1, RESP_REPLY_OK, 3); } static void test_reply_error_fragmented(void **state) { @@ -108,11 +118,13 @@ static void test_reply_long_err_trimmed_by_report(void **state) { static void test_mixture_and_fragmented(void **state) { UNUSED(state); RespRes res; - char bulk[] = "+OK\r\n$5\r\nmylib\r\n+OK\r\n+OK\r\n"; + int expReplies = 5; + char bulk[] = "*3\r\n$2\r\n12\r\n$1\r\nA\r\n$3\r\nABC\r\n" + "+OK\r\n$5\r\nmylib\r\n+OK\r\n+OK\r\n"; RespReaderCtx ctx; /* all responses in one bulk */ - test_resp_reader_common(&ctx, STR_AND_SIZE(bulk), 1, RESP_REPLY_OK, 4); + test_resp_reader_common(&ctx, STR_AND_SIZE(bulk), 1, RESP_REPLY_OK, expReplies); /* split stream to two bulks */ for (size_t firstFragLen = 1 ; firstFragLen < sizeof(bulk) - 1 ; ++firstFragLen) { @@ -121,7 +133,7 @@ static void test_mixture_and_fragmented(void **state) { assert_int_not_equal(res, RESP_REPLY_ERR); res = readRespReplies(&ctx, bulk + firstFragLen, (sizeof(bulk) - 1) - firstFragLen); assert_int_equal(res, RESP_REPLY_OK); - assert_int_equal(ctx.countReplies, 4); + assert_int_equal(ctx.countReplies, expReplies); } } @@ -130,6 +142,8 @@ int group_test_resp_reader(void) { const struct CMUnitTest tests[] = { cmocka_unit_test(test_single_status), cmocka_unit_test(test_single_int), + cmocka_unit_test(test_array_single_bulk), + cmocka_unit_test(test_array_3_bulks), cmocka_unit_test(test_two_statuses_and_partial_reply), cmocka_unit_test(test_reply_fragmented), cmocka_unit_test(test_reply_error),