Skip to content

Commit

Permalink
save axoflow#2
Browse files Browse the repository at this point in the history
  • Loading branch information
@bshifter
bshifter committed Oct 16, 2024
1 parent 2824570 commit 1885749
Show file tree
Hide file tree
Showing 12 changed files with 512 additions and 247 deletions.
91 changes: 69 additions & 22 deletions modules/cef/event-format-parser.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ parse_extensions(EventParserContext *ctx, const gchar *input, gint input_len, GE

static inline gboolean
_match_field_to_column(EventParserContext *ctx, Field *field, const gchar *input, gint input_len,
FilterXObject *fillable,
GError **error)
FilterXObject *fillable,
GError **error)
{
FilterXObject *val = NULL;

Expand All @@ -155,11 +155,11 @@ _match_field_to_column(EventParserContext *ctx, Field *field, const gchar *input

gboolean ok = FALSE;
if (!*error && val)
{
FilterXObject *key = filterx_string_new(field->name, -1);
ok = filterx_object_set_subscript(fillable, key, &val);
filterx_object_unref(key);
}
{
FilterXObject *key = filterx_string_new(field->name, -1);
ok = filterx_object_set_subscript(fillable, key, &val);
filterx_object_unref(key);
}

filterx_object_unref(val);
return ok;
Expand All @@ -175,28 +175,30 @@ _parse_column(EventParserContext *ctx, FilterXObject *fillable, GError **error)
Field field = field_by_index(ctx->parser, ctx->field_index);

while (!_match_field_to_column(ctx, &field, input, input_len, fillable, error) && !*error && field.optional)
{
ctx->field_index++;
if (ctx->field_index >= ctx->num_fields)
return FALSE;
field = field_by_index(ctx->parser, ctx->field_index);
}
{
ctx->field_index++;
if (ctx->field_index >= ctx->num_fields)
return FALSE;
field = field_by_index(ctx->parser, ctx->field_index);
}
ctx->column_index++;
return TRUE;
}

static EventParserContext
_new_context(FilterXFunctionEventFormatParser *self, CSVScanner *csv_scanner)
{
EventParserContext ctx = {
EventParserContext ctx =
{
.parser = self,
.num_fields = self->config.header.num_fields,
.field_index = 0,
.csv_scanner = csv_scanner,
.flags = 0,
.kv_parser_value_separator = self->config.extensions.value_separator,
.kv_parser_value_separator = self->kv_value_separator ? self->kv_value_separator[0] : self->config.extensions.value_separator,
};
g_strlcpy(ctx.kv_parser_pair_separator, self->config.extensions.pair_separator, EVENT_FORMAT_PARSER_SEPARATOR_MAX_LEN);
g_strlcpy(ctx.kv_parser_pair_separator, self->kv_pair_separator ? : self->config.extensions.pair_separator,
EVENT_FORMAT_PARSER_PAIR_SEPARATOR_MAX_LEN);
return ctx;
}

Expand All @@ -222,12 +224,12 @@ parse(FilterXFunctionEventFormatParser *self, const gchar *log, gsize len, Filte


if (ctx.field_index <= ctx.num_fields - 1)
{
csv_scanner_take_rest(&csv_scanner);
ok = _parse_column(&ctx, fillable, error);
if(!ok || *error)
goto exit;
}
{
csv_scanner_take_rest(&csv_scanner);
ok = _parse_column(&ctx, fillable, error);
if(!ok || *error)
goto exit;
}

if (ctx.column_index < ctx.num_fields-1)
{
Expand Down Expand Up @@ -301,6 +303,48 @@ _extract_msg_expr(FilterXFunctionArgs *args, GError **error)
return msg_expr;
}

static gboolean
_extract_optional_args(FilterXFunctionEventFormatParser *self, FilterXFunctionArgs *args, GError **error)
{
gboolean exists;
gsize len;
const gchar *value;

value = filterx_function_args_get_named_literal_string(args, EVENT_FORMAT_PARSER_ARG_NAME_PAIR_SEPARATOR, &len,
&exists);
if (exists)
{
if (len < 1 || !value)
{
g_set_error(error, FILTERX_FUNCTION_ERROR, FILTERX_FUNCTION_ERROR_CTOR_FAIL,
EVENT_FORMAT_PARSER_ERR_EMPTY_STRING, EVENT_FORMAT_PARSER_ARG_NAME_PAIR_SEPARATOR);
goto error;
}
if (len > EVENT_FORMAT_PARSER_PAIR_SEPARATOR_MAX_LEN)
{
g_set_error(error, FILTERX_FUNCTION_ERROR, FILTERX_FUNCTION_ERROR_CTOR_FAIL,
EVENT_FORMAT_PARSER_ERR_SEPARATOR_MAX_LENGTH_EXCEEDED, EVENT_FORMAT_PARSER_ARG_NAME_PAIR_SEPARATOR);
goto error;
}
self->kv_pair_separator = value;
}
value = filterx_function_args_get_named_literal_string(args, EVENT_FORMAT_PARSER_ARG_NAME_VALUE_SEPARATOR, &len,
&exists);
if (exists)
{
if (len < 1 || !value)
{
g_set_error(error, FILTERX_FUNCTION_ERROR, FILTERX_FUNCTION_ERROR_CTOR_FAIL,
EVENT_FORMAT_PARSER_ERR_EMPTY_STRING, EVENT_FORMAT_PARSER_ARG_NAME_VALUE_SEPARATOR);
goto error;
}
self->kv_value_separator = value;
}
return TRUE;
error:
return FALSE;
}

static gboolean
_extract_args(FilterXFunctionEventFormatParser *self, FilterXFunctionArgs *args, GError **error)
{
Expand All @@ -316,6 +360,9 @@ _extract_args(FilterXFunctionEventFormatParser *self, FilterXFunctionArgs *args,
if (!self->msg)
return FALSE;

if (!_extract_optional_args(self, args, error))
return FALSE;

return TRUE;
}

Expand Down
13 changes: 10 additions & 3 deletions modules/cef/event-format-parser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,16 @@
#define EVENT_FORMAT_PARSER_ERR_LOG_SIGN_DIFFERS_MSG "the log signature differs. actual:%s expected:%s"
#define EVENT_FORMAT_PARSER_ERR_MISSING_COLUMNS_MSG "not enough header columns provided. actual:%ld expected:%ld"
#define EVENT_FORMAT_PARSER_ERR_NOT_STRING_INPUT_MSG "input argument must be string"
#define EVENT_FORMAT_PARSER_ERR_EMPTY_STRING "%s must be a non-empty string literal"
#define EVENT_FORMAT_PARSER_ERR_SEPARATOR_MAX_LENGTH_EXCEEDED "%s max length exceeded"

#define EVENT_FORMAT_PARSER_ERROR event_format_parser_error_quark()
GQuark event_format_parser_error_quark(void);

#define EVENT_FORMAT_PARSER_SEPARATOR_MAX_LEN 0x04
#define EVENT_FORMAT_PARSER_PAIR_SEPARATOR_MAX_LEN 0x04

#define EVENT_FORMAT_PARSER_ARG_NAME_PAIR_SEPARATOR "pair_separator"
#define EVENT_FORMAT_PARSER_ARG_NAME_VALUE_SEPARATOR "value_separator"

enum EventFormatParserError
{
Expand All @@ -59,6 +64,8 @@ struct _FilterXFunctionEventFormatParser
FilterXExpr *msg;
CSVScannerOptions csv_opts;
Config config;
const gchar *kv_pair_separator;
const gchar *kv_value_separator;
};

struct _EventParserContext
Expand All @@ -69,7 +76,7 @@ struct _EventParserContext
guint64 column_index;
CSVScanner *csv_scanner;
guint64 flags;
gchar kv_parser_pair_separator[EVENT_FORMAT_PARSER_SEPARATOR_MAX_LEN];
gchar kv_parser_pair_separator[EVENT_FORMAT_PARSER_PAIR_SEPARATOR_MAX_LEN];
gchar kv_parser_value_separator;
};

Expand All @@ -88,7 +95,7 @@ static inline void append_error_message(GError **error, const char *extra_info)
if (error == NULL || *error == NULL)
return;

gchar *new_message = g_strdup_printf("%s: %s", (*error)->message, extra_info);
gchar *new_message = g_strdup_printf("%s %s", (*error)->message, extra_info);
GError *new_error = g_error_new((*error)->domain, (*error)->code, "%s", new_message);

g_error_free(*error);
Expand Down
4 changes: 3 additions & 1 deletion modules/cef/filterx-func-parse-cef.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@
#include "plugin.h"
#include "filterx/expr-function.h"

#define FILTERX_FUNC_PARSE_CEF_USAGE "Usage: parse_cef(str)"
#define FILTERX_FUNC_PARSE_CEF_USAGE "Usage: parse_cef(str [," \
EVENT_FORMAT_PARSER_ARG_NAME_PAIR_SEPARATOR"=boolean, " \
EVENT_FORMAT_PARSER_ARG_NAME_VALUE_SEPARATOR"=boolean])"

FILTERX_GENERATOR_FUNCTION_DECLARE(parse_cef);

Expand Down
44 changes: 29 additions & 15 deletions modules/cef/filterx-func-parse-leef.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,24 +35,25 @@
#include "filterx/func-flags.h"

DEFINE_FUNC_FLAGS(FilterXFunctionParseLeefFlags,
FILTERX_FUNC_PARSE_LEEF_FLAG_20,
FILTERX_FUNC_PARSE_LEEF_FLAG_CUSTOM_DELIMITER,
FILTERX_FUNC_PARSE_LEED_FLAG_FORCED_DELIMITER
FILTERX_FUNC_PARSE_LEEF_FLAG_20
);

static gboolean
_parse_hex_delimiter(const gchar* hexStr, gchar *delimiter)
_parse_hex_delimiter(const gchar *hexStr, gchar *delimiter)
{
errno = 0;
*delimiter = (gchar)strtol(hexStr, NULL, 16);
return (errno == 0);
}

static gboolean
_delimiter_multi_parser(const gchar* input, gint input_len, gchar *delimiter)
_delimiter_multi_parser(const gchar *input, gint input_len, gchar *delimiter, GError **error)
{
const gchar *hexStr = NULL;
switch (input_len) {
switch (input_len)
{
case 0:
return TRUE; // do not change
case 1:
*delimiter = input[0];
return TRUE;
Expand All @@ -66,28 +67,41 @@ _delimiter_multi_parser(const gchar* input, gint input_len, gchar *delimiter)
break;
default:
return FALSE; // no match
}
}
if (!hexStr)
return FALSE;
return _parse_hex_delimiter(hexStr, delimiter);
}

static gboolean
_is_pair_separator_forced(EventParserContext *ctx)
{
return ctx->parser->kv_pair_separator != NULL;
}

static gboolean
_is_delmiter_empty(gchar delimiter)
{
return delimiter == 0;
}

FilterXObject *
parse_delimiter(EventParserContext *ctx, const gchar *input, gint input_len, GError **error, gpointer user_data)
{
if (!check_flag(ctx->flags, FILTERX_FUNC_PARSE_LEEF_FLAG_20))
return NULL;
gchar delimiter = 0;
if (_delimiter_multi_parser(input, input_len, &delimiter))
{
if (!check_flag(ctx->flags, FILTERX_FUNC_PARSE_LEED_FLAG_FORCED_DELIMITER))
if (_delimiter_multi_parser(input, input_len, &delimiter, error))
{
set_flag(&ctx->flags, FILTERX_FUNC_PARSE_LEEF_FLAG_CUSTOM_DELIMITER, TRUE);
ctx->kv_parser_pair_separator[0] = delimiter;
ctx->kv_parser_pair_separator[1] = 0;
if (_is_delmiter_empty(delimiter))
return filterx_string_new("", 0);
if (!_is_pair_separator_forced(ctx))
{
ctx->kv_parser_pair_separator[0] = delimiter;
ctx->kv_parser_pair_separator[1] = 0;
}
return filterx_string_new(&delimiter, 1);
}
return filterx_string_new(&delimiter, 1);
}
return NULL;
}

Expand Down
4 changes: 3 additions & 1 deletion modules/cef/filterx-func-parse-leef.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@
#include "plugin.h"
#include "filterx/expr-function.h"

#define FILTERX_FUNC_PARSE_LEEF_USAGE "Usage: parse_leef(str)"
#define FILTERX_FUNC_PARSE_LEEF_USAGE "Usage: parse_leef(str [," \
EVENT_FORMAT_PARSER_ARG_NAME_PAIR_SEPARATOR"=boolean, " \
EVENT_FORMAT_PARSER_ARG_NAME_VALUE_SEPARATOR"=boolean])"

FILTERX_GENERATOR_FUNCTION_DECLARE(parse_leef);

Expand Down
9 changes: 7 additions & 2 deletions modules/cef/tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
set(TEST_HELPER_SOURCES
test_helpers.c
test_helpers.h
)

add_unit_test(LIBTEST CRITERION TARGET test-format-cef-extension DEPENDS cef)
add_unit_test(LIBTEST CRITERION TARGET test-filterx-function-parse-cef DEPENDS cef)
add_unit_test(LIBTEST CRITERION TARGET test-filterx-function-parse-leef DEPENDS cef)
add_unit_test(LIBTEST CRITERION TARGET test-filterx-function-parse-cef DEPENDS cef SOURCES test-filterx-function-parse-cef.c ${TEST_HELPER_SOURCES})
add_unit_test(LIBTEST CRITERION TARGET test-filterx-function-parse-leef DEPENDS cef SOURCES test-filterx-function-parse-leef.c ${TEST_HELPER_SOURCES})
12 changes: 11 additions & 1 deletion modules/cef/tests/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,25 @@ modules_cef_tests_test_format_cef_extension_LDFLAGS = \
EXTRA_modules_cef_tests_test_format_cef_extension_DEPENDENCIES = \
$(top_builddir)/modules/cef/libcef.la

modules_cef_tests_test_filterx_function_parse_cef_SOURCES = \
modules/cef/tests/test-filterx-function-parse-cef.c \
modules/cef/tests/test_helpers.c \
modules/cef/tests/test_helpers.h

modules_cef_tests_test_filterx_function_parse_cef_CFLAGS = $(TEST_CFLAGS) -I$(top_srcdir)/modules/cef
modules_cef_tests_test_filterx_function_parse_cef_LDADD = $(TEST_LDADD)
modules_cef_tests_test_filterx_function_parse_cef_LDADD = $(TEST_LDADD)
modules_cef_tests_test_filterx_function_parse_cef_LDFLAGS = \
$(PREOPEN_SYSLOGFORMAT) \
-dlpreopen $(top_builddir)/modules/cef/libcef.la

EXTRA_modules_cef_tests_test_filterx_function_parse_cef_DEPENDENCIES = \
$(top_builddir)/modules/cef/libcef.la

modules_cef_tests_test_filterx_function_parse_leef_SOURCES = \
modules/cef/tests/test-filterx-function-parse-leef.c \
modules/cef/tests/test_helpers.c \
modules/cef/tests/test_helpers.h

modules_cef_tests_test_filterx_function_parse_leef_CFLAGS = $(TEST_CFLAGS) -I$(top_srcdir)/modules/cef
modules_cef_tests_test_filterx_function_parse_leef_LDADD = $(TEST_LDADD)
modules_cef_tests_test_filterx_function_parse_leef_LDFLAGS = \
Expand Down
Loading

0 comments on commit 1885749

Please sign in to comment.