add configuration for tests

quickstart/docker-compose.yml

@@ -92,7 +92,24 @@ services:
       - ../quickstart/:/home/logprep/quickstart/
     entrypoint:
       - logprep
+      - run
       - /home/logprep/quickstart/exampledata/config/pipeline.yml
+  http-input:
+    build:
+      context: ..
+    image: logprep
+    container_name: http-input
+    profiles:
+      - http-input
+    expose:
+      - 8002
+    network_mode: host
+    volumes:
+      - ../quickstart/:/home/logprep/quickstart/
+    entrypoint:
+      - logprep
+      - run
+      - /home/logprep/quickstart/exampledata/config/http_pipeline.yml
   grafana:
     image: bitnami/grafana:latest
     container_name: grafana

quickstart/exampledata/artifacts/regex_mapping.yml

@@ -0,0 +1,4 @@
+RE_WHOLE_FIELD: (.*)
+RE_DOMAIN_BACKSLASH_USERNAME: \w+\\(.*)
+RE_ALL_NO_CAP: .*
+RE_IP4_COLON_PORT: ([\d.]+):\d+

quickstart/exampledata/artifacts/tree_config.json

@@ -0,0 +1,9 @@
+{
+  "priority_dict": {
+    "tags": "01",
+    "message": "02"
+  },
+  "tag_map": {
+    "field_name_to_check_for_in_rule": "TAG-TO-CHECK-IF-IN-EVENT"
+  }
+}

quickstart/exampledata/config/http_pipeline.yml

@@ -2,7 +2,7 @@ version: 1
 
 metrics:
   enabled: true
-  port: 8000
+  port: 8002
 
 input:
   httpinput:

quickstart/exampledata/config/pipeline.yml

@@ -1,3 +1,4 @@
+---
 version: 1
 process_count: 2
 timeout: 0.1
@@ -9,79 +10,196 @@ metrics:
   port: 8001
 
 pipeline:
-  - labelername:
-      type: labeler
-      schema: quickstart/exampledata/rules/labeler/schema.json
-      include_parent_labels: true
-      specific_rules:
-        - quickstart/exampledata/rules/labeler/specific
-      generic_rules:
-        - quickstart/exampledata/rules/labeler/generic
-
-  - normalizer:
-      type: normalizer
-      specific_rules:
-        - quickstart/exampledata/rules/normalizer/specific/
-      generic_rules:
-        - quickstart/exampledata/rules/normalizer/generic/
-      regex_mapping: quickstart/exampledata/rules/normalizer/normalizer_regex_mapping.yml
-
-  - dropper:
-      type: dropper
-      specific_rules:
-        - quickstart/exampledata/rules/dropper/specific
-      generic_rules:
-        - quickstart/exampledata/rules/dropper/generic
-        - filter: "test_dropper"
-          dropper:
-            drop:
-              - drop_me
-          description: "..."
-
-  - pre_detector:
-      type: pre_detector
-      specific_rules:
-        - quickstart/exampledata/rules/pre_detector/specific
-      generic_rules:
-        - quickstart/exampledata/rules/pre_detector/generic
-      outputs:
-        - opensearch: sre
-      tree_config: quickstart/exampledata/rules/pre_detector/tree_config.json
-      alert_ip_list_path: quickstart/exampledata/rules/pre_detector/alert_ips.yml
-
-  - amides:
-      type: amides
-      specific_rules:
-        - quickstart/exampledata/rules/amides/specific
-      generic_rules:
-        - quickstart/exampledata/rules/amides/generic
-      models_path: quickstart/exampledata/models/model.zip
-      num_rule_attributions: 10
-      max_cache_entries: 1000000
-      decision_threshold: 0.32
-
-  - pseudonymizer:
-      type: pseudonymizer
-      pubkey_analyst: quickstart/exampledata/rules/pseudonymizer/example_analyst_pub.pem
-      pubkey_depseudo: quickstart/exampledata/rules/pseudonymizer/example_depseudo_pub.pem
-      regex_mapping: quickstart/exampledata/rules/pseudonymizer/regex_mapping.yml
-      hash_salt: a_secret_tasty_ingredient
-      outputs:
-        - opensearch: pseudonyms
-      specific_rules:
-        - quickstart/exampledata/rules/pseudonymizer/specific/
-      generic_rules:
-        - quickstart/exampledata/rules/pseudonymizer/generic/
-      max_cached_pseudonyms: 1000000
-
-  - calculator:
-      type: calculator
-      specific_rules:
-        - filter: "test_label: execute"
-          calculator:
-            target_field: "calculation"
-            calc: "1 + 1"
-      generic_rules: []
+- dissector:
+    type: dissector
+    specific_rules: []
+    generic_rules:
+    - quickstart/exampledata/rules/030_dissector/rules_generic/
+
+- grokker:
+    type: grokker
+    specific_rules:
+    - quickstart/exampledata/rules/035_grokker/rules_specific/
+    generic_rules: []
+
+- field_manager_a:
+    type: field_manager
+    generic_rules:
+    - quickstart/exampledata/rules/041_field_manager/generic_rules
+    specific_rules:
+    - quickstart/exampledata/rules/041_field_manager/specific_rules
+
+- string_splitter:
+    type: string_splitter
+    specific_rules:
+    - quickstart/exampledata/rules/042_string_splitter/specific_rules/
+    generic_rules:
+    - quickstart/exampledata/rules/042_string_splitter/generic_rules/
+
+- timestamper:
+    type: timestamper
+    specific_rules:
+      - quickstart/exampledata/rules/043_timestamper/rules_specific/
+    generic_rules: []
+
+- calculator:
+    type: calculator
+    specific_rules:
+    - quickstart/exampledata/rules/045_calculator/rules_specific/
+    generic_rules:
+    - quickstart/exampledata/rules/045_calculator/rules_generic/
+
+- timestamp_differ:
+    type: timestamp_differ
+    specific_rules:
+    - quickstart/exampledata/rules/050_timestamp_differ/specific_rules/
+    generic_rules:
+    - quickstart/exampledata/rules/050_timestamp_differ/generic_rules/
+
+- labelername:
+    type: labeler
+    schema: quickstart/exampledata/rules/060_labeler/schema/schema.json
+    include_parent_labels: true
+    generic_rules:
+    - quickstart/exampledata/rules/060_labeler/generic_rules/
+    specific_rules:
+    - quickstart/exampledata/rules/060_labeler/specific_rules/
+
+- domain_resolver:
+    type: domain_resolver
+    specific_rules:
+    - quickstart/exampledata/rules/070_domain_resolver/specific_rules/
+    generic_rules:
+    - quickstart/exampledata/rules/070_domain_resolver/generic_rules/
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+    tld_lists: ["quickstart/exampledata/lists/public_suffix_list.dat"]
+    timeout: 10.0
+    hash_salt: "thisisasecureandrandomkey"
+    max_caching_days: 1
+    max_cached_domains: 20000
+
+- domain_label_extractor:
+    type: domain_label_extractor
+    tld_lists: ["quickstart/exampledata/lists/public_suffix_list.dat"]
+    specific_rules:
+    - quickstart/exampledata/rules/080_domain_label_extractor/specific_rules/
+    generic_rules:
+    - quickstart/exampledata/rules/080_domain_label_extractor/generic_rules/
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+
+- datetime_extractor:
+    type: datetime_extractor
+    generic_rules:
+    - quickstart/exampledata/rules/100_datetime_extractor/generic_rules/
+    specific_rules:
+    - quickstart/exampledata/rules/100_datetime_extractor/specific_rules/
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+
+- generic_adder:
+    type: generic_adder
+    generic_rules:
+    - quickstart/exampledata/rules/110_generic_adder/generic_rules
+    specific_rules:
+    - quickstart/exampledata/rules/110_generic_adder/specific_rules
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+
+- build_indexname:
+    type: concatenator
+    specific_rules:
+    - quickstart/exampledata/rules/115_concatenator/specific_rules/
+    generic_rules:
+    - quickstart/exampledata/rules/115_concatenator/generic_rules/
+
+- generic_resolver:
+    type: generic_resolver
+    generic_rules:
+    - quickstart/exampledata/rules/120_generic_resolver/generic_rules/
+    specific_rules:
+    - quickstart/exampledata/rules/120_generic_resolver/specific_rules/
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+
+- template_replacer:
+    type: template_replacer
+    generic_rules:
+    - quickstart/exampledata/rules/130_template_replacer/generic_rules/
+    specific_rules:
+    - quickstart/exampledata/rules/130_template_replacer/specific_rules/
+    template: quickstart/exampledata/rules/130_template_replacer/templates.yml
+    pattern:
+      delimiter: "-"
+      fields:
+      - winlog.channel
+      - winlog.provider_name
+      - winlog.event_id
+      allowed_delimiter_field: winlog.provider_name
+      target_field: message
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+
+- list_comparison:
+    type: list_comparison
+    generic_rules:
+    - quickstart/exampledata/rules/140_list_comparison/generic_rules/
+    specific_rules:
+    - quickstart/exampledata/rules/140_list_comparison/specific_rules/
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+    list_search_base_path: ./quickstart/exampledata/lists
+
+- amides:
+    type: amides
+    generic_rules:
+    - quickstart/exampledata/rules/145_amides/generic_rules/
+    specific_rules:
+    - quickstart/exampledata/rules/145_amides/specific_rules/
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+    models_path: quickstart/exampledata/models/model.zip
+    num_rule_attributions: 10
+    max_cache_entries: 1000000
+    decision_threshold: 0.32
+
+- pre_detector:
+    type: pre_detector
+    generic_rules:
+    - quickstart/exampledata/rules/150_pre_detector/generic_rules/
+    specific_rules:
+    - quickstart/exampledata/rules/150_pre_detector/specific_rules/
+    outputs:
+    - opensearch: sre
+    tree_config: quickstart/exampledata/artifacts/tree_config.json
+
+- pseudonymizer:
+    type: pseudonymizer
+    pubkey_analyst: quickstart/exampledata/rules/160_pseudonymizer/example_analyst_pub.pem
+    pubkey_depseudo: quickstart/exampledata/rules/160_pseudonymizer/example_depseudo_pub.pem
+    regex_mapping: quickstart/exampledata/artifacts/regex_mapping.yml
+    hash_salt: "thisisasecureandrandomkey"
+    outputs:
+    - opensearch: pseudonyms
+    specific_rules:
+    - quickstart/exampledata/rules/160_pseudonymizer/specific_rules/
+    generic_rules:
+    - quickstart/exampledata/rules/160_pseudonymizer/generic_rules/
+    max_cached_pseudonyms: 1000000
+
+- field_manager_b:
+    type: field_manager
+    generic_rules:
+    - quickstart/exampledata/rules/165_field_manager/generic_rules
+    specific_rules:
+    - quickstart/exampledata/rules/165_field_manager/specific_rules
+
+- selective_extractor:
+    type: selective_extractor
+    specific_rules:
+    - quickstart/exampledata/rules/170_selective_extractor/specific_rules/
+    generic_rules:
+    - quickstart/exampledata/rules/170_selective_extractor/generic_rules/
+
+- dropper:
+    type: dropper
+    specific_rules:
+    - quickstart/exampledata/rules/180_dropper/specific_rules/
+    generic_rules:
+    - quickstart/exampledata/rules/180_dropper/generic_rules/
 
 input:
   kafka:
@@ -90,12 +208,8 @@ input:
     kafka_config:
       bootstrap.servers: 127.0.0.1:9092
       group.id: cgroup3
-      enable.auto.commit: "true"
-      auto.commit.interval.ms: "10000"
+      enable.auto.commit: "false"
       enable.auto.offset.store: "false"
-      queued.min.messages: "100000"
-      queued.max.messages.kbytes: "65536"
-      statistics.interval.ms: "60000"
     preprocessing:
       version_info_target_field: Logprep_version_info
       log_arrival_time_target_field: event.ingested
@@ -113,9 +227,11 @@ output:
     error_index: errors
     message_backlog_size: 10000
     timeout: 10000
-    flush_timeout: 60
+    flush_timeout: 600
     max_retries: 3
-    parallel_bulk: false
+    thread_count: 8
+    queue_size: 8
+    chunk_size: 500
     user: admin
     secret: admin
   kafka:
@@ -126,4 +242,3 @@ output:
     flush_timeout: 300
     kafka_config:
       bootstrap.servers: 127.0.0.1:9092
-      statistics.interval.ms: "60000"

quickstart/exampledata/config/prometheus/prometheus.yml

@@ -29,7 +29,7 @@ scrape_configs:
       - targets: ["localhost:9090"]
   - job_name: "logprep"
     static_configs:
-      - targets: ["localhost:8000", "localhost:8001"]
+      - targets: ["localhost:8001", "localhost:8002"]
   - job_name: "kafka"
     metrics_path: "/metrics"
     static_configs:

quickstart/exampledata/lists/demo_rdp_ips.txt

@@ -0,0 +1,2 @@
+45.231.32.15
+45.231.32.11