Merge pull request #129 from sasagarw/clean-templates

LOG-1379: Cleanup ES index templates

namespaces/kubernetes.yml

@@ -194,6 +194,19 @@ namespace:
         example: Normal 
         description: >
           Type of this event (Normal, Warning), new types could be added in the future
+  
+  - name: flat_labels
+    type: keyword
+    example: [
+      "app=openshift-kube-scheduler",
+      "revision=8",
+      "scheduler=true"
+    ]
+    norms: true
+    description: >
+      Flattened Labels attached to the OpenShift object.
+      Each label name is a subfield of labels field.
+      It also contains custom labels added in CRDs. It will be an array.
 
 doc_sections:
   - ["pod_log", "Openshift logging metadata"]

templates/Makefile

@@ -2,27 +2,23 @@ TEMPLATE_NAME = com.redhat.viaq-openshift
 
 .PHONY: all templates index_pattern
 
-INDEX_PATTERN_DIRS = openshift,collectd_metrics
+INDEX_PATTERN_DIR = openshift
 
 all: templates index_pattern
 
 # Clean all index templates and index patterns generated during 'make'
 clean:
-	for dir in {$(INDEX_PATTERN_DIRS)}; do \
-		cd $$dir ; \
-		make clean; \
-		cd .. ; \
-	done
+	cd $(INDEX_PATTERN_DIR) ; \
+	make clean; \
+	cd .. ; \
 	rm *.index-pattern.json
 
 templates:
-	for dir in {$(INDEX_PATTERN_DIRS)}; do \
-		cd $$dir ; \
-		make all; \
-		cd .. ; \
-	done
+	cd $(INDEX_PATTERN_DIR) ; \
+	make all; \
+	cd .. ; \
 
 index_pattern:
 	python3 ../scripts/concat_index_pattern_fields.py \
-		"$(INDEX_PATTERN_DIRS)" "*.index-pattern.json" \
+		"$(INDEX_PATTERN_DIR)" "*.index-pattern.json" \
 		"$(TEMPLATE_NAME).<ES_VERSION>.index-pattern.json"

templates/openshift/README.md

@@ -4,10 +4,10 @@ viaq openshift index templates for ElasticSearch and index patterns for Kibana
 The template files are automatically generated.
 Please _do not edit_ the files directly.
 
-In order to edit the template please modify [objects.yml](objects.yml) and the respective object type files.
+In order to edit the template please modify [template-operations.yml](template-operations.yml), [template-project.yml](template-project.yml) and the respective namespace files referenced there.
 
 To rebuild the template, run:
-> python ../scripts/generate_template.py . ../../objects_dir
+> make
 
 For details about the mapping please see [ElasticSearch reference](https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html) and [Kibana reference](https://www.elastic.co/guide/en/kibana/current/index-patterns.html)
 
@@ -18,10 +18,10 @@ This file contains the skeleton of the template without the mappings section.
 by sections:  
 `aliases`: Aliases for the indices produces from the template. Aliases will be automatically generated along with the indices in future.  
 `mappings`: mappings section  
-  `_default_`: default mapping skeleton. It is the only mapping present in the skeleton. Other mappings from [fields.yml](fields.yml) will copy this skeleton.  
+  `_default_`: default mapping skeleton. It is the only mapping present in the skeleton. Other mappings from [template-operations.yml](template-operations.yml), [template-project.yml](template-project.yml) will copy this skeleton.  
   `date_detection`: we force no date detection in the unknown fields.  
   `dynamic_templates`: describes the mapping for autocreated fields.  
-  `properties`: empty section that is populated with the content from [fields.yml](fields.yml)  
+  `properties`: empty section that is populated with the content from [template-operations.yml](template-operations.yml), [template-project.yml](template-project.yml)  
 `order`: order of the template. lower order templates are applied first.  
 `settings`: various settings  
 `index_patterns`: indices that will be matched by this template

templates/openshift/template-operations.yml

@@ -8,12 +8,12 @@ elasticsearch_template:
   order: 10
 
 namespaces:
-  - rsyslog.yml
   - systemd.yml
   - kubernetes.yml
   - docker.yml
   - pipeline_metadata.yml
   - ovirt.yml
   - aushape.yml
   - tlog.yml
-#  - openshift/app_log.yml
+  # - rsyslog.yml
+  # - openshift/app_log.yml

templates/openshift/template-project.yml

@@ -8,12 +8,12 @@ elasticsearch_template:
   order: 10
 
 namespaces:
-  - rsyslog.yml
   - systemd.yml
   - kubernetes.yml
   - docker.yml
   - pipeline_metadata.yml
   - ovirt.yml
   - aushape.yml
   - tlog.yml
-#  - openshift/app_log.yml
+  # - rsyslog.yml
+  # - openshift/app_log.yml

templates/skeleton.json

@@ -69,8 +69,5 @@
         }
     },
     "order": 10,
-    "settings": {
-        "index.refresh_interval": "5s"
-    },
     "index_patterns": "logstash-*"
 }

templates/test/template-test.yml

@@ -11,14 +11,14 @@ elasticsearch_template:
 # TODO: use all available namespaces for tests
 namespaces:
   - aushape.yml
-#  - ci_job.yml
-  - collectd.yml
   - docker.yml
   - kubernetes.yml
   - ovirt.yml
   - pipeline_metadata.yml
-  - rsyslog.yml
   - systemd.yml
-#  - testcase.yml
   - tlog.yml
-#  - openshift/app_log.yml
+  # - ci_job.yml
+  # - collectd.yml
+  # - rsyslog.yml
+  # - testcase.yml
+  # - openshift/app_log.yml