Feature/additional elasticsearch options

Makefile

@@ -148,7 +148,7 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: go-deps ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.11.3)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.15.0)
 
 GINKGO = $(shell pwd)/bin/ginkgo
 ginkgo: go-deps ## Download controller-gen locally if necessary.

apis/fluentd/v1alpha1/plugins/output/es.go

@@ -35,6 +35,33 @@ type ElasticsearchCommon struct {
 	ClientKey *string `json:"clientKey,omitempty"`
 	// Optional, password for ClientKey file
 	ClientKeyPassword *plugins.Secret `json:"clientKeyPassword,omitempty"`
+	// Optional, Always update the template, even if it already exists (default: false)
+	TemplateOverwrite *bool `json:"templateOverwrite,omitempty"`
+	// Optional, You can specify times of retry putting template (default: 10)
+	MaxRetryPuttingTemplate *uint32 `json:"maxRetryPuttingTemplate,omitempty"`
+	// Optional, Indicates whether to fail when max_retry_putting_template is exceeded. If you have multiple output plugin, you could use this property to do not fail on fluentd statup (default: false)
+	FailOnPuttingTemplateRetryExceeded *bool `json:"failOnPuttingTemplateRetryExceeded,omitempty"`
+	// Optional, Indicates that the plugin should reset connection on any error (reconnect on next send) (default: false)
+	ReconnectOnError *bool `json:"reconnectOnError,omitempty"`
+	// Optional, Automatically reload connection after 10000 documents (default: true)
+	ReloadConnections *bool `json:"reloadConnections,omitempty"`
+	// Optional, Indicates that the elasticsearch-transport will try to reload the nodes addresses if there is a failure while making the request, this can be useful to quickly remove a dead node from the list of addresses (default: false)
+	ReloadOnFailure *bool `json:"reloadOnFailure,omitempty"`
+	// Optional, HTTP Timeout (default: 5)
+	// +kubebuilder:validation:Pattern:="^\\d+(s|m|h|d)$"
+	RequestTimeout *string `json:"requestTimeout,omitempty"`
+	// Optional, Suppress '[types removal]' warnings on elasticsearch 7.x
+	SuppressTypeName *bool `json:"suppressTypeName,omitempty"`
+	// Optional, Enable Index Lifecycle Management (ILM)
+	EnableIlm *bool `json:"enableIlm,omitempty"`
+	// Optional, Specify ILM policy id
+	IlmPolicyId *string `json:"ilmPolicyId,omitempty"`
+	// Optional, Specify ILM policy contents as Hash
+	IlmPolicy *string `json:"ilmPolicy,omitempty"`
+	// Optional, Specify whether overwriting ilm policy or not
+	IlmPolicyOverwrite *bool `json:"ilmPolicyOverride,omitempty"`
+	// Optional, Enable logging of 400 reason without enabling debug log level
+	LogEs400Reason *bool `json:"logEs400Reason,omitempty"`
 }
 
 type Elasticsearch struct {
@@ -53,4 +80,14 @@ type ElasticsearchDataStream struct {
 
 	// You can specify Elasticsearch data stream name by this parameter. This parameter is mandatory for elasticsearch_data_stream
 	DataStreamName *string `json:"dataStreamName"`
+	// Optional, You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template
+	DataStreamTemplateName *string `json:"dataStreamTemplateName,omitempty"`
+	// Optional, Specify whether index patterns should include a wildcard (*) when creating an index template. This is particularly useful to prevent errors in scenarios where index templates are generated automatically, and multiple services with distinct suffixes are in use
+	DataStreamTemplateUseIndexPatternsWildcard *bool `json:"dataStreamTemplateUseIndexPatternsWildcard,omitempty"`
+	// Optional, You can specify the name of an existing ILM policy, which will be applied to the data stream. If not present, it creates a new ILM default policy (unless data_stream_template_name is defined, in that case the ILM will be set to the one specified in the matching index template)
+	DataStreamIlmName *string `json:"dataStreamIlmName,omitempty"`
+	// Optional, You can specify the ILM policy contents as hash. If not present, it will apply the ILM default policy
+	DataStreamIlmPolicy *string `json:"dataStreamIlmPolicy,omitempty"`
+	// Optional, Specify whether the data stream ILM policy should be overwritten
+	DataStreamIlmPolicyOverwrite *bool `json:"dataStreamIlmPolicyOverwrite,omitempty"`
 }

apis/fluentd/v1alpha1/plugins/output/types.go

@@ -451,6 +451,58 @@ func (o *Output) elasticsearchPluginCommon(common *ElasticsearchCommon, parent *
 		parent.InsertPairs("path", fmt.Sprint(*common.Path))
 	}
 
+	if common.TemplateOverwrite != nil {
+		parent.InsertPairs("template_overwrite", fmt.Sprint(*common.TemplateOverwrite))
+	}
+
+	if common.MaxRetryPuttingTemplate != nil {
+		parent.InsertPairs("max_retry_putting_template", fmt.Sprint(*common.MaxRetryPuttingTemplate))
+	}
+
+	if common.FailOnPuttingTemplateRetryExceeded != nil {
+		parent.InsertPairs("fail_on_putting_template_retry_exceed", fmt.Sprint(*common.FailOnPuttingTemplateRetryExceeded))
+	}
+
+	if common.ReconnectOnError != nil {
+		parent.InsertPairs("reconnect_on_error", fmt.Sprint(*common.ReconnectOnError))
+	}
+
+	if common.ReloadConnections != nil {
+		parent.InsertPairs("reload_connections", fmt.Sprint(*common.ReloadConnections))
+	}
+
+	if common.ReloadOnFailure != nil {
+		parent.InsertPairs("reload_on_failure", fmt.Sprint(*common.ReloadOnFailure))
+	}
+
+	if common.RequestTimeout != nil {
+		parent.InsertPairs("request_timeout", fmt.Sprint(*common.RequestTimeout))
+	}
+
+	if common.SuppressTypeName != nil {
+		parent.InsertPairs("suppress_type_name", fmt.Sprint(*common.SuppressTypeName))
+	}
+
+	if common.EnableIlm != nil {
+		parent.InsertPairs("enable_ilm", fmt.Sprint(*common.EnableIlm))
+	}
+
+	if common.IlmPolicyId != nil {
+		parent.InsertPairs("ilm_policy_id", fmt.Sprint(*common.IlmPolicyId))
+	}
+
+	if common.IlmPolicy != nil {
+		parent.InsertPairs("ilm_policy", fmt.Sprint(*common.IlmPolicy))
+	}
+
+	if common.IlmPolicyOverwrite != nil {
+		parent.InsertPairs("ilm_policy_overwrite", fmt.Sprint(*common.IlmPolicyOverwrite))
+	}
+
+	if common.LogEs400Reason != nil {
+		parent.InsertPairs("log_es_400_reason", fmt.Sprint(*common.LogEs400Reason))
+	}
+
 	return parent, nil
 }
 
@@ -487,6 +539,26 @@ func (o *Output) elasticsearchDataStreamPlugin(parent *params.PluginStore, loade
 		parent.InsertPairs("data_stream_name", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamName))
 	}
 
+	if o.ElasticsearchDataStream.DataStreamTemplateName != nil {
+		parent.InsertPairs("data_stream_template_name", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamTemplateName))
+	}
+
+	if o.ElasticsearchDataStream.DataStreamTemplateUseIndexPatternsWildcard != nil {
+		parent.InsertPairs("data_stream_template_use_index_patterns_wildcard", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamTemplateUseIndexPatternsWildcard))
+	}
+
+	if o.ElasticsearchDataStream.DataStreamIlmName != nil {
+		parent.InsertPairs("data_stream_ilm_name", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamIlmName))
+	}
+
+	if o.ElasticsearchDataStream.DataStreamIlmPolicy != nil {
+		parent.InsertPairs("data_stream_ilm_policy", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamIlmPolicy))
+	}
+
+	if o.ElasticsearchDataStream.DataStreamIlmPolicyOverwrite != nil {
+		parent.InsertPairs("data_stream_ilm_policy_overwrite", fmt.Sprint(*o.ElasticsearchDataStream.DataStreamIlmPolicyOverwrite))
+	}
+
 	return parent, nil
 }