diff --git a/dss_metrics/README.md b/dss_metrics/README.md index 5bc52d8..3ba4c66 100644 --- a/dss_metrics/README.md +++ b/dss_metrics/README.md @@ -4,8 +4,11 @@ ## Overview The DSS Metrics Agent runs on a per node level. It can collect data from MINIO, target, and other sources. The data is then processed into metrics that can be stored into a Promotheus database or exposed at an endpoint. The metrics are associated with tags such as -cluster_id, susbsystem_id, target_id, etc.. in order to be able to correlate the metric with a given point/layer in the system. It -is assumed that all captured metrics will be time series data. +cluster_id, susbsystem_id, target_id, etc.. in order to be able to correlate the metric with a given point/layer in the system. It is assumed that all captured metrics will be time series data. + +## Dependencies +- install required python packages listed in requirements.txt +- set environment variable MINIO_REPORT_METRICS=1, will also need to add to MINIO startup script before running MINIO ## Execution Until the agent is deployed along with the deploy DSS playbook, it will need to be manually run. Currently, the agent supports diff --git a/dss_metrics/metrics_scope.json b/dss_metrics/metrics_scope.json index 0485469..bcd981d 100644 --- a/dss_metrics/metrics_scope.json +++ b/dss_metrics/metrics_scope.json @@ -2,5 +2,7 @@ "target.subsystem\\d+.kvio.(puts|gets|dels|putBandwidth|getBandwidth)": "target", "minio_upstream.outstanding.s3_get_req": "minio cluster", "minio_disk_storage_used_bytes": "minio cluster", - "minio_disk_storage_total_capacity_bytes": "minio cluster" + "minio_disk_storage_total_capacity_bytes": "minio cluster", + "minio_metrics_(get|put)_bw": "minio endpoint", + "minio_metrics_(del|get|put|)_iops": "minio endpoint" } \ No newline at end of file diff --git a/dss_metrics/minio_rest_collector.py b/dss_metrics/minio_rest_collector.py index 29ad244..9f3cd06 100644 --- a/dss_metrics/minio_rest_collector.py +++ b/dss_metrics/minio_rest_collector.py @@ -50,8 +50,12 @@ def __init__(self, configs, metrics_scopes, whitelist_patterns, self.metrics_scopes = metrics_scopes self.whitelist_patterns = whitelist_patterns self.filter = filter - self.minio_metrics = {'minio_disk_storage_used_bytes', - 'minio_disk_storage_total_capacity_bytes'} + + self.blacklist = { + 'minio_http_requests_duration_seconds_bucket', + 'minio_http_requests_duration_seconds_sum', + 'minio_http_requests_duration_seconds_count'} + self.url_prefix = "http://" self.cluster_id_url_suffix = "/minio/cluster_id" self.metrics_url_suffix = "/minio/prometheus/metrics" @@ -138,7 +142,8 @@ def get_minio_metrics_from_endpoint(self, endpoint): r = requests.get(url) metrics_data = [] for line in r.text.splitlines(): - if (any(line.startswith(metric) for metric in self.minio_metrics)): + if line.startswith("minio") and not any(line.startswith(x) + for x in self.blacklist): key, val = line.split(" ") metrics_data.append((key, float(val))) return metrics_data diff --git a/dss_metrics/whitelist.txt b/dss_metrics/whitelist.txt index d08872e..356bda2 100644 --- a/dss_metrics/whitelist.txt +++ b/dss_metrics/whitelist.txt @@ -1,4 +1,6 @@ target.subsystem\d+.kvio.(puts|gets|dels|putBandwidth|getBandwidth) minio_upstream.outstanding.s3_get_req minio_disk_storage_used_bytes -minio_disk_storage_total_capacity_bytes \ No newline at end of file +minio_disk_storage_total_capacity_bytes +minio_metrics_(del|get|put|)_iops +minio_metrics_(get|put)_bw \ No newline at end of file