alerta_prometheus plugin SSL - Self-signed certificate error

[arturkiwa]

Issue Summary
Alertmanager and Alerta are running on different K8s. SSL implemented using self-signed certs. CA certificates are added in the Alerta dockerfile, (copy and RUN openssl rehash /etc/ssl/certs/). From alerta pod i can connect to alertmanager web interface without any SSL errors. Attempting to silence an alert end with error.

Environment

• OS: Debian

• API version: 9.0.3

• Deployment: K8s 1.28.6, self-hosted

• For self-hosted, WSGI environment: [eg. nginx/uwsgi, apache/mod_wsgi]

• Database: Postgres

• web UI version: 8.7.1

alerta.app[3250]: [ERROR] Error while running action plugin 'alerta_prometheus': Alertmanager: ERROR - HTTPSConnectionPool(host='alertmanager-hostname.domain.tld', port=443): Max retries exceeded with url: /api/v1/silences (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1129)'))) request_id=eca88d64-1880-4829-b6b3-7bc7924b7a94 ip=10.233.75.84
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/urllib3/connectionpool.py", line 467, in _make_request
self._validate_conn(conn)
File "/venv/lib/python3.9/site-packages/urllib3/connectionpool.py", line 1099, in _validate_conn
conn.connect()
File "/venv/lib/python3.9/site-packages/urllib3/connection.py", line 653, in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
File "/venv/lib/python3.9/site-packages/urllib3/connection.py", line 806, in ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
File "/venv/lib/python3.9/site-packages/urllib3/util/ssl.py", line 465, in ssl_wrap_socket
ssl_sock = ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
File "/venv/lib/python3.9/site-packages/urllib3/util/ssl.py", line 509, in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
File "/usr/local/lib/python3.9/ssl.py", line 501, in wrap_socket
return self.sslsocket_class._create(
File "/usr/local/lib/python3.9/ssl.py", line 1041, in _create
self.do_handshake()
File "/usr/local/lib/python3.9/ssl.py", line 1310, in do_handshake
self._sslobj.do_handshake()
ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1129)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/urllib3/connectionpool.py", line 793, in urlopen
response = self._make_request(
File "/venv/lib/python3.9/site-packages/urllib3/connectionpool.py", line 491, in _make_request
raise new_e
urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1129)
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/venv/lib/python3.9/site-packages/urllib3/connectionpool.py", line 847, in urlopen
retries = retries.increment(
File "/venv/lib/python3.9/site-packages/urllib3/util/retry.py", line 515, in increment
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='alertmanager-hostname.domain.tld', port=443): Max retries exceeded with url: /api/v1/silences (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1129)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/alerta_prometheus.py", line 155, in take_action
r = requests.post(url, json=data, auth=self.auth, timeout=2)
File "/venv/lib/python3.9/site-packages/requests/api.py", line 115, in post
return request("post", url, data=data, json=json, **kwargs)
File "/venv/lib/python3.9/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/venv/lib/python3.9/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/venv/lib/python3.9/site-packages/requests/sessions.py", line 725, in send
history = [resp for resp in gen]
File "/venv/lib/python3.9/site-packages/requests/sessions.py", line 725, in
history = [resp for resp in gen]
File "/venv/lib/python3.9/site-packages/requests/sessions.py", line 266, in resolve_redirects
resp = self.send(
File "/venv/lib/python3.9/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/venv/lib/python3.9/site-packages/requests/adapters.py", line 517, in send
raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='alertmanager-hostname.domain.tld', port=443): Max retries exceeded with url: /api/v1/silences (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1129)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/alerta/utils/api.py", line 105, in process_action
updated = plugin.take_action(alert, action, text, timeout=timeout, config=wanted_config)
File "/venv/lib/python3.9/site-packages/alerta_prometheus.py", line 157, in take_action
raise RuntimeError('Alertmanager: ERROR - %s' % e)
RuntimeError: Alertmanager: ERROR - HTTPSConnectionPool(host='alertmanager-hostname.domain.tld', port=443): Max retries exceeded with url: /api/v1/silences (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1129)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/alerta/views/alerts.py", line 159, in action_alert
alert, action, text, timeout = process_action(alert, action, text, timeout)
File "/venv/lib/python3.9/site-packages/alerta/utils/api.py", line 112, in process_action
raise ApiError(f"Error while running action plugin '{plugin.name}': {str(e)}")
alerta.exceptions.ApiError: Error while running action plugin 'alerta_prometheus': Alertmanager: ERROR - HTTPSConnectionPool(host='alertmanager-hostname.domain.tld', port=443): Max retries exceeded with url: /api/v1/silences (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1129)')))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/venv/lib/python3.9/site-packages/flask/app.py", line 870, in full_dispatch_request
rv = self.dispatch_request()
File "/venv/lib/python3.9/site-packages/flask/app.py", line 855, in dispatch_request
return self.ensure_sync(self.view_functions[rule.endpoint])(**view_args) # type: ignore[no-any-return]
File "/venv/lib/python3.9/site-packages/flask_cors/decorator.py", line 130, in wrapped_function
resp = make_response(f(*args, **kwargs))
File "/venv/lib/python3.9/site-packages/alerta/auth/decorators.py", line 91, in wrapped
return f(*args, **kwargs)
File "/venv/lib/python3.9/site-packages/alerta/models/metrics.py", line 258, in wrapped
response = f(*args, **kwargs)
File "/venv/lib/python3.9/site-packages/alerta/utils/response.py", line 18, in decorated
return func(*args, **kwargs)
File "/venv/lib/python3.9/site-packages/alerta/views/alerts.py", line 173, in action_alert
raise ApiError(str(e), 500)
alerta.exceptions.ApiError: Error while running action plugin 'alerta_prometheus': Alertmanager: ERROR - HTTPSConnectionPool(host='alertmanager-hostname.domain.tld', port=443): Max retries exceeded with url: /api/v1/silences (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self signed certificate in certificate chain (_ssl.c:1129)')))