From 2bbf0a3fa799b52fa64ae51aa4465df90c8dc325 Mon Sep 17 00:00:00 2001 From: Nicolas Ochem Date: Thu, 12 Jan 2023 15:31:01 -0800 Subject: [PATCH] tezos signer forwarder chart The last remaining piece of https://github.com/midl-dev/tezos-on-gke/ to move into tezos-k8s, tezos-signer-forwarder is a terminating pod for ssh tunnels exposing a tezos signing endpoint from an on-prem location. --- charts/tezos-signer-forwarder/Chart.yaml | 24 ++++++ .../scripts/entrypoint.sh | 3 + .../scripts/signer_exporter.py | 28 +++++++ .../templates/_helpers.tpl | 62 ++++++++++++++ .../templates/alertmanagerconfig.yaml | 54 ++++++++++++ .../templates/config.yaml | 10 +++ .../templates/prometheusrule.yaml | 51 ++++++++++++ .../templates/secret.yaml | 7 ++ .../templates/service.yaml | 37 +++++++++ .../templates/servicemonitor.yaml | 38 +++++++++ .../templates/statefulset.yaml | 81 ++++++++++++++++++ charts/tezos-signer-forwarder/values.yaml | 82 +++++++++++++++++++ signerForwarder/Dockerfile | 23 ++++++ 13 files changed, 500 insertions(+) create mode 100644 charts/tezos-signer-forwarder/Chart.yaml create mode 100644 charts/tezos-signer-forwarder/scripts/entrypoint.sh create mode 100644 charts/tezos-signer-forwarder/scripts/signer_exporter.py create mode 100644 charts/tezos-signer-forwarder/templates/_helpers.tpl create mode 100644 charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml create mode 100644 charts/tezos-signer-forwarder/templates/config.yaml create mode 100644 charts/tezos-signer-forwarder/templates/prometheusrule.yaml create mode 100644 charts/tezos-signer-forwarder/templates/secret.yaml create mode 100644 charts/tezos-signer-forwarder/templates/service.yaml create mode 100644 charts/tezos-signer-forwarder/templates/servicemonitor.yaml create mode 100644 charts/tezos-signer-forwarder/templates/statefulset.yaml create mode 100644 charts/tezos-signer-forwarder/values.yaml create mode 100644 signerForwarder/Dockerfile diff --git a/charts/tezos-signer-forwarder/Chart.yaml b/charts/tezos-signer-forwarder/Chart.yaml new file mode 100644 index 000000000..00db4f455 --- /dev/null +++ b/charts/tezos-signer-forwarder/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: tezos-signer-forwarder +description: A chart for tezos-signer-forwarder + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "10.0" diff --git a/charts/tezos-signer-forwarder/scripts/entrypoint.sh b/charts/tezos-signer-forwarder/scripts/entrypoint.sh new file mode 100644 index 000000000..41233320e --- /dev/null +++ b/charts/tezos-signer-forwarder/scripts/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +/usr/sbin/sshd -D -e -p ${TUNNEL_ENDPOINT_PORT} diff --git a/charts/tezos-signer-forwarder/scripts/signer_exporter.py b/charts/tezos-signer-forwarder/scripts/signer_exporter.py new file mode 100644 index 000000000..4ded9a154 --- /dev/null +++ b/charts/tezos-signer-forwarder/scripts/signer_exporter.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +import os +from flask import Flask, request, jsonify +import requests + +import logging +log = logging.getLogger('werkzeug') +log.setLevel(logging.ERROR) + +application = Flask(__name__) + +readiness_probe_path = os.getenv("READINESS_PROBE_PATH") + +@application.route('/metrics', methods=['GET']) +def prometheus_metrics(): + ''' + Prometheus endpoint + ''' + try: + probe = requests.get(f"http://localhost:8443{readiness_probe_path}") + except requests.exceptions.RequestException: + probe = None + return f'''# number of unhealthy signers - should be 0 or 1 +unhealthy_signers_total {0 if probe else 1} +''' + +if __name__ == "__main__": + application.run(host = "0.0.0.0", port = 31732, debug = False) diff --git a/charts/tezos-signer-forwarder/templates/_helpers.tpl b/charts/tezos-signer-forwarder/templates/_helpers.tpl new file mode 100644 index 000000000..c3113e365 --- /dev/null +++ b/charts/tezos-signer-forwarder/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "tezos-signer-forwarder.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "tezos-signer-forwarder.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name $.Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" $.Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "tezos-signer-forwarder.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "tezos-signer-forwarder.labels" -}} +helm.sh/chart: {{ include "tezos-signer-forwarder.chart" . }} +{{ include "tezos-signer-forwarder.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "tezos-signer-forwarder.selectorLabels" -}} +app.kubernetes.io/name: {{ include "tezos-signer-forwarder.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "tezos-signer-forwarder.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "tezos-signer-forwarder.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml new file mode 100644 index 000000000..d5170fd4b --- /dev/null +++ b/charts/tezos-signer-forwarder/templates/alertmanagerconfig.yaml @@ -0,0 +1,54 @@ +{{- if .Values.alertmanagerConfig.enabled }} +{{- range .Values.endpoints }} +{{- if .monitoring_email }} +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: tezos-remote-signer-alerts-{{ .name }} + labels: +{{- toYaml $.Values.alertmanagerConfig.labels | nindent 4 }} +spec: + route: + groupBy: ['job'] + groupWait: 30s + groupInterval: 5m + repeatInterval: 12h + receiver: 'email_{{ .name }}' + matchers: + - name: service + value: tezos-remote-signer-{{ .name }} + regex: false + - name: alertType + value: tezos-remote-signer-alert + regex: false + continue: false + receivers: + - name: 'email_{{ .name }}' + emailConfigs: + - to: "{{ .monitoring_email }}" + sendResolved: true + headers: + - key: subject + value: '{{`[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}`}}' + html: >- + {{`{{ if eq .Status "firing" }} + Your attention is required regarding the following Tezos Remote Signer alert: + {{ else }} + The following Tezos Remote Signer Alert is resolved: + {{ end }} + {{ range .Alerts -}} + {{ .Annotations.summary }} + {{ end }}`}} + text: >- + {{`{{ if eq .Status "firing" }} + Your attention is required regarding the following Tezos Remote Signer alert: + {{ else }} + The following Tezos Remote Signer Alert is resolved: + {{ end }} + {{ range .Alerts -}} + {{ .Annotations.summary }} + {{ end }}`}} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/tezos-signer-forwarder/templates/config.yaml b/charts/tezos-signer-forwarder/templates/config.yaml new file mode 100644 index 000000000..0fb534f96 --- /dev/null +++ b/charts/tezos-signer-forwarder/templates/config.yaml @@ -0,0 +1,10 @@ +{{- range .Values.endpoints }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: tezos-signer-forwarder-config-{{ .name }} +data: + authorized_keys: "{{ .ssh_pubkey }} signer" + tunnel_endpoint_port: "{{ .tunnel_endpoint_port }}" +--- +{{- end }} diff --git a/charts/tezos-signer-forwarder/templates/prometheusrule.yaml b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml new file mode 100644 index 000000000..e6bafc3ab --- /dev/null +++ b/charts/tezos-signer-forwarder/templates/prometheusrule.yaml @@ -0,0 +1,51 @@ +{{- if .Values.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: +{{- toYaml .Values.prometheusRule.labels | nindent 4 }} + name: tezos-remote-signer-rules +spec: + groups: + - name: tezos-remote-signer.rules + rules: + - alert: SignerPowerLoss + annotations: + description: 'Remote signer has lost power' + summary: Tezos remote signer has lost power + expr: power{namespace="{{ .Release.Namespace }}"} != 0 + for: 1m + labels: + severity: critical + alertType: tezos-remote-signer-alert + - alert: SignerWiredNetworkLoss + annotations: + description: 'Remote signer has lost wired internet connection' + summary: Tezos remote signer has lost wired internet connection + expr: wired_network{namespace="{{ .Release.Namespace }}"} != 0 + for: 1m + labels: + severity: critical + alertType: tezos-remote-signer-alert +--- +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: +{{- toYaml .Values.prometheusRule.labels | nindent 4 }} + name: tezos-remote-signer-reachability-rules +spec: + groups: + - name: tezos-remote-signer.rules + rules: + - alert: NoRemoteSigner + annotations: + description: 'Remote signer is down' + summary: Remote signer is down or unable to sign. + expr: unhealthy_signers_total{namespace="{{ .Release.Namespace }}"} != 0 + for: 1m + labels: + severity: critical + alertType: tezos-remote-signer-alert +--- +{{- end }} diff --git a/charts/tezos-signer-forwarder/templates/secret.yaml b/charts/tezos-signer-forwarder/templates/secret.yaml new file mode 100644 index 000000000..b330b7cdb --- /dev/null +++ b/charts/tezos-signer-forwarder/templates/secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: tezos-signer-forwarder-secret-{{ .Values.name }} +data: + ssh_host_ecdsa_key: | +{{ println .Values.secrets.signer_target_host_key | b64enc | indent 4 -}} diff --git a/charts/tezos-signer-forwarder/templates/service.yaml b/charts/tezos-signer-forwarder/templates/service.yaml new file mode 100644 index 000000000..efdf727b7 --- /dev/null +++ b/charts/tezos-signer-forwarder/templates/service.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: tezos-remote-signer-ssh-ingress-{{ .Values.name }} + annotations: +{{ toYaml .Values.service_annotations | indent 4 }} +spec: + type: LoadBalancer + selector: + app.kubernetes.io/name: tezos-signer-forwarder + ports: +{{- range .Values.endpoints }} + - port: {{ .tunnel_endpoint_port }} + name: tunnel-{{ .name }} +{{- end }} + # ensures that remote signers can always ssh + publishNotReadyAddresses: true +--- +{{- range .Values.endpoints }} +apiVersion: v1 +kind: Service +metadata: + name: tezos-remote-signer-{{ .name }} + labels: + app.kubernetes.io/name: tezos-signer-forwarder + app.kubernetes.io/baker-name: {{ .name }} +spec: + selector: + app.kubernetes.io/name: tezos-signer-forwarder + app.kubernetes.io/baker-name: {{ .name }} + ports: + - port: 8443 + name: signer + - port: 31732 + name: metrics +--- +{{- end }} diff --git a/charts/tezos-signer-forwarder/templates/servicemonitor.yaml b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml new file mode 100644 index 000000000..a86932569 --- /dev/null +++ b/charts/tezos-signer-forwarder/templates/servicemonitor.yaml @@ -0,0 +1,38 @@ +{{- if .Values.serviceMonitor.enabled }} +{{- range .Values.endpoints }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/name: tezos-signer-forwarder + name: tezos-remote-signer-monitoring-{{ .name }} + namespace: {{ $.Release.Namespace }} +spec: + endpoints: + - interval: 20s + port: signer + path: /healthz + scrapeTimeout: 20s + selector: + matchLabels: + app.kubernetes.io/name: tezos-signer-forwarder + app.kubernetes.io/baker-name: {{ .name }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + app.kubernetes.io/name: tezos-signer-forwarder + name: tezos-remote-signer-reachability-{{ .name }} + namespace: {{ $.Release.Namespace }} +spec: + endpoints: + - port: metrics + path: /metrics + selector: + matchLabels: + app.kubernetes.io/name: tezos-signer-forwarder + app.kubernetes.io/baker-name: {{ .name }} +--- +{{- end }} +{{- end }} diff --git a/charts/tezos-signer-forwarder/templates/statefulset.yaml b/charts/tezos-signer-forwarder/templates/statefulset.yaml new file mode 100644 index 000000000..fd7a6cab3 --- /dev/null +++ b/charts/tezos-signer-forwarder/templates/statefulset.yaml @@ -0,0 +1,81 @@ +{{- range .Values.endpoints }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: tezos-signer-forwarder-{{ .name}} + annotations: + "pulumi.com/skipAwait": "true" +spec: + replicas: 1 + serviceName: tezos-remote-signer-{{ .name }} + selector: + matchLabels: + app.kubernetes.io/name: tezos-signer-forwarder + template: + metadata: + {{- with $.Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app.kubernetes.io/name: tezos-signer-forwarder + app.kubernetes.io/baker-name: {{ .name }} + spec: + volumes: + - name: config-volume + configMap: + name: tezos-signer-forwarder-config-{{ .name }} + - name: secret-volume + secret: + secretName: tezos-signer-forwarder-secret-{{ $.Values.name }} + defaultMode: 0400 + containers: + - name: tezos-signer-forwarder + image: {{ $.Values.images.tezos_signer_forwarder }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + args: + - "-c" + - | +{{ tpl ($.Files.Get (print "scripts/entrypoint.sh")) $ | indent 12 }} + volumeMounts: + - name: config-volume + mountPath: /home/signer/.ssh/authorized_keys + subPath: authorized_keys + - name: secret-volume + mountPath: /etc/ssh/ssh_host_ecdsa_key + subPath: ssh_host_ecdsa_key + env: + - name: TUNNEL_ENDPOINT_PORT + valueFrom: + configMapKeyRef: + name: tezos-signer-forwarder-config-{{ .name }} + key: tunnel_endpoint_port + ports: + - name: signer + containerPort: 8443 + protocol: TCP + readinessProbe: + httpGet: + path: {{ .readiness_probe_path }} + port: 8443 + - name: prom-exporter + image: {{ $.Values.tezos_k8s_images.utils }} + ports: + - name: metrics + containerPort: 31732 + protocol: TCP + env: + - name: READINESS_PROBE_PATH + value: {{ .readiness_probe_path | quote }} + command: + - /usr/local/bin/python + args: + - "-c" + - | +{{ tpl ($.Files.Get (print "scripts/signer_exporter.py")) $ | indent 12 }} + nodeSelector: + {{ toYaml $.Values.node_selector | indent 8 }} +--- +{{- end }} diff --git a/charts/tezos-signer-forwarder/values.yaml b/charts/tezos-signer-forwarder/values.yaml new file mode 100644 index 000000000..a2866c15a --- /dev/null +++ b/charts/tezos-signer-forwarder/values.yaml @@ -0,0 +1,82 @@ +images: + tezos_signer_forwarder: localhost/tezos-k8s-signerforwarder:dev +tezos_k8s_images: + utils: ghcr.io/oxheadalpha/tezos-k8s-utils:master + +# List the endpoints below. +# Each endpoint represents a ssh server. +# To handle several endpoints, you can either: +# * instantiate several replicas of this chart, or +# * list several endpoints below. + +# Since this chart instantiates a service of type Loadbalancer, +# it may be the case that each such service comes with its own +# auto-assigned IP, increasing costs. +# Listing several endpoints below will put all +# associated pods behind the same LoadBalancer service. +# Consequenty, the same IP will be re-used between signers. +# If you prefer to have one IP per signer, instantiate this chart +# several times. +endpoints: + # the public key that the server is expecting. + # The signer should authenticate with the corresponding secret key. +- ssh_pubkey: "ssh-rsa AAAA...." + + # ssh tunnel connection establishes to this port + tunnel_endpoint_port: 50000 + + # endpoint name - to disambiguate them + name: myendpoint + + # Set a readiness probe path for your signer. + # By default, it is the known path implemented by every signer "/authorized_keys" + # When using tezos-remote-signer-os, you can set it to a path that performs more + # checks, such as: + # "/statusz/${PUBLIC_BAKING_KEY_HASH}?ledger_url=${LEDGER_AUTHORIZED_PATH_ENCODED}" + readiness_probe_path: /authorized_keys + + # Enter email address to send alerts to. + # monitoring_email: + +# Name that goes into the service +# e.g tezos-signer-mybaker +# useful when one baker bakes for several addresses +# on different remote signers. +name: mybaker + +# to deploy in a specific node pool, put label here +node_selector: {} + +# LoadBalancer service annotations. On some cloud providers, it can +# be used to assign a static ip address. +service_annotations: {} + +secrets: + # The ssh host key must be passed as input. + # Otherwise, when destroying and respinning the infra, + # the signer would not recognize the host and refuse to + # connect. + signer_target_host_key: | + -----BEGIN OPENSSH PRIVATE KEY----- + xxx + -----END OPENSSH PRIVATE KEY----- + +# Prometheus Operator is required in your cluster in order to enable +# serviceMonitor and prometheusRule below. +# Enable service monitor to scrape the /healthz endpoint of your +# remote signer. +# The /healthz endpoint is exposed by tezos-remote-signer-os: +# https://github.com/midl-dev/tezos-remote-signer-os +serviceMonitor: + enabled: false +# Enable Prometheus Rule to be alerted when your hardware remote signer +# provisioned with tezos-remote-signer-os loses power or wired network +# connectivity. +# For rules to be picked up by the Prometheus Operator, proper labels need +# to be set below. Refer to Prometheus operator documentation for details. +prometheusRule: + enabled: false + labels: {} +alertmanagerConfig: + enabled: false + labels: {} diff --git a/signerForwarder/Dockerfile b/signerForwarder/Dockerfile new file mode 100644 index 000000000..d1a692b4a --- /dev/null +++ b/signerForwarder/Dockerfile @@ -0,0 +1,23 @@ +FROM alpine:edge + +# add openssh and clean +RUN apk add --no-cache openssh shadow + +RUN adduser --system signer +# * is the hash of the password. Effectively, password login is disabled. +# but I need to do that otherwise sshd says account is locked. see: +# https://unix.stackexchange.com/a/193131/81131 +RUN usermod -p '*' signer + +#allow forwarding +RUN sed -ri 's/^.*GatewayPorts.*$/GatewayPorts yes/g' /etc/ssh/sshd_config +RUN sed -ri 's/^.*AllowTcpForwarding.*$/AllowTcpForwarding yes/g' /etc/ssh/sshd_config +RUN sed -ri 's/^.*PasswordAuthentication.*$/PasswordAuthentication no/g' /etc/ssh/sshd_config +RUN sed -ri 's/^.*ClientAliveInterval.*$/ClientAliveInterval 10/g' /etc/ssh/sshd_config +RUN sed -ri 's/^.*ClientAliveCountMax.*$/ClientAliveCountMax 2/g' /etc/ssh/sshd_config +RUN printf "AllowUsers signer\n" >> /etc/ssh/sshd_config +RUN cat /etc/ssh/sshd_config + +RUN mkdir /home/signer/.ssh && chown -R signer /home/signer + +CMD ["/usr/sbin/sshd"]