feat: initial draft of custom metric tool and systemd timer

ic-os/components/guestos.bzl

@@ -85,6 +85,8 @@ component_files = {
     Label("monitoring/journald.conf"): "/etc/systemd/journald.conf",
     Label("monitoring/nft-exporter/nft-exporter.service"): "/etc/systemd/system/nft-exporter.service",
     Label("monitoring/nft-exporter/nft-exporter.timer"): "/etc/systemd/system/nft-exporter.timer",
+    Label("monitoring/custom-metrics/metrics_tool.service"): "/etc/systemd/system/metrics_tool.service",
+    Label("monitoring/custom-metrics/metrics_tool.timer"): "/etc/systemd/system/metrics_tool.timer",
 
     # networking
     Label("networking/generate-network-config/guestos/generate-network-config.service"): "/etc/systemd/system/generate-network-config.service",

ic-os/components/monitoring/custom-metrics/metrics_tool.service

@@ -0,0 +1,33 @@
+[Unit]
+Description=Report custom metrics once per minute
+
+[Service]
+Type=oneshot
+ExecStart=/opt/ic/bin/metrics_tool --metrics /run/node_exporter/collector_textfile/custom_metrics.prom
+DeviceAllow=/dev/vda
+IPAddressDeny=any
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+NoNewPrivileges=yes
+PrivateDevices=no
+PrivateNetwork=yes
+PrivateTmp=yes
+PrivateUsers=no
+ProtectClock=yes
+ProtectControlGroups=yes
+ProtectHome=yes
+ProtectHostname=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+ProtectSystem=strict
+ReadOnlyPaths=/proc/interrupts
+ReadWritePaths=/run/node_exporter/collector_textfile
+RestrictAddressFamilies=AF_UNIX
+RestrictAddressFamilies=~AF_UNIX
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+SystemCallArchitectures=native
+SystemCallErrorNumber=EPERM
+SystemCallFilter=@system-service
+UMask=022

ic-os/components/monitoring/custom-metrics/metrics_tool.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Collect custom metrics every minute
+
+[Timer]
+OnBootSec=60s
+OnUnitActiveSec=60s
+Unit=metrics_tool.service
+
+[Install]
+WantedBy=timers.target

ic-os/guestos/defs.bzl

@@ -50,7 +50,8 @@ def image_deps(mode, malicious = False):
             "//cpp:infogetty": "/opt/ic/bin/infogetty:0755",  # Terminal manager that replaces the login shell.
             "//cpp:prestorecon": "/opt/ic/bin/prestorecon:0755",  # Parallel restorecon replacement for filesystem relabeling.
             "//rs/ic_os/release:metrics-proxy": "/opt/ic/bin/metrics-proxy:0755",  # Proxies, filters, and serves public node metrics.
-
+            "//rs/ic_os/release:metrics_tool": "/opt/ic/bin/metrics_tool:0755",  # Collects and reports custom metrics.
+
             # additional libraries to install
             "//rs/ic_os/release:nss_icos": "/usr/lib/x86_64-linux-gnu/libnss_icos.so.2:0644",  # Allows referring to the guest IPv6 by name guestos from host, and host as hostos from guest.
         },

rs/ic_os/metrics_tool/BUILD.bazel

@@ -0,0 +1,54 @@
+load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test", "rust_test_suite")
+
+package(default_visibility = ["//rs:ic-os-pkg"])
+
+DEPENDENCIES = [
+    # Keep sorted.
+    "//rs/sys",
+    "@crate_index//:anyhow",
+    "@crate_index//:clap",    
+]
+
+DEV_DEPENDENCIES = [
+    # Keep sorted.
+]
+
+MACRO_DEPENDENCIES = []
+
+ALIASES = {}
+
+rust_library(
+    name = "metrics_tool",
+    srcs = glob(
+        ["src/**/*.rs"],
+        exclude = ["src/main.rs"],
+    ),
+    aliases = ALIASES,
+    crate_name = "ic_metrics_tool",
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    visibility = ["//rs:system-tests-pkg"],
+    deps = DEPENDENCIES,
+)
+
+rust_binary(
+    name = "metrics_tool_bin",
+    srcs = ["src/main.rs"],
+    aliases = ALIASES,
+    proc_macro_deps = MACRO_DEPENDENCIES,
+    deps = DEPENDENCIES + [":metrics_tool"],
+)
+
+rust_test(
+    name = "metrics_tool_test",
+    crate = ":metrics_tool",
+    deps = DEPENDENCIES + DEV_DEPENDENCIES,
+)
+
+rust_test_suite(
+    name = "metrics_tool_integration",
+    srcs = glob(["tests/**/*.rs"]),
+    target_compatible_with = [
+        "@platforms//os:linux",
+    ],
+    deps = [":metrics_tool_bin"] + DEPENDENCIES + DEV_DEPENDENCIES,
+)

rs/ic_os/metrics_tool/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "ic-metrics-tool"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "metrics_tool"
+path = "src/main.rs"
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true }

rs/ic_os/metrics_tool/src/lib.rs

@@ -0,0 +1,160 @@
+// TODO: refactor/merge this with fstrim_tool and guestos_tool metrics functionality
+use std::fs::File;
+use std::io::{self, Write};
+use std::path::Path;
+
+// TODO: everything is floating point for now
+pub struct Metric {
+    name: String,
+    value: f64,
+    annotation: String,
+    labels: Vec<(String, String)>,
+}
+
+impl Metric {
+    pub fn new(name: &str, value: f64) -> Self {
+        Self {
+            name: name.to_string(),
+            value,
+            annotation: "Custom metric".to_string(),
+            labels: Vec::new(),
+        }
+    }
+    pub fn with_annotation(name: &str, value: f64, annotation: &str) -> Self {
+        Self {
+            name: name.to_string(),
+            value,
+            annotation: annotation.to_string(),
+            labels: Vec::new(),
+        }
+    }
+
+    pub fn add_annotation(mut self, annotation: &str) -> Self {
+        self.annotation = annotation.to_string();
+        self
+    }
+
+    pub fn add_label(mut self, key: &str, value: &str) -> Self {
+        self.labels.push((key.to_string(), value.to_string()));
+        self
+    }
+
+    // TODO: formatting of floats
+    pub fn to_string(&self) -> String {
+        let labels_str = if self.labels.is_empty() {
+            String::new()
+        } else {
+            let labels: Vec<String> = self.labels.iter()
+                .map(|(k, v)| format!("{}=\"{}\"", k, v))
+                .collect();
+            format!("{{{}}}", labels.join(","))
+        };
+        format!("# HELP {} {}\n\
+             # TYPE {} counter\n\
+             {}{} {}", self.name, self.annotation, self.name, self.name, labels_str, self.value)
+    }
+}
+
+pub struct MetricsWriter {
+    file_path: String,
+}
+
+impl MetricsWriter {
+    pub fn new(file_path: &str) -> Self {
+        Self {
+            file_path: file_path.to_string(),
+        }
+    }
+
+    pub fn write_metrics(&self, metrics: &[Metric]) -> io::Result<()> {
+        let path = Path::new(&self.file_path);
+        let mut file = File::create(&path)?;
+        for metric in metrics {
+            writeln!(file, "{}", metric.to_string())?;
+        }
+        Ok(())
+    }
+}
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_metric_to_string() {
+        let metric = Metric::new("test_metric", 123.45)
+            .add_label("label1", "value1")
+            .add_label("label2", "value2");
+        assert_eq!(metric.to_string(), "# HELP test_metric Custom metric\n\
+             # TYPE test_metric counter\n\
+             test_metric{label1=\"value1\",label2=\"value2\"} 123.45");
+    }
+
+    #[test]
+    fn test_write_metrics() {
+        let metrics = vec![
+            Metric::new("metric1", 1.0),
+            Metric::new("metric2", 2.0).add_label("label", "value"),
+        ];
+        let writer = MetricsWriter::new("/tmp/test_metrics.prom");
+        writer.write_metrics(&metrics).unwrap();
+        let content = std::fs::read_to_string("/tmp/test_metrics.prom").unwrap();
+        assert!(content.contains("# HELP metric1 Custom metric\n\
+             # TYPE metric1 counter\n\
+             metric1 1"));
+        assert!(content.contains("# HELP metric2 Custom metric\n\
+             # TYPE metric2 counter\n\
+             metric2{label=\"value\"} 2"));
+    }
+
+    #[test]
+    fn test_metric_large_value() {
+        let metric = Metric::new("large_value_metric", 1.0e64);
+        assert_eq!(metric.to_string(), "# HELP large_value_metric Custom metric\n\
+             # TYPE large_value_metric counter\n\
+             large_value_metric 10000000000000000000000000000000000000000000000000000000000000000");
+    }
+
+
+    #[test]
+    fn test_metric_without_labels() {
+        let metric = Metric::new("no_label_metric", 42.0);
+        assert_eq!(metric.to_string(), "# HELP no_label_metric Custom metric\n\
+             # TYPE no_label_metric counter\n\
+             no_label_metric 42");
+    }
+
+    #[test]
+    fn test_metric_with_annotation() {
+        let metric = Metric::with_annotation("annotated_metric", 99.9, "This is a test metric");
+        assert_eq!(metric.to_string(), "# HELP annotated_metric This is a test metric\n\
+             # TYPE annotated_metric counter\n\
+             annotated_metric 99.9");
+    }
+
+    #[test]
+    fn test_write_empty_metrics() {
+        let metrics: Vec<Metric> = Vec::new();
+        let writer = MetricsWriter::new("/tmp/test_empty_metrics.prom");
+        writer.write_metrics(&metrics).unwrap();
+        let content = std::fs::read_to_string("/tmp/test_empty_metrics.prom").unwrap();
+        assert!(content.is_empty());
+    }
+
+    #[test]
+    fn test_metric_with_multiple_labels() {
+        let metric = Metric::new("multi_label_metric", 10.0)
+            .add_label("foo", "bar")
+            .add_label("version", "1.0.0");
+        assert_eq!(metric.to_string(), "# HELP multi_label_metric Custom metric\n\
+             # TYPE multi_label_metric counter\n\
+             multi_label_metric{foo=\"bar\",version=\"1.0.0\"} 10");
+    }
+
+    #[test]
+    fn test_metric_with_empty_annotation() {
+        let metric = Metric::with_annotation("empty_annotation_metric", 5.5, "");
+        assert_eq!(metric.to_string(), "# HELP empty_annotation_metric \n\
+             # TYPE empty_annotation_metric counter\n\
+             empty_annotation_metric 5.5");
+    }
+}

rs/ic_os/metrics_tool/src/main.rs

@@ -0,0 +1,61 @@
+use anyhow::Result;
+use clap::Parser;
+
+use std::path::Path;
+use std::fs::File;
+use std::io::{self, BufRead};
+
+use ic_metrics_tool::{Metric, MetricsWriter};
+
+const INTERRUPT_FILTER: &str = "TLB shootdowns";
+const INTERRUPT_SOURCE: &str = "/proc/interrupts";
+const CUSTOM_METRICS_PROM: &str = "/run/node_exporter/collector_textfile/custom_metrics.prom";
+const TLB_SHOOTDOWN_METRIC_NAME: &str = "sum_tlb_shootdowns";
+
+#[derive(Parser)]
+struct MetricToolArgs {
+    #[arg(
+        short = 'm',
+        long = "metrics",
+        default_value = CUSTOM_METRICS_PROM
+    )]
+    /// Filename to write the prometheus metrics for node_exporter generation.
+    /// Fails badly if the directory doesn't exist.
+    metrics_filename: String,
+}
+
+fn get_sum_tlb_shootdowns() -> Result<u64> {
+    let path = Path::new(INTERRUPT_SOURCE);
+    let file = File::open(&path)?;
+    let reader = io::BufReader::new(file);
+
+    let mut total_tlb_shootdowns = 0;
+
+    for line in reader.lines() {
+        let line = line?;
+        if line.contains(INTERRUPT_FILTER) {
+            let parts: Vec<&str> = line.split_whitespace().collect();
+            for part in parts.iter().skip(1) {
+                if let Ok(value) = part.parse::<u64>() {
+                    total_tlb_shootdowns += value;
+                }
+            }
+        }
+    }
+
+    Ok(total_tlb_shootdowns)
+}
+
+pub fn main() -> Result<()> {
+    let opts = MetricToolArgs::parse();
+    let mpath = Path::new(&opts.metrics_filename);
+    let tlb_shootdowns = get_sum_tlb_shootdowns()?;
+
+    let metrics = vec![
+        Metric::new(TLB_SHOOTDOWN_METRIC_NAME, tlb_shootdowns as f64).add_annotation("Total TLB shootdowns"),        
+    ];
+    let writer = MetricsWriter::new(mpath.to_str().unwrap());
+    writer.write_metrics(&metrics).unwrap();
+
+    Ok(())
+}

rs/ic_os/release/BUILD.bazel

@@ -13,6 +13,7 @@ OBJECTS = {
     "vsock_host": "//rs/ic_os/vsock/host:vsock_host",
     "metrics-proxy": "@crate_index//:metrics-proxy__metrics-proxy",
     "nss_icos": "//rs/ic_os/nss_icos",
+    "metrics_tool": "//rs/ic_os/metrics_tool:metrics_tool_bin",
 }
 
 [release_strip_binary(