From b1237bb1e0f7e487b1133aa455649806585c1bb6 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Wed, 21 Jun 2023 14:25:03 +0000 Subject: [PATCH 1/3] attester: Make Attester::get_evidence() async Marking get_evidence as async allows running async functions from an attester. There are two motivators for this: - TDX report->quote conversion can require an HTTP request, and it's a good idea to run that async. - as we move to support the RATS passport model better the attester itself might need to talk to MAA or Amber to fetch an attestation token. If get_evidence is not async, then it becomes tricky to use reqwest from get_evidence. reqwest::blocking::Client panics because it internally uses tokio and get_evidence is called from a tokio runtime. Signed-off-by: Jeremi Piotrowski --- attestation-agent/attester/Cargo.toml | 4 ++++ attestation-agent/attester/src/az_snp_vtpm/mod.rs | 3 ++- attestation-agent/attester/src/lib.rs | 3 ++- attestation-agent/attester/src/sample/mod.rs | 3 ++- attestation-agent/attester/src/sgx_occlum/mod.rs | 9 +++++---- attestation-agent/attester/src/snp/mod.rs | 3 ++- attestation-agent/attester/src/tdx/mod.rs | 9 +++++---- attestation-agent/kbs_protocol/src/lib.rs | 5 +++-- 8 files changed, 25 insertions(+), 14 deletions(-) diff --git a/attestation-agent/attester/Cargo.toml b/attestation-agent/attester/Cargo.toml index 9c85bd144..88cc92328 100644 --- a/attestation-agent/attester/Cargo.toml +++ b/attestation-agent/attester/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] anyhow.workspace = true +async-trait.workspace = true az-snp-vtpm = { git = "https://github.com/kinvolk/azure-cvm-tooling", rev = "2c2e411", default-features = false, features = ["attester"], optional = true } base64.workspace = true log.workspace = true @@ -17,6 +18,9 @@ sev = { git = "https://github.com/virtee/sev", rev = "3dca05d2c93388cb00534ad18f strum.workspace = true tdx-attest-rs = { git = "https://github.com/intel/SGXDataCenterAttestationPrimitives", rev = "cc582e8be0c9010295c66fb58c59f74744017600", optional = true } +[dev-dependencies] +tokio = { version = "1", features = ["macros"] } + [features] default = ["all-attesters"] all-attesters = ["tdx-attester", "occlum-attester", "az-snp-vtpm-attester", "snp-attester"] diff --git a/attestation-agent/attester/src/az_snp_vtpm/mod.rs b/attestation-agent/attester/src/az_snp_vtpm/mod.rs index 49cbcb79b..c3930bb8b 100644 --- a/attestation-agent/attester/src/az_snp_vtpm/mod.rs +++ b/attestation-agent/attester/src/az_snp_vtpm/mod.rs @@ -27,8 +27,9 @@ struct Evidence { vcek: String, } +#[async_trait::async_trait] impl Attester for AzSnpVtpmAttester { - fn get_evidence(&self, report_data: String) -> Result { + async fn get_evidence(&self, report_data: String) -> Result { let report = vtpm::get_report()?; let report_data_bin = base64::decode(report_data)?; let quote = vtpm::get_quote(&report_data_bin)?; diff --git a/attestation-agent/attester/src/lib.rs b/attestation-agent/attester/src/lib.rs index 58892622e..ca8f79b79 100644 --- a/attestation-agent/attester/src/lib.rs +++ b/attestation-agent/attester/src/lib.rs @@ -57,8 +57,9 @@ impl Tee { } } +#[async_trait::async_trait] pub trait Attester { - fn get_evidence(&self, report_data: String) -> Result; + async fn get_evidence(&self, report_data: String) -> Result; } // Detect which TEE platform the KBC running environment is. diff --git a/attestation-agent/attester/src/sample/mod.rs b/attestation-agent/attester/src/sample/mod.rs index 304859393..cea81b608 100644 --- a/attestation-agent/attester/src/sample/mod.rs +++ b/attestation-agent/attester/src/sample/mod.rs @@ -24,8 +24,9 @@ struct SampleQuote { #[derive(Debug, Default)] pub struct SampleAttester {} +#[async_trait::async_trait] impl Attester for SampleAttester { - fn get_evidence(&self, report_data: String) -> Result { + async fn get_evidence(&self, report_data: String) -> Result { let evidence = SampleQuote { svn: "1".to_string(), report_data, diff --git a/attestation-agent/attester/src/sgx_occlum/mod.rs b/attestation-agent/attester/src/sgx_occlum/mod.rs index b522692cc..8b4bab99b 100644 --- a/attestation-agent/attester/src/sgx_occlum/mod.rs +++ b/attestation-agent/attester/src/sgx_occlum/mod.rs @@ -23,8 +23,9 @@ struct SgxOcclumAttesterEvidence { #[derive(Debug, Default)] pub struct SgxOcclumAttester {} +#[async_trait::async_trait] impl Attester for SgxOcclumAttester { - fn get_evidence(&self, report_data: String) -> Result { + async fn get_evidence(&self, report_data: String) -> Result { let mut report_data_bin = base64::decode(report_data)?; if report_data_bin.len() != 48 { bail!("Occlum SGX Attester: Report data should be SHA384 base64 String"); @@ -56,13 +57,13 @@ mod tests { use super::*; #[ignore] - #[test] - fn test_sgx_get_evidence() { + #[tokio::test] + async fn test_sgx_get_evidence() { let attester = SgxOcclumAttester::default(); let report_data: Vec = vec![0; 48]; let report_data_base64 = base64::encode(report_data); - let evidence = attester.get_evidence(report_data_base64); + let evidence = attester.get_evidence(report_data_base64).await; assert!(evidence.is_ok()); } } diff --git a/attestation-agent/attester/src/snp/mod.rs b/attestation-agent/attester/src/snp/mod.rs index e1d45e760..ed5e5894b 100644 --- a/attestation-agent/attester/src/snp/mod.rs +++ b/attestation-agent/attester/src/snp/mod.rs @@ -24,8 +24,9 @@ struct SnpEvidence { #[derive(Debug, Default)] pub struct SnpAttester {} +#[async_trait::async_trait] impl Attester for SnpAttester { - fn get_evidence(&self, report_data: String) -> Result { + async fn get_evidence(&self, report_data: String) -> Result { let mut report_data_bin = base64::decode(report_data)?; if report_data_bin.len() != 48 { diff --git a/attestation-agent/attester/src/tdx/mod.rs b/attestation-agent/attester/src/tdx/mod.rs index 33f2bcac2..dff6091ee 100644 --- a/attestation-agent/attester/src/tdx/mod.rs +++ b/attestation-agent/attester/src/tdx/mod.rs @@ -27,8 +27,9 @@ struct TdxEvidence { #[derive(Debug, Default)] pub struct TdxAttester {} +#[async_trait::async_trait] impl Attester for TdxAttester { - fn get_evidence(&self, report_data: String) -> Result { + async fn get_evidence(&self, report_data: String) -> Result { let mut report_data_bin = base64::decode(report_data)?; if report_data_bin.len() != 48 { return Err(anyhow!( @@ -71,13 +72,13 @@ mod tests { use super::*; #[ignore] - #[test] - fn test_tdx_get_evidence() { + #[tokio::test] + async fn test_tdx_get_evidence() { let attester = TdxAttester::default(); let report_data: Vec = vec![0; 48]; let report_data_base64 = base64::encode(report_data); - let evidence = attester.get_evidence(report_data_base64); + let evidence = attester.get_evidence(report_data_base64).await; assert!(evidence.is_ok()); } } diff --git a/attestation-agent/kbs_protocol/src/lib.rs b/attestation-agent/kbs_protocol/src/lib.rs index 8ba6267c0..4bf1b5074 100644 --- a/attestation-agent/kbs_protocol/src/lib.rs +++ b/attestation-agent/kbs_protocol/src/lib.rs @@ -53,7 +53,7 @@ impl KbsProtocolWrapper { }) } - fn generate_evidence(&self) -> Result { + async fn generate_evidence(&self) -> Result { let key = self .tee_key .as_ref() @@ -77,6 +77,7 @@ impl KbsProtocolWrapper { let tee_evidence = attester .get_evidence(ehd) + .await .map_err(|e| anyhow!("Get TEE evidence failed: {:?}", e))?; Ok(Attestation { @@ -109,7 +110,7 @@ impl KbsProtocolWrapper { .http_client() .post(format!("{kbs_host_url}/{KBS_URL_PREFIX}/attest")) .header("Content-Type", "application/json") - .json(&self.generate_evidence()?) + .json(&self.generate_evidence().await?) .send() .await?; From 2cd372e1a93921f205186047a11bb88e64fc33f0 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Wed, 5 Jul 2023 09:23:20 +0000 Subject: [PATCH 2/3] attester: tdx: Detect platform using cpuid leaf This is equally good as a check if the TDX attester should be used. An additional consideration is that while we can check for "known" device node names, there is no way to specify which ones the Intel SGX libraries use (hardcoded in C source) and the ioctl API also differs for the different names, including for the /dev/tdx_guest device that was upstreamed. Signed-off-by: Jeremi Piotrowski --- attestation-agent/attester/src/tdx/mod.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/attestation-agent/attester/src/tdx/mod.rs b/attestation-agent/attester/src/tdx/mod.rs index dff6091ee..7fdd60ef4 100644 --- a/attestation-agent/attester/src/tdx/mod.rs +++ b/attestation-agent/attester/src/tdx/mod.rs @@ -5,14 +5,18 @@ use super::Attester; use anyhow::*; +use raw_cpuid::cpuid; use serde::{Deserialize, Serialize}; -use std::path::Path; use tdx_attest_rs; const CCEL_PATH: &str = "/sys/firmware/acpi/tables/data/CCEL"; pub fn detect_platform() -> bool { - Path::new("/dev/tdx-attest").exists() || Path::new("/dev/tdx-guest").exists() + const TDX_CPUID_LEAF: u32 = 0x21; + let c = cpuid!(TDX_CPUID_LEAF, 0); + let tdbytes = [c.ebx, c.edx, c.ecx].map(|x| x.to_le_bytes()).concat(); + let tdstring = String::from_utf8_lossy(&tdbytes); + tdstring == "IntelTDX " } #[derive(Serialize, Deserialize)] From 1b2ec062193c40df7f037410c353225bb50cc6a9 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Wed, 21 Jun 2023 15:02:58 +0000 Subject: [PATCH 3/3] attester: tdx: Support TDX CVM on Azure To get TDX attestation to work on Azure there are several changes needed: - the device node is called /dev/tdx_guest (upstream kernel name) - quote generation uses the IMDS (instance metadata service) instead of tdvmcall or vsock. This also means we can't use tdx_att_get_quote which combines quote and report fetching - no CCEL Implement the evidence gathering in a sub-module attester, but keep it within the TDX module because the evidence is fully compatible with the existing TDX verifier. It would be possible to use tdx_att_get_report, but calling an ioctl is easy enough that it doesn't make sense to add the dependency on the native library. The Intel library also seems to have an outdated definition of the ioctl. Signed-off-by: Jeremi Piotrowski --- attestation-agent/attester/Cargo.toml | 5 +- attestation-agent/attester/src/lib.rs | 2 +- attestation-agent/attester/src/tdx/az.rs | 105 ++++++++++++++++++++++ attestation-agent/attester/src/tdx/mod.rs | 27 ++++-- 4 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 attestation-agent/attester/src/tdx/az.rs diff --git a/attestation-agent/attester/Cargo.toml b/attestation-agent/attester/Cargo.toml index 88cc92328..78137ee82 100644 --- a/attestation-agent/attester/Cargo.toml +++ b/attestation-agent/attester/Cargo.toml @@ -10,8 +10,11 @@ anyhow.workspace = true async-trait.workspace = true az-snp-vtpm = { git = "https://github.com/kinvolk/azure-cvm-tooling", rev = "2c2e411", default-features = false, features = ["attester"], optional = true } base64.workspace = true +ioctl-sys = { version = "0.8.0", optional = true } log.workspace = true occlum_dcap = { git = "https://github.com/occlum/occlum", rev = "dbe404f", optional = true } +raw-cpuid = { version = "11", optional = true } +reqwest = { version = "0.11", features = ["json"], optional = true } serde.workspace = true serde_json.workspace = true sev = { git = "https://github.com/virtee/sev", rev = "3dca05d2c93388cb00534ad18f5928fd812e99cc", optional = true } @@ -25,7 +28,7 @@ tokio = { version = "1", features = ["macros"] } default = ["all-attesters"] all-attesters = ["tdx-attester", "occlum-attester", "az-snp-vtpm-attester", "snp-attester"] -tdx-attester = ["tdx-attest-rs"] +tdx-attester = ["tdx-attest-rs", "dep:reqwest", "dep:ioctl-sys", "dep:raw-cpuid"] occlum-attester = ["occlum_dcap"] az-snp-vtpm-attester = ["az-snp-vtpm"] snp-attester = ["sev"] diff --git a/attestation-agent/attester/src/lib.rs b/attestation-agent/attester/src/lib.rs index ca8f79b79..ff93c544f 100644 --- a/attestation-agent/attester/src/lib.rs +++ b/attestation-agent/attester/src/lib.rs @@ -45,7 +45,7 @@ impl Tee { match self { Tee::Sample => Ok(Box::::default()), #[cfg(feature = "tdx-attester")] - Tee::Tdx => Ok(Box::::default()), + Tee::Tdx => Ok(tdx::make_attester()), #[cfg(feature = "occlum-attester")] Tee::SgxOcclum => Ok(Box::::default()), #[cfg(feature = "az-snp-vtpm-attester")] diff --git a/attestation-agent/attester/src/tdx/az.rs b/attestation-agent/attester/src/tdx/az.rs new file mode 100644 index 000000000..fbdd3ee75 --- /dev/null +++ b/attestation-agent/attester/src/tdx/az.rs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// +// SPDX-License-Identifier: Apache-2.0 +// + +use anyhow::*; +use ioctl_sys::ioctl; +use raw_cpuid::{CpuId, Hypervisor}; +use serde::{Deserialize, Serialize}; +use std::fs::OpenOptions; +use std::io::Error; +use std::os::unix::io::AsRawFd; + +use crate::Attester; + +// Length of the REPORTDATA used in TDG.MR.REPORT TDCALL +const TDX_REPORTDATA_LEN: usize = 64; + +// Length of TDREPORT used in TDG.MR.REPORT TDCALL +const TDX_REPORT_LEN: usize = 1024; + +#[repr(C)] +#[derive(Debug)] +pub struct TdxReportReq { + reportdata: [u8; TDX_REPORTDATA_LEN], + tdreport: [u8; TDX_REPORT_LEN], +} + +impl TdxReportReq { + pub fn new(reportdata: [u8; TDX_REPORTDATA_LEN]) -> Self { + Self { + reportdata, + tdreport: [0; TDX_REPORT_LEN], + } + } +} + +#[derive(Serialize, Deserialize)] +struct AzTdQuoteRequest { + report: String, +} +#[derive(Serialize, Deserialize, Debug)] +struct AzTdQuoteResponse { + quote: String, +} + +impl Default for TdxReportReq { + fn default() -> Self { + Self { + reportdata: [0; TDX_REPORTDATA_LEN], + tdreport: [0; TDX_REPORT_LEN], + } + } +} + +ioctl!(readwrite tdx_cmd_get_report0 with b'T', 0x01; TdxReportReq); + +pub(super) async fn get_hyperv_tdx_evidence(report_data: &[u8]) -> Result { + let file = OpenOptions::new().write(true).open("/dev/tdx_guest")?; + let fd = file.as_raw_fd(); + let mut tdx_req = TdxReportReq::new(report_data.try_into()?); + unsafe { + let err = tdx_cmd_get_report0(fd, &mut tdx_req); + if err != 0 { + bail!("TDX Attester: ioctl failed: {}", Error::last_os_error()); + } + } + let report = base64::encode_config(tdx_req.tdreport, base64::URL_SAFE_NO_PAD); + let tdquotereq = AzTdQuoteRequest { report }; + let req = reqwest::Client::new() + .post("http://169.254.169.254/acc/tdquote") + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header(reqwest::header::ACCEPT, "application/json") + .body(serde_json::to_string(&tdquotereq)?) + .send() + .await?; + let tdquoteresp = req.json::().await?; + let quote = base64::decode_config(tdquoteresp.quote, base64::URL_SAFE_NO_PAD)?; + let evidence = super::TdxEvidence { + cc_eventlog: None, + quote: base64::encode(quote), + }; + serde_json::to_string(&evidence).context("TDX Attester: Failed to serialize evidence") +} + +pub(super) fn detect_platform() -> bool { + // check cpuid if we are in a Hyper-V guest + let cpuid = CpuId::new(); + if let Some(hypervisor) = cpuid.get_hypervisor_info() { + hypervisor.identify() == Hypervisor::HyperV + } else { + false + } +} + +#[derive(Debug, Default)] +pub struct TdxAttester {} + +#[async_trait::async_trait] +impl Attester for TdxAttester { + async fn get_evidence(&self, report_data: String) -> Result { + let report_data_bin = super::convert_report_data(report_data)?; + get_hyperv_tdx_evidence(&report_data_bin).await + } +} diff --git a/attestation-agent/attester/src/tdx/mod.rs b/attestation-agent/attester/src/tdx/mod.rs index 7fdd60ef4..c512b56d7 100644 --- a/attestation-agent/attester/src/tdx/mod.rs +++ b/attestation-agent/attester/src/tdx/mod.rs @@ -9,6 +9,8 @@ use raw_cpuid::cpuid; use serde::{Deserialize, Serialize}; use tdx_attest_rs; +mod az; + const CCEL_PATH: &str = "/sys/firmware/acpi/tables/data/CCEL"; pub fn detect_platform() -> bool { @@ -28,19 +30,30 @@ struct TdxEvidence { quote: String, } +pub fn make_attester() -> Box { + if az::detect_platform() { + Box::::default() + } else { + Box::::default() + } +} + +fn convert_report_data(report_data: String) -> Result> { + let mut report_data_bin = base64::decode(report_data)?; + if report_data_bin.len() != 48 { + bail!("TDX Attester: Report data should be SHA384 base64 String"); + } + report_data_bin.extend([0; 16]); + Ok(report_data_bin) +} + #[derive(Debug, Default)] pub struct TdxAttester {} #[async_trait::async_trait] impl Attester for TdxAttester { async fn get_evidence(&self, report_data: String) -> Result { - let mut report_data_bin = base64::decode(report_data)?; - if report_data_bin.len() != 48 { - return Err(anyhow!( - "TDX Attester: Report data should be SHA384 base64 String" - )); - } - report_data_bin.extend([0; 16]); + let report_data_bin = convert_report_data(report_data)?; let tdx_report_data = tdx_attest_rs::tdx_report_data_t { d: report_data_bin.as_slice().try_into()?,