feat(providers): expose actionable OAuth refresh failures

.agents/skills/openshell-cli/SKILL.md

@@ -207,6 +207,18 @@ output. Use `provider refresh rotate`, reconfigure refresh, or delete refresh
 before returning those keys to manual management. Unrelated provider fields
 remain updateable.
 
+When OAuth refresh fails, inspect the `RECOVERY` and `FAILURE_CODE` columns from
+`provider refresh status`; do not infer the remedy from HTTP status or parse
+`LAST_ERROR`. `retry` means the worker will try again, `reauthorize` means the
+user must obtain a new OAuth grant and run `provider refresh configure`,
+`fix_configuration` means an operator must repair the OAuth client, scopes, or
+administrator policy, and `investigate` means the issuer returned an
+unrecognized response. The gateway parks `reauthorize` records until a manual
+rotate or reconfiguration. It retries
+`fix_configuration` records hourly so externally repaired configuration can
+recover without rapid token-endpoint traffic. The existing access credential
+remains usable only until its recorded expiry.
+
 ---
 
 ## Workflow 3: Sandbox Lifecycle

.github/workflows/e2e-test.yml

@@ -157,6 +157,12 @@ jobs:
             podman_package_version: "5.7.0+ds2-3build1"
             conmon_package_version: "2.1.13+ds1-2"
             cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:podman:external-driver"
+          - suite: provider-refresh-keycloak
+            runner: ubuntu-26.04
+            podman_major: "5"
+            podman_package_version: "5.7.0+ds2-3build1"
+            conmon_package_version: "2.1.13+ds1-2"
+            cmd: "mise run --no-deps --skip-deps e2e:provider-refresh-keycloak"
     env:
       IMAGE_TAG: ${{ inputs.image-tag }}
       MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

architecture/gateway.md

@@ -343,6 +343,15 @@ Multi-replica deployments can use that default with a shared database and
 shared key-encryption key, or opt into an external backend such as Vault or
 Kubernetes Secrets.
 
+OAuth refresh failures retain a gateway-owned recovery classification alongside
+the refresh state. The gateway reads only a bounded error response and maps
+recognized OAuth codes to retry, reauthorization, configuration repair, or
+investigation without persisting issuer-controlled descriptions. Terminal
+reauthorization failures remain parked until a manual retry or explicit refresh
+reconfiguration. Configuration failures retry hourly so an externally repaired
+clock, policy, or stored credential can recover without rapid endpoint traffic;
+short-lived credentials still fail closed at their recorded expiry.
+
 Credential handles remain bound to the driver that created them. Before the
 0.1.0 compatibility boundary, gateways do not migrate inline refresh material
 or move handles between credential drivers; operators reconfigure affected

crates/openshell-cli/src/run.rs

@@ -46,7 +46,8 @@ use openshell_core::proto::{
     GetSandboxRequest, GetServiceRequest, GpuResourceRequirements, ImportProviderProfilesRequest,
     LintProviderProfilesRequest, ListProviderProfilesRequest, ListProvidersRequest,
     ListSandboxPoliciesRequest, ListSandboxProvidersRequest, ListSandboxesRequest,
-    ListServicesRequest, PolicySource, PolicyStatus, Provider, ProviderCredentialRefreshStatus,
+    ListServicesRequest, PolicySource, PolicyStatus, Provider,
+    ProviderCredentialRefreshRecoveryAction, ProviderCredentialRefreshStatus,
     ProviderCredentialRefreshStrategy, ProviderCredentialTokenGrantType, ProviderProfile,
     ProviderProfileDiagnostic, ProviderProfileImportItem, RejectDraftChunkRequest,
     ResourceRequirements, RevokeSshSessionRequest, RotateProviderCredentialRequest, Sandbox,
@@ -4338,14 +4339,16 @@ pub async fn provider_refresh_status(
 
 fn refresh_status_header() -> String {
     format!(
-        "{:<24}  {:<28}  {:<28}  {:<18}  {:<20}  {:<20}  {:<20}  {}",
+        "{:<24}  {:<28}  {:<28}  {:<24}  {:<18}  {:<20}  {:<20}  {:<20}  {:<44}  {}",
         "PROVIDER".bold(),
         "CREDENTIAL_KEY".bold(),
         "STRATEGY".bold(),
         "STATUS".bold(),
+        "RECOVERY".bold(),
         "EXPIRES_AT".bold(),
         "NEXT_REFRESH".bold(),
         "LAST_REFRESH".bold(),
+        "FAILURE_CODE".bold(),
         "LAST_ERROR".bold(),
     )
 }
@@ -4497,19 +4500,43 @@ fn print_refresh_status_row(status: &ProviderCredentialRefreshStatus) {
 fn refresh_status_row(status: &ProviderCredentialRefreshStatus) -> String {
     let strategy = ProviderCredentialRefreshStrategy::try_from(status.strategy)
         .unwrap_or(ProviderCredentialRefreshStrategy::Unspecified);
+    let recovery_action = ProviderCredentialRefreshRecoveryAction::try_from(status.recovery_action)
+        .unwrap_or(ProviderCredentialRefreshRecoveryAction::Unspecified);
     format!(
-        "{:<24}  {:<28}  {:<28}  {:<18}  {:<20}  {:<20}  {:<20}  {}",
+        "{:<24}  {:<28}  {:<28}  {:<24}  {:<18}  {:<20}  {:<20}  {:<20}  {:<44}  {}",
         status.provider_name,
         status.credential_key,
         provider_refresh_strategy_name(strategy),
         status.status,
+        provider_refresh_recovery_action_name(recovery_action),
         format_optional_epoch_ms(status.expires_at_ms),
-        format_optional_epoch_ms(status.next_refresh_at_ms),
+        format_refresh_next_at_ms(status.next_refresh_at_ms),
         format_optional_epoch_ms(status.last_refresh_at_ms),
+        status.failure_code,
         truncate_status_field(&status.last_error, 72),
     )
 }
 
+fn format_refresh_next_at_ms(next_refresh_at_ms: i64) -> String {
+    if next_refresh_at_ms == i64::MAX {
+        "-".to_string()
+    } else {
+        format_optional_epoch_ms(next_refresh_at_ms)
+    }
+}
+
+fn provider_refresh_recovery_action_name(
+    action: ProviderCredentialRefreshRecoveryAction,
+) -> &'static str {
+    match action {
+        ProviderCredentialRefreshRecoveryAction::Retry => "retry",
+        ProviderCredentialRefreshRecoveryAction::Reauthorize => "reauthorize",
+        ProviderCredentialRefreshRecoveryAction::FixConfiguration => "fix_configuration",
+        ProviderCredentialRefreshRecoveryAction::Investigate => "investigate",
+        ProviderCredentialRefreshRecoveryAction::Unspecified => "-",
+    }
+}
+
 fn provider_refresh_strategy_name(strategy: ProviderCredentialRefreshStrategy) -> &'static str {
     match strategy {
         ProviderCredentialRefreshStrategy::Static => "static",
@@ -7403,10 +7430,11 @@ mod tests {
     };
     use openshell_core::proto::{
         GetSandboxConfigResponse, GpuResourceRequirements, PolicySource, PolicyStatus, Provider,
-        ProviderCredentialRefresh, ProviderCredentialRefreshStatus,
-        ProviderCredentialRefreshStrategy, ProviderCredentialTokenGrant, ProviderProfile,
-        ProviderProfileCredential, ResourceRequirements, Sandbox, SandboxCondition, SandboxPhase,
-        SandboxPolicyRevision, SandboxStatus, datamodel::v1::ObjectMeta,
+        ProviderCredentialRefresh, ProviderCredentialRefreshRecoveryAction,
+        ProviderCredentialRefreshStatus, ProviderCredentialRefreshStrategy,
+        ProviderCredentialTokenGrant, ProviderProfile, ProviderProfileCredential,
+        ResourceRequirements, Sandbox, SandboxCondition, SandboxPhase, SandboxPolicyRevision,
+        SandboxStatus, datamodel::v1::ObjectMeta,
     };
 
     #[test]
@@ -7645,6 +7673,8 @@ mod tests {
         let header = refresh_status_header();
         assert!(header.contains("NEXT_REFRESH"));
         assert!(header.contains("LAST_REFRESH"));
+        assert!(header.contains("RECOVERY"));
+        assert!(header.contains("FAILURE_CODE"));
         assert!(header.contains("LAST_ERROR"));
 
         let row = refresh_status_row(&ProviderCredentialRefreshStatus {
@@ -7654,17 +7684,24 @@ mod tests {
             strategy: ProviderCredentialRefreshStrategy::Oauth2ClientCredentials as i32,
             status: "error".to_string(),
             expires_at_ms: 1_767_225_600_000,
-            next_refresh_at_ms: 1_767_225_660_000,
+            next_refresh_at_ms: i64::MAX,
             last_refresh_at_ms: 1_767_225_000_000,
             last_error: "token endpoint returned a very long error message that should be truncated for table readability"
                 .to_string(),
+            recovery_action: ProviderCredentialRefreshRecoveryAction::Reauthorize as i32,
+            failure_code: "oauth_rotated_refresh_token_handle_missing".to_string(),
+            provider_error_subtype: "invalid_rapt".to_string(),
+            last_error_at_ms: 1_767_225_000_000,
         });
 
         assert!(row.contains("my-graph"));
         assert!(row.contains("MS_GRAPH_ACCESS_TOKEN"));
         assert!(row.contains("oauth2_client_credentials"));
         assert!(row.contains("error"));
+        assert!(row.contains("reauthorize"));
+        assert!(row.contains("oauth_rotated_refresh_token_handle_missing"));
         assert!(row.contains("2026-01-01 00:00:00"));
+        assert!(!row.contains("292278994"));
         assert!(row.contains("..."));
     }
 

crates/openshell-cli/tests/provider_commands_integration.rs

@@ -750,6 +750,10 @@ impl OpenShell for TestOpenShell {
             next_refresh_at_ms: 0,
             last_refresh_at_ms: 0,
             last_error: String::new(),
+            recovery_action: 0,
+            failure_code: String::new(),
+            provider_error_subtype: String::new(),
+            last_error_at_ms: 0,
         };
         drop(providers);
         self.state

crates/openshell-server/src/grpc/provider.rs

@@ -4755,10 +4755,11 @@ mod tests {
         ImportProviderProfilesRequest, L7Allow, L7Rule, LintProviderProfilesRequest,
         ListProviderProfilesRequest, ListProvidersRequest, NetworkBinary, NetworkEndpoint,
         NetworkPolicyRule, ProviderCredentialRefresh, ProviderCredentialRefreshMaterial,
-        ProviderCredentialTokenGrant, ProviderCredentialTokenGrantAudienceOverride,
-        ProviderProfile, ProviderProfileCategory, ProviderProfileCredential,
-        ProviderProfileImportItem, RotateProviderCredentialRequest, Sandbox, SandboxPolicy,
-        SandboxSpec, StoredProviderProfile, UpdateProviderProfilesRequest, UpdateProviderRequest,
+        ProviderCredentialRefreshRecoveryAction, ProviderCredentialTokenGrant,
+        ProviderCredentialTokenGrantAudienceOverride, ProviderProfile, ProviderProfileCategory,
+        ProviderProfileCredential, ProviderProfileImportItem, RotateProviderCredentialRequest,
+        Sandbox, SandboxPolicy, SandboxSpec, StoredProviderProfile, UpdateProviderProfilesRequest,
+        UpdateProviderRequest,
     };
     use openshell_core::{ObjectId, ObjectName};
     use tonic::{Code, Request};
@@ -11988,7 +11989,16 @@ mod tests {
         .await
         .unwrap()
         .expect("refresh state should exist");
-        assert_eq!(refresh_state.status, "error");
+        assert_eq!(refresh_state.status, "configuration_required");
+        assert_eq!(
+            refresh_state.recovery_action,
+            ProviderCredentialRefreshRecoveryAction::FixConfiguration as i32
+        );
+        assert_eq!(refresh_state.failure_code, "refresh_configuration_invalid");
+        assert_eq!(
+            refresh_state.next_refresh_at_ms - refresh_state.last_error_at_ms,
+            60 * 60 * 1000
+        );
     }
 
     #[tokio::test]