Integrate vendor VFIO vGPUs into the instance lifecycle

cmd/api/api/instances.go

@@ -362,7 +362,18 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 
 	inst, err := s.InstanceManager.CreateInstance(ctx, domainReq)
 	if err != nil {
+		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
+		// Checked first: it wraps the original create error, so a later
+		// errors.Is case would match the cause and hide the pending vGPU cleanup.
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			message, inner := vgpuCleanupPendingDetail(vgpuPending, "create", "delete it to retry")
+			return oapi.CreateInstance500JSONResponse{
+				Code:       "vgpu_cleanup_pending",
+				Message:    message,
+				InnerError: inner,
+			}, nil
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
 				Code:    "image_not_ready",
@@ -424,6 +435,22 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 	return oapi.CreateInstance201JSONResponse(instanceToOAPI(*inst)), nil
 }
 
+// vgpuCleanupPendingDetail renders a pending vGPU cleanup into the message
+// and inner error detail shared by the create and start handlers. The
+// retained guidance names the verb-specific way to release the assignment.
+func vgpuCleanupPendingDetail(pending *instances.VGPUCleanupPendingError, action, retainedGuidance string) (string, *oapi.ErrorDetail) {
+	message := fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and instance %s retains the assignment, %s", action, pending.Err, pending.InstanceID, retainedGuidance)
+	innerCode := "vgpu_retained_instance"
+	if !pending.Retained {
+		message = fmt.Sprintf("failed to %s instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", action, pending.Err, pending.InstanceID)
+		innerCode = "vgpu_unretained_instance"
+	}
+	return message, &oapi.ErrorDetail{
+		Code:    lo.ToPtr(innerCode),
+		Message: lo.ToPtr(pending.InstanceID),
+	}
+}
+
 // GetInstance gets instance details
 // The id parameter can be an instance ID, name, or ID prefix
 // Note: Resolution is handled by ResolveResource middleware
@@ -823,7 +850,18 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan
 
 	result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq)
 	if err != nil {
+		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
+		// Checked first: it wraps the original start error, so a later
+		// errors.Is case would match the cause and hide the pending vGPU cleanup.
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to start instance", "error", err)
+			message, inner := vgpuCleanupPendingDetail(vgpuPending, "start", "delete it or retry start to release it")
+			return oapi.StartInstance500JSONResponse{
+				Code:       "vgpu_cleanup_pending",
+				Message:    message,
+				InnerError: inner,
+			}, nil
 		case errors.Is(err, instances.ErrInvalidState):
 			return oapi.StartInstance409JSONResponse{
 				Code:    "invalid_state",
@@ -1258,6 +1296,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance {
 		if inst.GPUMdevUUID != "" {
 			gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID)
 		}
+		if inst.GPUDevicePath != "" {
+			gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath)
+		}
 		oapiInst.Gpu = gpu
 	}
 

cmd/api/api/instances_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/instances/phasetracking"
 	mw "github.com/kernel/hypeman/lib/middleware"
+	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/paths"
 	restartpolicy "github.com/kernel/hypeman/lib/restart-policy"
@@ -47,6 +48,73 @@ func TestGetInstance_NotFound(t *testing.T) {
 	require.Error(t, err)
 }
 
+type createErrorInstanceManager struct {
+	instances.Manager
+	err error
+}
+
+func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) {
+	return nil, m.err
+}
+
+// A retained-assignment error must win over the mapping of the create error
+// it wraps, or the response omits the instance the caller has to delete.
+func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+		InstanceID: "inst-1",
+		Retained:   true,
+		Err:        network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "inst-1")
+	assert.Contains(t, pending.Message, network.ErrNameExists.Error(),
+		"the underlying create failure must survive the cleanup guidance")
+	assert.Contains(t, pending.Message, "delete it to retry")
+	require.NotNil(t, pending.InnerError)
+	require.NotNil(t, pending.InnerError.Code)
+	assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code)
+	require.NotNil(t, pending.InnerError.Message)
+	assert.Equal(t, "inst-1", *pending.InnerError.Message)
+}
+
+func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+		InstanceID: "inst-1",
+		Err:        network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+	assert.Contains(t, pending.Message, network.ErrNameExists.Error(),
+		"the underlying create failure must survive the cleanup guidance")
+	assert.Contains(t, pending.Message, "startup reconcile")
+	assert.NotContains(t, pending.Message, "delete")
+	require.NotNil(t, pending.InnerError)
+	require.NotNil(t, pending.InnerError.Code)
+	assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code)
+	require.NotNil(t, pending.InnerError.Message)
+	assert.Equal(t, "inst-1", *pending.InnerError.Message)
+}
+
 func TestCreateInstance_AutoPullImage(t *testing.T) {
 	t.Parallel()
 	if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {
@@ -923,6 +991,68 @@ func (m *errActionInstanceManager) RestoreSnapshot(context.Context, string, stri
 	return nil, m.err
 }
 
+// A retained-assignment error must win over the mapping of the start error
+// it wraps, or the response omits the pending vGPU cleanup the caller has to
+// resolve.
+func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+
+	resolved := &instances.Instance{
+		StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"},
+		State:          instances.StateStopped,
+	}
+
+	t.Run("retained", func(t *testing.T) {
+		t.Parallel()
+		svc := newTestService(t)
+		svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{
+			InstanceID: "inst-1",
+			Retained:   true,
+			Err:        fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources),
+		}}
+
+		resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id})
+		require.NoError(t, rerr)
+
+		pending, ok := resp.(oapi.StartInstance500JSONResponse)
+		require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+		assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+		assert.Contains(t, pending.Message, "inst-1")
+		assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(),
+			"the underlying start failure must survive the cleanup guidance")
+		assert.Contains(t, pending.Message, "delete it or retry start")
+		require.NotNil(t, pending.InnerError)
+		require.NotNil(t, pending.InnerError.Code)
+		assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code)
+		require.NotNil(t, pending.InnerError.Message)
+		assert.Equal(t, "inst-1", *pending.InnerError.Message)
+	})
+
+	t.Run("unretained", func(t *testing.T) {
+		t.Parallel()
+		svc := newTestService(t)
+		svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{
+			InstanceID: "inst-1",
+			Err:        fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources),
+		}}
+
+		resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id})
+		require.NoError(t, rerr)
+
+		pending, ok := resp.(oapi.StartInstance500JSONResponse)
+		require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+		assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+		assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+		assert.Contains(t, pending.Message, "startup reconcile")
+		assert.NotContains(t, pending.Message, "delete")
+		require.NotNil(t, pending.InnerError)
+		require.NotNil(t, pending.InnerError.Code)
+		assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code)
+		require.NotNil(t, pending.InnerError.Message)
+		assert.Equal(t, "inst-1", *pending.InnerError.Message)
+	})
+}
+
 func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) {
 	t.Parallel()
 

cmd/api/main.go

@@ -185,6 +185,63 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta
 	}, logger), nil
 }
 
+func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) {
+	allInstances, err := instanceManager.ListInstancesForReconcile(ctx)
+	if err != nil {
+		return nil, 0, err
+	}
+	protected := make(map[string]struct{})
+	var retryAfter time.Duration
+	for _, inst := range allInstances {
+		if inst.GPUDevicePath == "" {
+			continue
+		}
+		if inst.HypervisorPID != nil && instances.HypervisorMayBeAlive(inst.HypervisorProcessIdentity, inst.SocketPath) {
+			protected[inst.GPUDevicePath] = struct{}{}
+			continue
+		}
+		if inst.GPUAssignedAt == nil {
+			continue
+		}
+		remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt)
+		if remaining <= 0 {
+			continue
+		}
+		protected[inst.GPUDevicePath] = struct{}{}
+		if retryAfter == 0 || remaining < retryAfter {
+			retryAfter = remaining
+		}
+	}
+	return protected, retryAfter, nil
+}
+
+func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) {
+	protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager)
+	if err != nil {
+		// Operator-actionable: vendor VFIO reconciliation stays disabled
+		// host-wide (and releases fail closed on the same inventory) until
+		// the unreadable instance metadata is repaired.
+		logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err)
+		protected = nil
+		retryAfter = 0
+	}
+	if err := devices.ReconcileVGPUs(ctx, protected); err != nil {
+		logger.Warn("failed to reconcile vGPU devices", "error", err)
+	}
+	if retryAfter <= 0 {
+		return
+	}
+	go func() {
+		timer := time.NewTimer(retryAfter)
+		defer timer.Stop()
+		select {
+		case <-ctx.Done():
+		case <-timer.C:
+			reconcileVGPUs(ctx, instanceManager, logger)
+		}
+	}()
+}
+
 func run() error {
 	startupStarted := time.Now()
 	slog.Info("starting hypeman initialization")
@@ -384,12 +441,9 @@ func run() error {
 		return fmt.Errorf("reconcile device state: %w", err)
 	}
 
-	// Reconcile mdev devices (clears orphaned vGPUs from previous runs)
-	logger.Info("Reconciling mdev devices...")
-	if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil {
-		// Log but don't fail - mdev cleanup is best-effort
-		logger.Warn("failed to reconcile mdev devices", "error", err)
-	}
+	// Reconcile vGPU devices (clears orphaned vGPUs from previous runs)
+	logger.Info("Reconciling vGPU devices...")
+	reconcileVGPUs(ctx, app.InstanceManager, logger)
 
 	// Wire up resource validator for aggregate limit checking
 	// This enables the instance manager to validate CPU, memory, network, and GPU

cmd/api/main_test.go

@@ -2,15 +2,18 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os/exec"
 	"testing"
 	"time"
 
 	"github.com/getkin/kin-openapi/openapi3filter"
 	"github.com/go-chi/chi/v5"
 	"github.com/golang-jwt/jwt/v5"
+	"github.com/kernel/hypeman/lib/instances"
 	mw "github.com/kernel/hypeman/lib/middleware"
 	"github.com/kernel/hypeman/lib/oapi"
 	nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware"
@@ -338,3 +341,38 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) {
 		})
 	}
 }
+
+type vgpuReconcileManagerStub struct {
+	instances.Manager
+	list []instances.Instance
+}
+
+func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) {
+	return s.list, nil
+}
+
+func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) {
+	dead := exec.Command("true")
+	require.NoError(t, dead.Run())
+	deadPID := dead.Process.Pid
+	recent := time.Now().Add(-time.Minute)
+	stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute)
+
+	manager := vgpuReconcileManagerStub{list: []instances.Instance{
+		{StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}},
+		{StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}},
+		{StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}},
+		{StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}}},
+		{StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorProcessIdentity: instances.HypervisorProcessIdentity{HypervisorPID: &deadPID}, GPUAssignedAt: &recent}},
+	}}
+
+	protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
+	require.NoError(t, err)
+	require.Positive(t, retryAfter)
+	require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod)
+	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7")
+	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8")
+}

integration/vgpu_test.go

@@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) {
 	if framework == devices.VGPUFrameworkNone {
 		return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", ""
 	}
-	if framework == devices.VGPUFrameworkVendorVFIO {
-		// CreateVGPU rejects vendor VFIO until the instance lifecycle
-		// integration lands.
-		return "vGPU test requires the vendor VFIO instance lifecycle integration", ""
-	}
 
 	// Check for available profiles
 	profiles, err := devices.ListGPUProfiles()

lib/builds/manager_test.go

@@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc
 	return result, nil
 }
 
+func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) {
+	return m.ListInstances(ctx, nil)
+}
+
 func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) {
 	return nil, nil
 }