VMware Support

.gitignore

@@ -1,4 +1,6 @@
 /bin
 /test/e2e/e2e.test
 /test/e2e/ginkgo
+/cloudstack.ini
+/.idea
 cloud-config

README.md

@@ -40,6 +40,7 @@ api-url = <CloudStack API URL>
 api-key = <CloudStack API Key>
 secret-key = <CloudStack API Secret>
 ssl-no-verify = <Disable SSL certificate validation: true or false (optional)>
+project-id = <project ID>
 ```
 
 Create a secret named `cloudstack-secret` in namespace `kube-system`:
@@ -51,6 +52,18 @@ kubectl create secret generic \
   cloudstack-secret
 ```
 
+Set the correct hypervisor in the DaemonSet Env Vars:
+```
+            - name: NODE_HYPERVISOR
+              value: vmware
+```
+
+You can manually set the maximal attachable number of block volumes per node:
+```
+            - name: NODE_MAX_BLOCK_VOLUMES
+              value: "15" #Default value is 10 volumes per node
+```
+
 If you have also deployed the [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider),
 you may use the same secret for both tools.
 
@@ -88,11 +101,19 @@ disk offerings to Kubernetes storage classes.
 
 Example:
 
-```
+```bash
 kubectl apply -f ./examples/k8s/pvc.yaml
 kubectl apply -f ./examples/k8s/pod.yaml
 ```
 
+#### Reusing volumes
+
+1. Patch PV `reclaimPolicy` with `kubectl patch pv my-pv-name -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'`
+2. Delete Old Pod and PVC
+3. Patch PV `claimRef` with `kubectl patch pv my-pv-name -p '{"spec":{"claimRef": null}}'`
+4. Create new Pod and PVC with existing claimName `.spec.claimRef.name = my-pv-name`
+
+
 ## Building
 
 To build the driver binary:

clean-scsi-bus.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo 1 > /sys/class/scsi_device/0\:0\:$1\:0/device/delete

cmd/cloudstack-csi-driver/Dockerfile

@@ -11,7 +11,13 @@ RUN apk add --no-cache \
     # Provides mkfs.xfs
     xfsprogs \
     # Provides blkid, also used by k8s.io/mount-utils
-    blkid
+    blkid \
+    eudev \
+    bash
 
 COPY ./bin/cloudstack-csi-driver /cloudstack-csi-driver
+COPY rescan-scsi-bus.sh /usr/bin/
+RUN chmod +x /usr/bin/rescan-scsi-bus.sh
+COPY clean-scsi-bus.sh /usr/bin/
+RUN chmod +x /usr/bin/clean-scsi-bus.sh
 ENTRYPOINT ["/cloudstack-csi-driver"]

deploy/k8s/node-daemonset.yaml

@@ -2,101 +2,134 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: cloudstack-csi-node
-  namespace: kube-system
 spec:
+  revisionHistoryLimit: 10
   selector:
     matchLabels:
       app.kubernetes.io/name: cloudstack-csi-node
-  updateStrategy:
-    type: RollingUpdate
   template:
     metadata:
       labels:
         app.kubernetes.io/name: cloudstack-csi-node
         app.kubernetes.io/part-of: cloudstack-csi-driver
     spec:
-      nodeSelector:
-        kubernetes.io/os: linux
-      tolerations:
-        - effect: NoExecute
-          operator: Exists
-        - effect: NoSchedule
-          operator: Exists
-
       containers:
-        - name: cloudstack-csi-node
-          image: cloudstack-csi-driver
-          imagePullPolicy: Always
-          args:
-            - "-endpoint=$(CSI_ENDPOINT)"
-            - "-cloudstackconfig=/etc/cloudstack-csi-driver/cloud-config"
-            - "-nodeName=$(NODE_NAME)"
-            - "-debug"
+        - args:
+            - -endpoint=$(CSI_ENDPOINT)
+            - -cloudstackconfig=/etc/cloudstack-csi-driver/cloudstack.ini
+            - -nodeName=$(NODE_NAME)
+            - -debug
           env:
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
+            - name: NODE_HYPERVISOR
+              value: vmware
+            - name: NODE_MAX_BLOCK_VOLUMES
+              value: "10"
             - name: NODE_NAME
               valueFrom:
                 fieldRef:
+                  apiVersion: v1
                   fieldPath: spec.nodeName
+          image: cloudstack-csi-driver
+          imagePullPolicy: Always
+          name: cloudstack-csi-node
+          resources: {}
           securityContext:
             privileged: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
           volumeMounts:
-            - name: plugin-dir
-              mountPath: /csi
-            - name: kubelet-dir
-              mountPath: /var/lib/kubelet
-              # needed so that any mounts setup inside this container are
-              # propagated back to the host machine.
-              mountPropagation: Bidirectional
-            - name: device-dir
-              mountPath: /dev
-            - name: cloud-init-dir
-              mountPath: /run/cloud-init/
-            - name: cloudstack-conf
-              mountPath: /etc/cloudstack-csi-driver
-
-        - name: node-driver-registrar
-          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
-          imagePullPolicy: IfNotPresent
-          args:
-            - "--csi-address=$(ADDRESS)"
-            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
-            - "--v=5"
+          - mountPath: /csi
+            name: plugin-dir
+          - mountPath: /var/lib/kubelet
+            mountPropagation: Bidirectional
+            name: kubelet-dir
+          - mountPath: /dev
+            name: device-dir
+          - mountPath: /run/cloud-init/
+            name: cloud-init-dir
+          - mountPath: /etc/cloudstack-csi-driver
+            name: cloudstack-conf
+          - mountPath: /sys/class/scsi_host/
+            name: sys-class-scsi-host-dir
+          - mountPath: /sys/class/scsi_device/
+            name: sys-class-scsi-device-dir
+          - mountPath: /sys/devices
+            name: sys-devices
+        - args:
+            - --csi-address=$(ADDRESS)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --v=5
           env:
             - name: ADDRESS
               value: /csi/csi.sock
             - name: DRIVER_REG_SOCK_PATH
               value: /var/lib/kubelet/plugins/csi.cloudstack.apache.org/csi.sock
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.0.1
+          imagePullPolicy: IfNotPresent
+          name: node-driver-registrar
+          resources: {}
           securityContext:
             privileged: true
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
           volumeMounts:
-            - name: plugin-dir
-              mountPath: /csi
-            - name: registration-dir
-              mountPath: /registration
-
+            - mountPath: /csi
+              name: plugin-dir
+            - mountPath: /registration
+              name: registration-dir
+      dnsPolicy: ClusterFirst
+      nodeSelector:
+        kubernetes.io/os: linux
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      serviceAccount: cloudstack-csi-node
+      serviceAccountName: cloudstack-csi-node
+      terminationGracePeriodSeconds: 30
+      tolerations:
+        - effect: NoExecute
+          operator: Exists
+        - effect: NoSchedule
+          operator: Exists
       volumes:
-        - name: plugin-dir
-          hostPath:
+        - hostPath:
             path: /var/lib/kubelet/plugins/csi.cloudstack.apache.org/
             type: DirectoryOrCreate
-        - name: kubelet-dir
-          hostPath:
+          name: plugin-dir
+        - hostPath:
             path: /var/lib/kubelet
             type: Directory
-        - name: device-dir
-          hostPath:
+          name: kubelet-dir
+        - hostPath:
             path: /dev
             type: Directory
-        - name: registration-dir
-          hostPath:
+          name: device-dir
+        - hostPath:
             path: /var/lib/kubelet/plugins_registry
             type: Directory
-        - name: cloud-init-dir
-          hostPath:
+          name: registration-dir
+        - hostPath:
             path: /run/cloud-init/
             type: Directory
+          name: cloud-init-dir
+        - hostPath:
+            path: /sys/class/scsi_host
+            type: Directory
+          name: sys-class-scsi-host-dir
+        - hostPath:
+            path: /sys/class/scsi_device
+            type: Directory
+          name: sys-class-scsi-device-dir
+        - hostPath:
+            path: /sys/devices
+            type: Directory
+          name: sys-devices
         - name: cloudstack-conf
           secret:
-            secretName: cloudstack-secret
+            defaultMode: 420
+            secretName: csi-cloudstack-secret
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/hashicorp/go-uuid v1.0.2
 	github.com/kubernetes-csi/csi-test/v4 v4.2.0
 	go.uber.org/zap v1.16.0
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
 	golang.org/x/text v0.3.6
 	google.golang.org/genproto v0.0.0-20210726200206-e7812ac95cc0 // indirect
 	google.golang.org/grpc v1.39.0

pkg/cloud/cloud.go

@@ -16,9 +16,13 @@ type Interface interface {
 
 	ListZonesID(ctx context.Context) ([]string, error)
 
+	GetDomainID(ctx context.Context) (string, error)
+	GetProjectID() string
+
 	GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error)
 	GetVolumeByName(ctx context.Context, name string) (*Volume, error)
-	CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error)
+	ListVolumesForVM(ctx context.Context, virtualMachineID, projectID string) ([]*Volume, error)
+	CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error)
 	DeleteVolume(ctx context.Context, id string) error
 	AttachVolume(ctx context.Context, volumeID, vmID string) (string, error)
 	DetachVolume(ctx context.Context, volumeID string) error
@@ -37,6 +41,7 @@ type Volume struct {
 
 	VirtualMachineID string
 	DeviceID         string
+	Hypervisor       string
 }
 
 // VM represents a CloudStack Virtual Machine.
@@ -54,10 +59,11 @@ var (
 // client is the implementation of Interface.
 type client struct {
 	*cloudstack.CloudStackClient
+	ProjectID string
 }
 
 // New creates a new cloud connector, given its configuration.
 func New(config *Config) Interface {
 	csClient := cloudstack.NewAsyncClient(config.APIURL, config.APIKey, config.SecretKey, config.VerifySSL)
-	return &client{csClient}
+	return &client{csClient, config.ProjectID}
 }

pkg/cloud/config.go

@@ -12,6 +12,7 @@ type Config struct {
 	APIKey    string
 	SecretKey string
 	VerifySSL bool
+	ProjectID string
 }
 
 // csConfig wraps the config for the CloudStack cloud provider.
@@ -42,5 +43,6 @@ func ReadConfig(configFilePath string) (*Config, error) {
 		APIKey:    cfg.Global.APIKey,
 		SecretKey: cfg.Global.SecretKey,
 		VerifySSL: cfg.Global.SSLNoVerify,
+		ProjectID: cfg.Global.ProjectID,
 	}, nil
 }

pkg/cloud/fake/fake.go

@@ -73,7 +73,7 @@ func (f *fakeConnector) GetVolumeByName(ctx context.Context, name string) (*clou
 	return nil, cloud.ErrNotFound
 }
 
-func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) {
+func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, projectID, domainID, zoneID, name string, sizeInGB int64) (string, error) {
 	id, _ := uuid.GenerateUUID()
 	vol := cloud.Volume{
 		ID:             id,
@@ -101,3 +101,11 @@ func (f *fakeConnector) AttachVolume(ctx context.Context, volumeID, vmID string)
 }
 
 func (f *fakeConnector) DetachVolume(ctx context.Context, volumeID string) error { return nil }
+
+func (f *fakeConnector) GetDomainID(ctx context.Context) (string, error) {
+	return "domain", nil
+}
+
+func (f *fakeConnector) GetProjectID() string {
+	return "test"
+}

pkg/cloud/project.go

@@ -0,0 +1,23 @@
+package cloud
+
+import (
+	"context"
+	"github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap"
+)
+
+func (c *client) GetDomainID(ctx context.Context) (string, error) {
+	p, _, err := c.Project.GetProjectByID(c.ProjectID)
+	ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "GetProjectByID", "params", map[string]string{
+		"projectID": c.ProjectID,
+	})
+
+	if err != nil {
+		return "", err
+	}
+
+	return p.Domainid, nil
+}
+
+func (c *client) GetProjectID() string {
+	return c.ProjectID
+}

pkg/cloud/vms.go

@@ -23,6 +23,7 @@ func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) {
 		return nil, ErrTooManyResults
 	}
 	vm := l.VirtualMachines[0]
+
 	return &VM{
 		ID:     vm.Id,
 		ZoneID: vm.Zoneid,