From 718cea7160ac188c93456f5683400d9d14bafc91 Mon Sep 17 00:00:00 2001 From: Yuan Peng Date: Tue, 20 Feb 2024 15:23:40 +0800 Subject: [PATCH 1/2] Only update when NodeLocalStorage status changed --- pkg/agent/discovery/discovery.go | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/pkg/agent/discovery/discovery.go b/pkg/agent/discovery/discovery.go index 776f4f43..11607d59 100644 --- a/pkg/agent/discovery/discovery.go +++ b/pkg/agent/discovery/discovery.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "os" + "reflect" "regexp" "strconv" "strings" @@ -149,16 +150,35 @@ func (d *Discoverer) Discover() { d.createSpdkBdevs(&nlsCopy.Status.FilteredStorageInfo.Devices) } - // only update status - log.Infof("update nls %s", nlsCopy.Name) - _, err = d.localclientset.CsiV1alpha1().NodeLocalStorages().UpdateStatus(context.Background(), nlsCopy, metav1.UpdateOptions{}) - if err != nil { - log.Errorf("local storage CRD updateStatus error: %s", err.Error()) - return + // only update when NodeLocalStorage status changed, reduce pressure to apiserver in large cluster + oldStatus := &nls.Status + newStatusCopy := nlsCopy.Status.DeepCopy() + if isNlsStatusChanged(oldStatus, newStatusCopy) { + // only update status + log.Infof("update nls %s", nlsCopy.Name) + _, err = d.localclientset.CsiV1alpha1().NodeLocalStorages().UpdateStatus(context.Background(), nlsCopy, metav1.UpdateOptions{}) + if err != nil { + log.Errorf("local storage CRD updateStatus error: %s", err.Error()) + return + } + } else { + log.Infof("nls %s status not changed", nlsCopy.Name) } } } +func isNlsStatusChanged(oldStatus, newStatus *localv1alpha1.NodeLocalStorageStatus) bool { + // Remove time-related fields and then compare + oldStatus.NodeStorageInfo.State.LastHeartbeatTime = nil + oldStatus.NodeStorageInfo.State.LastTransitionTime = nil + oldStatus.FilteredStorageInfo.UpdateStatus.LastUpdateTime = nil + newStatus.NodeStorageInfo.State.LastHeartbeatTime = nil + newStatus.NodeStorageInfo.State.LastTransitionTime = nil + newStatus.FilteredStorageInfo.UpdateStatus.LastUpdateTime = nil + + return !reflect.DeepEqual(oldStatus, newStatus) +} + // Create AIO bdevs func (d *Discoverer) createSpdkBdevs(devices *[]string) { var found bool From 6bc30467881d192bdb3fed8a21471dcb7b4ab471 Mon Sep 17 00:00:00 2001 From: Yuan Peng Date: Tue, 20 Feb 2024 15:27:48 +0800 Subject: [PATCH 2/2] Reduce pressure to apiserver in large cluster The checkSPDKSupport frequency is too high, which puts a lot of pressure on apiserver in large cluster --- pkg/agent/discovery/discovery_test.go | 116 ++++++++++++++++++++++++++ pkg/csi/nodeserver.go | 2 +- 2 files changed, 117 insertions(+), 1 deletion(-) diff --git a/pkg/agent/discovery/discovery_test.go b/pkg/agent/discovery/discovery_test.go index 0ffd941c..963ea026 100644 --- a/pkg/agent/discovery/discovery_test.go +++ b/pkg/agent/discovery/discovery_test.go @@ -17,7 +17,10 @@ limitations under the License. package discovery import ( + localv1alpha1 "github.com/alibaba/open-local/pkg/apis/storage/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "testing" + "time" ) func TestFilterInfo(t *testing.T) { @@ -81,3 +84,116 @@ func sameStringSlice(x, y []string) bool { } return len(diff) == 0 } + +func TestIsNlsStatusChanged(t *testing.T) { + type args struct { + oldStatus *localv1alpha1.NodeLocalStorageStatus + newStatus *localv1alpha1.NodeLocalStorageStatus + } + updatedTime := metav1.Now() + oldStatus := &localv1alpha1.NodeLocalStorageStatus{ + NodeStorageInfo: localv1alpha1.NodeStorageInfo{ + DeviceInfos: []localv1alpha1.DeviceInfo{ + { + Name: "/dev/sdb", + MediaType: "hdd", + Total: 100000000, + }, + }, + VolumeGroups: []localv1alpha1.VolumeGroup{ + { + Name: "pool0", + PhysicalVolumes: []string{ + "/dev/sdb", + }, + Total: 100000000, + Allocatable: 100000000, + Available: 100000000, + }, + }, + Phase: "Running", + State: localv1alpha1.StorageState{ + Status: localv1alpha1.ConditionTrue, + Type: localv1alpha1.StorageReady, + }, + }, + FilteredStorageInfo: localv1alpha1.FilteredStorageInfo{ + UpdateStatus: localv1alpha1.UpdateStatusInfo{ + LastUpdateTime: &updatedTime, + Status: "accepted", + }, + VolumeGroups: []string{"pool0"}, + }, + } + + newStatus := &localv1alpha1.NodeLocalStorageStatus{ + NodeStorageInfo: localv1alpha1.NodeStorageInfo{ + DeviceInfos: []localv1alpha1.DeviceInfo{ + { + Name: "/dev/sdb", + MediaType: "hdd", + Total: 100000000, + }, + }, + VolumeGroups: []localv1alpha1.VolumeGroup{ + { + Name: "pool0", + PhysicalVolumes: []string{ + "/dev/sdb", + }, + Total: 100000000, + Allocatable: 50000000, // changed + Available: 50000000, // changed + }, + }, + Phase: "Running", + State: localv1alpha1.StorageState{ + Status: localv1alpha1.ConditionTrue, + Type: localv1alpha1.StorageReady, + }, + }, + FilteredStorageInfo: localv1alpha1.FilteredStorageInfo{ + UpdateStatus: localv1alpha1.UpdateStatusInfo{ + LastUpdateTime: &updatedTime, + Status: "accepted", + }, + VolumeGroups: []string{"pool0"}, + }, + } + + newStatusOnlyTimeChanged := oldStatus + newUpdateTime := updatedTime.Add(1*time.Minute) + newStatusOnlyTimeChanged.FilteredStorageInfo.UpdateStatus.LastUpdateTime = &metav1.Time{Time: newUpdateTime} + newStatusOnlyTimeChanged.NodeStorageInfo.State.LastTransitionTime = &metav1.Time{Time: newUpdateTime} + newStatusOnlyTimeChanged.NodeStorageInfo.State.LastHeartbeatTime = &metav1.Time{Time: newUpdateTime} + + tests := []struct { + name string + args args + want bool + }{ + { + name: "test1", + args: args{ + oldStatus: oldStatus, + newStatus: newStatus, + }, + want: true, + }, + { + name: "test2", + args: args{ + oldStatus: oldStatus, + newStatus: newStatusOnlyTimeChanged, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isNlsStatusChanged(tt.args.oldStatus, tt.args.newStatus); got != tt.want { + t.Errorf("isNlsStatusChanged() = %v, want %v", got, tt.want) + } + }) + } +} \ No newline at end of file diff --git a/pkg/csi/nodeserver.go b/pkg/csi/nodeserver.go index 7d0d64a4..98c3d435 100644 --- a/pkg/csi/nodeserver.go +++ b/pkg/csi/nodeserver.go @@ -723,6 +723,6 @@ func (ns *nodeServer) checkSPDKSupport() { } } - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Second * 5) } }