Make control client retry able in e2e tests

tools/integration_tests/util/client/control_client.go

@@ -18,14 +18,45 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"reflect"
 	"strings"
+	"time"
 
 	control "cloud.google.com/go/storage/control/apiv2"
 	"cloud.google.com/go/storage/control/apiv2/controlpb"
+	"github.com/googleapis/gax-go/v2"
+	"google.golang.org/grpc/codes"
 )
 
+func storageControlClientRetryOptions(clientConfig *control.StorageControlClient) []gax.CallOption {
+	return []gax.CallOption{
+		gax.WithTimeout(300000 * time.Millisecond),
+		gax.WithRetry(func() gax.Retryer {
+			return gax.OnCodes([]codes.Code{
+				codes.ResourceExhausted,
+				codes.Unavailable,
+				codes.DeadlineExceeded,
+				codes.Internal,
+				codes.Unknown,
+			}, gax.Backoff{
+				Max:        30 * time.Second,
+				Multiplier: 2,
+			})
+		}),
+	}
+}
+
 func CreateControlClient(ctx context.Context) (client *control.StorageControlClient, err error) {
 	client, err = control.NewStorageControlClient(ctx)
+	opts := &control.StorageControlCallOptions{}
+
+	// Loop over all fields in the struct and assign retry options
+	v := reflect.ValueOf(opts).Elem()
+	for i := 0; i < v.NumField(); i++ {
+		field := v.Field(i)
+		field.Set(reflect.ValueOf(storageControlClientRetryOptions(client)))
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf("control.NewStorageControlClient: #{err}")
 	}