handle the missing mlmd service-ca cert gracefully

When mlmd is being reconciled, it expects a secret with certs that is
created by the service-ca (when podtopodtls is enabled). This is an
expected outcome, so we should not be reporting stacktraces for this.
This change instead catches this scenario via a custom error for such
lagging dependencies, and logs it at info level without a stack trace.

Signed-off-by: Humair Khan <HumairAK@users.noreply.github.com>

controllers/dspipeline_controller.go

@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/opendatahub-io/data-science-pipelines-operator/controllers/dspastatus"
@@ -313,7 +314,14 @@ func (r *DSPAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 		err = r.ReconcileMLMD(ctx, dspa, params)
 		if err != nil {
 			r.setStatusAsNotReady(config.MLMDProxyReady, err, dspaStatus.SetMLMDProxyStatus)
-			return ctrl.Result{}, err
+			// TODO: this (and other components) should handle these scenarios via states or statuses instead of error
+			var depErr *util.LaggingDependencyCreationError
+			if errors.As(err, &depErr) {
+				log.Info(depErr.Message)
+				return ctrl.Result{}, nil
+			} else {
+				return ctrl.Result{}, err
+			}
 		} else {
 			r.setStatus(ctx, params.MlmdProxyDefaultResourceName, config.MLMDProxyReady, dspa,
 				dspaStatus.SetMLMDProxyStatus, log)

controllers/mlmd.go

@@ -17,8 +17,8 @@ package controllers
 
 import (
 	"context"
-	"errors"
 	dspav1alpha1 "github.com/opendatahub-io/data-science-pipelines-operator/api/v1alpha1"
+	"github.com/opendatahub-io/data-science-pipelines-operator/controllers/util"
 )
 
 const (
@@ -72,9 +72,8 @@ func (r *DSPAReconciler) ReconcileMLMD(ctx context.Context, dsp *dspav1alpha1.Da
 			if err != nil {
 				return err
 			}
-
 			if !certificatesExist {
-				return errors.New("secret containing the certificate for MLMD gRPC Server was not created yet")
+				return &util.LaggingDependencyCreationError{Message: "MLMD gRPC Server cert secret not found, this is likely because it has not been created yet"}
 			}
 		}
 

controllers/util/err.go

@@ -0,0 +1,31 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+)
+
+// LaggingDependencyCreationError should be used if a dependency that is
+// created by a third party is not found (e.g. service-ca secrets).
+type LaggingDependencyCreationError struct {
+	Message string
+}
+
+func (e *LaggingDependencyCreationError) Error() string {
+	return fmt.Sprintf("Missing dependency error: %s", e.Message)
+}