[PLAT-15725] User visible tasks submitted by background jobs must hav…

…e their own correlation ID such that UI can pull the task logs correctly

Summary: Generate a new correlation ID for each task submission to track only that task instead of using the one in the background job.

Test Plan:
1. Manually created a 5 node, RF3 universe.
2. Made auto-master failover to run by stopping a node.
3. Check the log by clicking on the task.

{F300261}

Reviewers: cwang, yshchetinin, amalyshev, sanketh, anijhawan

Reviewed By: amalyshev

Subscribers: yugaware

Differential Revision: https://phorge.dev.yugabyte.com/D39081

managed/src/main/java/com/yugabyte/yw/commissioner/AutoMasterFailover.java

@@ -15,6 +15,7 @@
 import com.yugabyte.yw.commissioner.tasks.subtasks.CheckClusterConsistency;
 import com.yugabyte.yw.commissioner.tasks.subtasks.CheckFollowerLag;
 import com.yugabyte.yw.common.CustomerTaskManager;
+import com.yugabyte.yw.common.Util;
 import com.yugabyte.yw.common.config.UniverseConfKeys;
 import com.yugabyte.yw.common.nodeui.MetricGroup;
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
@@ -155,8 +156,10 @@ public Optional<TaskInfo> maybeFailoverMaster(
         log.debug(
             "Retrying task {} for universe {}", action.getTaskType(), universe.getUniverseUUID());
         customerTask =
-            customerTaskManager.retryCustomerTask(
-                customer.getUuid(), universeDetails.placementModificationTaskUuid);
+            Util.doWithCorrelationId(
+                id ->
+                    customerTaskManager.retryCustomerTask(
+                        customer.getUuid(), universeDetails.placementModificationTaskUuid));
       }
       if (customerTask == null) {
         return Optional.empty();
@@ -504,19 +507,22 @@ private CustomerTask submitMasterFailoverTask(
     taskParams.clusters = universe.getUniverseDetails().clusters;
     taskParams.rootCA = universe.getUniverseDetails().rootCA;
     // Submit the task to initiate master failover.
-    UUID taskUUID = getCommissioner().submit(TaskType.MasterFailover, taskParams);
-    log.info(
-        "Submitted master failover for universe {} node {}, task uuid = {}.",
-        universe.getUniverseUUID(),
-        action.getNodeName(),
-        taskUUID);
-    return CustomerTask.create(
-        customer,
-        universe.getUniverseUUID(),
-        taskUUID,
-        CustomerTask.TargetType.Universe,
-        CustomerTask.TaskType.MasterFailover,
-        universe.getName());
+    return Util.doWithCorrelationId(
+        id -> {
+          UUID taskUUID = getCommissioner().submit(TaskType.MasterFailover, taskParams);
+          log.info(
+              "Submitted master failover for universe {} node {}, task uuid = {}.",
+              universe.getUniverseUUID(),
+              action.getNodeName(),
+              taskUUID);
+          return CustomerTask.create(
+              customer,
+              universe.getUniverseUUID(),
+              taskUUID,
+              CustomerTask.TargetType.Universe,
+              CustomerTask.TaskType.MasterFailover,
+              universe.getName());
+        });
   }
 
   private CustomerTask submitSyncMasterAddressesTask(Customer customer, Universe universe) {
@@ -525,18 +531,21 @@ private CustomerTask submitSyncMasterAddressesTask(Customer customer, Universe u
     taskParams.expectedUniverseVersion = universe.getVersion();
     taskParams.clusters = universe.getUniverseDetails().clusters;
     taskParams.rootCA = universe.getUniverseDetails().rootCA;
-    UUID taskUUID = getCommissioner().submit(TaskType.SyncMasterAddresses, taskParams);
-    log.info(
-        "Submitted sync master addresses task {} for universe {}",
-        taskUUID,
-        universe.getUniverseUUID());
-    return CustomerTask.create(
-        customer,
-        universe.getUniverseUUID(),
-        taskUUID,
-        CustomerTask.TargetType.Universe,
-        CustomerTask.TaskType.SyncMasterAddresses,
-        universe.getName());
+    return Util.doWithCorrelationId(
+        id -> {
+          UUID taskUUID = getCommissioner().submit(TaskType.SyncMasterAddresses, taskParams);
+          log.info(
+              "Submitted sync master addresses task {} for universe {}",
+              taskUUID,
+              universe.getUniverseUUID());
+          return CustomerTask.create(
+              customer,
+              universe.getUniverseUUID(),
+              taskUUID,
+              CustomerTask.TargetType.Universe,
+              CustomerTask.TaskType.SyncMasterAddresses,
+              universe.getName());
+        });
   }
 
   /**

managed/src/main/java/com/yugabyte/yw/commissioner/NodeAgentInstallerImpl.java

@@ -9,6 +9,7 @@
 import com.yugabyte.yw.commissioner.tasks.subtasks.InstallNodeAgent;
 import com.yugabyte.yw.common.ImageBundleUtil;
 import com.yugabyte.yw.common.NodeAgentClient;
+import com.yugabyte.yw.common.Util;
 import com.yugabyte.yw.common.config.GlobalConfKeys;
 import com.yugabyte.yw.common.config.RuntimeConfGetter;
 import com.yugabyte.yw.forms.UniverseDefinitionTaskParams;
@@ -101,18 +102,21 @@ public boolean migrate(UUID customerUuid, UUID universeUuid) {
     Customer customer = Customer.getOrBadRequest(customerUuid);
     UniverseDefinitionTaskParams params = new UniverseDefinitionTaskParams();
     params.setUniverseUUID(universeUuid);
-    UUID taskUuid = commissioner.submit(TaskType.EnableNodeAgentInUniverse, params);
-    CustomerTask customerTask =
-        CustomerTask.create(
-            customer,
-            universeUuid,
-            taskUuid,
-            CustomerTask.TargetType.Universe,
-            CustomerTask.TaskType.EnableNodeAgent,
-            universeOpt.get().getName());
-    commissioner.waitForTask(customerTask.getTaskUUID());
-    TaskInfo taskInfo = TaskInfo.getOrBadRequest(taskUuid);
-    return taskInfo.getTaskState() == TaskInfo.State.Success;
+    return Util.doWithCorrelationId(
+        id -> {
+          UUID taskUuid = commissioner.submit(TaskType.EnableNodeAgentInUniverse, params);
+          CustomerTask customerTask =
+              CustomerTask.create(
+                  customer,
+                  universeUuid,
+                  taskUuid,
+                  CustomerTask.TargetType.Universe,
+                  CustomerTask.TaskType.EnableNodeAgent,
+                  universeOpt.get().getName());
+          commissioner.waitForTask(customerTask.getTaskUUID());
+          TaskInfo taskInfo = TaskInfo.getOrBadRequest(taskUuid);
+          return taskInfo.getTaskState() == TaskInfo.State.Success;
+        });
   }
 
   private InstallNodeAgent.Params createInstallParams(

managed/src/main/java/com/yugabyte/yw/common/CustomerTaskManager.java

@@ -306,6 +306,7 @@ public void handlePendingTask(CustomerTask customerTask, TaskInfo taskInfo) {
                   customerTask.resetCompletionTime();
                   customerTask.setCorrelationId(corrId);
                   commitTransaction();
+                  return customerTask;
                 } catch (Exception e) {
                   throw new RuntimeException(
                       "Unable to delete the previous task info: " + taskUUID);

managed/src/main/java/com/yugabyte/yw/common/Util.java

@@ -76,7 +76,6 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
@@ -1368,7 +1367,7 @@ public static <T> Predicate<T> not(Predicate<T> t) {
     return t.negate();
   }
 
-  public static void doWithCorrelationId(Consumer<String> consumer) {
+  public static <T> T doWithCorrelationId(Function<String, T> function) {
     Map<String, String> originalContext = MDC.getCopyOfContextMap();
     try {
       String correlationId = UUID.randomUUID().toString();
@@ -1378,7 +1377,7 @@ public static void doWithCorrelationId(Consumer<String> consumer) {
       }
       context.put(LogUtil.CORRELATION_ID, correlationId);
       MDC.setContextMap(context);
-      consumer.accept(correlationId);
+      return function.apply(correlationId);
     } finally {
       if (MapUtils.isEmpty(originalContext)) {
         MDC.clear();