Merge branch 'master' into gtid-recovery

openark · Mar 28, 2017 · fdaa885 · fdaa885
2 parents 0f728d4 + 0333dfa
commit fdaa885
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 5 deletions.
diff --git a/go/inst/instance_topology_dao.go b/go/inst/instance_topology_dao.go
@@ -246,16 +246,21 @@ func StopSlaveNicely(instanceKey *InstanceKey, timeout time.Duration) (*Instance
 		return instance, fmt.Errorf("instance is not a replica: %+v", instanceKey)
 	}
 
-	_, err = ExecInstanceNoPrepare(instanceKey, `stop slave io_thread`)
-	_, err = ExecInstanceNoPrepare(instanceKey, `start slave sql_thread`)
+	// stop io_thread, start sql_thread but catch any errors
+	for _, cmd := range []string{`stop slave io_thread`, `start slave sql_thread`} {
+		if _, err = ExecInstanceNoPrepare(instanceKey, cmd); err != nil {
+			return nil, log.Errorf("%+v: StopSlaveNicely: %q failed: %+v", *instanceKey, cmd, err)
+		}
+	}
 
 	if instance.SQLDelay == 0 {
 		// Otherwise we don't bother.
 		startTime := time.Now()
 		for upToDate := false; !upToDate; {
-			if timeout > 0 && time.Since(startTime) >= timeout {
+			timeSinceStartTime := time.Since(startTime)
+			if timeout > 0 && timeSinceStartTime >= timeout {
 				// timeout
-				return nil, log.Errorf("StopSlaveNicely timeout on %+v", *instanceKey)
+				return nil, log.Errorf("%+v: StopSlaveNicely timeout after %+v", *instanceKey, timeSinceStartTime)
 			}
 			instance, err = ReadTopologyInstance(instanceKey)
 			if err != nil {

diff --git a/go/logic/orchestrator.go b/go/logic/orchestrator.go
@@ -45,6 +45,7 @@ var discoveryQueue *discovery.Queue
 
 var discoveriesCounter = metrics.NewCounter()
 var failedDiscoveriesCounter = metrics.NewCounter()
+var instancePollSecondsExceededCounter = metrics.NewCounter()
 var discoveryQueueLengthGauge = metrics.NewGauge()
 var discoveryRecentCountGauge = metrics.NewGauge()
 var isElectedGauge = metrics.NewGauge()
@@ -57,6 +58,7 @@ var recentDiscoveryOperationKeys *cache.Cache
 func init() {
 	metrics.Register("discoveries.attempt", discoveriesCounter)
 	metrics.Register("discoveries.fail", failedDiscoveriesCounter)
+	metrics.Register("discoveries.instance_poll_seconds_exceeded", instancePollSecondsExceededCounter)
 	metrics.Register("discoveries.queue_length", discoveryQueueLengthGauge)
 	metrics.Register("discoveries.recent_count", discoveryRecentCountGauge)
 	metrics.Register("elect.is_elected", isElectedGauge)
@@ -146,7 +148,8 @@ func discoverInstance(instanceKey inst.InstanceKey) {
 		latency.Stop("total")
 		discoveryTime := latency.Elapsed("total")
 		if discoveryTime > instancePollSecondsDuration() {
-			log.Warningf("discoverInstance for key %v took %.4fs", instanceKey, discoveryTime.Seconds())
+			instancePollSecondsExceededCounter.Inc(1)
+			log.Warningf("discoverInstance exceeded InstancePollSeconds for %+v, took %.4fs", instanceKey, discoveryTime.Seconds())
 		}
 	}()