mysqlcluster: when storage size shrink, and change replicas, It will reconcile fail forever. #436

Author: acekingke

mysqlcluster/syncer/statefulset.go

@@ -47,6 +47,9 @@ import (
 // The wait time limit for pod upgrade.
 const waitLimit = 2 * 60 * 60
 
+var startTime time.Time
+var failCount int = 0
+
 // StatefulSetSyncer used to operate statefulset.
 type StatefulSetSyncer struct {
 	*mysqlcluster.MysqlCluster
@@ -146,6 +149,7 @@ func (s *StatefulSetSyncer) Sync(ctx context.Context) (syncer.SyncResult, error)
 		result.SetEventData("Normal", basicEventReason(s.Name, err),
 			fmt.Sprintf("%s %s %s successfully", kind, key, result.Operation))
 		log.Info(string(result.Operation), "key", key, "kind", kind)
+		failCount = 0
 	}
 	return result, err
 }
@@ -167,8 +171,22 @@ func (s *StatefulSetSyncer) canExpandPVC(ctx context.Context) bool {
 	// If newStorage is not greater than oldStorage, do not expand.
 	if newStorage.Cmp(*oldRequest.Storage()) != 1 {
 		log.Info("canExpandPVC", "result", "can not expand", "reason", "new pvc is not larger than old pvc")
-		return false
+		// Should return true, because if change replicas and storage size, it will fail again and again
+		// during expanding PVCs, not great then oldStorage it will not truely expand, just recreate
+		// statefulset.
+		if failCount > 0 && time.Since(startTime) > time.Duration(30)*time.Second {
+			log.Info("canExpandPVC", "result", "reason", "over 30 seconds, trying to expand")
+			failCount = 0
+			return true
+		} else {
+			if failCount == 0 {
+				startTime = time.Now()
+			}
+			failCount++
+			return false
+		}
 	}
+	failCount = 0
 	return true
 }
 

sidecar/config.go

@@ -333,7 +333,7 @@ func (cfg *Config) buildXenonConf() []byte {
 			"admin": "root",
 			"ping-timeout": %d,
 			"passwd": "%s",
-			"host": "localhost",
+			"host": "127.0.0.1",
 			"version": "%s",
 			"master-sysvars": "%s",
 			"slave-sysvars": "%s",