fix the deadlock

controller/monitor.go

@@ -69,7 +69,8 @@ type MonitorMgr struct {
 	ctrl     *Controller
 	consul   *ConsulMon
 
-	sync.Mutex
+	monMu sync.Mutex
+	clMu  sync.Mutex
 }
 
 func NewMonitor(config *c.Config) *MonitorMgr {
@@ -123,7 +124,7 @@ func (m *MonitorMgr) consulMon() {
 			}
 			// remove currently running apps that are not discovered in this pass
 			var toRemove []string
-			m.Lock()
+			m.monMu.Lock()
 			for name, mon := range m.monitors {
 				if mon.app.Source != "consul" {
 					continue
@@ -140,10 +141,10 @@ func (m *MonitorMgr) consulMon() {
 					toRemove = append(toRemove, name)
 				}
 			}
+			m.monMu.Unlock()
 			for _, tr := range toRemove {
 				m.Remove(tr)
 			}
-			m.Unlock()
 		}
 		<-time.After(m.config.Agent.ConsulQueryInterval)
 	}
@@ -152,8 +153,7 @@ func (m *MonitorMgr) consulMon() {
 // Add adds a new app into monitor manager
 func (m *MonitorMgr) Add(app *App) {
 	// check if already running
-	m.Lock()
-	defer m.Unlock()
+	m.monMu.Lock()
 	for _, appMon := range m.monitors {
 		if appMon.app.Equal(app) && appMon.checkOn {
 			glog.V(2).Infof("App %s already exists", app.Name)
@@ -164,6 +164,7 @@ func (m *MonitorMgr) Add(app *App) {
 			return
 		}
 	}
+	m.monMu.Unlock()
 	m.Remove(app.Name)
 	appMon := &appMon{app: app, done: make(chan bool)}
 	m.monitors[app.Name] = appMon
@@ -174,6 +175,8 @@ func (m *MonitorMgr) Add(app *App) {
 // Remove removes an app from monitor manager, stops BGP
 /// announcement and cleans up state
 func (m *MonitorMgr) Remove(appName string) {
+	m.monMu.Lock()
+	defer m.monMu.Unlock()
 	if a, ok := m.monitors[appName]; ok {
 		if a.checkOn {
 			a.done <- true
@@ -223,8 +226,8 @@ func (m *MonitorMgr) runMonitors(app *App) bool {
 
 func (m *MonitorMgr) checkCond(am *appMon) error {
 	app := am.app
-	m.Lock()
-	defer m.Unlock()
+	m.clMu.Lock()
+	defer m.clMu.Unlock()
 	if m.runMonitors(app) {
 		glog.V(2).Infof("All Monitors for app: %s succeeded", app.Name)
 		if !am.announced {
@@ -246,6 +249,7 @@ func (m *MonitorMgr) checkCond(am *appMon) error {
 			am.announced = true
 			if exit, ok := m.cleanups[app.Name]; ok {
 				exit <- true
+				delete(m.cleanups, app.Name)
 			}
 		}
 	} else {
@@ -313,9 +317,7 @@ func (m *MonitorMgr) Cleanup(app string, exit chan bool) {
 		select {
 		case <-t.C:
 			glog.Infof("Cleaning up app %s", app)
-			m.Lock()
 			m.Remove(app)
-			m.Unlock()
 		case <-exit:
 			return
 		}