From 6fdff2871601c128a66664fd834cb49cb7f97497 Mon Sep 17 00:00:00 2001 From: Mayuresh Gaitonde Date: Mon, 18 Oct 2021 15:50:13 -0700 Subject: [PATCH 1/3] fix the deadlock --- controller/monitor.go | 20 +++++++++++--------- server/server.go | 2 -- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/controller/monitor.go b/controller/monitor.go index 392e984..41a62e6 100644 --- a/controller/monitor.go +++ b/controller/monitor.go @@ -69,7 +69,8 @@ type MonitorMgr struct { ctrl *Controller consul *ConsulMon - sync.Mutex + monMu sync.Mutex + clMu sync.Mutex } func NewMonitor(config *c.Config) *MonitorMgr { @@ -123,7 +124,7 @@ func (m *MonitorMgr) consulMon() { } // remove currently running apps that are not discovered in this pass var toRemove []string - m.Lock() + m.monMu.Lock() for name, mon := range m.monitors { if mon.app.Source != "consul" { continue @@ -140,10 +141,10 @@ func (m *MonitorMgr) consulMon() { toRemove = append(toRemove, name) } } + m.monMu.Unlock() for _, tr := range toRemove { m.Remove(tr) } - m.Unlock() } <-time.After(m.config.Agent.ConsulQueryInterval) } @@ -152,8 +153,7 @@ func (m *MonitorMgr) consulMon() { // Add adds a new app into monitor manager func (m *MonitorMgr) Add(app *App) { // check if already running - m.Lock() - defer m.Unlock() + m.monMu.Lock() for _, appMon := range m.monitors { if appMon.app.Equal(app) && appMon.checkOn { glog.V(2).Infof("App %s already exists", app.Name) @@ -164,6 +164,7 @@ func (m *MonitorMgr) Add(app *App) { return } } + m.monMu.Unlock() m.Remove(app.Name) appMon := &appMon{app: app, done: make(chan bool)} m.monitors[app.Name] = appMon @@ -174,6 +175,8 @@ func (m *MonitorMgr) Add(app *App) { // Remove removes an app from monitor manager, stops BGP /// announcement and cleans up state func (m *MonitorMgr) Remove(appName string) { + m.monMu.Lock() + defer m.monMu.Unlock() if a, ok := m.monitors[appName]; ok { if a.checkOn { a.done <- true @@ -223,8 +226,8 @@ func (m *MonitorMgr) runMonitors(app *App) bool { func (m *MonitorMgr) checkCond(am *appMon) error { app := am.app - m.Lock() - defer m.Unlock() + m.clMu.Lock() + defer m.clMu.Unlock() if m.runMonitors(app) { glog.V(2).Infof("All Monitors for app: %s succeeded", app.Name) if !am.announced { @@ -246,6 +249,7 @@ func (m *MonitorMgr) checkCond(am *appMon) error { am.announced = true if exit, ok := m.cleanups[app.Name]; ok { exit <- true + delete(m.cleanups, app.Name) } } } else { @@ -313,9 +317,7 @@ func (m *MonitorMgr) Cleanup(app string, exit chan bool) { select { case <-t.C: glog.Infof("Cleaning up app %s", app) - m.Lock() m.Remove(app) - m.Unlock() case <-exit: return } diff --git a/server/server.go b/server/server.go index 75edb6c..bf50c36 100644 --- a/server/server.go +++ b/server/server.go @@ -68,9 +68,7 @@ func (s *Server) unregisterHandler(w http.ResponseWriter, r *http.Request) { http.Error(w, "Invalid request, need app name specified", http.StatusBadRequest) return } - s.mon.Lock() s.mon.Remove(appName[0]) - s.mon.Unlock() } func (s *Server) infoHandler(w http.ResponseWriter, r *http.Request) { From 5ac02c373ba379572bf566ff91688536e628466d Mon Sep 17 00:00:00 2001 From: Mayuresh Gaitonde Date: Mon, 18 Oct 2021 18:16:22 -0700 Subject: [PATCH 2/3] dockerfile fix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bcfdb58..1a3bc67 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:alpine as builder +FROM golang:1.14-alpine as builder RUN apk update && \ apk upgrade && \ apk add --no-cache git && \ From a99f92e9a5e7312e3b224ddd8186e5297b11b907 Mon Sep 17 00:00:00 2001 From: Mayuresh Gaitonde Date: Mon, 18 Oct 2021 18:21:01 -0700 Subject: [PATCH 3/3] fix --- controller/monitor.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controller/monitor.go b/controller/monitor.go index 41a62e6..bd0708a 100644 --- a/controller/monitor.go +++ b/controller/monitor.go @@ -157,10 +157,12 @@ func (m *MonitorMgr) Add(app *App) { for _, appMon := range m.monitors { if appMon.app.Equal(app) && appMon.checkOn { glog.V(2).Infof("App %s already exists", app.Name) + m.monMu.Unlock() return } if appMon.app.Vip.Net.String() == app.Vip.Net.String() && appMon.app.Name != app.Name { glog.Errorf("Error: Vip %s is already being announced by app: %s", app.Vip.Net.String(), appMon.app.Name) + m.monMu.Unlock() return } }