Nat support and fix monitoring issues

controller/app.go

@@ -52,6 +52,7 @@ type App struct {
 	Name     string
 	Vip      *net.IPNet
 	Monitors Monitors
+	Nats     []string
 }
 
 func (a *App) Equal(other *App) bool {
@@ -66,42 +67,37 @@ func (a *App) Equal(other *App) bool {
 	return a.Name == other.Name && a.Vip.String() == other.Vip.String()
 }
 
-func (a *App) needsNatRule() (bool, *Monitor) {
-	for _, m := range a.Monitors {
-		if m.Type == Monitor_CONSUL && m.Port != "" {
-			return true, m
-		}
-	}
-	return false, nil
-}
-
-func NewApp(appName, vip string, monitors []string) (*App, error) {
+func NewApp(appName, vip string, monitors []string, nats []string) (*App, error) {
 	if appName == "" {
 		return nil, fmt.Errorf("Invalid app name")
 	}
-	app := &App{Name: appName}
+	app := &App{Name: appName, Nats: nats}
 	_, ipnet, err := net.ParseCIDR(vip)
 	if err != nil {
 		return nil, fmt.Errorf("Invalid VIP specified, need ip/mask")
 	}
 	app.Vip = ipnet
 	for _, m := range monitors {
+		// valid monitor formats:
+		// "port:tcp:123" , "exec:/local/check.sh", "consul"
 		parts := strings.Split(m, ":")
-		if len(parts) != 2 && len(parts) != 3 {
-			glog.Errorf("Invalid monitor specified, ignoring")
-			continue
-		}
 		mon := &Monitor{Type: MonitorMap[parts[0]]}
 		switch mon.Type.String() {
 		case "port":
+			if len(parts) != 3 {
+				return nil, fmt.Errorf("Invalid port monitor, must specify proto:port")
+			}
 			mon.Protocol = parts[1]
 			mon.Port = parts[2]
 		case "exec":
+			if len(parts) != 2 {
+				return nil, fmt.Errorf("Invalid exec monitor, must specify command")
+			}
 			mon.Cmd = parts[1]
 		case "consul":
-			glog.V(2).Infof("Using consul health monitor")
+			glog.V(2).Infof("Will use consul healthcheck monitor")
 		default:
-			glog.V(2).Infof("No monitor specified")
+			glog.V(2).Infof("Invalid monitor specified")
 		}
 		app.Monitors = append(app.Monitors, mon)
 	}

controller/consul.go

@@ -65,6 +65,7 @@ func (c *ConsulMon) queryServices() ([]*App, error) {
 		var (
 			vip      string
 			monitors []string
+			nats     []string
 		)
 		for _, tag := range service.Tags {
 			// try to find the requires tags. Only vip is mandatory
@@ -73,17 +74,19 @@ func (c *ConsulMon) queryServices() ([]*App, error) {
 				continue
 			}
 			switch parts[0] {
-			case "vip":
+			case "gocast_vip":
 				vip = parts[1]
-			case "monitor":
+			case "gocast_monitor":
 				monitors = append(monitors, parts[1])
+			case "gocast_nat":
+				nats = append(nats, parts[1])
 			}
 		}
 		if vip == "" {
 			glog.Errorf("No vip Tag found in matched service :%s", service.Service)
 			continue
 		}
-		app, err := NewApp(service.Service, vip, monitors)
+		app, err := NewApp(service.Service, vip, monitors, nats)
 		if err != nil {
 			glog.Errorf("Unable to add consul app: %v", err)
 			continue

controller/monitor.go

@@ -7,6 +7,7 @@ import (
 	api "github.com/osrg/gobgp/api"
 	"net"
 	"os/exec"
+	"strings"
 	"sync"
 	"time"
 )
@@ -137,6 +138,7 @@ func (m *MonitorMgr) Add(app *App) {
 	defer m.Unlock()
 	for _, appMon := range m.monitors {
 		if appMon.app.Equal(app) && appMon.checkOn {
+			glog.V(2).Infof("App %s already exists", app.Name)
 			return
 		}
 		if appMon.app.Vip.String() == app.Vip.String() && appMon.app.Name != app.Name {
@@ -161,16 +163,24 @@ func (m *MonitorMgr) Remove(appName string) {
 				glog.Errorf("Failed to withdraw route: %v", err)
 			}
 		}
-		deleteLoopback(a.app.Vip)
-		if ok, mon := a.app.needsNatRule(); ok {
-			natRule("D", a.app.Vip.IP, m.ctrl.localIP, mon.Port, mon.Protocol)
+		if err := deleteLoopback(a.app.Vip); err != nil {
+			glog.Errorf("Failed to remove app: %s: %v", a.app.Name, err)
+		}
+		for _, nat := range a.app.Nats {
+			parts := strings.Split(nat, ":")
+			if len(parts) != 2 {
+				continue
+			}
+			if err := natRule("D", a.app.Vip.IP, m.ctrl.localIP, parts[0], parts[1]); err != nil {
+				glog.Errorf("Failed to remove app: %s: %v", a.app.Name, err)
+			}
 		}
 	}
 	delete(m.monitors, appName)
 }
 func (m *MonitorMgr) runMonitors(app *App) bool {
-	var check bool
 	for _, mon := range app.Monitors {
+		var check bool
 		switch mon.Type {
 		case Monitor_PORT:
 			check = portMonitor(mon.Protocol, mon.Port)
@@ -201,8 +211,12 @@ func (m *MonitorMgr) checkCond(am *appMon) error {
 			if err := addLoopback(app.Name, app.Vip); err != nil {
 				return err
 			}
-			if ok, mon := app.needsNatRule(); ok {
-				if err := natRule("A", app.Vip.IP, m.ctrl.localIP, mon.Port, mon.Protocol); err != nil {
+			for _, nat := range app.Nats {
+				parts := strings.Split(nat, ":")
+				if len(parts) != 2 {
+					continue
+				}
+				if err := natRule("A", app.Vip.IP, m.ctrl.localIP, parts[0], parts[1]); err != nil {
 					return err
 				}
 			}
@@ -258,8 +272,12 @@ func (m *MonitorMgr) CloseAll() {
 			am.done <- true
 		}
 		deleteLoopback(am.app.Vip)
-		if ok, mon := am.app.needsNatRule(); ok {
-			natRule("D", am.app.Vip.IP, m.ctrl.localIP, mon.Port, mon.Protocol)
+		for _, nat := range am.app.Nats {
+			parts := strings.Split(nat, ":")
+			if len(parts) != 2 {
+				continue
+			}
+			natRule("D", am.app.Vip.IP, m.ctrl.localIP, parts[0], parts[1])
 		}
 	}
 }

controller/system.go

@@ -33,6 +33,7 @@ func localAddress(gw net.IP) (net.IP, error) {
 }
 
 func addLoopback(name string, addr *net.IPNet) error {
+	deleteLoopback(addr)
 	prefixLen, _ := addr.Mask.Size()
 	label := fmt.Sprintf("lo:%s", name)
 	// linux kernel limits labels to 15 chars
@@ -57,7 +58,7 @@ func deleteLoopback(addr *net.IPNet) error {
 	return nil
 }
 
-func natRule(op string, vip, localAddr net.IP, port, protocol string) error {
+func natRule(op string, vip, localAddr net.IP, protocol, port string) error {
 	cmd := fmt.Sprintf(
 		"iptables -t nat -%s PREROUTING -p %s -d %s --dport %s -j DNAT --to-destination %s:%s",
 		op, protocol, vip.String(), port, localAddr.String(), port,