ben/gocast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ADd consul support , multiple monitors, config file

Browse Source
This commit is contained in:
Mayuresh Gaitonde
2018-10-25 22:23:04 -07:00
parent e20f691de5
commit 6fafdbbd16
11 changed files with 441 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
controller/app.go
View File

@@ -10,14 +10,15 @@ import (
type MonitorType int
const (
Monitor_PORT MonitorType = 1
Monitor_EXEC MonitorType = 2
Monitor_PORT MonitorType = 1
Monitor_EXEC MonitorType = 2
Monitor_CONSUL MonitorType = 3
)
var Monitors = map[string]MonitorType{"port": Monitor_PORT, "exec": Monitor_EXEC}
var MonitorMap = map[string]MonitorType{"port": Monitor_PORT, "exec": Monitor_EXEC, "consul": Monitor_CONSUL}
func (m MonitorType) String() string {
for str, mtr := range Monitors {
for str, mtr := range MonitorMap {
if m == mtr {
return str
}
@@ -32,13 +33,49 @@ type Monitor struct {
Cmd string
}
type App struct {
Name string
Vip *net.IPNet
Monitor Monitor
func (m *Monitor) Equal(other *Monitor) bool {
return m.Type == other.Type && m.Port == other.Port && m.Protocol == other.Protocol && m.Cmd == other.Cmd
}
func NewApp(appName, vip, monitor, monitorType string) (*App, error) {
type Monitors []*Monitor
func (m Monitors) Contains(elem *Monitor) bool {
for _, mon := range m {
if mon.Equal(elem) {
return true
}
}
return false
}
type App struct {
Name string
Vip *net.IPNet
Monitors Monitors
}
func (a *App) Equal(other *App) bool {
if len(a.Monitors) != len(other.Monitors) {
return false
}
for _, m := range other.Monitors {
if !a.Monitors.Contains(m) {
return false
}
}
return a.Name == other.Name && a.Vip.String() == other.Vip.String()
}
func (a *App) needsNatRule() (bool, *Monitor) {
for _, m := range a.Monitors {
if m.Type == Monitor_CONSUL && m.Port != "" {
return true, m
}
}
return false, nil
}
func NewApp(appName, vip string, monitors []string) (*App, error) {
if appName == "" {
return nil, fmt.Errorf("Invalid app name")
}
@@ -48,17 +85,25 @@ func NewApp(appName, vip, monitor, monitorType string) (*App, error) {
return nil, fmt.Errorf("Invalid VIP specified, need ip/mask")
}
app.Vip = ipnet
m := Monitor{Type: Monitors[monitorType]}
switch monitorType {
case "port":
parts := strings.Split(monitor, ":")
m.Protocol = parts[0]
m.Port = parts[1]
case "exec":
m.Cmd = monitor
default:
glog.V(2).Infof("No monitor specified")
for _, m := range monitors {
parts := strings.Split(m, ":")
if len(parts) != 2 && len(parts) != 3 {
glog.Errorf("Invalid monitor specified, ignoring")
continue
}
mon := &Monitor{Type: MonitorMap[parts[0]]}
switch mon.Type.String() {
case "port":
mon.Protocol = parts[1]
mon.Port = parts[2]
case "exec":
mon.Cmd = parts[1]
case "consul":
glog.V(2).Infof("Using consul health monitor")
default:
glog.V(2).Infof("No monitor specified")
}
app.Monitors = append(app.Monitors, mon)
}
app.Monitor = m
return app, nil
}

42
controller/bgp.go
View File

@@ -5,31 +5,45 @@ import (
"fmt"
"github.com/golang/protobuf/ptypes"
"github.com/golang/protobuf/ptypes/any"
c "github.com/mayuresh82/gocast/config"
api "github.com/osrg/gobgp/api"
gobgp "github.com/osrg/gobgp/pkg/server"
"net"
"strconv"
"strings"
)
type Controller struct {
peerAS int
localIP, peerIP net.IP
communities []string
origin uint32
s *gobgp.BgpServer
}
func NewController(localAS, peerAS int, peerIP string) (*Controller, error) {
func NewController(config *c.Config) (*Controller, error) {
c := &Controller{}
if peerIP == "" {
if config.Bgp.PeerIP == "" {
gw, err := gateway()
if err != nil {
return nil, err
}
c.peerIP = gw
} else {
c.peerIP = net.ParseIP(peerIP)
c.peerIP = net.ParseIP(config.Bgp.PeerIP)
}
if c.peerIP == nil {
return nil, fmt.Errorf("Unable to get peer IP")
}
c.communities = config.Bgp.Communities
switch config.Bgp.Origin {
case "igp":
c.origin = 0
case "egp":
c.origin = 1
case "unknown":
c.origin = 2
}
s := gobgp.NewBgpServer()
go s.Serve()
localAddr, err := localAddress(c.peerIP)
@@ -39,7 +53,7 @@ func NewController(localAS, peerAS int, peerIP string) (*Controller, error) {
c.localIP = localAddr
if err := s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
As: uint32(localAS),
As: uint32(config.Bgp.LocalAS),
RouterId: localAddr.String(),
ListenPort: -1, // gobgp won't listen on tcp:179
},
@@ -47,7 +61,7 @@ func NewController(localAS, peerAS int, peerIP string) (*Controller, error) {
return nil, fmt.Errorf("Unable to start bgp: %v", err)
}
c.s = s
c.peerAS = peerAS
c.peerAS = config.Bgp.PeerAS
return c, nil
}
@@ -72,12 +86,19 @@ func (c *Controller) getApiPath(route *net.IPNet) *api.Path {
PrefixLen: uint32(prefixlen),
})
a1, _ := ptypes.MarshalAny(&api.OriginAttribute{
Origin: 0,
Origin: c.origin,
})
a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{
NextHop: c.localIP.String(),
})
attrs := []*any.Any{a1, a2}
var communities []uint32
for _, comm := range c.communities {
communities = append(communities, convertCommunity(comm))
}
a3, _ := ptypes.MarshalAny(&api.CommunitiesAttribute{
Communities: communities,
})
attrs := []*any.Any{a1, a2, a3}
return &api.Path{
Family: &api.Family{Afi: afi, Safi: api.Family_SAFI_UNICAST},
AnyNlri: nlri,
@@ -134,3 +155,10 @@ func (c *Controller) Shutdown() error {
}
return nil
}
func convertCommunity(comm string) uint32 {
parts := strings.Split(comm, ":")
first, _ := strconv.ParseUint(parts[0], 10, 32)
second, _ := strconv.ParseUint(parts[1], 10, 32)
return uint32(first)<<16 | uint32(second)
}

115
controller/consul.go Normal file
View File

@@ -0,0 +1,115 @@
package controller
import (
"encoding/json"
"fmt"
"github.com/golang/glog"
"net/http"
"os"
"strings"
)
const (
consulNodeEnv = "CONSUL_NODE"
matchTag = "enable_gocast"
nodeUrl = "/catalog/node"
healthCheckurl = "/health/checks"
)
type ConsulMon struct {
addr string
node string
}
type ConsulServiceData struct {
Services map[string]struct {
ID string
Service string
Tags []string
}
}
func contains(inp []string, elem string) bool {
for _, a := range inp {
if a == elem {
return true
}
}
return false
}
func NewConsulMon(addr string) (*ConsulMon, error) {
node := os.Getenv(consulNodeEnv)
if node == "" {
return nil, fmt.Errorf("%s env variable not set", consulNodeEnv)
}
return &ConsulMon{addr: addr, node: node}, nil
}
func (c *ConsulMon) queryServices() ([]*App, error) {
var apps []*App
addr := c.addr + fmt.Sprintf("%s/%s", nodeUrl, c.node)
resp, err := http.Get(addr)
if err != nil {
return apps, err
}
defer resp.Body.Close()
var consulData ConsulServiceData
if err := json.NewDecoder(resp.Body).Decode(&consulData); err != nil {
return apps, fmt.Errorf("Unable to decode consul data: %v", err)
}
for _, service := range consulData.Services {
if !contains(service.Tags, matchTag) {
continue
}
var (
vip string
monitors []string
)
for _, tag := range service.Tags {
// try to find the requires tags. Only vip is mandatory
parts := strings.Split(tag, "=")
if len(parts) != 2 {
continue
}
switch parts[0] {
case "vip":
vip = parts[1]
case "monitor":
monitors = append(monitors, parts[1])
}
}
if vip == "" {
glog.Errorf("No vip Tag found in matched service :%s", service.Service)
continue
}
app, err := NewApp(service.Service, vip, monitors)
if err != nil {
glog.Errorf("Unable to add consul app: %v", err)
continue
}
apps = append(apps, app)
}
return apps, nil
}
func (c *ConsulMon) healthCheck(service string) (bool, error) {
addr := c.addr + fmt.Sprintf("%s/%s", healthCheckurl, service)
resp, err := http.Get(addr)
if err != nil {
return false, err
}
defer resp.Body.Close()
var data []interface{}
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
return false, err
}
for _, nodeInfo := range data {
n := nodeInfo.(map[string]interface{})
if n["Node"] == c.node && n["Status"].(string) == "passing" {
return true, nil
}
}
return false, fmt.Errorf("No healcheck info found for node %s in consul", c.node)
}

170
controller/monitor.go
View File

@@ -3,6 +3,7 @@ package controller
import (
"fmt"
"github.com/golang/glog"
c "github.com/mayuresh82/gocast/config"
api "github.com/osrg/gobgp/api"
"net"
"os/exec"
@@ -10,6 +11,11 @@ import (
"time"
)
const (
defaultMonitorInterval = 10 * time.Second
defaultCleanupTimer = 15 * time.Minute
)
func portMonitor(protocol, port string) bool {
switch protocol {
case "tcp":
@@ -18,14 +24,14 @@ func portMonitor(protocol, port string) bool {
glog.V(4).Infof("Monitor tcp port up")
return true
}
defer conn.Close()
conn.Close()
case "udp":
conn, err := net.ListenPacket(protocol, ":"+port)
if err != nil {
glog.V(4).Infof("Monitor udp port up")
return true
}
defer conn.Close()
conn.Close()
}
return false
}
@@ -53,74 +59,154 @@ type appMon struct {
}
type MonitorMgr struct {
monitors map[string]*appMon
cleanups map[string]chan bool
c *Controller
monitorInterval time.Duration
cleanupTimer time.Duration
monitors map[string]*appMon
cleanups map[string]chan bool
config *c.Config
ctrl *Controller
consul *ConsulMon
sync.Mutex
}
func NewMonitor(localAS, peerAS int, monitorInterval time.Duration, peerIP string, cleanup time.Duration) *MonitorMgr {
c, err := NewController(localAS, peerAS, peerIP)
func NewMonitor(config *c.Config) *MonitorMgr {
ctrl, err := NewController(config)
if err != nil {
glog.Exitf("Failed to start BGP controller: %v", err)
}
return &MonitorMgr{
c: c,
monitors: make(map[string]*appMon),
cleanups: make(map[string]chan bool),
monitorInterval: monitorInterval,
cleanupTimer: cleanup,
mon := &MonitorMgr{
ctrl: ctrl,
monitors: make(map[string]*appMon),
cleanups: make(map[string]chan bool),
}
if config.Agent.ConsulAddr != "" {
cmon, err := NewConsulMon(config.Agent.ConsulAddr)
if err != nil {
glog.Errorf("Failed to start consul monitor: %v", err)
} else {
mon.consul = cmon
go mon.consulMon()
}
}
if config.Agent.MonitorInterval == 0 {
config.Agent.MonitorInterval = defaultMonitorInterval
}
if config.Agent.CleanupTimer == 0 {
config.Agent.CleanupTimer = defaultCleanupTimer
}
mon.config = config
return mon
}
func (m *MonitorMgr) consulMon() {
for {
apps, err := m.consul.queryServices()
if err != nil {
glog.Errorf("Failed to query consul: %v", err)
} else {
for _, app := range apps {
m.Add(app)
}
// remove currently running apps that are not discovered in this pass
var toRemove []string
m.Lock()
for name := range m.monitors {
var found bool
for _, app := range apps {
if name == app.Name {
found = true
break
}
}
if !found {
glog.V(2).Infof("Removing app: %s as it was not found in consul", name)
toRemove = append(toRemove, name)
}
}
for _, tr := range toRemove {
m.Remove(tr)
}
m.Unlock()
}
<-time.After(m.config.Agent.ConsulQueryInterval)
}
}
func (m *MonitorMgr) Add(app *App) {
// stop and start a new one if one already running
// check if already running
m.Lock()
defer m.Unlock()
for _, appMon := range m.monitors {
if appMon.app.Equal(app) && appMon.checkOn {
return
}
if appMon.app.Vip.String() == app.Vip.String() && appMon.app.Name != app.Name {
glog.Errorf("Error: Vip %s is already being announced by app: %s", app.Vip.String(), appMon.app.Name)
return
}
}
m.Remove(app.Name)
appMon := &appMon{app: app, done: make(chan bool)}
m.Lock()
m.monitors[app.Name] = appMon
m.Unlock()
go m.runLoop(appMon)
glog.Infof("Registered a new app: %v", app)
}
func (m *MonitorMgr) Remove(appName string) {
m.Lock()
defer m.Unlock()
if a, ok := m.monitors[appName]; ok {
if a.checkOn {
a.done <- true
}
if a.announced {
if err := m.c.Withdraw(a.app.Vip); err != nil {
if err := m.ctrl.Withdraw(a.app.Vip); err != nil {
glog.Errorf("Failed to withdraw route: %v", err)
}
}
deleteLoopback(appName)
deleteLoopback(a.app.Vip)
if ok, mon := a.app.needsNatRule(); ok {
natRule("D", a.app.Vip.IP, m.ctrl.localIP, mon.Port, mon.Protocol)
}
}
delete(m.monitors, appName)
}
func (m *MonitorMgr) runMonitors(app *App) bool {
var check bool
for _, mon := range app.Monitors {
switch mon.Type {
case Monitor_PORT:
check = portMonitor(mon.Protocol, mon.Port)
case Monitor_EXEC:
check = execMonitor(mon.Cmd)
case Monitor_CONSUL:
c, err := m.consul.healthCheck(app.Name)
if err != nil {
glog.Errorf("Failed to perform consul healthcheck for %s: %v", app.Name, err)
}
check = c
}
if !check {
glog.V(2).Infof("%s Monitor for app: %s Failed", mon.Type.String(), app.Name)
return false
}
}
return true
}
func (m *MonitorMgr) checkCond(am *appMon) error {
var cond bool
app := am.app
switch app.Monitor.Type {
case Monitor_PORT:
cond = portMonitor(app.Monitor.Protocol, app.Monitor.Port)
case Monitor_EXEC:
cond = execMonitor(app.Monitor.Cmd)
}
m.Lock()
defer m.Unlock()
if cond {
glog.V(2).Infof("%s Monitor for app: %s succeeded", app.Monitor.Type.String(), app.Name)
if m.runMonitors(app) {
glog.V(2).Infof("All Monitors for app: %s succeeded", app.Name)
if !am.announced {
if err := addLoopback(app.Name, app.Vip); err != nil {
return err
}
if err := m.c.Announce(app.Vip); err != nil {
if ok, mon := app.needsNatRule(); ok {
if err := natRule("A", app.Vip.IP, m.ctrl.localIP, mon.Port, mon.Protocol); err != nil {
return err
}
}
if err := m.ctrl.Announce(app.Vip); err != nil {
return fmt.Errorf("Failed to announce route: %v", err)
}
am.announced = true
@@ -129,9 +215,8 @@ func (m *MonitorMgr) checkCond(am *appMon) error {
}
}
} else {
glog.V(2).Infof("%s Monitor for app: %s Failed", app.Monitor.Type.String(), app.Name)
if am.announced {
if err := m.c.Withdraw(app.Vip); err != nil {
if err := m.ctrl.Withdraw(app.Vip); err != nil {
return fmt.Errorf("Failed to withdraw route: %v", err)
}
am.announced = false
@@ -148,7 +233,7 @@ func (m *MonitorMgr) runLoop(am *appMon) {
if err := m.checkCond(am); err != nil {
glog.Errorln(err)
}
t := time.NewTicker(m.monitorInterval)
t := time.NewTicker(m.config.Agent.MonitorInterval)
defer t.Stop()
for {
select {
@@ -165,25 +250,30 @@ func (m *MonitorMgr) runLoop(am *appMon) {
func (m *MonitorMgr) CloseAll() {
glog.Infof("Shutting down all open bgp sessions")
if err := m.c.Shutdown(); err != nil {
if err := m.ctrl.Shutdown(); err != nil {
glog.Errorf("Failed to shut-down BGP: %v", err)
}
for name, am := range m.monitors {
for _, am := range m.monitors {
if am.checkOn {
am.done <- true
}
deleteLoopback(name)
deleteLoopback(am.app.Vip)
if ok, mon := am.app.needsNatRule(); ok {
natRule("D", am.app.Vip.IP, m.ctrl.localIP, mon.Port, mon.Protocol)
}
}
}
func (m *MonitorMgr) Cleanup(app string, exit chan bool) {
t := time.NewTimer(m.cleanupTimer)
t := time.NewTimer(m.config.Agent.CleanupTimer)
defer t.Stop()
for {
select {
case <-t.C:
glog.Infof("Cleaning up app %s", app)
m.Lock()
m.Remove(app)
m.Unlock()
case <-exit:
return
}
@@ -191,5 +281,5 @@ func (m *MonitorMgr) Cleanup(app string, exit chan bool) {
}
func (m *MonitorMgr) GetInfo() (*api.Peer, error) {
return m.c.PeerInfo()
return m.ctrl.PeerInfo()
}

30
controller/system.go
View File

@@ -33,20 +33,38 @@ func localAddress(gw net.IP) (net.IP, error) {
}
func addLoopback(name string, addr *net.IPNet) error {
mask := fmt.Sprintf("%d.%d.%d.%d", addr.Mask[0], addr.Mask[1], addr.Mask[2], addr.Mask[3])
cmd := fmt.Sprintf("ifconfig lo:%s %s netmask %s up", name, addr.IP.String(), mask)
prefixLen, _ := addr.Mask.Size()
label := fmt.Sprintf("lo:%s", name)
// linux kernel limits labels to 15 chars
if len(label) > 15 {
label = label[:15]
}
cmd := fmt.Sprintf("ip address add %s/%d dev lo label %s", addr.IP.String(), prefixLen, label)
_, err := exec.Command("bash", "-c", cmd).Output()
if err != nil {
return fmt.Errorf("Failed to Add loopback command: %v", err)
return fmt.Errorf("Failed to Add loopback command: %s: %v", cmd, err)
}
return nil
}
func deleteLoopback(name string) error {
cmd := fmt.Sprintf("ifconfig lo:%s down", name)
func deleteLoopback(addr *net.IPNet) error {
prefixLen, _ := addr.Mask.Size()
cmd := fmt.Sprintf("ip address delete %s/%d dev lo", addr.IP.String(), prefixLen)
_, err := exec.Command("bash", "-c", cmd).Output()
if err != nil {
return fmt.Errorf("Failed to delete loopback command: %v", err)
return fmt.Errorf("Failed to delete loopback command: %s: %v", cmd, err)
}
return nil
}
func natRule(op string, vip, localAddr net.IP, port, protocol string) error {
cmd := fmt.Sprintf(
"iptables -t nat -%s PREROUTING -p %s -d %s --dport %s -j DNAT --to-destination %s:%s",
op, protocol, vip.String(), port, localAddr.String(), port,
)
_, err := exec.Command("bash", "-c", cmd).Output()
if err != nil {
return fmt.Errorf("Failed to %s nat rule: %s: %v", op, cmd, err)
}
return nil
}