ben/gocast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add ipv6 support and enable go mod

Browse Source
This commit is contained in:
Mayuresh Gaitonde
2019-04-24 11:21:22 -07:00
parent e15172111e
commit e729608b90
11 changed files with 467 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
controller/app.go
View File

@@ -2,17 +2,19 @@ package controller
import (
"fmt"
"github.com/golang/glog"
"net"
"strings"
"github.com/golang/glog"
)
type MonitorType int
const (
Monitor_PORT MonitorType = 1
Monitor_EXEC MonitorType = 2
Monitor_CONSUL MonitorType = 3
Monitor_PORT MonitorType = 1
Monitor_EXEC MonitorType = 2
Monitor_CONSUL MonitorType = 3
defaultFailThreshold = 3
)
var MonitorMap = map[string]MonitorType{"port": Monitor_PORT, "exec": Monitor_EXEC, "consul": Monitor_CONSUL}
@@ -27,10 +29,11 @@ func (m MonitorType) String() string {
}
type Monitor struct {
Type MonitorType
Port string
Protocol string
Cmd string
Type MonitorType
Port string
Protocol string
Cmd string
FailCount int
}
func (m *Monitor) Equal(other *Monitor) bool {
@@ -48,9 +51,19 @@ func (m Monitors) Contains(elem *Monitor) bool {
return false
}
type Vip struct {
IP net.IP
Net *net.IPNet
Family string
}
func (v *Vip) Equal(other *Vip) bool {
return v.IP.Equal(other.IP)
}
type App struct {
Name string
Vip *net.IPNet
Vip *Vip
Monitors Monitors
Nats []string
}
@@ -64,7 +77,7 @@ func (a *App) Equal(other *App) bool {
return false
}
}
return a.Name == other.Name && a.Vip.String() == other.Vip.String()
return a.Name == other.Name && a.Vip.Equal(other.Vip)
}
func NewApp(appName, vip string, monitors []string, nats []string) (*App, error) {
@@ -72,11 +85,16 @@ func NewApp(appName, vip string, monitors []string, nats []string) (*App, error)
return nil, fmt.Errorf("Invalid app name")
}
app := &App{Name: appName, Nats: nats}
_, ipnet, err := net.ParseCIDR(vip)
ip, ipnet, err := net.ParseCIDR(vip)
if err != nil {
return nil, fmt.Errorf("Invalid VIP specified, need ip/mask")
}
app.Vip = ipnet
app.Vip = &Vip{IP: ip, Net: ipnet}
if ip.To4() != nil {
app.Vip.Family = "4"
} else {
app.Vip.Family = "6"
}
for _, m := range monitors {
// valid monitor formats:
// "port:tcp:123" , "exec:/local/check.sh", "consul"

240
controller/bgp.go
View File

@@ -3,160 +3,214 @@ package controller
import (
"context"
"fmt"
"net"
"strconv"
"strings"
"github.com/golang/protobuf/ptypes"
"github.com/golang/protobuf/ptypes/any"
c "github.com/mayuresh82/gocast/config"
api "github.com/osrg/gobgp/api"
gobgp "github.com/osrg/gobgp/pkg/server"
"net"
"strconv"
"strings"
)
type Controller struct {
type Peer struct {
peerAS int
localIP, peerIP net.IP
communities []string
origin uint32
multiHop bool
s *gobgp.BgpServer
family string
announced []string
}
type Controller struct {
peers []*Peer
s *gobgp.BgpServer
}
func NewController(config *c.Config) (*Controller, error) {
c := &Controller{}
var gw net.IP
var err error
if config.Bgp.PeerIP == "" {
gw, err = gateway()
c.peerIP = gw
} else {
c.peerIP = net.ParseIP(config.Bgp.PeerIP)
gw, err = via(c.peerIP)
}
if err != nil || c.peerIP == nil {
return nil, fmt.Errorf("Unable to get peer IP : %v", err)
}
c.communities = config.Bgp.Communities
switch config.Bgp.Origin {
case "igp":
c.origin = 0
case "egp":
c.origin = 1
case "unknown":
c.origin = 2
}
s := gobgp.NewBgpServer()
go s.Serve()
localAddr, err := localAddress(gw)
if err != nil {
return nil, err
}
c.localIP = localAddr
if err := s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
As: uint32(config.Bgp.LocalAS),
RouterId: localAddr.String(),
ListenPort: -1, // gobgp won't listen on tcp:179
},
}); err != nil {
return nil, fmt.Errorf("Unable to start bgp: %v", err)
}
c.s = s
c.peerAS = config.Bgp.PeerAS
// set mh by default for all ebgp peers
if c.peerAS != config.Bgp.LocalAS {
c.multiHop = true
c := &Controller{s: gobgp.NewBgpServer()}
go c.s.Serve()
for _, bgpConf := range config.Bgp {
p := &Peer{}
var err error
if bgpConf.PeerIP == "" {
p.peerIP, err = gateway(bgpConf.AddrFamily)
} else {
p.peerIP = net.ParseIP(bgpConf.PeerIP)
}
if err != nil || p.peerIP == nil {
return nil, fmt.Errorf("Unable to get peer IP : %v", err)
}
p.communities = bgpConf.Communities
switch bgpConf.Origin {
case "igp":
p.origin = 0
case "egp":
p.origin = 1
case "unknown":
p.origin = 2
}
dev, err := via(p.peerIP)
if err != nil {
return nil, err
}
localAddr, err := localAddress(dev, bgpConf.AddrFamily)
if err != nil {
return nil, err
}
p.localIP = localAddr
p.peerAS = bgpConf.PeerAS
// set mh by default for all ebgp peers
if p.peerAS != bgpConf.LocalAS {
p.multiHop = true
}
localAddr4, _ := localAddress(dev, "4") // for router-id
if err := c.s.StartBgp(context.Background(), &api.StartBgpRequest{
Global: &api.Global{
As: uint32(bgpConf.LocalAS),
RouterId: localAddr4.String(),
ListenPort: -1, // gobgp won't listen on tcp:179
},
}); err != nil {
return nil, fmt.Errorf("Unable to start bgp: %v", err)
}
p.family = "6"
if p.peerIP.To4() != nil {
p.family = "4"
}
c.peers = append(c.peers, p)
}
return c, nil
}
func (c *Controller) AddPeer(peer string) error {
func (c *Controller) localIP(family string) net.IP {
for _, peer := range c.peers {
if peer.family == family {
return peer.localIP
}
}
return nil
}
func (c *Controller) AddPeer(p *Peer) error {
n := &api.Peer{
Conf: &api.PeerConf{
NeighborAddress: peer,
PeerAs: uint32(c.peerAS),
NeighborAddress: p.peerIP.String(),
PeerAs: uint32(p.peerAS),
},
}
if c.multiHop {
if p.multiHop {
n.EbgpMultihop = &api.EbgpMultihop{Enabled: true, MultihopTtl: uint32(255)}
}
return c.s.AddPeer(context.Background(), &api.AddPeerRequest{Peer: n})
}
func (c *Controller) getApiPath(route *net.IPNet) *api.Path {
func (c *Controller) getApiPath(p *Peer, route *net.IPNet, withdraw bool) *api.Path {
afi := api.Family_AFI_IP
if route.IP.To4() == nil {
afi = api.Family_AFI_IP6
}
family := &api.Family{Afi: afi, Safi: api.Family_SAFI_UNICAST}
prefixlen, _ := route.Mask.Size()
nlri, _ := ptypes.MarshalAny(&api.IPAddressPrefix{
Prefix: route.IP.String(),
PrefixLen: uint32(prefixlen),
})
a1, _ := ptypes.MarshalAny(&api.OriginAttribute{
Origin: c.origin,
})
a2, _ := ptypes.MarshalAny(&api.NextHopAttribute{
NextHop: c.localIP.String(),
Origin: p.origin,
})
var communities []uint32
for _, comm := range c.communities {
for _, comm := range p.communities {
communities = append(communities, convertCommunity(comm))
}
a3, _ := ptypes.MarshalAny(&api.CommunitiesAttribute{
a2, _ := ptypes.MarshalAny(&api.CommunitiesAttribute{
Communities: communities,
})
attrs := []*any.Any{a1, a2, a3}
return &api.Path{
Family: &api.Family{Afi: afi, Safi: api.Family_SAFI_UNICAST},
AnyNlri: nlri,
AnyPattrs: attrs,
attrs := []*any.Any{a1, a2}
path := &api.Path{AnyNlri: nlri, Family: family, AnyPattrs: attrs, IsWithdraw: withdraw}
switch afi {
case api.Family_AFI_IP:
nh, _ := ptypes.MarshalAny(&api.NextHopAttribute{
NextHop: p.localIP.String(),
})
path.AnyPattrs = append(path.AnyPattrs, nh)
case api.Family_AFI_IP6:
mpReachAttr, _ := ptypes.MarshalAny(&api.MpReachNLRIAttribute{
Family: family,
NextHops: []string{p.localIP.String()},
Nlris: []*any.Any{nlri},
})
mpUnreachAttr, _ := ptypes.MarshalAny(&api.MpUnreachNLRIAttribute{
Family: family,
Nlris: []*any.Any{nlri},
})
if withdraw {
path.AnyPattrs = append(path.AnyPattrs, mpUnreachAttr)
} else {
path.AnyPattrs = append(path.AnyPattrs, mpReachAttr)
}
}
return path
}
func (c *Controller) Announce(route *net.IPNet) error {
peers, err := c.s.ListPeer(context.Background(), &api.ListPeerRequest{})
if err != nil {
return err
family := "6"
if route.IP.To4() != nil {
family = "4"
}
var found bool
for _, p := range peers {
if p.Conf.NeighborAddress == c.peerIP.String() {
found = true
break
for _, peer := range c.peers {
if peer.family != family {
continue
}
}
if !found {
if err := c.AddPeer(c.peerIP.String()); err != nil {
peers, err := c.s.ListPeer(context.Background(), &api.ListPeerRequest{})
if err != nil {
return err
}
var found bool
for _, p := range peers {
if p.Conf.NeighborAddress == peer.peerIP.String() {
found = true
break
}
}
if !found {
if err := c.AddPeer(peer); err != nil {
return err
}
}
if _, err := c.s.AddPath(context.Background(), &api.AddPathRequest{Path: c.getApiPath(peer, route, false)}); err != nil {
return err
}
peer.announced = append(peer.announced, route.String())
}
_, err = c.s.AddPath(context.Background(), &api.AddPathRequest{Path: c.getApiPath(route)})
return err
return nil
}
func (c *Controller) Withdraw(route *net.IPNet) error {
return c.s.DeletePath(context.Background(), &api.DeletePathRequest{Path: c.getApiPath(route)})
}
func (c *Controller) PeerInfo() (*api.Peer, error) {
peers, err := c.s.ListPeer(context.Background(), &api.ListPeerRequest{})
if err != nil {
return nil, err
}
for _, p := range peers {
if p.Conf.NeighborAddress == c.peerIP.String() {
return p, nil
for _, peer := range c.peers {
if !contains(peer.announced, route.String()) {
continue
}
if err := c.s.DeletePath(context.Background(), &api.DeletePathRequest{Path: c.getApiPath(peer, route, true)}); err != nil {
return err
}
}
return nil, nil
return nil
}
func (c *Controller) PeerInfo() ([]*api.Peer, error) {
return c.s.ListPeer(context.Background(), &api.ListPeerRequest{})
}
func (c *Controller) Shutdown() error {
if err := c.s.ShutdownPeer(context.Background(), &api.ShutdownPeerRequest{
Address: c.peerIP.String(),
}); err != nil {
return err
for _, peer := range c.peers {
if err := c.s.ShutdownPeer(context.Background(), &api.ShutdownPeerRequest{
Address: peer.peerIP.String(),
}); err != nil {
return err
}
}
if err := c.s.StopBgp(context.Background(), &api.StopBgpRequest{}); err != nil {
return err

76
controller/monitor.go
View File

@@ -2,18 +2,19 @@ package controller
import (
"fmt"
"github.com/golang/glog"
c "github.com/mayuresh82/gocast/config"
api "github.com/osrg/gobgp/api"
"net"
"os/exec"
"strings"
"sync"
"time"
"github.com/golang/glog"
c "github.com/mayuresh82/gocast/config"
api "github.com/osrg/gobgp/api"
)
const (
defaultMonitorInterval = 10 * time.Second
defaultMonitorInterval = 20 * time.Second
defaultCleanupTimer = 15 * time.Minute
)
@@ -53,12 +54,14 @@ func execMonitor(cmd string) bool {
}
type appMon struct {
app *App
done chan bool
announced bool
checkOn bool
app *App
done chan bool
announced bool
vipCreated bool
checkOn bool
}
// MonitorMgr manages all registered apps and their healthcheck monitors
type MonitorMgr struct {
monitors map[string]*appMon
cleanups map[string]chan bool
@@ -69,6 +72,7 @@ type MonitorMgr struct {
sync.Mutex
}
// NewMonitor returns a new instance of MonitorMgr
func NewMonitor(config *c.Config) *MonitorMgr {
ctrl, err := NewController(config)
if err != nil {
@@ -141,6 +145,7 @@ func (m *MonitorMgr) consulMon() {
}
}
// Add adds a new app to be monitored
func (m *MonitorMgr) Add(app *App) {
// check if already running
m.Lock()
@@ -150,8 +155,8 @@ func (m *MonitorMgr) Add(app *App) {
glog.V(2).Infof("App %s already exists", app.Name)
return
}
if appMon.app.Vip.String() == app.Vip.String() && appMon.app.Name != app.Name {
glog.Errorf("Error: Vip %s is already being announced by app: %s", app.Vip.String(), appMon.app.Name)
if appMon.app.Vip.Equal(app.Vip) && appMon.app.Name != app.Name {
glog.Errorf("Error: Vip %s is already being announced by app: %s", app.Vip.IP.String(), appMon.app.Name)
return
}
}
@@ -162,16 +167,20 @@ func (m *MonitorMgr) Add(app *App) {
glog.Infof("Registered a new app: %v", app)
}
// Remove removes an existing app and withdraws the bgp vip
func (m *MonitorMgr) Remove(appName string) {
if a, ok := m.monitors[appName]; ok {
if a.checkOn {
a.done <- true
}
if a.announced {
if err := m.ctrl.Withdraw(a.app.Vip); err != nil {
if err := m.ctrl.Withdraw(a.app.Vip.Net); err != nil {
glog.Errorf("Failed to withdraw route: %v", err)
}
}
if !a.vipCreated {
return
}
if err := deleteLoopback(a.app.Vip); err != nil {
glog.Errorf("Failed to remove app: %s: %v", a.app.Name, err)
}
@@ -180,7 +189,12 @@ func (m *MonitorMgr) Remove(appName string) {
if len(parts) != 2 {
continue
}
if err := natRule("D", a.app.Vip.IP, m.ctrl.localIP, parts[0], parts[1]); err != nil {
localIP := m.ctrl.localIP(a.app.Vip.Family)
if localIP == nil {
glog.Errorf("Failed to get local IP for family %s", a.app.Vip.Family)
continue
}
if err := natRule("D", a.app.Vip.IP, localIP, parts[0], parts[1]); err != nil {
glog.Errorf("Failed to remove app: %s: %v", a.app.Name, err)
}
}
@@ -204,9 +218,16 @@ func (m *MonitorMgr) runMonitors(app *App) bool {
}
if !check {
glog.V(2).Infof("%s Monitor for app: %s Failed", mon.Type.String(), app.Name)
return false
if mon.FailCount >= defaultFailThreshold {
return false
}
mon.FailCount++
}
if check {
mon.FailCount = 0
}
}
glog.V(2).Infof("All Monitors for app: %s succeeded", app.Name)
return true
}
@@ -215,8 +236,7 @@ func (m *MonitorMgr) checkCond(am *appMon) error {
m.Lock()
defer m.Unlock()
if m.runMonitors(app) {
glog.V(2).Infof("All Monitors for app: %s succeeded", app.Name)
if !am.announced {
if !am.vipCreated {
if err := addLoopback(app.Name, app.Vip); err != nil {
return err
}
@@ -225,11 +245,19 @@ func (m *MonitorMgr) checkCond(am *appMon) error {
if len(parts) != 2 {
continue
}
if err := natRule("A", app.Vip.IP, m.ctrl.localIP, parts[0], parts[1]); err != nil {
localIP := m.ctrl.localIP(app.Vip.Family)
if localIP == nil {
glog.Errorf("Failed to get local IP for family %s", app.Vip.Family)
continue
}
if err := natRule("A", app.Vip.IP, localIP, parts[0], parts[1]); err != nil {
return err
}
}
if err := m.ctrl.Announce(app.Vip); err != nil {
am.vipCreated = true
}
if !am.announced {
if err := m.ctrl.Announce(app.Vip.Net); err != nil {
return fmt.Errorf("Failed to announce route: %v", err)
}
am.announced = true
@@ -239,7 +267,7 @@ func (m *MonitorMgr) checkCond(am *appMon) error {
}
} else {
if am.announced {
if err := m.ctrl.Withdraw(app.Vip); err != nil {
if err := m.ctrl.Withdraw(app.Vip.Net); err != nil {
return fmt.Errorf("Failed to withdraw route: %v", err)
}
am.announced = false
@@ -271,6 +299,7 @@ func (m *MonitorMgr) runLoop(am *appMon) {
}
}
// CloseAll closes all open bgp sessions and cleans up all apps and their VIPs
func (m *MonitorMgr) CloseAll() {
glog.Infof("Shutting down all open bgp sessions")
if err := m.ctrl.Shutdown(); err != nil {
@@ -286,11 +315,17 @@ func (m *MonitorMgr) CloseAll() {
if len(parts) != 2 {
continue
}
natRule("D", am.app.Vip.IP, m.ctrl.localIP, parts[0], parts[1])
localIP := m.ctrl.localIP(am.app.Vip.Family)
if localIP == nil {
glog.Errorf("Failed to get local IP for family %s", am.app.Vip.Family)
continue
}
natRule("D", am.app.Vip.IP, localIP, parts[0], parts[1])
}
}
}
// Cleanup waits for cleanuptimer to expire and then removes the app
func (m *MonitorMgr) Cleanup(app string, exit chan bool) {
t := time.NewTimer(m.config.Agent.CleanupTimer)
defer t.Stop()
@@ -307,6 +342,7 @@ func (m *MonitorMgr) Cleanup(app string, exit chan bool) {
}
}
func (m *MonitorMgr) GetInfo() (*api.Peer, error) {
// GetInfo returns BGP peer info for a specific peer
func (m *MonitorMgr) GetInfo() ([]*api.Peer, error) {
return m.ctrl.PeerInfo()
}

73
controller/system.go
View File

@@ -7,8 +7,12 @@ import (
"strings"
)
func gateway() (net.IP, error) {
cmd := `ip route | grep "^default" | cut -d" " -f3`
func gateway(family string) (net.IP, error) {
ipCmd := "ip"
if family == "6" {
ipCmd = "ip -6"
}
cmd := fmt.Sprintf(`%s route | grep "^default" | cut -d" " -f3`, ipCmd)
out, err := exec.Command("bash", "-c", cmd).Output()
if err != nil {
return nil, fmt.Errorf("Failed to execute command: %s", cmd)
@@ -16,40 +20,59 @@ func gateway() (net.IP, error) {
return net.ParseIP(strings.TrimSpace(string(out))), nil
}
func via(dest net.IP) (net.IP, error) {
cmd := fmt.Sprintf(`ip route get %s | grep via | cut -d" " -f3`, dest.String())
func via(dest net.IP) (string, error) {
ipCmd := "ip"
if dest.To4() == nil {
ipCmd = "ip -6"
}
cmd := fmt.Sprintf(`%s route get %s | grep src | cut -d" " -f3`, ipCmd, dest.String())
out, err := exec.Command("bash", "-c", cmd).Output()
if err != nil {
return nil, fmt.Errorf("Failed to execute command: %s", cmd)
return "", fmt.Errorf("Failed to execute command: %s", cmd)
}
return net.ParseIP(strings.TrimSpace(string(out))), nil
return strings.TrimSpace(string(out)), nil
}
func localAddress(gw net.IP) (net.IP, error) {
addrs, err := net.InterfaceAddrs()
func localAddress(dev string, family string) (net.IP, error) {
iface, err := net.InterfaceByName(dev)
if err != nil {
return nil, err
}
addrs, err := iface.Addrs()
if err != nil {
return nil, err
}
for _, addr := range addrs {
var ip net.IP
switch v := addr.(type) {
case *net.IPNet:
if v.Contains(gw) {
return v.IP, nil
}
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
if family == "4" && ip.To4() != nil {
return ip, nil
}
if family == "6" && ip.To4() == nil && !ip.IsLinkLocalUnicast() {
return ip, nil
}
}
return nil, fmt.Errorf("Unable to find local address")
}
func addLoopback(name string, addr *net.IPNet) error {
deleteLoopback(addr)
prefixLen, _ := addr.Mask.Size()
func addLoopback(name string, vip *Vip) error {
deleteLoopback(vip)
prefixLen, _ := vip.Net.Mask.Size()
label := fmt.Sprintf("lo:%s", name)
// linux kernel limits labels to 15 chars
if len(label) > 15 {
label = label[:15]
}
cmd := fmt.Sprintf("ip address add %s/%d dev lo label %s", addr.IP.String(), prefixLen, label)
ipCmd := "ip"
if vip.Family == "6" {
ipCmd = "ip -6"
}
cmd := fmt.Sprintf("%s address add %s/%d dev lo label %s", ipCmd, vip.IP.String(), prefixLen, label)
_, err := exec.Command("bash", "-c", cmd).Output()
if err != nil {
return fmt.Errorf("Failed to Add loopback command: %s: %v", cmd, err)
@@ -57,9 +80,13 @@ func addLoopback(name string, addr *net.IPNet) error {
return nil
}
func deleteLoopback(addr *net.IPNet) error {
prefixLen, _ := addr.Mask.Size()
cmd := fmt.Sprintf("ip address delete %s/%d dev lo", addr.IP.String(), prefixLen)
func deleteLoopback(vip *Vip) error {
prefixLen, _ := vip.Net.Mask.Size()
ipCmd := "ip"
if vip.Family == "6" {
ipCmd = "ip -6"
}
cmd := fmt.Sprintf("%s address delete %s/%d dev lo", ipCmd, vip.IP.String(), prefixLen)
_, err := exec.Command("bash", "-c", cmd).Output()
if err != nil {
return fmt.Errorf("Failed to delete loopback command: %s: %v", cmd, err)
@@ -68,9 +95,15 @@ func deleteLoopback(addr *net.IPNet) error {
}
func natRule(op string, vip, localAddr net.IP, protocol, port string) error {
iptCmd := "iptables"
toDest := fmt.Sprintf("%s:%s", localAddr.String(), port)
if vip.To4() == nil {
iptCmd = "ip6tables"
toDest = fmt.Sprintf("[%s]:%s", localAddr.String(), port)
}
cmd := fmt.Sprintf(
"iptables -t nat -%s PREROUTING -p %s -d %s --dport %s -j DNAT --to-destination %s:%s",
op, protocol, vip.String(), port, localAddr.String(), port,
"%s -t nat -%s PREROUTING -p %s -d %s --dport %s -j DNAT --to-destination %s",
iptCmd, op, protocol, vip.String(), port, toDest,
)
_, err := exec.Command("bash", "-c", cmd).Output()
if err != nil {