Add support for local consul agent checking

Direct healthchecking can be expensive at scale. Using the local
agent is a best practice.

controller/consul.go

@@ -12,8 +12,9 @@ import (
 const (
 	consulNodeEnv        = "CONSUL_NODE"
 	matchTag             = "enable_gocast"
-	nodeUrl        = "/catalog/node"
-	healthCheckurl = "/health/checks"
+	nodeURL              = "/catalog/node"
+	remoteHealthCheckurl = "/health/checks"
+	localHealthCheckurl  = "/agent/checks"
 )
 
 type ConsulMon struct {
@@ -48,7 +49,7 @@ func NewConsulMon(addr string) (*ConsulMon, error) {
 
 func (c *ConsulMon) queryServices() ([]*App, error) {
 	var apps []*App
-	addr := c.addr + fmt.Sprintf("%s/%s", nodeUrl, c.node)
+	addr := c.addr + fmt.Sprintf("%s/%s", nodeURL, c.node)
 	resp, err := http.Get(addr)
 	if err != nil {
 		return apps, err
@@ -97,27 +98,75 @@ func (c *ConsulMon) queryServices() ([]*App, error) {
 	return apps, nil
 }
 
-func (c *ConsulMon) healthCheck(service string) (bool, error) {
-	addr := c.addr + fmt.Sprintf("%s/%s", healthCheckurl, service)
+// Returns a *Response to mimic http.Get with some minimal error handling.
+// https://golang.org/pkg/net/http/#Get for docs on Get.
+func getResp(addr) (resp *Response) {
 	resp, err := http.Get(addr)
 	if err != nil {
-		return false, err
+		glog.V(2).Errorf("Error getting %s with %s", addr, err)
 	}
 	defer resp.Body.Close()
+	return resp
+}
+
+// healthCheckLocal queries a node's local consul agent to perform service healthchecks
+// This is the underlying api call: https://www.consul.io/api/agent/check.html
+func (c *ConsulMon) healthCheckLocal(service string) (bool, error) {
+	addr := c.addr + fmt.Sprintf("%s/%s", localHealthCheckurl, service)
+	resp, err = getResp(addr)
+
+	var data interface{}
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return false, err
+	}
+
+	services := data.(map[string]interface{})
+	for _, serviceInfo := range services {
+		s := serviceInfo.(map[string]interface{})
+		serviceName := s["ServiceName"].(string)
+		if serviceName == service {
+			status := s["Status"].(string)
+			if status == "passing" {
+				return true, nil
+			}
+			glog.V(2).Infof("Consul local healthcheck returned %s status", status)
+			return false, nil
+		}
+		node := serviceReport["Node"].(string)
+		return false, fmt.Errorf("No local healcheck info found for service %s on node %s in consul", service, node)
+	}
+}
+
+// healthCheckRemote queries the consul cluster's healthcheck endpoint to perform service healthchecks
+// This is the underlying api call: https://www.consul.io/api/health.html
+func (c *ConsulMon) healthCheckRemote(service string) (bool, error) {
+	addr := c.addr + fmt.Sprintf("%s/%s", remoteHealthCheckurl, service)
+	resp, err = getResp(addr)
+
 	var data []interface{}
 	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
 		return false, err
 	}
+
 	for _, nodeInfo := range data {
 		n := nodeInfo.(map[string]interface{})
-		if n["Node"] == c.node {
+		if n["Node"].(string) == c.node {
 			if n["Status"].(string) == "passing" {
 				return true, nil
-			} else {
-				glog.V(2).Infof("Consul Healthcheck returned %s status", n["Status"].(string))
-				return false, nil
 			}
+			glog.V(2).Infof("Consul healthcheck returned %s status", n["Status"].(string))
+			return false, nil
 		}
 	}
 	return false, fmt.Errorf("No healcheck info found for node %s in consul", c.node)
 }
+
+// healthCheck determines if we should use the local agent
+// If the address contains "localhost", then it presumes that the local agent is to be used.
+func (c *ConsulMon) healthCheck(service string) (bool, error) {
+	usingLocalAgent := strings.Contains(c.addr, "localhost")
+	if (usingLocalAgent) {
+		return c.healthCheckLocal(service)
+	}
+	return c.healthCheckRemote(service)
+}

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/eapache/queue v1.1.0
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/golang/protobuf v0.0.0-20181022004443-7be363195599
+	github.com/golang/protobuf v1.2.0
 	github.com/hashicorp/hcl v1.0.0
 	github.com/influxdata/influxdb v1.6.4
 	github.com/konsorten/go-windows-terminal-sequences v1.0.1