From d7f4cc520fb3037a4d00934c70467796c5ad86df Mon Sep 17 00:00:00 2001 From: muradm Date: Fri, 30 Sep 2016 01:23:46 +0400 Subject: [PATCH 1/5] use docker info as source for host name considering swarm mode --- bridge/bridge.go | 43 ++++++++++++++++++++++--------------------- bridge/types.go | 1 + registrator.go | 22 ++++++++++++++++++++++ 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/bridge/bridge.go b/bridge/bridge.go index 2387b769f..d0e1f8b28 100644 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -3,9 +3,9 @@ package bridge import ( "errors" "log" - "net" +// "net" "net/url" - "os" +// "os" "path" "regexp" "strconv" @@ -157,7 +157,7 @@ func (b *Bridge) Sync(quiet bool) { continue } serviceHostname := matches[1] - if serviceHostname != Hostname { + if serviceHostname != b.config.NodeId { // use node id as reference instead of hostname // ignore because registered on a different host continue } @@ -251,16 +251,16 @@ func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { defaultName := strings.Split(path.Base(container.Config.Image), ":")[0] // not sure about this logic. kind of want to remove it. - hostname := Hostname - if hostname == "" { - hostname = port.HostIP - } - if port.HostIP == "0.0.0.0" { - ip, err := net.ResolveIPAddr("ip", hostname) - if err == nil { - port.HostIP = ip.String() - } - } + //hostname := Hostname + //if hostname == "" { + // hostname = port.HostIP + //} + //if port.HostIP == "0.0.0.0" { + // ip, err := net.ResolveIPAddr("ip", hostname) + // if err == nil { + // port.HostIP = ip.String() + // } + //} if b.config.HostIp != "" { port.HostIP = b.config.HostIp @@ -275,7 +275,8 @@ func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { service := new(Service) service.Origin = port - service.ID = hostname + ":" + container.Name[1:] + ":" + port.ExposedPort + // use node id, which is more reliable + service.ID = b.config.NodeId + ":" + container.Name[1:] + ":" + port.ExposedPort service.Name = mapDefault(metadata, "name", defaultName) if isgroup && !metadataFromPort["name"] { service.Name += "-" + port.ExposedPort @@ -371,10 +372,10 @@ func (b *Bridge) shouldRemove(containerId string) bool { return false } -var Hostname string - -func init() { - // It's ok for Hostname to ultimately be an empty string - // An empty string will fall back to trying to make a best guess - Hostname, _ = os.Hostname() -} +//var Hostname string +// +//func init() { +// // It's ok for Hostname to ultimately be an empty string +// // An empty string will fall back to trying to make a best guess +// Hostname, _ = os.Hostname() +//} diff --git a/bridge/types.go b/bridge/types.go index b1611127e..656b599ee 100644 --- a/bridge/types.go +++ b/bridge/types.go @@ -20,6 +20,7 @@ type RegistryAdapter interface { } type Config struct { + NodeId string HostIp string Internal bool ForceTags string diff --git a/registrator.go b/registrator.go index b76dc9441..9c452e769 100644 --- a/registrator.go +++ b/registrator.go @@ -12,6 +12,7 @@ import ( dockerapi "github.com/fsouza/go-dockerclient" "github.com/gliderlabs/pkg/usage" "github.com/gliderlabs/registrator/bridge" + "github.com/docker/engine-api/types/swarm" ) var Version string @@ -95,7 +96,28 @@ func main() { assert(errors.New("-deregister must be \"always\" or \"on-success\"")) } + // use docker info to determine node id that will be used as prefix to service id + dockerInfo, err := docker.Info() + assert(err) + + nodeId := new(string) + if dockerInfo.Swarm.LocalNodeState != swarm.LocalNodeStateInactive { + // swarm mode identifies each node uniquely + *nodeId = dockerInfo.Swarm.NodeID + if *hostIp == "" { + // in case of swarm mode, docker host has information about ip + // although it won't be always useful, we can use it if not provided by user + *hostIp = dockerInfo.Swarm.NodeAddr + } + log.Printf("Docker host in Swarm Mode: %s (%s)", *nodeId, *hostIp) + } else { + // docker host name normally is hostname + *nodeId = dockerInfo.Name + log.Printf("Docker host: %s (%s)", *nodeId, *hostIp) + } + b, err := bridge.New(docker, flag.Arg(0), bridge.Config{ + NodeId: *nodeId, HostIp: *hostIp, Internal: *internal, ForceTags: *forceTags, From 4167e41fe88f4d5e38d696171178456fa7713e24 Mon Sep 17 00:00:00 2001 From: muradm Date: Fri, 30 Sep 2016 01:36:44 +0400 Subject: [PATCH 2/5] remove old code --- bridge/bridge.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/bridge/bridge.go b/bridge/bridge.go index d0e1f8b28..9e2f8005f 100644 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -3,9 +3,7 @@ package bridge import ( "errors" "log" -// "net" "net/url" -// "os" "path" "regexp" "strconv" @@ -250,18 +248,6 @@ func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { container := port.container defaultName := strings.Split(path.Base(container.Config.Image), ":")[0] - // not sure about this logic. kind of want to remove it. - //hostname := Hostname - //if hostname == "" { - // hostname = port.HostIP - //} - //if port.HostIP == "0.0.0.0" { - // ip, err := net.ResolveIPAddr("ip", hostname) - // if err == nil { - // port.HostIP = ip.String() - // } - //} - if b.config.HostIp != "" { port.HostIP = b.config.HostIp } @@ -371,11 +357,3 @@ func (b *Bridge) shouldRemove(containerId string) bool { } return false } - -//var Hostname string -// -//func init() { -// // It's ok for Hostname to ultimately be an empty string -// // An empty string will fall back to trying to make a best guess -// Hostname, _ = os.Hostname() -//} From 6d8b69e3eef1d8e816072ce9a7ac6093dc9bbd5c Mon Sep 17 00:00:00 2001 From: muradm Date: Fri, 30 Sep 2016 06:39:21 +0400 Subject: [PATCH 3/5] implement docker swarm mode support, no extra configuration required --- bridge/bridge.go | 118 ++++++++++++++++++++++++++++++++++++++++++++++- bridge/util.go | 8 +++- registrator.go | 6 +-- 3 files changed, 124 insertions(+), 8 deletions(-) diff --git a/bridge/bridge.go b/bridge/bridge.go index 9e2f8005f..1a7392d58 100644 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -11,6 +11,7 @@ import ( "sync" dockerapi "github.com/fsouza/go-dockerclient" + "github.com/docker/engine-api/types/swarm" ) var serviceIDPattern = regexp.MustCompile(`^(.+?):([a-zA-Z0-9][a-zA-Z0-9_.-]+):[0-9]+(?::udp)?$`) @@ -242,6 +243,21 @@ func (b *Bridge) add(containerId string, quiet bool) { b.services[container.ID] = append(b.services[container.ID], service) log.Println("added:", container.ID[:12], service.ID) } + + // if swarm container belongs to swarm mode service, publish VIP services + if swarmServiceName, ok := container.Config.Labels["com.docker.swarm.service.name"]; ok { + filters := map[string][]string{"name": {swarmServiceName}} + services, err := b.docker.ListServices(dockerapi.ListServicesOptions{Filters: filters}) + if err != nil { + log.Println("error listing swarm services, wont register VIP service", err) + } else if len(services) == 1 { // container cannot belong to no or more than one service + if services[0].Spec.EndpointSpec.Mode == swarm.ResolutionModeVIP { // endpoint should be VIP + if (len(services[0].Endpoint.VirtualIPs) > 0) { + b.registerSwarmVipServices(services[0]) + } + } + } + } } func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { @@ -261,9 +277,17 @@ func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { service := new(Service) service.Origin = port - // use node id, which is more reliable + + // consider swarm mode + if swarmServiceName, ok := port.container.Config.Labels["com.docker.swarm.service.name"]; ok { + // swarm mode has concept of services + service.Name = mapDefault(metadata, "name", swarmServiceName) + } else { + // use node id, which is more reliable + service.Name = mapDefault(metadata, "name", defaultName) + } + service.ID = b.config.NodeId + ":" + container.Name[1:] + ":" + port.ExposedPort - service.Name = mapDefault(metadata, "name", defaultName) if isgroup && !metadataFromPort["name"] { service.Name += "-" + port.ExposedPort } @@ -300,6 +324,96 @@ func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { return service } +// there are two types of endpoints VIP and DNS rr based +// DNS rr happens implicitly by registering multiple services with the same name +// so that no extra effort is required +// in case of VIP based services, user specifies the published ports +// which are equivalent of docker port binding, but works differently +// swarm mode provides ingress network, where services are load-balanced +// behind VIP address. From inside network (if there any) perspective +// only one service is need, with swarm mode assigned VIP address. +// From outside perspective, every docker host IP address becomes an entry point +// for load-balancer, so published ports shall be registered for each docker host +func (b *Bridge) registerSwarmVipServices(service swarm.Service) { + // if internal, register the internal VIP services + if b.config.Internal { + for _, vip := range service.Endpoint.VirtualIPs { + if network, err := b.docker.NetworkInfo(vip.NetworkID); err != nil { + log.Println("unable to inspect network while evaluating VIPs for service:", service.Spec.Name, err) + } else { + // no point to publish docker swarm internal ingress network VIP + if network.Name != "ingress" && len(vip.Addr) > 0 && strings.Contains(vip.Addr, "/") { + vipAddr := strings.Split(vip.Addr, "/")[0] + if len(service.Spec.EndpointSpec.Ports) > 0 { + b.registerSwarmVipServicePorts(service.Spec.Name, true, vipAddr, service.Spec.EndpointSpec.Ports) + } + // publish VIP in with out ports in any case + b.registerSwarmVipService(service.Spec.Name, true, vipAddr, false, 0, "ip") + } + } + } + } else { + // if there is no published ports, no point to register it out side + if len(service.Spec.EndpointSpec.Ports) > 0 { + b.registerSwarmVipServicePorts(service.Spec.Name, false, b.config.HostIp, service.Spec.EndpointSpec.Ports) + } + } +} + +// current implementation attempts to register VIP service every container add event +// better way could be to listen for service create events, however according to +// docker configuration there is no such events +// registrations created here are unique, and not based on containers +// so we will just create them and forget, i don't see proper way to cleanup them at the moment +func (b *Bridge) registerSwarmVipServicePorts(serviceName string, inside bool, vip string, ports []swarm.PortConfig) { + for _, port := range ports { + var portNum uint32 + if portNum = port.PublishedPort; inside { + // inside port is not translated to published port + portNum = port.TargetPort + } + + b.registerSwarmVipService(serviceName, inside, vip, true, int(portNum), port.Protocol) + } +} + +func (b *Bridge) registerSwarmVipService(serviceName string, inside bool, vip string, isGroup bool, port int, protocol swarm.PortConfigProtocol) { + var tag string + if tag = "vip-outside"; inside { + tag = "vip-inside" + } + + svcReg := new(Service) + + svcReg.Name = serviceName + "-" + tag + if isGroup { + svcReg.Name = svcReg.Name + "-" + strconv.Itoa(port) + } + + if inside { + // VIP is global and singleton, so we can use service name as service id + svcReg.ID = svcReg.Name + } else { + // VIP is actually host ip address or whatever provided by user + svcReg.ID = b.config.NodeId + "-" + svcReg.Name + } + // tag it for convenience + if protocol != swarm.PortConfigProtocolTCP { + svcReg.Tags = combineTags(tag, b.config.ForceTags, protocol) + } else { + svcReg.Tags = combineTags(tag, b.config.ForceTags) + } + + svcReg.IP = vip + svcReg.Port = port + + err := b.registry.Register(svcReg) + if err != nil { + log.Println("register failed:", svcReg.Name, err) + } + log.Println("added:", svcReg.Name) +} + func (b *Bridge) remove(containerId string, deregister bool) { b.Lock() defer b.Unlock() diff --git a/bridge/util.go b/bridge/util.go index 3b450faff..67b24f4e4 100644 --- a/bridge/util.go +++ b/bridge/util.go @@ -89,8 +89,12 @@ func servicePort(container *dockerapi.Container, port dockerapi.Port, published // Nir: support docker NetworkSettings eip = container.NetworkSettings.IPAddress if eip == "" { - for _, network := range container.NetworkSettings.Networks { - eip = network.IPAddress + for name, network := range container.NetworkSettings.Networks { + // if in swarm mode and network name is ingress, + // no point to publish service with ingress network IP address + if _, ok := container.Config.Labels["com.docker.swarm.service.name"]; ok && name != "ingress" { + eip = network.IPAddress + } } } diff --git a/registrator.go b/registrator.go index 9c452e769..35afdbd95 100644 --- a/registrator.go +++ b/registrator.go @@ -101,9 +101,9 @@ func main() { assert(err) nodeId := new(string) + // docker host name normally is hostname + *nodeId = dockerInfo.Name if dockerInfo.Swarm.LocalNodeState != swarm.LocalNodeStateInactive { - // swarm mode identifies each node uniquely - *nodeId = dockerInfo.Swarm.NodeID if *hostIp == "" { // in case of swarm mode, docker host has information about ip // although it won't be always useful, we can use it if not provided by user @@ -111,8 +111,6 @@ func main() { } log.Printf("Docker host in Swarm Mode: %s (%s)", *nodeId, *hostIp) } else { - // docker host name normally is hostname - *nodeId = dockerInfo.Name log.Printf("Docker host: %s (%s)", *nodeId, *hostIp) } From 97f2cb3753b28fadff762c18c06c933f72c35247 Mon Sep 17 00:00:00 2001 From: muradm Date: Fri, 30 Sep 2016 08:36:32 +0400 Subject: [PATCH 4/5] last moment typo --- bridge/bridge.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridge/bridge.go b/bridge/bridge.go index 1a7392d58..b9388ad00 100644 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -399,7 +399,7 @@ func (b *Bridge) registerSwarmVipService(serviceName string, inside bool, vip st } // tag it for convenience if protocol != swarm.PortConfigProtocolTCP { - svcReg.Tags = combineTags(tag, b.config.ForceTags, protocol) + svcReg.Tags = combineTags(tag, b.config.ForceTags, string(protocol)) } else { svcReg.Tags = combineTags(tag, b.config.ForceTags) } From 13723404cf3115d7f3d14306933f20db996c3577 Mon Sep 17 00:00:00 2001 From: muradm Date: Fri, 30 Sep 2016 17:35:45 +0400 Subject: [PATCH 5/5] fixing wrong network filtering condition --- bridge/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bridge/util.go b/bridge/util.go index 67b24f4e4..f589c38b3 100644 --- a/bridge/util.go +++ b/bridge/util.go @@ -92,7 +92,7 @@ func servicePort(container *dockerapi.Container, port dockerapi.Port, published for name, network := range container.NetworkSettings.Networks { // if in swarm mode and network name is ingress, // no point to publish service with ingress network IP address - if _, ok := container.Config.Labels["com.docker.swarm.service.name"]; ok && name != "ingress" { + if _, ok := container.Config.Labels["com.docker.swarm.service.name"]; !(ok && name == "ingress") { eip = network.IPAddress } }