diff --git a/bridge/bridge.go b/bridge/bridge.go index 2387b769f..b9388ad00 100644 --- a/bridge/bridge.go +++ b/bridge/bridge.go @@ -3,9 +3,7 @@ package bridge import ( "errors" "log" - "net" "net/url" - "os" "path" "regexp" "strconv" @@ -13,6 +11,7 @@ import ( "sync" dockerapi "github.com/fsouza/go-dockerclient" + "github.com/docker/engine-api/types/swarm" ) var serviceIDPattern = regexp.MustCompile(`^(.+?):([a-zA-Z0-9][a-zA-Z0-9_.-]+):[0-9]+(?::udp)?$`) @@ -157,7 +156,7 @@ func (b *Bridge) Sync(quiet bool) { continue } serviceHostname := matches[1] - if serviceHostname != Hostname { + if serviceHostname != b.config.NodeId { // use node id as reference instead of hostname // ignore because registered on a different host continue } @@ -244,24 +243,27 @@ func (b *Bridge) add(containerId string, quiet bool) { b.services[container.ID] = append(b.services[container.ID], service) log.Println("added:", container.ID[:12], service.ID) } + + // if swarm container belongs to swarm mode service, publish VIP services + if swarmServiceName, ok := container.Config.Labels["com.docker.swarm.service.name"]; ok { + filters := map[string][]string{"name": {swarmServiceName}} + services, err := b.docker.ListServices(dockerapi.ListServicesOptions{Filters: filters}) + if err != nil { + log.Println("error listing swarm services, wont register VIP service", err) + } else if len(services) == 1 { // container cannot belong to no or more than one service + if services[0].Spec.EndpointSpec.Mode == swarm.ResolutionModeVIP { // endpoint should be VIP + if (len(services[0].Endpoint.VirtualIPs) > 0) { + b.registerSwarmVipServices(services[0]) + } + } + } + } } func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { container := port.container defaultName := strings.Split(path.Base(container.Config.Image), ":")[0] - // not sure about this logic. kind of want to remove it. - hostname := Hostname - if hostname == "" { - hostname = port.HostIP - } - if port.HostIP == "0.0.0.0" { - ip, err := net.ResolveIPAddr("ip", hostname) - if err == nil { - port.HostIP = ip.String() - } - } - if b.config.HostIp != "" { port.HostIP = b.config.HostIp } @@ -275,8 +277,17 @@ func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { service := new(Service) service.Origin = port - service.ID = hostname + ":" + container.Name[1:] + ":" + port.ExposedPort - service.Name = mapDefault(metadata, "name", defaultName) + + // consider swarm mode + if swarmServiceName, ok := port.container.Config.Labels["com.docker.swarm.service.name"]; ok { + // swarm mode has concept of services + service.Name = mapDefault(metadata, "name", swarmServiceName) + } else { + // use node id, which is more reliable + service.Name = mapDefault(metadata, "name", defaultName) + } + + service.ID = b.config.NodeId + ":" + container.Name[1:] + ":" + port.ExposedPort if isgroup && !metadataFromPort["name"] { service.Name += "-" + port.ExposedPort } @@ -313,6 +324,96 @@ func (b *Bridge) newService(port ServicePort, isgroup bool) *Service { return service } +// there are two types of endpoints VIP and DNS rr based +// DNS rr happens implicitly by registering multiple services with the same name +// so that no extra effort is required +// in case of VIP based services, user specifies the published ports +// which are equivalent of docker port binding, but works differently +// swarm mode provides ingress network, where services are load-balanced +// behind VIP address. From inside network (if there any) perspective +// only one service is need, with swarm mode assigned VIP address. +// From outside perspective, every docker host IP address becomes an entry point +// for load-balancer, so published ports shall be registered for each docker host +func (b *Bridge) registerSwarmVipServices(service swarm.Service) { + // if internal, register the internal VIP services + if b.config.Internal { + for _, vip := range service.Endpoint.VirtualIPs { + if network, err := b.docker.NetworkInfo(vip.NetworkID); err != nil { + log.Println("unable to inspect network while evaluating VIPs for service:", service.Spec.Name, err) + } else { + // no point to publish docker swarm internal ingress network VIP + if network.Name != "ingress" && len(vip.Addr) > 0 && strings.Contains(vip.Addr, "/") { + vipAddr := strings.Split(vip.Addr, "/")[0] + if len(service.Spec.EndpointSpec.Ports) > 0 { + b.registerSwarmVipServicePorts(service.Spec.Name, true, vipAddr, service.Spec.EndpointSpec.Ports) + } + // publish VIP in with out ports in any case + b.registerSwarmVipService(service.Spec.Name, true, vipAddr, false, 0, "ip") + } + } + } + } else { + // if there is no published ports, no point to register it out side + if len(service.Spec.EndpointSpec.Ports) > 0 { + b.registerSwarmVipServicePorts(service.Spec.Name, false, b.config.HostIp, service.Spec.EndpointSpec.Ports) + } + } +} + +// current implementation attempts to register VIP service every container add event +// better way could be to listen for service create events, however according to +// docker configuration there is no such events +// registrations created here are unique, and not based on containers +// so we will just create them and forget, i don't see proper way to cleanup them at the moment +func (b *Bridge) registerSwarmVipServicePorts(serviceName string, inside bool, vip string, ports []swarm.PortConfig) { + for _, port := range ports { + var portNum uint32 + if portNum = port.PublishedPort; inside { + // inside port is not translated to published port + portNum = port.TargetPort + } + + b.registerSwarmVipService(serviceName, inside, vip, true, int(portNum), port.Protocol) + } +} + +func (b *Bridge) registerSwarmVipService(serviceName string, inside bool, vip string, isGroup bool, port int, protocol swarm.PortConfigProtocol) { + var tag string + if tag = "vip-outside"; inside { + tag = "vip-inside" + } + + svcReg := new(Service) + + svcReg.Name = serviceName + "-" + tag + if isGroup { + svcReg.Name = svcReg.Name + "-" + strconv.Itoa(port) + } + + if inside { + // VIP is global and singleton, so we can use service name as service id + svcReg.ID = svcReg.Name + } else { + // VIP is actually host ip address or whatever provided by user + svcReg.ID = b.config.NodeId + "-" + svcReg.Name + } + // tag it for convenience + if protocol != swarm.PortConfigProtocolTCP { + svcReg.Tags = combineTags(tag, b.config.ForceTags, string(protocol)) + } else { + svcReg.Tags = combineTags(tag, b.config.ForceTags) + } + + svcReg.IP = vip + svcReg.Port = port + + err := b.registry.Register(svcReg) + if err != nil { + log.Println("register failed:", svcReg.Name, err) + } + log.Println("added:", svcReg.Name) +} + func (b *Bridge) remove(containerId string, deregister bool) { b.Lock() defer b.Unlock() @@ -370,11 +471,3 @@ func (b *Bridge) shouldRemove(containerId string) bool { } return false } - -var Hostname string - -func init() { - // It's ok for Hostname to ultimately be an empty string - // An empty string will fall back to trying to make a best guess - Hostname, _ = os.Hostname() -} diff --git a/bridge/types.go b/bridge/types.go index b1611127e..656b599ee 100644 --- a/bridge/types.go +++ b/bridge/types.go @@ -20,6 +20,7 @@ type RegistryAdapter interface { } type Config struct { + NodeId string HostIp string Internal bool ForceTags string diff --git a/bridge/util.go b/bridge/util.go index 3b450faff..f589c38b3 100644 --- a/bridge/util.go +++ b/bridge/util.go @@ -89,8 +89,12 @@ func servicePort(container *dockerapi.Container, port dockerapi.Port, published // Nir: support docker NetworkSettings eip = container.NetworkSettings.IPAddress if eip == "" { - for _, network := range container.NetworkSettings.Networks { - eip = network.IPAddress + for name, network := range container.NetworkSettings.Networks { + // if in swarm mode and network name is ingress, + // no point to publish service with ingress network IP address + if _, ok := container.Config.Labels["com.docker.swarm.service.name"]; !(ok && name == "ingress") { + eip = network.IPAddress + } } } diff --git a/registrator.go b/registrator.go index b76dc9441..35afdbd95 100644 --- a/registrator.go +++ b/registrator.go @@ -12,6 +12,7 @@ import ( dockerapi "github.com/fsouza/go-dockerclient" "github.com/gliderlabs/pkg/usage" "github.com/gliderlabs/registrator/bridge" + "github.com/docker/engine-api/types/swarm" ) var Version string @@ -95,7 +96,26 @@ func main() { assert(errors.New("-deregister must be \"always\" or \"on-success\"")) } + // use docker info to determine node id that will be used as prefix to service id + dockerInfo, err := docker.Info() + assert(err) + + nodeId := new(string) + // docker host name normally is hostname + *nodeId = dockerInfo.Name + if dockerInfo.Swarm.LocalNodeState != swarm.LocalNodeStateInactive { + if *hostIp == "" { + // in case of swarm mode, docker host has information about ip + // although it won't be always useful, we can use it if not provided by user + *hostIp = dockerInfo.Swarm.NodeAddr + } + log.Printf("Docker host in Swarm Mode: %s (%s)", *nodeId, *hostIp) + } else { + log.Printf("Docker host: %s (%s)", *nodeId, *hostIp) + } + b, err := bridge.New(docker, flag.Arg(0), bridge.Config{ + NodeId: *nodeId, HostIp: *hostIp, Internal: *internal, ForceTags: *forceTags,