From a31e6ac5adc21e0158a108984b6a12666d39e350 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Wed, 26 Oct 2016 12:47:07 +0100 Subject: [PATCH 01/20] Install kubernetes Ubuntu 16.04 packages --- admin/acceptance.py | 94 ++++++++++++++++++++++++++++++++++- flocker/provision/_install.py | 61 +++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/admin/acceptance.py b/admin/acceptance.py index 84f2d52787..dc6497dd1b 100644 --- a/admin/acceptance.py +++ b/admin/acceptance.py @@ -48,6 +48,7 @@ ) from flocker.provision._install import ( ManagedNode, + install_kubernetes, task_pull_docker_images, uninstall_flocker, install_flocker, @@ -335,6 +336,7 @@ def install(ignored): install_flocker(nodes, package_source), ) installing = uninstalling.addCallback(install) + return installing def ensure_keys(self, reactor): @@ -355,7 +357,7 @@ def start_cluster(self, reactor): ) else: upgrading = succeed(None) - + return upgrading def configure(ignored): return configured_cluster_for_nodes( reactor, @@ -388,6 +390,80 @@ def extend_cluster(self, reactor, cluster, count, tag, starting_index): raise UsageError("Extending a cluster with managed nodes " "is not implemented yet.") +@implementer(IClusterRunner) +class KubernetesRunner(object): + """ + """ + def __init__(self, node_addresses, package_source, distribution, + dataset_backend, dataset_backend_configuration, identity, + cert_path, logging_config): + """ + :param list: A ``list`` of public IP addresses or + ``[private_address, public_address]`` lists. + + See ``ManagedRunner`` and ``ManagedNode`` for other parameter + documentation. + """ + self._nodes = pvector( + make_managed_nodes(node_addresses, distribution) + ) + self.package_source = package_source + self.dataset_backend = dataset_backend + self.dataset_backend_configuration = dataset_backend_configuration + self.identity = identity + self.cert_path = cert_path + self.logging_config = logging_config + + def ensure_keys(self, reactor): + """ + Assume we have keys, since there's no way of asking the nodes what keys + they'll accept. + """ + return succeed(None) + + def start_cluster(self, reactor): + """ + Don't start any nodes. Give back the addresses of the configured, + already-started nodes. + """ + dispatcher = make_dispatcher(reactor) + installing = perform( + dispatcher, + install_kubernetes(self._nodes, self.package_source), + ) + + def configure(ignored): + return configured_cluster_for_nodes( + reactor, + generate_certificates( + self.identity.name, + self.identity.id, + self._nodes, + self.cert_path, + ), + self._nodes, + self.dataset_backend, + self.dataset_backend_configuration, + save_backend_configuration( + self.dataset_backend, + self.dataset_backend_configuration + ), + provider="managed", + logging_config=self.logging_config, + ) + configuring = installing.addCallback(configure) + return configuring + + def stop_cluster(self, reactor): + """ + Don't stop any nodes. + """ + return succeed(None) + + def extend_cluster(self, reactor, cluster, count, tag, starting_index): + raise UsageError("Extending a cluster with managed nodes " + "is not implemented yet.") + def generate_certificates(cluster_name, cluster_id, nodes, cert_path): """ @@ -1094,6 +1170,22 @@ def _runner_MANAGED(self, package_source, dataset_backend, logging_config=self['config'].get('logging'), ) + def _runner_KUBERNETES(self, package_source, dataset_backend, + provider_config): + if provider_config is None: + self._provider_config_missing("kubernetes") + + return KubernetesRunner( + node_addresses=provider_config['addresses'], + package_source=None, + distribution=self['distribution'], + dataset_backend=dataset_backend, + dataset_backend_configuration=self.dataset_backend_configuration(), + identity=self._make_cluster_identity(dataset_backend), + cert_path=self['cert-directory'], + logging_config=self['config'].get('logging'), + ) + def _libcloud_runner(self, package_source, dataset_backend, provider, provider_config): """ diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 3e7f9304a9..d28535cb37 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1384,6 +1384,55 @@ def task_install_docker(distribution): timeout=5.0 * 60.0, ) +GOOGLE_CLOUD_PACKAGES_KEY = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQENBFUd6rIBCAD6mhKRHDn3UrCeLDp7U5IE7AhhrOCPpqGF7mfTemZYHf/5Jdjx +cOxoSFlK7zwmFr3lVqJ+tJ9L1wd1K6P7RrtaNwCiZyeNPf/Y86AJ5NJwBe0VD0xH +TXzPNTqRSByVYtdN94NoltXUYFAAPZYQls0x0nUD1hLMlOlC2HdTPrD1PMCnYq/N +uL/Vk8sWrcUt4DIS+0RDQ8tKKe5PSV0+PnmaJvdF5CKawhh0qGTklS2MXTyKFoqj +XgYDfY2EodI9ogT/LGr9Lm/+u4OFPvmN9VN6UG+s0DgJjWvpbmuHL/ZIRwMEn/tp +uneaLTO7h1dCrXC849PiJ8wSkGzBnuJQUbXnABEBAAG0QEdvb2dsZSBDbG91ZCBQ +YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv +bT6JAT4EEwECACgFAlUd6rICGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B +AheAAAoJEDdGwginMXsPcLcIAKi2yNhJMbu4zWQ2tM/rJFovazcY28MF2rDWGOnc +9giHXOH0/BoMBcd8rw0lgjmOosBdM2JT0HWZIxC/Gdt7NSRA0WOlJe04u82/o3OH +WDgTdm9MS42noSP0mvNzNALBbQnlZHU0kvt3sV1YsnrxljoIuvxKWLLwren/GVsh +FLPwONjw3f9Fan6GWxJyn/dkX3OSUGaduzcygw51vksBQiUZLCD2Tlxyr9NvkZYT +qiaWW78L6regvATsLc9L/dQUiSMQZIK6NglmHE+cuSaoK0H4ruNKeTiQUw/EGFaL +ecay6Qy/s3Hk7K0QLd+gl0hZ1w1VzIeXLo2BRlqnjOYFX4A= +=HVTm +-----END PGP PUBLIC KEY BLOCK----- +""" + + +def task_install_kubernetes(distribution): + """ + Return an ``Effect`` for installing Kubernetes + """ + key_path = b"/etc/gc-team@google.com.gpg.pub" + source_path = b"/etc/apt/sources.list.d/kubernetes.list" + + return sequence([ + # Upload the public key rather than downloading from the kubernetes + # servers every time. + put(GOOGLE_CLOUD_PACKAGES_KEY, key_path), + # Upload the apt repo URL + put( + b"deb http://apt.kubernetes.io/ kubernetes-xenial main\n", + source_path + ), + # Install the key + run(command=b"apt-key add {}".format(key_path)), + # Install Kubernetes packages + run(command=b"apt-get update"), + run(command=( + b"apt-get install -y " + b"kubelet kubeadm kubectl kubernetes-cni" + )) + ]) + def task_install_flocker( distribution=None, @@ -1495,6 +1544,7 @@ def provision(distribution, package_source, variants): if Variants.DOCKER_HEAD in variants: commands.append(task_enable_docker_head_repository(distribution)) commands.append(task_install_docker(distribution)) + commands.append(task_install_kubernetes(distribution)) commands.append( task_install_flocker( package_source=package_source, distribution=distribution)) @@ -1552,6 +1602,17 @@ def install_flocker(nodes, package_source): ) +def install_kubernetes(nodes, package_source): + return _run_on_all_nodes( + nodes, + task=lambda node: sequence([ + task_install_kubernetes( + distribution=node.distribution, + ), + ]), + ) + + def configure_cluster( cluster, dataset_backend_configuration, provider, logging_config=None ): From 6de873b5ee36d93cec90cb66b18e4d6a45a70d70 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Wed, 26 Oct 2016 13:18:25 +0100 Subject: [PATCH 02/20] Attempt to configure a master and nodes --- admin/acceptance.py | 4 ++- flocker/provision/_install.py | 66 ++++++++++++++++++++++++++++++----- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/admin/acceptance.py b/admin/acceptance.py index dc6497dd1b..cec517187e 100644 --- a/admin/acceptance.py +++ b/admin/acceptance.py @@ -11,6 +11,7 @@ from base64 import b32encode from pipes import quote as shell_quote from tempfile import mkdtemp +from uuid import uuid4 from zope.interface import Interface, implementer from characteristic import attributes @@ -426,10 +427,11 @@ def start_cluster(self, reactor): Don't start any nodes. Give back the addresses of the configured, already-started nodes. """ + token = unicode(uuid4()) dispatcher = make_dispatcher(reactor) installing = perform( dispatcher, - install_kubernetes(self._nodes, self.package_source), + install_kubernetes(self._nodes, self.package_source, token), ) def configure(ignored): diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index d28535cb37..2c79838e15 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1434,6 +1434,31 @@ def task_install_kubernetes(distribution): ]) +def task_configure_kubernetes_master(distribution, token): + """ + Return an ``Effect`` for installing Kubernetes + """ + return sequence([ + run( + command=b"kubeadm init --token {}".format(token) + ) + ]) + + +def task_configure_kubernetes_node(distribution, token, master_ip): + """ + Return an ``Effect`` for installing Kubernetes + """ + return sequence([ + run( + command=b"kubeadm join --token {} {}".format( + token, + master_ip, + ) + ) + ]) + + def task_install_flocker( distribution=None, package_source=PackageSource(), @@ -1602,15 +1627,38 @@ def install_flocker(nodes, package_source): ) -def install_kubernetes(nodes, package_source): - return _run_on_all_nodes( - nodes, - task=lambda node: sequence([ - task_install_kubernetes( - distribution=node.distribution, - ), - ]), - ) +def install_kubernetes(nodes, package_source, token): + master = nodes[0] + workers = nodes[1:] + return sequence([ + _run_on_all_nodes( + nodes, + task=lambda node: sequence([ + task_install_kubernetes( + distribution=node.distribution, + ), + ]), + ), + _run_on_all_nodes( + [master], + command=lambda node: sequence([ + task_configure_kubernetes_master( + distribution=node.distribution, + token=token, + ) + ]), + ), + _run_on_all_nodes( + workers, + command=lambda node: sequence([ + task_configure_kubernetes_node( + distribution=node.distribution, + token=token, + master.public_ip + ) + ]), + ), + ]) def configure_cluster( From 2498c24303786b4e0375aa21e0344cb21b3bb5af Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Wed, 26 Oct 2016 16:06:25 +0100 Subject: [PATCH 03/20] uninstall kubernetes....for easier testing --- admin/acceptance.py | 19 +++++++++++---- flocker/provision/_install.py | 44 ++++++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/admin/acceptance.py b/admin/acceptance.py index cec517187e..2e955f9fe5 100644 --- a/admin/acceptance.py +++ b/admin/acceptance.py @@ -9,9 +9,10 @@ import json from itertools import repeat from base64 import b32encode +from hashlib import sha256 from pipes import quote as shell_quote +from random import getrandbits from tempfile import mkdtemp -from uuid import uuid4 from zope.interface import Interface, implementer from characteristic import attributes @@ -50,6 +51,7 @@ from flocker.provision._install import ( ManagedNode, install_kubernetes, + uninstall_kubernetes, task_pull_docker_images, uninstall_flocker, install_flocker, @@ -427,13 +429,20 @@ def start_cluster(self, reactor): Don't start any nodes. Give back the addresses of the configured, already-started nodes. """ - token = unicode(uuid4()) + # See https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go + random_bytes = sha256(bytes(getrandbits(64))).hexdigest().encode('ascii') + token = random_bytes[:6] + b"." + random_bytes[-16:] dispatcher = make_dispatcher(reactor) - installing = perform( + uninstalling = perform( dispatcher, - install_kubernetes(self._nodes, self.package_source, token), + uninstall_kubernetes(self._nodes, self.package_source, token), + ) + installing = uninstalling.addCallback( + lambda _ignored: perform( + dispatcher, + install_kubernetes(self._nodes, self.package_source, token), + ) ) - def configure(ignored): return configured_cluster_for_nodes( reactor, diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 2c79838e15..f6ea2caf05 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1630,6 +1630,7 @@ def install_flocker(nodes, package_source): def install_kubernetes(nodes, package_source, token): master = nodes[0] workers = nodes[1:] + return sequence([ _run_on_all_nodes( nodes, @@ -1641,7 +1642,7 @@ def install_kubernetes(nodes, package_source, token): ), _run_on_all_nodes( [master], - command=lambda node: sequence([ + task=lambda node: sequence([ task_configure_kubernetes_master( distribution=node.distribution, token=token, @@ -1650,17 +1651,54 @@ def install_kubernetes(nodes, package_source, token): ), _run_on_all_nodes( workers, - command=lambda node: sequence([ + task=lambda node: sequence([ task_configure_kubernetes_node( distribution=node.distribution, token=token, - master.public_ip + master_ip=master.address ) ]), ), ]) +def uninstall_kubernetes(nodes, package_source, token): + master = nodes[0] + workers = nodes[1:] + + return sequence([ + _run_on_all_nodes( + nodes, + task=lambda node: sequence([ + run( + command=( + b"apt-get -y remove " + b"kubelet kubeadm kubectl kubernetes-cni" + ) + ), + run( + command=( + b"docker ps --all --quiet | " + b"xargs --no-run-if-empty docker rm --force" + ) + ), + run( + command=( + b"find /var/lib/kubelet " + b"| xargs -n 1 findmnt -n -t tmpfs -o TARGET -T " + b"| uniq | xargs -r umount -v" + ) + ), + run( + command=( + b"rm -rf /etc/kubernetes /var/lib/kubelet /var/lib/etcd" + ) + ), + ]), + ), + ]) + + def configure_cluster( cluster, dataset_backend_configuration, provider, logging_config=None ): From 5bf1611f18f4ac1a26bde56c4f4fe1851571cc17 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Wed, 26 Oct 2016 17:03:14 +0100 Subject: [PATCH 04/20] Add weave networking after the nodes have joined --- flocker/provision/_install.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index f6ea2caf05..73ee202931 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1441,7 +1441,8 @@ def task_configure_kubernetes_master(distribution, token): return sequence([ run( command=b"kubeadm init --token {}".format(token) - ) + ), + run(command=b"kubectl taint nodes --all dedicated-"), ]) @@ -1646,7 +1647,8 @@ def install_kubernetes(nodes, package_source, token): task_configure_kubernetes_master( distribution=node.distribution, token=token, - ) + ), + ]), ), _run_on_all_nodes( @@ -1659,6 +1661,12 @@ def install_kubernetes(nodes, package_source, token): ) ]), ), + _run_on_all_nodes( + [master], + task=lambda node: sequence([ + run(command=b"kubectl apply -f https://git.io/weave-kube") + ]), + ), ]) From af1e19f438888cbe6df28691df743ee06f17c639 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Thu, 27 Oct 2016 12:18:36 +0100 Subject: [PATCH 05/20] Integrate the kubernetes configuration into the existing configure_cluster function --- admin/acceptance.py | 21 ++--- flocker/provision/_install.py | 141 +++++++++++++++------------------- 2 files changed, 66 insertions(+), 96 deletions(-) diff --git a/admin/acceptance.py b/admin/acceptance.py index 2e955f9fe5..9ad5f05eca 100644 --- a/admin/acceptance.py +++ b/admin/acceptance.py @@ -9,9 +9,7 @@ import json from itertools import repeat from base64 import b32encode -from hashlib import sha256 from pipes import quote as shell_quote -from random import getrandbits from tempfile import mkdtemp from zope.interface import Interface, implementer @@ -50,8 +48,7 @@ ) from flocker.provision._install import ( ManagedNode, - install_kubernetes, - uninstall_kubernetes, + deconfigure_kubernetes, task_pull_docker_images, uninstall_flocker, install_flocker, @@ -393,6 +390,7 @@ def extend_cluster(self, reactor, cluster, count, tag, starting_index): raise UsageError("Extending a cluster with managed nodes " "is not implemented yet.") + @implementer(IClusterRunner) class KubernetesRunner(object): """ @@ -429,19 +427,10 @@ def start_cluster(self, reactor): Don't start any nodes. Give back the addresses of the configured, already-started nodes. """ - # See https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go - random_bytes = sha256(bytes(getrandbits(64))).hexdigest().encode('ascii') - token = random_bytes[:6] + b"." + random_bytes[-16:] dispatcher = make_dispatcher(reactor) - uninstalling = perform( + deconfiguring = perform( dispatcher, - uninstall_kubernetes(self._nodes, self.package_source, token), - ) - installing = uninstalling.addCallback( - lambda _ignored: perform( - dispatcher, - install_kubernetes(self._nodes, self.package_source, token), - ) + deconfigure_kubernetes(self._nodes, self.package_source), ) def configure(ignored): return configured_cluster_for_nodes( @@ -462,7 +451,7 @@ def configure(ignored): provider="managed", logging_config=self.logging_config, ) - configuring = installing.addCallback(configure) + configuring = deconfiguring.addCallback(configure) return configuring def stop_cluster(self, reactor): diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 73ee202931..6f6727d9d2 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -13,6 +13,8 @@ from effect.retry import retry from time import time import yaml +from hashlib import sha256 +from random import getrandbits from zope.interface import implementer @@ -1407,6 +1409,15 @@ def task_install_docker(distribution): """ +def kubeadm_token_from_cluster(cluster): + # See https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go # noqa + hash_bytes = sha256( + cluster.certificates.cluster.certificate.getContent() + ).hexdigest().encode('ascii') + token = hash_bytes[:6] + b"." + hash_bytes[-16:] + return token + + def task_install_kubernetes(distribution): """ Return an ``Effect`` for installing Kubernetes @@ -1628,62 +1639,12 @@ def install_flocker(nodes, package_source): ) -def install_kubernetes(nodes, package_source, token): - master = nodes[0] - workers = nodes[1:] - - return sequence([ - _run_on_all_nodes( - nodes, - task=lambda node: sequence([ - task_install_kubernetes( - distribution=node.distribution, - ), - ]), - ), - _run_on_all_nodes( - [master], - task=lambda node: sequence([ - task_configure_kubernetes_master( - distribution=node.distribution, - token=token, - ), - - ]), - ), - _run_on_all_nodes( - workers, - task=lambda node: sequence([ - task_configure_kubernetes_node( - distribution=node.distribution, - token=token, - master_ip=master.address - ) - ]), - ), - _run_on_all_nodes( - [master], - task=lambda node: sequence([ - run(command=b"kubectl apply -f https://git.io/weave-kube") - ]), - ), - ]) - - -def uninstall_kubernetes(nodes, package_source, token): - master = nodes[0] - workers = nodes[1:] - +def deconfigure_kubernetes(nodes, package_source): return sequence([ _run_on_all_nodes( nodes, task=lambda node: sequence([ - run( - command=( - b"apt-get -y remove " - b"kubelet kubeadm kubectl kubernetes-cni" - ) - ), + run(command=b"systemctl stop kubelet"), run( command=( b"docker ps --all --quiet | " @@ -1693,7 +1654,9 @@ def uninstall_kubernetes(nodes, package_source, token): run( command=( b"find /var/lib/kubelet " - b"| xargs -n 1 findmnt -n -t tmpfs -o TARGET -T " + b"| xargs --no-run-if-empty " + b" --max-args 1 " + b" findmnt -n -t tmpfs -o TARGET -T " b"| uniq | xargs -r umount -v" ) ), @@ -1702,6 +1665,7 @@ def uninstall_kubernetes(nodes, package_source, token): b"rm -rf /etc/kubernetes /var/lib/kubelet /var/lib/etcd" ) ), + run(command=b"systemctl start kubelet"), ]), ), ]) @@ -1886,6 +1850,10 @@ def configure_control_node( cluster.control_node.distribution ) ), + task_configure_kubernetes_master( + distribution=cluster.control_node.distribution, + token=kubeadm_token_from_cluster(cluster), + ) ]), ) @@ -1913,36 +1881,49 @@ def configure_node( if provider == "managed": setup_action = 'restart' + if node is cluster.control_node: + commands = [] + else: + commands = [ + task_configure_kubernetes_node( + distribution=node.distribution, + token=kubeadm_token_from_cluster(cluster), + master_ip=cluster.control_node.address, + ), + ] + + commands.extend([ + task_install_node_certificates( + cluster.certificates.cluster.certificate, + certnkey.certificate, + certnkey.key), + task_install_api_certificates( + cluster.certificates.user.certificate, + cluster.certificates.user.key), + task_enable_docker(node.distribution), + if_firewall_available( + node.distribution, + open_firewall_for_docker_api(node.distribution), + ), + task_configure_flocker_agent( + control_node=cluster.control_node.address, + dataset_backend=cluster.dataset_backend, + dataset_backend_configuration=( + dataset_backend_configuration + ), + logging_config=logging_config, + ), + task_enable_docker_plugin(node.distribution), + task_enable_flocker_agent( + distribution=node.distribution, + action=setup_action, + ), + ]) + return run_remotely( username='root', address=node.address, - commands=sequence([ - task_install_node_certificates( - cluster.certificates.cluster.certificate, - certnkey.certificate, - certnkey.key), - task_install_api_certificates( - cluster.certificates.user.certificate, - cluster.certificates.user.key), - task_enable_docker(node.distribution), - if_firewall_available( - node.distribution, - open_firewall_for_docker_api(node.distribution), - ), - task_configure_flocker_agent( - control_node=cluster.control_node.address, - dataset_backend=cluster.dataset_backend, - dataset_backend_configuration=( - dataset_backend_configuration - ), - logging_config=logging_config, - ), - task_enable_docker_plugin(node.distribution), - task_enable_flocker_agent( - distribution=node.distribution, - action=setup_action, - ), - ]), + commands=sequence(commands), ) From eb2f70812f3dd86197e71d86ce9c91ee559ea2e2 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Thu, 27 Oct 2016 12:48:37 +0100 Subject: [PATCH 06/20] Do the kubernetes provisioning from the manages runner. --- admin/acceptance.py | 102 ++++------------------------------ flocker/provision/_install.py | 20 +++++-- 2 files changed, 25 insertions(+), 97 deletions(-) diff --git a/admin/acceptance.py b/admin/acceptance.py index 9ad5f05eca..8c5b63c864 100644 --- a/admin/acceptance.py +++ b/admin/acceptance.py @@ -306,7 +306,7 @@ def __init__(self, node_addresses, package_source, distribution, self.cert_path = cert_path self.logging_config = logging_config - def _upgrade_flocker(self, reactor, nodes, package_source): + def _upgrade_flocker(self, dispatcher, nodes, package_source): """ Put the version of Flocker indicated by ``package_source`` onto all of the given nodes. @@ -325,7 +325,7 @@ def _upgrade_flocker(self, reactor, nodes, package_source): :return: A ``Deferred`` that fires when the software has been upgraded. """ - dispatcher = make_dispatcher(reactor) + uninstalling = perform(dispatcher, uninstall_flocker(nodes)) uninstalling.addErrback(write_failure, logger=None) @@ -351,87 +351,21 @@ def start_cluster(self, reactor): Don't start any nodes. Give back the addresses of the configured, already-started nodes. """ + dispatcher = make_dispatcher(reactor) if self.package_source is not None: upgrading = self._upgrade_flocker( - reactor, self._nodes, self.package_source + dispatcher, self._nodes, self.package_source ) else: upgrading = succeed(None) - return upgrading - def configure(ignored): - return configured_cluster_for_nodes( - reactor, - generate_certificates( - self.identity.name, - self.identity.id, - self._nodes, - self.cert_path, - ), - self._nodes, - self.dataset_backend, - self.dataset_backend_configuration, - save_backend_configuration( - self.dataset_backend, - self.dataset_backend_configuration - ), - provider="managed", - logging_config=self.logging_config, - ) - configuring = upgrading.addCallback(configure) - return configuring - - def stop_cluster(self, reactor): - """ - Don't stop any nodes. - """ - return succeed(None) - - def extend_cluster(self, reactor, cluster, count, tag, starting_index): - raise UsageError("Extending a cluster with managed nodes " - "is not implemented yet.") - - -@implementer(IClusterRunner) -class KubernetesRunner(object): - """ - """ - def __init__(self, node_addresses, package_source, distribution, - dataset_backend, dataset_backend_configuration, identity, - cert_path, logging_config): - """ - :param list: A ``list`` of public IP addresses or - ``[private_address, public_address]`` lists. - See ``ManagedRunner`` and ``ManagedNode`` for other parameter - documentation. - """ - self._nodes = pvector( - make_managed_nodes(node_addresses, distribution) + deconfiguring_kubernetes = upgrading.addCallback( + lambda _ignored: perform( + dispatcher, + deconfigure_kubernetes(self._nodes), + ) ) - self.package_source = package_source - self.dataset_backend = dataset_backend - self.dataset_backend_configuration = dataset_backend_configuration - self.identity = identity - self.cert_path = cert_path - self.logging_config = logging_config - def ensure_keys(self, reactor): - """ - Assume we have keys, since there's no way of asking the nodes what keys - they'll accept. - """ - return succeed(None) - - def start_cluster(self, reactor): - """ - Don't start any nodes. Give back the addresses of the configured, - already-started nodes. - """ - dispatcher = make_dispatcher(reactor) - deconfiguring = perform( - dispatcher, - deconfigure_kubernetes(self._nodes, self.package_source), - ) def configure(ignored): return configured_cluster_for_nodes( reactor, @@ -451,7 +385,7 @@ def configure(ignored): provider="managed", logging_config=self.logging_config, ) - configuring = deconfiguring.addCallback(configure) + configuring = deconfiguring_kubernetes.addCallback(configure) return configuring def stop_cluster(self, reactor): @@ -1170,22 +1104,6 @@ def _runner_MANAGED(self, package_source, dataset_backend, logging_config=self['config'].get('logging'), ) - def _runner_KUBERNETES(self, package_source, dataset_backend, - provider_config): - if provider_config is None: - self._provider_config_missing("kubernetes") - - return KubernetesRunner( - node_addresses=provider_config['addresses'], - package_source=None, - distribution=self['distribution'], - dataset_backend=dataset_backend, - dataset_backend_configuration=self.dataset_backend_configuration(), - identity=self._make_cluster_identity(dataset_backend), - cert_path=self['cert-directory'], - logging_config=self['config'].get('logging'), - ) - def _libcloud_runner(self, package_source, dataset_backend, provider, provider_config): """ diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 6f6727d9d2..3751af64cc 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -14,7 +14,6 @@ from time import time import yaml from hashlib import sha256 -from random import getrandbits from zope.interface import implementer @@ -1639,7 +1638,16 @@ def install_flocker(nodes, package_source): ) -def deconfigure_kubernetes(nodes, package_source): +def deconfigure_kubernetes(nodes): + """ + See: http://kubernetes.io/docs/getting-started-guides/kubeadm/#cleanup + + :param nodes: An iterable of ``Node`` instances on which to de-configure + Kubernetes. + + :return: An ``Effect`` which removes all Kubernetes pods / containers and + configuration. + """ return sequence([ _run_on_all_nodes( nodes, @@ -1656,13 +1664,15 @@ def deconfigure_kubernetes(nodes, package_source): b"find /var/lib/kubelet " b"| xargs --no-run-if-empty " b" --max-args 1 " - b" findmnt -n -t tmpfs -o TARGET -T " - b"| uniq | xargs -r umount -v" + b" findmnt --noheadings --types tmpfs " + b" --output TARGET --target " + b"| uniq | xargs --no-run-if-empty umount" ) ), run( command=( - b"rm -rf /etc/kubernetes /var/lib/kubelet /var/lib/etcd" + b"rm -rf " + b"/etc/kubernetes /var/lib/kubelet /var/lib/etcd" ) ), run(command=b"systemctl start kubelet"), From b10116f867c9c41c3efb7f58cd14169ae041bdbb Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Thu, 27 Oct 2016 15:41:47 +0100 Subject: [PATCH 07/20] Import the repo key to a separate keyring and add docstrings. --- admin/acceptance.py | 2 -- flocker/provision/_install.py | 56 +++++++++++++++++++++++++++++------ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/admin/acceptance.py b/admin/acceptance.py index 8c5b63c864..bb203ab0c1 100644 --- a/admin/acceptance.py +++ b/admin/acceptance.py @@ -325,8 +325,6 @@ def _upgrade_flocker(self, dispatcher, nodes, package_source): :return: A ``Deferred`` that fires when the software has been upgraded. """ - - uninstalling = perform(dispatcher, uninstall_flocker(nodes)) uninstalling.addErrback(write_failure, logger=None) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 3751af64cc..2804e9d836 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1385,6 +1385,8 @@ def task_install_docker(distribution): timeout=5.0 * 60.0, ) +# Hard coded Kubernetes repository key. +# In PGP ASCII armor. GOOGLE_CLOUD_PACKAGES_KEY = """ -----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 @@ -1409,7 +1411,17 @@ def task_install_docker(distribution): def kubeadm_token_from_cluster(cluster): - # See https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go # noqa + """ + Generate a stable ``kubeadmin --token`` parameter for the supplied + ``cluster``. + + See: https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go # noqa + + :param Cluster cluster: The cluster supplied by the acceptance test runner. + :returns: A "6.16" byte string in the format expected by + kubeadm --token. + """ + # hash_bytes = sha256( cluster.certificates.cluster.certificate.getContent() ).hexdigest().encode('ascii') @@ -1419,22 +1431,32 @@ def kubeadm_token_from_cluster(cluster): def task_install_kubernetes(distribution): """ - Return an ``Effect`` for installing Kubernetes + Install Kubernetes packages. + + :param unicode distribution: The name of the target OS distribution. + :returns: an ``Effect`` for installing Kubernetes packages from the + Kubernetes repository. """ - key_path = b"/etc/gc-team@google.com.gpg.pub" - source_path = b"/etc/apt/sources.list.d/kubernetes.list" + if distribution not in ("ubuntu-16.04",): + return sequence([]) + key_path = b"/etc/apt/trusted.gpg.d/gc-team@google.com.gpg" + source_path = b"/etc/apt/sources.list.d/kubernetes.list" return sequence([ # Upload the public key rather than downloading from the kubernetes # servers every time. - put(GOOGLE_CLOUD_PACKAGES_KEY, key_path), + put(GOOGLE_CLOUD_PACKAGES_KEY, key_path + b".asc"), # Upload the apt repo URL put( b"deb http://apt.kubernetes.io/ kubernetes-xenial main\n", source_path ), - # Install the key - run(command=b"apt-key add {}".format(key_path)), + # Install the key Kubernetes key + run( + command=b"apt-key --keyring {} add {}.asc".format( + key_path, key_path + ) + ), # Install Kubernetes packages run(command=b"apt-get update"), run(command=( @@ -1446,20 +1468,36 @@ def task_install_kubernetes(distribution): def task_configure_kubernetes_master(distribution, token): """ - Return an ``Effect`` for installing Kubernetes + Configure a Kubernetes master and allow that node to also run pods. + + :param unicode distribution: The name of the target OS distribution. + :param bytes token: A ``kubeadm`` token. + :returns: an ``Effect`` for configuring the Kubernetes master node. """ + if distribution not in ("ubuntu-16.04",): + return sequence([]) + return sequence([ run( command=b"kubeadm init --token {}".format(token) ), + # Allow pods to be scheduled to the master node too. run(command=b"kubectl taint nodes --all dedicated-"), ]) def task_configure_kubernetes_node(distribution, token, master_ip): """ - Return an ``Effect`` for installing Kubernetes + Configure a Kubernetes worker node and join it to the master configured in + ``task_configure_kubernetes_master``. + + :param unicode distribution: The name of the target OS distribution. + :param bytes token: A ``kubeadm`` token. + :returns: an ``Effect`` for running ``kubeadm --join``. """ + if distribution not in ("ubuntu-16.04",): + return sequence([]) + return sequence([ run( command=b"kubeadm join --token {} {}".format( From 163132d2a09b1a80d3b3ef8f2216f96faab97e87 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Mon, 31 Oct 2016 16:39:18 +0000 Subject: [PATCH 08/20] Install Weave networking daemonset --- flocker/provision/_install.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 2804e9d836..2803a8705f 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1465,6 +1465,13 @@ def task_install_kubernetes(distribution): )) ]) +# XXX Maybe copy the entire configuration here to avoid failures due to flaky +# downloads. +KUBERNETES_ADDON_WEAVE = ( + b"https://raw.githubusercontent.com" + b"/weaveworks/weave-kube/v1.7.2/weave-daemonset.yaml" +) + def task_configure_kubernetes_master(distribution, token): """ @@ -1483,6 +1490,11 @@ def task_configure_kubernetes_master(distribution, token): ), # Allow pods to be scheduled to the master node too. run(command=b"kubectl taint nodes --all dedicated-"), + # Install a network addon. The weave daemonset will be started on the + # nodes as they join the cluster in ``task_configure_kubernetes_node``. + run( + command=b"kubectl apply --filename " + KUBERNETES_ADDON_WEAVE + ), ]) From b1b803e21ea102a905c383939a4243bfd5dc6582 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Tue, 1 Nov 2016 14:08:40 +0000 Subject: [PATCH 09/20] Non-working Centos installation tasks --- flocker/provision/_install.py | 147 +++++++++++++++++++++++++++------- 1 file changed, 117 insertions(+), 30 deletions(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 2803a8705f..b00f3f8619 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -133,6 +133,17 @@ def is_systemd_distribution(distribution): distribution == "ubuntu-16.04" ) +_distribution_to_package_format = { + "centos-7": "rpm", + "rhel-7.2": "rpm", + "ubuntu-16.04": "deb", + "ubuntu-14.04": "deb", +} + + +def package_format_for_distribution(distribution): + return _distribution_to_package_format[distribution] + def _from_args(sudo): """ @@ -1385,9 +1396,10 @@ def task_install_docker(distribution): timeout=5.0 * 60.0, ) -# Hard coded Kubernetes repository key. -# In PGP ASCII armor. -GOOGLE_CLOUD_PACKAGES_KEY = """ +# Used for signing yum and apt repo metadata +# pub 2048R/A7317B0F 2015-04-03 [expires: 2018-04-02] +# uid Google Cloud Packages Automatic Signing Key +GOOGLE_CLOUD_PACKAGES_KEY_AUTOMATIC = """ -----BEGIN PGP PUBLIC KEY BLOCK----- Version: GnuPG v1 @@ -1409,6 +1421,50 @@ def task_install_docker(distribution): -----END PGP PUBLIC KEY BLOCK----- """ +# Used for signing RPM packages. +# pub 2048R/3E1BA8D5 2015-06-24 +# uid Google Cloud Packages RPM Signing Key +GOOGLE_CLOUD_PACKAGES_KEY_RPM = """ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQENBFWKtqgBCADmKQWYQF9YoPxLEQZ5XA6DFVg9ZHG4HIuehsSJETMPQ+W9K5c5 +Us5assCZBjG/k5i62SmWb09eHtWsbbEgexURBWJ7IxA8kM3kpTo7bx+LqySDsSC3 +/8JRkiyibVV0dDNv/EzRQsGDxmk5Xl8SbQJ/C2ECSUT2ok225f079m2VJsUGHG+5 +RpyHHgoMaRNedYP8ksYBPSD6sA3Xqpsh/0cF4sm8QtmsxkBmCCIjBa0B0LybDtdX +XIq5kPJsIrC2zvERIPm1ez/9FyGmZKEFnBGeFC45z5U//pHdB1z03dYKGrKdDpID +17kNbC5wl24k/IeYyTY9IutMXvuNbVSXaVtRABEBAAG0Okdvb2dsZSBDbG91ZCBQ +YWNrYWdlcyBSUE0gU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNvbT6JATgE +EwECACIFAlWKtqgCGy8GCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEPCcOUw+ +G6jV+QwH/0wRH+XovIwLGfkg6kYLEvNPvOIYNQWnrT6zZ+XcV47WkJ+i5SR+QpUI +udMSWVf4nkv+XVHruxydafRIeocaXY0E8EuIHGBSB2KR3HxG6JbgUiWlCVRNt4Qd +6udC6Ep7maKEIpO40M8UHRuKrp4iLGIhPm3ELGO6uc8rks8qOBMH4ozU+3PB9a0b +GnPBEsZdOBI1phyftLyyuEvG8PeUYD+uzSx8jp9xbMg66gQRMP9XGzcCkD+b8w1o +7v3J3juKKpgvx5Lqwvwv2ywqn/Wr5d5OBCHEw8KtU/tfxycz/oo6XUIshgEbS/+P +6yKDuYhRp6qxrYXjmAszIT25cftb4d4= +=/PbX +-----END PGP PUBLIC KEY BLOCK----- +""" + +KUBERNETES_REPO_APT = """ +deb http://apt.kubernetes.io/ kubernetes-xenial main +""" + +KUBERNETES_REPO_PATH_APT = "/etc/apt/sources.list.d/kubernetes.list" + +KUBERNETES_REPO_YUM = """ +[kubernetes] +name=Kubernetes +baseurl=http://yum.kubernetes.io/repos/kubernetes-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +""" + +KUBERNETES_REPO_PATH_YUM = "/etc/yum.repos.d/kubernetes.repo" + def kubeadm_token_from_cluster(cluster): """ @@ -1429,34 +1485,23 @@ def kubeadm_token_from_cluster(cluster): return token -def task_install_kubernetes(distribution): - """ - Install Kubernetes packages. - - :param unicode distribution: The name of the target OS distribution. - :returns: an ``Effect`` for installing Kubernetes packages from the - Kubernetes repository. - """ - if distribution not in ("ubuntu-16.04",): - return sequence([]) - - key_path = b"/etc/apt/trusted.gpg.d/gc-team@google.com.gpg" - source_path = b"/etc/apt/sources.list.d/kubernetes.list" +def task_install_kubernetes_apt(): + key_path = b"/etc/apt/trusted.gpg.d/google_cloud_packages_automatic" return sequence([ # Upload the public key rather than downloading from the kubernetes # servers every time. - put(GOOGLE_CLOUD_PACKAGES_KEY, key_path + b".asc"), - # Upload the apt repo URL - put( - b"deb http://apt.kubernetes.io/ kubernetes-xenial main\n", - source_path - ), + put(GOOGLE_CLOUD_PACKAGES_KEY_AUTOMATIC, key_path + b".asc"), # Install the key Kubernetes key run( command=b"apt-key --keyring {} add {}.asc".format( key_path, key_path ) ), + # Upload the repo file + put( + KUBERNETES_REPO_APT, + KUBERNETES_REPO_PATH_APT + ), # Install Kubernetes packages run(command=b"apt-get update"), run(command=( @@ -1465,6 +1510,54 @@ def task_install_kubernetes(distribution): )) ]) + +def task_install_kubernetes_yum(): + key_paths = [ + (GOOGLE_CLOUD_PACKAGES_KEY_AUTOMATIC, + b"/etc/pki/rpm-gpg/google_cloud_packages_automatic"), + (GOOGLE_CLOUD_PACKAGES_KEY_RPM, + b"/etc/pki/rpm-gpg/google_cloud_packages_rpm"), + ] + key_operations = [] + for key_content, key_path in key_paths: + key_operations += [ + put(key_content, key_path), + run(b"rpmkeys --import " + key_path) + ] + return sequence( + # Upload and import YUM and RPM signing keys + key_operations + [ + # Upload the repo file + put( + KUBERNETES_REPO_YUM, + KUBERNETES_REPO_PATH_YUM + ), + # Install Kubernetes packages + run(command=( + b"yum install -y " + b"kubelet kubeadm kubectl kubernetes-cni" + )), + ] + ) + +_task_install_kubernetes_variants = { + 'deb': task_install_kubernetes_apt, + 'rpm': task_install_kubernetes_yum, +} + + +def task_install_kubernetes(distribution): + """ + Install Kubernetes packages. + + :param unicode distribution: The name of the target OS distribution. + :returns: an ``Effect`` for installing Kubernetes packages from the + Kubernetes repository. + """ + package_format = package_format_for_distribution(distribution) + return _task_install_kubernetes_variants[package_format]() + + # XXX Maybe copy the entire configuration here to avoid failures due to flaky # downloads. KUBERNETES_ADDON_WEAVE = ( @@ -1481,9 +1574,6 @@ def task_configure_kubernetes_master(distribution, token): :param bytes token: A ``kubeadm`` token. :returns: an ``Effect`` for configuring the Kubernetes master node. """ - if distribution not in ("ubuntu-16.04",): - return sequence([]) - return sequence([ run( command=b"kubeadm init --token {}".format(token) @@ -1507,16 +1597,13 @@ def task_configure_kubernetes_node(distribution, token, master_ip): :param bytes token: A ``kubeadm`` token. :returns: an ``Effect`` for running ``kubeadm --join``. """ - if distribution not in ("ubuntu-16.04",): - return sequence([]) - return sequence([ run( command=b"kubeadm join --token {} {}".format( token, master_ip, ) - ) + ), ]) @@ -1913,7 +2000,7 @@ def configure_control_node( task_configure_kubernetes_master( distribution=cluster.control_node.distribution, token=kubeadm_token_from_cluster(cluster), - ) + ), ]), ) From 23e6361c46c2be237e70b3d4d7fd9ed1e878ab09 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Mon, 7 Nov 2016 17:52:51 +0000 Subject: [PATCH 10/20] Start kublet and docker daemon separately. --- flocker/provision/_install.py | 41 ++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index cec1e69f38..0de5e7c0e4 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -829,7 +829,7 @@ def task_install_api_certificates(api_cert, api_key): ]) -def task_enable_docker(distribution): +def task_configure_docker(distribution): """ Configure docker. @@ -885,6 +885,37 @@ def task_enable_docker(distribution): raise DistributionNotSupported(distribution=distribution) +def task_start_docker(distribution): + """ + foo + """ + if is_systemd_distribution(distribution): + commands = [ + run_from_args(['systemctl', START, 'docker']), + ] + elif is_ubuntu(distribution): + commands = [ + run_from_args(['service', 'docker', 'restart']), + ] + return sequence(commands) + + +def task_start_kubelet(distribution): + """ + foo + """ + if is_systemd_distribution(distribution): + commands = [ + run_from_args(['systemctl', 'enable' 'kubelet']), + run_from_args(['systemctl', START, 'kubelet']), + ] + elif is_ubuntu(distribution): + commands = [ + run_from_args(['service', 'kubelet', 'restart']), + ] + return sequence(commands) + + def open_firewalld(service): """ Open firewalld port for a service. @@ -974,12 +1005,10 @@ def task_enable_docker_plugin(distribution): return sequence([ run_from_args(['systemctl', 'enable', 'flocker-docker-plugin']), run_from_args(['systemctl', START, 'flocker-docker-plugin']), - run_from_args(['systemctl', START, 'docker']), ]) elif is_ubuntu(distribution): return sequence([ run_from_args(['service', 'flocker-docker-plugin', 'restart']), - run_from_args(['service', 'docker', 'restart']), ]) else: raise DistributionNotSupported(distribution=distribution) @@ -1717,7 +1746,9 @@ def provision(distribution, package_source, variants): commands.append( task_install_docker_plugin( package_source=package_source, distribution=distribution)) - commands.append(task_enable_docker(distribution)) + commands.append(task_configure_docker(distribution)) + commands.append(task_start_docker(distribution)) + commands.append(task_start_kubelet(distribution)) return sequence(commands) @@ -2040,7 +2071,7 @@ def configure_node( task_install_api_certificates( cluster.certificates.user.certificate, cluster.certificates.user.key), - task_enable_docker(node.distribution), + task_configure_docker(node.distribution), if_firewall_available( node.distribution, open_firewall_for_docker_api(node.distribution), From 36637e01965c1fa67f7d2c9c353f79d9f4e5ef8a Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Thu, 10 Nov 2016 16:16:17 +0000 Subject: [PATCH 11/20] Juggle the setup steps and disable pre-flight checks to get kubadm working on Centos7 --- flocker/provision/_install.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 0de5e7c0e4..e4ffc58ce4 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -872,7 +872,6 @@ def task_configure_docker(distribution): ExecStart= ExecStart=/usr/bin/dockerd {} {} """.format(unixsock_opt, docker_tls_options))), - run_from_args(["systemctl", "enable", "docker.service"]), ]) elif is_ubuntu(distribution): return sequence([ @@ -891,6 +890,7 @@ def task_start_docker(distribution): """ if is_systemd_distribution(distribution): commands = [ + run_from_args(["systemctl", "enable", "docker.service"]), run_from_args(['systemctl', START, 'docker']), ] elif is_ubuntu(distribution): @@ -906,7 +906,7 @@ def task_start_kubelet(distribution): """ if is_systemd_distribution(distribution): commands = [ - run_from_args(['systemctl', 'enable' 'kubelet']), + run_from_args(['systemctl', 'enable', 'kubelet']), run_from_args(['systemctl', START, 'kubelet']), ] elif is_ubuntu(distribution): @@ -1615,15 +1615,19 @@ def task_configure_kubernetes_node(distribution, token, master_ip): Configure a Kubernetes worker node and join it to the master configured in ``task_configure_kubernetes_master``. + XXX Skip preflight checks until + https://github.com/kubernetes/kubernetes/issues/36301 is resolved. + :param unicode distribution: The name of the target OS distribution. :param bytes token: A ``kubeadm`` token. :returns: an ``Effect`` for running ``kubeadm --join``. """ return sequence([ run( - command=b"kubeadm join --token {} {}".format( - token, - master_ip, + command=( + b"kubeadm join --skip-preflight-checks --token " + + token + b" " + + master_ip ) ), ]) @@ -1746,7 +1750,6 @@ def provision(distribution, package_source, variants): commands.append( task_install_docker_plugin( package_source=package_source, distribution=distribution)) - commands.append(task_configure_docker(distribution)) commands.append(task_start_docker(distribution)) commands.append(task_start_kubelet(distribution)) return sequence(commands) @@ -2089,6 +2092,10 @@ def configure_node( distribution=node.distribution, action=setup_action, ), + # Restart docker after pushing the Flocker certificates and the Docker + # configuration modifications which make it use Flocker certificates + # for listening on a TLS / TCP port. + task_start_docker(node.distribution) ]) return run_remotely( From cbaa8270e7a3f5752c512b7a76f06fe92f259b51 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Mon, 14 Nov 2016 11:49:46 +0000 Subject: [PATCH 12/20] Fix task names for documentation tasks import. --- flocker/provision/_tasks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flocker/provision/_tasks.py b/flocker/provision/_tasks.py index da729324a7..a8aab35884 100644 --- a/flocker/provision/_tasks.py +++ b/flocker/provision/_tasks.py @@ -6,7 +6,8 @@ from ._install import ( task_create_flocker_pool_file, - task_enable_docker, + task_configure_docker, + task_start_docker, task_install_flocker, task_install_ssh_key, task_cli_pkg_install, @@ -22,7 +23,8 @@ __all__ = [ 'task_create_flocker_pool_file', - 'task_enable_docker', + 'task_configure_docker', + 'task_start_docker', 'task_install_flocker', 'task_install_ssh_key', 'task_cli_pkg_install', From 05bcc3ccf3665b4497d8b6c32e1bce1b73a377c9 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Mon, 14 Nov 2016 13:38:34 +0000 Subject: [PATCH 13/20] Docstrings --- flocker/provision/_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index e4ffc58ce4..28114309cf 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -886,7 +886,7 @@ def task_configure_docker(distribution): def task_start_docker(distribution): """ - foo + Enable and (re)start the Docker daemon. """ if is_systemd_distribution(distribution): commands = [ @@ -902,7 +902,7 @@ def task_start_docker(distribution): def task_start_kubelet(distribution): """ - foo + Enable and (re)start the Kubernetes Kubelet. """ if is_systemd_distribution(distribution): commands = [ From 2bfa812f4ae60e5b88675ad16d9f926202d2798d Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Mon, 14 Nov 2016 14:06:13 +0000 Subject: [PATCH 14/20] Use the correct Ubuntu 14.04 repository --- flocker/provision/_install.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 28114309cf..77480937fa 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1468,12 +1468,6 @@ def task_install_docker(distribution): -----END PGP PUBLIC KEY BLOCK----- """ -KUBERNETES_REPO_APT = """ -deb http://apt.kubernetes.io/ kubernetes-xenial main -""" - -KUBERNETES_REPO_PATH_APT = "/etc/apt/sources.list.d/kubernetes.list" - KUBERNETES_REPO_YUM = """ [kubernetes] name=Kubernetes @@ -1519,13 +1513,19 @@ def task_install_kubernetes_apt(): key_path, key_path ) ), - # Upload the repo file - put( - KUBERNETES_REPO_APT, - KUBERNETES_REPO_PATH_APT - ), - # Install Kubernetes packages + # Install Kubernetes repository + run(command=b"apt-get update"), + run(command=( + b"apt-get install -y " + b"apt-transport-https software-properties-common" + )), + run(command=( + b"add-apt-repository -y " + b"deb http://apt.kubernetes.io/ " + b"kubernetes-$(lsb_release --codename --short) main" + )), run(command=b"apt-get update"), + # Install Kubernetes packages run(command=( b"apt-get install -y " b"kubelet kubeadm kubectl kubernetes-cni" From ac4233241ee83a96a2f93f72aaa41624c0e5a8ec Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Mon, 14 Nov 2016 14:45:59 +0000 Subject: [PATCH 15/20] Use a single argument to add-apt-repository --- flocker/provision/_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index 77480937fa..abac8acd14 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1521,8 +1521,8 @@ def task_install_kubernetes_apt(): )), run(command=( b"add-apt-repository -y " - b"deb http://apt.kubernetes.io/ " - b"kubernetes-$(lsb_release --codename --short) main" + b'"deb http://apt.kubernetes.io/ ' + b'kubernetes-$(lsb_release --codename --short)"' )), run(command=b"apt-get update"), # Install Kubernetes packages From 6ac35170146e7a08a0f882600a805814bebd2de1 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Mon, 14 Nov 2016 15:01:22 +0000 Subject: [PATCH 16/20] Don't attempt to delete kubernetes related containers --- flocker/acceptance/testtools.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flocker/acceptance/testtools.py b/flocker/acceptance/testtools.py index b353ad224b..e644a79063 100644 --- a/flocker/acceptance/testtools.py +++ b/flocker/acceptance/testtools.py @@ -874,7 +874,15 @@ def cleanup_all_containers(_): # they're left over from previous test; they might e.g. # have a volume bind-mounted, preventing its destruction. for container in client.containers(): - client.remove_container(container["Id"], force=True) + # Don't attempt to remove containers related to + # orchestration frameworks + protected_container = False + label_keys = container["Labels"].keys() + for key in label_keys: + if key.startswith("io.kubernetes."): + protected_container = True + if not protected_container: + client.remove_container(container["Id"], force=True) def cleanup_flocker_containers(_): cleaning_containers = api_clean_state( From e98e40c144223b332f220739db2418848190bc93 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Mon, 14 Nov 2016 15:48:49 +0000 Subject: [PATCH 17/20] fix repo line --- flocker/provision/_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index abac8acd14..b4a6feefe4 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1522,7 +1522,7 @@ def task_install_kubernetes_apt(): run(command=( b"add-apt-repository -y " b'"deb http://apt.kubernetes.io/ ' - b'kubernetes-$(lsb_release --codename --short)"' + b'kubernetes-$(lsb_release --codename --short) main"' )), run(command=b"apt-get update"), # Install Kubernetes packages From 59463a78c0417fa3bbb02e84d7e0ed09e63a5a03 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Tue, 15 Nov 2016 11:05:31 +0000 Subject: [PATCH 18/20] The key must have a .gpg extension or it won't be used by the apt tools --- flocker/provision/_install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index b4a6feefe4..f9d1b28a58 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1502,7 +1502,8 @@ def kubeadm_token_from_cluster(cluster): def task_install_kubernetes_apt(): - key_path = b"/etc/apt/trusted.gpg.d/google_cloud_packages_automatic" + # The de-armored key must have a .gpg file extension + key_path = b"/etc/apt/trusted.gpg.d/google_cloud_packages_automatic.gpg" return sequence([ # Upload the public key rather than downloading from the kubernetes # servers every time. From b23ef6946d39cb006dfae6f78231de5c66052900 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Tue, 15 Nov 2016 11:07:28 +0000 Subject: [PATCH 19/20] Force ebtables to be installed since it's not a declared dependency of kubelet or kubeadm. See https://github.com/kubernetes/release/pull/197. --- flocker/provision/_install.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py index f9d1b28a58..9a807ba512 100644 --- a/flocker/provision/_install.py +++ b/flocker/provision/_install.py @@ -1556,9 +1556,11 @@ def task_install_kubernetes_yum(): KUBERNETES_REPO_PATH_YUM ), # Install Kubernetes packages + # XXX The ebtables dependency isn't declared by the kubelet + # package. See https://github.com/kubernetes/release/pull/197 run(command=( b"yum install -y " - b"kubelet kubeadm kubectl kubernetes-cni" + b"ebtables kubelet kubeadm kubectl kubernetes-cni" )), ] ) From 219f5da3eac2d53c95b62951c978d23a6b57fba7 Mon Sep 17 00:00:00 2001 From: Richard Wall Date: Tue, 15 Nov 2016 11:13:31 +0000 Subject: [PATCH 20/20] Don't run acceptance tests on Ubuntu 14.04 --- build.yaml | 113 ----------------------------------------------------- 1 file changed, 113 deletions(-) diff --git a/build.yaml b/build.yaml index 5a646eab1b..3251cfeb81 100644 --- a/build.yaml +++ b/build.yaml @@ -1063,31 +1063,6 @@ job_type: timeout: 45 directories_to_delete: *run_acceptance_directories_to_delete - run_acceptance_loopback_on_AWS_Ubuntu_Trusty_for: - on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium' - with_modules: *run_full_acceptance_modules - with_steps: - - { type: 'shell', - cli: [ *hashbang, *add_shell_functions, - *cleanup, *setup_venv, *setup_flocker_modules, - *check_version, - 'export DISTRIBUTION_NAME=ubuntu-14.04', - *build_sdist, *build_package, - *build_repo_metadata, - *setup_authentication, - 'export ACCEPTANCE_TEST_MODULE=${MODULE}', - 'export ACCEPTANCE_TEST_PROVIDER=aws', - *run_acceptance_loopback_tests, - *convert_results_to_junit, - *clean_packages, - *exit_with_return_code_from_test ] - } - clean_repo: true - archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case - publish_test_results: true - timeout: 45 - directories_to_delete: *run_acceptance_directories_to_delete - run_acceptance_loopback_on_AWS_Ubuntu_Xenial_for: on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium' with_modules: *run_full_acceptance_modules @@ -1301,36 +1276,6 @@ job_type: directories_to_delete: [] notify_slack: '#nightly-builds' - run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS: - at: '0 6 * * *' - # flocker.provision is responsible for creating the test nodes on - # so we can actually run run-acceptance-tests from GCE - on_nodes_with_labels: 'gce-ubuntu16' - with_steps: - - { type: 'shell', - cli: [ *hashbang, *add_shell_functions, - *cleanup, *setup_venv, *setup_flocker_modules, - *check_version, - 'export DISTRIBUTION_NAME=ubuntu-14.04', - *build_sdist, *build_package, - *build_repo_metadata, - *setup_authentication, - 'export ACCEPTANCE_TEST_MODULE=flocker.acceptance', - 'export ACCEPTANCE_TEST_PROVIDER=aws', - *run_acceptance_tests, - *convert_results_to_junit, - *clean_packages, - *exit_with_return_code_from_test ] - } - clean_repo: true - archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case - publish_test_results: true - # Similar to the reasoning for run_acceptance_on_AWS_CentOS_7_with_EBS - # but slightly shorter since Ubuntu runs the tests faster. - timeout: 90 - directories_to_delete: [] - notify_slack: '#nightly-builds' - run_acceptance_on_AWS_Ubuntu_Xenial_with_EBS: at: '0 6 * * *' # flocker.provision is responsible for creating the test nodes on @@ -1390,35 +1335,6 @@ job_type: directories_to_delete: [] notify_slack: '#nightly-builds' - run_acceptance_on_GCE_Ubuntu_Trusty_with_GCE: - at: '0 6 * * *' - # flocker.provision is responsible for creating the test nodes on - # so we can actually run run-acceptance-tests from GCE - on_nodes_with_labels: 'gce-ubuntu16' - with_steps: - - { type: 'shell', - cli: [ *hashbang, *add_shell_functions, - *cleanup, *setup_venv, *setup_flocker_modules, - *check_version, - 'export DISTRIBUTION_NAME=ubuntu-14.04', - *build_sdist, *build_package, - *build_repo_metadata, - *setup_authentication, - 'export ACCEPTANCE_TEST_MODULE=flocker.acceptance', - 'export ACCEPTANCE_TEST_PROVIDER=gce', - *run_acceptance_tests, - *convert_results_to_junit, - *clean_packages, - *exit_with_return_code_from_test ] - } - clean_repo: true - archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case - publish_test_results: true - # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS - timeout: 90 - directories_to_delete: [] - notify_slack: '#nightly-builds' - run_acceptance_on_GCE_Ubuntu_Xenial_with_GCE: at: '0 6 * * *' # flocker.provision is responsible for creating the test nodes on @@ -1477,35 +1393,6 @@ job_type: directories_to_delete: [] notify_slack: '#nightly-builds' - run_acceptance_on_Rackspace_Ubuntu_Trusty_with_Cinder: - at: '0 6 * * *' - # flocker.provision is responsible for creating the test nodes on - # so we can actually run run-acceptance-tests from GCE - on_nodes_with_labels: 'gce-ubuntu16' - with_steps: - - { type: 'shell', - cli: [ *hashbang, *add_shell_functions, - *cleanup, *setup_venv, *setup_flocker_modules, - *check_version, - 'export DISTRIBUTION_NAME=ubuntu-14.04', - *build_sdist, *build_package, - *build_repo_metadata, - *setup_authentication, - 'export ACCEPTANCE_TEST_MODULE=flocker.acceptance', - 'export ACCEPTANCE_TEST_PROVIDER=rackspace', - *run_acceptance_tests, - *convert_results_to_junit, - *clean_packages, - *exit_with_return_code_from_test ] - } - clean_repo: true - archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case - publish_test_results: true - # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS - timeout: 90 - directories_to_delete: [] - notify_slack: '#nightly-builds' - run_acceptance_on_Rackspace_Ubuntu_Xenial_with_Cinder: at: '0 6 * * *' # flocker.provision is responsible for creating the test nodes on