diff options
Diffstat (limited to '')
| -rw-r--r-- | qa/tasks/kubeadm.py | 563 |
1 files changed, 563 insertions, 0 deletions
diff --git a/qa/tasks/kubeadm.py b/qa/tasks/kubeadm.py new file mode 100644 index 000000000..00417fc86 --- /dev/null +++ b/qa/tasks/kubeadm.py @@ -0,0 +1,563 @@ +""" +Kubernetes cluster task, deployed via kubeadm +""" +import argparse +import contextlib +import ipaddress +import json +import logging +import random +import yaml +from io import BytesIO + +from teuthology import misc as teuthology +from teuthology import contextutil +from teuthology.config import config as teuth_config +from teuthology.orchestra import run + +log = logging.getLogger(__name__) + + +def _kubectl(ctx, config, args, **kwargs): + cluster_name = config['cluster'] + ctx.kubeadm[cluster_name].bootstrap_remote.run( + args=['kubectl'] + args, + **kwargs, + ) + + +def kubectl(ctx, config): + if isinstance(config, str): + config = [config] + assert isinstance(config, list) + for c in config: + if isinstance(c, str): + _kubectl(ctx, config, c.split(' ')) + else: + _kubectl(ctx, config, c) + + +@contextlib.contextmanager +def preflight(ctx, config): + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'modprobe', 'br_netfilter', + run.Raw('&&'), + 'sudo', 'sysctl', 'net.bridge.bridge-nf-call-ip6tables=1', + run.Raw('&&'), + 'sudo', 'sysctl', 'net.bridge.bridge-nf-call-iptables=1', + run.Raw('&&'), + 'sudo', 'sysctl', 'net.ipv4.ip_forward=1', + run.Raw('&&'), + 'sudo', 'swapoff', '-a', + ], + wait=False, + ) + ) + + # set docker cgroup driver = systemd + # see https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker + # see https://github.com/kubernetes/kubeadm/issues/2066 + for remote in ctx.cluster.remotes.keys(): + try: + orig = remote.read_file('/etc/docker/daemon.json', sudo=True) + j = json.loads(orig) + except Exception as e: + log.info(f'Failed to pull old daemon.json: {e}') + j = {} + j["exec-opts"] = ["native.cgroupdriver=systemd"] + j["log-driver"] = "json-file" + j["log-opts"] = {"max-size": "100m"} + j["storage-driver"] = "overlay2" + remote.write_file('/etc/docker/daemon.json', json.dumps(j), sudo=True) + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'systemctl', 'restart', 'docker', + run.Raw('||'), + 'true', + ], + wait=False, + ) + ) + yield + + +@contextlib.contextmanager +def kubeadm_install(ctx, config): + version = config.get('version', '1.21') + + os_type = teuthology.get_distro(ctx) + os_version = teuthology.get_distro_version(ctx) + + try: + if os_type in ['centos', 'rhel']: + os = f"CentOS_{os_version.split('.')[0]}" + log.info('Installing cri-o') + run.wait( + ctx.cluster.run( + args=[ + 'sudo', + 'curl', '-L', '-o', + '/etc/yum.repos.d/devel:kubic:libcontainers:stable.repo', + f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{os}/devel:kubic:libcontainers:stable.repo', + run.Raw('&&'), + 'sudo', + 'curl', '-L', '-o', + f'/etc/yum.repos.d/devel:kubic:libcontainers:stable:cri-o:{version}.repo', + f'https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{version}/{os}/devel:kubic:libcontainers:stable:cri-o:{version}.repo', + run.Raw('&&'), + 'sudo', 'dnf', 'install', '-y', 'cri-o', + ], + wait=False, + ) + ) + + log.info('Installing kube{adm,ctl,let}') + repo = """[kubernetes] +name=Kubernetes +baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +""" + for remote in ctx.cluster.remotes.keys(): + remote.write_file( + '/etc/yum.repos.d/kubernetes.repo', + repo, + sudo=True, + ) + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'dnf', 'install', '-y', + 'kubelet', 'kubeadm', 'kubectl', + 'iproute-tc', 'bridge-utils', + ], + wait=False, + ) + ) + + # fix cni config + for remote in ctx.cluster.remotes.keys(): + conf = """# from https://github.com/cri-o/cri-o/blob/master/tutorials/kubernetes.md#flannel-network +{ + "name": "crio", + "type": "flannel" +} +""" + remote.write_file('/etc/cni/net.d/10-crio-flannel.conf', conf, sudo=True) + remote.run(args=[ + 'sudo', 'rm', '-f', + '/etc/cni/net.d/87-podman-bridge.conflist', + '/etc/cni/net.d/100-crio-bridge.conf', + ]) + + # start crio + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'systemctl', 'daemon-reload', + run.Raw('&&'), + 'sudo', 'systemctl', 'enable', 'crio', '--now', + ], + wait=False, + ) + ) + + elif os_type == 'ubuntu': + os = f"xUbuntu_{os_version}" + log.info('Installing kube{adm,ctl,let}') + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'apt', 'update', + run.Raw('&&'), + 'sudo', 'apt', 'install', '-y', + 'apt-transport-https', 'ca-certificates', 'curl', + run.Raw('&&'), + 'sudo', 'curl', '-fsSLo', + '/usr/share/keyrings/kubernetes-archive-keyring.gpg', + 'https://packages.cloud.google.com/apt/doc/apt-key.gpg', + run.Raw('&&'), + 'echo', 'deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main', + run.Raw('|'), + 'sudo', 'tee', '/etc/apt/sources.list.d/kubernetes.list', + run.Raw('&&'), + 'sudo', 'apt', 'update', + run.Raw('&&'), + 'sudo', 'apt', 'install', '-y', + 'kubelet', 'kubeadm', 'kubectl', + 'bridge-utils', + ], + wait=False, + ) + ) + + else: + raise RuntimeError(f'unsupported distro {os_type} for cri-o') + + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'systemctl', 'enable', '--now', 'kubelet', + run.Raw('&&'), + 'sudo', 'kubeadm', 'config', 'images', 'pull', + ], + wait=False, + ) + ) + + yield + + finally: + if config.get('uninstall', True): + log.info('Uninstalling kube{adm,let,ctl}') + if os_type in ['centos', 'rhel']: + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'rm', '-f', + '/etc/yum.repos.d/kubernetes.repo', + run.Raw('&&'), + 'sudo', 'dnf', 'remove', '-y', + 'kubeadm', 'kubelet', 'kubectl', 'cri-o', + ], + wait=False + ) + ) + elif os_type == 'ubuntu' and False: + run.wait( + ctx.cluster.run( + args=[ + 'sudo', 'rm', '-f', + '/etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list', + f'/etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:{version}.list', + '/etc/apt/trusted.gpg.d/libcontainers-cri-o.gpg', + run.Raw('&&'), + 'sudo', 'apt', 'remove', '-y', + 'kkubeadm', 'kubelet', 'kubectl', 'cri-o', 'cri-o-runc', + ], + wait=False, + ) + ) + + +@contextlib.contextmanager +def kubeadm_init_join(ctx, config): + cluster_name = config['cluster'] + + bootstrap_remote = None + remotes = {} # remote -> ip + for remote, roles in ctx.cluster.remotes.items(): + for role in roles: + if role.startswith('host.'): + if not bootstrap_remote: + bootstrap_remote = remote + if remote not in remotes: + remotes[remote] = remote.ssh.get_transport().getpeername()[0] + if not bootstrap_remote: + raise RuntimeError('must define at least one host.something role') + ctx.kubeadm[cluster_name].bootstrap_remote = bootstrap_remote + ctx.kubeadm[cluster_name].remotes = remotes + ctx.kubeadm[cluster_name].token = 'abcdef.' + ''.join([ + random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for _ in range(16) + ]) + log.info(f'Token: {ctx.kubeadm[cluster_name].token}') + log.info(f'Remotes: {ctx.kubeadm[cluster_name].remotes}') + + try: + # init + cmd = [ + 'sudo', 'kubeadm', 'init', + '--node-name', ctx.kubeadm[cluster_name].bootstrap_remote.shortname, + '--token', ctx.kubeadm[cluster_name].token, + '--pod-network-cidr', str(ctx.kubeadm[cluster_name].pod_subnet), + ] + bootstrap_remote.run(args=cmd) + + # join additional nodes + joins = [] + for remote, ip in ctx.kubeadm[cluster_name].remotes.items(): + if remote == bootstrap_remote: + continue + cmd = [ + 'sudo', 'kubeadm', 'join', + ctx.kubeadm[cluster_name].remotes[ctx.kubeadm[cluster_name].bootstrap_remote] + ':6443', + '--node-name', remote.shortname, + '--token', ctx.kubeadm[cluster_name].token, + '--discovery-token-unsafe-skip-ca-verification', + ] + joins.append(remote.run(args=cmd, wait=False)) + run.wait(joins) + yield + + except Exception as e: + log.exception(e) + raise + + finally: + log.info('Cleaning up node') + run.wait( + ctx.cluster.run( + args=['sudo', 'kubeadm', 'reset', 'cleanup-node', '-f'], + wait=False, + ) + ) + + +@contextlib.contextmanager +def kubectl_config(ctx, config): + cluster_name = config['cluster'] + bootstrap_remote = ctx.kubeadm[cluster_name].bootstrap_remote + + ctx.kubeadm[cluster_name].admin_conf = \ + bootstrap_remote.read_file('/etc/kubernetes/admin.conf', sudo=True) + + log.info('Setting up kubectl') + try: + ctx.cluster.run(args=[ + 'mkdir', '-p', '.kube', + run.Raw('&&'), + 'sudo', 'mkdir', '-p', '/root/.kube', + ]) + for remote in ctx.kubeadm[cluster_name].remotes.keys(): + remote.write_file('.kube/config', ctx.kubeadm[cluster_name].admin_conf) + remote.sudo_write_file('/root/.kube/config', + ctx.kubeadm[cluster_name].admin_conf) + yield + + except Exception as e: + log.exception(e) + raise + + finally: + log.info('Deconfiguring kubectl') + ctx.cluster.run(args=[ + 'rm', '-rf', '.kube', + run.Raw('&&'), + 'sudo', 'rm', '-rf', '/root/.kube', + ]) + + +def map_vnet(mip): + for mapping in teuth_config.get('vnet', []): + mnet = ipaddress.ip_network(mapping['machine_subnet']) + vnet = ipaddress.ip_network(mapping['virtual_subnet']) + if vnet.prefixlen >= mnet.prefixlen: + log.error(f"virtual_subnet {vnet} prefix >= machine_subnet {mnet} prefix") + return None + if mip in mnet: + pos = list(mnet.hosts()).index(mip) + log.info(f"{mip} is in {mnet} at pos {pos}") + sub = list(vnet.subnets(32 - mnet.prefixlen))[pos] + return sub + return None + + +@contextlib.contextmanager +def allocate_pod_subnet(ctx, config): + """ + Allocate a private subnet that will not collide with other test machines/clusters + """ + cluster_name = config['cluster'] + assert cluster_name == 'kubeadm', 'multiple subnets not yet implemented' + + log.info('Identifying pod subnet') + remote = list(ctx.cluster.remotes.keys())[0] + ip = remote.ssh.get_transport().getpeername()[0] + mip = ipaddress.ip_address(ip) + vnet = map_vnet(mip) + assert vnet + log.info(f'Pod subnet: {vnet}') + ctx.kubeadm[cluster_name].pod_subnet = vnet + yield + + +@contextlib.contextmanager +def pod_network(ctx, config): + cluster_name = config['cluster'] + pnet = config.get('pod_network', 'calico') + if pnet == 'flannel': + r = ctx.kubeadm[cluster_name].bootstrap_remote.run( + args=[ + 'curl', + 'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml', + ], + stdout=BytesIO(), + ) + assert r.exitstatus == 0 + flannel = list(yaml.load_all(r.stdout.getvalue(), Loader=yaml.FullLoader)) + for o in flannel: + if o.get('data', {}).get('net-conf.json'): + log.info(f'Updating {o}') + o['data']['net-conf.json'] = o['data']['net-conf.json'].replace( + '10.244.0.0/16', + str(ctx.kubeadm[cluster_name].pod_subnet) + ) + log.info(f'Now {o}') + flannel_yaml = yaml.dump_all(flannel) + log.debug(f'Flannel:\n{flannel_yaml}') + _kubectl(ctx, config, ['apply', '-f', '-'], stdin=flannel_yaml) + + elif pnet == 'calico': + _kubectl(ctx, config, [ + 'create', '-f', + 'https://docs.projectcalico.org/manifests/tigera-operator.yaml' + ]) + cr = { + 'apiVersion': 'operator.tigera.io/v1', + 'kind': 'Installation', + 'metadata': {'name': 'default'}, + 'spec': { + 'calicoNetwork': { + 'ipPools': [ + { + 'blockSize': 26, + 'cidr': str(ctx.kubeadm[cluster_name].pod_subnet), + 'encapsulation': 'IPIPCrossSubnet', + 'natOutgoing': 'Enabled', + 'nodeSelector': 'all()', + } + ] + } + } + } + _kubectl(ctx, config, ['create', '-f', '-'], stdin=yaml.dump(cr)) + + else: + raise RuntimeError(f'unrecognized pod_network {pnet}') + + try: + yield + + finally: + if pnet == 'flannel': + _kubectl(ctx, config, [ + 'delete', '-f', + 'https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml', + ]) + + elif pnet == 'calico': + _kubectl(ctx, config, ['delete', 'installation', 'default']) + _kubectl(ctx, config, [ + 'delete', '-f', + 'https://docs.projectcalico.org/manifests/tigera-operator.yaml' + ]) + + +@contextlib.contextmanager +def setup_pvs(ctx, config): + """ + Create PVs for all scratch LVs and set up a trivial provisioner + """ + log.info('Scanning for scratch devices') + crs = [] + for remote in ctx.cluster.remotes.keys(): + ls = remote.read_file('/scratch_devs').decode('utf-8').strip().splitlines() + log.info(f'Scratch devices on {remote.shortname}: {ls}') + for dev in ls: + devname = dev.split('/')[-1].replace("_", "-") + crs.append({ + 'apiVersion': 'v1', + 'kind': 'PersistentVolume', + 'metadata': {'name': f'{remote.shortname}-{devname}'}, + 'spec': { + 'volumeMode': 'Block', + 'accessModes': ['ReadWriteOnce'], + 'capacity': {'storage': '100Gi'}, # doesn't matter? + 'persistentVolumeReclaimPolicy': 'Retain', + 'storageClassName': 'scratch', + 'local': {'path': dev}, + 'nodeAffinity': { + 'required': { + 'nodeSelectorTerms': [ + { + 'matchExpressions': [ + { + 'key': 'kubernetes.io/hostname', + 'operator': 'In', + 'values': [remote.shortname] + } + ] + } + ] + } + } + } + }) + # overwriting first few MB is enough to make k8s happy + remote.run(args=[ + 'sudo', 'dd', 'if=/dev/zero', f'of={dev}', 'bs=1M', 'count=10' + ]) + crs.append({ + 'kind': 'StorageClass', + 'apiVersion': 'storage.k8s.io/v1', + 'metadata': {'name': 'scratch'}, + 'provisioner': 'kubernetes.io/no-provisioner', + 'volumeBindingMode': 'WaitForFirstConsumer', + }) + y = yaml.dump_all(crs) + log.info('Creating PVs + StorageClass') + log.debug(y) + _kubectl(ctx, config, ['create', '-f', '-'], stdin=y) + + yield + + +@contextlib.contextmanager +def final(ctx, config): + cluster_name = config['cluster'] + + # remove master node taint + _kubectl(ctx, config, [ + 'taint', 'node', + ctx.kubeadm[cluster_name].bootstrap_remote.shortname, + 'node-role.kubernetes.io/master-', + run.Raw('||'), + 'true', + ]) + + yield + + +@contextlib.contextmanager +def task(ctx, config): + if not config: + config = {} + assert isinstance(config, dict), \ + "task only supports a dictionary for configuration" + + log.info('Kubeadm start') + + overrides = ctx.config.get('overrides', {}) + teuthology.deep_merge(config, overrides.get('kubeadm', {})) + log.info('Config: ' + str(config)) + + # set up cluster context + if not hasattr(ctx, 'kubeadm'): + ctx.kubeadm = {} + if 'cluster' not in config: + config['cluster'] = 'kubeadm' + cluster_name = config['cluster'] + if cluster_name not in ctx.kubeadm: + ctx.kubeadm[cluster_name] = argparse.Namespace() + + with contextutil.nested( + lambda: preflight(ctx, config), + lambda: allocate_pod_subnet(ctx, config), + lambda: kubeadm_install(ctx, config), + lambda: kubeadm_init_join(ctx, config), + lambda: kubectl_config(ctx, config), + lambda: pod_network(ctx, config), + lambda: setup_pvs(ctx, config), + lambda: final(ctx, config), + ): + try: + log.info('Kubeadm complete, yielding') + yield + + finally: + log.info('Tearing down kubeadm') |