From: Cian Johnston Date: Thu, 14 Jan 2021 09:59:07 +0000 (+0000) Subject: Add k8-calico-dualstack and k8-multus-dualstack scenarios X-Git-Url: https://gerrit.nordix.org/gitweb?a=commitdiff_plain;ds=inline;h=refs%2Fchanges%2F55%2F7155%2F9;p=infra%2Fstack%2Fkubernetes.git Add k8-calico-dualstack and k8-multus-dualstack scenarios This commit adds two self-contained scenarios that support the following: * Assigning both IPv4 and IPv6 addresses to pods using the Calico CNI (kubernetes_version >= v1.16) * Setting both IPv4 and IPv6 kubelet addresses (kubernetes_version >= v1.20) One of these scenarios installs the Calico CNI only, and the other installs the Multus CNI with Calico as the backing CNI. *** NOTE: this depends on upstream features incompatible with k8s < 1.18. *** DO NOT MERGE with 1.15, 1.16, or 1.17. Deploy-Scenario: k8-calico-dualstack Change-Id: I5b9e21f6e0b5aaff276c8e86c19d5ae75d0c4569 Signed-off-by: Cian Johnston --- diff --git a/scenarios/k8-calico-dualstack.yaml b/scenarios/k8-calico-dualstack.yaml new file mode 100644 index 0000000..6bd00bb --- /dev/null +++ b/scenarios/k8-calico-dualstack.yaml @@ -0,0 +1,288 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2021 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +# ============README_START======================================================== +# This scenario configures a Kubernetes cluster with the calico network plugin +# and configured with IPv6 addresses on the 'public' network. Currently, only +# the ubuntu1804 base OS is supported, using the bifrost provisioner. +# +# WARNING: DUAL-STACK IS AN ALPHA FEATURE AS OF 1.16. HERE BE DRAGONS. +# +# To run this scenario, the following is required: +# - Kubernetes version >= 1.16. [1] +# - Usage of bifrost provisioner +# - Base OS == ubuntu1804 +# - A compatible version of Kubespray [2]. +# - Additional Ansible variables `libvirt_ipv6_networks`, `node_ipv6_addresses` +# specified (for example, by the environment variable ENGINE_ANSIBLE_PARAMS). +# Default values are provided below. +# +# The following tasks are performed prior to running the Kubespray installer: +# - libvirt networks specified in `libvirt_ipv6_networks` are re-created with +# the addition of an IPv6 stanza. +# - The hosts specified in `node_ipv6_addresses` have the provided address +# configured by dropping an additional file `$INTERFACE-ipv6.cfg` into +# `/etc/network/interfaces.d/`. An additional reboot of the instances is +# carried out to ensure the interfaces are configured properly. +# - The parameter `enable_dual_stack_networks: true` is set in the Kubespray +# configuration file on-disk. +# - The Kubelet configuration file `/etc/kubernetes/kubelet.env` is modified to +# append the corresponding IPv6 address of the node's interface to the node-ip +# parameter. This ensures that nodes are aware of both their IPv4 and IPv6 +# interfaces. +# - The feature gate is added: `IPv6DualStack=true` +# +# NOTES: +# [1] Assigning dual-stack addresses to kubelet via --node-ip=$IPV4,$IPV6 +# requires v1.20, see: https://github.com/kubernetes/kubernetes/issues/95239) +# [2] Kubespray supports IPv4/IPv6 DualStack as of commit c2c97c3. +# ============README_END========================================================== + +# NOTE (cian): Scenario-specific preinstall tasks +- hosts: localhost + connection: local + gather_facts: false + become: false + tags: + - preinstall + + # Below vars are only for documentation purposes and can be overridden as required. + vars: + libvirt_ipv6_networks: + - name: public + address: "2001:db8:0:2::/64" + node_ipv6_addresses: + master0: + interface: ens4 + address: "2001:db8:0:2::3/64" + node0: + interface: ens4 + address: "2001:db8:0:2::4/64" + + tasks: + - name: ensure kubernetes version is at least 1.16 + fail: + msg: "Dual-Stack operation requires a Kubernetes version of at least 1.16" + when: kubernetes_version is version('v1.16', operator='<', strict=False) + + - name: Set network plugin to Calico + lineinfile: + path: "{{ engine_cache }}/repos/kubespray/inventory/engine/group_vars/k8s-cluster/k8s-cluster.yml" + regexp: "^kube_network_plugin:.*" + line: "kube_network_plugin: calico" + + # NOTE (cian): this should really be in the installer configure-network role + - name: add IPv6 configuration stanzas to /etc/network/interfaces.d/ + become: true + when: provisioner_type == "bifrost" + copy: + owner: root + group: root + mode: 0644 + content: | + iface {{ item.value.interface }} inet6 static + address {{ item.value.address | ipaddr("address") }} + netmask {{ item.value.address | ipaddr("prefix") }} + gateway {{ item.value.address | ipaddr(1) | ipaddr("address") }} + dest: "/etc/network/interfaces.d/{{ item.value.interface }}-ipv6.cfg" + delegate_to: "{{ item.key }}" + with_items: "{{ node_ipv6_addresses | dict2items }}" + + # NOTE (cian): this should really be in the provisioner create-libvirt-networks role + - name: add IPv6 to existing libvirt networks + when: provisioner_type == "bifrost" + become: true + block: + - name: dump libvirt networks needing IPv6 + become: true + virt_net: + command: get_xml + name: "{{ item.name }}" + with_items: "{{ libvirt_ipv6_networks }}" + register: libvirt_net_xml + + - name: dump libvirt networks needing IPv6 to engine cache dir + copy: + mode: 0644 + content: "{{ item.get_xml }}" + dest: "{{ engine_cache }}/config/libvirt_net_{{ item.item.name }}.xml" + with_items: "{{ libvirt_net_xml.results }}" + register: libvirt_net_dumpxml + + - name: add IPv6 stanza to libvirt networks requiring IPv6 + blockinfile: + path: "{{ engine_cache }}/config/libvirt_net_{{ item.name }}.xml" + marker: "" + insertbefore: "" + block: | + + + + + + loop: "{{ libvirt_ipv6_networks | list }}" + register: libvirt_nets_blockinfile + + - name: redefine libvirt networks + become: true + when: libvirt_nets_blockinfile is changed + block: + - name: stop libvirt VMs + virt: + command: shutdown + name: "{{ item }}" + loop: "{{ engine.installers.kubespray.hostnames | dict2items | map(attribute='value') | list }}" + + - name: destroy affected libvirt networks + virt_net: + command: destroy + name: "{{ item.name }}" + with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}" + + - name: undefine affected libvirt networks + virt_net: + command: undefine + name: "{{ item.name }}" + with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}" + + - name: redefine affected libvirt networks + virt_net: + command: define + name: "{{ item.name }}" + xml: "{{ lookup('file', item.path) }}" + with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{ name: item.item.name, path: dest }') | list }}" + + - name: start affected libvirt networks + virt_net: + command: start + name: "{{ item.name }}" + with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}" + + - name: start libvirt VMs + virt: + command: start + name: "{{ item }}" + loop: "{{ engine.installers.kubespray.hostnames | dict2items | map(attribute='value') | list }}" + + # NOTE (cian): this will not modify the file if regexp does not match because backrefs=true + - name: Enable IPv6DualStack feature gate + when: provisioner_type == "bifrost" + block: + - name: Append IPv6DualStack feature gate if feature gates are already set + lineinfile: + path: "{{ engine_cache }}/config/kubespray-vars.yml" + regexp: "^kube_feature_gates:\\s*\\[(.+)\\]\\s*$" + line: "kube_feature_gates: [\"IPv6DualStack=true\",\\1]" + backrefs: true + + - name: Set IPv6DualStack feature gate if no feature gates are already set + lineinfile: + path: "{{ engine_cache }}/config/kubespray-vars.yml" + regexp: "^kube_feature_gates:\\s*\\[\\]\\s*$" + line: "kube_feature_gates: [\"IPv6DualStack=true\"]" + + - name: enable dual-stack networks + lineinfile: + path: "{{ engine_cache }}/config/kubespray-vars.yml" + regexp: "^enable_dual_stack_networks:.*" + line: "enable_dual_stack_networks: true" + +- hosts: k8s-nodes + tags: + - preinstall + tasks: + - name: wait for libvirt VMs to come back up + wait_for_connection: + connect_timeout: 10 + delay: 30 + timeout: 300 + register: result + until: result is succeeded + retries: 3 + when: provisioner_type == "bifrost" + +# NOTE (cian): scenario-specific postinstall tasks +- hosts: k8s-cluster + tags: + - postinstall + tasks: + - name: set node-ip with both IPv4 and IPv6 addresses + block: + - name: Determine existing IPv4 address for kubelet + shell: | + set -o pipefail && \ + egrep -o -- 'KUBELET_ADDRESS="--node-ip=[0-9\.]+' /etc/kubernetes/kubelet.env \ + | awk -F '=' '{print $3}' + args: + executable: "/bin/bash" + register: kubelet_ipv4_address + changed_when: true + + - name: Determine corresponding interface for IPv4 address {{ kubelet_ipv4_address.stdout }} + shell: | + set -o pipefail && \ + ip -4 -o a \ + | cut -d ' ' -f 2,7 \ + | cut -d '/' -f 1 \ + | grep {{ kubelet_ipv4_address.stdout }} \ + | cut -d ' ' -f 1 + args: + executable: "/bin/bash" + register: kubelet_ipv4_interface + changed_when: true + + # NOTE (cian): this will default to the first IPv6 address assigned to the given interface + # excluding link-local addresses. If there are multiple IPv6 addresses assigned to one interface, + # this will probably not do what we want. + - name: Determine first IPv6 address assigned to interface {{ kubelet_ipv4_interface.stdout }} + shell: | + set -o pipefail && \ + ip -6 -o a \ + | cut -d ' ' -f 2,7 \ + | cut -d '/' -f 1 \ + | grep {{ kubelet_ipv4_interface.stdout }} \ + | grep -v "fe80::" \ + | cut -d ' ' -f 2 \ + | head -n 1 + args: + executable: "/bin/bash" + register: kubelet_ipv6_address + changed_when: true + + # NOTE (cian): Dual-Stack NodeIP handling was added in v1.20.0 + - name: Add comma-separated dualstack NodeIP parameter to Kubelet + become: true + lineinfile: + path: "/etc/kubernetes/kubelet.env" + regexp: "^KUBELET_ADDRESS=\"--node-ip=[0-9\\.]+\"\\s*$" + line: "KUBELET_ADDRESS=\"--node-ip={{ kubelet_ipv4_address.stdout }},{{ kubelet_ipv6_address.stdout }}\"" + register: etc_kubernetes_kubelet_env + + - name: Restart kubelet + become: true + service: name=kubelet state=restarted + when: + - provisioner_type == "bifrost" + - kubernetes_version is version('v1.20', operator='>=', strict=False) + +- name: Execute common postinstall tasks + import_playbook: "../playbooks/postinstall.yaml" + tags: postinstall + +# vim: set ts=2 sw=2 expandtab: diff --git a/scenarios/k8-multus-dualstack.yaml b/scenarios/k8-multus-dualstack.yaml new file mode 100644 index 0000000..9e03d70 --- /dev/null +++ b/scenarios/k8-multus-dualstack.yaml @@ -0,0 +1,293 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2021 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +# ============README_START======================================================== +# This scenario configures a Kubernetes cluster with the Multus CNI backed by +# Calico, and configured with IPv6 addresses on the 'public' network. Currently +# only the ubuntu1804 base OS is supported, using the bifrost provisioner. +# +# WARNING: DUAL-STACK IS AN ALPHA FEATURE AS OF 1.16. HERE BE DRAGONS. +# +# To run this scenario, the following is required: +# - Kubernetes version >= 1.16. [1] +# - Usage of bifrost provisioner +# - Base OS == ubuntu1804 +# - A compatible version of Kubespray [2]. +# - Additional Ansible variables `libvirt_ipv6_networks`, `node_ipv6_addresses` +# specified (for example, by the environment variable ENGINE_ANSIBLE_PARAMS). +# Default values are provided below. +# +# The following tasks are performed prior to running the Kubespray installer: +# - libvirt networks specified in `libvirt_ipv6_networks` are re-created with +# the addition of an IPv6 stanza. +# - The hosts specified in `node_ipv6_addresses` have the provided address +# configured by dropping an additional file `$INTERFACE-ipv6.cfg` into +# `/etc/network/interfaces.d/`. An additional reboot of the instances is +# carried out to ensure the interfaces are configured properly. +# - The parameter `enable_dual_stack_networks: true` is set in the Kubespray +# configuration file on-disk. +# - The Kubelet configuration file `/etc/kubernetes/kubelet.env` is modified to +# append the corresponding IPv6 address of the node's interface to the node-ip +# parameter. This ensures that nodes are aware of both their IPv4 and IPv6 +# interfaces. +# - The feature gate is added: `IPv6DualStack=true` +# +# NOTES: +# [1] Assigning dual-stack addresses to kubelet via --node-ip=$IPV4,$IPV6 +# requires v1.20, see: https://github.com/kubernetes/kubernetes/issues/95239) +# [2] Kubespray supports IPv4/IPv6 DualStack as of commit c2c97c3. +# ============README_END========================================================== + +- hosts: localhost + connection: local + gather_facts: false + become: false + tags: + - preinstall + + # Below vars are only for documentation purposes and can be overridden as required. + vars: + libvirt_ipv6_networks: + - name: public + address: "2001:db8:0:2::/64" + node_ipv6_addresses: + master0: + interface: ens4 + address: "2001:db8:0:2::3/64" + node0: + interface: ens4 + address: "2001:db8:0:2::4/64" + + tasks: + - name: ensure kubernetes version is at least 1.16 + fail: + msg: "Dual-Stack operation requires a Kubernetes version of at least 1.16" + when: kubernetes_version is version('v1.16', operator='<', strict=False) + + - name: Configure Multus to use Calico as the primary network plugin + lineinfile: + path: "{{ engine_cache }}/repos/kubespray/inventory/engine/group_vars/k8s-cluster/k8s-cluster.yml" + regexp: "^kube_network_plugin:.*" + line: "kube_network_plugin: calico" + + - name: Enable the Multus network plugin + lineinfile: + path: "{{ engine_cache }}/repos/kubespray/inventory/engine/group_vars/k8s-cluster/k8s-cluster.yml" + regexp: "^kube_network_plugin_multus:.*" + line: "kube_network_plugin_multus: true" + + # NOTE (cian): this should really be in the installer configure-network role + - name: add IPv6 configuration stanzas to /etc/network/interfaces.d/ + become: true + when: provisioner_type == "bifrost" + copy: + owner: root + group: root + mode: 0644 + content: | + iface {{ item.value.interface }} inet6 static + address {{ item.value.address | ipaddr("address") }} + netmask {{ item.value.address | ipaddr("prefix") }} + gateway {{ item.value.address | ipaddr(1) | ipaddr("address") }} + dest: "/etc/network/interfaces.d/{{ item.value.interface }}-ipv6.cfg" + delegate_to: "{{ item.key }}" + with_items: "{{ node_ipv6_addresses | dict2items }}" + + # NOTE (cian): this should really be in the provisioner create-libvirt-networks role + - name: add IPv6 to existing libvirt networks + when: provisioner_type == "bifrost" + become: true + block: + - name: dump libvirt networks needing IPv6 + become: true + virt_net: + command: get_xml + name: "{{ item.name }}" + with_items: "{{ libvirt_ipv6_networks }}" + register: libvirt_net_xml + + - name: dump libvirt networks needing IPv6 to engine cache dir + copy: + mode: 0644 + content: "{{ item.get_xml }}" + dest: "{{ engine_cache }}/config/libvirt_net_{{ item.item.name }}.xml" + with_items: "{{ libvirt_net_xml.results }}" + register: libvirt_net_dumpxml + + - name: add IPv6 stanza to libvirt networks requiring IPv6 + blockinfile: + path: "{{ engine_cache }}/config/libvirt_net_{{ item.name }}.xml" + marker: "" + insertbefore: "" + block: | + + + + + + loop: "{{ libvirt_ipv6_networks | list }}" + register: libvirt_nets_blockinfile + + - name: redefine libvirt networks + become: true + when: libvirt_nets_blockinfile is changed + block: + - name: stop libvirt VMs + virt: + command: shutdown + name: "{{ item }}" + loop: "{{ engine.installers.kubespray.hostnames | dict2items | map(attribute='value') | list }}" + + - name: destroy affected libvirt networks + virt_net: + command: destroy + name: "{{ item.name }}" + with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}" + + - name: undefine affected libvirt networks + virt_net: + command: undefine + name: "{{ item.name }}" + with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}" + + - name: redefine affected libvirt networks + virt_net: + command: define + name: "{{ item.name }}" + xml: "{{ lookup('file', item.path) }}" + with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{ name: item.item.name, path: dest }') | list }}" + + - name: start affected libvirt networks + virt_net: + command: start + name: "{{ item.name }}" + with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}" + + - name: start libvirt VMs + virt: + command: start + name: "{{ item }}" + loop: "{{ engine.installers.kubespray.hostnames | dict2items | map(attribute='value') | list }}" + + # NOTE (cian): this will not modify the file if regexp does not match because backrefs=true + - name: Enable IPv6DualStack feature gate + when: provisioner_type == "bifrost" + block: + - name: Append IPv6DualStack feature gate if feature gates are already set + lineinfile: + path: "{{ engine_cache }}/config/kubespray-vars.yml" + regexp: "^kube_feature_gates:\\s*\\[(.+)\\]\\s*$" + line: "kube_feature_gates: [\"IPv6DualStack=true\",\\1]" + backrefs: true + + - name: Set IPv6DualStack feature gate if no feature gates are already set + lineinfile: + path: "{{ engine_cache }}/config/kubespray-vars.yml" + regexp: "^kube_feature_gates:\\s*\\[\\]\\s*$" + line: "kube_feature_gates: [\"IPv6DualStack=true\"]" + + - name: enable dual-stack networks + lineinfile: + path: "{{ engine_cache }}/config/kubespray-vars.yml" + regexp: "^enable_dual_stack_networks:.*" + line: "enable_dual_stack_networks: true" + +- hosts: k8s-nodes + tags: + - preinstall + tasks: + - name: wait for libvirt VMs to come back up + wait_for_connection: + connect_timeout: 10 + delay: 30 + timeout: 300 + register: result + until: result is succeeded + retries: 3 + when: provisioner_type == "bifrost" + +# NOTE (cian): scenario-specific postinstall tasks +- hosts: k8s-cluster + tags: + - postinstall + tasks: + - name: set node-ip with both IPv4 and IPv6 addresses + block: + - name: Determine existing IPv4 address for kubelet + shell: | + set -o pipefail && \ + egrep -o -- 'KUBELET_ADDRESS="--node-ip=[0-9\.]+' /etc/kubernetes/kubelet.env \ + | awk -F '=' '{print $3}' + args: + executable: "/bin/bash" + register: kubelet_ipv4_address + changed_when: true + + - name: Determine corresponding interface for IPv4 address {{ kubelet_ipv4_address.stdout }} + shell: | + set -o pipefail && \ + ip -4 -o a \ + | cut -d ' ' -f 2,7 \ + | cut -d '/' -f 1 \ + | grep {{ kubelet_ipv4_address.stdout }} \ + | cut -d ' ' -f 1 + args: + executable: "/bin/bash" + register: kubelet_ipv4_interface + changed_when: true + + # NOTE (cian): this will default to the first IPv6 address assigned to the given interface + # excluding link-local addresses. If there are multiple IPv6 addresses assigned to one interface, + # this will probably not do what we want. + - name: Determine first IPv6 address assigned to interface {{ kubelet_ipv4_interface.stdout }} + shell: | + set -o pipefail && \ + ip -6 -o a \ + | cut -d ' ' -f 2,7 \ + | cut -d '/' -f 1 \ + | grep {{ kubelet_ipv4_interface.stdout }} \ + | grep -v "fe80::" \ + | cut -d ' ' -f 2 \ + | head -n 1 + args: + executable: "/bin/bash" + register: kubelet_ipv6_address + changed_when: true + + # NOTE (cian): Dual-Stack NodeIP handling was added in v1.20.0 + - name: Add comma-separated dualstack NodeIP parameter to Kubelet + become: true + lineinfile: + path: "/etc/kubernetes/kubelet.env" + regexp: "^KUBELET_ADDRESS=\"--node-ip=[0-9\\.]+\"\\s*$" + line: "KUBELET_ADDRESS=\"--node-ip={{ kubelet_ipv4_address.stdout }},{{ kubelet_ipv6_address.stdout }}\"" + register: etc_kubernetes_kubelet_env + + - name: Restart kubelet + become: true + service: name=kubelet state=restarted + when: + - provisioner_type == "bifrost" + - kubernetes_version is version('v1.20', operator='>=', strict=False) + +- name: Execute common postinstall tasks + import_playbook: "../playbooks/postinstall.yaml" + tags: postinstall + +# vim: set ts=2 sw=2 expandtab: diff --git a/vars/kubernetes.yaml b/vars/kubernetes.yaml index 849859e..2023452 100644 --- a/vars/kubernetes.yaml +++ b/vars/kubernetes.yaml @@ -45,12 +45,14 @@ installers: # repository. Curated apps are not listed separately either as they are # similar to scenarios; they are available in installer repository. scenarios: + - k8-calico-dualstack - k8-calico-istio - k8-calico-nofeature - k8-calico-spinnaker - k8-canal-nofeature - k8-cilium-nofeature - k8-flannel-nofeature + - k8-multus-dualstack - k8-multus-nofeature - k8-multus-plugins - k8-weave-nofeature