2 # ============LICENSE_START=======================================================
3 # Copyright (C) 2021 The Nordix Foundation. All rights reserved.
4 # ================================================================================
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
17 # SPDX-License-Identifier: Apache-2.0
18 # ============LICENSE_END=========================================================
20 # ============README_START========================================================
21 # This scenario configures a Kubernetes cluster with the Multus CNI backed by
22 # Calico, and configured with IPv6 addresses on the 'public' network. Currently
23 # only the ubuntu1804 base OS is supported, using the bifrost provisioner.
25 # WARNING: DUAL-STACK IS AN ALPHA FEATURE AS OF 1.16. HERE BE DRAGONS.
27 # To run this scenario, the following is required:
28 # - Kubernetes version >= 1.16. [1]
29 # - Usage of bifrost provisioner
30 # - Base OS == ubuntu1804
31 # - A compatible version of Kubespray [2].
32 # - Additional Ansible variables `libvirt_ipv6_networks`, `node_ipv6_addresses`
33 # specified (for example, by the environment variable ENGINE_ANSIBLE_PARAMS).
34 # Default values are provided below.
36 # The following tasks are performed prior to running the Kubespray installer:
37 # - libvirt networks specified in `libvirt_ipv6_networks` are re-created with
38 # the addition of an IPv6 stanza.
39 # - The hosts specified in `node_ipv6_addresses` have the provided address
40 # configured by dropping an additional file `$INTERFACE-ipv6.cfg` into
41 # `/etc/network/interfaces.d/`. An additional reboot of the instances is
42 # carried out to ensure the interfaces are configured properly.
43 # - The parameter `enable_dual_stack_networks: true` is set in the Kubespray
44 # configuration file on-disk.
45 # - The Kubelet configuration file `/etc/kubernetes/kubelet.env` is modified to
46 # append the corresponding IPv6 address of the node's interface to the node-ip
47 # parameter. This ensures that nodes are aware of both their IPv4 and IPv6
49 # - The feature gate is added: `IPv6DualStack=true`
52 # [1] Assigning dual-stack addresses to kubelet via --node-ip=$IPV4,$IPV6
53 # requires v1.20, see: https://github.com/kubernetes/kubernetes/issues/95239)
54 # [2] Kubespray supports IPv4/IPv6 DualStack as of commit c2c97c3.
55 # ============README_END==========================================================
64 # Below vars are only for documentation purposes and can be overridden as required.
66 libvirt_ipv6_networks:
68 address: "2001:db8:0:2::/64"
72 address: "2001:db8:0:2::3/64"
75 address: "2001:db8:0:2::4/64"
78 - name: ensure kubernetes version is at least 1.16
80 msg: "Dual-Stack operation requires a Kubernetes version of at least 1.16"
81 when: kubernetes_version is version('v1.16', operator='<', strict=False)
83 - name: Configure Multus to use Calico as the primary network plugin
85 path: "{{ engine_cache }}/repos/kubespray/inventory/engine/group_vars/k8s_cluster/k8s-cluster.yml"
86 regexp: "^kube_network_plugin:.*"
87 line: "kube_network_plugin: calico"
89 - name: Enable the Multus network plugin
91 path: "{{ engine_cache }}/repos/kubespray/inventory/engine/group_vars/k8s_cluster/k8s-cluster.yml"
92 regexp: "^kube_network_plugin_multus:.*"
93 line: "kube_network_plugin_multus: true"
95 # NOTE (cian): this should really be in the installer configure-network role
96 - name: add IPv6 configuration stanzas to /etc/network/interfaces.d/
98 when: provisioner_type == "bifrost"
104 iface {{ item.value.interface }} inet6 static
105 address {{ item.value.address | ipaddr("address") }}
106 netmask {{ item.value.address | ipaddr("prefix") }}
107 gateway {{ item.value.address | ipaddr(1) | ipaddr("address") }}
108 dest: "/etc/network/interfaces.d/{{ item.value.interface }}-ipv6.cfg"
109 delegate_to: "{{ item.key }}"
110 with_items: "{{ node_ipv6_addresses | dict2items }}"
112 # NOTE (cian): this should really be in the provisioner create-libvirt-networks role
113 - name: add IPv6 to existing libvirt networks
114 when: provisioner_type == "bifrost"
117 - name: dump libvirt networks needing IPv6
121 name: "{{ item.name }}"
122 with_items: "{{ libvirt_ipv6_networks }}"
123 register: libvirt_net_xml
125 - name: dump libvirt networks needing IPv6 to engine cache dir
128 content: "{{ item.get_xml }}"
129 dest: "{{ engine_cache }}/config/libvirt_net_{{ item.item.name }}.xml"
130 with_items: "{{ libvirt_net_xml.results }}"
131 register: libvirt_net_dumpxml
133 - name: add IPv6 stanza to libvirt networks requiring IPv6
135 path: "{{ engine_cache }}/config/libvirt_net_{{ item.name }}.xml"
136 marker: "<!-- {mark} ANSIBLE MANAGED BLOCK -->"
137 insertbefore: "</network>"
139 <ip family='ipv6' address='{{ item.address | ipaddr(1) | ipaddr("address") }}' prefix='{{ item.address | ipaddr("prefix") }}'>
141 <range start='{{ item.address | ipaddr(2) | ipaddr("address") }}' end='{{ item.address | ipaddr(255) | ipaddr("address") }}'/><!-- ' -->
144 loop: "{{ libvirt_ipv6_networks | list }}"
145 register: libvirt_nets_blockinfile
147 - name: redefine libvirt networks
149 when: libvirt_nets_blockinfile is changed
151 - name: stop libvirt VMs
155 loop: "{{ engine.installers.kubespray.hostnames | dict2items | map(attribute='value') | list }}"
157 - name: destroy affected libvirt networks
160 name: "{{ item.name }}"
161 with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}"
163 - name: undefine affected libvirt networks
166 name: "{{ item.name }}"
167 with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}"
169 - name: redefine affected libvirt networks
172 name: "{{ item.name }}"
173 xml: "{{ lookup('file', item.path) }}"
174 with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{ name: item.item.name, path: dest }') | list }}"
176 - name: start affected libvirt networks
179 name: "{{ item.name }}"
180 with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}"
182 - name: start libvirt VMs
186 loop: "{{ engine.installers.kubespray.hostnames | dict2items | map(attribute='value') | list }}"
188 # NOTE (cian): this will not modify the file if regexp does not match because backrefs=true
189 - name: Enable IPv6DualStack feature gate
190 when: provisioner_type == "bifrost"
192 - name: Append IPv6DualStack feature gate if feature gates are already set
194 path: "{{ engine_cache }}/config/kubespray-vars.yml"
195 regexp: "^kube_feature_gates:\\s*\\[(.+)\\]\\s*$"
196 line: "kube_feature_gates: [\"IPv6DualStack=true\",\\1]"
199 - name: Set IPv6DualStack feature gate if no feature gates are already set
201 path: "{{ engine_cache }}/config/kubespray-vars.yml"
202 regexp: "^kube_feature_gates:\\s*\\[\\]\\s*$"
203 line: "kube_feature_gates: [\"IPv6DualStack=true\"]"
205 - name: enable dual-stack networks
207 path: "{{ engine_cache }}/config/kubespray-vars.yml"
208 regexp: "^enable_dual_stack_networks:.*"
209 line: "enable_dual_stack_networks: true"
215 - name: wait for libvirt VMs to come back up
221 until: result is succeeded
223 when: provisioner_type == "bifrost"
225 # NOTE (cian): scenario-specific postinstall tasks
230 - name: set node-ip with both IPv4 and IPv6 addresses
232 - name: Determine existing IPv4 address for kubelet
235 egrep -o -- 'KUBELET_ADDRESS="--node-ip=[0-9\.]+' /etc/kubernetes/kubelet.env \
236 | awk -F '=' '{print $3}'
238 executable: "/bin/bash"
239 register: kubelet_ipv4_address
242 - name: Determine corresponding interface for IPv4 address {{ kubelet_ipv4_address.stdout }}
246 | cut -d ' ' -f 2,7 \
248 | grep {{ kubelet_ipv4_address.stdout }} \
251 executable: "/bin/bash"
252 register: kubelet_ipv4_interface
255 # NOTE (cian): this will default to the first IPv6 address assigned to the given interface
256 # excluding link-local addresses. If there are multiple IPv6 addresses assigned to one interface,
257 # this will probably not do what we want.
258 - name: Determine first IPv6 address assigned to interface {{ kubelet_ipv4_interface.stdout }}
262 | cut -d ' ' -f 2,7 \
264 | grep {{ kubelet_ipv4_interface.stdout }} \
269 executable: "/bin/bash"
270 register: kubelet_ipv6_address
273 # NOTE (cian): Dual-Stack NodeIP handling was added in v1.20.0
274 - name: Add comma-separated dualstack NodeIP parameter to Kubelet
277 path: "/etc/kubernetes/kubelet.env"
278 regexp: "^KUBELET_ADDRESS=\"--node-ip=[0-9\\.]+\"\\s*$"
279 line: "KUBELET_ADDRESS=\"--node-ip={{ kubelet_ipv4_address.stdout }},{{ kubelet_ipv6_address.stdout }}\""
280 register: etc_kubernetes_kubelet_env
282 - name: Restart kubelet
284 service: name=kubelet state=restarted
286 - provisioner_type == "bifrost"
287 - kubernetes_version is version('v1.20', operator='>=', strict=False)
289 - name: Execute common postinstall tasks
290 import_playbook: "../playbooks/postinstall.yaml"
293 # vim: set ts=2 sw=2 expandtab: