scenarios/k8-multus-dualstack.yaml

---
# ============LICENSE_START=======================================================
#  Copyright (C) 2021 The Nordix Foundation. All rights reserved.
# ================================================================================
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
# ============LICENSE_END=========================================================

# ============README_START========================================================
# This scenario configures a Kubernetes cluster with the Multus CNI backed by
# Calico, and configured with IPv6 addresses on the 'public' network. Currently
# only the ubuntu1804 base OS is supported, using the bifrost provisioner.
#
# WARNING: DUAL-STACK IS AN ALPHA FEATURE AS OF 1.16. HERE BE DRAGONS.
#
# To run this scenario, the following is required:
#   - Kubernetes version >= 1.16. [1]
#   - Usage of bifrost provisioner
#   - Base OS == ubuntu1804
#   - A compatible version of Kubespray [2].
#   - Additional Ansible variables `libvirt_ipv6_networks`, `node_ipv6_addresses`
#     specified (for example, by the environment variable ENGINE_ANSIBLE_PARAMS).
#     Default values are provided below.
#
# The following tasks are performed prior to running the Kubespray installer:
#   - libvirt networks specified in `libvirt_ipv6_networks` are re-created with
#     the addition of an IPv6 stanza.
#   - The hosts specified in `node_ipv6_addresses` have the provided address
#     configured by dropping an additional file `$INTERFACE-ipv6.cfg` into
#     `/etc/network/interfaces.d/`. An additional reboot of the instances is
#     carried out to ensure the interfaces are configured properly.
#   - The parameter `enable_dual_stack_networks: true` is set in the Kubespray
#     configuration file on-disk.
#   - The Kubelet configuration file `/etc/kubernetes/kubelet.env` is modified to
#     append the corresponding IPv6 address of the node's interface to the node-ip
#     parameter. This ensures that nodes are aware of both their IPv4 and IPv6
#     interfaces.
#   - The feature gate is added: `IPv6DualStack=true`
#
# NOTES:
# [1] Assigning dual-stack addresses to kubelet via --node-ip=$IPV4,$IPV6
#     requires v1.20, see: https://github.com/kubernetes/kubernetes/issues/95239)
# [2] Kubespray supports IPv4/IPv6 DualStack as of commit c2c97c3.
# ============README_END==========================================================

- hosts: localhost
  connection: local
  gather_facts: false
  become: false
  tags:
    - preinstall

  # Below vars are only for documentation purposes and can be overridden as required.
  vars:
    libvirt_ipv6_networks:
      - name: public
        address: "2001:db8:0:2::/64"
    node_ipv6_addresses:
      master0:
        interface: ens4
        address: "2001:db8:0:2::3/64"
      node0:
        interface: ens4
        address: "2001:db8:0:2::4/64"

  tasks:
    - name: ensure kubernetes version is at least 1.16
      fail:
        msg: "Dual-Stack operation requires a Kubernetes version of at least 1.16"
      when: kubernetes_version is version('v1.16', operator='<', strict=False)

    - name: Configure Multus to use Calico as the primary network plugin
      lineinfile:
        path: "{{ engine_cache }}/repos/kubespray/inventory/engine/group_vars/k8s_cluster/k8s-cluster.yml"
        regexp: "^kube_network_plugin:.*"
        line: "kube_network_plugin: calico"

    - name: Enable the Multus network plugin
      lineinfile:
        path: "{{ engine_cache }}/repos/kubespray/inventory/engine/group_vars/k8s_cluster/k8s-cluster.yml"
        regexp: "^kube_network_plugin_multus:.*"
        line: "kube_network_plugin_multus: true"

    # NOTE (cian): this should really be in the installer configure-network role
    - name: add IPv6 configuration stanzas to /etc/network/interfaces.d/
      become: true
      when: provisioner_type == "bifrost"
      copy:
        owner: root
        group: root
        mode: 0644
        content: |
          iface {{ item.value.interface }} inet6 static
          address {{ item.value.address | ipaddr("address") }}
          netmask {{ item.value.address | ipaddr("prefix") }}
          gateway {{ item.value.address | ipaddr(1) | ipaddr("address") }}
        dest: "/etc/network/interfaces.d/{{ item.value.interface }}-ipv6.cfg"
      delegate_to: "{{ item.key }}"
      with_items: "{{ node_ipv6_addresses | dict2items }}"

    # NOTE (cian): this should really be in the provisioner create-libvirt-networks role
    - name: add IPv6 to existing libvirt networks
      when: provisioner_type == "bifrost"
      become: true
      block:
        - name: dump libvirt networks needing IPv6
          become: true
          virt_net:
            command: get_xml
            name: "{{ item.name }}"
          with_items: "{{ libvirt_ipv6_networks }}"
          register: libvirt_net_xml

        - name: dump libvirt networks needing IPv6 to engine cache dir
          copy:
            mode: 0644
            content: "{{ item.get_xml }}"
            dest: "{{ engine_cache }}/config/libvirt_net_{{ item.item.name }}.xml"
          with_items: "{{ libvirt_net_xml.results }}"
          register: libvirt_net_dumpxml

        - name: add IPv6 stanza to libvirt networks requiring IPv6
          blockinfile:
            path: "{{ engine_cache }}/config/libvirt_net_{{ item.name }}.xml"
            marker: "<!-- {mark} ANSIBLE MANAGED BLOCK -->"
            insertbefore: "</network>"
            block: |
              <ip family='ipv6' address='{{ item.address | ipaddr(1) | ipaddr("address") }}' prefix='{{ item.address | ipaddr("prefix") }}'>
                <dhcp>
                  <range start='{{ item.address | ipaddr(2) | ipaddr("address") }}' end='{{ item.address | ipaddr(255) | ipaddr("address") }}'/><!-- ' -->
                </dhcp>
              </ip>
          loop: "{{ libvirt_ipv6_networks | list }}"
          register: libvirt_nets_blockinfile

        - name: redefine libvirt networks
          become: true
          when: libvirt_nets_blockinfile is changed
          block:
            - name: stop libvirt VMs
              virt:
                command: shutdown
                name: "{{ item }}"
              loop: "{{ engine.installers.kubespray.hostnames | dict2items | map(attribute='value') | list }}"

            - name: destroy affected libvirt networks
              virt_net:
                command: destroy
                name: "{{ item.name }}"
              with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}"

            - name: undefine affected libvirt networks
              virt_net:
                command: undefine
                name: "{{ item.name }}"
              with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}"

            - name: redefine affected libvirt networks
              virt_net:
                command: define
                name: "{{ item.name }}"
                xml: "{{ lookup('file', item.path) }}"
              with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{ name: item.item.name, path: dest }') | list }}"

            - name: start affected libvirt networks
              virt_net:
                command: start
                name: "{{ item.name }}"
              with_items: "{{ libvirt_net_dumpxml.results | json_query('[].{name: item.item.name}') | list }}"

            - name: start libvirt VMs
              virt:
                command: start
                name: "{{ item }}"
              loop: "{{ engine.installers.kubespray.hostnames | dict2items | map(attribute='value') | list }}"

    # NOTE (cian): this will not modify the file if regexp does not match because backrefs=true
    - name: Enable IPv6DualStack feature gate
      when: provisioner_type == "bifrost"
      block:
        - name: Append IPv6DualStack feature gate if feature gates are already set
          lineinfile:
            path: "{{ engine_cache }}/config/kubespray-vars.yml"
            regexp: "^kube_feature_gates:\\s*\\[(.+)\\]\\s*$"
            line: "kube_feature_gates: [\"IPv6DualStack=true\",\\1]"
            backrefs: true

        - name: Set IPv6DualStack feature gate if no feature gates are already set
          lineinfile:
            path: "{{ engine_cache }}/config/kubespray-vars.yml"
            regexp: "^kube_feature_gates:\\s*\\[\\]\\s*$"
            line: "kube_feature_gates: [\"IPv6DualStack=true\"]"

        - name: enable dual-stack networks
          lineinfile:
            path: "{{ engine_cache }}/config/kubespray-vars.yml"
            regexp: "^enable_dual_stack_networks:.*"
            line: "enable_dual_stack_networks: true"

- hosts: k8s-nodes
  tags:
    - preinstall
  tasks:
    - name: wait for libvirt VMs to come back up
      wait_for_connection:
        connect_timeout: 10
        delay: 30
        timeout: 300
      register: result
      until: result is succeeded
      retries: 3
      when: provisioner_type == "bifrost"

# NOTE (cian): scenario-specific postinstall tasks
- hosts: k8s-cluster
  tags:
    - postinstall
  tasks:
    - name: set node-ip with both IPv4 and IPv6 addresses
      block:
        - name: Determine existing IPv4 address for kubelet
          shell: |
            set -o pipefail && \
            egrep -o -- 'KUBELET_ADDRESS="--node-ip=[0-9\.]+' /etc/kubernetes/kubelet.env \
              | awk -F '=' '{print $3}'
          args:
            executable: "/bin/bash"
          register: kubelet_ipv4_address
          changed_when: true

        - name: Determine corresponding interface for IPv4 address {{ kubelet_ipv4_address.stdout }}
          shell: |
            set -o pipefail && \
            ip -4 -o a \
              | cut -d ' ' -f 2,7 \
              | cut -d '/' -f 1 \
              | grep {{ kubelet_ipv4_address.stdout }} \
              | cut -d ' ' -f 1
          args:
            executable: "/bin/bash"
          register: kubelet_ipv4_interface
          changed_when: true

          # NOTE (cian): this will default to the first IPv6 address assigned to the given interface
          # excluding link-local addresses. If there are multiple IPv6 addresses assigned to one interface,
          # this will probably not do what we want.
        - name: Determine first IPv6 address assigned to interface {{ kubelet_ipv4_interface.stdout }}
          shell: |
            set -o pipefail && \
            ip -6 -o a \
              | cut -d ' ' -f 2,7 \
              | cut -d '/' -f 1 \
              | grep {{ kubelet_ipv4_interface.stdout }} \
              | grep -v "fe80::" \
              | cut -d ' ' -f 2 \
              | head -n 1
          args:
            executable: "/bin/bash"
          register: kubelet_ipv6_address
          changed_when: true

        # NOTE (cian): Dual-Stack NodeIP handling was added in v1.20.0
        - name: Add comma-separated dualstack NodeIP parameter to Kubelet
          become: true
          lineinfile:
            path: "/etc/kubernetes/kubelet.env"
            regexp: "^KUBELET_ADDRESS=\"--node-ip=[0-9\\.]+\"\\s*$"
            line: "KUBELET_ADDRESS=\"--node-ip={{ kubelet_ipv4_address.stdout }},{{ kubelet_ipv6_address.stdout }}\""
          register: etc_kubernetes_kubelet_env

        - name: Restart kubelet
          become: true
          service: name=kubelet state=restarted
      when:
        - provisioner_type == "bifrost"
        - kubernetes_version is version('v1.20', operator='>=', strict=False)

- name: Execute common postinstall tasks
  import_playbook: "../playbooks/postinstall.yaml"
  tags: postinstall

# vim: set ts=2 sw=2 expandtab: