Add molecule tests for rke role
This change adds molecule tests for the rke role, and
modifies the rke role itself to be more idempotent/to
pass linter.
Note that this molecule test case uses a separate
role to install docker in containers, that runs
docker daemon inside of them instead of using host docker.
Issue-ID: OOM-1778
Change-Id: I875f3ff2ab961e5428acee5a02287a8d2d6e9969
Signed-off-by: Michal Zegan <m.zegan@samsung.com>
diff --git a/ansible/roles/rke/.yamllint b/ansible/roles/rke/.yamllint
new file mode 100644
index 0000000..ad0be76
--- /dev/null
+++ b/ansible/roles/rke/.yamllint
@@ -0,0 +1,11 @@
+extends: default
+
+rules:
+ braces:
+ max-spaces-inside: 1
+ level: error
+ brackets:
+ max-spaces-inside: 1
+ level: error
+ line-length: disable
+ truthy: disable
diff --git a/ansible/roles/rke/molecule/default/destroy.yml b/ansible/roles/rke/molecule/default/destroy.yml
new file mode 100644
index 0000000..591da82
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/destroy.yml
@@ -0,0 +1,34 @@
+---
+- name: Destroy
+ hosts: localhost
+ connection: local
+ gather_facts: false
+ no_log: "{{ not (lookup('env', 'MOLECULE_DEBUG') | bool or molecule_yml.provisioner.log|default(false) | bool) }}"
+ tasks:
+ - name: Destroy molecule instance(s)
+ docker_container:
+ name: "{{ item.name }}"
+ docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+ state: absent
+ force_kill: "{{ item.force_kill | default(true) }}"
+ # Modification: we want to clean up old volumes.
+ keep_volumes: false
+ register: server
+ with_items: "{{ molecule_yml.platforms }}"
+ async: 7200
+ poll: 0
+
+ - name: Wait for instance(s) deletion to complete
+ async_status:
+ jid: "{{ item.ansible_job_id }}"
+ register: docker_jobs
+ until: docker_jobs.finished
+ retries: 300
+ with_items: "{{ server.results }}"
+
+ - name: Delete docker network(s)
+ docker_network:
+ name: "{{ item }}"
+ docker_host: "{{ item.docker_host | default(lookup('env', 'DOCKER_HOST') or 'unix://var/run/docker.sock') }}"
+ state: absent
+ with_items: "{{ molecule_yml.platforms | molecule_get_docker_networks }}"
diff --git a/ansible/roles/rke/molecule/default/molecule.yml b/ansible/roles/rke/molecule/default/molecule.yml
new file mode 100644
index 0000000..e8e5ad7
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/molecule.yml
@@ -0,0 +1,78 @@
+---
+dependency:
+ name: galaxy
+driver:
+ name: docker
+lint:
+ name: yamllint
+platforms:
+ - name: infrastructure-server
+ image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+ pre_build_image: true
+ privileged: true
+ override_command: false
+ restart_policy: unless-stopped
+ volumes:
+ - /var/lib/kubelet
+ - /var/lib/docker
+ env:
+ container: docker
+ groups:
+ - infrastructure
+ - kubernetes-control-plane
+ networks:
+ - name: rke
+ purge_networks: true
+
+ - name: kubernetes-node-1
+ image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+ pre_build_image: true
+ privileged: true
+ override_command: false
+ restart_policy: unless-stopped
+ env:
+ container: docker
+ volumes:
+ - /var/lib/kubelet
+ - /var/lib/docker
+ groups:
+ - kubernetes
+ networks:
+ - name: rke
+ purge_networks: true
+
+ - name: kubernetes-node-2
+ image: molecule-${PREBUILD_PLATFORM_DISTRO:-centos}:${PREBUILD_DISTRO_VERSION:-centos7.6}
+ pre_build_image: true
+ privileged: true
+ override_command: false
+ restart_policy: unless-stopped
+ env:
+ container: docker
+ volumes:
+ - /var/lib/kubelet
+ - /var/lib/docker
+ groups:
+ - kubernetes
+ networks:
+ - name: rke
+ purge_networks: true
+
+provisioner:
+ name: ansible
+ env:
+ ANSIBLE_ROLES_PATH: ../../../../test/roles
+ ANSIBLE_LIBRARY: ../../../../library
+ inventory:
+ links:
+ group_vars: ../../../../group_vars
+ options:
+ e: "app_data_path=/opt/onap"
+ lint:
+ name: ansible-lint
+scenario:
+ name: default
+verifier:
+ name: testinfra
+ lint:
+ name: flake8
diff --git a/ansible/roles/rke/molecule/default/playbook.yml b/ansible/roles/rke/molecule/default/playbook.yml
new file mode 100644
index 0000000..09dbfb8
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/playbook.yml
@@ -0,0 +1,30 @@
+---
+- name: "Set cluster_ip"
+ hosts: all
+ tasks:
+ - name: "Set cluster_ip fact"
+ set_fact:
+ cluster_ip: "{{ ansible_default_ipv4.address }}"
+
+- name: Configure kubernetes cluster (RKE)
+ hosts: infrastructure
+ roles:
+ - role: rke
+ vars:
+ mode: config
+
+- name: Prepare kubernetes nodes (RKE)
+ hosts:
+ - kubernetes
+ - kubernetes-control-plane
+ roles:
+ - role: rke
+ vars:
+ mode: node
+
+- name: Deploy kubernetes cluster (RKE)
+ hosts: infrastructure
+ roles:
+ - role: rke
+ vars:
+ mode: deploy
diff --git a/ansible/roles/rke/molecule/default/prepare.yml b/ansible/roles/rke/molecule/default/prepare.yml
new file mode 100644
index 0000000..6bad2b8
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/prepare.yml
@@ -0,0 +1,15 @@
+---
+- name: "Prepare hosts"
+ hosts: all
+ roles:
+ - role: prepare-rke
+ vars:
+ mode: all
+ - prepare-docker-dind
+
+- name: "Infra specific preparations"
+ hosts: infrastructure
+ roles:
+ - role: prepare-rke
+ vars:
+ mode: infra
diff --git a/ansible/roles/rke/molecule/default/tests/test_controlplane.py b/ansible/roles/rke/molecule/default/tests/test_controlplane.py
new file mode 100644
index 0000000..0bfbca2
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_controlplane.py
@@ -0,0 +1,14 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+ os.environ['MOLECULE_INVENTORY_FILE']).get_hosts(
+ 'kubernetes-control-plane')
+
+
+@pytest.mark.parametrize('container_name', [
+ 'kube-apiserver', 'kube-controller-manager', 'kube-scheduler', 'kubelet'])
+def test_container_running(host, container_name):
+ assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/molecule/default/tests/test_infrastructure.py b/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
new file mode 100644
index 0000000..9ba11d6
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_infrastructure.py
@@ -0,0 +1,56 @@
+import os
+import pytest
+import json
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+ os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('infrastructure')
+
+
+@pytest.mark.parametrize('filename', [
+ '/root/.kube/config',
+ '/opt/onap/cluster/cluster.yml',
+ '/opt/onap/cluster/cluster.rkestate'])
+def test_file_existence(host, filename):
+ assert host.file(filename).exists
+
+
+def test_rke_in_path(host):
+ assert host.find_command('rke') == '/usr/local/bin/rke'
+
+
+def test_rke_version_works(host):
+ # Note that we need to cd to the cluster data dir first, really.
+ assert host.run('cd /opt/onap/cluster && rke version').rc == 0
+
+
+def test_nodes_ready(host):
+ # Retrieve all node names.
+ nodecmdres = host.run('kubectl get nodes -o name')
+ assert nodecmdres.rc == 0
+ nodes = nodecmdres.stdout.split('\n')
+ for node in nodes:
+ assert host.run(
+ 'kubectl wait --timeout=0 --for=condition=ready ' + node).rc == 0
+
+
+def test_pods_ready(host):
+ # Retrieve all pods from all namespaces.
+ # Because we need pod and namespace name, we get full json representation.
+ podcmdres = host.run('kubectl get pods --all-namespaces -o json')
+ assert podcmdres.rc == 0
+ pods = json.loads(podcmdres.stdout)['items']
+ for pod in pods:
+ # Each pod may be either created by a job or not.
+ # In job case they should already be completed
+ # when we are here so we ignore them.
+ namespace = pod['metadata']['namespace']
+ podname = pod['metadata']['name']
+ condition = 'Ready'
+ if len(pod['metadata']['ownerReferences']) == 1 and pod[
+ 'metadata']['ownerReferences'][0]['kind'] == 'Job':
+ continue
+ assert host.run(
+ 'kubectl wait --timeout=120s --for=condition=' + condition + ' -n ' +
+ namespace + ' pods/' + podname).rc == 0
diff --git a/ansible/roles/rke/molecule/default/tests/test_kubernetes.py b/ansible/roles/rke/molecule/default/tests/test_kubernetes.py
new file mode 100644
index 0000000..887494f
--- /dev/null
+++ b/ansible/roles/rke/molecule/default/tests/test_kubernetes.py
@@ -0,0 +1,13 @@
+import os
+import pytest
+
+import testinfra.utils.ansible_runner
+
+testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
+ os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('kubernetes')
+
+
+@pytest.mark.parametrize('container_name', [
+ 'etcd', 'kubelet', 'kube-proxy'])
+def test_container_running(host, container_name):
+ assert host.docker(container_name).is_running
diff --git a/ansible/roles/rke/tasks/rke_config.yml b/ansible/roles/rke/tasks/rke_config.yml
index 4112e10..9dc0d8c 100644
--- a/ansible/roles/rke/tasks/rke_config.yml
+++ b/ansible/roles/rke/tasks/rke_config.yml
@@ -37,6 +37,7 @@
template:
src: cluster.yml.j2
dest: "{{ cluster_config_dir }}/cluster.yml"
+ register: cluster_yml
- name: Prepare rke addon manifest (dashboard)
template:
diff --git a/ansible/roles/rke/tasks/rke_deploy.yml b/ansible/roles/rke/tasks/rke_deploy.yml
index 9983d08..7b36f55 100644
--- a/ansible/roles/rke/tasks/rke_deploy.yml
+++ b/ansible/roles/rke/tasks/rke_deploy.yml
@@ -1,8 +1,17 @@
---
+- name: "Check if rke is deployed"
+ command: "rke version"
+ args:
+ chdir: "{{ cluster_config_dir }}"
+ failed_when: false
+ changed_when: false
+ register: rke_deployed
+
- name: Run rke up
command: "{{ rke_bin_dir }}/rke up --config cluster.yml"
args:
chdir: "{{ cluster_config_dir }}"
+ when: rke_deployed.rc != 0 or cluster_yml.changed # noqa 503
- name: Ensure .kube directory is present
file:
diff --git a/ansible/test/roles/prepare-rke/defaults/main.yml b/ansible/test/roles/prepare-rke/defaults/main.yml
new file mode 100644
index 0000000..2cf8563
--- /dev/null
+++ b/ansible/test/roles/prepare-rke/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+#The rke version.
+rke_version: 0.2.0
+#The kubectl version.
+kubectl_version: 1.13.5
diff --git a/ansible/test/roles/prepare-rke/tasks/all.yml b/ansible/test/roles/prepare-rke/tasks/all.yml
new file mode 100644
index 0000000..d4b67c1
--- /dev/null
+++ b/ansible/test/roles/prepare-rke/tasks/all.yml
@@ -0,0 +1,6 @@
+#This is needed because login from non root is blocked by default.
+- name: "Allow non root logins"
+ service:
+ name: systemd-user-sessions
+ state: started
+
diff --git a/ansible/test/roles/prepare-rke/tasks/infra.yml b/ansible/test/roles/prepare-rke/tasks/infra.yml
new file mode 100644
index 0000000..55ab7f1
--- /dev/null
+++ b/ansible/test/roles/prepare-rke/tasks/infra.yml
@@ -0,0 +1,16 @@
+---
+- name: "Ensure {{ app_data_path }} exists"
+ file:
+ path: "{{ app_data_path }}/downloads"
+ state: directory
+
+- name: "Install rke-{{ rke_version }}"
+ get_url:
+ url: "https://github.com/rancher/rke/releases/download/v{{ rke_version }}/rke_linux-amd64"
+ dest: "{{ app_data_path }}/downloads/rke"
+
+- name: "Install kubectl-{{ kubectl_version }}"
+ get_url:
+ url: "https://storage.googleapis.com/kubernetes-release/release/v{{ kubectl_version }}/bin/linux/amd64/kubectl"
+ dest: "/usr/local/bin/kubectl"
+ mode: 0755
diff --git a/ansible/test/roles/prepare-rke/tasks/main.yml b/ansible/test/roles/prepare-rke/tasks/main.yml
new file mode 100644
index 0000000..210c9b5
--- /dev/null
+++ b/ansible/test/roles/prepare-rke/tasks/main.yml
@@ -0,0 +1,2 @@
+---
+- include_tasks: "{{ mode }}.yml"