Merge "create a XTesting directory to first add the required files for a RIC platform deployment and healthcheck work flow demonstration"
diff --git a/XTesting/XTesting-demo/README.md b/XTesting/XTesting-demo/README.md
new file mode 100644
index 0000000..7f84b35
--- /dev/null
+++ b/XTesting/XTesting-demo/README.md
@@ -0,0 +1,8 @@
+# XTesting-demo
+
+This repo is to demonstrate the XTesting work flow against the deployment of the OSC RIC platform followed up with a health check test case.
+
+To run the demo, clone the repo on the XTesting host and run the following command:
+
+sudo ./install_dependencies.sh # if the XTesting is not yet set up on the host
+sudo ./demo.sh target-ip private-key-file-path [working-directory]
diff --git a/XTesting/XTesting-demo/demo.sh b/XTesting/XTesting-demo/demo.sh
new file mode 100755
index 0000000..8cac36a
--- /dev/null
+++ b/XTesting/XTesting-demo/demo.sh
@@ -0,0 +1,90 @@
+#! /bin/bash
+set -x
+
+echo "This is to demonstrate the XTesting work flow against the OSC RIC platform deployment and perform health check"
+
+if [ $# -lt 2 ]
+then
+ echo "Usage: $0 target-ip private-key-file-path [working-directory]"
+ exit 1
+fi
+
+# pick up the input parameters from command line
+IP=$1
+KEYFILE=$2
+ORIG=$PWD
+if [ $# -ge 3 ]
+then
+ WORKDIR=$3
+else
+ WORKDIR=$PWD
+fi
+
+# make the working directory if it's not there yet
+if [ ! -d $WORKDIR ]
+then
+ mkdir -p $WORKDIR
+fi
+
+# copy over the health check test case to the working directory
+cp healthcheck.robot $WORKDIR
+
+cd $WORKDIR
+
+# replace it with the target IP address for health check
+sed -i "s/TARGET-IP/${IP}/" healthcheck.robot
+
+###### step 1 deploy Kubernetes cluster and obtain the Kube config
+
+# remove the old one if it's already there
+if [ -d kubeadm ]
+then
+ rm -rf kubeadm
+fi
+
+# copy over the content to build the kubeadm container
+mkdir kubeadm
+cp -r $ORIG/../kubeadm/* kubeadm/
+
+cd kubeadm
+# update the host IP in the environment file
+TMPFILE=/tmp/tmp`date +%s`
+cat sample_env | sed -e '/ANSIBLE_HOST_IP/d' > $TMPFILE
+echo "ANSIBLE_HOST_IP=${IP}" > sample_env
+cat $TMPFILE >> sample_env
+
+# copy the private key to the inventory/sample folder as id_rsa
+cp $KEYFILE id_rsa
+chmod 400 id_rsa
+
+# build the Docker image
+docker build -t kubeadm .
+
+# run the docker container to deploy Kubernetes onto the SUT specified by the IP address
+docker run -v ~/.kube:/kubeadm/config kubeadm
+
+###### step 2 complete the deployment based on the Kube config from step 1
+
+cd $WORKDIR
+# remove the old one if it's already there
+if [ -d richelm ]
+then
+ rm -rf richelm
+fi
+
+#copy over the content to build the richelm container
+mkdir richelm
+cp -r $ORIG/../richelm/* richelm/
+
+#build the container
+cd richelm && ./build.sh static
+
+#run the container with the kubeconfig to complete the RIC platform deployment
+sudo docker run -ti --rm -w /apps -v ~/.kube:/root/.kube -t richelmlegacy:1.19.16
+
+# sometimes some RIC platform containers are not up right away so wait a bit
+sleep 60
+
+cd $WORKDIR
+###### step 3 run the health check test case to complete the demo
+ansible-playbook healthcheck.robot
diff --git a/XTesting/XTesting-demo/healthcheck.robot b/XTesting/XTesting-demo/healthcheck.robot
new file mode 100644
index 0000000..7014c0a
--- /dev/null
+++ b/XTesting/XTesting-demo/healthcheck.robot
@@ -0,0 +1,12 @@
+---
+- name: Shell module example
+ hosts: 127.0.0.1
+ tasks:
+
+ - name: Check system information
+ shell:
+ "curl -v http://TARGET-IP:32080/appmgr/ric/v1/health/ready 2>&1"
+ register: os_info
+
+ - debug:
+ msg: "{{os_info.stdout_lines}}"
diff --git a/XTesting/XTesting-demo/install-dependencies.sh b/XTesting/XTesting-demo/install-dependencies.sh
new file mode 100755
index 0000000..7238571
--- /dev/null
+++ b/XTesting/XTesting-demo/install-dependencies.sh
@@ -0,0 +1,35 @@
+#! /bin/bash -x
+
+# openssh
+apt install -y openssh-server
+systemctl status ssh
+ufw allow ssh
+
+# docker
+apt-get remove docker docker-engine docker.io containerd runc
+apt-get update
+apt-get install -y \
+ ca-certificates \
+ curl \
+ gnupg \
+ lsb-release
+
+mkdir -p /etc/apt/keyrings
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+echo \
+ "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+
+apt-get update
+apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+systemctl enable docker
+systemctl start docker
+
+# XTesting dependencies
+apt update && apt install git -y
+[ -z "$VIRTUAL_ENV" ] && apt install python3-pip -y && pip3 install ansible
+ansible-galaxy install collivier.xtesting
+ansible-galaxy collection install ansible.posix community.general community.grafana \
+ community.kubernetes community.docker community.postgresql
diff --git a/XTesting/kubeadm/Dockerfile b/XTesting/kubeadm/Dockerfile
new file mode 100644
index 0000000..83cdbeb
--- /dev/null
+++ b/XTesting/kubeadm/Dockerfile
@@ -0,0 +1,31 @@
+# Use imutable image tags rather than mutable tags (like ubuntu:20.04)
+FROM ubuntu:focal-20220531
+
+ARG ARCH=amd64
+ARG KUBE_VERSION=v1.19.16
+ARG TZ=Etc/UTC
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN apt update -y \
+ && apt install -y \
+ libssl-dev python3-dev sshpass apt-transport-https jq moreutils \
+ ca-certificates curl gnupg2 software-properties-common python3-pip unzip rsync git \
+ && rm -rf /var/lib/apt/lists/*
+
+
+ENV LANG=C.UTF-8
+
+WORKDIR /kubeadm
+COPY . .
+
+RUN /usr/bin/python3 -m pip install --no-cache-dir pip -U \
+ && python3 -m pip install --no-cache-dir -r requirements.txt \
+ && update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$ARCH/kubectl \
+ && chmod a+x kubectl \
+ && mv kubectl /usr/local/bin/kubectl
+
+RUN chmod a+x deploy.sh
+
+CMD /kubeadm/deploy.sh
diff --git a/XTesting/kubeadm/cluster.yml b/XTesting/kubeadm/cluster.yml
new file mode 100644
index 0000000..15db72c
--- /dev/null
+++ b/XTesting/kubeadm/cluster.yml
@@ -0,0 +1,132 @@
+- hosts: all
+ become: true
+ tasks:
+ - name: Install packages that allow apt to be used over HTTPS
+ apt:
+ name: "{{ packages }}"
+ state: present
+ vars:
+ packages:
+ - apt-transport-https
+ - ca-certificates
+ - curl
+ - gnupg-agent
+ - software-properties-common
+
+ - name: Add the apt signing key for Docker
+ apt_key:
+ url: https://download.docker.com/linux/ubuntu/gpg
+ state: present
+
+ - name: Add the apt repository for the stable Docker version
+ apt_repository:
+ repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable
+ state: present
+
+ - name: Install Docker and its dependencies
+ apt:
+ name: "{{ packages }}"
+ state: present
+ update_cache: yes
+ vars:
+ packages:
+ - docker-ce
+ - docker-ce-cli
+ - containerd.io
+
+ - name: Ensure user group exists
+ group:
+ name: "{{ ansible_user }}"
+
+ - name: Add user to the docker group
+ user:
+ name: "{{ ansible_user }}"
+ groups: docker,{{ ansible_user }}
+ append: yes
+
+ - name: Remove swapfile from /etc/fstab
+ mount:
+ name: "{{ item }}"
+ fstype: swap
+ state: absent
+ with_items:
+ - swap
+ - none
+
+ - name: Disable swap
+ command: swapoff -a
+ when: ansible_swaptotal_mb > 0
+
+ - name: Add the apt signing key for Kubernetes
+ apt_key:
+ url: https://packages.cloud.google.com/apt/doc/apt-key.gpg
+ state: present
+
+ - name: Add the apt repository for Kubernetes
+ apt_repository:
+ repo: deb https://apt.kubernetes.io/ kubernetes-xenial main
+ state: present
+ filename: kubernetes.list
+
+ - name: Install Kubernetes binaries
+ apt:
+ name: "{{ packages }}"
+ update_cache: yes
+ vars:
+ packages:
+ - kubelet=1.19.16-00
+ - kubeadm=1.19.16-00
+ - kubectl=1.19.16-00
+
+ - name: Restart kubelet
+ service:
+ name: kubelet
+ daemon_reload: yes
+ state: restarted
+
+ - name: Making systemd as Cgroup Driver
+ copy:
+ src: daemon.json
+ dest: /etc/docker/daemon.json
+
+ - name: Restaring Docker...
+ service:
+ name: docker
+ state: restarted
+
+ - name: update sysctl param
+ copy:
+ src: k8s.conf
+ dest: /etc/sysctl.d/k8s.conf
+ owner: root
+ group: root
+
+ - name: System reload
+ shell: sysctl --system
+
+ - name: Initialize the Kubernetes cluster
+ become: true
+ command: kubeadm init --pod-network-cidr=10.244.0.0/16
+
+ - name: Setup kubeconfig
+ command: "{{ item }}"
+ with_items:
+ - mkdir -p "$HOME/.kube"
+ - sudo cp /etc/kubernetes/admin.conf "$HOME/.kube/config"
+ - sudo chown "{{ ansible_user }}:{{ ansible_user }}" "$HOME/.kube/config"
+
+ - name: Deploy Flannel
+ become: true
+ command: kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
+
+ - name: Remove taints from master node
+ become: true
+ command: kubectl taint nodes --all node-role.kubernetes.io/master-
+
+ - name: Create Tiller service account
+ become: true
+ command: kubectl -n kube-system create serviceaccount tiller
+
+ - name: Setup Tiller ClusterBinding
+ become: true
+ command: kubectl create clusterrolebinding tiller --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
diff --git a/XTesting/kubeadm/daemon.json b/XTesting/kubeadm/daemon.json
new file mode 100644
index 0000000..5d18abc
--- /dev/null
+++ b/XTesting/kubeadm/daemon.json
@@ -0,0 +1,8 @@
+{
+ "exec-opts": ["native.cgroupdriver=systemd"],
+ "log-driver": "json-file",
+ "log-opts": {
+ "max-size": "100m"
+ },
+ "storage-driver": "overlay2"
+}
diff --git a/XTesting/kubeadm/deploy.sh b/XTesting/kubeadm/deploy.sh
new file mode 100644
index 0000000..198f69e
--- /dev/null
+++ b/XTesting/kubeadm/deploy.sh
@@ -0,0 +1,12 @@
+#!/bin/bash -x
+
+#cp -rfp inventory/sample inventory/oransc-cluster
+. sample_env
+declare -a IPS=($ANSIBLE_HOST_IP)
+CONFIG_FILE=hosts.yaml python3 inventory.py ${IPS[@]}
+export ANSIBLE_HOST_KEY_CHECKING=False
+ansible-playbook -i hosts.yaml --become --private-key ${ANSIBLE_SSH_KEY} cluster.yml
+
+#sshpass -p $ANSIBLE_PASSWORD scp -o StrictHostKeyChecking=no -i ${ANSIBLE_SSH_KEY} -q root@$ANSIBLE_HOST_IP:/root/.kube/config ${PROJECT_ROOT}/config
+scp -o StrictHostKeyChecking=no -i ${ANSIBLE_SSH_KEY} -q root@$ANSIBLE_HOST_IP:/root/.kube/config ${PROJECT_ROOT}/config
+sed -i "s/127.0.0.1/${ANSIBLE_HOST_IP}/g" "${PROJECT_ROOT}"/config
diff --git a/XTesting/kubeadm/inventory.py b/XTesting/kubeadm/inventory.py
new file mode 100644
index 0000000..0a00599
--- /dev/null
+++ b/XTesting/kubeadm/inventory.py
@@ -0,0 +1,493 @@
+#!/usr/bin/env python3
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Usage: inventory.py ip1 [ip2 ...]
+# Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+#
+# Advanced usage:
+# Add another host after initial creation: inventory.py 10.10.1.5
+# Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
+# Add hosts with different ip and access ip:
+# inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.1.3
+# Add hosts with a specific hostname, ip, and optional access ip:
+# inventory.py first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
+# Delete a host: inventory.py -10.10.1.3
+# Delete a host by id: inventory.py -node1
+#
+# Load a YAML or JSON file with inventory data: inventory.py load hosts.yaml
+# YAML file should be in the following format:
+# group1:
+# host1:
+# ip: X.X.X.X
+# var: val
+# group2:
+# host2:
+# ip: X.X.X.X
+
+from collections import OrderedDict
+from ipaddress import ip_address
+from ruamel.yaml import YAML
+
+import os
+import re
+import subprocess
+import sys
+
+ROLES = ['all', 'kube_control_plane', 'kube_node', 'etcd', 'k8s_cluster',
+ 'calico_rr']
+PROTECTED_NAMES = ROLES
+AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'print_hostnames',
+ 'load', 'add']
+_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
+ '0': False, 'no': False, 'false': False, 'off': False}
+yaml = YAML()
+yaml.Representer.add_representer(OrderedDict, yaml.Representer.represent_dict)
+
+
+def get_var_as_bool(name, default):
+ value = os.environ.get(name, '')
+ return _boolean_states.get(value.lower(), default)
+
+# Configurable as shell vars start
+
+
+CONFIG_FILE = os.environ.get("CONFIG_FILE", "./hosts.yaml")
+# Remove the reference of KUBE_MASTERS after some deprecation cycles.
+KUBE_CONTROL_HOSTS = int(os.environ.get("KUBE_CONTROL_HOSTS",
+ os.environ.get("KUBE_MASTERS", 2)))
+# Reconfigures cluster distribution at scale
+SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50))
+MASSIVE_SCALE_THRESHOLD = int(os.environ.get("MASSIVE_SCALE_THRESHOLD", 200))
+
+DEBUG = get_var_as_bool("DEBUG", True)
+HOST_PREFIX = os.environ.get("HOST_PREFIX", "node")
+USE_REAL_HOSTNAME = get_var_as_bool("USE_REAL_HOSTNAME", False)
+
+# Configurable as shell vars end
+
+
+class KubesprayInventory(object):
+
+ def __init__(self, changed_hosts=None, config_file=None):
+ self.config_file = config_file
+ self.yaml_config = {}
+ loadPreviousConfig = False
+ printHostnames = False
+ # See whether there are any commands to process
+ if changed_hosts and changed_hosts[0] in AVAILABLE_COMMANDS:
+ if changed_hosts[0] == "add":
+ loadPreviousConfig = True
+ changed_hosts = changed_hosts[1:]
+ elif changed_hosts[0] == "print_hostnames":
+ loadPreviousConfig = True
+ printHostnames = True
+ else:
+ self.parse_command(changed_hosts[0], changed_hosts[1:])
+ sys.exit(0)
+
+ # If the user wants to remove a node, we need to load the config anyway
+ if changed_hosts and changed_hosts[0][0] == "-":
+ loadPreviousConfig = True
+
+ if self.config_file and loadPreviousConfig: # Load previous YAML file
+ try:
+ self.hosts_file = open(config_file, 'r')
+ self.yaml_config = yaml.load(self.hosts_file)
+ except OSError as e:
+ # I am assuming we are catching "cannot open file" exceptions
+ print(e)
+ sys.exit(1)
+
+ if printHostnames:
+ self.print_hostnames()
+ sys.exit(0)
+
+ self.ensure_required_groups(ROLES)
+
+ if changed_hosts:
+ changed_hosts = self.range2ips(changed_hosts)
+ self.hosts = self.build_hostnames(changed_hosts,
+ loadPreviousConfig)
+ self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
+ self.set_all(self.hosts)
+ self.set_k8s_cluster()
+ etcd_hosts_count = 3 if len(self.hosts.keys()) >= 3 else 1
+ self.set_etcd(list(self.hosts.keys())[:etcd_hosts_count])
+ if len(self.hosts) >= SCALE_THRESHOLD:
+ self.set_kube_control_plane(list(self.hosts.keys())[
+ etcd_hosts_count:(etcd_hosts_count + KUBE_CONTROL_HOSTS)])
+ else:
+ self.set_kube_control_plane(
+ list(self.hosts.keys())[:KUBE_CONTROL_HOSTS])
+ self.set_kube_node(self.hosts.keys())
+ if len(self.hosts) >= SCALE_THRESHOLD:
+ self.set_calico_rr(list(self.hosts.keys())[:etcd_hosts_count])
+ else: # Show help if no options
+ self.show_help()
+ sys.exit(0)
+
+ self.write_config(self.config_file)
+
+ def write_config(self, config_file):
+ if config_file:
+ with open(self.config_file, 'w') as f:
+ yaml.dump(self.yaml_config, f)
+
+ else:
+ print("WARNING: Unable to save config. Make sure you set "
+ "CONFIG_FILE env var.")
+
+ def debug(self, msg):
+ if DEBUG:
+ print("DEBUG: {0}".format(msg))
+
+ def get_ip_from_opts(self, optstring):
+ if 'ip' in optstring:
+ return optstring['ip']
+ else:
+ raise ValueError("IP parameter not found in options")
+
+ def ensure_required_groups(self, groups):
+ for group in groups:
+ if group == 'all':
+ self.debug("Adding group {0}".format(group))
+ if group not in self.yaml_config:
+ all_dict = OrderedDict([('hosts', OrderedDict({})),
+ ('children', OrderedDict({}))])
+ self.yaml_config = {'all': all_dict}
+ else:
+ self.debug("Adding group {0}".format(group))
+ if group not in self.yaml_config['all']['children']:
+ self.yaml_config['all']['children'][group] = {'hosts': {}}
+
+ def get_host_id(self, host):
+ '''Returns integer host ID (without padding) from a given hostname.'''
+ try:
+ short_hostname = host.split('.')[0]
+ return int(re.findall("\\d+$", short_hostname)[-1])
+ except IndexError:
+ raise ValueError("Host name must end in an integer")
+
+ # Keeps already specified hosts,
+ # and adds or removes the hosts provided as an argument
+ def build_hostnames(self, changed_hosts, loadPreviousConfig=False):
+ existing_hosts = OrderedDict()
+ highest_host_id = 0
+ # Load already existing hosts from the YAML
+ if loadPreviousConfig:
+ try:
+ for host in self.yaml_config['all']['hosts']:
+ # Read configuration of an existing host
+ hostConfig = self.yaml_config['all']['hosts'][host]
+ existing_hosts[host] = hostConfig
+ # If the existing host seems
+ # to have been created automatically, detect its ID
+ if host.startswith(HOST_PREFIX):
+ host_id = self.get_host_id(host)
+ if host_id > highest_host_id:
+ highest_host_id = host_id
+ except Exception as e:
+ # I am assuming we are catching automatically
+ # created hosts without IDs
+ print(e)
+ sys.exit(1)
+
+ # FIXME(mattymo): Fix condition where delete then add reuses highest id
+ next_host_id = highest_host_id + 1
+ next_host = ""
+
+ username = os.environ.get("ANSIBLE_USER", 'osc_int')
+ password = os.environ.get("ANSIBLE_PASSWORD", 'osc_int')
+
+
+ all_hosts = existing_hosts.copy()
+ for host in changed_hosts:
+ # Delete the host from config the hostname/IP has a "-" prefix
+ if host[0] == "-":
+ realhost = host[1:]
+ if self.exists_hostname(all_hosts, realhost):
+ self.debug("Marked {0} for deletion.".format(realhost))
+ all_hosts.pop(realhost)
+ elif self.exists_ip(all_hosts, realhost):
+ self.debug("Marked {0} for deletion.".format(realhost))
+ self.delete_host_by_ip(all_hosts, realhost)
+ # Host/Argument starts with a digit,
+ # then we assume its an IP address
+ elif host[0].isdigit():
+ if ',' in host:
+ ip, access_ip = host.split(',')
+ else:
+ ip = host
+ access_ip = host
+ if self.exists_hostname(all_hosts, host):
+ self.debug("Skipping existing host {0}.".format(host))
+ continue
+ elif self.exists_ip(all_hosts, ip):
+ self.debug("Skipping existing host {0}.".format(ip))
+ continue
+
+ if USE_REAL_HOSTNAME:
+ cmd = ("ssh -oStrictHostKeyChecking=no "
+ + access_ip + " 'hostname -s'")
+ next_host = subprocess.check_output(cmd, shell=True)
+ next_host = next_host.strip().decode('ascii')
+ else:
+ # Generates a hostname because we have only an IP address
+ next_host = "{0}{1}".format(HOST_PREFIX, next_host_id)
+ next_host_id += 1
+ # Uses automatically generated node name
+ # in case we dont provide it.
+ if os.getenv('ANSIBLE_SSH_KEY'):
+ all_hosts[next_host] = {'ansible_host': access_ip,
+ 'ansible_user': username,
+ 'ansible_ssh_private_key_file': os.getenv('ANSIBLE_SSH_KEY'),
+ 'ip': ip,
+ 'access_ip': access_ip}
+ else:
+ all_hosts[next_host] = {'ansible_host': access_ip,
+ 'ansible_user': username,
+ 'ansible_password': password,
+ 'ip': ip,
+ 'access_ip': access_ip}
+ # Host/Argument starts with a letter, then we assume its a hostname
+ elif host[0].isalpha():
+ if ',' in host:
+ try:
+ hostname, ip, access_ip = host.split(',')
+ except Exception:
+ hostname, ip = host.split(',')
+ access_ip = ip
+ if self.exists_hostname(all_hosts, host):
+ self.debug("Skipping existing host {0}.".format(host))
+ continue
+ elif self.exists_ip(all_hosts, ip):
+ self.debug("Skipping existing host {0}.".format(ip))
+ continue
+ all_hosts[hostname] = {'ansible_host': access_ip,
+ 'ip': ip,
+ 'access_ip': access_ip}
+ return all_hosts
+
+ # Expand IP ranges into individual addresses
+ def range2ips(self, hosts):
+ reworked_hosts = []
+
+ def ips(start_address, end_address):
+ try:
+ # Python 3.x
+ start = int(ip_address(start_address))
+ end = int(ip_address(end_address))
+ except Exception:
+ # Python 2.7
+ start = int(ip_address(str(start_address)))
+ end = int(ip_address(str(end_address)))
+ return [ip_address(ip).exploded for ip in range(start, end + 1)]
+
+ for host in hosts:
+ if '-' in host and not (host.startswith('-') or host[0].isalpha()):
+ start, end = host.strip().split('-')
+ try:
+ reworked_hosts.extend(ips(start, end))
+ except ValueError:
+ raise Exception("Range of ip_addresses isn't valid")
+ else:
+ reworked_hosts.append(host)
+ return reworked_hosts
+
+ def exists_hostname(self, existing_hosts, hostname):
+ return hostname in existing_hosts.keys()
+
+ def exists_ip(self, existing_hosts, ip):
+ for host_opts in existing_hosts.values():
+ if ip == self.get_ip_from_opts(host_opts):
+ return True
+ return False
+
+ def delete_host_by_ip(self, existing_hosts, ip):
+ for hostname, host_opts in existing_hosts.items():
+ if ip == self.get_ip_from_opts(host_opts):
+ del existing_hosts[hostname]
+ return
+ raise ValueError("Unable to find host by IP: {0}".format(ip))
+
+ def purge_invalid_hosts(self, hostnames, protected_names=[]):
+ for role in self.yaml_config['all']['children']:
+ if role != 'k8s_cluster' and self.yaml_config['all']['children'][role]['hosts']: # noqa
+ all_hosts = self.yaml_config['all']['children'][role]['hosts'].copy() # noqa
+ for host in all_hosts.keys():
+ if host not in hostnames and host not in protected_names:
+ self.debug(
+ "Host {0} removed from role {1}".format(host, role)) # noqa
+ del self.yaml_config['all']['children'][role]['hosts'][host] # noqa
+ # purge from all
+ if self.yaml_config['all']['hosts']:
+ all_hosts = self.yaml_config['all']['hosts'].copy()
+ for host in all_hosts.keys():
+ if host not in hostnames and host not in protected_names:
+ self.debug("Host {0} removed from role all".format(host))
+ del self.yaml_config['all']['hosts'][host]
+
+ def add_host_to_group(self, group, host, opts=""):
+ self.debug("adding host {0} to group {1}".format(host, group))
+ if group == 'all':
+ if self.yaml_config['all']['hosts'] is None:
+ self.yaml_config['all']['hosts'] = {host: None}
+ self.yaml_config['all']['hosts'][host] = opts
+ elif group != 'k8s_cluster:children':
+ if self.yaml_config['all']['children'][group]['hosts'] is None:
+ self.yaml_config['all']['children'][group]['hosts'] = {
+ host: None}
+ else:
+ self.yaml_config['all']['children'][group]['hosts'][host] = None # noqa
+
+ def set_kube_control_plane(self, hosts):
+ for host in hosts:
+ self.add_host_to_group('kube_control_plane', host)
+
+ def set_all(self, hosts):
+ for host, opts in hosts.items():
+ self.add_host_to_group('all', host, opts)
+
+ def set_k8s_cluster(self):
+ k8s_cluster = {'children': {'kube_control_plane': None,
+ 'kube_node': None}}
+ self.yaml_config['all']['children']['k8s_cluster'] = k8s_cluster
+
+ def set_calico_rr(self, hosts):
+ for host in hosts:
+ if host in self.yaml_config['all']['children']['kube_control_plane']: # noqa
+ self.debug("Not adding {0} to calico_rr group because it "
+ "conflicts with kube_control_plane "
+ "group".format(host))
+ continue
+ if host in self.yaml_config['all']['children']['kube_node']:
+ self.debug("Not adding {0} to calico_rr group because it "
+ "conflicts with kube_node group".format(host))
+ continue
+ self.add_host_to_group('calico_rr', host)
+
+ def set_kube_node(self, hosts):
+ for host in hosts:
+ if len(self.yaml_config['all']['hosts']) >= SCALE_THRESHOLD:
+ if host in self.yaml_config['all']['children']['etcd']['hosts']: # noqa
+ self.debug("Not adding {0} to kube_node group because of "
+ "scale deployment and host is in etcd "
+ "group.".format(host))
+ continue
+ if len(self.yaml_config['all']['hosts']) >= MASSIVE_SCALE_THRESHOLD: # noqa
+ if host in self.yaml_config['all']['children']['kube_control_plane']['hosts']: # noqa
+ self.debug("Not adding {0} to kube_node group because of "
+ "scale deployment and host is in "
+ "kube_control_plane group.".format(host))
+ continue
+ self.add_host_to_group('kube_node', host)
+
+ def set_etcd(self, hosts):
+ for host in hosts:
+ self.add_host_to_group('etcd', host)
+
+ def load_file(self, files=None):
+ '''Directly loads JSON to inventory.'''
+
+ if not files:
+ raise Exception("No input file specified.")
+
+ import json
+
+ for filename in list(files):
+ # Try JSON
+ try:
+ with open(filename, 'r') as f:
+ data = json.load(f)
+ except ValueError:
+ raise Exception("Cannot read %s as JSON, or CSV", filename)
+
+ self.ensure_required_groups(ROLES)
+ self.set_k8s_cluster()
+ for group, hosts in data.items():
+ self.ensure_required_groups([group])
+ for host, opts in hosts.items():
+ optstring = {'ansible_host': opts['ip'],
+ 'ip': opts['ip'],
+ 'access_ip': opts['ip']}
+ self.add_host_to_group('all', host, optstring)
+ self.add_host_to_group(group, host)
+ self.write_config(self.config_file)
+
+ def parse_command(self, command, args=None):
+ if command == 'help':
+ self.show_help()
+ elif command == 'print_cfg':
+ self.print_config()
+ elif command == 'print_ips':
+ self.print_ips()
+ elif command == 'print_hostnames':
+ self.print_hostnames()
+ elif command == 'load':
+ self.load_file(args)
+ else:
+ raise Exception("Invalid command specified.")
+
+ def show_help(self):
+ help_text = '''Usage: inventory.py ip1 [ip2 ...]
+Examples: inventory.py 10.10.1.3 10.10.1.4 10.10.1.5
+
+Available commands:
+help - Display this message
+print_cfg - Write inventory file to stdout
+print_ips - Write a space-delimited list of IPs from "all" group
+print_hostnames - Write a space-delimited list of Hostnames from "all" group
+add - Adds specified hosts into an already existing inventory
+
+Advanced usage:
+Create new or overwrite old inventory file: inventory.py 10.10.1.5
+Add another host after initial creation: inventory.py add 10.10.1.6
+Add range of hosts: inventory.py 10.10.1.3-10.10.1.5
+Add hosts with different ip and access ip: inventory.py 10.0.0.1,192.168.10.1 10.0.0.2,192.168.10.2 10.0.0.3,192.168.10.3
+Add hosts with a specific hostname, ip, and optional access ip: first,10.0.0.1,192.168.10.1 second,10.0.0.2 last,10.0.0.3
+Delete a host: inventory.py -10.10.1.3
+Delete a host by id: inventory.py -node1
+
+Configurable env vars:
+DEBUG Enable debug printing. Default: True
+CONFIG_FILE File to write config to Default: ./inventory/sample/hosts.yaml
+HOST_PREFIX Host prefix for generated hosts. Default: node
+KUBE_CONTROL_HOSTS Set the number of kube-control-planes. Default: 2
+SCALE_THRESHOLD Separate ETCD role if # of nodes >= 50
+MASSIVE_SCALE_THRESHOLD Separate K8s control-plane and ETCD if # of nodes >= 200
+''' # noqa
+ print(help_text)
+
+ def print_config(self):
+ yaml.dump(self.yaml_config, sys.stdout)
+
+ def print_hostnames(self):
+ print(' '.join(self.yaml_config['all']['hosts'].keys()))
+
+ def print_ips(self):
+ ips = []
+ for host, opts in self.yaml_config['all']['hosts'].items():
+ ips.append(self.get_ip_from_opts(opts))
+ print(' '.join(ips))
+
+
+def main(argv=None):
+ if not argv:
+ argv = sys.argv[1:]
+ KubesprayInventory(argv, CONFIG_FILE)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/XTesting/kubeadm/k8s.conf b/XTesting/kubeadm/k8s.conf
new file mode 100644
index 0000000..d46bab8
--- /dev/null
+++ b/XTesting/kubeadm/k8s.conf
@@ -0,0 +1,3 @@
+net.bridge.bridge-nf-call-ip6tables = 1
+net.bridge.bridge-nf-call-iptables = 1
+net.ipv4.ip_forward = 1
diff --git a/XTesting/kubeadm/requirements.txt b/XTesting/kubeadm/requirements.txt
new file mode 100644
index 0000000..f8b1e50
--- /dev/null
+++ b/XTesting/kubeadm/requirements.txt
@@ -0,0 +1,10 @@
+ansible==5.7.1
+ansible-core==2.12.5
+cryptography==3.4.8
+jinja2==2.11.3
+netaddr==0.7.19
+pbr==5.4.4
+jmespath==0.9.5
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.6
+MarkupSafe==1.1.1
diff --git a/XTesting/kubeadm/sample_env b/XTesting/kubeadm/sample_env
new file mode 100644
index 0000000..00c4891
--- /dev/null
+++ b/XTesting/kubeadm/sample_env
@@ -0,0 +1,5 @@
+ANSIBLE_HOST_IP=155.98.36.91
+ANSIBLE_USERNAME=osc_int
+ANSIBLE_PASSWORD=osc_int
+PROJECT_ROOT=/kubeadm/config
+ANSIBLE_SSH_KEY=id_rsa
diff --git a/XTesting/richelm/Dockerfile b/XTesting/richelm/Dockerfile
new file mode 100644
index 0000000..b07c377
--- /dev/null
+++ b/XTesting/richelm/Dockerfile
@@ -0,0 +1,74 @@
+FROM alpine
+
+# Ignore to update versions here
+# docker build --no-cache --build-arg KUBECTL_VERSION=${tag} --build-arg HELM_VERSION=${helm} --build-arg KUSTOMIZE_VERSION=${kustomize_version} -t ${image}:${tag} .
+ARG HELM_VERSION=2.17.0
+ARG KUBECTL_VERSION=1.19.16
+ARG KUSTOMIZE_VERSION=v3.8.1
+ARG KUBESEAL_VERSION=0.18.1
+
+# Install helm (latest release)
+# ENV BASE_URL="https://storage.googleapis.com/kubernetes-helm"
+ENV BASE_URL="https://get.helm.sh"
+ENV TAR_FILE="helm-v${HELM_VERSION}-linux-amd64.tar.gz"
+RUN apk add --update --no-cache curl ca-certificates bash git && \
+ curl -sL ${BASE_URL}/${TAR_FILE} | tar -xvz && \
+ mv linux-amd64/helm /usr/bin/helm && \
+ chmod +x /usr/bin/helm && \
+ rm -rf linux-amd64
+
+RUN helm init --client-only
+
+# add helm-diff
+RUN helm plugin install https://github.com/databus23/helm-diff && rm -rf /tmp/helm-*
+
+# add helm-unittest
+RUN helm plugin install https://github.com/quintush/helm-unittest && rm -rf /tmp/helm-*
+
+# add helm-push
+RUN helm plugin install https://github.com/chartmuseum/helm-push && rm -rf /tmp/helm-*
+
+# Install kubectl (same version of aws esk)
+RUN curl -sLO https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \
+ mv kubectl /usr/bin/kubectl && \
+ chmod +x /usr/bin/kubectl
+
+# Install kustomize (latest release)
+RUN curl -sLO https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \
+ tar xvzf kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \
+ mv kustomize /usr/bin/kustomize && \
+ chmod +x /usr/bin/kustomize
+
+# Install eksctl (latest version)
+RUN curl -sL "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp && \
+ mv /tmp/eksctl /usr/bin && \
+ chmod +x /usr/bin/eksctl
+
+# Install awscli
+RUN apk add --update --no-cache python3 && \
+ python3 -m ensurepip && \
+ pip3 install --upgrade pip && \
+ pip3 install awscli && \
+ pip3 cache purge
+
+# https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html
+# Install aws-iam-authenticator
+RUN authenticator=$(aws --no-sign-request s3 ls s3://amazon-eks --recursive |grep aws-iam-authenticator$|grep amd64 |awk '{print $NF}' |sort -V|tail -1) && \
+ aws --no-sign-request s3 cp s3://amazon-eks/${authenticator} /usr/bin/aws-iam-authenticator && \
+ chmod +x /usr/bin/aws-iam-authenticator
+
+# Install jq
+RUN apk add --update --no-cache jq yq
+
+# Install for envsubst
+RUN apk add --update --no-cache gettext
+
+# Install kubeseal
+RUN curl -L https://github.com/bitnami-labs/sealed-secrets/releases/download/v${KUBESEAL_VERSION}/kubeseal-${KUBESEAL_VERSION}-linux-amd64.tar.gz -o - | tar xz -C /usr/bin/ && \
+ chmod +x /usr/bin/kubeseal
+
+WORKDIR /apps
+COPY . .
+
+RUN chmod a+x deploy.sh
+CMD ./deploy.sh
diff --git a/XTesting/richelm/README.md b/XTesting/richelm/README.md
new file mode 100644
index 0000000..1c4e86c
--- /dev/null
+++ b/XTesting/richelm/README.md
@@ -0,0 +1,38 @@
+# Kubernetes tools for RIC CI/CD
+
+kubernetes images with necessary tools that can be used as normal kubectl tool along with AWS EKS.
+
+### Installed tools
+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) (eks versions: https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html)
+- [kustomize](https://github.com/kubernetes-sigs/kustomize) (latest release: https://github.com/kubernetes-sigs/kustomize/releases/latest)
+- [helm](https://github.com/helm/helm) (latest release: https://github.com/helm/helm/releases/latest)
+- [helm-diff](https://github.com/databus23/helm-diff) (latest commit)
+- [helm-unittest](https://github.com/quintush/helm-unittest) (latest commit)
+- [helm-push](https://github.com/chartmuseum/helm-push) (latest commit)
+- [aws-iam-authenticator](https://github.com/kubernetes-sigs/aws-iam-authenticator) (latest version when run the build)
+- [eksctl](https://github.com/weaveworks/eksctl) (latest version when run the build)
+- [awscli v1](https://github.com/aws/aws-cli) (latest version when run the build)
+- [kubeseal](https://github.com/bitnami-labs/sealed-secrets) (latest version when run the build)
+- General tools, such as bash, curl
+
+# Why we need it
+
+Mostly it is used during CI/CD (continuous integration and continuous delivery) or as part of an automated build/deployment
+
+# kubectl versions
+
+You should check in [kubernetes versions](https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html), it lists the kubectl version and used as image tags.
+
+# Involve with developing and testing
+
+If you want to build these images by yourself, please follow below commands.
+
+```
+./build.sh static
+```
+
+# Usage
+
+ # mount local folder with kube config in container.
+ docker run -ti --rm -w /apps -v ~/.kube:/root/.kube -t richelmlegacy:1.19.16
diff --git a/XTesting/richelm/build.sh b/XTesting/richelm/build.sh
new file mode 100755
index 0000000..3e19e90
--- /dev/null
+++ b/XTesting/richelm/build.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# Prerequisite
+# Make sure you set secret enviroment variables in CI
+# DOCKER_USERNAME
+# DOCKER_PASSWORD
+
+# set -ex
+
+set -e
+
+build() {
+
+ # helm latest
+ helm=$(curl -s https://github.com/helm/helm/releases)
+ helm=$(echo $helm\" |grep -oP '(?<=tag\/v)[0-9][^"]*'|grep -v \-|sort -Vr|head -1)
+ echo "helm version is $helm"
+
+ # jq 1.6
+ DEBIAN_FRONTEND=noninteractive
+ #sudo apt-get update && sudo apt-get -q -y install jq
+ curl -sL https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o jq
+ sudo mv jq /usr/bin/jq
+ sudo chmod +x /usr/bin/jq
+
+ # kustomize latest
+ kustomize_release=$(curl -s https://api.github.com/repos/kubernetes-sigs/kustomize/releases | /usr/bin/jq -r '.[].tag_name | select(contains("kustomize"))' \
+ | sort -rV | head -n 1)
+ kustomize_version=$(basename ${kustomize_release})
+ echo "kustomize version is $kustomize_version"
+
+ # kubeseal latest
+ kubeseal_version=$(curl -s https://api.github.com/repos/bitnami-labs/sealed-secrets/releases | /usr/bin/jq -r '.[].tag_name | select(startswith("v"))' \
+ | sort -rV | head -n 1 |sed 's/v//')
+ echo "kubeseal version is $kubeseal_version"
+
+ docker build --no-cache \
+ --build-arg KUBECTL_VERSION=${tag} \
+ --build-arg HELM_VERSION=${helm} \
+ --build-arg KUSTOMIZE_VERSION=${kustomize_version} \
+ --build-arg KUBESEAL_VERSION=${kubeseal_version} \
+ -t ${image}:${tag} .
+
+ # run test
+ echo "Detected Helm3+"
+ version=$(docker run --rm ${image}:${tag} helm version)
+ # version.BuildInfo{Version:"v3.6.3", GitCommit:"d506314abfb5d21419df8c7e7e68012379db2354", GitTreeState:"clean", GoVersion:"go1.16.5"}
+
+ version=$(echo ${version}| awk -F \" '{print $2}')
+ if [ "${version}" == "v${helm}" ]; then
+ echo "matched"
+ else
+ echo "unmatched"
+ exit
+ fi
+
+ if [[ "$CIRCLE_BRANCH" == "master" ]]; then
+ docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+ docker push ${image}:${tag}
+ fi
+}
+
+build_static() {
+ docker build --no-cache \
+ -t ${image}:${tag} .
+
+}
+
+image="richelmlegacy"
+bimage="alpine/k8s"
+if [ $# -eq 0 ]
+ then
+ curl -s https://raw.githubusercontent.com/awsdocs/amazon-eks-user-guide/master/doc_source/kubernetes-versions.md |egrep -A 10 "The following Kubernetes versions"|awk -F \` '/^\+/ {print $2}'|sort -Vr | while read tag
+ do
+ echo ${tag}
+ status=$(curl -sL https://hub.docker.com/v2/repositories/${bimage}/tags/${tag})
+ echo $status
+ if [[ ( "${status}" =~ "not found" ) || ( ${REBUILD} == "true" ) ]]; then
+ build
+ break
+ fi
+ done
+ else
+ tag=$(awk -F '=' '/ARG\ KUBECTL_VERSION/{print $NF}' Dockerfile)
+ echo ${tag}
+ build_static
+fi
diff --git a/XTesting/richelm/deploy.sh b/XTesting/richelm/deploy.sh
new file mode 100644
index 0000000..c46bb34
--- /dev/null
+++ b/XTesting/richelm/deploy.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -x
+
+# initiate the tiller
+helm init --service-account tiller
+git clone "https://gerrit.o-ran-sc.org/r/ric-plt/ric-dep"
+
+export VERIFY_CHECKSUM=false
+cd ric-dep/bin && ./verify-ric-charts && cat ../RECIPE_EXAMPLE/example_recipe_oran_f_release.yaml | sed -e 's/10\.0\.0\.1//g' > ../RECIPE_EXAMPLE/example_recipe_oran_f_release.yaml.overwrite && cat install | sed -e 's/bash/bash -x/' | sed -e 's/helm install/helm install --debug/' > install2 && chmod +x install2 && ./install2 -f ../RECIPE_EXAMPLE/example_recipe_oran_f_release.yaml.overwrite