Merge "Updating list of docker images for ONAP"
diff --git a/build/creating_data/docker-images-collector.sh b/build/creating_data/docker-images-collector.sh
index e13b915..9206b0b 100755
--- a/build/creating_data/docker-images-collector.sh
+++ b/build/creating_data/docker-images-collector.sh
@@ -30,9 +30,9 @@
echo " "
echo " This script is preparing docker images list based on kubernetes project"
echo " Usage:"
- echo " ./$(basename $0) <project version> <path to project> [<output list file>]"
+ echo " ./$(basename $0) <path to project> [<output list file>]"
echo " "
- echo " Example: ./$(basename $0) onap_3.0.2 /root/oom/kubernetes/onap"
+ echo " Example: ./$(basename $0) /root/oom/kubernetes/onap"
echo " "
echo " Dependencies: helm, python-yaml, make"
echo " "
@@ -55,26 +55,35 @@
}
create_list() {
- helm template "${PROJECT_DIR}/../${1}" | grep 'image:\ \|tag_version:\ \|h._image' |
+ if [ -d "${PROJECT_DIR}/../${1}" ]; then
+ SUBSYS_DIR="${PROJECT_DIR}/../${1}"
+ elif [ -d "${PROJECT_DIR}/../common/${1}" ]; then
+ SUBSYS_DIR="${PROJECT_DIR}/../common/${1}"
+ else
+ >&2 echo -e \n" !!! ${1} sybsystem does not exist !!!"\n
+ fi
+ helm template "${SUBSYS_DIR}" | grep 'image:\ \|tag_version:\ \|h._image' |
sed -e 's/^.*\"h._image\"\ :\ //; s/^.*\"\(.*\)\".*$/\1/' \
-e 's/\x27\|,//g; s/^.*\(image\|tag_version\):\ //' | tr -d '\r'
}
# Configuration
-TAG="${1}"
-PROJECT_DIR="${2}"
-LIST="${3}"
+if [ "${1}" == "-h" ] || [ "${1}" == "--help" ] || [ $# -lt 1 ]; then
+ usage
+fi
+
+PROJECT_DIR="${1}"
+LIST="${2}"
LISTS_DIR="$(readlink -f $(dirname ${0}))/../data_lists"
HELM_REPO="local http://127.0.0.1:8879"
+PROJECT="$(basename ${1})"
-if [ "${1}" == "-h" ] || [ "${1}" == "--help" ] || [ $# -lt 2 ]; then
- usage
-elif [ ! -f "${PROJECT_DIR}/../Makefile" ]; then
+if [ ! -f "${PROJECT_DIR}/../Makefile" ]; then
echo "Wrong path to project directory entered"
exit 1
elif [ -z "${LIST}" ]; then
mkdir -p ${LISTS_DIR}
- LIST="${LISTS_DIR}/${TAG}-docker_images.list"
+ LIST="${LISTS_DIR}/${PROJECT}_docker_images.list"
fi
if [ -e "${LIST}" ]; then
@@ -82,8 +91,6 @@
MSG="$(realpath ${LIST}) already existed\nCreated backup $(realpath ${LIST}).bk\n"
fi
-PROJECT="$(basename ${2})"
-
# Setup helm
if pgrep -x "helm" > /dev/null; then
echo "helm is already running"
@@ -106,12 +113,17 @@
# Create the list from all enabled subsystems
echo "Creating the list..."
if [ "${PROJECT}" == "onap" ]; then
+ COMMENT="OOM commit $(git --git-dir="${PROJECT_DIR}/../../.git" rev-parse HEAD)"
for subsystem in `parse_yaml "${PROJECT_DIR}/values.yaml"`; do
create_list ${subsystem}
- done
+ done | sort -u > ${LIST}
else
- create_list ${PROJECT}
-fi | sort -u > ${LIST}
+ COMMENT="${PROJECT}"
+ create_list ${PROJECT} | sort -u > ${LIST}
+fi
+
+# Add comment reffering to the project
+sed -i "1i# generated from ${COMMENT}" "${LIST}"
echo -e ${MSG}
echo -e 'The list has been created:\n '"${LIST}"
diff --git a/build/data_lists/onap_pip_packages.list b/build/data_lists/onap_pip_packages.list
index ba5cdf7..ab4949b 100644
--- a/build/data_lists/onap_pip_packages.list
+++ b/build/data_lists/onap_pip_packages.list
@@ -7,6 +7,5 @@
Jinja2==2.10.1
MarkupSafe==1.1.1
requests==2.22.0
-setuptools==40.7.1
urllib3==1.25.3
Werkzeug==0.15.4
diff --git a/docs/InstallGuide.rst b/docs/InstallGuide.rst
index 7849047..95b5749 100644
--- a/docs/InstallGuide.rst
+++ b/docs/InstallGuide.rst
@@ -406,23 +406,23 @@
.. _oooi_installguide_postinstall:
-Part 4. Postinstallation and troubleshooting
---------------------------------------------
+Part 4. Post-installation and troubleshooting
+---------------------------------------------
-After all the playbooks are finished, it will still take a lot of time until all pods will be up and running. You can monitor your newly created kubernetes cluster for example like this::
+After all of the playbooks are run successfully, it will still take a lot of time until all pods are up and running. You can monitor your newly created kubernetes cluster for example like this::
$ ssh -i ~/.ssh/offline_ssh_key root@10.8.8.4 # tailor this command to connect to your infra-node
$ watch -d -n 5 'kubectl get pods --all-namespaces'
-Alternatively you can monitor progress with ``helm_deployment_status.py`` script located in offline-installer directory. While on infra-node this can be achieved like this::
+Alternatively you can monitor progress with ``helm_deployment_status.py`` script located in offline-installer directory. Transfer it to infra-node and run::
$ python helm_deployment_status.py -n <namespace_name> # namespace defaults to onap
-To automatically verify functionality, after deployment becomes ready or after timeout period expires, add path to healthcheck scripts::
+To automatically verify functionality with healthchecks after deployment becomes ready or after timeout period expires, append ``-hp`` switch followed by the full path to the healthcheck script and ``--health-mode`` optional switch with appropriate mode supported by that script (``health`` by default, ``--help`` displays available modes)::
- $ python helm_deployment_status.py -hp <app_data_path>/<app_name>/helm_charts/robot/ete-k8s.sh
+ $ python helm_deployment_status.py -hp <app_data_path>/<app_name>/helm_charts/robot/ete-k8s.sh --health-mode <healthcheck mode>
-It is strongly recommended to check help for ``helm_deployment_status.py`` to be able to tailor monitoring to your needs since default values might not be what you are looking for.
+It is strongly recommended to tailor ``helm_deployment_status.py`` to your needs since default values might not be what you'd expect. The defaults can be displayed with ``--help`` switch.
Final result of installation varies based on number of k8s nodes used and distribution of pods. In some dev envs we quite frequently hit problems with not all pods properly deployed. In successful deployments all jobs should be in successful state.
This can be verified using ::
diff --git a/helm_deployment_status.py b/helm_deployment_status.py
index 448263d..8f9a931 100755
--- a/helm_deployment_status.py
+++ b/helm_deployment_status.py
@@ -25,30 +25,21 @@
import argparse
import yaml
import requests
-import subprocess
+from subprocess import Popen,STDOUT,PIPE
import datetime
from time import sleep
from os.path import expanduser
from itertools import chain
import csv
from requests.packages.urllib3.exceptions import InsecureRequestWarning
-
+from base64 import b64decode
+from tempfile import NamedTemporaryFile
def add_resource_kind(resources, kind):
for item in resources:
item['kind'] = kind
return resources
-def get_resources(server, namespace, api, kind, ssl_verify=False):
- url = '/'.join([server, api, 'namespaces', namespace, kind])
- try:
- req = requests.get(url, verify=ssl_verify)
- except requests.exceptions.ConnectionError as err:
- sys.exit('Could not connect to {}'.format(server))
- json = req.json()
- # kind is <resource>List in response so [:-4] removes 'List' from value
- return add_resource_kind(json['items'], json['kind'][:-4])
-
def pods_by_parent(pods, parent):
for pod in pods:
if pod['metadata']['labels']['app'] == parent:
@@ -87,48 +78,44 @@
return resources
-def get_k8s_controllers(namespace, k8s_url):
+def get_k8s_controllers(k8s):
k8s_controllers = {}
- k8s_controllers['deployments'] = {'data': get_resources(k8s_url, namespace,
+ k8s_controllers['deployments'] = {'data': k8s.get_resources(
'apis/apps/v1', 'deployments')}
- k8s_controllers['deployments'].update(analyze_k8s_controllers(k8s_controllers['deployments']['data']))
+ k8s_controllers['deployments'].update(analyze_k8s_controllers(
+ k8s_controllers['deployments']['data']))
- k8s_controllers['statefulsets'] = {'data': get_resources(k8s_url, namespace,
+ k8s_controllers['statefulsets'] = {'data': k8s.get_resources(
'apis/apps/v1', 'statefulsets')}
- k8s_controllers['statefulsets'].update(analyze_k8s_controllers(k8s_controllers['statefulsets']['data']))
+ k8s_controllers['statefulsets'].update(analyze_k8s_controllers(
+ k8s_controllers['statefulsets']['data']))
- k8s_controllers['jobs'] = {'data': get_resources(k8s_url, namespace,
+ k8s_controllers['jobs'] = {'data': k8s.get_resources(
'apis/batch/v1', 'jobs')}
- k8s_controllers['jobs'].update(analyze_k8s_controllers(k8s_controllers['jobs']['data']))
+ k8s_controllers['jobs'].update(analyze_k8s_controllers(
+ k8s_controllers['jobs']['data']))
not_ready_controllers = chain.from_iterable(
k8s_controllers[x]['not_ready_list'] for x in k8s_controllers)
return k8s_controllers, list(not_ready_controllers)
-def get_k8s_url(kube_config):
- # TODO: Get login info
- with open(kube_config) as f:
- config = yaml.load(f)
- # TODO: Support cluster by name
- return config['clusters'][0]['cluster']['server']
+def exec_healthcheck(hp_script, namespace, hp_mode):
+ # spawn healthcheck script and redirect it's stderr to stdout
+ hc = Popen(['sh',hp_script,namespace,hp_mode],stdout=PIPE,stderr=STDOUT)
+ # Trace the output of subprocess until it has finished
+ for line in iter(hc.stdout.readline, ''):
+ print(line.strip())
+ hc.poll() # set returncode in Popen object
+ return hc.returncode
-def exec_healthcheck(hp_script, namespace):
- try:
- hc = subprocess.check_output(
- ['sh', hp_script, namespace, 'health'],
- stderr=subprocess.STDOUT)
- return 0, hc.output
- except subprocess.CalledProcessError as err:
- return err.returncode, err.output
-
-def check_readiness(k8s_url, namespace, verbosity):
- k8s_controllers, not_ready_controllers = get_k8s_controllers(namespace, k8s_url)
+def check_readiness(k8s, verbosity):
+ k8s_controllers, not_ready_controllers = get_k8s_controllers(k8s)
# check pods only when it is explicitly wanted (judging readiness by deployment status)
if verbosity > 1:
- pods = get_resources(k8s_url, namespace, 'api/v1', 'pods')
+ pods = k8s.get_resources('api/v1', 'pods')
unready_pods = chain.from_iterable(
get_names(not_ready_pods(
pods_by_parent(pods, x)))
@@ -139,11 +126,11 @@
print_status(verbosity, k8s_controllers, unready_pods)
return not not_ready_controllers
-def check_in_loop(k8s_url, namespace, max_time, sleep_time, verbosity):
+def check_in_loop(k8s, max_time, sleep_time, verbosity):
max_end_time = datetime.datetime.now() + datetime.timedelta(minutes=max_time)
ready = False
while datetime.datetime.now() < max_end_time:
- ready = check_readiness(k8s_url, namespace, verbosity)
+ ready = check_readiness(k8s, verbosity)
if ready:
return ready
sleep(sleep_time)
@@ -184,7 +171,8 @@
print('\n'.join(status_strings), '\n')
def parse_args():
- parser = argparse.ArgumentParser(description='Monitor ONAP deployment progress')
+ parser = argparse.ArgumentParser(description='Monitor ONAP deployment progress',
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--namespace', '-n', default='onap',
help='Kubernetes namespace of ONAP')
parser.add_argument('--server', '-s', help='address of Kubernetes cluster')
@@ -192,6 +180,9 @@
default=expanduser('~') + '/.kube/config',
help='path to .kube/config file')
parser.add_argument('--health-path', '-hp', help='path to ONAP robot ete-k8s.sh')
+ parser.add_argument('--health-mode', default='health', help='healthcheck mode',
+ choices=('health','healthdist','distribute','instantiate','instantiateVFWCL',
+ 'instantiateDemoVFWCL','portal'))
parser.add_argument('--no-helm', action='store_true', help='Do not check Helm')
parser.add_argument('--check-frequency', '-w', default=300, type=int,
help='time between readiness checks in seconds')
@@ -201,9 +192,86 @@
help='run check loop only once')
parser.add_argument('-v', dest='verbosity', action='count', default=0,
help='increase output verbosity, e.g. -vv is more verbose than -v')
+ parser.add_argument('--no-ssl-auth', action='store_true',
+ help='Disable SSL certificate based authentication while connecting to server')
return parser.parse_args()
+class Kubernetes:
+ '''Class exposing get_resources() routine for connecting to kube API.
+ It keeps all attributes required by that call as an internal
+ object state.'''
+
+ requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
+ def __init__(self,args):
+
+ self.config = args.kubeconfig
+ self.url = args.server if args.server is not None else \
+ self._get_k8s_url()
+ self.no_ssl_auth = args.no_ssl_auth
+ self.certs = self._get_k8s_certs() if not self.no_ssl_auth else {}
+ self.namespace = args.namespace
+
+ # Setup tmp file with ca chain only if certs were gathered successfully
+ # and --no-ssl-auth wasn't set
+ if self.certs and not self.no_ssl_auth:
+ self._setup_cert_files()
+
+ def get_resources(self, api, kind):
+ '''Performs actual API call'''
+ url = '/'.join([self.url, api, 'namespaces', self.namespace, kind])
+ try:
+ if self.no_ssl_auth:
+ req = requests.get(url, verify=False)
+ else:
+ req = requests.get(url, verify=self.crt_tmp_file.name, cert=self.crt_tmp_file.name)
+ except requests.exceptions.ConnectionError as err:
+ sys.exit('Error: Could not connect to {}'.format(self.url))
+ if req.status_code == 200:
+ json = req.json()
+ # kind is <resource>List in response so [:-4] removes 'List' from value
+ return add_resource_kind(json['items'], json['kind'][:-4])
+ elif (req.status_code == 401):
+ sys.exit('Error: Server replied with "401 Unauthorized" while making connection')
+ else:
+ sys.exit("Error: There's been an unspecified issue while making a request to the API")
+
+ def _setup_cert_files(self):
+ '''Helper funtion to setup named file for requests.get() call
+ in self.get_resources() which is able read certificate only
+ from file'''
+ ca_chain = NamedTemporaryFile()
+ for crt in self.certs.values():
+ ca_chain.write(crt)
+ ca_chain.read() # flush the file buffer
+ self.crt_tmp_file = ca_chain
+
+ def _get_k8s_url(self):
+ # TODO: Get login info
+ with open(self.config) as f:
+ config = yaml.load(f)
+ # TODO: Support cluster by name
+ return config['clusters'][0]['cluster']['server']
+
+ def _get_k8s_certs(self):
+ '''Helper function to read and decode certificates from kube config'''
+ with open(self.config) as f:
+ config = yaml.load(f)
+ certs = {}
+ try:
+ certs.update(dict(ca_cert=b64decode(
+ config['clusters'][0]['cluster']['certificate-authority-data'])))
+ certs.update(dict(client_cert=b64decode(
+ config['users'][0]['user']['client-certificate-data'])))
+ certs.update(dict(client_key=b64decode(
+ config['users'][0]['user']['client-key-data'])))
+ except KeyError as err:
+ print('Warning: could not get Kubernetes config for certificates. ' \
+ 'Turning off SSL authentication.')
+ self.no_ssl_auth = True
+ return certs
+
def main():
args = parse_args()
@@ -218,25 +286,20 @@
except IOError as err:
sys.exit(err.strerror)
- requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
- k8s_url = args.server if args.server is not None else get_k8s_url(args.kubeconfig)
+ k8s = Kubernetes(args)
ready = False
if args.single_run:
- ready = check_readiness(k8s_url, args.namespace, args.verbosity)
+ ready = check_readiness(k8s, args.verbosity)
else:
- if not check_in_loop(k8s_url, args.namespace, args.max_time, args.check_frequency, args.verbosity):
+ if not check_in_loop(k8s, args.max_time, args.check_frequency, args.verbosity):
# Double-check last 5 minutes and write verbosely in case it is not ready
- ready = check_readiness(k8s_url, args.namespace, 2)
+ ready = check_readiness(k8s, 2)
if args.health_path is not None:
- try:
- hc_rc, hc_output = exec_healthcheck(args.health_path, args.namespace)
- except IOError as err:
- sys.exit(err.strerror)
- if args.verbosity > 1 or hc_rc > 0:
- print(hc_output.decode('utf-8'))
- sys.exit(hc_rc)
+ hc_rc = exec_healthcheck(args.health_path, args.namespace, args.health_mode)
+ if hc_rc:
+ sys.exit(hc_rc)
if not ready:
sys.exit('Deployment is not ready')