Reduce OOM k8s footprint further
Reduce OOM k8s footprint by using the Rancher VM
as the orchestration node.
Also cleaned up the rancher deployment script.
Change-Id: Ic5594eac2830d83d58d2b0a1fa5eee770d97063d
Issue-ID: INT-586
Signed-off-by: Gary Wu <gary.i.wu@huawei.com>
diff --git a/deployment/heat/onap-oom/k8s_vm_entrypoint.sh b/deployment/heat/onap-oom/k8s_vm_entrypoint.sh
index b93eab3..e93622c 100644
--- a/deployment/heat/onap-oom/k8s_vm_entrypoint.sh
+++ b/deployment/heat/onap-oom/k8s_vm_entrypoint.sh
@@ -10,14 +10,14 @@
#
export DEBIAN_FRONTEND=noninteractive
+HOST_IP=$(hostname -I)
+echo $HOST_IP `hostname` >> /etc/hosts
printenv
mkdir -p /opt/config
echo "__docker_version__" > /opt/config/docker_version.txt
echo "__rancher_ip_addr__" > /opt/config/rancher_ip_addr.txt
echo "__rancher_private_ip_addr__" > /opt/config/rancher_private_ip_addr.txt
-HOST_IP=$(hostname -I)
-echo $HOST_IP `hostname` >> /etc/hosts
mkdir -p /etc/docker
if [ ! -z "__docker_proxy__" ]; then
diff --git a/deployment/heat/onap-oom/onap-oom.yaml b/deployment/heat/onap-oom/onap-oom.yaml
index dc06512..436c67f 100644
--- a/deployment/heat/onap-oom/onap-oom.yaml
+++ b/deployment/heat/onap-oom/onap-oom.yaml
@@ -713,43 +713,6 @@
template:
get_file: k8s_vm_entrypoint.sh
- orch_1_private_port:
- type: OS::Neutron::Port
- properties:
- network: { get_resource: oam_network }
- fixed_ips: [{"subnet": { get_resource: oam_subnet }}]
- security_groups:
- - { get_resource: onap_sg }
-
- orch_1_floating_ip:
- type: OS::Neutron::FloatingIP
- properties:
- floating_network_id: { get_param: public_net_id }
- port_id: { get_resource: orch_1_private_port }
-
- orch_1_vm:
- type: OS::Nova::Server
- properties:
- name:
- list_join: ['-', [ { get_param: 'OS::stack_name' }, 'orch', '1' ] ]
- image: { get_param: ubuntu_1604_image }
- flavor: { get_param: orch_vm_flavor }
- key_name: { get_param: key_name }
- networks:
- - port: { get_resource: orch_1_private_port }
- user_data_format: RAW
- user_data:
- str_replace:
- params:
- __docker_proxy__: { get_param: docker_proxy }
- __apt_proxy__: { get_param: apt_proxy }
- __docker_version__: { get_param: docker_version }
- __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
- __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
- __host_label__: 'orchestration'
- template:
- get_file: k8s_vm_entrypoint.sh
-
outputs:
rancher_vm_ip:
description: The IP address of the rancher instance
diff --git a/deployment/heat/onap-oom/rancher_vm_entrypoint.sh b/deployment/heat/onap-oom/rancher_vm_entrypoint.sh
index 3bc811b..e8aae2e 100644
--- a/deployment/heat/onap-oom/rancher_vm_entrypoint.sh
+++ b/deployment/heat/onap-oom/rancher_vm_entrypoint.sh
@@ -10,6 +10,8 @@
#
export DEBIAN_FRONTEND=noninteractive
+HOST_IP=$(hostname -I)
+echo $HOST_IP `hostname` >> /etc/hosts
printenv
mkdir -p /opt/config
@@ -114,10 +116,15 @@
mkdir ~/.kube
# install helm __helm_version__
+mkdir -p helm
+pushd helm
wget -q http://storage.googleapis.com/kubernetes-helm/helm-v__helm_version__-linux-amd64.tar.gz
tar -zxvf helm-v__helm_version__-linux-amd64.tar.gz
-sudo mv linux-amd64/helm /usr/local/bin/helm
+sudo cp linux-amd64/helm /usr/local/bin/helm
+popd
+mkdir -p rancher
+pushd rancher
echo export RANCHER_IP=__rancher_private_ip_addr__ > api-keys-rc
source api-keys-rc
@@ -178,11 +185,18 @@
jq -r .command token.json > rancher_agent_cmd.sh
chmod +x rancher_agent_cmd.sh
cp rancher_agent_cmd.sh /dockerdata-nfs
+popd
+
cd /dockerdata-nfs
git add -A
git commit -a -m "Add rancher agent command file"
cd ~
+cp /dockerdata-nfs/rancher_agent_cmd.sh .
+sed -i "s/docker run/docker run -e CATTLE_HOST_LABELS='orchestration=true' -e CATTLE_AGENT_IP=${HOST_IP}/g" rancher_agent_cmd.sh
+source rancher_agent_cmd.sh
+
+
KUBETOKEN=$(echo -n 'Basic '$(echo -n "$CATTLE_ACCESS_KEY:$CATTLE_SECRET_KEY" | base64 -w 0) | base64 -w 0)
@@ -211,6 +225,8 @@
export KUBECONFIG=/root/.kube/config
kubectl config view
+
+
# Enable auto-completion for kubectl
echo "source <(kubectl completion bash)" >> ~/.bashrc
diff --git a/deployment/heat/onap-oom/scripts/deploy.sh b/deployment/heat/onap-oom/scripts/deploy.sh
index 459af34..c3a73c7 100755
--- a/deployment/heat/onap-oom/scripts/deploy.sh
+++ b/deployment/heat/onap-oom/scripts/deploy.sh
@@ -156,28 +156,36 @@
sleep 2m
ssh -o StrictHostKeychecking=no -i $SSH_KEY ubuntu@$RANCHER_IP "sed -u '/Cloud-init.*finished/q' <(tail -n+0 -f /var/log/cloud-init-output.log)"
-for n in $(seq 1 8); do
- echo "Wait count $n of 8"
- sleep 15m
- ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap health"'
- RESULT=$?
- if [ $RESULT -eq 0 ]; then
- break
- fi
+PREV_RESULT=0
+for n in $(seq 1 20); do
+ RESULT=$(ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'Running|Complete|NAME' | wc -l)
+ if [[ $? -eq 0 && ( $RESULT -eq 0 || $RESULT -eq $PREV_RESULT ) ]]; then
+ break
+ fi
+ sleep 15m
+ PREV_RESULT=$RESULT
done
-ROBOT_POD=$(ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -c "kubectl --namespace onap get pods"' | grep robot | sed 's/ .*//')
+
+PREV_RESULT=0
+for n in $(seq 1 20); do
+ echo "Wait for HEALTHCHECK count $n of 10"
+ ROBOT_POD=$(ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -c "kubectl --namespace onap get pods"' | grep robot | sed 's/ .*//')
+ ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap health"'
+ RESULT=$?
+ if [[ $RESULT -lt 10 && ( $RESULT -eq 0 || $RESULT -eq $PREV_RESULT ) ]]; then
+ break
+ fi
+ sleep 15m
+ PREV_RESULT=$RESULT
+done
if [ "$ROBOT_POD" == "" ]; then
- exit 1
+ exit 1
fi
LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep health | head -1" | ssh -i $SSH_KEY ubuntu@$RANCHER_IP sudo su)
-if [ "$LOG_DIR" == "" ]; then
- exit 1
-fi
-
echo "kubectl cp -n onap $ROBOT_POD:share/logs/$LOG_DIR /tmp/robot/logs/$LOG_DIR" | ssh -i $SSH_KEY ubuntu@$RANCHER_IP sudo su
-rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$RANCHER_IP:/tmp/robot/logs/$LOG_DIR/ $WORKSPACE/archives/
-
echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+mkdir -p $WORKSPACE/archives/healthcheck
+rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$RANCHER_IP:/tmp/robot/logs/$LOG_DIR/ $WORKSPACE/archives/healthcheck
exit 0
diff --git a/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh b/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
index 5700111..928142f 100755
--- a/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
+++ b/deployment/heat/onap-oom/scripts/gen-onap-oom-yaml.sh
@@ -101,7 +101,7 @@
VM_TYPE=etcd HOST_LABEL=etcd VM_NUM=$VM_NUM envsubst < $PARTS_DIR/onap-oom-2.yaml
done
-for VM_NUM in $(seq 1); do
+for VM_NUM in $(seq 0); do
VM_TYPE=orch HOST_LABEL=orchestration VM_NUM=$VM_NUM envsubst < $PARTS_DIR/onap-oom-2.yaml
done