First draft of k8s deployment using RKE
Change-Id: Ifa0eb52b64438df64692aaf58b9ef8e5dd7fd32c
Issue-ID: INT-993
Signed-off-by: Gary Wu <gary.i.wu@huawei.com>
diff --git a/deployment/heat/onap-rke/scripts/Crypto.java b/deployment/heat/onap-rke/scripts/Crypto.java
new file mode 100644
index 0000000..a9bad50
--- /dev/null
+++ b/deployment/heat/onap-rke/scripts/Crypto.java
@@ -0,0 +1,82 @@
+import javax.crypto.Cipher;
+import javax.crypto.spec.GCMParameterSpec;
+import javax.crypto.spec.SecretKeySpec;
+import java.security.GeneralSecurityException;
+import java.security.SecureRandom;
+import java.util.Arrays;
+
+public class Crypto {
+
+ private static final String AES = "AES";
+ private static final int GCM_TAG_LENGTH = 16;
+ private static final int GCM_IV_LENGTH = 12;
+ private static final String AES_GCM_NO_PADDING = "AES/GCM/NoPadding";
+
+ public static void main(String[] args) {
+ if(args.length != 2) {
+ System.out.println("Usage: java Crypto value_to_encrypt key");
+ System.out.println("exit(1)");
+ System.exit(1);
+ }
+
+ String value = args[0];
+ String key = args[1];
+ String encrypted = encryptCloudConfigPassword(value, key);
+ System.out.println(encrypted);
+ }
+
+ /**
+ * encrypt a value and generate a keyfile
+ * if the keyfile is not found then a new one is created
+ *
+ * @throws GeneralSecurityException
+ */
+ public static String encrypt (String value, String keyString) throws GeneralSecurityException {
+ SecretKeySpec sks = getSecretKeySpec (keyString);
+ Cipher cipher = Cipher.getInstance(AES_GCM_NO_PADDING);
+ byte[] initVector = new byte[GCM_IV_LENGTH];
+ (new SecureRandom()).nextBytes(initVector);
+ GCMParameterSpec spec = new GCMParameterSpec(GCM_TAG_LENGTH * java.lang.Byte.SIZE, initVector);
+ cipher.init(Cipher.ENCRYPT_MODE, sks, spec);
+ byte[] encoded = value.getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ byte[] cipherText = new byte[initVector.length + cipher.getOutputSize(encoded.length)];
+ System.arraycopy(initVector, 0, cipherText, 0, initVector.length);
+ cipher.doFinal(encoded, 0, encoded.length, cipherText, initVector.length);
+ return byteArrayToHexString(cipherText);
+ }
+
+ public static String encryptCloudConfigPassword(String message, String key) {
+ try {
+ return Crypto.encrypt(message, key);
+ } catch (GeneralSecurityException e) {
+ return null;
+ }
+ }
+
+ private static SecretKeySpec getSecretKeySpec (String keyString) {
+ byte[] key = hexStringToByteArray (keyString);
+ return new SecretKeySpec (key, AES);
+ }
+
+ public static String byteArrayToHexString (byte[] b) {
+ StringBuilder sb = new StringBuilder(b.length * 2);
+ for (byte aB : b) {
+ int v = aB & 0xff;
+ if (v < 16) {
+ sb.append('0');
+ }
+ sb.append(Integer.toHexString(v));
+ }
+ return sb.toString ().toUpperCase ();
+ }
+
+ private static byte[] hexStringToByteArray (String s) {
+ byte[] b = new byte[s.length () / 2];
+ for (int i = 0; i < b.length; i++) {
+ int index = i * 2;
+ int v = Integer.parseInt (s.substring (index, index + 2), 16);
+ b[i] = (byte) v;
+ }
+ return b;
+ }
+}
\ No newline at end of file
diff --git a/deployment/heat/onap-rke/scripts/cleanup.sh b/deployment/heat/onap-rke/scripts/cleanup.sh
new file mode 100755
index 0000000..7c2a1e2
--- /dev/null
+++ b/deployment/heat/onap-rke/scripts/cleanup.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+IFS='
+'
+
+if [ -z $1 ]; then
+ echo "ONAP component name missing"
+ echo "Usage: ./cleanup.sh onap_component_name"
+ exit 1
+fi
+
+COMPONENT=$1
+
+if [ $COMPONENT == "dcae" ] || [ $COMPONENT == "DCAE" ]; then
+ kubectl delete service consul -n onap
+fi
+
+for op in secrets configmaps pvc pv services deployments statefulsets clusterrolebinding; do
+ ARRAY=(`kubectl get $op -n onap | grep dev-$COMPONENT | awk '{print $1}'`)
+ for i in ${ARRAY[*]}; do
+ kubectl delete $op -n onap $i
+ done
+done
diff --git a/deployment/heat/onap-rke/scripts/deploy.sh b/deployment/heat/onap-rke/scripts/deploy.sh
new file mode 100755
index 0000000..c4475b5
--- /dev/null
+++ b/deployment/heat/onap-rke/scripts/deploy.sh
@@ -0,0 +1,405 @@
+#!/bin/bash
+#
+# Copyright 2018 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+stack_name="oom"
+portal_hostname="portal.api.simpledemo.onap.org"
+full_deletion=false
+
+if [ -z "$WORKSPACE" ]; then
+ export WORKSPACE=`git rev-parse --show-toplevel`
+fi
+
+usage() {
+ echo "Usage: $0 [ -n <number of VMs {2-15}> ][ -s <stack name> ][ -m <manifest> ][ -d <domain> ][ -r ][ -q ] <env>" 1>&2;
+
+ echo "n: Set the number of VM's that will be installed. This number must be between 2 and 15" 1>&2;
+ echo "s: Set the name to be used for stack. This name will be used for naming of resources" 1>&2;
+ echo "d: Set the base domain name to be used in portal UI URLs" 1>&2;
+ echo "m: The docker manifest to apply; must be either \"docker-manifest-staging.csv\" or \"docker-manifest.csv\"." 1>&2;
+ echo "r: Delete all resources relating to ONAP within enviroment." 1>&2;
+ echo "q: Quiet Delete of all ONAP resources." 1>&2;
+
+ exit 1;
+}
+
+
+while getopts ":n:s:d:m:rq" o; do
+ case "${o}" in
+ n)
+ if [[ ${OPTARG} =~ ^[0-9]+$ ]];then
+ if [ ${OPTARG} -ge 2 -a ${OPTARG} -le 15 ]; then
+ vm_num=${OPTARG}
+ else
+ usage
+ fi
+ else
+ usage
+ fi
+ ;;
+ s)
+ if [[ ! ${OPTARG} =~ ^[0-9]+$ ]];then
+ stack_name=${OPTARG}
+ else
+ usage
+ fi
+ ;;
+ d)
+ if [[ ! ${OPTARG} =~ ^[0-9]+$ ]];then
+ portal_hostname=${OPTARG}
+ else
+ usage
+ fi
+ ;;
+ m)
+ if [ -f $WORKSPACE/version-manifest/src/main/resources/${OPTARG} ]; then
+ docker_manifest=${OPTARG}
+ else
+ usage
+ fi
+ ;;
+ r)
+ echo "The following command will delete all information relating to onap within your enviroment"
+ read -p "Are you certain this is what you want? (type y to confirm):" answer
+
+ if [ $answer = "y" ] || [ $answer = "Y" ] || [ $answer = "yes" ] || [ $answer = "Yes"]; then
+ echo "This may delete the work of other colleages within the same enviroment"
+ read -p "Are you certain this is what you want? (type y to confirm):" answer2
+
+ if [ $answer2 = "y" ] || [ $answer2 = "Y" ] || [ $answer2 = "yes" ] || [ $answer2 = "Yes"]; then
+ full_deletion=true
+ else
+ echo "Ending program"
+ exit 1
+ fi
+ else
+ echo "Ending program"
+ exit 1
+ fi
+ ;;
+ q)
+ full_deletion=true
+ ;;
+ *)
+ usage
+ ;;
+ esac
+done
+shift $((OPTIND-1))
+
+if [ "$#" -ne 1 ]; then
+ usage
+fi
+
+ENV_FILE=$1
+
+if [ ! -f $ENV_FILE ];then
+ echo ENV file does not exist or was not given
+ exit 1
+fi
+
+set -x
+
+SSH_KEY=~/.ssh/onap_key
+
+source $WORKSPACE/test/ete/scripts/install_openstack_cli.sh
+
+#SO_ENCRYPTION_KEY=aa3871669d893c7fb8abbcda31b88b4f
+#export OS_PASSWORD_ENCRYPTED=$(echo -n "$OS_PASSWORD" | openssl aes-128-ecb -e -K "$SO_ENCRYPTION_KEY" -nosalt | xxd -c 256 -p)
+
+#Use new encryption method
+pushd $WORKSPACE/deployment/heat/onap-rke/scripts
+javac Crypto.java
+SO_ENCRYPTION_KEY=aa3871669d893c7fb8abbcda31b88b4f
+export OS_PASSWORD_ENCRYPTED=$(java Crypto "$OS_PASSWORD" "$SO_ENCRYPTION_KEY")
+popd
+
+for n in $(seq 1 5); do
+ if [ $full_deletion = true ] ; then
+ $WORKSPACE/test/ete/scripts/teardown-onap.sh -n $stack_name -q
+ else
+ $WORKSPACE/test/ete/scripts/teardown-onap.sh -n $stack_name
+ fi
+
+ cd $WORKSPACE/deployment/heat/onap-rke
+ envsubst < $ENV_FILE > $ENV_FILE~
+ if [ -z "$vm_num" ]; then
+ cp onap-oom.yaml onap-oom.yaml~
+ else
+ ./scripts/gen-onap-oom-yaml.sh $vm_num > onap-oom.yaml~
+ fi
+
+ if ! openstack stack create -t ./onap-oom.yaml~ -e $ENV_FILE~ $stack_name --parameter docker_manifest=$docker_manifest --parameter portal_hostname=$portal_hostname; then
+ break
+ fi
+
+ while [ "CREATE_IN_PROGRESS" == "$(openstack stack show -c stack_status -f value $stack_name)" ]; do
+ sleep 20
+ done
+
+ STATUS=$(openstack stack show -c stack_status -f value $stack_name)
+ echo $STATUS
+ if [ "CREATE_COMPLETE" != "$STATUS" ]; then
+ break
+ fi
+
+ for i in $(seq 1 30); do
+ sleep 30
+ RANCHER_IP=$(openstack stack output show $stack_name rancher_vm_ip -c output_value -f value)
+ K8S_IP=$(openstack stack output show $stack_name k8s_01_vm_ip -c output_value -f value)
+ timeout 1 ping -c 1 "$RANCHER_IP" && break
+ done
+
+ timeout 1 ping -c 1 "$RANCHER_IP" && break
+
+ echo Error: OpenStack infrastructure issue: unable to reach rancher "$RANCHER_IP"
+ sleep 10
+done
+
+if ! timeout 1 ping -c 1 "$RANCHER_IP"; then
+ exit 2
+fi
+
+# wait until all k8s VMs have fully initialized
+for VM_NAME in $(grep _vm: ./onap-oom.yaml~ | cut -d: -f1); do
+ echo $VM_NAME
+ VM_IP=$(openstack stack output show $stack_name ${VM_NAME}_ip -c output_value -f value)
+ ssh-keygen -R $VM_IP
+ until ssh -o StrictHostKeychecking=no -i $SSH_KEY ubuntu@$VM_IP ls -ad /dockerdata-nfs/.git; do
+ sleep 1m
+ done
+done
+
+cat > ./cluster.yml~ <<EOF
+# If you intened to deploy Kubernetes in an air-gapped environment,
+# please consult the documentation on how to configure custom RKE images.
+nodes:
+EOF
+
+for VM_NAME in $(grep -E 'k8s_.+_vm:' ./onap-oom.yaml~ | cut -d: -f1); do
+ echo $VM_NAME
+ VM_IP=$(openstack stack output show $stack_name ${VM_NAME}_ip -c output_value -f value)
+ VM_PRIVATE_IP=$(openstack stack output show $stack_name ${VM_NAME}_private_ip -c output_value -f value)
+ VM_HOSTNAME=$stack_name-$(echo $VM_NAME | tr '_' '-' | cut -d- -f1,2)
+ cat >> ./cluster.yml~ <<EOF
+- address: $VM_IP
+ port: "22"
+ internal_address: $VM_PRIVATE_IP
+ role:
+ - worker
+ hostname_override: "$VM_HOSTNAME"
+ user: ubuntu
+ docker_socket: /var/run/docker.sock
+ ssh_key: ""
+ ssh_key_path: ~/.ssh/onap_key
+ ssh_cert: ""
+ ssh_cert_path: ""
+ labels: {}
+EOF
+done
+
+for VM_NAME in $(grep -E 'orch_.+_vm:' ./onap-oom.yaml~ | cut -d: -f1); do
+ echo $VM_NAME
+ VM_IP=$(openstack stack output show $stack_name ${VM_NAME}_ip -c output_value -f value)
+ VM_PRIVATE_IP=$(openstack stack output show $stack_name ${VM_NAME}_private_ip -c output_value -f value)
+ VM_HOSTNAME=$stack_name-$(echo $VM_NAME | tr '_' '-' | cut -d- -f1,2)
+ cat >> ./cluster.yml~ <<EOF
+- address: $VM_IP
+ port: "22"
+ internal_address: $VM_PRIVATE_IP
+ role:
+ - controlplane
+ - etcd
+ hostname_override: "$VM_HOSTNAME"
+ user: ubuntu
+ docker_socket: /var/run/docker.sock
+ ssh_key: ""
+ ssh_key_path: ~/.ssh/onap_key
+ ssh_cert: ""
+ ssh_cert_path: ""
+ labels: {}
+EOF
+done
+
+cat >> ./cluster.yml~ <<EOF
+services:
+ etcd:
+ image: ""
+ extra_args: {}
+ extra_binds: []
+ extra_env: []
+ external_urls: []
+ ca_cert: ""
+ cert: ""
+ key: ""
+ path: ""
+ snapshot: null
+ retention: ""
+ creation: ""
+ backup_config: null
+ kube-api:
+ image: ""
+ extra_args: {}
+ extra_binds: []
+ extra_env: []
+ service_cluster_ip_range: 10.43.0.0/16
+ service_node_port_range: ""
+ pod_security_policy: false
+ always_pull_images: false
+ kube-controller:
+ image: ""
+ extra_args: {}
+ extra_binds: []
+ extra_env: []
+ cluster_cidr: 10.42.0.0/16
+ service_cluster_ip_range: 10.43.0.0/16
+ scheduler:
+ image: ""
+ extra_args: {}
+ extra_binds: []
+ extra_env: []
+ kubelet:
+ image: ""
+ extra_args: {}
+ extra_binds: []
+ extra_env: []
+ cluster_domain: cluster.local
+ infra_container_image: ""
+ cluster_dns_server: 10.43.0.10
+ fail_swap_on: false
+ kubeproxy:
+ image: ""
+ extra_args: {}
+ extra_binds: []
+ extra_env: []
+network:
+ plugin: canal
+ options: {}
+authentication:
+ strategy: x509
+ sans: []
+ webhook: null
+addons: ""
+addons_include: []
+system_images:
+ etcd: rancher/coreos-etcd:v3.2.24-rancher1
+ alpine: rancher/rke-tools:v0.1.27
+ nginx_proxy: rancher/rke-tools:v0.1.27
+ cert_downloader: rancher/rke-tools:v0.1.27
+ kubernetes_services_sidecar: rancher/rke-tools:v0.1.27
+ kubedns: rancher/k8s-dns-kube-dns:1.15.0
+ dnsmasq: rancher/k8s-dns-dnsmasq-nanny:1.15.0
+ kubedns_sidecar: rancher/k8s-dns-sidecar:1.15.0
+ kubedns_autoscaler: rancher/cluster-proportional-autoscaler:1.0.0
+ coredns: coredns/coredns:1.2.6
+ coredns_autoscaler: rancher/cluster-proportional-autoscaler:1.0.0
+ kubernetes: rancher/hyperkube:v1.13.4-rancher1
+ flannel: rancher/coreos-flannel:v0.10.0-rancher1
+ flannel_cni: rancher/flannel-cni:v0.3.0-rancher1
+ calico_node: rancher/calico-node:v3.4.0
+ calico_cni: rancher/calico-cni:v3.4.0
+ calico_controllers: ""
+ calico_ctl: rancher/calico-ctl:v2.0.0
+ canal_node: rancher/calico-node:v3.4.0
+ canal_cni: rancher/calico-cni:v3.4.0
+ canal_flannel: rancher/coreos-flannel:v0.10.0
+ weave_node: weaveworks/weave-kube:2.5.0
+ weave_cni: weaveworks/weave-npc:2.5.0
+ pod_infra_container: rancher/pause:3.1
+ ingress: rancher/nginx-ingress-controller:0.21.0-rancher3
+ ingress_backend: rancher/nginx-ingress-controller-defaultbackend:1.4-rancher1
+ metrics_server: rancher/metrics-server:v0.3.1
+ssh_key_path: ~/.ssh/onap_key
+ssh_cert_path: ""
+ssh_agent_auth: false
+authorization:
+ mode: rbac
+ options: {}
+ignore_docker_version: false
+kubernetes_version: ""
+private_registries: []
+ingress:
+ provider: ""
+ options: {}
+ node_selector: {}
+ extra_args: {}
+cluster_name: "$stack_name"
+cloud_provider:
+ name: ""
+prefix_path: ""
+addon_job_timeout: 0
+bastion_host:
+ address: ""
+ port: ""
+ user: ""
+ ssh_key: ""
+ ssh_key_path: ""
+ ssh_cert: ""
+ ssh_cert_path: ""
+monitoring:
+ provider: ""
+ options: {}
+restore:
+ restore: false
+ snapshot_name: ""
+dns: null
+EOF
+
+rm -rf ./target
+mkdir -p ./target
+cp ./cluster.yml~ ./target/cluster.yml
+pushd ./target
+
+# spin up k8s with RKE
+until rke up; do
+ sleep 1m
+ rke remove
+done
+
+scp ./kube_config_cluster.yml root@$RANCHER_IP:/root/.kube/config
+popd
+
+
+sleep 2m
+ssh -o StrictHostKeychecking=no -i $SSH_KEY ubuntu@$RANCHER_IP "sed -u '/Cloud-init.*finished/q' <(tail -n+0 -f /var/log/cloud-init-output.log)"
+
+PREV_RESULT=0
+for n in $(seq 1 20); do
+ RESULT=$(ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -c "kubectl -n onap get pods"' | grep -vE 'Running|Complete|NAME' | wc -l)
+ if [[ $? -eq 0 && ( $RESULT -eq 0 || $RESULT -eq $PREV_RESULT ) ]]; then
+ break
+ fi
+ sleep 15m
+ PREV_RESULT=$RESULT
+done
+
+PREV_RESULT=0
+for n in $(seq 1 20); do
+ echo "Wait for HEALTHCHECK count $n of 10"
+ ROBOT_POD=$(ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -c "kubectl --namespace onap get pods"' | grep robot | sed 's/ .*//')
+ ssh -i $SSH_KEY ubuntu@$RANCHER_IP 'sudo su -l root -c "/root/oom/kubernetes/robot/ete-k8s.sh onap health"'
+ RESULT=$?
+ if [[ $RESULT -lt 10 && ( $RESULT -eq 0 || $RESULT -eq $PREV_RESULT ) ]]; then
+ break
+ fi
+ sleep 15m
+ PREV_RESULT=$RESULT
+done
+if [ "$ROBOT_POD" == "" ]; then
+ exit 1
+fi
+
+LOG_DIR=$(echo "kubectl exec -n onap $ROBOT_POD -- ls -1t /share/logs | grep health | head -1" | ssh -i $SSH_KEY ubuntu@$RANCHER_IP sudo su)
+echo "kubectl cp -n onap $ROBOT_POD:share/logs/$LOG_DIR /tmp/robot/logs/$LOG_DIR" | ssh -i $SSH_KEY ubuntu@$RANCHER_IP sudo su
+echo "Browse Robot results at http://$K8S_IP:30209/logs/$LOG_DIR/"
+mkdir -p $WORKSPACE/archives/healthcheck
+rsync -e "ssh -i $SSH_KEY" -avtz ubuntu@$RANCHER_IP:/tmp/robot/logs/$LOG_DIR/ $WORKSPACE/archives/healthcheck
+
+exit 0
diff --git a/deployment/heat/onap-rke/scripts/gen-onap-oom-yaml.sh b/deployment/heat/onap-rke/scripts/gen-onap-oom-yaml.sh
new file mode 100755
index 0000000..b700d89
--- /dev/null
+++ b/deployment/heat/onap-rke/scripts/gen-onap-oom-yaml.sh
@@ -0,0 +1,137 @@
+#!/bin/bash
+#
+# Copyright 2018 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+if [ "$#" -ne 1 ]; then
+ echo This script generates the HEAT template for X number of k8s VMs
+ echo "$0 <num k8s vms>"
+ exit 1
+fi
+NUM_K8S_VMS=$1
+
+if [ -z "$WORKSPACE" ]; then
+ export WORKSPACE=`git rev-parse --show-toplevel`
+fi
+PARTS_DIR=$WORKSPACE/deployment/heat/onap-rke/parts
+
+cat <<EOF
+#
+# Generated by scripts/gen-onap-oom-yaml.sh; MANUAL CHANGES WILL BE LOST
+#
+EOF
+
+cat $PARTS_DIR/onap-oom-1.yaml
+
+cat <<EOF
+ rancher_vm:
+ type: OS::Nova::Server
+ properties:
+ name:
+ list_join: ['-', [{ get_param: 'OS::stack_name' }, 'rancher']]
+ image: { get_param: ubuntu_1804_image }
+ flavor: { get_param: rancher_vm_flavor }
+ key_name: { get_param: key_name }
+ networks:
+ - port: { get_resource: rancher_private_port }
+ user_data_format: RAW
+ user_data:
+ str_replace:
+ template:
+ get_file: rancher_vm_entrypoint.sh
+ params:
+ __docker_proxy__: { get_param: docker_proxy }
+ __apt_proxy__: { get_param: apt_proxy }
+ __rancher_ip_addr__: { get_attr: [rancher_floating_ip, floating_ip_address] }
+ __rancher_private_ip_addr__: { get_attr: [rancher_floating_ip, fixed_ip_address] }
+ __integration_override_yaml__: { get_param: integration_override_yaml }
+ __integration_gerrit_branch__: { get_param: integration_gerrit_branch }
+ __integration_gerrit_refspec__: { get_param: integration_gerrit_refspec }
+ __oom_gerrit_branch__: { get_param: oom_gerrit_branch }
+ __oom_gerrit_refspec__: { get_param: oom_gerrit_refspec }
+ __docker_manifest__: { get_param: docker_manifest }
+ __docker_version__: { get_param: docker_version }
+ __rancher_version__: { get_param: rancher_version }
+ __rancher_agent_version__: { get_param: rancher_agent_version }
+ __kubectl_version__: { get_param: kubectl_version }
+ __helm_version__: { get_param: helm_version }
+ __helm_deploy_delay__: { get_param: helm_deploy_delay }
+ __use_ramdisk__: { get_param: use_ramdisk }
+ __mtu__: { get_param: mtu }
+ __portal_hostname__: { get_param: portal_hostname }
+ __public_net_id__: { get_param: public_net_id }
+ __oam_network_cidr__: { get_param: oam_network_cidr }
+ __oam_network_id__: { get_resource: oam_network }
+ __oam_subnet_id__: { get_resource: oam_subnet }
+ __sec_group__: { get_resource: onap_sg }
+ __k8s_01_vm_ip__: { get_attr: [k8s_01_floating_ip, floating_ip_address] }
+ __k8s_vm_ips__: [
+EOF
+
+for VM_NUM in $(seq -f %02g $NUM_K8S_VMS); do
+ K8S_VM_NAME=k8s_$VM_NUM
+ cat <<EOF
+ get_attr: [${K8S_VM_NAME}_floating_ip, floating_ip_address],
+EOF
+done
+
+cat <<EOF
+ ]
+ __k8s_private_ips__: [
+EOF
+
+for VM_NUM in $(seq -f %02g $NUM_K8S_VMS); do
+ K8S_VM_NAME=k8s_$VM_NUM
+ cat <<EOF
+ get_attr: [${K8S_VM_NAME}_floating_ip, fixed_ip_address],
+EOF
+done
+
+cat <<EOF
+ ]
+EOF
+
+for VM_NUM in $(seq -f %02g $NUM_K8S_VMS); do
+ VM_TYPE=k8s HOST_LABEL=compute VM_NUM=$VM_NUM envsubst < $PARTS_DIR/onap-oom-2.yaml
+done
+
+for VM_NUM in $(seq 3); do
+ VM_TYPE=orch HOST_LABEL=orchestration VM_NUM=$VM_NUM envsubst < $PARTS_DIR/onap-oom-2.yaml
+done
+
+
+cat $PARTS_DIR/onap-oom-3.yaml
+
+for VM_NUM in $(seq -f %02g $NUM_K8S_VMS); do
+ K8S_VM_NAME=k8s_$VM_NUM
+ cat <<EOF
+ ${K8S_VM_NAME}_vm_ip:
+ description: The IP address of the ${K8S_VM_NAME} instance
+ value: { get_attr: [${K8S_VM_NAME}_floating_ip, floating_ip_address] }
+
+ ${K8S_VM_NAME}_vm_private_ip:
+ description: The private IP address of the ${K8S_VM_NAME} instance
+ value: { get_attr: [${K8S_VM_NAME}_floating_ip, fixed_ip_address] }
+
+EOF
+done
+
+for VM_NUM in $(seq 3); do
+ K8S_VM_NAME=orch_$VM_NUM
+ cat <<EOF
+ ${K8S_VM_NAME}_vm_ip:
+ description: The IP address of the ${K8S_VM_NAME} instance
+ value: { get_attr: [${K8S_VM_NAME}_floating_ip, floating_ip_address] }
+
+ ${K8S_VM_NAME}_vm_private_ip:
+ description: The private IP address of the ${K8S_VM_NAME} instance
+ value: { get_attr: [${K8S_VM_NAME}_floating_ip, fixed_ip_address] }
+
+EOF
+done
diff --git a/deployment/heat/onap-rke/scripts/prepull-docker.sh b/deployment/heat/onap-rke/scripts/prepull-docker.sh
new file mode 100755
index 0000000..37385dd
--- /dev/null
+++ b/deployment/heat/onap-rke/scripts/prepull-docker.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -x
+#
+# Copyright 2018 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+if [ -z "$WORKSPACE" ]; then
+ export WORKSPACE=`git rev-parse --show-toplevel`
+fi
+
+if [ "$#" -ne 1 ]; then
+ echo "Usage: $0 <docker-proxy>"
+ exit 1
+fi
+DOCKER_PROXY=$1
+
+for MANIFEST in docker-manifest.csv docker-manifest-staging.csv; do
+ for DOCKER_IMAGE in $(tail -n +2 $WORKSPACE/version-manifest/src/main/resources/$MANIFEST | tr ',' ':'); do
+ docker pull $DOCKER_PROXY/$DOCKER_IMAGE
+ done
+done
diff --git a/deployment/heat/onap-rke/scripts/redeploy-module.sh b/deployment/heat/onap-rke/scripts/redeploy-module.sh
new file mode 100644
index 0000000..ab52831
--- /dev/null
+++ b/deployment/heat/onap-rke/scripts/redeploy-module.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Copyright 2019 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+if [ "$#" -ne 1 ]; then
+ echo "Please specify module name, i.e. $0 robot"
+ exit 1
+fi
+
+module=$1
+deploy=dev-$1
+cd /root/oom/kubernetes
+helm delete $deploy --purge
+/root/integration/deployment/heat/onap-oom/scripts/cleanup.sh $module
+rm -rf /dockerdata-nfs/$deploy
+make $module
+make onap
+helm deploy $deploy local/onap -f /root/oom/kubernetes/onap/resources/environments/public-cloud.yaml -f /root/integration-override.yaml --namespace onap
diff --git a/deployment/heat/onap-rke/scripts/redeploy.sh b/deployment/heat/onap-rke/scripts/redeploy.sh
new file mode 100755
index 0000000..1d46f02
--- /dev/null
+++ b/deployment/heat/onap-rke/scripts/redeploy.sh
@@ -0,0 +1,106 @@
+#!/bin/bash -x
+#
+# Copyright 2018 Huawei Technologies Co., Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+# This is meant to be run from within the Rancher VM to completely
+# redeploy ONAP while reusing the existing k8s stack.
+#
+# This assumes that /root/integration-override.yaml is up-to-date.
+#
+# This script can also be used after a VM reboot, and will restart
+# helm server accordingly.
+
+export DEBIAN_FRONTEND=noninteractive
+
+usage() {
+ echo "Usage: $0 <namespace>" 1>&2;
+ echo "This will completely re-deploy ONAP, and delete and re-clone oom/ and integration/ directories."
+ exit 1;
+}
+
+if [ "$#" -ne 1 ]; then
+ usage
+fi
+
+
+NS=$1
+OOM_GERRIT_BRANCH=master
+OOM_GERRIT_REFSPEC=refs/heads/master
+INTEGRATION_GERRIT_BRANCH=master
+INTEGRATION_GERRIT_REFSPEC=refs/heads/master
+DOCKER_MANIFEST=""
+
+# Verify that k8s works
+if [ $(kubectl get pods --namespace kube-system | tail -n +2 | grep -c Running) -lt 6 ]; then
+ echo "[ERROR] Kubernetes is not healthy; aborting"
+ exit 1
+fi
+
+if [ ! -f /dockerdata-nfs/rancher_agent_cmd.sh ]; then
+ cp /root/rancher_agent_cmd.sh /dockerdata-nfs
+fi
+
+
+kubectl delete namespace $NS
+for op in secrets configmaps pvc pv services deployments statefulsets clusterrolebinding; do
+ kubectl delete $op -n $NS --all
+done
+helm undeploy dev --purge
+rm -rf /dockerdata-nfs/dev-*/
+
+
+# Clone OOM:
+cd ~
+rm -rf oom/
+git clone -b $OOM_GERRIT_BRANCH https://gerrit.onap.org/r/oom
+cd oom
+git fetch https://gerrit.onap.org/r/oom $OOM_GERRIT_REFSPEC
+git checkout FETCH_HEAD
+git checkout -b workarounds
+git log -1
+
+# Clone integration
+cd ~
+rm -rf integration/
+git clone -b $INTEGRATION_GERRIT_BRANCH https://gerrit.onap.org/r/integration
+cd integration
+git fetch https://gerrit.onap.org/r/integration $INTEGRATION_GERRIT_REFSPEC
+git checkout FETCH_HEAD
+git checkout -b workarounds
+git log -1
+
+if [ ! -z "$DOCKER_MANIFEST" ]; then
+ cd version-manifest/src/main/scripts
+ ./update-oom-image-versions.sh ../resources/$DOCKER_MANIFEST ~/oom/
+fi
+
+cd ~/oom
+git diff
+git commit -a -m "apply manifest versions"
+git tag -a "deploy0" -m "initial deployment"
+
+
+# Run ONAP:
+cd ~/oom/kubernetes/
+
+if [ $(curl -s -o /dev/null -w "%{http_code}" 127.0.0.1:8879) -ne 200 ]; then
+ helm init --client-only
+ helm init --upgrade
+ helm serve &
+ sleep 10
+ helm repo add local http://127.0.0.1:8879
+ helm repo list
+fi
+make all
+rsync -avt ~/oom/kubernetes/helm/plugins ~/.helm/
+helm search -l | grep local
+helm deploy dev local/onap -f ~/oom/kubernetes/onap/resources/environments/public-cloud.yaml -f ~/integration-override.yaml --namespace onap | ts | tee -a ~/helm-deploy.log
+helm list
+