Add code to enhance k8s bootstrap
- Make bootstrap container continue to run after initial bootstrap done
- Do deployments in parallel where feasible
- Accommodate deployments with no input files
- Test to see if actions (uploads, installs, etc.) are needed before doing them
Issue-ID: DCAEGEN2-594
Change-Id: Ie188c1fd69695479593aa82b516e5504a849099c
Signed-off-by: Jack Lucas <jflucas@research.att.com>
diff --git a/k8s-bootstrap-container/bootstrap.sh b/k8s-bootstrap-container/bootstrap.sh
index 6d39404..fb092ce 100755
--- a/k8s-bootstrap-container/bootstrap.sh
+++ b/k8s-bootstrap-container/bootstrap.sh
@@ -26,8 +26,99 @@
# Blueprints for components to be installed in /blueprints
# Input files for components to be installed in /inputs
# Configuration JSON files that need to be loaded into Consul in /dcae-configs
+# Consul is installed in /opt/consul/bin/consul, with base config in /opt/consul/config/00consul.json
-set -ex
+### FUNCTION DEFINITIONS ###
+
+# keep_running: Keep running after bootstrap finishes or after error
+keep_running() {
+ echo $1
+ sleep infinity &
+ wait
+}
+
+# cm_hasany: Query Cloudify Manager and return 0 (true) if there are any entities matching the query
+# Used to see if something is already present on CM
+# $1 -- query fragment, for instance "plugins?archive_name=xyz.wgn" to get
+# the number of plugins that came from the archive file "xyz.wgn"
+function cm_hasany {
+ # We use _include=id to limit the amount of data the CM sends back
+ # We rely on the "metadata.pagination.total" field in the response
+ # for the total number of matching entities
+ COUNT=$(curl -Ss -H "Tenant: default_tenant" --user admin:${CMPASS} "${CMADDR}/api/v3.1/$1&_include=id" \
+ | /bin/jq .metadata.pagination.total)
+ if (( $COUNT > 0 ))
+ then
+ return 0
+ else
+ return 1
+ fi
+}
+
+# deploy: Deploy components if they're not already deployed
+# $1 -- name (for bp and deployment)
+# $2 -- blueprint file name
+# $3 -- inputs file name (optional)
+function deploy {
+ # Don't crash the script on error
+ set +e
+
+ # Upload blueprint if it's not already there
+ if cm_hasany "blueprints?id=$1"
+ then
+ echo blueprint $1 is already installed on ${CMADDR}
+ else
+ cfy blueprints upload -b $1 /blueprints/$2
+ fi
+
+ # Create deployment if it doesn't already exist
+ if cm_hasany "deployments?id=$1"
+ then
+ echo deployment $1 has already been created on ${CMADDR}
+ else
+ INPUTS=
+ if [ -n "$3" ]
+ then
+ INPUTS="-i/inputs/$3"
+ fi
+ cfy deployments create -b $1 ${INPUTS} $1
+ fi
+
+ # Run the install workflow if it hasn't been run already
+ # We don't have a completely certain way of determining this.
+ # We check to see if the deployment has any node instances
+ # that are in the 'uninitialized' or 'deleted' states. (Note that
+ # the & in the query acts as a logical OR for the multiple state values.)
+ # We'll try to install when a deployment has node instances in those states
+ if cm_hasany "node-instances?deployment_id=$1&state=uninitialized&state=deleted"
+ then
+ cfy executions start -d $1 install
+ else
+ echo deployment $1 appears to have had an install workflow executed already or is not ready for an install
+ fi
+}
+
+# Install plugin if it's not already installed
+# $1 -- path to wagon file for plugin
+function install_plugin {
+ ARCHIVE=$(basename $1)
+ # See if it's already installed
+ if cm_hasany "plugins?archive_name=$ARCHIVE"
+ then
+ echo plugin $1 already installed on ${CMADDR}
+ else
+ cfy plugin upload $1
+ fi
+}
+
+### END FUNCTION DEFINTIONS ###
+
+set -x
+
+# Make sure we keep the container alive after an error
+trap keep_running ERR
+
+set -e
# Consul service registration data
CBS_REG='{"ID": "dcae-cbs0", "Name": "config_binding_service", "Address": "config-binding-service", "Port": 10000}'
@@ -48,20 +139,35 @@
fi
PH_REG="${PH_REG}\"}"
-# Deploy components
-# $1 -- name (for bp and deployment)
-# $2 -- blueprint name
-# $3 -- inputs file name
-function deploy {
- cfy install -b $1 -d $1 -i /inputs/$3 /blueprints/$2
-}
+
+
# Set up profile to access Cloudify Manager
cfy profiles use -u admin -t default_tenant -p "${CMPASS}" "${CMADDR}"
# Output status, for debugging purposes
cfy status
-# Load configurations into Consul
+# Check Consul readiness
+# The readiness container waits for a "consul-server" container to be ready,
+# but this isn't always enough. We need the Consul API to be up and for
+# the cluster to be formed, otherwise our Consul accesses might fail.
+# (Note in ONAP R2, we never saw a problem, but occasionally in R3 we
+# have seen Consul not be fully ready, so we add these checks, originally
+# used in the R1 HEAT-based deployment.)
+# Wait for Consul API to come up
+until curl http://${CONSUL}/v1/agent/services
+do
+ echo Waiting for Consul API
+ sleep 60
+done
+# Wait for a leader to be elected
+until [[ "$(curl -Ss http://{$CONSUL}/v1/status/leader)" != '""' ]]
+do
+ echo Waiting for leader
+ sleep 30
+done
+
+# Load configurations into Consul KV store
for config in /dcae-configs/*.json
do
# The basename of the file is the Consul key
@@ -71,47 +177,57 @@
curl -v -X PUT -H "Content-Type: application/json" --data-binary @/tmp/dcae-upload ${CONSUL}/v1/kv/${key}
done
-# For backward compatibility, load some platform services into Consul service registry
-# Some components still rely on looking up a service in Consul
-curl -v -X PUT -H "Content-Type: application/json" --data "${CBS_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${CBS_REG1}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${CM_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${INV_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${PH_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${HE_REG}" ${CONSUL}/v1/agent/service/register
-curl -v -X PUT -H "Content-Type: application/json" --data "${HR_REG}" ${CONSUL}/v1/agent/service/register
+# Put service registrations into the local Consul configuration directory
+for sr in CBS_REG CBS_REG1 INV_REG HE_REG HR_REG CM_REG PH_REG
+do
+ echo '{"service" : ' ${!sr} ' }'> /opt/consul/config/${sr}.json
+done
+
+# Start the local consul agent instance
+/opt/consul/bin/consul agent --config-dir /opt/consul/config 2>&1 | tee /opt/consul/consul.log &
# Store the CM password into a Cloudify secret
cfy secret create -s ${CMPASS} cmpass
# Load plugins onto CM
-# Allow "already loaded" error
-# (If there are other problems, will
-# be caught in deployments.)
-set +e
for wagon in /wagons/*.wgn
do
- cfy plugins upload ${wagon}
+ install_plugin ${wagon}
done
-set -e
set +e
-# (don't let failure of one stop the script. this is likely due to image pull taking too long)
+# (Don't let failure of one stop the script. This is likely due to image pull taking too long.)
+
# Deploy platform components
-deploy config_binding_service k8s-config_binding_service.yaml k8s-config_binding_service-inputs.yaml
-deploy inventory k8s-inventory.yaml k8s-inventory-inputs.yaml
-deploy deployment_handler k8s-deployment_handler.yaml k8s-deployment_handler-inputs.yaml
-deploy policy_handler k8s-policy_handler.yaml k8s-policy_handler-inputs.yaml
-deploy pgaas_initdb k8s-pgaas-initdb.yaml k8s-pgaas-initdb-inputs.yaml
+# Allow for some parallelism to speed up the process. Probably could be somewhat more aggressive.
+# config_binding_service and pgaas_initdb needed by others, but can execute in parallel
+deploy config_binding_service k8s-config_binding_service.yaml k8s-config_binding_service-inputs.yaml &
+CBS_PID=$!
+deploy pgaas_initdb k8s-pgaas-initdb.yaml k8s-pgaas-initdb-inputs.yaml &
+PG_PID=$!
+wait ${CBS_PID} ${PG_PID}
+# inventory, deployment_handler, and policy_handler can be deployed simultaneously
+deploy inventory k8s-inventory.yaml k8s-inventory-inputs.yaml &
+INV_PID=$!
+deploy deployment_handler k8s-deployment_handler.yaml k8s-deployment_handler-inputs.yaml &
+DH_PID=$!
+deploy policy_handler k8s-policy_handler.yaml k8s-policy_handler-inputs.yaml&
+PH_PID=$!
+wait ${INV_PID} ${DH_PID} ${PH_PID}
# Deploy service components
-deploy tca k8s-tca.yaml k8s-tca-inputs.yaml
-deploy ves k8s-ves.yaml k8s-ves-inputs.yaml
-deploy prh k8s-prh.yaml k8s-prh-inputs.yaml
-# holmes_rules must be deployed before holmes_engine
+# tca, ves, prh can be deployed simultaneously
+deploy tca k8s-tca.yaml k8s-tca-inputs.yaml &
+deploy ves k8s-ves.yaml k8s-ves-inputs.yaml &
+deploy prh k8s-prh.yaml &
+# holmes_rules must be deployed before holmes_engine, but holmes_rules can go in parallel with other service components
deploy holmes_rules k8s-holmes-rules.yaml k8s-holmes_rules-inputs.yaml
deploy holmes_engine k8s-holmes-engine.yaml k8s-holmes_engine-inputs.yaml
set -e
# Display deployments, for debugging purposes
cfy deployments list
+
+# Continue running
+keep_running "Finished bootstrap steps."
+echo "Exiting!"
\ No newline at end of file