From: afenner Date: Tue, 11 Aug 2020 16:02:41 +0000 (+0100) Subject: Add job to cleanup cicd stacks and keypairs X-Git-Url: https://gerrit.nordix.org/gitweb?a=commitdiff_plain;h=b252d14afc71426da984abc144a26b630b0a2296;p=infra%2Fcicd.git Add job to cleanup cicd stacks and keypairs If DELETE_STACK is set to false for the cicd jobs then stacks can hang around. This job deletes stacks and keypairs based on a pattern that are older than a time Change-Id: I92be06944a7c88e5c9f3dd61ed588025102947f2 Signed-off-by: afenner --- diff --git a/jjb/cloud-infra/common/cloud-infra-cleanup-stacks.yaml b/jjb/cloud-infra/common/cloud-infra-cleanup-stacks.yaml new file mode 100644 index 000000000..91d5e6ec8 --- /dev/null +++ b/jjb/cloud-infra/common/cloud-infra-cleanup-stacks.yaml @@ -0,0 +1,73 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2020 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +# A project to remove any CI stacks that are left overnight. The domain +# reason they would be left is for troubleshooting where the +# DELETE_STACK option was set to false +- project: + name: 'cloud-infra-cleanup-stacks' + + project: 'cloud-infra-cleanup-stacks' + + cloud: + - city: + openrc-credentials-file: 'est-jenkins-openrcfile-city-frankfurt-cicd' + + jobs: + - 'cloud-infra-cleanup-stack-{cloud}' + +- job-template: + name: 'cloud-infra-cleanup-stack-{cloud}' + + disabled: '{obj:disabled}' + + concurrent: false + + parameters: + - label: + name: SLAVE_LABEL + default: "engine-package-ubuntu1804-fra" + description: "tieing to specific cloud-infra slave so it doesn't end up on another openstack slave randomly" + - string: + name: STACK_START_STR + default: "nordix-cicd-" + description: "A pattern of stack names to search for" + - string: + name: TIMETOLIVE + default: "180" + description: "Don't delete stacks created in this timeframe of minutes" + properties: + - logrotate + + triggers: + - timed: "@midnight" + + wrappers: + - build-timeout: + timeout: 180 + - credentials-binding: + - file: + credential-id: '{openrc-credentials-file}' + variable: OPENRC_FILE + - mask-passwords + - openstack: + single-use: True + + builders: + - 'cloud-infra-cleanup-stack-macro' diff --git a/jjb/cloud-infra/engine-macros.yaml b/jjb/cloud-infra/engine-macros.yaml index 27381e033..4218645e9 100644 --- a/jjb/cloud-infra/engine-macros.yaml +++ b/jjb/cloud-infra/engine-macros.yaml @@ -93,6 +93,12 @@ - shell: !include-raw: ./scripts/delete-heat-stack.sh +- builder: + name: 'cloud-infra-cleanup-stack-macro' + builders: + - shell: + !include-raw: ./scripts/cleanup-heat-stacks.sh + - builder: name: 'wait-pkg-mgr-macro' builders: @@ -120,7 +126,7 @@ name: 'generate-k8sdeployment-environment-macro' builders: - shell: - !include-raw: ./scripts/generate-k8sdeployment-environment.sh + !include-raw: ./scripts/generate-k8sdeployment-environment.sh - builder: name: 'promote-macro' diff --git a/jjb/cloud-infra/scripts/cleanup-heat-stacks.sh b/jjb/cloud-infra/scripts/cleanup-heat-stacks.sh new file mode 100644 index 000000000..d9e49b9c3 --- /dev/null +++ b/jjb/cloud-infra/scripts/cleanup-heat-stacks.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# ============LICENSE_START======================================================= +# Copyright (C) 2020 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +set -o nounset +set -o errexit +set -o pipefail + +# prepare venv for openstack cli +/bin/rm -rf $WORKSPACE/.venv +virtualenv --python python3 --never-download $WORKSPACE/.venv +set +u +source $WORKSPACE/.venv +set -u +pip install --quiet --upgrade pip +pip install --quiet python-cinderclient==6.0.0 +pip install --quiet python-glanceclient==3.0.0 +pip install --quiet python-heatclient==2.0.0 +pip install --quiet python-ironic-inspector-client==4.0.0 +pip install --quiet python-ironicclient==4.0.0 +pip install --quiet python-keystoneclient==3.22.0 +pip install --quiet python-neutronclient==7.1.0 +pip install --quiet python-novaclient==16.0.0 +pip install --quiet python-openstackclient==5.1.0 +pip install --quiet python-swiftclient==3.9.0 + +echo "Info: Find Stacks $STACK_START_STR and keypair keypair-$STACK_START_STR" +echo "-------------------------------------------------------------------------" +source $OPENRC_FILE +STACKS=$(openstack stack list -c "Stack Name" -c "Creation Time" -c ID -f value) +KEYPAIRS=$(openstack keypair list -c "Name" -f value) +# iternate of all the stacks +while IFS= read -r stackline +do + # split line which comes in order ID,"Stack Name","Creation Time" + stackdetails=($stackline) + # find stacks starting with the search string + if [[ ${stackdetails[1]} =~ ^$STACK_START_STR.* ]] + then + whenstackcreated=$(date -d ${stackdetails[2]} +%s) + nowinsecond=$(date +%s) + ageofstack=$(($nowinsecond-$whenstackcreated)) + # find stacks older than the allowed time + if [[ $ageofstack > $TIMETOLIVE*60 ]] + then + echo "Info: Going to delete stack " ${stackdetails[1]} " ID " ${stackdetails[0]} + openstack stack delete ${stackdetails[0]} --wait --yes + echo "Info: Deleted stack " ${stackdetails[1]} " ID " ${stackdetails[0]} + # check keypair is there before trying to delete + if [[ "${KEYPAIRS[@]}" =~ "keypair-${stackdetails[1]}" ]]; then + echo "Info: Going to delete keypair " keypair-${stackdetails[1]} + openstack keypair delete keypair-${stackdetails[1]} + echo "Info: Deleted keypair " keypair-${stackdetails[1]} + fi + fi + fi +done <<< $(echo "$STACKS") + +echo "-------------------------------------------------------------------------" +echo "Info: Finished deleting stacks and keypairs" + +deactivate +# vim: set ts=2 sw=2 expandtab: