Add job to cleanup cicd stacks and keypairs

If DELETE_STACK is set to false for the cicd jobs then stacks can hang around.
This job deletes stacks and keypairs based on a pattern that are older than a time

Change-Id: I92be06944a7c88e5c9f3dd61ed588025102947f2
Signed-off-by: afenner <andrew.fenner@est.tech>

diff --git a/jjb/cloud-infra/common/cloud-infra-cleanup-stacks.yaml b/jjb/cloud-infra/common/cloud-infra-cleanup-stacks.yaml
new file mode 100644 (file)
index 0000000..91d5e6e
--- /dev/null
+++ b/jjb/cloud-infra/common/cloud-infra-cleanup-stacks.yaml
@@ -0,0 +1,73 @@
+---
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2020 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+# A project to remove any CI stacks that are left overnight. The domain
+# reason they would be left is for troubleshooting where the
+# DELETE_STACK option was set to false
+- project:
+    name: 'cloud-infra-cleanup-stacks'
+
+    project: 'cloud-infra-cleanup-stacks'
+
+    cloud:
+      - city:
+          openrc-credentials-file: 'est-jenkins-openrcfile-city-frankfurt-cicd'
+
+    jobs:
+      - 'cloud-infra-cleanup-stack-{cloud}'
+
+- job-template:
+    name: 'cloud-infra-cleanup-stack-{cloud}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: false
+
+    parameters:
+      - label:
+          name: SLAVE_LABEL
+          default: "engine-package-ubuntu1804-fra"
+          description: "tieing to specific cloud-infra slave so it doesn't end up on another openstack slave randomly"
+      - string:
+          name: STACK_START_STR
+          default: "nordix-cicd-"
+          description: "A pattern of stack names to search for"
+      - string:
+          name: TIMETOLIVE
+          default: "180"
+          description: "Don't delete stacks created in this timeframe of minutes"
+    properties:
+      - logrotate
+
+    triggers:
+      - timed: "@midnight"
+
+    wrappers:
+      - build-timeout:
+          timeout: 180
+      - credentials-binding:
+          - file:
+              credential-id: '{openrc-credentials-file}'
+              variable: OPENRC_FILE
+      - mask-passwords
+      - openstack:
+          single-use: True
+
+    builders:
+      - 'cloud-infra-cleanup-stack-macro'

diff --git a/jjb/cloud-infra/engine-macros.yaml b/jjb/cloud-infra/engine-macros.yaml
index 27381e033370c03b98abc812148a9deb9850a89a..4218645e9fa730fb72d1874f1c3087c1a5ba39bf 100644 (file)
--- a/jjb/cloud-infra/engine-macros.yaml
+++ b/jjb/cloud-infra/engine-macros.yaml
@@ -93,6 +93,12 @@
       - shell:
           !include-raw: ./scripts/delete-heat-stack.sh
 
+- builder:
+    name: 'cloud-infra-cleanup-stack-macro'
+    builders:
+      - shell:
+          !include-raw: ./scripts/cleanup-heat-stacks.sh
+
 - builder:
     name: 'wait-pkg-mgr-macro'
     builders:
@@ -120,7 +126,7 @@
     name: 'generate-k8sdeployment-environment-macro'
     builders:
       - shell:
-          !include-raw: ./scripts/generate-k8sdeployment-environment.sh          
+          !include-raw: ./scripts/generate-k8sdeployment-environment.sh
 
 - builder:
     name: 'promote-macro'

diff --git a/jjb/cloud-infra/scripts/cleanup-heat-stacks.sh b/jjb/cloud-infra/scripts/cleanup-heat-stacks.sh
new file mode 100644 (file)
index 0000000..d9e49b9
--- /dev/null
+++ b/jjb/cloud-infra/scripts/cleanup-heat-stacks.sh
@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2020 The Nordix Foundation. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+# prepare venv for openstack cli
+/bin/rm -rf $WORKSPACE/.venv
+virtualenv --python python3 --never-download $WORKSPACE/.venv
+set +u
+source $WORKSPACE/.venv
+set -u
+pip install --quiet --upgrade pip
+pip install --quiet python-cinderclient==6.0.0
+pip install --quiet python-glanceclient==3.0.0
+pip install --quiet python-heatclient==2.0.0
+pip install --quiet python-ironic-inspector-client==4.0.0
+pip install --quiet python-ironicclient==4.0.0
+pip install --quiet python-keystoneclient==3.22.0
+pip install --quiet python-neutronclient==7.1.0
+pip install --quiet python-novaclient==16.0.0
+pip install --quiet python-openstackclient==5.1.0
+pip install --quiet python-swiftclient==3.9.0
+
+echo "Info: Find Stacks $STACK_START_STR and keypair keypair-$STACK_START_STR"
+echo "-------------------------------------------------------------------------"
+source $OPENRC_FILE
+STACKS=$(openstack stack list -c "Stack Name" -c "Creation Time" -c ID -f value)
+KEYPAIRS=$(openstack keypair list -c "Name" -f value)
+# iternate of all the stacks
+while IFS= read -r stackline
+do
+  # split line which comes in order ID,"Stack Name","Creation Time"
+  stackdetails=($stackline)
+  # find stacks starting with the search string
+  if [[ ${stackdetails[1]} =~ ^$STACK_START_STR.* ]]
+  then
+    whenstackcreated=$(date -d ${stackdetails[2]} +%s)
+    nowinsecond=$(date +%s)
+    ageofstack=$(($nowinsecond-$whenstackcreated))
+    # find stacks older than the allowed time
+    if [[ $ageofstack > $TIMETOLIVE*60 ]]
+    then
+      echo "Info: Going to delete stack " ${stackdetails[1]} " ID " ${stackdetails[0]}
+      openstack stack delete ${stackdetails[0]} --wait --yes
+      echo "Info: Deleted stack " ${stackdetails[1]} " ID " ${stackdetails[0]}
+      # check keypair is there before trying to delete
+      if [[ "${KEYPAIRS[@]}" =~ "keypair-${stackdetails[1]}" ]]; then
+         echo "Info: Going to delete keypair " keypair-${stackdetails[1]}
+         openstack keypair delete keypair-${stackdetails[1]}
+         echo "Info: Deleted keypair " keypair-${stackdetails[1]}
+      fi
+    fi
+  fi
+done <<< $(echo "$STACKS")
+
+echo "-------------------------------------------------------------------------"
+echo "Info: Finished deleting stacks and keypairs"
+
+deactivate
+# vim: set ts=2 sw=2 expandtab: