From 299841ac0e55baa3f49aab1e0e55d353fe1d75ca Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Sun, 22 Mar 2020 09:04:13 +0000 Subject: [PATCH] engine: Enable deployment and testing for release jobs Change-Id: I28f455dee369eaca9f91b5210401fa24f57707c4 --- jjb/engine/engine-macros.yaml | 21 +++++-- .../engine-periodic-offline-deploy-test.yaml | 13 ++-- .../engine-periodic-release-jobs.yaml | 61 ++++++++++++++----- jjb/engine/scripts/deploy.sh | 7 ++- jjb/engine/scripts/download-artifact.sh | 23 +++++-- jjb/engine/scripts/{test.sh => functest.sh} | 11 +++- ...metadata.sh => generate-build-metadata.sh} | 4 +- jjb/engine/scripts/promote.sh | 46 ++++++++++++++ jjb/engine/scripts/upload-artifact.sh | 28 +++++---- jjb/engine/scripts/yardstick.sh | 1 + .../engine-verify-deployment-parent-jobs.yaml | 10 +-- .../engine-verify-offline-deploy-test.yaml | 12 ++-- .../engine-verify-online-deploy-test.yaml | 12 ++-- 13 files changed, 184 insertions(+), 65 deletions(-) rename jjb/engine/scripts/{test.sh => functest.sh} (67%) rename jjb/engine/scripts/{generate-job-metadata.sh => generate-build-metadata.sh} (94%) create mode 100755 jjb/engine/scripts/promote.sh create mode 120000 jjb/engine/scripts/yardstick.sh diff --git a/jjb/engine/engine-macros.yaml b/jjb/engine/engine-macros.yaml index bb45b068..aa893b5b 100644 --- a/jjb/engine/engine-macros.yaml +++ b/jjb/engine/engine-macros.yaml @@ -59,10 +59,16 @@ !include-raw: ./scripts/deploy.sh - builder: - name: 'test-macro' + name: 'functest-macro' builders: - shell: - !include-raw: ./scripts/test.sh + !include-raw: ./scripts/functest.sh + +- builder: + name: 'yardstick-macro' + builders: + - shell: + !include-raw: ./scripts/yardstick.sh - builder: name: 'delete-stack-macro' @@ -75,9 +81,16 @@ builders: - shell: !include-raw: ./scripts/wait-for-pkg-manager.sh + +- builder: + name: 'generate-build-metadata-macro' + builders: + - shell: + !include-raw: ./scripts/generate-build-metadata.sh + - builder: - name: 'generate-job-metadata-macro' + name: 'promote-macro' builders: - shell: - !include-raw: ./scripts/generate-job-metadata.sh + !include-raw: ./scripts/promote.sh diff --git a/jjb/engine/periodic-jobs/engine-periodic-offline-deploy-test.yaml b/jjb/engine/periodic-jobs/engine-periodic-offline-deploy-test.yaml index e3e68c1f..5a77c400 100644 --- a/jjb/engine/periodic-jobs/engine-periodic-offline-deploy-test.yaml +++ b/jjb/engine/periodic-jobs/engine-periodic-offline-deploy-test.yaml @@ -120,14 +120,15 @@ option: project wrappers: - - build-timeout: - timeout: 550 +# - build-timeout: +# timeout: 550 - mask-passwords - fix-workspace-permissions builders: - description-setter: description: $PROVISIONER_TYPE | $INSTALLER_TYPE | $DEPLOY_SCENARIO + - 'download-artifact-macro' - multijob: name: "Offline Deployment on {type}" condition: SUCCESSFUL @@ -170,12 +171,10 @@ option: project wrappers: - - build-timeout: - timeout: '{build-timeout}' +# - build-timeout: +# timeout: '{build-timeout}' - mask-passwords - fix-workspace-permissions builders: - - shell: | - #!/bin/bash - env + - '{phase}-macro' diff --git a/jjb/engine/periodic-jobs/engine-periodic-release-jobs.yaml b/jjb/engine/periodic-jobs/engine-periodic-release-jobs.yaml index 65eaa57c..f4c3bfef 100644 --- a/jjb/engine/periodic-jobs/engine-periodic-release-jobs.yaml +++ b/jjb/engine/periodic-jobs/engine-periodic-release-jobs.yaml @@ -40,7 +40,8 @@ jobs: - 'kubernetes-periodic-release-{environment-type}-{distro}' - - 'kubernetes-periodic-{phase}-{distro}' + - 'kubernetes-periodic-package-{distro}' + - 'kubernetes-periodic-promote-{distro}' - job-template: name: 'kubernetes-periodic-release-{environment-type}-{distro}' @@ -105,8 +106,8 @@ - logrotate wrappers: - - build-timeout: - timeout: 180 +# - build-timeout: +# timeout: 180 - mask-passwords - fix-workspace-permissions @@ -123,9 +124,9 @@ wipe-workspace: true builders: - - 'wait-pkg-mgr-macro' - 'generate-ssh-keypair-macro' - - 'generate-job-metadata-macro' + - 'generate-build-metadata-macro' + - 'wait-pkg-mgr-macro' - multijob: name: package condition: SUCCESSFUL @@ -149,6 +150,7 @@ node-parameters: false property-file: "$WORKSPACE/build.properties" predefined-parameters: | + EXECUTION_MODE=$EXECUTION_MODE NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD VERBOSITY=$VERBOSITY @@ -165,6 +167,7 @@ node-parameters: false property-file: "$WORKSPACE/build.properties" predefined-parameters: | + EXECUTION_MODE=$EXECUTION_MODE NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD VERBOSITY=$VERBOSITY @@ -184,7 +187,40 @@ abort-all-job: true - job-template: - name: 'kubernetes-periodic-{phase}-{distro}' + name: 'kubernetes-periodic-package-{distro}' + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - logrotate + + wrappers: +# - build-timeout: +# timeout: 70 + - credentials-binding: + - username-password-separated: + credential-id: 'nordix-cicd-arm-credentials' + username: NORDIX_ARM_USERNAME + password: NORDIX_ARM_TOKEN + - mask-passwords + - fix-workspace-permissions + + scm: + - git: + url: https://gerrit.nordix.org/infra/engine.git + branches: + - master + timeout: 15 + wipe-workspace: true + + builders: + - 'package-macro' + - 'upload-artifact-macro' + +- job-template: + name: 'kubernetes-periodic-promote-{distro}' disabled: '{obj:disabled}' @@ -194,8 +230,8 @@ - logrotate wrappers: - - build-timeout: - timeout: 70 +# - build-timeout: +# timeout: 70 - credentials-binding: - username-password-separated: credential-id: 'nordix-cicd-arm-credentials' @@ -213,11 +249,4 @@ wipe-workspace: true builders: - - shell: | - #!/bin/bash - env - -# - 'wait-pkg-mgr-macro' -# - 'generate-ssh-keypair-macro' -# - 'package-macro' -# - 'upload-artifact-macro' + - 'promote-macro' diff --git a/jjb/engine/scripts/deploy.sh b/jjb/engine/scripts/deploy.sh index cfaa8c07..0f4ab73d 100755 --- a/jjb/engine/scripts/deploy.sh +++ b/jjb/engine/scripts/deploy.sh @@ -33,7 +33,12 @@ else ENGINE_ROOT_DIR="$WORKSPACE" fi -/bin/cp -f $OPENRC_FILE /tmp/openrc +# NOTE (fdegir): for bifrost based deployments, we don't need these files so we fake them +OPENRC_FILE=${OPENRC_FILE:-/tmp/openrc} +HEAT_ENVIRONMENT_FILENAME=${HEAT_ENVIRONMENT_FILENAME:-/tmp/heat} + +[[ ${PROVISIONER_TYPE} == "heat" ]] && /bin/cp -f $OPENRC_FILE /tmp/openrc + cd "$ENGINE_ROOT_DIR/engine" ./deploy.sh -d "$INSTALLER_TYPE" -s "$DEPLOY_SCENARIO" -b "file://$ENGINE_ROOT_DIR/engine/inventory/group_vars/all/sdf.yaml" \ -r "$PROVISIONER_TYPE" -p "$PDF" -i "$IDF" -u /tmp/openrc \ diff --git a/jjb/engine/scripts/download-artifact.sh b/jjb/engine/scripts/download-artifact.sh index 2cd405ef..3c559bae 100755 --- a/jjb/engine/scripts/download-artifact.sh +++ b/jjb/engine/scripts/download-artifact.sh @@ -20,21 +20,32 @@ set -o nounset set -o errexit -set -o pipefail if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then + # ensure we delete leftovers due to lack of disk space + sudo /bin/rm -rf /tmp/$ARTIFACT_TO_DOWNLOAD \ + /opt/engine /opt/test /opt/stack /httpboot || true + sudo docker ps -a | grep -v CONTAINER | awk '{print $1}' | xargs -n 1 sudo docker stop || true + sudo docker ps -a | grep -v CONTAINER | awk '{print $1}' | xargs -n 1 sudo docker rm || true + sudo docker images -a | grep -v CONTAINER | awk '{print $1":"$2}' | \ + xargs -n 1 sudo docker rmi -f || true + + # offline nodes can't resolve artifactory.nordix.org echo "Info : Offline deployment. Adding Nordix ARM Public IP to /etc/hosts" sudo bash -c "cat << EOF >> /etc/hosts # managed by engine $NORDIX_ARM_PUBLIC_IP artifactory.nordix.org # managed by engine EOF" - echo "Info : Downloading artifact from Nordix ARM." + echo "Info : Downloading artifact from Nordix ARM..." + echo " $NORDIX_ARTIFACT_URL/$ARTIFACT_TO_DOWNLOAD" cd /tmp - curl -s -O $NORDIX_ARTIFACT_URL - echo "Info : Downloaded the artifact for offline deployment. Extracting." - chmod +x $ARTIFACT_TO_DOWNLOAD - $ARTIFACT_TO_DOWNLOAD + curl -s -O $NORDIX_ARTIFACT_URL/$ARTIFACT_TO_DOWNLOAD + echo "Info : Downloaded the artifact for offline deployment. Extracting..." + echo " $(ls -al /tmp/$ARTIFACT_TO_DOWNLOAD)" + chmod +x /tmp/$ARTIFACT_TO_DOWNLOAD + /tmp/$ARTIFACT_TO_DOWNLOAD + sudo /bin/rm -rf /tmp/$ARTIFACT_TO_DOWNLOAD else echo "Info : Online deployment. Skipping artifact download." fi diff --git a/jjb/engine/scripts/test.sh b/jjb/engine/scripts/functest.sh similarity index 67% rename from jjb/engine/scripts/test.sh rename to jjb/engine/scripts/functest.sh index cc793530..654c419d 100755 --- a/jjb/engine/scripts/test.sh +++ b/jjb/engine/scripts/functest.sh @@ -22,14 +22,21 @@ set -o nounset set -o errexit set -o pipefail -if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then +if [[ "$EXECUTION_MODE" == "offline-deployment" && "$SLAVE_LABEL" != "baremetal-deploy" ]]; then echo "Info : Testing of offline deployments is not currently supported!" exit 0 fi +# NOTE (fdegir): delete unnecessary stuff to free up space on baremetal jumphost +sudo /bin/rm -rf /tmp/$ARTIFACT_TO_DOWNLOAD \ + /opt/test /opt/stack /httpboot /opt/engine/offline/binaries /opt/engine/offline/containers + /opt/engine/offline/dib /opt/engine/offline/pkg +sudo docker images -a | grep -v CONTAINER | awk '{print $1":"$2}' | \ + xargs -n 1 sudo docker rmi -f || true + echo "Info : Cloning infra/test repository" cd $HOME && /bin/rm -rf $WORKSPACE git clone -q $TEST_REPO_URL $WORKSPACE cd $WORKSPACE/test -./test.sh -f $TEST_FW -t $TEST_SUITE +./test.sh -f $TEST_FW -t $TEST_SUITE -s $DEPLOY_SCENARIO -d $YARDSTICK_DISPATCHER diff --git a/jjb/engine/scripts/generate-job-metadata.sh b/jjb/engine/scripts/generate-build-metadata.sh similarity index 94% rename from jjb/engine/scripts/generate-job-metadata.sh rename to jjb/engine/scripts/generate-build-metadata.sh index df160e3d..88a89f54 100755 --- a/jjb/engine/scripts/generate-job-metadata.sh +++ b/jjb/engine/scripts/generate-build-metadata.sh @@ -26,8 +26,10 @@ cd $WORKSPACE # need to generate the identifier before writing it if [[ "$CI_LOOP" == "verify" ]]; then BUILD_IDENTIFIER=$GERRIT_CHANGE_NUMBER + ARTIFACT_ARM_FOLDER=gerrit elif [[ "$CI_LOOP" == "periodic" ]]; then BUILD_IDENTIFIER="$(date '+%Y%m%d%H%M')" + ARTIFACT_ARM_FOLDER=release else echo "Error : Unrecognized CI Loop $CI_LOOP" exit 1 @@ -40,7 +42,7 @@ BUILT_BY=$BUILD_URL BUILD_IDENTIFIER=$BUILD_IDENTIFIER CI_LOOP=$CI_LOOP ENGINE_SHA=$(git rev-parse HEAD) -NORDIX_ARTIFACT_URL=$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/release/$BUILD_IDENTIFIER +NORDIX_ARTIFACT_URL=$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/$ARTIFACT_ARM_FOLDER/$BUILD_IDENTIFIER EOF echo "Info : Build metadata" diff --git a/jjb/engine/scripts/promote.sh b/jjb/engine/scripts/promote.sh new file mode 100755 index 00000000..ed56c6ad --- /dev/null +++ b/jjb/engine/scripts/promote.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# ============LICENSE_START======================================================= +# Copyright (C) 2019 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +set -o nounset +set -o errexit +set -o pipefail + +if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then + cd /tmp + URL_TO_UPLOAD=$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/$ARTIFACT_ARM_FOLDER/latest + for ARTIFACT_TO_UPLOAD in $ARTIFACTS_TO_UPLOAD; do + echo "Info : Generating checksums for $ARTIFACT_TO_UPLOAD" + ARTIFACT_MD5SUM=$(md5sum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) + ARTIFACT_SHASUM=$(shasum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) + ARTIFACT_SHA256SUM=$(sha256sum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) + + echo "Info : Promoting $ARTIFACT_TO_UPLOAD" + curl -s \ + -H "X-JFrog-Art-Api:$NORDIX_ARM_TOKEN" \ + -H "X-Checksum-MD5:$ARTIFACT_MD5SUM" \ + -H "X-Checksum-Sha1:$ARTIFACT_SHASUM" \ + -H "X-Checksum-Sha256:$ARTIFACT_SHA256SUM" \ + -T $ARTIFACT_TO_UPLOAD \ + "$URL_TO_UPLOAD/$ARTIFACT_TO_UPLOAD" + echo "Info : Artifact $ARTIFACT_TO_UPLOAD is available on $URL_TO_UPLOAD/$ARTIFACT_TO_UPLOAD" + done +else + echo "Info : Online deployment. No artifact to upload." +fi diff --git a/jjb/engine/scripts/upload-artifact.sh b/jjb/engine/scripts/upload-artifact.sh index 37d9d80c..79ba0a9d 100755 --- a/jjb/engine/scripts/upload-artifact.sh +++ b/jjb/engine/scripts/upload-artifact.sh @@ -23,21 +23,23 @@ set -o errexit set -o pipefail if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then - echo "Info : Generating artifact checksums" cd /tmp - ARTIFACT_MD5SUM=$(md5sum k8s-installer-ubuntu1804.bsx | cut -d' ' -f1) - ARTIFACT_SHASUM=$(shasum k8s-installer-ubuntu1804.bsx | cut -d' ' -f1) - ARTIFACT_SHA256SUM=$(sha256sum k8s-installer-ubuntu1804.bsx | cut -d' ' -f1) + for ARTIFACT_TO_UPLOAD in $ARTIFACTS_TO_UPLOAD; do + echo "Info : Generating checksums for $ARTIFACT_TO_UPLOAD" + ARTIFACT_MD5SUM=$(md5sum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) + ARTIFACT_SHASUM=$(shasum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) + ARTIFACT_SHA256SUM=$(sha256sum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) - echo "Info : Uploading artifact to Nordix Artifactory" - curl -s \ - -H "X-JFrog-Art-Api:$NORDIX_ARM_TOKEN" \ - -H "X-Checksum-MD5:$ARTIFACT_MD5SUM" \ - -H "X-Checksum-Sha1:$ARTIFACT_SHASUM" \ - -H "X-Checksum-Sha256:$ARTIFACT_SHA256SUM" \ - -T k8s-installer-ubuntu1804.bsx \ - "$NORDIX_ARTIFACT_URL" - echo "Info : Artifact is available on $NORDIX_ARTIFACT_URL" + echo "Info : Uploading $ARTIFACT_TO_UPLOAD to Nordix Artifactory" + curl -s \ + -H "X-JFrog-Art-Api:$NORDIX_ARM_TOKEN" \ + -H "X-Checksum-MD5:$ARTIFACT_MD5SUM" \ + -H "X-Checksum-Sha1:$ARTIFACT_SHASUM" \ + -H "X-Checksum-Sha256:$ARTIFACT_SHA256SUM" \ + -T $ARTIFACT_TO_UPLOAD \ + "$NORDIX_ARTIFACT_URL/$ARTIFACT_TO_UPLOAD" + echo "Info : Artifact $ARTIFACT_TO_UPLOAD is available on $NORDIX_ARTIFACT_URL/$ARTIFACT_TO_UPLOAD" + done else echo "Info : Online deployment. No artifact to upload." fi diff --git a/jjb/engine/scripts/yardstick.sh b/jjb/engine/scripts/yardstick.sh new file mode 120000 index 00000000..284dd767 --- /dev/null +++ b/jjb/engine/scripts/yardstick.sh @@ -0,0 +1 @@ +functest.sh \ No newline at end of file diff --git a/jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml b/jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml index 16943527..cbcde2ac 100644 --- a/jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml +++ b/jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml @@ -86,7 +86,7 @@ description: 'Verbosity setting for the engine - true or false - default is false' - string: name: ARTIFACTS_TO_UPLOAD - default: "k8s-installer-ubuntu1804.bsx" + default: '"k8s-installer-ubuntu1804.bsx"' description: 'Artifacts to upload' - string: name: ARTIFACT_TO_DOWNLOAD @@ -173,9 +173,9 @@ readable-message: 'true' builders: - - 'wait-pkg-mgr-macro' - 'generate-ssh-keypair-macro' - - 'generate-job-metadata-macro' + - 'generate-build-metadata-macro' + - 'wait-pkg-mgr-macro' - multijob: name: package condition: SUCCESSFUL @@ -212,8 +212,6 @@ DISTRO=$DISTRO INSTALLER_TYPE=$INSTALLER_TYPE DEPLOY_SCENARIO=$DEPLOY_SCENARIO - PDF=$PDF - IDF=$IDF property-file: "$WORKSPACE/build.properties" kill-phase-on: FAILURE abort-all-job: false @@ -237,8 +235,6 @@ DISTRO=$DISTRO INSTALLER_TYPE=$INSTALLER_TYPE DEPLOY_SCENARIO=$DEPLOY_SCENARIO - PDF=$PDF - IDF=$IDF property-file: "$WORKSPACE/build.properties" kill-phase-on: FAILURE abort-all-job: false diff --git a/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml index 561a8950..f14a861f 100644 --- a/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml +++ b/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml @@ -30,7 +30,7 @@ phase: - deploy - - test + - functest type: - virtual: @@ -109,6 +109,10 @@ name: TEST_SUITE default: 'healthcheck' description: 'Test suite to run' + - string: + name: YARDSTICK_DISPATCHER + default: 'file' + description: 'Dispatcher to use for Yardstick reporting' - string: name: DELETE_STACK default: 'true' @@ -138,12 +142,12 @@ single-use: True builders: - - 'wait-pkg-mgr-macro' - shell: | #!/bin/bash sudo sed -i 's/^search/# search/' /etc/resolv.conf - - 'generate-ssh-keypair-macro' - 'download-artifact-macro' + - 'generate-ssh-keypair-macro' + - 'wait-pkg-mgr-macro' - multijob: name: "offline deployment on {type} instances" condition: SUCCESSFUL @@ -159,7 +163,7 @@ condition: SUCCESSFUL execution-type: PARALLEL projects: - - name: 'engine-verify-offline-test-{distro}-city-{type}-master' + - name: 'engine-verify-offline-functest-{distro}-city-{type}-master' current-parameters: true node-parameters: true kill-phase-on: FAILURE diff --git a/jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml index bd53c61a..c48db3fa 100644 --- a/jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml +++ b/jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml @@ -30,7 +30,7 @@ phase: - deploy - - test + - functest type: - virtual: @@ -109,6 +109,10 @@ name: TEST_SUITE default: 'healthcheck' description: 'Test suite to run' + - string: + name: YARDSTICK_DISPATCHER + default: 'file' + description: 'Dispatcher to use for Yardstick reporting' - string: name: DELETE_STACK default: 'true' @@ -146,9 +150,9 @@ clean_before: false builders: - - 'wait-pkg-mgr-macro' - - 'generate-ssh-keypair-macro' - 'download-artifact-macro' + - 'generate-ssh-keypair-macro' + - 'wait-pkg-mgr-macro' - multijob: name: "online deployment on {type} instances" condition: SUCCESSFUL @@ -164,7 +168,7 @@ condition: SUCCESSFUL execution-type: PARALLEL projects: - - name: 'engine-verify-online-test-{distro}-city-{type}-master' + - name: 'engine-verify-online-functest-{distro}-city-{type}-master' current-parameters: true node-parameters: true kill-phase-on: FAILURE -- 2.25.1