From 459c348dc363fab68f05fdb874147be83b7fa8ec Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Mon, 15 Jun 2020 21:15:43 +0000 Subject: [PATCH] Adapt engine-verify jobs for engine refactoring Change-Id: I80cea4d5b7e0e01d58dddfa5de20f16487cb60ae --- jjb/engine/engine-macros.yaml | 6 + jjb/engine/scripts/delete-heat-stack.sh | 15 +-- jjb/engine/scripts/deploy.sh | 30 +++-- jjb/engine/scripts/download-artifact.sh | 41 ++---- .../scripts/generate-change-metadata.sh | 120 ++++++++++++++++++ jjb/engine/scripts/package.sh | 9 +- jjb/engine/scripts/upload-artifact.sh | 37 ++---- .../engine-verify-deploy-test.yaml | 76 +++-------- .../engine-verify-package.yaml | 32 +---- .../engine-verify-tox.yaml | 10 +- 10 files changed, 202 insertions(+), 174 deletions(-) create mode 100755 jjb/engine/scripts/generate-change-metadata.sh rename jjb/engine/{engine-verify-jobs => verify-jobs}/engine-verify-deploy-test.yaml (80%) rename jjb/engine/{engine-verify-jobs => verify-jobs}/engine-verify-package.yaml (81%) rename jjb/engine/{engine-verify-jobs => verify-jobs}/engine-verify-tox.yaml (91%) diff --git a/jjb/engine/engine-macros.yaml b/jjb/engine/engine-macros.yaml index 6dbbb20d5..3361c740d 100644 --- a/jjb/engine/engine-macros.yaml +++ b/jjb/engine/engine-macros.yaml @@ -94,6 +94,12 @@ - shell: !include-raw: ./scripts/generate-build-metadata.sh +- builder: + name: 'generate-change-metadata-macro' + builders: + - shell: + !include-raw: ./scripts/generate-change-metadata.sh + - builder: name: 'promote-macro' builders: diff --git a/jjb/engine/scripts/delete-heat-stack.sh b/jjb/engine/scripts/delete-heat-stack.sh index 9b76d8828..2e162fae7 100755 --- a/jjb/engine/scripts/delete-heat-stack.sh +++ b/jjb/engine/scripts/delete-heat-stack.sh @@ -22,12 +22,7 @@ set -o nounset set -o errexit set -o pipefail -if [[ "$EXECUTION_MODE" == "offline-deployment" && "$PROVISIONER_TYPE" == "heat" ]]; then - echo "Info : Offline deployments on cloud instances is not currently supported!" - exit 0 -fi - -if [[ "$DELETE_STACK" != "true" || "$PROVISIONER_TYPE" != "heat" ]]; then +if [[ "$DELETE_STACK" != "true" || "$HW_TYPE" != "cloud" ]]; then echo "Info : Skipping stack delete as it is either not a cloud deployment or slack deletion is disabled!" exit 0 fi @@ -36,10 +31,12 @@ set +u source /opt/engine/.venv/bin/activate set -u -echo "Info: Deleting stack $STACK_NAME and keypair $KEYPAIR_NAME" +echo "Info: Deleting stack $HEAT_STACK_NAME and keypair $KEYPAIR_NAME" echo "-------------------------------------------------------------------------" source /tmp/openrc && /bin/rm -f /tmp/openrc -openstack stack delete $STACK_NAME --wait --yes +openstack stack delete $HEAT_STACK_NAME --wait --yes openstack keypair delete $KEYPAIR_NAME echo "-------------------------------------------------------------------------" -echo "Info: Stack $STACK_NAME and keypair $KEYPAIR_NAME have been deleted!" +echo "Info: Stack $HEAT_STACK_NAME and keypair $KEYPAIR_NAME have been deleted!" + +# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/engine/scripts/deploy.sh b/jjb/engine/scripts/deploy.sh index 56010749b..0589a8274 100755 --- a/jjb/engine/scripts/deploy.sh +++ b/jjb/engine/scripts/deploy.sh @@ -22,15 +22,13 @@ set -o nounset set -o errexit set -o pipefail -if [[ "$EXECUTION_MODE" == "offline-deployment" && "$PROVISIONER_TYPE" == "heat" ]]; then - echo "Info : Offline deployments on cloud instances is not currently supported!" - exit 0 -fi - -if [[ "$GERRIT_PROJECT" == "infra/installer/kolla" ]]; then - echo "Info : Setting installer to kolla and scenario to os-nosdn-nofeature" - export INSTALLER_TYPE="kolla" - export DEPLOY_SCENARIO="os-nosdn-nofeature" +# NOTE (fdegir): we need to remove $WORKSPACE and clone infra/engine repository +# as infra/engine is what drives the deployment +if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then + cd "$HOME" && /bin/rm -rf "$WORKSPACE" + echo "Info : Cloning infra/engine repository" + git clone -q https://gerrit.nordix.org/infra/engine.git "$WORKSPACE" + cd "$WORKSPACE" fi if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then @@ -39,13 +37,19 @@ else ENGINE_ROOT_DIR="$WORKSPACE" fi +# TODO (fdegir): this is temporarily here to develop the jobs +if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then + cd $ENGINE_ROOT_DIR + git fetch "https://gerrit.nordix.org/infra/engine" refs/changes/23/4823/18 && git checkout FETCH_HEAD +fi + # NOTE (fdegir): for bifrost based deployments, we don't need these files so we fake them OPENRC_FILE=${OPENRC_FILE:-/tmp/openrc} HEAT_ENVIRONMENT_FILENAME=${HEAT_ENVIRONMENT_FILENAME:-/tmp/heat} -[[ ${PROVISIONER_TYPE} == "heat" ]] && /bin/cp -f $OPENRC_FILE /tmp/openrc +/bin/cp -f $OPENRC_FILE /tmp/openrc cd "$ENGINE_ROOT_DIR/engine" -./deploy.sh -d "$INSTALLER_TYPE" -s "$DEPLOY_SCENARIO" -b "file://$ENGINE_ROOT_DIR/engine/inventory/group_vars/all/sdf.yaml" \ - -r "$PROVISIONER_TYPE" -p "$PDF" -i "$IDF" -u /tmp/openrc \ - -e "file://$ENGINE_ROOT_DIR/engine/provisioner/heat/playbooks/roles/bootstrap-hwconfig/files/$HEAT_ENVIRONMENT_FILENAME" +./deploy.sh -d -s "$STACK_TYPE" -v "$STACK_VERSION" -p "$PDF" -i "$IDF" -o /tmp/openrc + +# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/engine/scripts/download-artifact.sh b/jjb/engine/scripts/download-artifact.sh index 332ccfb78..2257c011e 100755 --- a/jjb/engine/scripts/download-artifact.sh +++ b/jjb/engine/scripts/download-artifact.sh @@ -21,49 +21,26 @@ set -o nounset set -o errexit -if [[ "$GERRIT_PROJECT" == "infra/installer/kolla" ]]; then - export ARTIFACT_TO_DOWNLOAD="os-installer-ubuntu1804.bsx" - export NORDIX_ARM_REPO="nordix-openstack" -else - export ARTIFACT_TO_DOWNLOAD="k8s-installer-ubuntu1804.bsx" - export NORDIX_ARM_REPO="nordix-kubernetes" -fi - -if [[ "$CI_LOOP" == "periodic" ]]; then - export ARTIFACTS_TO_UPLOAD="$ARTIFACTS_TO_UPLOAD offline-package.tgz" -else - export NORDIX_ARTIFACT_URL="$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/gerrit/$GERRIT_CHANGE_NUMBER" -fi - if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then # ensure we delete leftovers due to lack of disk space - sudo /bin/rm -rf /tmp/$ARTIFACT_TO_DOWNLOAD ~/.ara \ + sudo /bin/rm -rf /tmp/$INSTALLER_FILE ~/.ara \ /opt/engine /opt/test /opt/stack /httpboot > /dev/null 2>&1 || true sudo docker ps -a --format {{.ID}} | xargs -n 1 sudo docker stop > /dev/null 2>&1 || true sudo docker ps -a --format {{.ID}} | xargs -n 1 sudo docker rm > /dev/null 2>&1 || true sudo docker images -a --format {{.ID}} | xargs -n 1 sudo docker rmi -f > /dev/null 2>&1 || true sudo docker volume rm $(sudo docker volume ls -qf dangling=true) > /dev/null 2>&1 || true - # offline nodes can't resolve artifactory.nordix.org - echo "Info : Offline deployment. Adding Nordix ARM Public IP to /etc/hosts" - sudo bash -c "cat << EOF >> /etc/hosts -# managed by engine -$NORDIX_ARM_PUBLIC_IP artifactory.nordix.org -# managed by engine -EOF" - - # we need to get rid of search domain as well - sudo sed -i 's/^search/# search/' /etc/resolv.conf - echo "Info : Downloading artifact from Nordix ARM..." - echo " $NORDIX_ARTIFACT_URL/$ARTIFACT_TO_DOWNLOAD" + echo " $NORDIX_ARTIFACT_URL/$INSTALLER_FILE" cd /tmp - curl -s -S -O $NORDIX_ARTIFACT_URL/$ARTIFACT_TO_DOWNLOAD + curl -s -S -O $NORDIX_ARTIFACT_URL/$INSTALLER_FILE echo "Info : Downloaded the artifact for offline deployment. Extracting..." - echo " $(ls -al /tmp/$ARTIFACT_TO_DOWNLOAD)" - chmod +x /tmp/$ARTIFACT_TO_DOWNLOAD - /tmp/$ARTIFACT_TO_DOWNLOAD - sudo /bin/rm -rf /tmp/$ARTIFACT_TO_DOWNLOAD + echo " $(ls -al /tmp/$INSTALLER_FILE)" + chmod +x /tmp/$INSTALLER_FILE + /tmp/$INSTALLER_FILE + sudo /bin/rm -rf /tmp/$INSTALLER_FILE else echo "Info : Online deployment. Skipping artifact download." fi + +# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/engine/scripts/generate-change-metadata.sh b/jjb/engine/scripts/generate-change-metadata.sh new file mode 100755 index 000000000..f02b34806 --- /dev/null +++ b/jjb/engine/scripts/generate-change-metadata.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +# ============LICENSE_START======================================================= +# Copyright (C) 2019 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + + +# NOTES (fdegir): this script only works for +# - infra/engine +# - infra/provisioner/heat + +set -o nounset +set -o errexit +set -o pipefail + +# NOTE (fdegir): we need to remove $WORKSPACE and clone infra/engine repository +# as infra/engine is what drives the deployment +if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then + cd "$HOME" && /bin/rm -rf "$WORKSPACE" + echo "Info : Cloning infra/engine repository" + git clone -q https://gerrit.nordix.org/infra/engine.git "$WORKSPACE" + cd "$WORKSPACE" + # TODO (fdegir): this is temporarily here to develop the jobs + git fetch "https://gerrit.nordix.org/infra/engine" refs/changes/23/4823/18 && git checkout FETCH_HEAD +fi + +# file to record change metadata + export CHANGE_METADATA_FILE="$WORKSPACE/change.properties" +/bin/rm -f "$CHANGE_METADATA_FILE" + +echo "Info : Generating change metadata" + +# TODO (fdegir): this should match to the branch of the job for {stack}-verify jobs +export STACK_VERSION=master + +# determine the project +# this scripts is used by all engine repos so we need to determine what this change is for +export ENGINE_COMPONENT="${GERRIT_PROJECT//*\//}" + +if [[ "$GERRIT_PROJECT" =~ /stack/ ]]; then + export STACK_TYPE="$ENGINE_COMPONENT" + echo "Info : Setting STACK_TYPE, STACK_VERSION and STACK_REFSPEC" + echo " STACK_TYPE=$STACK_TYPE" + echo " STACK_VERSION=$GERRIT_PATCHSET_REVISION" + echo " STACK_REFSPEC=$GERRIT_REFSPEC" + echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE" + echo "STACK_VERSION=$GERRIT_PATCHSET_REVISION" >> "$CHANGE_METADATA_FILE" + echo "STACK_REFSPEC=$GERRIT_REFSPEC" >> "$CHANGE_METADATA_FILE" +elif [[ "$GERRIT_PROJECT" =~ /installer/ ]]; then + case "${ENGINE_COMPONENT}" in + kolla) + export STACK_TYPE="openstack" + ;; + kubespray) + export STACK_TYPE="kubernetes" + ;; + oom) + export STACK_TYPE="onap" + ;; + *) echo "ERROR : Invalid engine component $ENGINE_COMPONENT"; exit 1;; + esac + + echo "Info : Setting STACK_TYPE, STACK_VERSION and STACK_REFSPEC" + echo " STACK_TYPE=$STACK_TYPE" + echo " STACK_VERSION=$STACK_VERSION" + echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION" + echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC" + echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE" + echo "STACK_VERSION=$STACK_VERSION" >> "$CHANGE_METADATA_FILE" + echo "NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION" >> "$CHANGE_METADATA_FILE" + echo "NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC" >> "$CHANGE_METADATA_FILE" +else + export STACK_TYPE=kubernetes + export STACK_VERSION=$STACK_VERSION + echo "STACK_TYPE=kubernetes" >> "$CHANGE_METADATA_FILE" + echo "NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION" >> "$CHANGE_METADATA_FILE" + echo "NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC" >> "$CHANGE_METADATA_FILE" +fi + +# global vars +export NORDIX_ARM_HTTPS_URL="https://artifactory.nordix.org/artifactory" +export ARTIFACT_ARM_FOLDER="gerrit" +export BUILD_IDENTIFIER=$GERRIT_CHANGE_NUMBER +export INSTALLER_FILE="${STACK_TYPE}-ubuntu1804.bsx" +export BUILD_ARTIFACTS="${STACK_TYPE}-ubuntu1804.bsx" +export NORDIX_ARM_REPO="nordix-${STACK_TYPE}" +export NORDIX_ARTIFACT_URL="$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/$ARTIFACT_ARM_FOLDER/$BUILD_IDENTIFIER" + +cat << EOF >> "$CHANGE_METADATA_FILE" +BUILD_IDENTIFIER=$BUILD_IDENTIFIER +BUILD_DATE=$(date '+%Y%m%d%H%M') +BUILT_BY=$BUILD_URL +CI_LOOP=$CI_LOOP +ENGINE_SHA=$(git rev-parse HEAD) +INSTALLER_FILE=${STACK_TYPE}-ubuntu1804.bsx +BUILD_ARTIFACTS=${STACK_TYPE}-ubuntu1804.bsx +NORDIX_ARM_REPO=nordix-${STACK_TYPE} +NORDIX_ARTIFACT_URL=$NORDIX_ARTIFACT_URL +EOF + +echo "Info : Change metadata" +echo "-------------------------------------------------------------------------" +cat "$CHANGE_METADATA_FILE" +echo "-------------------------------------------------------------------------" + +# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/engine/scripts/package.sh b/jjb/engine/scripts/package.sh index 565826ca3..11700a957 100755 --- a/jjb/engine/scripts/package.sh +++ b/jjb/engine/scripts/package.sh @@ -22,15 +22,10 @@ set -o nounset set -o errexit set -o pipefail -if [[ "$GERRIT_PROJECT" == "infra/installer/kolla" ]]; then - export STACK_TO_PACKAGE="kolla" -else - export STACK_TO_PACKAGE="kubespray" -fi - +source "$WORKSPACE/change.properties" echo "Info : Packaging stack $STACK_TO_PACKAGE" cd "$WORKSPACE/engine" -./package.sh -s "$STACK_TO_PACKAGE" -v +./package.sh -s "$STACK_TO_PACKAGE" -d # vim: set ts=2 sw=2 expandtab: diff --git a/jjb/engine/scripts/upload-artifact.sh b/jjb/engine/scripts/upload-artifact.sh index f1b58be8b..fe2374217 100755 --- a/jjb/engine/scripts/upload-artifact.sh +++ b/jjb/engine/scripts/upload-artifact.sh @@ -22,41 +22,28 @@ set -o nounset set -o errexit set -o pipefail -if [[ "$GERRIT_PROJECT" == "infra/installer/kolla" ]]; then - export ARTIFACTS_TO_UPLOAD="os-installer-ubuntu1804.bsx" - export NORDIX_ARM_REPO="nordix-openstack" -else - export ARTIFACTS_TO_UPLOAD="k8s-installer-ubuntu1804.bsx" - export NORDIX_ARM_REPO="nordix-kubernetes" -fi - - -if [[ "$CI_LOOP" == "periodic" ]]; then - export ARTIFACTS_TO_UPLOAD="$ARTIFACTS_TO_UPLOAD offline-package.tgz" -else - export NORDIX_ARTIFACT_URL="$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/gerrit/$GERRIT_CHANGE_NUMBER" -fi - -echo "Info : Uploading $ARTIFACTS_TO_UPLOAD to Nordix ARM $NORDIX_ARM_REPO repository" +echo "Info : Uploading $BUILD_ARTIFACTS to Nordix ARM $NORDIX_ARM_REPO repository" if [[ "$EXECUTION_MODE" == "packaging" || "$EXECUTION_MODE" == "offline-deployment" ]]; then cd /tmp - for ARTIFACT_TO_UPLOAD in $ARTIFACTS_TO_UPLOAD; do - echo "Info : Generating checksums for $ARTIFACT_TO_UPLOAD" - ARTIFACT_MD5SUM=$(md5sum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) - ARTIFACT_SHASUM=$(shasum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) - ARTIFACT_SHA256SUM=$(sha256sum $ARTIFACT_TO_UPLOAD | cut -d' ' -f1) + for BUILD_ARTIFACT in $BUILD_ARTIFACTS; do + echo "Info : Generating checksums for $BUILD_ARTIFACT" + ARTIFACT_MD5SUM=$(md5sum $BUILD_ARTIFACT | cut -d' ' -f1) + ARTIFACT_SHASUM=$(shasum $BUILD_ARTIFACT | cut -d' ' -f1) + ARTIFACT_SHA256SUM=$(sha256sum $BUILD_ARTIFACT | cut -d' ' -f1) - echo "Info : Uploading $ARTIFACT_TO_UPLOAD to Nordix Artifactory" + echo "Info : Uploading $BUILD_ARTIFACT to Nordix Artifactory" curl -s -S \ -H "X-JFrog-Art-Api:$NORDIX_ARM_TOKEN" \ -H "X-Checksum-MD5:$ARTIFACT_MD5SUM" \ -H "X-Checksum-Sha1:$ARTIFACT_SHASUM" \ -H "X-Checksum-Sha256:$ARTIFACT_SHA256SUM" \ - -T $ARTIFACT_TO_UPLOAD \ - "$NORDIX_ARTIFACT_URL/$ARTIFACT_TO_UPLOAD" - echo "Info : Artifact $ARTIFACT_TO_UPLOAD is available on $NORDIX_ARTIFACT_URL/$ARTIFACT_TO_UPLOAD" + -T $BUILD_ARTIFACT \ + "$NORDIX_ARTIFACT_URL/$BUILD_ARTIFACT" + echo "Info : Artifact $BUILD_ARTIFACT is available on $NORDIX_ARTIFACT_URL/$BUILD_ARTIFACT" done else echo "Info : Online deployment. No artifact to upload." fi + +# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/engine/engine-verify-jobs/engine-verify-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-deploy-test.yaml similarity index 80% rename from jjb/engine/engine-verify-jobs/engine-verify-deploy-test.yaml rename to jjb/engine/verify-jobs/engine-verify-deploy-test.yaml index b96fec875..faf27d461 100644 --- a/jjb/engine/engine-verify-jobs/engine-verify-deploy-test.yaml +++ b/jjb/engine/verify-jobs/engine-verify-deploy-test.yaml @@ -54,11 +54,11 @@ - virtual: provisioner-type: 'bifrost' slave-type: 'vpod' - gated-projects: 'infra/engine|infra/provisioner/bifrost|infra/installer/kolla|infra/installer/kubespray' + gated-projects: 'infra/engine|infra/provisioner/bifrost' - cloud: provisioner-type: 'heat' slave-type: 'jumphost' - gated-projects: 'infra/engine|infra/provisioner/heat|infra/installer/kubespray' + gated-projects: 'infra/engine|infra/provisioner/heat' distro: - ubuntu1804 @@ -89,25 +89,13 @@ default: 'engine-{slave-type}-{environment-type}-{distro}-fra' description: 'Slave label to schedule this job on.' - string: - name: ENGINE_REPO_URL - default: 'https://gerrit.nordix.org/infra/engine.git' - description: 'URL to Nordix Cloud Infra engine repository' + name: HW_TYPE + default: '{type}' + description: 'Hardware type - could be virtual, cloud, or baremetal' - string: name: EXECUTION_MODE default: '{execution-mode}' description: 'Execution mode to run in' - - string: - name: PROVISIONER_TYPE - default: '{provisioner-type}' - description: 'Provisioner type to use' - - string: - name: INSTALLER_TYPE - default: 'kubespray' - description: 'Installer type to use' - - string: - name: DEPLOY_SCENARIO - default: 'k8-calico-nofeature' - description: 'Scenario to deploy' - string: name: PDF default: '{pdf}' @@ -121,7 +109,7 @@ default: 'heat-environment-{cloud}-{distro}.yaml' description: 'Name of heat-environment file to use.' - string: - name: STACK_NAME + name: HEAT_STACK_NAME default: "nordix-cicd-verify-engine-online-{distro}-$GERRIT_CHANGE_NUMBER" description: 'Name of the created stack' - string: @@ -156,30 +144,6 @@ name: VERBOSITY default: 'true' description: 'Verbosity setting for the engine - true or false - default is false' - - string: - name: NORDIX_GERRIT_PUBLIC_IP - default: '91.106.198.142' - description: 'Public IP of Nordix Gerrit. Offline deployments can not resolve domain name' - - string: - name: NORDIX_ARM_PUBLIC_IP - default: '91.106.198.25' - description: 'Public IP of Nordix Artifactory. Offline deployments can not resolve domain name' - - string: - name: NORDIX_ARM_HTTPS_URL - default: 'https://artifactory.nordix.org/artifactory' - description: 'URL to Nordix Artifactory' - - string: - name: NORDIX_ARM_REPO - default: 'nordix-kubernetes' - description: 'Name of the repository to upload and download artifact to/from. Depends on the stack' - - string: - name: ARTIFACT_TO_DOWNLOAD - default: "k8s-installer-ubuntu1804.bsx" - description: 'Artifact to download' - - string: - name: NORDIX_ARTIFACT_URL - default: "$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/gerrit/$GERRIT_CHANGE_NUMBER" - description: 'Artifact to download' - string: name: CI_LOOP default: "verify" @@ -195,8 +159,8 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - 'engine-verify-online-deploy-test-.*' - - 'engine-verify-offline-deploy-test-.*' + - '.*?-verify-online-deploy-test-.*' + - '.*?-verify-offline-deploy-test-.*' block-level: 'NODE' - throttle: max-per-node: 1 @@ -206,7 +170,7 @@ scm: - git-scm-gerrit: ssh-credentials-id: nordixinfra-nordix-gerrit-ssh - branch: refs/heads/$GERRIT_BRANCH + branch: $GERRIT_BRANCH refspec: $GERRIT_REFSPEC wipe_workspace: true clean_before: false @@ -271,18 +235,23 @@ - shell: | #!/bin/bash sudo chown -R $USER:$USER $WORKSPACE || exit 1 - # offline nodes can not resolve Nordix Gerrit + # offline nodes can not resolve Nordix Artifactory and Gerrit if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then - echo "Info : Offline deployment. Adding Nordix Gerrit Public IP to /etc/hosts" + echo "Info : Offline deployment. Adding City OpenStack Endpoint, Nordix Artifactory and Gerrit Public IP to /etc/hosts" sudo sed -i 's/^search/# search/' /etc/resolv.conf sudo bash -c "cat << EOF >> /etc/hosts # managed by engine - $NORDIX_GERRIT_PUBLIC_IP gerrit.nordix.org + 91.106.198.25 artifactory.nordix.org + 91.106.198.142 gerrit.nordix.org + 86.107.242.50 fra1.citycloud.com # managed by engine EOF" + # we need to get rid of search domain as well + sudo sed -i 's/^search/# search/' /etc/resolv.conf fi builders: + - 'generate-change-metadata-macro' - multijob: name: "{environment-type} deployment on {type} instances" condition: SUCCESSFUL @@ -294,6 +263,7 @@ node-parameters: true kill-phase-on: FAILURE abort-all-job: false + property-file: $WORKSPACE/change.properties - multijob: name: "{environment-type} testing on {type} instances" condition: SUCCESSFUL @@ -305,6 +275,7 @@ node-parameters: true kill-phase-on: FAILURE abort-all-job: false + property-file: $WORKSPACE/change.properties publishers: - postbuildscript: @@ -337,7 +308,7 @@ scm: - git-scm-gerrit: ssh-credentials-id: nordixinfra-nordix-gerrit-ssh - branch: refs/heads/$GERRIT_BRANCH + branch: $GERRIT_BRANCH refspec: $GERRIT_REFSPEC wipe_workspace: true clean_before: false @@ -357,13 +328,6 @@ - fix-workspace-permissions builders: - - shell: | - #!/bin/bash - if [[ "$EXECUTION_MODE" == "offline-deployment" && "$PROVISIONER_TYPE" == "heat" ]]; then - echo "Info : Offline deployment and testing on cloud instances is not currently supported!" - exit 0 - fi - - 'generate-sdf-for-change-macro' - 'generate-ssh-keypair-macro' - 'wait-pkg-mgr-macro' - 'download-artifact-macro' diff --git a/jjb/engine/engine-verify-jobs/engine-verify-package.yaml b/jjb/engine/verify-jobs/engine-verify-package.yaml similarity index 81% rename from jjb/engine/engine-verify-jobs/engine-verify-package.yaml rename to jjb/engine/verify-jobs/engine-verify-package.yaml index 94fec2cbc..f8ccece82 100644 --- a/jjb/engine/engine-verify-jobs/engine-verify-package.yaml +++ b/jjb/engine/verify-jobs/engine-verify-package.yaml @@ -27,7 +27,7 @@ branch: '{stream}' openrc-credentials-file: 'est-jenkins-openrcfile-city-frankfurt-cicd' execution-mode: 'packaging' - gated-projects: 'infra/engine|infra/provisioner/bifrost|infra/provisioner/heat|infra/installer/kolla|infra/installer/kubespray' + gated-projects: 'infra/engine|infra/provisioner/bifrost|infra/provisioner/heat' successful: false failed: false unstable: false @@ -62,30 +62,6 @@ name: VERBOSITY default: 'true' description: 'Verbosity setting for the engine - true or false - default is false' - - string: - name: ENGINE_REPO_URL - default: 'https://gerrit.nordix.org/infra/engine.git' - description: 'URL to Nordix Cloud Infra engine repository' - - string: - name: ARTIFACTS_TO_UPLOAD - default: "k8s-installer-ubuntu1804.bsx" - description: 'Artifacts to upload' - - string: - name: NORDIX_ARM_PUBLIC_IP - default: '91.106.198.25' - description: 'Public IP of Nordix Artifactory. Offline deployments can not resolve domain name' - - string: - name: NORDIX_ARM_HTTPS_URL - default: 'https://artifactory.nordix.org/artifactory' - description: 'URL to Nordix Artifactory' - - string: - name: NORDIX_ARM_REPO - default: 'nordix-kubernetes' - description: 'Name of the repository to upload and download artifact to/from. Depends on the stack' - - string: - name: NORDIX_ARTIFACT_URL - default: "$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/gerrit/$GERRIT_CHANGE_NUMBER" - description: 'Artifact to download' - string: name: CI_LOOP default: 'verify' @@ -118,7 +94,7 @@ scm: - git-scm-gerrit: ssh-credentials-id: nordixinfra-nordix-gerrit-ssh - branch: refs/heads/$GERRIT_BRANCH + branch: $GERRIT_BRANCH refspec: $GERRIT_REFSPEC wipe_workspace: true clean_before: false @@ -183,7 +159,9 @@ builders: - - 'generate-sdf-for-change-macro' + - 'generate-change-metadata-macro' + - inject: + properties-file: "$WORKSPACE/change.properties" - 'generate-ssh-keypair-macro' - 'wait-pkg-mgr-macro' - 'package-macro' diff --git a/jjb/engine/engine-verify-jobs/engine-verify-tox.yaml b/jjb/engine/verify-jobs/engine-verify-tox.yaml similarity index 91% rename from jjb/engine/engine-verify-jobs/engine-verify-tox.yaml rename to jjb/engine/verify-jobs/engine-verify-tox.yaml index 3d34add81..d115415ba 100644 --- a/jjb/engine/engine-verify-jobs/engine-verify-tox.yaml +++ b/jjb/engine/verify-jobs/engine-verify-tox.yaml @@ -25,7 +25,7 @@ stream: - 'master': branch: '{stream}' - gated-projects: 'infra/engine|infra/hwconfig|infra/swconfig|infra/test|infra/provisioner/bifrost|infra/provisioner/heat|infra/installer/kolla|infra/installer/kubespray' + gated-projects: 'infra/engine|infra/hwconfig|infra/test|infra/provisioner/bifrost|infra/provisioner/heat' lint-type: - ansible-lint: @@ -33,7 +33,7 @@ repos-to-ignore: 'infra/hwconfig' - docs: disabled: false - repos-to-ignore: 'infra/hwconfig|infra/swconfig|infra/test|infra/provisioner/bifrost|infra/provisioner/heat|infra/installer/kolla|infra/installer/kubespray' + repos-to-ignore: 'infra/hwconfig|infra/test|infra/provisioner/bifrost|infra/provisioner/heat' - shellcheck: disabled: false repos-to-ignore: 'infra/hwconfig' @@ -76,7 +76,7 @@ scm: - git-scm-gerrit: ssh-credentials-id: nordixinfra-nordix-gerrit-ssh - branch: refs/heads/$GERRIT_BRANCH + branch: $GERRIT_BRANCH refspec: $GERRIT_REFSPEC wipe_workspace: true clean_before: false @@ -108,7 +108,7 @@ builders: - multijob: - name: Engine Lint + name: Static Analysis condition: ALWAYS execution-type: PARALLEL projects: @@ -159,7 +159,7 @@ scm: - git-scm-gerrit: ssh-credentials-id: nordixinfra-nordix-gerrit-ssh - branch: refs/heads/$GERRIT_BRANCH + branch: $GERRIT_BRANCH refspec: $GERRIT_REFSPEC wipe_workspace: true clean_before: false -- 2.25.1