From 8d3e682d48c4444d324e2a87d1e9939becc39e27 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Thu, 19 Mar 2020 14:55:23 +0000 Subject: [PATCH] engine: Fine tune release jobs Change-Id: Ide647fe777fc2a23b5565c0ea09edc4a71ccb6fd --- jjb/engine/engine-macros.yaml | 20 +++---- .../engine-periodic-offline-deploy-test.yaml | 30 +++++++--- .../engine-periodic-release-jobs.yaml | 58 ++++++++++--------- jjb/engine/scripts/download-artifact.sh | 4 +- jjb/engine/scripts/generate-job-metadata.sh | 28 ++++++--- .../engine-verify-deployment-parent-jobs.yaml | 36 +++++++----- .../engine-verify-offline-deploy-test.yaml | 12 ++-- .../engine-verify-online-deploy-test.yaml | 12 ++-- .../engine-verify-tox.yaml | 2 +- 9 files changed, 122 insertions(+), 80 deletions(-) rename jjb/engine/{engine-periodic-jobs => periodic-jobs}/engine-periodic-offline-deploy-test.yaml (80%) rename jjb/engine/{engine-periodic-jobs => periodic-jobs}/engine-periodic-release-jobs.yaml (81%) rename jjb/engine/{engine-verify-jobs => verify-jobs}/engine-verify-deployment-parent-jobs.yaml (90%) rename jjb/engine/{engine-verify-jobs => verify-jobs}/engine-verify-offline-deploy-test.yaml (96%) rename jjb/engine/{engine-verify-jobs => verify-jobs}/engine-verify-online-deploy-test.yaml (96%) rename jjb/engine/{engine-verify-jobs => verify-jobs}/engine-verify-tox.yaml (99%) diff --git a/jjb/engine/engine-macros.yaml b/jjb/engine/engine-macros.yaml index d9fd1714..bb45b068 100644 --- a/jjb/engine/engine-macros.yaml +++ b/jjb/engine/engine-macros.yaml @@ -22,61 +22,61 @@ # the need to using include-raw-escape #------------------------------------------------------------------------------- - builder: - name: 'engine-verify-tox-macro' + name: 'tox-macro' builders: - shell: !include-raw: ./scripts/tox.sh - builder: - name: 'engine-generate-ssh-keypair-macro' + name: 'generate-ssh-keypair-macro' builders: - shell: !include-raw: ./scripts/generate-ssh-keypair.sh - builder: - name: 'engine-packaging-macro' + name: 'package-macro' builders: - shell: !include-raw: ./scripts/package.sh - builder: - name: 'engine-upload-artifact-macro' + name: 'upload-artifact-macro' builders: - shell: !include-raw: ./scripts/upload-artifact.sh - builder: - name: 'engine-download-artifact-macro' + name: 'download-artifact-macro' builders: - shell: !include-raw: ./scripts/download-artifact.sh - builder: - name: 'engine-deploy-macro' + name: 'deploy-macro' builders: - shell: !include-raw: ./scripts/deploy.sh - builder: - name: 'engine-test-macro' + name: 'test-macro' builders: - shell: !include-raw: ./scripts/test.sh - builder: - name: 'engine-delete-stack-macro' + name: 'delete-stack-macro' builders: - shell: !include-raw: ./scripts/delete-heat-stack.sh - builder: - name: 'engine-wait-pkg-mgr-macro' + name: 'wait-pkg-mgr-macro' builders: - shell: !include-raw: ./scripts/wait-for-pkg-manager.sh - builder: - name: 'engine-generate-job-metadata-macro' + name: 'generate-job-metadata-macro' builders: - shell: !include-raw: ./scripts/generate-job-metadata.sh diff --git a/jjb/engine/engine-periodic-jobs/engine-periodic-offline-deploy-test.yaml b/jjb/engine/periodic-jobs/engine-periodic-offline-deploy-test.yaml similarity index 80% rename from jjb/engine/engine-periodic-jobs/engine-periodic-offline-deploy-test.yaml rename to jjb/engine/periodic-jobs/engine-periodic-offline-deploy-test.yaml index 43f6e65f..e3e68c1f 100644 --- a/jjb/engine/engine-periodic-jobs/engine-periodic-offline-deploy-test.yaml +++ b/jjb/engine/periodic-jobs/engine-periodic-offline-deploy-test.yaml @@ -33,6 +33,7 @@ - k8-calico-nofeature - k8-multus-nofeature + type: - baremetal: pod-name: est-baremetal @@ -41,6 +42,13 @@ slave-label: baremetal-deploy yardstick-dispatcher: 'influxdb' + environment-type: + - offline: + disabled: false + execution-mode: offline-deployment + pdf: 'file:///opt/engine/offline/git/hwconfig/pods' + idf: 'file:///opt/engine/offline/git/hwconfig/pods' + phase: - deploy: build-timeout: 90 @@ -50,11 +58,11 @@ build-timeout: 120 jobs: - - '{deploy-scenario}-offline-{distro}-{type}-periodic' - - 'kubernetes-offline-{phase}-{distro}-{type}-periodic' + - '{deploy-scenario}-{environment-type}-{distro}-{type}-periodic' + - 'kubernetes-{environment-type}-{phase}-{distro}-{type}-periodic' - job-template: - name: '{deploy-scenario}-offline-{distro}-{type}-periodic' + name: '{deploy-scenario}-{environment-type}-{distro}-{type}-periodic' project-type: multijob @@ -79,6 +87,14 @@ name: DEPLOY_SCENARIO default: '{deploy-scenario}' description: 'Scenario to deploy and test' + - string: + name: PDF + default: '{pdf}/${{NODE_NAME}}-pdf.yml' + description: 'PDF file to use for deployment' + - string: + name: IDF + default: '{idf}/${{NODE_NAME}}-idf.yml' + description: 'IDF file to use for deployment' - string: name: TEST_REPO_URL default: 'https://gerrit.nordix.org/infra/test.git' @@ -117,7 +133,7 @@ condition: SUCCESSFUL execution-type: PARALLEL projects: - - name: 'kubernetes-offline-deploy-{distro}-{type}-periodic' + - name: 'kubernetes-{environment-type}-deploy-{distro}-{type}-periodic' current-parameters: true node-parameters: true kill-phase-on: FAILURE @@ -126,7 +142,7 @@ name: functest condition: ALWAYS projects: - - name: 'kubernetes-offline-functest-{distro}-{type}-periodic' + - name: 'kubernetes-{environment-type}-functest-{distro}-{type}-periodic' current-parameters: true node-parameters: true kill-phase-on: FAILURE @@ -135,14 +151,14 @@ name: yardstick condition: ALWAYS projects: - - name: 'kubernetes-offline-yardstick-{distro}-{type}-periodic' + - name: 'kubernetes-{environment-type}-yardstick-{distro}-{type}-periodic' current-parameters: true node-parameters: true kill-phase-on: FAILURE abort-all-job: true - job-template: - name: 'kubernetes-offline-{phase}-{distro}-{type}-periodic' + name: 'kubernetes-{environment-type}-{phase}-{distro}-{type}-periodic' disabled: '{obj:disabled}' diff --git a/jjb/engine/engine-periodic-jobs/engine-periodic-release-jobs.yaml b/jjb/engine/periodic-jobs/engine-periodic-release-jobs.yaml similarity index 81% rename from jjb/engine/engine-periodic-jobs/engine-periodic-release-jobs.yaml rename to jjb/engine/periodic-jobs/engine-periodic-release-jobs.yaml index eb5d875e..65eaa57c 100644 --- a/jjb/engine/engine-periodic-jobs/engine-periodic-release-jobs.yaml +++ b/jjb/engine/periodic-jobs/engine-periodic-release-jobs.yaml @@ -26,8 +26,6 @@ - offline: disabled: false execution-mode: offline-deployment - pdf: 'file:///opt/engine/offline/git/hwconfig/pods' - idf: 'file:///opt/engine/offline/git/hwconfig/pods' distro: - ubuntu1804 @@ -70,18 +68,18 @@ name: DEPLOY_SCENARIO default: 'k8-calico-nofeature' description: 'Scenario to deploy and test' - - string: - name: PDF - default: '{pdf}/${{NODE_NAME}}-pdf.yml' - description: 'PDF file to use for deployment' - - string: - name: IDF - default: '{idf}/${{NODE_NAME}}-idf.yml' - description: 'IDF file to use for deployment' - string: name: VERBOSITY default: 'true' description: 'Verbosity setting for the engine - true or false - default is false' + - string: + name: ARTIFACTS_TO_UPLOAD + default: "k8s-installer-ubuntu1804.bsx offline-package.tgz" + description: 'Artifacts to upload' + - string: + name: ARTIFACT_TO_DOWNLOAD + default: "k8s-installer-ubuntu1804.bsx" + description: 'Artifact to download' - string: name: NORDIX_ARM_PUBLIC_IP default: '91.106.198.25' @@ -95,9 +93,9 @@ default: 'nordix-kubernetes' description: 'Name of the repository to upload and download artifact to/from. Depends on the stack' - string: - name: NORDIX_ARTIFACT_URL - default: "$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/change/$GERRIT_CHANGE_NUMBER/k8s-installer-ubuntu1804.bsx" - description: 'Name of the repository to upload and download artifact to/from. Depends on the stack' + name: CI_LOOP + default: 'periodic' + description: 'CI loop type' - label: name: SLAVE_LABEL default: 'engine-build-ubuntu1804' @@ -125,11 +123,11 @@ wipe-workspace: true builders: - - 'engine-wait-pkg-mgr-macro' - - 'engine-generate-ssh-keypair-macro' - - 'engine-generate-job-metadata-macro' + - 'wait-pkg-mgr-macro' + - 'generate-ssh-keypair-macro' + - 'generate-job-metadata-macro' - multijob: - name: packaging + name: package condition: SUCCESSFUL execution-type: PARALLEL projects: @@ -137,7 +135,7 @@ current-parameters: true git-revision: true node-parameters: true - property-file: "$WORKSPACE/job.properties" + property-file: "$WORKSPACE/build.properties" kill-phase-on: FAILURE abort-all-job: true - multijob: @@ -149,7 +147,11 @@ current-parameters: false git-revision: false node-parameters: false - property-file: "$WORKSPACE/job.properties" + property-file: "$WORKSPACE/build.properties" + predefined-parameters: | + NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP + ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD + VERBOSITY=$VERBOSITY kill-phase-on: FAILURE abort-all-job: true - multijob: @@ -161,7 +163,11 @@ current-parameters: false git-revision: false node-parameters: false - property-file: "$WORKSPACE/job.properties" + property-file: "$WORKSPACE/build.properties" + predefined-parameters: | + NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP + ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD + VERBOSITY=$VERBOSITY kill-phase-on: FAILURE abort-all-job: true - multijob: @@ -169,11 +175,11 @@ condition: SUCCESSFUL execution-type: PARALLEL projects: - - name: 'kubernetes-periodic-promote-package-{distro}' + - name: 'kubernetes-periodic-promote-{distro}' current-parameters: true git-revision: true node-parameters: true - property-file: "$WORKSPACE/job.properties" + property-file: "$WORKSPACE/build.properties" kill-phase-on: FAILURE abort-all-job: true @@ -211,7 +217,7 @@ #!/bin/bash env -# - 'engine-wait-pkg-mgr-macro' -# - 'engine-generate-ssh-keypair-macro' -# - 'engine-packaging-macro' -# - 'engine-upload-artifact-macro' +# - 'wait-pkg-mgr-macro' +# - 'generate-ssh-keypair-macro' +# - 'package-macro' +# - 'upload-artifact-macro' diff --git a/jjb/engine/scripts/download-artifact.sh b/jjb/engine/scripts/download-artifact.sh index d67405a9..2cd405ef 100755 --- a/jjb/engine/scripts/download-artifact.sh +++ b/jjb/engine/scripts/download-artifact.sh @@ -33,8 +33,8 @@ EOF" cd /tmp curl -s -O $NORDIX_ARTIFACT_URL echo "Info : Downloaded the artifact for offline deployment. Extracting." - chmod +x /tmp/k8s-installer-ubuntu1804.bsx - /tmp/k8s-installer-ubuntu1804.bsx + chmod +x $ARTIFACT_TO_DOWNLOAD + $ARTIFACT_TO_DOWNLOAD else echo "Info : Online deployment. Skipping artifact download." fi diff --git a/jjb/engine/scripts/generate-job-metadata.sh b/jjb/engine/scripts/generate-job-metadata.sh index eaa0aca7..df160e3d 100755 --- a/jjb/engine/scripts/generate-job-metadata.sh +++ b/jjb/engine/scripts/generate-job-metadata.sh @@ -23,15 +23,27 @@ set -o errexit set -o pipefail cd $WORKSPACE -echo "Info : Generating job metadata" -cat << EOF > "$WORKSPACE/job.properties" -BUILD_IDENTIFIER="$(date '+%Y%m%d%H%M')" -PROMOTED_BY="$BUILD_URL" -NORDIX_ARTIFACT_URL="$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/release/$BUILD_IDENTIFIER" -ENGINE_SHA="$(git rev-parse HEAD)" +# need to generate the identifier before writing it +if [[ "$CI_LOOP" == "verify" ]]; then + BUILD_IDENTIFIER=$GERRIT_CHANGE_NUMBER +elif [[ "$CI_LOOP" == "periodic" ]]; then + BUILD_IDENTIFIER="$(date '+%Y%m%d%H%M')" +else + echo "Error : Unrecognized CI Loop $CI_LOOP" + exit 1 +fi + +echo "Info : Generating build metadata" +cat << EOF > "$WORKSPACE/build.properties" +BUILD_DATE=$(date '+%Y%m%d%H%M') +BUILT_BY=$BUILD_URL +BUILD_IDENTIFIER=$BUILD_IDENTIFIER +CI_LOOP=$CI_LOOP +ENGINE_SHA=$(git rev-parse HEAD) +NORDIX_ARTIFACT_URL=$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/release/$BUILD_IDENTIFIER EOF -echo "Info : Job metadata" +echo "Info : Build metadata" echo "-------------------------------------------------------------------------" -cat "$WORKSPACE/job.properties" +cat "$WORKSPACE/build.properties" echo "-------------------------------------------------------------------------" diff --git a/jjb/engine/engine-verify-jobs/engine-verify-deployment-parent-jobs.yaml b/jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml similarity index 90% rename from jjb/engine/engine-verify-jobs/engine-verify-deployment-parent-jobs.yaml rename to jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml index ab95a5b0..16943527 100644 --- a/jjb/engine/engine-verify-jobs/engine-verify-deployment-parent-jobs.yaml +++ b/jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml @@ -52,7 +52,7 @@ jobs: - 'engine-verify-{environment-type}-{distro}-{cloud}-{stream}' - - 'engine-verify-packaging-{distro}-{stream}' + - 'engine-verify-package-{distro}-{stream}' - job-template: name: 'engine-verify-{environment-type}-{distro}-{cloud}-{stream}' @@ -84,6 +84,14 @@ name: VERBOSITY default: 'true' description: 'Verbosity setting for the engine - true or false - default is false' + - string: + name: ARTIFACTS_TO_UPLOAD + default: "k8s-installer-ubuntu1804.bsx" + description: 'Artifacts to upload' + - string: + name: ARTIFACT_TO_DOWNLOAD + default: "k8s-installer-ubuntu1804.bsx" + description: 'Artifact to download' - string: name: NORDIX_ARM_PUBLIC_IP default: '91.106.198.25' @@ -96,10 +104,6 @@ name: NORDIX_ARM_REPO default: 'nordix-kubernetes' description: 'Name of the repository to upload and download artifact to/from. Depends on the stack' - - string: - name: NORDIX_ARTIFACT_URL - default: "$NORDIX_ARM_HTTPS_URL/$NORDIX_ARM_REPO/installer/oss/change/$GERRIT_CHANGE_NUMBER/k8s-installer-ubuntu1804.bsx" - description: 'Name of the repository to upload and download artifact to/from. Depends on the stack' - string: name: CI_LOOP default: 'verify' @@ -169,14 +173,18 @@ readable-message: 'true' builders: + - 'wait-pkg-mgr-macro' + - 'generate-ssh-keypair-macro' + - 'generate-job-metadata-macro' - multijob: - name: packaging + name: package condition: SUCCESSFUL execution-type: PARALLEL projects: - - name: 'engine-verify-packaging-{distro}-{stream}' + - name: 'engine-verify-package-{distro}-{stream}' current-parameters: true node-parameters: true + property-file: "$WORKSPACE/build.properties" kill-phase-on: FAILURE abort-all-job: true - multijob: @@ -197,8 +205,8 @@ GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER PROJECT=$GERRIT_PROJECT BRANCH=$BRANCH + ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP - NORDIX_ARTIFACT_URL=$NORDIX_ARTIFACT_URL EXECUTION_MODE=$EXECUTION_MODE VERBOSITY=$VERBOSITY DISTRO=$DISTRO @@ -206,6 +214,7 @@ DEPLOY_SCENARIO=$DEPLOY_SCENARIO PDF=$PDF IDF=$IDF + property-file: "$WORKSPACE/build.properties" kill-phase-on: FAILURE abort-all-job: false - name: 'engine-verify-{environment-type}-deploy-test-{distro}-city-cloud-master' @@ -221,8 +230,8 @@ GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER PROJECT=$GERRIT_PROJECT BRANCH=$BRANCH + ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP - NORDIX_ARTIFACT_URL=$NORDIX_ARTIFACT_URL EXECUTION_MODE=$EXECUTION_MODE VERBOSITY=$VERBOSITY DISTRO=$DISTRO @@ -230,11 +239,12 @@ DEPLOY_SCENARIO=$DEPLOY_SCENARIO PDF=$PDF IDF=$IDF + property-file: "$WORKSPACE/build.properties" kill-phase-on: FAILURE abort-all-job: false - job-template: - name: 'engine-verify-packaging-{distro}-{stream}' + name: 'engine-verify-package-{distro}-{stream}' disabled: '{obj:disabled}' @@ -267,7 +277,5 @@ clean_before: false builders: - - 'engine-wait-pkg-mgr-macro' - - 'engine-generate-ssh-keypair-macro' - - 'engine-packaging-macro' - - 'engine-upload-artifact-macro' + - 'package-macro' + - 'upload-artifact-macro' diff --git a/jjb/engine/engine-verify-jobs/engine-verify-offline-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml similarity index 96% rename from jjb/engine/engine-verify-jobs/engine-verify-offline-deploy-test.yaml rename to jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml index e16398a6..561a8950 100644 --- a/jjb/engine/engine-verify-jobs/engine-verify-offline-deploy-test.yaml +++ b/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml @@ -138,12 +138,12 @@ single-use: True builders: - - 'engine-wait-pkg-mgr-macro' + - 'wait-pkg-mgr-macro' - shell: | #!/bin/bash sudo sed -i 's/^search/# search/' /etc/resolv.conf - - 'engine-generate-ssh-keypair-macro' - - 'engine-download-artifact-macro' + - 'generate-ssh-keypair-macro' + - 'download-artifact-macro' - multijob: name: "offline deployment on {type} instances" condition: SUCCESSFUL @@ -176,8 +176,8 @@ - NOT_BUILT - UNSTABLE build-steps: - - 'engine-wait-pkg-mgr-macro' - - 'engine-delete-stack-macro' + - 'wait-pkg-mgr-macro' + - 'delete-stack-macro' - job-template: name: 'engine-verify-offline-{phase}-{distro}-{cloud}-{type}-{stream}' @@ -208,4 +208,4 @@ - fix-workspace-permissions builders: - - 'engine-{phase}-macro' + - '{phase}-macro' diff --git a/jjb/engine/engine-verify-jobs/engine-verify-online-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml similarity index 96% rename from jjb/engine/engine-verify-jobs/engine-verify-online-deploy-test.yaml rename to jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml index 3118e34d..bd53c61a 100644 --- a/jjb/engine/engine-verify-jobs/engine-verify-online-deploy-test.yaml +++ b/jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml @@ -146,9 +146,9 @@ clean_before: false builders: - - 'engine-wait-pkg-mgr-macro' - - 'engine-generate-ssh-keypair-macro' - - 'engine-download-artifact-macro' + - 'wait-pkg-mgr-macro' + - 'generate-ssh-keypair-macro' + - 'download-artifact-macro' - multijob: name: "online deployment on {type} instances" condition: SUCCESSFUL @@ -181,8 +181,8 @@ - NOT_BUILT - UNSTABLE build-steps: - - 'engine-wait-pkg-mgr-macro' - - 'engine-delete-stack-macro' + - 'wait-pkg-mgr-macro' + - 'delete-stack-macro' - job-template: name: 'engine-verify-online-{phase}-{distro}-{cloud}-{type}-{stream}' @@ -221,4 +221,4 @@ clean_before: false builders: - - 'engine-{phase}-macro' + - '{phase}-macro' diff --git a/jjb/engine/engine-verify-jobs/engine-verify-tox.yaml b/jjb/engine/verify-jobs/engine-verify-tox.yaml similarity index 99% rename from jjb/engine/engine-verify-jobs/engine-verify-tox.yaml rename to jjb/engine/verify-jobs/engine-verify-tox.yaml index 1cf39621..4561c13e 100644 --- a/jjb/engine/engine-verify-jobs/engine-verify-tox.yaml +++ b/jjb/engine/verify-jobs/engine-verify-tox.yaml @@ -164,6 +164,6 @@ clean_before: false builders: - - 'engine-verify-tox-macro' + - 'tox-macro' # vim: set ts=2 sw=2 expandtab: -- 2.25.1