From 4a7b8bbfa2bcfb080107108e32abd43dd8e6cfe6 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Fri, 17 Apr 2020 10:29:47 +0000 Subject: [PATCH] engine: Split verification jobs Engine verification jobs are split in order to run jobs based on the repo the change is proposed for. Packaging is also made into a separate step since we introduced job dependencies now. Change-Id: I16c658ffcbb4ff0d45acf5fe5670e3cf9e3b0f2b --- jjb/engine/scripts/download-artifact.sh | 4 + ...st.yaml => engine-verify-deploy-test.yaml} | 112 +++++++-- .../engine-verify-offline-deploy-test.yaml | 215 ------------------ ...t-jobs.yaml => engine-verify-package.yaml} | 134 +---------- 4 files changed, 112 insertions(+), 353 deletions(-) rename jjb/engine/verify-jobs/{engine-verify-online-deploy-test.yaml => engine-verify-deploy-test.yaml} (60%) delete mode 100644 jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml rename jjb/engine/verify-jobs/{engine-verify-deployment-parent-jobs.yaml => engine-verify-package.yaml} (53%) diff --git a/jjb/engine/scripts/download-artifact.sh b/jjb/engine/scripts/download-artifact.sh index b6602d39..9b1f667f 100755 --- a/jjb/engine/scripts/download-artifact.sh +++ b/jjb/engine/scripts/download-artifact.sh @@ -37,6 +37,10 @@ if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then $NORDIX_ARM_PUBLIC_IP artifactory.nordix.org # managed by engine EOF" + + # we need to get rid of search domain as well + sudo sed -i 's/^search/# search/' /etc/resolv.conf + echo "Info : Downloading artifact from Nordix ARM..." echo " $NORDIX_ARTIFACT_URL/$ARTIFACT_TO_DOWNLOAD" cd /tmp diff --git a/jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-deploy-test.yaml similarity index 60% rename from jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml rename to jjb/engine/verify-jobs/engine-verify-deploy-test.yaml index c48db3fa..e0a47b0a 100644 --- a/jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml +++ b/jjb/engine/verify-jobs/engine-verify-deploy-test.yaml @@ -18,15 +18,33 @@ # ============LICENSE_END========================================================= - project: - name: 'engine-verify-online-deploy-test' + name: 'engine-verify-deploy-test' - project: 'engine-verify-online-deploy-test' + project: 'engine-verify-deploy-test' stream: - 'master': branch: '{stream}' + + environment-type: + - online: + disabled: false + execution-mode: online-deployment pdf: 'https://gerrit.nordix.org/gitweb?p=infra/hwconfig.git;a=blob_plain;f=pods/nordix-vpod1-pdf.yml' idf: 'https://gerrit.nordix.org/gitweb?p=infra/hwconfig.git;a=blob_plain;f=pods/nordix-vpod1-idf.yml' + successful: false + failed: false + unstable: false + notbuilt: false + - offline: + disabled: false + execution-mode: offline-deployment + pdf: 'file:///opt/engine/offline/git/hwconfig/pods/nordix-vpod1-pdf.yml' + idf: 'file:///opt/engine/offline/git/hwconfig/pods/nordix-vpod1-idf.yml' + successful: false + failed: false + unstable: false + notbuilt: false phase: - deploy @@ -36,9 +54,11 @@ - virtual: provisioner-type: 'bifrost' slave-type: 'vpod' + gated-projects: 'infra\/engine|infra\/provisioner\/bifrost\/|infra\/installer\/kolla\/|infra\/installer\/kubespray\/' - cloud: provisioner-type: 'heat' slave-type: 'jumphost' + gated-projects: 'infra\/engine|infra\/provisioner\/heat\/|infra\/installer\/kolla\/|infra\/installer\/kubespray\/' distro: - ubuntu1804 @@ -48,11 +68,11 @@ openrc-credentials-file: 'est-jenkins-openrcfile-city-karlskrona-onap' jobs: - - 'engine-verify-online-deploy-test-{distro}-{cloud}-{type}-{stream}' - - 'engine-verify-online-{phase}-{distro}-{cloud}-{type}-{stream}' + - 'engine-verify-{environment-type}-deploy-test-{distro}-{cloud}-{type}-{stream}' + - 'engine-verify-{environment-type}-{phase}-{distro}-{cloud}-{type}-{stream}' - job-template: - name: 'engine-verify-online-deploy-test-{distro}-{cloud}-{type}-{stream}' + name: 'engine-verify-{environment-type}-deploy-test-{distro}-{cloud}-{type}-{stream}' project-type: multijob @@ -69,10 +89,22 @@ name: SLAVE_LABEL default: 'engine-{slave-type}-online-{distro}' description: 'Slave label to schedule this job on.' + - string: + name: EXECUTION_MODE + default: '{execution-mode}' + description: 'Execution mode to run in' - string: name: PROVISIONER_TYPE default: '{provisioner-type}' description: 'Provisioner type to use' + - string: + name: INSTALLER_TYPE + default: 'kubespray' + description: 'Installer type to use' + - string: + name: DEPLOY_SCENARIO + default: 'k8-calico-nofeature' + description: 'Scenario to deploy' - string: name: PDF default: '{pdf}' @@ -117,6 +149,26 @@ name: DELETE_STACK default: 'true' description: 'Delete the stack upon completion of the build' + - string: + name: VERBOSITY + default: 'true' + description: 'Verbosity setting for the engine - true or false - default is false' + - string: + name: NORDIX_ARM_PUBLIC_IP + default: '91.106.198.25' + description: 'Public IP of Nordix Artifactory. Offline deployments can not resolve domain name' + - string: + name: NORDIX_ARM_HTTPS_URL + default: 'https://artifactory.nordix.org/artifactory' + description: 'URL to Nordix Artifactory' + - string: + name: NORDIX_ARM_REPO + default: 'nordix-kubernetes' + description: 'Name of the repository to upload and download artifact to/from. Depends on the stack' + - string: + name: ARTIFACT_TO_DOWNLOAD + default: "https://artifactory.nordix.org/artifactory/nordix-kubernetes/installer/oss/gerrit/$GERRIT_CHANGE_ID/k8s-installer-ubuntu1804.bsx" + description: 'Artifact to download' properties: - logrotate @@ -149,27 +201,58 @@ wipe_workspace: true clean_before: false + triggers: + - gerrit: + server-name: 'Nordix Gerrit' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: '{gated-projects}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|releasenotes/**' + skip-vote: + successful: '{obj:successful}' + failed: '{obj:failed}' + unstable: '{obj:unstable}' + notbuilt: '{obj:notbuilt}' + custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' + dependency-jobs: 'engine-verify-tox-{stream},engine-verify-package-{distro}-{stream}' + builders: - - 'download-artifact-macro' - - 'generate-ssh-keypair-macro' - - 'wait-pkg-mgr-macro' - multijob: - name: "online deployment on {type} instances" + name: "{environment-type} deployment on {type} instances" condition: SUCCESSFUL execution-type: PARALLEL projects: - - name: 'engine-verify-online-deploy-{distro}-city-{type}-master' + - name: 'engine-verify-{environment-type}-deploy-{distro}-city-{type}-master' current-parameters: true + git-revision: true node-parameters: true kill-phase-on: FAILURE abort-all-job: false - multijob: - name: "online testing on {type} instances" + name: "{environment-type} testing on {type} instances" condition: SUCCESSFUL execution-type: PARALLEL projects: - - name: 'engine-verify-online-functest-{distro}-city-{type}-master' + - name: 'engine-verify-{environment-type}-functest-{distro}-city-{type}-master' current-parameters: true + git-revision: false node-parameters: true kill-phase-on: FAILURE abort-all-job: false @@ -189,7 +272,7 @@ - 'delete-stack-macro' - job-template: - name: 'engine-verify-online-{phase}-{distro}-{cloud}-{type}-{stream}' + name: 'engine-verify-{environment-type}-{phase}-{distro}-{cloud}-{type}-{stream}' disabled: '{obj:disabled}' @@ -225,4 +308,7 @@ clean_before: false builders: + - 'generate-ssh-keypair-macro' + - 'wait-pkg-mgr-macro' + - 'download-artifact-macro' - '{phase}-macro' diff --git a/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml deleted file mode 100644 index f14a861f..00000000 --- a/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml +++ /dev/null @@ -1,215 +0,0 @@ ---- -# ============LICENSE_START======================================================= -# Copyright (C) 2019 Nordix Foundation. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# ============LICENSE_END========================================================= - -- project: - name: 'engine-verify-offline-deploy-test' - - project: 'engine-verify-offline-deploy-test' - - stream: - - 'master': - branch: '{stream}' - pdf: 'file:///opt/engine/offline/git/hwconfig/pods/nordix-vpod1-pdf.yml' - idf: 'file:///opt/engine/offline/git/hwconfig/pods/nordix-vpod1-idf.yml' - - phase: - - deploy - - functest - - type: - - virtual: - provisioner-type: 'bifrost' - slave-type: 'vpod' - - cloud: - provisioner-type: 'heat' - slave-type: 'jumphost' - - distro: - - ubuntu1804 - - cloud: - - city: - openrc-credentials-file: 'est-jenkins-openrcfile-city-karlskrona-onap' - - jobs: - - 'engine-verify-offline-deploy-test-{distro}-{cloud}-{type}-{stream}' - - 'engine-verify-offline-{phase}-{distro}-{cloud}-{type}-{stream}' - -- job-template: - name: 'engine-verify-offline-deploy-test-{distro}-{cloud}-{type}-{stream}' - - project-type: multijob - - disabled: '{obj:disabled}' - - concurrent: true - - parameters: - - project-parameters: - project: $GERRIT_PROJECT - branch: '{branch}' - description: 'The project and branch which the change is proposed for' - - label: - name: SLAVE_LABEL - default: 'engine-{slave-type}-offline-{distro}' - description: 'Slave label to schedule this job on.' - - string: - name: PROVISIONER_TYPE - default: '{provisioner-type}' - description: 'Provisioner type to use' - - string: - name: PDF - default: '{pdf}' - description: 'PDF file to use for deployment' - - string: - name: IDF - default: '{idf}' - description: 'IDF file to use for deployment' - - string: - name: HEAT_ENVIRONMENT_FILENAME - default: 'heat-environment-{cloud}-{distro}.yaml' - description: 'Name of heat-environment file to use.' - - string: - name: STACK_NAME - default: "nordix-cicd-verify-engine-offline-{distro}-$GERRIT_CHANGE_NUMBER" - description: 'Name of the created stack' - - string: - name: KEYPAIR_NAME - default: "keypair-nordix-cicd-verify-engine-offline-{distro}-$GERRIT_CHANGE_NUMBER" - description: 'Name of the created keypair' - - string: - name: USE_PREBUILT_DEPLOYMENT_IMAGE - default: 'true' - description: 'Use prebuilt deployment image while running as part of CI/CD' - - string: - name: TEST_REPO_URL - default: 'https://gerrit.nordix.org/infra/test.git' - description: 'URL to clone the test repo from' - - string: - name: TEST_FW - default: 'functest' - description: 'Test framework to use' - - string: - name: TEST_SUITE - default: 'healthcheck' - description: 'Test suite to run' - - string: - name: YARDSTICK_DISPATCHER - default: 'file' - description: 'Dispatcher to use for Yardstick reporting' - - string: - name: DELETE_STACK - default: 'true' - description: 'Delete the stack upon completion of the build' - - properties: - - logrotate - - throttle: - max-per-node: 1 - max-total: 3 - option: project - - wrappers: - - build-timeout: - timeout: 180 - - credentials-binding: - - file: - credential-id: '{openrc-credentials-file}' - variable: OPENRC_FILE - - username-password-separated: - credential-id: 'nordix-cicd-arm-credentials' - username: NORDIX_ARM_USERNAME - password: NORDIX_ARM_TOKEN - - mask-passwords - - fix-workspace-permissions - - openstack: - single-use: True - - builders: - - shell: | - #!/bin/bash - sudo sed -i 's/^search/# search/' /etc/resolv.conf - - 'download-artifact-macro' - - 'generate-ssh-keypair-macro' - - 'wait-pkg-mgr-macro' - - multijob: - name: "offline deployment on {type} instances" - condition: SUCCESSFUL - execution-type: PARALLEL - projects: - - name: 'engine-verify-offline-deploy-{distro}-city-{type}-master' - current-parameters: true - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: false - - multijob: - name: "offline testing on {type} instances" - condition: SUCCESSFUL - execution-type: PARALLEL - projects: - - name: 'engine-verify-offline-functest-{distro}-city-{type}-master' - current-parameters: true - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: false - - publishers: - - postbuildscript: - builders: - - role: SLAVE - build-on: - - SUCCESS - - FAILURE - - ABORTED - - NOT_BUILT - - UNSTABLE - build-steps: - - 'wait-pkg-mgr-macro' - - 'delete-stack-macro' - -- job-template: - name: 'engine-verify-offline-{phase}-{distro}-{cloud}-{type}-{stream}' - - disabled: '{obj:disabled}' - - concurrent: true - - properties: - - logrotate - - throttle: - max-per-node: 1 - max-total: 3 - option: project - - wrappers: - - build-timeout: - timeout: 180 - - credentials-binding: - - file: - credential-id: '{openrc-credentials-file}' - variable: OPENRC_FILE - - username-password-separated: - credential-id: 'nordix-cicd-arm-credentials' - username: NORDIX_ARM_USERNAME - password: NORDIX_ARM_TOKEN - - mask-passwords - - fix-workspace-permissions - - builders: - - '{phase}-macro' diff --git a/jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml b/jjb/engine/verify-jobs/engine-verify-package.yaml similarity index 53% rename from jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml rename to jjb/engine/verify-jobs/engine-verify-package.yaml index b35df867..4a44bd03 100644 --- a/jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml +++ b/jjb/engine/verify-jobs/engine-verify-package.yaml @@ -18,26 +18,16 @@ # ============LICENSE_END========================================================= - project: - name: 'engine-deployment' + name: 'engine-verify-package' - project: 'engine-deployment' + project: 'engine-verify-package' stream: - 'master': branch: '{stream}' + openrc-credentials-file: 'est-jenkins-openrcfile-city-karlskrona-onap' + execution-mode: 'packaging' gated-projects: 'infra\/engine' - - environment-type: - - online: - disabled: false - execution-mode: online-deployment - successful: false - failed: false - unstable: false - notbuilt: false - - offline: - disabled: false - execution-mode: offline-deployment successful: false failed: false unstable: false @@ -46,18 +36,11 @@ distro: - ubuntu1804 - cloud: - - city: - openrc-credentials-file: 'est-jenkins-openrcfile-city-karlskrona-onap' - jobs: - - 'engine-verify-{environment-type}-{distro}-{cloud}-{stream}' - 'engine-verify-package-{distro}-{stream}' - job-template: - name: 'engine-verify-{environment-type}-{distro}-{cloud}-{stream}' - - project-type: multijob + name: 'engine-verify-package-{distro}-{stream}' disabled: '{obj:disabled}' @@ -67,19 +50,11 @@ - string: name: EXECUTION_MODE default: '{execution-mode}' - description: 'Engine execution mode - online-deployment or offline-deployment - default is online-deployment' + description: 'Engine execution mode - packaging' - string: name: DISTRO default: '{distro}' description: 'Distribution to use for configuring target nodes (networking etc.)' - - string: - name: INSTALLER_TYPE - default: 'kubespray' - description: 'Installer type to use' - - string: - name: DEPLOY_SCENARIO - default: 'k8-calico-nofeature' - description: 'Scenario to deploy and test' - string: name: VERBOSITY default: 'true' @@ -88,10 +63,6 @@ name: ARTIFACTS_TO_UPLOAD default: "k8s-installer-ubuntu1804.bsx" description: 'Artifacts to upload' - - string: - name: ARTIFACT_TO_DOWNLOAD - default: "k8s-installer-ubuntu1804.bsx" - description: 'Artifact to download' - string: name: NORDIX_ARM_PUBLIC_IP default: '91.106.198.25' @@ -110,7 +81,7 @@ description: 'CI loop type' - label: name: SLAVE_LABEL - default: 'engine-master-{environment-type}-{distro}' + default: 'engine-master-offline-{distro}' description: 'Slave label to schedule this job on.' - project-parameters: project: $GERRIT_PROJECT @@ -173,87 +144,6 @@ readable-message: 'true' dependency-jobs: 'engine-verify-tox-{stream}' - builders: - - 'generate-ssh-keypair-macro' - - 'generate-build-metadata-macro' - - 'wait-pkg-mgr-macro' - - multijob: - name: package - condition: SUCCESSFUL - execution-type: PARALLEL - projects: - - name: 'engine-verify-package-{distro}-{stream}' - current-parameters: true - node-parameters: true - property-file: "$WORKSPACE/build.properties" - kill-phase-on: FAILURE - abort-all-job: true - - multijob: - name: "{environment-type} deployment and testing" - condition: SUCCESSFUL - execution-type: PARALLEL - projects: - - name: 'engine-verify-{environment-type}-deploy-test-{distro}-city-virtual-master' - current-parameters: false - node-parameters: false - predefined-parameters: | - GIT_BASE_SSH=$GIT_BASE_SSH - GERRIT_PROJECT=$GERRIT_PROJECT - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - PROJECT=$GERRIT_PROJECT - BRANCH=$BRANCH - ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD - NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP - EXECUTION_MODE=$EXECUTION_MODE - VERBOSITY=$VERBOSITY - DISTRO=$DISTRO - INSTALLER_TYPE=$INSTALLER_TYPE - DEPLOY_SCENARIO=$DEPLOY_SCENARIO - property-file: "$WORKSPACE/build.properties" - kill-phase-on: FAILURE - abort-all-job: false - - name: 'engine-verify-{environment-type}-deploy-test-{distro}-city-cloud-master' - current-parameters: false - node-parameters: false - predefined-parameters: | - GIT_BASE_SSH=$GIT_BASE_SSH - GERRIT_PROJECT=$GERRIT_PROJECT - GERRIT_BRANCH=$GERRIT_BRANCH - GERRIT_REFSPEC=$GERRIT_REFSPEC - GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE - GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID - GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER - PROJECT=$GERRIT_PROJECT - BRANCH=$BRANCH - ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD - NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP - EXECUTION_MODE=$EXECUTION_MODE - VERBOSITY=$VERBOSITY - DISTRO=$DISTRO - INSTALLER_TYPE=$INSTALLER_TYPE - DEPLOY_SCENARIO=$DEPLOY_SCENARIO - property-file: "$WORKSPACE/build.properties" - kill-phase-on: FAILURE - abort-all-job: false - -- job-template: - name: 'engine-verify-package-{distro}-{stream}' - - disabled: '{obj:disabled}' - - concurrent: true - - properties: - - logrotate - - throttle: - max-per-node: 1 - max-total: 3 - option: project - wrappers: - build-timeout: timeout: 70 @@ -265,14 +155,8 @@ - mask-passwords - fix-workspace-permissions - scm: - - git-scm-gerrit: - ssh-credentials-id: nordixinfra-nordix-gerrit-ssh - branch: '{branch}' - refspec: $GERRIT_REFSPEC - wipe_workspace: true - clean_before: false - builders: + - 'generate-ssh-keypair-macro' + - 'wait-pkg-mgr-macro' - 'package-macro' - 'upload-artifact-macro' -- 2.25.1