From b2205c4e7fafbadc724db1da0c2c7c1e58370e9c Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Tue, 23 Jun 2020 19:46:23 +0000 Subject: [PATCH] engine: Create verify jobs for onap stack Change-Id: I5180c19b07a67087e069f9004598ca22d51cca35 --- .../engine-verify-deploy-test.yaml | 5 +- .../verify-jobs/engine-verify-package.yaml | 1 + .../kubernetes-verify-deploy-test.yaml | 5 +- .../kubernetes-verify-package.yaml | 1 + .../verify-jobs/onap-verify-deploy-test.yaml | 333 ++++++++++++++++++ .../verify-jobs/onap-verify-package.yaml | 185 ++++++++++ jjb/engine/verify-jobs/onap-verify-tox.yaml | 169 +++++++++ 7 files changed, 695 insertions(+), 4 deletions(-) create mode 100644 jjb/engine/verify-jobs/onap-verify-deploy-test.yaml create mode 100644 jjb/engine/verify-jobs/onap-verify-package.yaml create mode 100644 jjb/engine/verify-jobs/onap-verify-tox.yaml diff --git a/jjb/engine/verify-jobs/engine-verify-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-deploy-test.yaml index 2702466c1..3ffdfebab 100644 --- a/jjb/engine/verify-jobs/engine-verify-deploy-test.yaml +++ b/jjb/engine/verify-jobs/engine-verify-deploy-test.yaml @@ -159,8 +159,9 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - '.*?-verify-online-deploy-test-.*' - - '.*?-verify-offline-deploy-test-.*' + - 'engine-verify-online-deploy-test-.*' + - 'kubernetes-verify-offline-deploy-test-.*' + - 'onap-verify-offline-deploy-test-.*' block-level: 'NODE' - throttle: max-per-node: 1 diff --git a/jjb/engine/verify-jobs/engine-verify-package.yaml b/jjb/engine/verify-jobs/engine-verify-package.yaml index f486228a3..22b1e0b21 100644 --- a/jjb/engine/verify-jobs/engine-verify-package.yaml +++ b/jjb/engine/verify-jobs/engine-verify-package.yaml @@ -83,6 +83,7 @@ blocking-jobs: - 'engine-verify-package-.*' - 'kubernetes-verify-package-.*' + - 'onap-verify-package-.*' block-level: 'NODE' - throttle: max-per-node: 1 diff --git a/jjb/engine/verify-jobs/kubernetes-verify-deploy-test.yaml b/jjb/engine/verify-jobs/kubernetes-verify-deploy-test.yaml index 658f80ab5..ec8ed704a 100644 --- a/jjb/engine/verify-jobs/kubernetes-verify-deploy-test.yaml +++ b/jjb/engine/verify-jobs/kubernetes-verify-deploy-test.yaml @@ -163,8 +163,9 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - '.*?-verify-online-deploy-test-.*' - - '.*?-verify-offline-deploy-test-.*' + - 'engine-verify-online-deploy-test-.*' + - 'kubernetes-verify-offline-deploy-test-.*' + - 'onap-verify-offline-deploy-test-.*' block-level: 'NODE' - throttle: max-per-node: 1 diff --git a/jjb/engine/verify-jobs/kubernetes-verify-package.yaml b/jjb/engine/verify-jobs/kubernetes-verify-package.yaml index 68506869e..2b6bd3dee 100644 --- a/jjb/engine/verify-jobs/kubernetes-verify-package.yaml +++ b/jjb/engine/verify-jobs/kubernetes-verify-package.yaml @@ -101,6 +101,7 @@ blocking-jobs: - 'engine-verify-package-.*' - 'kubernetes-verify-package-.*' + - 'onap-verify-package-.*' block-level: 'NODE' - throttle: max-per-node: 1 diff --git a/jjb/engine/verify-jobs/onap-verify-deploy-test.yaml b/jjb/engine/verify-jobs/onap-verify-deploy-test.yaml new file mode 100644 index 000000000..0f1774597 --- /dev/null +++ b/jjb/engine/verify-jobs/onap-verify-deploy-test.yaml @@ -0,0 +1,333 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'onap-verify-deploy-test' + + project: 'onap-verify-deploy-test' + + stream: + - 'master': + branch: '{stream}' + - 'frankfurt': + branch: '{stream}' + - 'elalto': + branch: '{stream}' + + environment-type: + - online: + disabled: false + execution-mode: online-deployment + successful: false + failed: false + unstable: false + notbuilt: false + - offline: + disabled: false + execution-mode: offline-deployment + successful: false + failed: false + unstable: false + notbuilt: false + + phase: + - deploy + - functest + + type: + - cloud: + provisioner-type: 'heat' + slave-type: 'jumphost' + gated-projects: 'infra/stack/onap|infra/installer/oom' + pdf: "https://gerrit.nordix.org/gitweb?p=infra/provisioner/heat.git;a=blob_plain;f=playbooks/roles/create-stack/files/heat-environment-city-ubuntu1804.yaml" + idf: "https://gerrit.nordix.org/gitweb?p=infra/provisioner/heat.git;a=blob_plain;f=playbooks/roles/create-stack/files/heat-idf.yaml" + + distro: + - ubuntu1804 + + cloud: + - city: + openrc-credentials-file: 'est-jenkins-openrcfile-city-frankfurt-cicd' + + dependency-jobs: 'onap-verify-package-{distro}-{stream}' + dependency-job-var: 'TRIGGER_onap_verify_package_{distro}_{stream}_BUILD_RESULT' + + jobs: + - 'onap-verify-{environment-type}-deploy-test-{distro}-{cloud}-{type}-{stream}' + - 'onap-verify-{environment-type}-{phase}-{distro}-{cloud}-{type}-{stream}' + +- job-template: + name: 'onap-verify-{environment-type}-deploy-test-{distro}-{cloud}-{type}-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + parameters: + - label: + name: SLAVE_LABEL + default: 'engine-{slave-type}-{environment-type}-{distro}-fra' + description: 'Slave label to schedule this job on.' + - string: + name: HW_TYPE + default: '{type}' + description: 'Hardware type - could be virtual, cloud, or baremetal' + - string: + name: EXECUTION_MODE + default: '{execution-mode}' + description: 'Execution mode to run in' + - string: + name: PDF + default: '{pdf}' + description: 'PDF file to use for deployment' + - string: + name: IDF + default: '{idf}' + description: 'IDF file to use for deployment' + - string: + name: HEAT_ENVIRONMENT_FILENAME + default: 'heat-environment-{cloud}-{distro}.yaml' + description: 'Name of heat-environment file to use.' + - string: + name: HEAT_STACK_NAME + default: "nordix-cicd-verify-onap-{environment-type}-{distro}-$GERRIT_CHANGE_NUMBER" + description: 'Name of the created stack' + - string: + name: KEYPAIR_NAME + default: "keypair-nordix-cicd-verify-onap-{environment-type}-{distro}-$GERRIT_CHANGE_NUMBER" + description: 'Name of the created keypair' + - string: + name: USE_PREBUILT_DEPLOYMENT_IMAGE + default: 'true' + description: 'Use prebuilt deployment image while running as part of CI/CD' + - string: + name: TEST_REPO_URL + default: 'https://gerrit.nordix.org/infra/test.git' + description: 'URL to clone the test repo from' + - string: + name: TEST_FW + default: 'functest' + description: 'Test framework to use' + - string: + name: TEST_SUITE + default: 'healthcheck' + description: 'Test suite to run' + - string: + name: YARDSTICK_DISPATCHER + default: 'file' + description: 'Dispatcher to use for Yardstick reporting' + - string: + name: DELETE_STACK + default: 'true' + description: 'Delete the stack upon completion of the build' + - string: + name: VERBOSITY + default: 'true' + description: 'Verbosity setting for the engine - true or false - default is false' + - string: + name: CI_LOOP + default: "verify" + description: 'CI loop the job belongs to' + - project-parameters: + project: $GERRIT_PROJECT + branch: '{branch}' + description: 'The project and branch which the change is proposed for' + - nordix-gerrit-parameters + + properties: + - logrotate + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'engine-verify-online-deploy-test-.*' + - 'kubernetes-verify-offline-deploy-test-.*' + - 'onap-verify-offline-deploy-test-.*' + block-level: 'NODE' + - throttle: + max-per-node: 1 + max-total: 3 + option: project + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: $GERRIT_BRANCH + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false + + triggers: + - gerrit: + server-name: 'Nordix Gerrit' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: '{gated-projects}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|releasenotes/**' + skip-vote: + successful: '{obj:successful}' + failed: '{obj:failed}' + unstable: '{obj:unstable}' + notbuilt: '{obj:notbuilt}' + custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' + dependency-jobs: '{dependency-jobs}' + + wrappers: + - build-timeout: + timeout: 180 + - credentials-binding: + - file: + credential-id: '{openrc-credentials-file}' + variable: OPENRC_FILE + - username-password-separated: + credential-id: 'nordix-cicd-arm-credentials' + username: NORDIX_ARM_USERNAME + password: NORDIX_ARM_TOKEN + - mask-passwords + - openstack: + single-use: True + - pre-scm-buildstep: + failOnError: true + buildsteps: + - system-groovy: + command: | + def build = this.getProperty('binding').getVariable('build') + def listener = this.getProperty('binding').getVariable('listener') + if (build.getEnvironment(listener).{dependency-job-var} != "SUCCESS") {{ + println "WARN : Aborting build since dependent job did not succeed!" + throw new InterruptedException() + }} + - shell: | + #!/bin/bash + sudo chown -R $USER:$USER $WORKSPACE || exit 1 + # offline nodes can not resolve Nordix Artifactory and Gerrit + if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then + echo "Info : Offline deployment. Adding City OpenStack Endpoint, Nordix Artifactory and Gerrit Public IP to /etc/hosts" + sudo sed -i 's/^search/# search/' /etc/resolv.conf + sudo bash -c "cat << EOF >> /etc/hosts + # managed by engine + 91.106.198.25 artifactory.nordix.org + 91.106.198.142 gerrit.nordix.org + 86.107.242.50 fra1.citycloud.com + # managed by engine + EOF" + # we need to get rid of search domain as well + sudo sed -i 's/^search/# search/' /etc/resolv.conf + fi + + builders: + - 'generate-change-metadata-macro' + - multijob: + name: "{environment-type} deployment on {type} instances" + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'onap-verify-{environment-type}-deploy-{distro}-city-{type}-master' + current-parameters: true + git-revision: true + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: false + property-file: $WORKSPACE/change.properties + - multijob: + name: "{environment-type} testing on {type} instances" + condition: SUCCESSFUL + execution-type: PARALLEL + projects: + - name: 'onap-verify-{environment-type}-functest-{distro}-city-{type}-master' + current-parameters: true + git-revision: false + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: false + property-file: $WORKSPACE/change.properties + + publishers: + - postbuildscript: + builders: + - role: SLAVE + build-on: + - SUCCESS + - FAILURE + - ABORTED + - NOT_BUILT + - UNSTABLE + build-steps: + - 'wait-pkg-mgr-macro' + - 'delete-stack-macro' + +- job-template: + name: 'onap-verify-{environment-type}-{phase}-{distro}-{cloud}-{type}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - logrotate + - throttle: + max-per-node: 1 + max-total: 3 + option: project + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: $GERRIT_BRANCH + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false + + wrappers: + - build-timeout: + timeout: 180 + - credentials-binding: + - file: + credential-id: '{openrc-credentials-file}' + variable: OPENRC_FILE + - username-password-separated: + credential-id: 'nordix-cicd-arm-credentials' + username: NORDIX_ARM_USERNAME + password: NORDIX_ARM_TOKEN + - mask-passwords + - fix-workspace-permissions + + builders: + - 'generate-ssh-keypair-macro' + - 'wait-pkg-mgr-macro' + - 'download-artifact-macro' + - '{phase}-macro' diff --git a/jjb/engine/verify-jobs/onap-verify-package.yaml b/jjb/engine/verify-jobs/onap-verify-package.yaml new file mode 100644 index 000000000..11bb7b589 --- /dev/null +++ b/jjb/engine/verify-jobs/onap-verify-package.yaml @@ -0,0 +1,185 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'onap-verify-package' + + project: 'onap-verify-package' + + stream: + - 'master': + branch: '{stream}' + openrc-credentials-file: 'est-jenkins-openrcfile-city-frankfurt-cicd' + execution-mode: 'packaging' + gated-projects: 'infra/stack/onap|infra/installer/oom' + successful: false + failed: false + unstable: false + notbuilt: false + - '1.17': + branch: '{stream}' + openrc-credentials-file: 'est-jenkins-openrcfile-city-frankfurt-cicd' + execution-mode: 'packaging' + gated-projects: 'infra/stack/onap|infra/installer/oom' + successful: false + failed: false + unstable: false + notbuilt: false + - '1.15': + branch: '{stream}' + openrc-credentials-file: 'est-jenkins-openrcfile-city-frankfurt-cicd' + execution-mode: 'packaging' + gated-projects: 'infra/stack/onap|infra/installer/oom' + successful: false + failed: false + unstable: false + notbuilt: false + + distro: + - ubuntu1804 + + dependency-jobs: 'onap-verify-tox-{stream}' + dependency-job-var: 'TRIGGER_onap_verify_tox_{stream}_BUILD_RESULT' + + jobs: + - 'onap-verify-package-{distro}-{stream}' + +- job-template: + name: 'onap-verify-package-{distro}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + parameters: + - string: + name: EXECUTION_MODE + default: '{execution-mode}' + description: 'Engine execution mode - packaging' + - string: + name: DISTRO + default: '{distro}' + description: 'Distribution to use for configuring target nodes (networking etc.)' + - string: + name: VERBOSITY + default: 'true' + description: 'Verbosity setting for the engine - true or false - default is false' + - string: + name: CI_LOOP + default: 'verify' + description: 'CI loop type' + - label: + name: SLAVE_LABEL + default: 'engine-package-{distro}-fra' + description: 'Slave label to schedule this job on.' + - project-parameters: + project: $GERRIT_PROJECT + branch: '{branch}' + description: 'The project and branch which the change is proposed for' + - nordix-gerrit-parameters + + properties: + - logrotate + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'engine-verify-package-.*' + - 'kubernetes-verify-package-.*' + - 'onap-verify-package-.*' + block-level: 'NODE' + - throttle: + max-per-node: 1 + max-total: 3 + option: project + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: $GERRIT_BRANCH + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false + + triggers: + - gerrit: + server-name: 'Nordix Gerrit' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: '{gated-projects}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'true' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|releasenotes/**' + skip-vote: + successful: '{obj:successful}' + failed: '{obj:failed}' + unstable: '{obj:unstable}' + notbuilt: '{obj:notbuilt}' + custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' + dependency-jobs: '{dependency-jobs}' + + wrappers: + - openstack: + single-use: True + - build-timeout: + timeout: 240 + - credentials-binding: + - username-password-separated: + credential-id: 'nordix-cicd-arm-credentials' + username: NORDIX_ARM_USERNAME + password: NORDIX_ARM_TOKEN + - mask-passwords + - pre-scm-buildstep: + failOnError: true + buildsteps: + - system-groovy: + command: | + def build = this.getProperty('binding').getVariable('build') + def listener = this.getProperty('binding').getVariable('listener') + if (build.getEnvironment(listener).{dependency-job-var} != "SUCCESS") {{ + println "WARN : Aborting build since dependent job did not succeed!" + throw new InterruptedException() + }} + - shell: | + #!/bin/bash + sudo chown -R $USER:$USER $WORKSPACE || exit 1 + + + builders: + - 'generate-change-metadata-macro' + - inject: + properties-file: "$WORKSPACE/change.properties" + - 'generate-ssh-keypair-macro' + - 'wait-pkg-mgr-macro' + - 'package-macro' + - 'upload-artifact-macro' diff --git a/jjb/engine/verify-jobs/onap-verify-tox.yaml b/jjb/engine/verify-jobs/onap-verify-tox.yaml new file mode 100644 index 000000000..6bd6df247 --- /dev/null +++ b/jjb/engine/verify-jobs/onap-verify-tox.yaml @@ -0,0 +1,169 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'onap-tox' + + project: 'onap-tox' + + stream: + - 'master': + branch: '{stream}' + gated-projects: 'infra/stack/onap|infra/installer/oom' + - '1.17': + branch: '{stream}' + gated-projects: 'infra/stack/onap|infra/installer/oom' + - '1.15': + branch: '{stream}' + gated-projects: 'infra/stack/onap|infra/installer/oom' + + lint-type: + - ansible-lint: + disabled: false + repos-to-ignore: '' + - shellcheck: + disabled: false + repos-to-ignore: '' + - yamllint: + disabled: false + repos-to-ignore: '' + + jobs: + - 'onap-verify-tox-{stream}' + - 'onap-verify-{lint-type}-{stream}' + +- job-template: + name: 'onap-verify-tox-{stream}' + + project-type: multijob + + concurrent: true + + node: infra-tools-docker-slave-ubuntu1804 + + parameters: + - project-parameters: + project: $GERRIT_PROJECT + branch: '{branch}' + description: 'The project and branch which the change is proposed for' + - string: + name: VERBOSITY + default: 'false' + description: 'Verbosity setting for the tox script' + - nordix-gerrit-parameters + + properties: + - logrotate + + wrappers: + - build-timeout: + timeout: 60 + - fix-workspace-permissions + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: $GERRIT_BRANCH + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false + + triggers: + - gerrit: + server-name: 'Nordix Gerrit' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: '{gated-projects}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'false' + forbidden-file-paths: + - compare-type: ANT + pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' + custom-url: '* $JOB_NAME $BUILD_URL' + + builders: + - multijob: + name: Static Analysis + condition: ALWAYS + execution-type: PARALLEL + projects: + - name: 'onap-verify-ansible-lint-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + - name: 'onap-verify-shellcheck-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + - name: 'onap-verify-yamllint-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + +- job-template: + name: 'onap-verify-{lint-type}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + node: infra-tools-docker-slave-ubuntu1804 + + parameters: + - string: + name: LINT_TYPE + default: '{lint-type}' + description: 'Lint type to run' + - string: + name: REPOS_TO_IGNORE + default: '{repos-to-ignore}' + description: 'Repos to ignore for the specific lint type' + + properties: + - logrotate + + wrappers: + - build-timeout: + timeout: 150 + - fix-workspace-permissions + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: $GERRIT_BRANCH + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false + + builders: + - 'tox-macro' + +# vim: set ts=2 sw=2 expandtab: -- 2.25.1