engine: Split verification jobs 71/4471/4
authorFatih Degirmenci <fdegir@gmail.com>
Fri, 17 Apr 2020 10:29:47 +0000 (10:29 +0000)
committerFatih Degirmenci <fdegir@gmail.com>
Fri, 17 Apr 2020 14:28:19 +0000 (14:28 +0000)
Engine verification jobs are split in order to run jobs based
on the repo the change is proposed for. Packaging is also made
into a separate step since we introduced job dependencies now.

Change-Id: I16c658ffcbb4ff0d45acf5fe5670e3cf9e3b0f2b

jjb/engine/scripts/download-artifact.sh
jjb/engine/verify-jobs/engine-verify-deploy-test.yaml [moved from jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml with 60% similarity]
jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml [deleted file]
jjb/engine/verify-jobs/engine-verify-package.yaml [moved from jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml with 53% similarity]

index b6602d39442fd3f1ab5b25413baecfb030cb9951..9b1f667f2b4958047325bf2b3589f5fda5aed734 100755 (executable)
@@ -37,6 +37,10 @@ if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then
 $NORDIX_ARM_PUBLIC_IP artifactory.nordix.org
 # managed by engine
 EOF"
+
+  # we need to get rid of search domain as well
+  sudo sed -i 's/^search/# search/' /etc/resolv.conf
+
   echo "Info  : Downloading artifact from Nordix ARM..."
   echo "        $NORDIX_ARTIFACT_URL/$ARTIFACT_TO_DOWNLOAD"
   cd /tmp
similarity index 60%
rename from jjb/engine/verify-jobs/engine-verify-online-deploy-test.yaml
rename to jjb/engine/verify-jobs/engine-verify-deploy-test.yaml
index c48db3fa1398d8199e898a016448213eb3a3d9f2..e0a47b0a790d7fb0e657b763e6f21f665276537c 100644 (file)
 # ============LICENSE_END=========================================================
 
 - project:
-    name: 'engine-verify-online-deploy-test'
+    name: 'engine-verify-deploy-test'
 
-    project: 'engine-verify-online-deploy-test'
+    project: 'engine-verify-deploy-test'
 
     stream:
       - 'master':
           branch: '{stream}'
+
+    environment-type:
+      - online:
+          disabled: false
+          execution-mode: online-deployment
           pdf: 'https://gerrit.nordix.org/gitweb?p=infra/hwconfig.git;a=blob_plain;f=pods/nordix-vpod1-pdf.yml'
           idf: 'https://gerrit.nordix.org/gitweb?p=infra/hwconfig.git;a=blob_plain;f=pods/nordix-vpod1-idf.yml'
+          successful: false
+          failed: false
+          unstable: false
+          notbuilt: false
+      - offline:
+          disabled: false
+          execution-mode: offline-deployment
+          pdf: 'file:///opt/engine/offline/git/hwconfig/pods/nordix-vpod1-pdf.yml'
+          idf: 'file:///opt/engine/offline/git/hwconfig/pods/nordix-vpod1-idf.yml'
+          successful: false
+          failed: false
+          unstable: false
+          notbuilt: false
 
     phase:
       - deploy
       - virtual:
           provisioner-type: 'bifrost'
           slave-type: 'vpod'
+          gated-projects: 'infra\/engine|infra\/provisioner\/bifrost\/|infra\/installer\/kolla\/|infra\/installer\/kubespray\/'
       - cloud:
           provisioner-type: 'heat'
           slave-type: 'jumphost'
+          gated-projects: 'infra\/engine|infra\/provisioner\/heat\/|infra\/installer\/kolla\/|infra\/installer\/kubespray\/'
 
     distro:
       - ubuntu1804
           openrc-credentials-file: 'est-jenkins-openrcfile-city-karlskrona-onap'
 
     jobs:
-      - 'engine-verify-online-deploy-test-{distro}-{cloud}-{type}-{stream}'
-      - 'engine-verify-online-{phase}-{distro}-{cloud}-{type}-{stream}'
+      - 'engine-verify-{environment-type}-deploy-test-{distro}-{cloud}-{type}-{stream}'
+      - 'engine-verify-{environment-type}-{phase}-{distro}-{cloud}-{type}-{stream}'
 
 - job-template:
-    name: 'engine-verify-online-deploy-test-{distro}-{cloud}-{type}-{stream}'
+    name: 'engine-verify-{environment-type}-deploy-test-{distro}-{cloud}-{type}-{stream}'
 
     project-type: multijob
 
           name: SLAVE_LABEL
           default: 'engine-{slave-type}-online-{distro}'
           description: 'Slave label to schedule this job on.'
+      - string:
+          name: EXECUTION_MODE
+          default: '{execution-mode}'
+          description: 'Execution mode to run in'
       - string:
           name: PROVISIONER_TYPE
           default: '{provisioner-type}'
           description: 'Provisioner type to use'
+      - string:
+          name: INSTALLER_TYPE
+          default: 'kubespray'
+          description: 'Installer type to use'
+      - string:
+          name: DEPLOY_SCENARIO
+          default: 'k8-calico-nofeature'
+          description: 'Scenario to deploy'
       - string:
           name: PDF
           default: '{pdf}'
           name: DELETE_STACK
           default: 'true'
           description: 'Delete the stack upon completion of the build'
+      - string:
+          name: VERBOSITY
+          default: 'true'
+          description: 'Verbosity setting for the engine - true or false - default is false'
+      - string:
+          name: NORDIX_ARM_PUBLIC_IP
+          default: '91.106.198.25'
+          description: 'Public IP of Nordix Artifactory. Offline deployments can not resolve domain name'
+      - string:
+          name: NORDIX_ARM_HTTPS_URL
+          default: 'https://artifactory.nordix.org/artifactory'
+          description: 'URL to Nordix Artifactory'
+      - string:
+          name: NORDIX_ARM_REPO
+          default: 'nordix-kubernetes'
+          description: 'Name of the repository to upload and download artifact to/from. Depends on the stack'
+      - string:
+          name: ARTIFACT_TO_DOWNLOAD
+          default: "https://artifactory.nordix.org/artifactory/nordix-kubernetes/installer/oss/gerrit/$GERRIT_CHANGE_ID/k8s-installer-ubuntu1804.bsx"
+          description: 'Artifact to download'
 
     properties:
       - logrotate
           wipe_workspace: true
           clean_before: false
 
+    triggers:
+      - gerrit:
+          server-name: 'Nordix Gerrit'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+            - comment-added-contains-event:
+                comment-contains-value: 'reverify'
+          projects:
+            - project-compare-type: 'REG_EXP'
+              project-pattern: '{gated-projects}'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              disable-strict-forbidden-file-verification: 'true'
+              forbidden-file-paths:
+                - compare-type: ANT
+                  pattern: 'docs/**|releasenotes/**'
+          skip-vote:
+            successful: '{obj:successful}'
+            failed: '{obj:failed}'
+            unstable: '{obj:unstable}'
+            notbuilt: '{obj:notbuilt}'
+          custom-url: '* $JOB_NAME $BUILD_URL'
+          readable-message: 'true'
+          dependency-jobs: 'engine-verify-tox-{stream},engine-verify-package-{distro}-{stream}'
+
     builders:
-      - 'download-artifact-macro'
-      - 'generate-ssh-keypair-macro'
-      - 'wait-pkg-mgr-macro'
       - multijob:
-          name: "online deployment on {type} instances"
+          name: "{environment-type} deployment on {type} instances"
           condition: SUCCESSFUL
           execution-type: PARALLEL
           projects:
-            - name: 'engine-verify-online-deploy-{distro}-city-{type}-master'
+            - name: 'engine-verify-{environment-type}-deploy-{distro}-city-{type}-master'
               current-parameters: true
+              git-revision: true
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: false
       - multijob:
-          name: "online testing on {type} instances"
+          name: "{environment-type} testing on {type} instances"
           condition: SUCCESSFUL
           execution-type: PARALLEL
           projects:
-            - name: 'engine-verify-online-functest-{distro}-city-{type}-master'
+            - name: 'engine-verify-{environment-type}-functest-{distro}-city-{type}-master'
               current-parameters: true
+              git-revision: false
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: false
                 - 'delete-stack-macro'
 
 - job-template:
-    name: 'engine-verify-online-{phase}-{distro}-{cloud}-{type}-{stream}'
+    name: 'engine-verify-{environment-type}-{phase}-{distro}-{cloud}-{type}-{stream}'
 
     disabled: '{obj:disabled}'
 
           clean_before: false
 
     builders:
+      - 'generate-ssh-keypair-macro'
+      - 'wait-pkg-mgr-macro'
+      - 'download-artifact-macro'
       - '{phase}-macro'
diff --git a/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml b/jjb/engine/verify-jobs/engine-verify-offline-deploy-test.yaml
deleted file mode 100644 (file)
index f14a861..0000000
+++ /dev/null
@@ -1,215 +0,0 @@
----
-# ============LICENSE_START=======================================================
-#  Copyright (C) 2019 Nordix Foundation.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# SPDX-License-Identifier: Apache-2.0
-# ============LICENSE_END=========================================================
-
-- project:
-    name: 'engine-verify-offline-deploy-test'
-
-    project: 'engine-verify-offline-deploy-test'
-
-    stream:
-      - 'master':
-          branch: '{stream}'
-          pdf: 'file:///opt/engine/offline/git/hwconfig/pods/nordix-vpod1-pdf.yml'
-          idf: 'file:///opt/engine/offline/git/hwconfig/pods/nordix-vpod1-idf.yml'
-
-    phase:
-      - deploy
-      - functest
-
-    type:
-      - virtual:
-          provisioner-type: 'bifrost'
-          slave-type: 'vpod'
-      - cloud:
-          provisioner-type: 'heat'
-          slave-type: 'jumphost'
-
-    distro:
-      - ubuntu1804
-
-    cloud:
-      - city:
-          openrc-credentials-file: 'est-jenkins-openrcfile-city-karlskrona-onap'
-
-    jobs:
-      - 'engine-verify-offline-deploy-test-{distro}-{cloud}-{type}-{stream}'
-      - 'engine-verify-offline-{phase}-{distro}-{cloud}-{type}-{stream}'
-
-- job-template:
-    name: 'engine-verify-offline-deploy-test-{distro}-{cloud}-{type}-{stream}'
-
-    project-type: multijob
-
-    disabled: '{obj:disabled}'
-
-    concurrent: true
-
-    parameters:
-      - project-parameters:
-          project: $GERRIT_PROJECT
-          branch: '{branch}'
-          description: 'The project and branch which the change is proposed for'
-      - label:
-          name: SLAVE_LABEL
-          default: 'engine-{slave-type}-offline-{distro}'
-          description: 'Slave label to schedule this job on.'
-      - string:
-          name: PROVISIONER_TYPE
-          default: '{provisioner-type}'
-          description: 'Provisioner type to use'
-      - string:
-          name: PDF
-          default: '{pdf}'
-          description: 'PDF file to use for deployment'
-      - string:
-          name: IDF
-          default: '{idf}'
-          description: 'IDF file to use for deployment'
-      - string:
-          name: HEAT_ENVIRONMENT_FILENAME
-          default: 'heat-environment-{cloud}-{distro}.yaml'
-          description: 'Name of heat-environment file to use.'
-      - string:
-          name: STACK_NAME
-          default: "nordix-cicd-verify-engine-offline-{distro}-$GERRIT_CHANGE_NUMBER"
-          description: 'Name of the created stack'
-      - string:
-          name: KEYPAIR_NAME
-          default: "keypair-nordix-cicd-verify-engine-offline-{distro}-$GERRIT_CHANGE_NUMBER"
-          description: 'Name of the created keypair'
-      - string:
-          name: USE_PREBUILT_DEPLOYMENT_IMAGE
-          default: 'true'
-          description: 'Use prebuilt deployment image while running as part of CI/CD'
-      - string:
-          name: TEST_REPO_URL
-          default: 'https://gerrit.nordix.org/infra/test.git'
-          description: 'URL to clone the test repo from'
-      - string:
-          name: TEST_FW
-          default: 'functest'
-          description: 'Test framework to use'
-      - string:
-          name: TEST_SUITE
-          default: 'healthcheck'
-          description: 'Test suite to run'
-      - string:
-          name: YARDSTICK_DISPATCHER
-          default: 'file'
-          description: 'Dispatcher to use for Yardstick reporting'
-      - string:
-          name: DELETE_STACK
-          default: 'true'
-          description: 'Delete the stack upon completion of the build'
-
-    properties:
-      - logrotate
-      - throttle:
-          max-per-node: 1
-          max-total: 3
-          option: project
-
-    wrappers:
-      - build-timeout:
-          timeout: 180
-      - credentials-binding:
-          - file:
-              credential-id: '{openrc-credentials-file}'
-              variable: OPENRC_FILE
-          - username-password-separated:
-              credential-id: 'nordix-cicd-arm-credentials'
-              username: NORDIX_ARM_USERNAME
-              password: NORDIX_ARM_TOKEN
-      - mask-passwords
-      - fix-workspace-permissions
-      - openstack:
-          single-use: True
-
-    builders:
-      - shell: |
-          #!/bin/bash
-          sudo sed -i 's/^search/# search/' /etc/resolv.conf
-      - 'download-artifact-macro'
-      - 'generate-ssh-keypair-macro'
-      - 'wait-pkg-mgr-macro'
-      - multijob:
-          name: "offline deployment on {type} instances"
-          condition: SUCCESSFUL
-          execution-type: PARALLEL
-          projects:
-            - name: 'engine-verify-offline-deploy-{distro}-city-{type}-master'
-              current-parameters: true
-              node-parameters: true
-              kill-phase-on: FAILURE
-              abort-all-job: false
-      - multijob:
-          name: "offline testing on {type} instances"
-          condition: SUCCESSFUL
-          execution-type: PARALLEL
-          projects:
-            - name: 'engine-verify-offline-functest-{distro}-city-{type}-master'
-              current-parameters: true
-              node-parameters: true
-              kill-phase-on: FAILURE
-              abort-all-job: false
-
-    publishers:
-      - postbuildscript:
-          builders:
-            - role: SLAVE
-              build-on:
-                - SUCCESS
-                - FAILURE
-                - ABORTED
-                - NOT_BUILT
-                - UNSTABLE
-              build-steps:
-                - 'wait-pkg-mgr-macro'
-                - 'delete-stack-macro'
-
-- job-template:
-    name: 'engine-verify-offline-{phase}-{distro}-{cloud}-{type}-{stream}'
-
-    disabled: '{obj:disabled}'
-
-    concurrent: true
-
-    properties:
-      - logrotate
-      - throttle:
-          max-per-node: 1
-          max-total: 3
-          option: project
-
-    wrappers:
-      - build-timeout:
-          timeout: 180
-      - credentials-binding:
-          - file:
-              credential-id: '{openrc-credentials-file}'
-              variable: OPENRC_FILE
-          - username-password-separated:
-              credential-id: 'nordix-cicd-arm-credentials'
-              username: NORDIX_ARM_USERNAME
-              password: NORDIX_ARM_TOKEN
-      - mask-passwords
-      - fix-workspace-permissions
-
-    builders:
-      - '{phase}-macro'
similarity index 53%
rename from jjb/engine/verify-jobs/engine-verify-deployment-parent-jobs.yaml
rename to jjb/engine/verify-jobs/engine-verify-package.yaml
index b35df8672378845d078e435a51decf0c35c00bb6..4a44bd03117aeaf2334f2f09cdab64acf37a3552 100644 (file)
 # ============LICENSE_END=========================================================
 
 - project:
-    name: 'engine-deployment'
+    name: 'engine-verify-package'
 
-    project: 'engine-deployment'
+    project: 'engine-verify-package'
 
     stream:
       - 'master':
           branch: '{stream}'
+          openrc-credentials-file: 'est-jenkins-openrcfile-city-karlskrona-onap'
+          execution-mode: 'packaging'
           gated-projects: 'infra\/engine'
-
-    environment-type:
-      - online:
-          disabled: false
-          execution-mode: online-deployment
-          successful: false
-          failed: false
-          unstable: false
-          notbuilt: false
-      - offline:
-          disabled: false
-          execution-mode: offline-deployment
           successful: false
           failed: false
           unstable: false
     distro:
       - ubuntu1804
 
-    cloud:
-      - city:
-          openrc-credentials-file: 'est-jenkins-openrcfile-city-karlskrona-onap'
-
     jobs:
-      - 'engine-verify-{environment-type}-{distro}-{cloud}-{stream}'
       - 'engine-verify-package-{distro}-{stream}'
 
 - job-template:
-    name: 'engine-verify-{environment-type}-{distro}-{cloud}-{stream}'
-
-    project-type: multijob
+    name: 'engine-verify-package-{distro}-{stream}'
 
     disabled: '{obj:disabled}'
 
       - string:
           name: EXECUTION_MODE
           default: '{execution-mode}'
-          description: 'Engine execution mode - online-deployment or offline-deployment - default is online-deployment'
+          description: 'Engine execution mode - packaging'
       - string:
           name: DISTRO
           default: '{distro}'
           description: 'Distribution to use for configuring target nodes (networking etc.)'
-      - string:
-          name: INSTALLER_TYPE
-          default: 'kubespray'
-          description: 'Installer type to use'
-      - string:
-          name: DEPLOY_SCENARIO
-          default: 'k8-calico-nofeature'
-          description: 'Scenario to deploy and test'
       - string:
           name: VERBOSITY
           default: 'true'
           name: ARTIFACTS_TO_UPLOAD
           default: "k8s-installer-ubuntu1804.bsx"
           description: 'Artifacts to upload'
-      - string:
-          name: ARTIFACT_TO_DOWNLOAD
-          default: "k8s-installer-ubuntu1804.bsx"
-          description: 'Artifact to download'
       - string:
           name: NORDIX_ARM_PUBLIC_IP
           default: '91.106.198.25'
           description: 'CI loop type'
       - label:
           name: SLAVE_LABEL
-          default: 'engine-master-{environment-type}-{distro}'
+          default: 'engine-master-offline-{distro}'
           description: 'Slave label to schedule this job on.'
       - project-parameters:
           project: $GERRIT_PROJECT
           readable-message: 'true'
           dependency-jobs: 'engine-verify-tox-{stream}'
 
-    builders:
-      - 'generate-ssh-keypair-macro'
-      - 'generate-build-metadata-macro'
-      - 'wait-pkg-mgr-macro'
-      - multijob:
-          name: package
-          condition: SUCCESSFUL
-          execution-type: PARALLEL
-          projects:
-            - name: 'engine-verify-package-{distro}-{stream}'
-              current-parameters: true
-              node-parameters: true
-              property-file: "$WORKSPACE/build.properties"
-              kill-phase-on: FAILURE
-              abort-all-job: true
-      - multijob:
-          name: "{environment-type} deployment and testing"
-          condition: SUCCESSFUL
-          execution-type: PARALLEL
-          projects:
-            - name: 'engine-verify-{environment-type}-deploy-test-{distro}-city-virtual-master'
-              current-parameters: false
-              node-parameters: false
-              predefined-parameters: |
-                GIT_BASE_SSH=$GIT_BASE_SSH
-                GERRIT_PROJECT=$GERRIT_PROJECT
-                GERRIT_BRANCH=$GERRIT_BRANCH
-                GERRIT_REFSPEC=$GERRIT_REFSPEC
-                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
-                GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID
-                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
-                PROJECT=$GERRIT_PROJECT
-                BRANCH=$BRANCH
-                ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD
-                NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP
-                EXECUTION_MODE=$EXECUTION_MODE
-                VERBOSITY=$VERBOSITY
-                DISTRO=$DISTRO
-                INSTALLER_TYPE=$INSTALLER_TYPE
-                DEPLOY_SCENARIO=$DEPLOY_SCENARIO
-              property-file: "$WORKSPACE/build.properties"
-              kill-phase-on: FAILURE
-              abort-all-job: false
-            - name: 'engine-verify-{environment-type}-deploy-test-{distro}-city-cloud-master'
-              current-parameters: false
-              node-parameters: false
-              predefined-parameters: |
-                GIT_BASE_SSH=$GIT_BASE_SSH
-                GERRIT_PROJECT=$GERRIT_PROJECT
-                GERRIT_BRANCH=$GERRIT_BRANCH
-                GERRIT_REFSPEC=$GERRIT_REFSPEC
-                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
-                GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID
-                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
-                PROJECT=$GERRIT_PROJECT
-                BRANCH=$BRANCH
-                ARTIFACT_TO_DOWNLOAD=$ARTIFACT_TO_DOWNLOAD
-                NORDIX_ARM_PUBLIC_IP=$NORDIX_ARM_PUBLIC_IP
-                EXECUTION_MODE=$EXECUTION_MODE
-                VERBOSITY=$VERBOSITY
-                DISTRO=$DISTRO
-                INSTALLER_TYPE=$INSTALLER_TYPE
-                DEPLOY_SCENARIO=$DEPLOY_SCENARIO
-              property-file: "$WORKSPACE/build.properties"
-              kill-phase-on: FAILURE
-              abort-all-job: false
-
-- job-template:
-    name: 'engine-verify-package-{distro}-{stream}'
-
-    disabled: '{obj:disabled}'
-
-    concurrent: true
-
-    properties:
-      - logrotate
-      - throttle:
-          max-per-node: 1
-          max-total: 3
-          option: project
-
     wrappers:
       - build-timeout:
           timeout: 70
       - mask-passwords
       - fix-workspace-permissions
 
-    scm:
-      - git-scm-gerrit:
-          ssh-credentials-id: nordixinfra-nordix-gerrit-ssh
-          branch: '{branch}'
-          refspec: $GERRIT_REFSPEC
-          wipe_workspace: true
-          clean_before: false
-
     builders:
+      - 'generate-ssh-keypair-macro'
+      - 'wait-pkg-mgr-macro'
       - 'package-macro'
       - 'upload-artifact-macro'