From 85c872e9d1f48762565c1ac5f696bc21fa9dfd7f Mon Sep 17 00:00:00 2001 From: Lionel Jouin Date: Wed, 10 Aug 2022 15:08:42 +0200 Subject: [PATCH] Meridio: New PR and periodic jobs * Now periodic and PR jobs are based on the same generic pipeline * The pipeline can handles Git, Static analysis (code generated, linter, unit tests), Images build/tag/push (base image and then others), Github commit status * E2e will be added later Change-Id: I01731637c940f7585c3d08008ffd062c53f4f2b1 --- jjb/nsm/Jenkinsfile | 267 ++++++++++++++++++ ...io-periodic.yaml => meridio-periodic.yaml} | 82 +++--- jjb/nsm/meridio-pull-request.yaml | 102 +++++++ jjb/nsm/nordix-nsm-meridio-verify.yaml | 175 ------------ jjb/nsm/scripts/build.sh | 47 --- jjb/nsm/scripts/static-analysis.sh | 43 --- 6 files changed, 406 insertions(+), 310 deletions(-) create mode 100644 jjb/nsm/Jenkinsfile rename jjb/nsm/{nordix-nsm-meridio-periodic.yaml => meridio-periodic.yaml} (54%) create mode 100644 jjb/nsm/meridio-pull-request.yaml delete mode 100644 jjb/nsm/nordix-nsm-meridio-verify.yaml delete mode 100755 jjb/nsm/scripts/build.sh delete mode 100755 jjb/nsm/scripts/static-analysis.sh diff --git a/jjb/nsm/Jenkinsfile b/jjb/nsm/Jenkinsfile new file mode 100644 index 00000000..72246762 --- /dev/null +++ b/jjb/nsm/Jenkinsfile @@ -0,0 +1,267 @@ +/* +Copyright (c) 2022 Nordix Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import org.jenkinsci.plugins.pipeline.modeldefinition.Utils + +pending = 'PENDING' +success = 'SUCCESS' +failure = 'FAILURE' +base_image = 'base-image' +in_progress = 'In Progress.' +completed = 'Completed.' +failed = 'Failed' + +node { + build_number = env.BUILD_NUMBER + workspace = env.WORKSPACE + ws("${workspace}/${build_number}") { + def image_names = params.IMAGE_NAMES.split(' ') + def version = params.IMAGE_VERSION + def e2e_enabled = params.E2E_ENABLED + def git_project = params.GIT_PROJECT + def current_branch = params.CURRENT_BRANCH + def default_branch = params.DEFAULT_BRANCH + def build_steps = params.BUILD_STEPS + def image_registry = params.IMAGE_REGISTRY + def local_version = "${env.JOB_NAME}-${build_number}" + + stage('Clone/Checkout') { + git branch: default_branch, url: git_project + checkout([ + $class: 'GitSCM', + branches: [[name: current_branch]], + extensions: [], + userRemoteConfigs: [[ + refspec: '+refs/pull/*/head:refs/remotes/origin/pr/*', + url: git_project + ]] + ]) + sh 'git show' + } + stage('Verify') { + Verify().call() + } + stage('Docker login') { + wrap([$class: 'MaskPasswordsBuildWrapper', varPasswordPairs: [[password: env.HARBOR_USERNAME, var: 'HARBOR_USERNAME'], [password: env.HARBOR_PASSWORD, var: 'HARBOR_PASSWORD'], [password: image_registry, var: 'IMAGE_REGISTRY']]]) { + sh '''#!/bin/bash -eu + echo ${HARBOR_PASSWORD} | docker login --username ${HARBOR_USERNAME} --password-stdin ${IMAGE_REGISTRY} + ''' + } + } + stage('Base Image') { + BaseImage(version, build_steps, image_registry, local_version).call() + } + stage('Images') { + Images(image_names, version, build_steps, image_registry, local_version).call() + } + stage('E2E') { + E2e(e2e_enabled).call() + } + stage('Cleanup') { + Cleanup() + } + } +} + +// Static analysis: Runs the GeneratedCode function and then UnitTests and Linter in parallel +def Verify() { + return { + GeneratedCode().call() // cannot generate code and run the linter and tests at the same time + // Linter().call() + // UnitTests().call() + def stages = [:] + stages.put('Unit Tests', UnitTests()) + stages.put('Linter', Linter()) + // stages.put('Generated code verification', GeneratedCode()) + parallel(stages) + } +} + +// Runs the unit tests and set the github commit status +def UnitTests() { + return { + def context = 'Unit Tests' + stage('Unit Tests') { + try { + SetBuildStatus(in_progress, context, pending) + sh 'make test' + SetBuildStatus(completed, context, success) + } catch (Exception e) { + SetBuildStatus(failed, context, failure) + Error(e).call() + } + } + } +} + +// Runs the linter and set the github commit status +def Linter() { + return { + def context = 'Linter' + stage('Linter') { + try { + SetBuildStatus(in_progress, context, pending) + sh 'make lint' + SetBuildStatus(completed, context, success) + } catch (Exception e) { + SetBuildStatus(failed, context, failure) + Error(e).call() + } + } + } +} + +// Check if code has been generated correctly and set the github commit status: +// go.mod: runs "go mod tidy" +// go generate ./...: Code should be generated using "make genrate" command +// proto: skipped due to version of protoc +// If files are generated correctly then GetModifiedFiles function should return an empty string +def GeneratedCode() { + return { + def context = 'Generated code verification' + def exception_message = 'Generated code verification failed' + SetBuildStatus(in_progress, context, pending) + stage('go mod tidy') { + try { + sh 'go mod tidy' + if (GetModifiedFiles() != '') { + throw new Exception(exception_message) + } + } catch (Exception e) { + SetBuildStatus(failed, context, failure) + sh 'git diff' + sh 'git status -s' + Error(e).call() + } + } + stage('go generate ./...') { + try { + sh 'make generate' + if (GetModifiedFiles() != '') { + throw new Exception(exception_message) + } + } catch (Exception e) { + SetBuildStatus(failed, context, failure) + sh 'git diff' + sh 'git status -s' + Error(e).call() + } + } + stage('Proto') { + // TODO: protoc version could be different + Utils.markStageSkippedForConditional('Proto') + // try { + // sh 'make proto' + // if (GetModifiedFiles() != '') { + // throw new Exception(exception_message) + // } + // } catch (Exception e) { + // SetBuildStatus(failed, context, failure) + // sh 'git diff' + // sh 'git status -s' + // Error(e).call() + // } + } + SetBuildStatus(completed, context, success) + } +} + +def BaseImage(version, build_steps, registry, local_version) { + return { + Build(base_image, version, build_steps, registry, local_version).call() + } +} + +// Call Build function for every images in parallel +def Images(images, version, build_steps, registry, local_version) { + return { + def stages = [:] + for (i in images) { + stages.put(i, Build(i, version, build_steps, registry, local_version)) + } + parallel(stages) + } +} + +// Build set the github commit status +def Build(image, version, build_steps, registry, local_version) { + return { + stage("${image} (${version}): ${build_steps}") { + def context = "Image: ${image}" + def in_progress_message = "${in_progress} (${build_steps})" + def completed_message = "${completed} (${build_steps})" + def failed_message = "${failed} (${build_steps})" + try { + SetBuildStatus(in_progress_message, context, pending) + sh "make ${image} VERSION=${version} BUILD_STEPS='${build_steps}' REGISTRY=${registry} LOCAL_VERSION=${local_version} BASE_IMAGE=${base_image}:${local_version}" + SetBuildStatus(completed_message, context, success) + } catch (Exception e) { + SetBuildStatus(failed_message, context, failure) + Error(e).call() + } + } + } +} + +// Run the E2e Tests +// Currently skipped +def E2e(e2e_enabled) { + if (e2e_enabled == 'true') { + return { + echo 'make e2e' // todo + } + } else { + return { + Utils.markStageSkippedForConditional('E2E') + } + } +} + +// Raise error in Jenkins job +def Error(e) { + return { + Cleanup() + error e + } +} + +// Cleanup directory +def Cleanup() { + cleanWs() +} + +// Set the commit status on Github +// https://plugins.jenkins.io/github/#plugin-content-pipeline-examples +def SetBuildStatus(String message, String context, String state) { + step([ + $class: 'GitHubCommitStatusSetter', + reposSource: [$class: 'ManuallyEnteredRepositorySource', url: 'https://github.com/Nordix/Meridio'], + commitShaSource: [$class: 'ManuallyEnteredShaSource', sha: GetCommitSha()], + contextSource: [$class: 'ManuallyEnteredCommitContextSource', context: context], + errorHandlers: [[$class: 'ChangingBuildStatusErrorHandler', result: 'UNSTABLE']], + statusResultSource: [ $class: 'ConditionalStatusResultSource', results: [[$class: 'AnyBuildResult', message: message, state: state]] ] + ]) +} + +// Return the current commit sha +def GetCommitSha() { + return sh(script: 'git rev-parse HEAD', returnStdout: true).trim() +} + +// Returns if any files has been modified/added/removed +def GetModifiedFiles() { + return sh(script: 'git status -s', returnStdout: true).trim() +} diff --git a/jjb/nsm/nordix-nsm-meridio-periodic.yaml b/jjb/nsm/meridio-periodic.yaml similarity index 54% rename from jjb/nsm/nordix-nsm-meridio-periodic.yaml rename to jjb/nsm/meridio-periodic.yaml index 5101ddbb..9ffe7e9d 100644 --- a/jjb/nsm/nordix-nsm-meridio-periodic.yaml +++ b/jjb/nsm/meridio-periodic.yaml @@ -18,35 +18,26 @@ # ============LICENSE_END========================================================= - project: - name: 'nsm-meridio-periodic' + name: 'meridio-periodic' - project: 'nsm-meridio-periodic' + project: 'meridio-periodic' - stream: - - 'master': - branch: 'master' - - image-name: - - ctraffic - - ipam - - load-balancer - - nsp - - proxy - - tapa - - frontend + # NOTE (fdegir): auth-id is taken from Jenkins Global Configuration + # by clicking Auth ID button for the desired GitHub Server API URL + # Please ensure you are looking at GitHub Pull Request Builder part + # of global configuration and not to GitHub Server configuration + ghprb-auth-id: 'cdfd2452-a9e2-41a3-8ee6-9058512b4aff' jobs: - - 'nordix-nsm-meridio-{image-name}-periodic-build-{stream}' + - 'meridio-periodic' - job-template: - name: 'nordix-nsm-meridio-{image-name}-periodic-build-{stream}' - + name: 'meridio-periodic' + project-type: pipeline disabled: '{obj:disabled}' concurrent: true - node: nordix-nsm-build-ubuntu1804 - properties: - github: url: https://github.com/Nordix/Meridio @@ -58,45 +49,46 @@ description: JJB configured parameter to identify GitHub Organization - string: name: PROJECT - default: 'meridio' + default: 'Meridio' description: JJB configured PROJECT parameter to identify a Nordix GitHub project - string: - name: IMAGE_NAME - default: '{image-name}' - description: Meridio image to verify build + name: IMAGE_NAMES + default: 'load-balancer proxy tapa ipam nsp ctraffic frontend' + description: Images to compile - string: name: IMAGE_VERSION default: 'latest' - description: Meridio image version to build with + description: Version of the images - string: - name: IMAGE_REGISTRY - default: 'registry.nordix.org/cloud-native/$PROJECT' - description: Meridio image version to build with + name: E2E_ENABLED + default: "false" + description: Is E2E Tests enabled? + - string: + name: GIT_PROJECT + default: "https://github.com/Nordix/Meridio.git" + description: Git URL of the project + - string: + name: CURRENT_BRANCH + default: "master" + description: Current Git branch + - string: + name: DEFAULT_BRANCH + default: "master" + description: default branch - string: name: BUILD_STEPS - default: 'build tag push' + default: "build tag push" description: Steps to run during build - - scm: - - git: - url: 'https://github.com/$GITHUB_ORGANIZATION/$PROJECT.git' - branches: - - master - timeout: 15 - wipe-workspace: true + - string: + name: IMAGE_REGISTRY + default: 'registry.nordix.org/cloud-native/meridio' + description: Meridio image regsitry triggers: - pollscm: cron: '@midnight' - wrappers: - - build-timeout: - timeout: 30 - - nordixinfra-harbor-creds-wrapper - - mask-passwords - - builders: - - shell: - !include-raw: ./scripts/build.sh + dsl: + !include-raw-escape: Jenkinsfile # vim: set ts=2 sw=2 expandtab: diff --git a/jjb/nsm/meridio-pull-request.yaml b/jjb/nsm/meridio-pull-request.yaml new file mode 100644 index 00000000..61e341f4 --- /dev/null +++ b/jjb/nsm/meridio-pull-request.yaml @@ -0,0 +1,102 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2021 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'meridio-pull-request' + + project: 'meridio-pull-request' + + # NOTE (fdegir): auth-id is taken from Jenkins Global Configuration + # by clicking Auth ID button for the desired GitHub Server API URL + # Please ensure you are looking at GitHub Pull Request Builder part + # of global configuration and not to GitHub Server configuration + ghprb-auth-id: 'cdfd2452-a9e2-41a3-8ee6-9058512b4aff' + ghprb-pull-id: ${{ghprbPullId}} + + jobs: + - 'meridio-pull-request' + +- job-template: + name: 'meridio-pull-request' + project-type: pipeline + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - github: + url: https://github.com/Nordix/Meridio + + parameters: + - string: + name: GITHUB_ORGANIZATION + default: 'Nordix' + description: JJB configured parameter to identify GitHub Organization + - string: + name: PROJECT + default: 'Meridio' + description: JJB configured PROJECT parameter to identify a Nordix GitHub project + - string: + name: IMAGE_NAMES + default: 'load-balancer proxy tapa ipam nsp ctraffic frontend' + description: Images to compile + - string: + name: IMAGE_VERSION + default: 'latest' + description: Version of the images + - string: + name: E2E_ENABLED + default: "false" + description: Is E2E Tests enabled? + - string: + name: GIT_PROJECT + default: "https://github.com/Nordix/Meridio.git" + description: Git URL of the project + - string: + name: CURRENT_BRANCH + default: "pr/{ghprb-pull-id}" + description: Current Git branch + - string: + name: DEFAULT_BRANCH + default: "master" + description: default branch + - string: + name: BUILD_STEPS + default: "build" + description: Steps to run during build + - string: + name: IMAGE_REGISTRY + default: 'registry.nordix.org/cloud-native/meridio' + description: Meridio image regsitry + + # https://opendev.org/jjb/jenkins-job-builder/src/commit/be422af6bb5edc32886a98d78340051f71244c41/jenkins_jobs/modules/triggers.py#L1235 + triggers: + - github-pull-request: + admin-list: + - LionelJouin + auth-id: '{ghprb-auth-id}' + github-hooks: true + permit-all: true + no-commit-status: true + trigger-phrase: '/reverify' + only-trigger-phrase: false + cron: 'H/5 * * * *' + + dsl: + !include-raw-escape: Jenkinsfile diff --git a/jjb/nsm/nordix-nsm-meridio-verify.yaml b/jjb/nsm/nordix-nsm-meridio-verify.yaml deleted file mode 100644 index effecc83..00000000 --- a/jjb/nsm/nordix-nsm-meridio-verify.yaml +++ /dev/null @@ -1,175 +0,0 @@ ---- -# ============LICENSE_START======================================================= -# Copyright (C) 2021 Nordix Foundation. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# ============LICENSE_END========================================================= - -- project: - name: 'nsm-meridio-verify' - - project: 'nsm-meridio-verify' - - stream: - - 'master': - branch: '${{ghprbActualCommit}}' - - # NOTE (fdegir): auth-id is taken from Jenkins Global Configuration - # by clicking Auth ID button for the desired GitHub Server API URL - # Please ensure you are looking at GitHub Pull Request Builder part - # of global configuration and not to GitHub Server configuration - ghprb-auth-id: 'cdfd2452-a9e2-41a3-8ee6-9058512b4aff' - image-name: - - ctraffic - - ipam - - load-balancer - - nsp - - proxy - - tapa - - frontend - - jobs: - - 'nordix-nsm-meridio-verify-static-analysis-{stream}' - - 'nordix-nsm-meridio-{image-name}-verify-build-{stream}' - -- job-template: - name: 'nordix-nsm-meridio-verify-static-analysis-{stream}' - - disabled: '{obj:disabled}' - - concurrent: true - - node: nordix-nsm-build-ubuntu1804 - - properties: - - github: - url: https://github.com/Nordix/Meridio - - parameters: - - string: - name: GITHUB_ORGANIZATION - default: 'Nordix' - description: JJB configured parameter to identify GitHub Organization - - string: - name: PROJECT - default: 'meridio' - description: JJB configured PROJECT parameter to identify a Nordix GitHub project - - scm: - - git: - url: 'https://github.com/$GITHUB_ORGANIZATION/$PROJECT.git' - branches: - - '{branch}' - refspec: '+refs/pull/*:refs/remotes/origin/pr/*' - timeout: 15 - wipe-workspace: true - choosing-strategy: 'default' - clean_before: false - - triggers: - - github-pull-request: - admin-list: - - nordixinfra - - uablrek - - LionelJouin - - fdegir - auth-id: '{ghprb-auth-id}' - github-hooks: true - permit-all: true - trigger-phrase: '/reverify' - only-trigger-phrase: false - cron: 'H/5 * * * *' - status-context: 'Nordix Jenkins Static Analysis' - - builders: - - shell: - !include-raw: ./scripts/static-analysis.sh - -- job-template: - name: 'nordix-nsm-meridio-{image-name}-verify-build-{stream}' - - disabled: '{obj:disabled}' - - concurrent: true - - node: nordix-nsm-build-ubuntu1804 - - properties: - - github: - url: https://github.com/Nordix/Meridio - - parameters: - - string: - name: GITHUB_ORGANIZATION - default: 'Nordix' - description: JJB configured parameter to identify GitHub Organization - - string: - name: PROJECT - default: 'meridio' - description: JJB configured PROJECT parameter to identify a Nordix GitHub project - - string: - name: IMAGE_NAME - default: '{image-name}' - description: Meridio image to verify build - - string: - name: IMAGE_VERSION - default: 'dev' - description: Meridio image version to build with - - string: - name: IMAGE_REGISTRY - default: 'registry.nordix.org/cloud-native/$PROJECT' - description: Meridio image version to build with - - string: - name: BUILD_STEPS - default: 'build tag' - description: Steps to run during build - - scm: - - git: - url: 'https://github.com/$GITHUB_ORGANIZATION/$PROJECT.git' - branches: - - '{branch}' - refspec: '+refs/pull/*:refs/remotes/origin/pr/*' - timeout: 15 - wipe-workspace: true - choosing-strategy: 'default' - clean_before: false - - triggers: - - github-pull-request: - admin-list: - - nordixinfra - - uablrek - - LionelJouin - - fdegir - auth-id: '{ghprb-auth-id}' - github-hooks: true - permit-all: true - trigger-phrase: '/reverify' - only-trigger-phrase: false - cron: 'H/5 * * * *' - status-context: 'Nordix Jenkins Image Build: {image-name}' - - wrappers: - - build-timeout: - timeout: 30 - - nordixinfra-harbor-creds-wrapper - - mask-passwords - - builders: - - shell: - !include-raw: ./scripts/build.sh - -# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/nsm/scripts/build.sh b/jjb/nsm/scripts/build.sh deleted file mode 100755 index 7a57bfa0..00000000 --- a/jjb/nsm/scripts/build.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash - -# ============LICENSE_START======================================================= -# Copyright (C) 2021 The Nordix Foundation. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# ============LICENSE_END========================================================= - -set -o errexit -set -o pipefail -set -o nounset - -# navigate to WORKSPACE which is the root of the Git clone -cd $WORKSPACE - -# git show for information purposes -echo "--------------------------------------------" -git show -echo "--------------------------------------------" - -# login to image registry -echo "Info : Logging in to $IMAGE_REGISTRY" -echo "--------------------------------------------" -echo $HARBOR_PASSWORD | docker login --username $HARBOR_USERNAME --password-stdin $IMAGE_REGISTRY -echo "--------------------------------------------" - -# build the image -# the env vars are set by the job using parameters so update them instead -echo "Info : Starting the build using make $BUILD_STEPS" -echo "--------------------------------------------" -IMAGE=$IMAGE_NAME VERSION=$IMAGE_VERSION REGISTRY=$IMAGE_REGISTRY make $BUILD_STEPS -echo "--------------------------------------------" -echo "Info : Done!" - -# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/nsm/scripts/static-analysis.sh b/jjb/nsm/scripts/static-analysis.sh deleted file mode 100755 index 57a68613..00000000 --- a/jjb/nsm/scripts/static-analysis.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# ============LICENSE_START======================================================= -# Copyright (C) 2021 The Nordix Foundation. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# ============LICENSE_END========================================================= - -set -o errexit -set -o pipefail -set -o nounset - -# source .profile -source "$HOME/.profile" - -# navigate to WORKSPACE which is the root of the Git clone -cd "$WORKSPACE" - -# git show for information purposes -echo "--------------------------------------------" -git show -echo "--------------------------------------------" - -# run lint -echo "Info : Starting lint using make check" -echo "--------------------------------------------" -make check -echo "--------------------------------------------" -echo "Info : Done!" - -# vim: set ts=2 sw=2 expandtab: -- 2.25.1