From 4926d1c13e51b8ff20871b309821be2aaa69d675 Mon Sep 17 00:00:00 2001 From: "robert.tomczyk" Date: Fri, 5 Jul 2019 09:52:58 +0100 Subject: [PATCH] OpenDaylight: Improve jenkins jobs for OpenDaylight projects - Mirror branches from OpenDaylight upstream project to Nordix. - Do not force the git user.name to be the gerrit id. Git authorship can be independent from the gerrit review ownership now. - Cache the repository on the Jenkins slave nodes when rebasing repositories. - Make sure the commit hash in the local repo is the same as in nordix Gerrit and the same as the commit hash in the upstream gerrit. - Decrease number of jobs required for project in Jenkins Change-Id: Ia0bb2094375ec3ccf14af28ce65f09ad0613fb3f --- .../global-templates-acumos-rebase.yaml | 4 - .../controller-push-upstream-jobs.yml | 7 +- .../controller/controller-rebase-jobs.yml | 7 +- ...-templates-opendaylight-push-upstream.yaml | 13 +-- .../global-templates-opendaylight-rebase.yaml | 15 ++-- .../integration-test-push-upstream-jobs.yml | 7 +- .../integration-test-rebase-jobs.yml | 7 +- .../mdsal/mdsal-push-upstream-jobs.yml | 7 +- jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml | 7 +- .../netconf/netconf-push-upstream-jobs.yml | 7 +- .../netconf/netconf-rebase-jobs.yml | 7 +- jjb/opendaylight/push-upstream-jobs.sh | 88 ++++++++++--------- jjb/opendaylight/rebase-jobs.sh | 40 +++++++++ 13 files changed, 115 insertions(+), 101 deletions(-) create mode 100755 jjb/opendaylight/rebase-jobs.sh diff --git a/jjb/acumos/global-templates-acumos-rebase.yaml b/jjb/acumos/global-templates-acumos-rebase.yaml index 5ae2c88fe..29865cfa7 100644 --- a/jjb/acumos/global-templates-acumos-rebase.yaml +++ b/jjb/acumos/global-templates-acumos-rebase.yaml @@ -12,10 +12,6 @@ parameters: - acumos-gerrit-parameters - - string: - name: PROJECT - default: '{project}' - description: JJB configured PROJECT parameter to identify a Nordix Gerrit project - string: name: NORDIX_REMOTE default: 'ssh://infra@gerrit.nordix.org:29418/acumos/{project}.git' diff --git a/jjb/opendaylight/controller/controller-push-upstream-jobs.yml b/jjb/opendaylight/controller/controller-push-upstream-jobs.yml index 57f434603..2fea72623 100644 --- a/jjb/opendaylight/controller/controller-push-upstream-jobs.yml +++ b/jjb/opendaylight/controller/controller-push-upstream-jobs.yml @@ -18,12 +18,9 @@ # ============LICENSE_END========================================================= # - project: - name: controller-push-upstream + name: opendaylight-controller-push-upstream project: controller project-name: controller - stream: - - master: - branch: '{stream}' jobs: - - 'opendaylight-{project-name}-{stream}-push-upstream' + - 'opendaylight-{project-name}-push-upstream' diff --git a/jjb/opendaylight/controller/controller-rebase-jobs.yml b/jjb/opendaylight/controller/controller-rebase-jobs.yml index a8a671944..67f66cd38 100644 --- a/jjb/opendaylight/controller/controller-rebase-jobs.yml +++ b/jjb/opendaylight/controller/controller-rebase-jobs.yml @@ -18,12 +18,9 @@ # ============LICENSE_END========================================================= # - project: - name: controller-rebase + name: opendaylight-controller-rebase project: controller project-name: controller - stream: - - master: - branch: '{stream}' jobs: - - 'opendaylight-{project-name}-{stream}-rebase' + - 'opendaylight-{project-name}-rebase' diff --git a/jjb/opendaylight/global-templates-opendaylight-push-upstream.yaml b/jjb/opendaylight/global-templates-opendaylight-push-upstream.yaml index f3915b2bb..63743d751 100644 --- a/jjb/opendaylight/global-templates-opendaylight-push-upstream.yaml +++ b/jjb/opendaylight/global-templates-opendaylight-push-upstream.yaml @@ -1,5 +1,5 @@ - job-template: - name: 'opendaylight-{project-name}-{stream}-push-upstream' + name: 'opendaylight-{project-name}-push-upstream' node: infra-ubuntu1804 @@ -11,15 +11,16 @@ - logrotate parameters: - - project-parameters: - project: 'opendaylight/{project}' - branch: '{branch}' + - string: + name: PROJECT + default: 'opendaylight/{project}' + description: JJB configured PROJECT parameter to identify a Nordix Gerrit project - nordix-gerrit-parameters scm: - git-scm-gerrit: ssh-credentials-id: nordixinfra-nordix-gerrit-ssh - branch: '{branch}' + branch: $GERRIT_BRANCH refspec: $GERRIT_REFSPEC wipe_workspace: true clean_before: false @@ -27,7 +28,7 @@ triggers: - nordix-gerrit-push-upstream: project: 'opendaylight/{project}' - branch: '{branch}' + branch: '**' files: '**' wrappers: diff --git a/jjb/opendaylight/global-templates-opendaylight-rebase.yaml b/jjb/opendaylight/global-templates-opendaylight-rebase.yaml index f7dab86c1..e6921391a 100644 --- a/jjb/opendaylight/global-templates-opendaylight-rebase.yaml +++ b/jjb/opendaylight/global-templates-opendaylight-rebase.yaml @@ -1,5 +1,5 @@ - job-template: - name: 'opendaylight-{project-name}-{stream}-rebase' + name: 'opendaylight-{project-name}-rebase' node: infra-ubuntu1804 @@ -11,9 +11,6 @@ - logrotate parameters: - - project-parameters: - project: '{project}' - branch: '{branch}' - odl-gerrit-parameters - string: name: NORDIX_REMOTE @@ -23,15 +20,15 @@ scm: - git-scm: ssh-credentials-id: nordixinfra-lfn-gerrit-ssh - branch: '{branch}' + branch: 'master' refspec: '' - wipe_workspace: true - clean_before: false + wipe_workspace: false + clean_before: true triggers: - odl-gerrit-trigger-change-merged: project: '{project}' - branch: '{branch}' + branch: '**' files: '**' - timed: "@midnight" @@ -41,7 +38,7 @@ builders: - shell: - !include-raw-escape: ../global/rebase-jobs.sh + !include-raw-escape: ./rebase-jobs.sh publishers: - notify-slack: diff --git a/jjb/opendaylight/integration-test/integration-test-push-upstream-jobs.yml b/jjb/opendaylight/integration-test/integration-test-push-upstream-jobs.yml index ed5af0c3c..e1a3d4780 100644 --- a/jjb/opendaylight/integration-test/integration-test-push-upstream-jobs.yml +++ b/jjb/opendaylight/integration-test/integration-test-push-upstream-jobs.yml @@ -18,12 +18,9 @@ # ============LICENSE_END========================================================= # - project: - name: integration-test-push-upstream + name: opendaylight-integration-test-push-upstream project: integration/test project-name: integration-test - stream: - - master: - branch: '{stream}' jobs: - - 'opendaylight-{project-name}-{stream}-push-upstream' + - 'opendaylight-{project-name}-push-upstream' diff --git a/jjb/opendaylight/integration-test/integration-test-rebase-jobs.yml b/jjb/opendaylight/integration-test/integration-test-rebase-jobs.yml index 5734206c8..40465ed42 100644 --- a/jjb/opendaylight/integration-test/integration-test-rebase-jobs.yml +++ b/jjb/opendaylight/integration-test/integration-test-rebase-jobs.yml @@ -18,12 +18,9 @@ # ============LICENSE_END========================================================= # - project: - name: integration-test-rebase + name: opendaylight-integration-test-rebase project: integration/test project-name: integration-test - stream: - - master: - branch: '{stream}' jobs: - - 'opendaylight-{project-name}-{stream}-rebase' + - 'opendaylight-{project-name}-rebase' diff --git a/jjb/opendaylight/mdsal/mdsal-push-upstream-jobs.yml b/jjb/opendaylight/mdsal/mdsal-push-upstream-jobs.yml index 9fce64697..10347b162 100644 --- a/jjb/opendaylight/mdsal/mdsal-push-upstream-jobs.yml +++ b/jjb/opendaylight/mdsal/mdsal-push-upstream-jobs.yml @@ -18,12 +18,9 @@ # ============LICENSE_END========================================================= # - project: - name: mdsal-push-upstream + name: opendaylight-mdsal-push-upstream project: mdsal project-name: mdsal - stream: - - master: - branch: '{stream}' jobs: - - 'opendaylight-{project-name}-{stream}-push-upstream' + - 'opendaylight-{project-name}-push-upstream' diff --git a/jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml b/jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml index 9053eda44..81a9561b8 100644 --- a/jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml +++ b/jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml @@ -18,12 +18,9 @@ # ============LICENSE_END========================================================= # - project: - name: mdsal-rebase + name: opendaylight-mdsal-rebase project: mdsal project-name: mdsal - stream: - - master: - branch: '{stream}' jobs: - - 'opendaylight-{project-name}-{stream}-rebase' + - 'opendaylight-{project-name}-rebase' diff --git a/jjb/opendaylight/netconf/netconf-push-upstream-jobs.yml b/jjb/opendaylight/netconf/netconf-push-upstream-jobs.yml index b6fb0d1a1..9ef7db3ad 100644 --- a/jjb/opendaylight/netconf/netconf-push-upstream-jobs.yml +++ b/jjb/opendaylight/netconf/netconf-push-upstream-jobs.yml @@ -18,12 +18,9 @@ # ============LICENSE_END========================================================= # - project: - name: netconf-push-upstream + name: opendaylight-netconf-push-upstream project: netconf project-name: netconf - stream: - - master: - branch: '{stream}' jobs: - - 'opendaylight-{project-name}-{stream}-push-upstream' + - 'opendaylight-{project-name}-push-upstream' diff --git a/jjb/opendaylight/netconf/netconf-rebase-jobs.yml b/jjb/opendaylight/netconf/netconf-rebase-jobs.yml index 2fbbcc482..e23d049dd 100644 --- a/jjb/opendaylight/netconf/netconf-rebase-jobs.yml +++ b/jjb/opendaylight/netconf/netconf-rebase-jobs.yml @@ -18,12 +18,9 @@ # ============LICENSE_END========================================================= # - project: - name: netconf-rebase + name: opendaylight-netconf-rebase project: netconf project-name: netconf - stream: - - master: - branch: '{stream}' jobs: - - 'opendaylight-{project-name}-{stream}-rebase' + - 'opendaylight-{project-name}-rebase' diff --git a/jjb/opendaylight/push-upstream-jobs.sh b/jjb/opendaylight/push-upstream-jobs.sh index 7b6fbaaaa..a7122ab8e 100755 --- a/jjb/opendaylight/push-upstream-jobs.sh +++ b/jjb/opendaylight/push-upstream-jobs.sh @@ -19,58 +19,62 @@ # ============LICENSE_END========================================================= # -# This script will take the changes that are pushed to Nordix Gerrit and push upstream to Opendaylight +# This script will take the single commit form the given Nordix Gerrit review +# and push it, and all its ancestor commit till $GERRIT_BRANCH, to OpenDaylight +# Gerrit. # Pre-requisites for script to run successfully: -# - Author name in Nordix Gerrit equals LFID i.e. need to run "git config --global user.name " -# - infra public key on build server needs to be added to your users SSH Public Keys in Opendaylight Gerrit +# - The Gerrit username in Nordix needs to match with the Gerrit username in +# OpenDaylight +# - infra public key on build server needs to be added to your users SSH +# Public Keys in OpenDaylight Gerrit # -set -o nounset -set -o pipefail +set -euxo pipefail -cd $WORKSPACE -echo "Retrieving information from commit to push towards Opendaylight" -project=$(git config --local remote.origin.url | awk -F "29418/opendaylight/" '{print $2}') -message=$(git show -s --pretty=%B | grep -vi 'Signed-off-by') -username=$(git show -s --pretty=%an) +cd "$WORKSPACE" -pattern=" " -if [[ "$username" =~ $pattern ]] -then - echo "Incorrect username, use Linux Foundation ID as git user.name when pushing to Nordix" - exit 1 -fi +opendaylight_gerrit_base='git.opendaylight.org:29418' +nordix_gerrit_rest='https://gerrit.nordix.org' -echo "Setting user name and email" -git config user.email $(git show -s --pretty=%ae) -git config user.name $username +echo >&2 "Collecting information about what and how to push towards OpenDaylight" -echo "Checking out branch on master with new changes" -git reset HEAD~1 --soft -git checkout -b delivery_branch origin/$BRANCH +# The git hash of the commit in the review this job runs on +commit_hash="$GERRIT_PATCHSET_REVISION" -git config --get remote.upstream.url -retVal=$? +# GERRIT_PROJECT is in the form of opendaylight/ +opendaylight_project=$(echo "$GERRIT_PROJECT" | cut -d/ -f2- ) -if [[ $retVal -eq 0 ]] -then - git remote rm upstream - git remote add upstream "ssh://$username@git.opendaylight.org:29418/$project.git" -else - git remote add upstream "ssh://$username@git.opendaylight.org:29418/$project.git" -fi +# Nordix Gerrit can map the email address of the user pushed the change to +# the review to the Gerrit username of the account in Nordix Gerrit. This +# should match with the Gerrit username in the OpenDaylight Gerrit. +username=$( + curl -s -H 'Accept: application/json' \ + "$nordix_gerrit_rest/accounts/?q=email:$GERRIT_EVENT_ACCOUNT_EMAIL&o=DETAILS" \ + | tee /dev/stderr | tail -1 | jq -r '.[0].username') -echo "Committing changes and pushing upstream" -git commit -as -m "$message" -git push upstream HEAD:refs/for/$BRANCH +opendaylight_remote_url="ssh://$username@$opendaylight_gerrit_base/$opendaylight_project" -retVal1=$? -if [[ $retVal1 -eq 0 ]] -then - echo "Push upstream to Opendaylight succeeded" +# GERRIT_BRANCH is the _intended_ branch of the commit under review +# E.g. git push HEAD:refs/for/master => GERRIT_BRANCH='master' +# GERRIT_TOPIC is the rest of the refspec after GERRIT_BRANCH when the topic +# is set for the change. +if [ -z "${GERRIT_TOPIC:-}" ] ; then + refspec="$commit_hash:refs/for/$GERRIT_BRANCH" else - echo "Push upstream to Opendaylight failed" - exit 2 + refspec="$commit_hash:refs/for/$GERRIT_BRANCH/$GERRIT_TOPIC" fi -git checkout $BRANCH -git branch -D delivery_branch +echo >&2 "Pushing to OpenDaylight" + +# do not fail if git push fails as in case of no new changes we want to succeed. See below. +set +o pipefail + +git push "$opendaylight_remote_url" "$refspec" 2>&1 | tee push_result.txt +push_result=${PIPESTATUS[0]} + +# make the job a success if the above git push fails due to no new changes are needed +# to be created upstream. This will be useful when we switch to automatic triggering +# of the push-upstream for each patch set and jobs on different commits in a same chain +# will race with each other to push parts of the chain +if [ $push_result -ne 0 ]; then + grep '(no new changes)' push_result.txt +fi diff --git a/jjb/opendaylight/rebase-jobs.sh b/jjb/opendaylight/rebase-jobs.sh new file mode 100755 index 000000000..8cec63a02 --- /dev/null +++ b/jjb/opendaylight/rebase-jobs.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +cd $WORKSPACE +# Make sure the Nordix is set from Jenkins Job configuration +# parameter: NORDIX_REMOTE and not from previous job execution +if git remote | grep nordix > /dev/null ; then + git remote rm nordix +fi +# When the GERRIT_BRANCH is not set the rebase job wasn't triggered by merge +# but it's executed as part of timer. In this case we will make sure that all +# branches from remote are synced with Nordix Gerrit. +if [[ -z "${GERRIT_BRANCH:-}" ]] ; then + echo "-- Fetch all branches from OpenDaylight for project: ${PROJECT} --" + REMOTE_BRANCHES=$(git branch -r | grep -v 'HEAD -> ' | awk '/origin/ {gsub("origin/","");print}') + for branch in ${REMOTE_BRANCHES} ; do + git branch -f --track ${branch} origin/${branch} + done +# The job was triggered by merge on OpenDaylight. We will make sure that branch set +# in GERRIT_BRANCH is in sync with Nordix Gerrit. +else + echo "-- Fetch branch: ${GERRIT_BRANCH} from OpenDaylight for project: ${PROJECT} --" + REMOTE_BRANCHES=${GERRIT_BRANCH} + git branch -f --track ${GERRIT_BRANCH} origin/${GERRIT_BRANCH} +fi +git fetch origin -v +echo "-- Adding new remote $NORDIX_REMOTE for sync --" +git remote add nordix $NORDIX_REMOTE +git remote -v +echo "-- Merging changes from OpenDaylight remote branches to local branch: --" +for branch in ${REMOTE_BRANCHES} ; do + echo "------ Branch name: ${branch}" + git checkout ${branch} -q + git merge origin/${branch} --ff-only --stat +done +echo "-- Pushing changes to Nordix Gerrit --" +git push nordix --all -v +echo "--------------------------------------------------------------------" -- 2.25.1