OpenDaylight: Improve jenkins jobs for OpenDaylight projects
- Mirror branches from OpenDaylight upstream project to Nordix.
- Do not force the git user.name to be the gerrit id.
Git authorship can be independent from the
gerrit review ownership now.
- Cache the repository on the Jenkins slave nodes
when rebasing repositories.
- Make sure the commit hash in the local repo is the same
as in nordix Gerrit and the same as the commit hash
in the upstream gerrit.
- Decrease number of jobs required for project in Jenkins
Change-Id: Ia0bb2094375ec3ccf14af28ce65f09ad0613fb3f
diff --git a/jjb/acumos/global-templates-acumos-rebase.yaml b/jjb/acumos/global-templates-acumos-rebase.yaml
index 5ae2c88..29865cf 100644
--- a/jjb/acumos/global-templates-acumos-rebase.yaml
+++ b/jjb/acumos/global-templates-acumos-rebase.yaml
@@ -13,10 +13,6 @@
parameters:
- acumos-gerrit-parameters
- string:
- name: PROJECT
- default: '{project}'
- description: JJB configured PROJECT parameter to identify a Nordix Gerrit project
- - string:
name: NORDIX_REMOTE
default: 'ssh://infra@gerrit.nordix.org:29418/acumos/{project}.git'
description: The remote to add for Nordix.
diff --git a/jjb/opendaylight/controller/controller-push-upstream-jobs.yml b/jjb/opendaylight/controller/controller-push-upstream-jobs.yml
index 57f4346..2fea726 100644
--- a/jjb/opendaylight/controller/controller-push-upstream-jobs.yml
+++ b/jjb/opendaylight/controller/controller-push-upstream-jobs.yml
@@ -18,12 +18,9 @@
# ============LICENSE_END=========================================================
#
- project:
- name: controller-push-upstream
+ name: opendaylight-controller-push-upstream
project: controller
project-name: controller
- stream:
- - master:
- branch: '{stream}'
jobs:
- - 'opendaylight-{project-name}-{stream}-push-upstream'
+ - 'opendaylight-{project-name}-push-upstream'
diff --git a/jjb/opendaylight/controller/controller-rebase-jobs.yml b/jjb/opendaylight/controller/controller-rebase-jobs.yml
index a8a6719..67f66cd 100644
--- a/jjb/opendaylight/controller/controller-rebase-jobs.yml
+++ b/jjb/opendaylight/controller/controller-rebase-jobs.yml
@@ -18,12 +18,9 @@
# ============LICENSE_END=========================================================
#
- project:
- name: controller-rebase
+ name: opendaylight-controller-rebase
project: controller
project-name: controller
- stream:
- - master:
- branch: '{stream}'
jobs:
- - 'opendaylight-{project-name}-{stream}-rebase'
+ - 'opendaylight-{project-name}-rebase'
diff --git a/jjb/opendaylight/global-templates-opendaylight-push-upstream.yaml b/jjb/opendaylight/global-templates-opendaylight-push-upstream.yaml
index f3915b2..63743d7 100644
--- a/jjb/opendaylight/global-templates-opendaylight-push-upstream.yaml
+++ b/jjb/opendaylight/global-templates-opendaylight-push-upstream.yaml
@@ -1,5 +1,5 @@
- job-template:
- name: 'opendaylight-{project-name}-{stream}-push-upstream'
+ name: 'opendaylight-{project-name}-push-upstream'
node: infra-ubuntu1804
@@ -11,15 +11,16 @@
- logrotate
parameters:
- - project-parameters:
- project: 'opendaylight/{project}'
- branch: '{branch}'
+ - string:
+ name: PROJECT
+ default: 'opendaylight/{project}'
+ description: JJB configured PROJECT parameter to identify a Nordix Gerrit project
- nordix-gerrit-parameters
scm:
- git-scm-gerrit:
ssh-credentials-id: nordixinfra-nordix-gerrit-ssh
- branch: '{branch}'
+ branch: $GERRIT_BRANCH
refspec: $GERRIT_REFSPEC
wipe_workspace: true
clean_before: false
@@ -27,7 +28,7 @@
triggers:
- nordix-gerrit-push-upstream:
project: 'opendaylight/{project}'
- branch: '{branch}'
+ branch: '**'
files: '**'
wrappers:
diff --git a/jjb/opendaylight/global-templates-opendaylight-rebase.yaml b/jjb/opendaylight/global-templates-opendaylight-rebase.yaml
index f7dab86..e692139 100644
--- a/jjb/opendaylight/global-templates-opendaylight-rebase.yaml
+++ b/jjb/opendaylight/global-templates-opendaylight-rebase.yaml
@@ -1,5 +1,5 @@
- job-template:
- name: 'opendaylight-{project-name}-{stream}-rebase'
+ name: 'opendaylight-{project-name}-rebase'
node: infra-ubuntu1804
@@ -11,9 +11,6 @@
- logrotate
parameters:
- - project-parameters:
- project: '{project}'
- branch: '{branch}'
- odl-gerrit-parameters
- string:
name: NORDIX_REMOTE
@@ -23,15 +20,15 @@
scm:
- git-scm:
ssh-credentials-id: nordixinfra-lfn-gerrit-ssh
- branch: '{branch}'
+ branch: 'master'
refspec: ''
- wipe_workspace: true
- clean_before: false
+ wipe_workspace: false
+ clean_before: true
triggers:
- odl-gerrit-trigger-change-merged:
project: '{project}'
- branch: '{branch}'
+ branch: '**'
files: '**'
- timed: "@midnight"
@@ -41,7 +38,7 @@
builders:
- shell:
- !include-raw-escape: ../global/rebase-jobs.sh
+ !include-raw-escape: ./rebase-jobs.sh
publishers:
- notify-slack:
diff --git a/jjb/opendaylight/integration-test/integration-test-push-upstream-jobs.yml b/jjb/opendaylight/integration-test/integration-test-push-upstream-jobs.yml
index ed5af0c..e1a3d47 100644
--- a/jjb/opendaylight/integration-test/integration-test-push-upstream-jobs.yml
+++ b/jjb/opendaylight/integration-test/integration-test-push-upstream-jobs.yml
@@ -18,12 +18,9 @@
# ============LICENSE_END=========================================================
#
- project:
- name: integration-test-push-upstream
+ name: opendaylight-integration-test-push-upstream
project: integration/test
project-name: integration-test
- stream:
- - master:
- branch: '{stream}'
jobs:
- - 'opendaylight-{project-name}-{stream}-push-upstream'
+ - 'opendaylight-{project-name}-push-upstream'
diff --git a/jjb/opendaylight/integration-test/integration-test-rebase-jobs.yml b/jjb/opendaylight/integration-test/integration-test-rebase-jobs.yml
index 5734206..40465ed 100644
--- a/jjb/opendaylight/integration-test/integration-test-rebase-jobs.yml
+++ b/jjb/opendaylight/integration-test/integration-test-rebase-jobs.yml
@@ -18,12 +18,9 @@
# ============LICENSE_END=========================================================
#
- project:
- name: integration-test-rebase
+ name: opendaylight-integration-test-rebase
project: integration/test
project-name: integration-test
- stream:
- - master:
- branch: '{stream}'
jobs:
- - 'opendaylight-{project-name}-{stream}-rebase'
+ - 'opendaylight-{project-name}-rebase'
diff --git a/jjb/opendaylight/mdsal/mdsal-push-upstream-jobs.yml b/jjb/opendaylight/mdsal/mdsal-push-upstream-jobs.yml
index 9fce646..10347b1 100644
--- a/jjb/opendaylight/mdsal/mdsal-push-upstream-jobs.yml
+++ b/jjb/opendaylight/mdsal/mdsal-push-upstream-jobs.yml
@@ -18,12 +18,9 @@
# ============LICENSE_END=========================================================
#
- project:
- name: mdsal-push-upstream
+ name: opendaylight-mdsal-push-upstream
project: mdsal
project-name: mdsal
- stream:
- - master:
- branch: '{stream}'
jobs:
- - 'opendaylight-{project-name}-{stream}-push-upstream'
+ - 'opendaylight-{project-name}-push-upstream'
diff --git a/jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml b/jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml
index 9053eda..81a9561 100644
--- a/jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml
+++ b/jjb/opendaylight/mdsal/mdsal-rebase-jobs.yml
@@ -18,12 +18,9 @@
# ============LICENSE_END=========================================================
#
- project:
- name: mdsal-rebase
+ name: opendaylight-mdsal-rebase
project: mdsal
project-name: mdsal
- stream:
- - master:
- branch: '{stream}'
jobs:
- - 'opendaylight-{project-name}-{stream}-rebase'
+ - 'opendaylight-{project-name}-rebase'
diff --git a/jjb/opendaylight/netconf/netconf-push-upstream-jobs.yml b/jjb/opendaylight/netconf/netconf-push-upstream-jobs.yml
index b6fb0d1..9ef7db3 100644
--- a/jjb/opendaylight/netconf/netconf-push-upstream-jobs.yml
+++ b/jjb/opendaylight/netconf/netconf-push-upstream-jobs.yml
@@ -18,12 +18,9 @@
# ============LICENSE_END=========================================================
#
- project:
- name: netconf-push-upstream
+ name: opendaylight-netconf-push-upstream
project: netconf
project-name: netconf
- stream:
- - master:
- branch: '{stream}'
jobs:
- - 'opendaylight-{project-name}-{stream}-push-upstream'
+ - 'opendaylight-{project-name}-push-upstream'
diff --git a/jjb/opendaylight/netconf/netconf-rebase-jobs.yml b/jjb/opendaylight/netconf/netconf-rebase-jobs.yml
index 2fbbcc4..e23d049 100644
--- a/jjb/opendaylight/netconf/netconf-rebase-jobs.yml
+++ b/jjb/opendaylight/netconf/netconf-rebase-jobs.yml
@@ -18,12 +18,9 @@
# ============LICENSE_END=========================================================
#
- project:
- name: netconf-rebase
+ name: opendaylight-netconf-rebase
project: netconf
project-name: netconf
- stream:
- - master:
- branch: '{stream}'
jobs:
- - 'opendaylight-{project-name}-{stream}-rebase'
+ - 'opendaylight-{project-name}-rebase'
diff --git a/jjb/opendaylight/push-upstream-jobs.sh b/jjb/opendaylight/push-upstream-jobs.sh
index 7b6fbaa..a7122ab 100755
--- a/jjb/opendaylight/push-upstream-jobs.sh
+++ b/jjb/opendaylight/push-upstream-jobs.sh
@@ -19,58 +19,62 @@
# ============LICENSE_END=========================================================
#
-# This script will take the changes that are pushed to Nordix Gerrit and push upstream to Opendaylight
+# This script will take the single commit form the given Nordix Gerrit review
+# and push it, and all its ancestor commit till $GERRIT_BRANCH, to OpenDaylight
+# Gerrit.
# Pre-requisites for script to run successfully:
-# - Author name in Nordix Gerrit equals LFID i.e. need to run "git config --global user.name <LFID>"
-# - infra public key on build server needs to be added to your users SSH Public Keys in Opendaylight Gerrit
+# - The Gerrit username in Nordix needs to match with the Gerrit username in
+# OpenDaylight
+# - infra public key on build server needs to be added to your users SSH
+# Public Keys in OpenDaylight Gerrit
#
-set -o nounset
-set -o pipefail
+set -euxo pipefail
-cd $WORKSPACE
-echo "Retrieving information from commit to push towards Opendaylight"
-project=$(git config --local remote.origin.url | awk -F "29418/opendaylight/" '{print $2}')
-message=$(git show -s --pretty=%B | grep -vi 'Signed-off-by')
-username=$(git show -s --pretty=%an)
+cd "$WORKSPACE"
-pattern=" "
-if [[ "$username" =~ $pattern ]]
-then
- echo "Incorrect username, use Linux Foundation ID as git user.name when pushing to Nordix"
- exit 1
-fi
+opendaylight_gerrit_base='git.opendaylight.org:29418'
+nordix_gerrit_rest='https://gerrit.nordix.org'
-echo "Setting user name and email"
-git config user.email $(git show -s --pretty=%ae)
-git config user.name $username
+echo >&2 "Collecting information about what and how to push towards OpenDaylight"
-echo "Checking out branch on master with new changes"
-git reset HEAD~1 --soft
-git checkout -b delivery_branch origin/$BRANCH
+# The git hash of the commit in the review this job runs on
+commit_hash="$GERRIT_PATCHSET_REVISION"
-git config --get remote.upstream.url
-retVal=$?
+# GERRIT_PROJECT is in the form of opendaylight/<opendaylight-project>
+opendaylight_project=$(echo "$GERRIT_PROJECT" | cut -d/ -f2- )
-if [[ $retVal -eq 0 ]]
-then
- git remote rm upstream
- git remote add upstream "ssh://$username@git.opendaylight.org:29418/$project.git"
+# Nordix Gerrit can map the email address of the user pushed the change to
+# the review to the Gerrit username of the account in Nordix Gerrit. This
+# should match with the Gerrit username in the OpenDaylight Gerrit.
+username=$(
+ curl -s -H 'Accept: application/json' \
+ "$nordix_gerrit_rest/accounts/?q=email:$GERRIT_EVENT_ACCOUNT_EMAIL&o=DETAILS" \
+ | tee /dev/stderr | tail -1 | jq -r '.[0].username')
+
+opendaylight_remote_url="ssh://$username@$opendaylight_gerrit_base/$opendaylight_project"
+
+# GERRIT_BRANCH is the _intended_ branch of the commit under review
+# E.g. git push HEAD:refs/for/master => GERRIT_BRANCH='master'
+# GERRIT_TOPIC is the rest of the refspec after GERRIT_BRANCH when the topic
+# is set for the change.
+if [ -z "${GERRIT_TOPIC:-}" ] ; then
+ refspec="$commit_hash:refs/for/$GERRIT_BRANCH"
else
- git remote add upstream "ssh://$username@git.opendaylight.org:29418/$project.git"
+ refspec="$commit_hash:refs/for/$GERRIT_BRANCH/$GERRIT_TOPIC"
fi
-echo "Committing changes and pushing upstream"
-git commit -as -m "$message"
-git push upstream HEAD:refs/for/$BRANCH
+echo >&2 "Pushing to OpenDaylight"
-retVal1=$?
-if [[ $retVal1 -eq 0 ]]
-then
- echo "Push upstream to Opendaylight succeeded"
-else
- echo "Push upstream to Opendaylight failed"
- exit 2
+# do not fail if git push fails as in case of no new changes we want to succeed. See below.
+set +o pipefail
+
+git push "$opendaylight_remote_url" "$refspec" 2>&1 | tee push_result.txt
+push_result=${PIPESTATUS[0]}
+
+# make the job a success if the above git push fails due to no new changes are needed
+# to be created upstream. This will be useful when we switch to automatic triggering
+# of the push-upstream for each patch set and jobs on different commits in a same chain
+# will race with each other to push parts of the chain
+if [ $push_result -ne 0 ]; then
+ grep '(no new changes)' push_result.txt
fi
-
-git checkout $BRANCH
-git branch -D delivery_branch
diff --git a/jjb/opendaylight/rebase-jobs.sh b/jjb/opendaylight/rebase-jobs.sh
new file mode 100755
index 0000000..8cec63a
--- /dev/null
+++ b/jjb/opendaylight/rebase-jobs.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+
+cd $WORKSPACE
+# Make sure the Nordix is set from Jenkins Job configuration
+# parameter: NORDIX_REMOTE and not from previous job execution
+if git remote | grep nordix > /dev/null ; then
+ git remote rm nordix
+fi
+# When the GERRIT_BRANCH is not set the rebase job wasn't triggered by merge
+# but it's executed as part of timer. In this case we will make sure that all
+# branches from remote are synced with Nordix Gerrit.
+if [[ -z "${GERRIT_BRANCH:-}" ]] ; then
+ echo "-- Fetch all branches from OpenDaylight for project: ${PROJECT} --"
+ REMOTE_BRANCHES=$(git branch -r | grep -v 'HEAD -> ' | awk '/origin/ {gsub("origin/","");print}')
+ for branch in ${REMOTE_BRANCHES} ; do
+ git branch -f --track ${branch} origin/${branch}
+ done
+# The job was triggered by merge on OpenDaylight. We will make sure that branch set
+# in GERRIT_BRANCH is in sync with Nordix Gerrit.
+else
+ echo "-- Fetch branch: ${GERRIT_BRANCH} from OpenDaylight for project: ${PROJECT} --"
+ REMOTE_BRANCHES=${GERRIT_BRANCH}
+ git branch -f --track ${GERRIT_BRANCH} origin/${GERRIT_BRANCH}
+fi
+git fetch origin -v
+echo "-- Adding new remote $NORDIX_REMOTE for sync --"
+git remote add nordix $NORDIX_REMOTE
+git remote -v
+echo "-- Merging changes from OpenDaylight remote branches to local branch: --"
+for branch in ${REMOTE_BRANCHES} ; do
+ echo "------ Branch name: ${branch}"
+ git checkout ${branch} -q
+ git merge origin/${branch} --ff-only --stat
+done
+echo "-- Pushing changes to Nordix Gerrit --"
+git push nordix --all -v
+echo "--------------------------------------------------------------------"