From 6881a59a7e27c049d996f426e0e154eee9366d9a Mon Sep 17 00:00:00 2001 From: "robert.tomczyk" Date: Thu, 8 Oct 2020 13:50:22 +0100 Subject: [PATCH] FDIO: Fast Data Project introduction to Nordix Change-Id: Ieab38096276b2f5b0a198894c020f62bb13b2fbf --- .../global-templates-fdio-push-upstream.yaml | 40 ++++++++++ .../global-templates-fdio-rebase.yaml | 50 ++++++++++++ jjb/fast-data-project/push-upstream-jobs.sh | 79 +++++++++++++++++++ .../vpp/vpp-push-upstream-jobs.yml | 26 ++++++ jjb/fast-data-project/vpp/vpp-rebase-jobs.yml | 26 ++++++ jjb/global/jjb-macros.yaml | 38 +++++++++ 6 files changed, 259 insertions(+) create mode 100644 jjb/fast-data-project/global-templates-fdio-push-upstream.yaml create mode 100644 jjb/fast-data-project/global-templates-fdio-rebase.yaml create mode 100755 jjb/fast-data-project/push-upstream-jobs.sh create mode 100644 jjb/fast-data-project/vpp/vpp-push-upstream-jobs.yml create mode 100644 jjb/fast-data-project/vpp/vpp-rebase-jobs.yml diff --git a/jjb/fast-data-project/global-templates-fdio-push-upstream.yaml b/jjb/fast-data-project/global-templates-fdio-push-upstream.yaml new file mode 100644 index 000000000..b3e4d9a52 --- /dev/null +++ b/jjb/fast-data-project/global-templates-fdio-push-upstream.yaml @@ -0,0 +1,40 @@ +- job-template: + name: 'fdio-{project-name}-push-upstream' + node: infra-ubuntu1804 + disabled: false + concurrent: false + + properties: + - logrotate + + parameters: + - string: + name: PROJECT + default: 'fdio/{project}' + description: JJB configured PROJECT parameter to identify a Nordix Gerrit fast data project + - nordix-gerrit-parameters + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: $GERRIT_BRANCH + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false + + triggers: + - nordix-gerrit-push-upstream: + project: 'fdio/{project}' + branch: '**' + files: '**' + + wrappers: + - build-timeout: + timeout: 10 + - ssh-agent-credentials: + users: + - 'infra-upstream-gerrit-ssh-passphrase' + + builders: + - shell: + !include-raw-escape: ./push-upstream-jobs.sh diff --git a/jjb/fast-data-project/global-templates-fdio-rebase.yaml b/jjb/fast-data-project/global-templates-fdio-rebase.yaml new file mode 100644 index 000000000..d4a6cebc6 --- /dev/null +++ b/jjb/fast-data-project/global-templates-fdio-rebase.yaml @@ -0,0 +1,50 @@ +- job-template: + name: 'fdio-{project-name}-rebase' + node: infra-ubuntu1804 + disabled: false + concurrent: false + + properties: + - logrotate + + parameters: + - fdio-gerrit-parameters + - string: + name: PROJECT + default: '{project}' + description: JJB configured PROJECT parameter to identify a Nordix Gerrit fast data project + - string: + name: NORDIX_REMOTE + default: 'ssh://infra@gerrit.nordix.org:29418/fdio/{project}.git' + description: The remote to add for Nordix. + + scm: + - git-scm: + ssh-credentials-id: nordixinfra-lfn-gerrit-ssh + branch: 'master' + refspec: '' + wipe_workspace: false + clean_before: true + + triggers: + - fdio-gerrit-trigger-change-merged: + project: '{project}' + branch: '**' + files: '**' + - timed: "@midnight" + + wrappers: + - build-timeout: + timeout: 10 + + builders: + - shell: + !include-raw-escape: ../global/rebase-jobs.sh + + publishers: + - notify-slack: + slack-channel: '#infra' + notify-start: false + notify-success: false + notify-failure: true + notify-back-to-normal: true diff --git a/jjb/fast-data-project/push-upstream-jobs.sh b/jjb/fast-data-project/push-upstream-jobs.sh new file mode 100755 index 000000000..810e1b309 --- /dev/null +++ b/jjb/fast-data-project/push-upstream-jobs.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# ============LICENSE_START======================================================= +# Copyright (C) 2020 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +# +# This script will take the single commit form the given Nordix Gerrit review +# and push it, and all its ancestor commit till $GERRIT_BRANCH, to FDio Gerrit. +# Pre-requisites for script to run successfully: +# - The Gerrit username in Nordix needs to match with the Gerrit +# username in FDio Gerrit +# - infra public key on build server needs to be added to your users SSH +# Public Keys in FDio Gerrit +# +set -euxo pipefail + +cd "$WORKSPACE" + +fdio_gerrit_base='gerrit.fd.io:29418' +nordix_gerrit_rest='https://gerrit.nordix.org' + +echo >&2 "Collecting information about what and how to push towards Fast Data Project (FDio)" + +# The git hash of the commit in the review this job runs on +commit_hash="$GERRIT_PATCHSET_REVISION" + +# GERRIT_PROJECT is in the form of fdio/ +fdio_project=$(echo "$GERRIT_PROJECT" | cut -d/ -f2- ) + +# Nordix Gerrit can map the email address of the user pushed the change to +# the review to the Gerrit username of the account in Nordix Gerrit. This +# should match with the Gerrit username in the FDio Gerrit. +username=$( + curl -s -H 'Accept: application/json' \ + "$nordix_gerrit_rest/accounts/?q=email:$GERRIT_EVENT_ACCOUNT_EMAIL&o=DETAILS" \ + | tee /dev/stderr | tail -1 | jq -r '.[0].username') + +fdio_remote_url="ssh://$username@$fdio_gerrit_base/$fdio_project" + +# GERRIT_BRANCH is the _intended_ branch of the commit under review +# E.g. git push HEAD:refs/for/master => GERRIT_BRANCH='master' +# GERRIT_TOPIC is the rest of the refspec after GERRIT_BRANCH when the topic +# is set for the change. +if [ -z "${GERRIT_TOPIC:-}" ] ; then + refspec="$commit_hash:refs/for/$GERRIT_BRANCH" +else + refspec="$commit_hash:refs/for/$GERRIT_BRANCH/$GERRIT_TOPIC" +fi + +echo >&2 "Pushing to FDio Gerrit" + +# do not fail if git push fails as in case of no new changes we want to succeed. See below. +set +o pipefail + +git push "$fdio_remote_url" "$refspec" 2>&1 | tee push_result.txt +push_result=${PIPESTATUS[0]} + +# make the job a success if the above git push fails due to no new changes are needed +# to be created upstream. This will be useful when we switch to automatic triggering +# of the push-upstream for each patch set and jobs on different commits in a same chain +# will race with each other to push parts of the chain +if [ $push_result -ne 0 ]; then + grep '(no new changes)' push_result.txt +fi diff --git a/jjb/fast-data-project/vpp/vpp-push-upstream-jobs.yml b/jjb/fast-data-project/vpp/vpp-push-upstream-jobs.yml new file mode 100644 index 000000000..4b7689948 --- /dev/null +++ b/jjb/fast-data-project/vpp/vpp-push-upstream-jobs.yml @@ -0,0 +1,26 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2020 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +# +- project: + name: fdio-vpp-push-upstream + project: fdio/vpp + project-name: vpp + + jobs: + - 'fdio-{project-name}-push-upstream' \ No newline at end of file diff --git a/jjb/fast-data-project/vpp/vpp-rebase-jobs.yml b/jjb/fast-data-project/vpp/vpp-rebase-jobs.yml new file mode 100644 index 000000000..d243e536e --- /dev/null +++ b/jjb/fast-data-project/vpp/vpp-rebase-jobs.yml @@ -0,0 +1,26 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2020 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +# +- project: + name: fdio-vpp-rebase + project: fdio/vpp + project-name: vpp + + jobs: + - 'fdio-{project-name}-rebase' \ No newline at end of file diff --git a/jjb/global/jjb-macros.yaml b/jjb/global/jjb-macros.yaml index c98268cf7..eaaa69e61 100644 --- a/jjb/global/jjb-macros.yaml +++ b/jjb/global/jjb-macros.yaml @@ -126,6 +126,22 @@ default: ssh://$GERRIT_USER@gerrit.o-ran-sc.org:29418 description: SSH URL to Project Repo on ORAN-SC Gerrit to clone repo +- parameter: + name: fdio-gerrit-parameters + parameters: + - string: + name: GERRIT_USER + default: nordix + description: Non-interactive user to stream Gerrit Events on Fast Data Project Gerrit + - string: + name: GIT_BASE_HTTPS + default: https://gerrit.fd.io/r + description: HTTPS URL to Project Repo on Fast Data Project Gerrit to clone repo + - string: + name: GIT_BASE_SSH + default: ssh://$GERRIT_USER@gerrit.fd.io:29418 + description: SSH URL to Project Repo on Fast Data Project Gerrit to clone repo + # Verify job for Geode git tags to make sure there are in agreed format - builder: name: 'verify-geode-tag-macro' @@ -408,6 +424,28 @@ silent: true silent-start: true +# triggers for Fast Data Project Gerrit +- trigger: + name: fdio-gerrit-trigger-change-merged + triggers: + - gerrit: + server-name: 'FDio Gerrit' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: 'ANT' + pattern: '{files}' + silent: true + silent-start: true + # wrappers to use for jobs - wrapper: name: nordixinfra-jjb-creds-wrapper -- 2.25.1