From 6cd44a7fea816effd76c19fd95ea7cdcf21c1536 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Mon, 6 Jul 2020 07:14:17 +0000 Subject: [PATCH] engine: Implement change dependency handling The new functionality enables specifying dependency between open changes sent for different projects on Gerrit. When this functionality is used, the generate-change-metadata.sh determines the change properties such as change project, revision, and refspec and sets the variables accordingly for the framework to use. In order to use this feature, the dependent change needs to be specified in commit message as below. Depends-On: For example Depends-On: Iea9c21b9898f3733aa5486932ca6d5abffd8b803 Additional notes about the feature are - the format to specify dependent change must follow the above example - the dependent change must be specified on its own line - the dependent change must be specified at the beginning of the line - only single dependency can be specified at the moment but this can be improved further if it is deemed necessary - the specified change must be open (not merged or abandoned) - only the current/latest patchset can be used (as the earlier patchsets are already outdated) - dependent change must not be on same repository as the change under test See the generate-change-metadata.sh determine_dependency function for how things work. Change-Id: I2e5cfa23fb4a46ff49a4bdf137054b11202d39eb --- jjb/engine/scripts/deploy.sh | 8 ++ .../scripts/generate-change-metadata.sh | 101 +++++++++++++----- jjb/engine/scripts/package.sh | 18 +++- jjb/engine/verify-jobs/engine-verify-tox.yaml | 1 + jjb/engine/verify-jobs/ironic-verify-tox.yaml | 1 + .../verify-jobs/kubernetes-verify-tox.yaml | 1 + jjb/engine/verify-jobs/onap-verify-tox.yaml | 1 + .../verify-jobs/openstack-verify-tox.yaml | 1 + 8 files changed, 103 insertions(+), 29 deletions(-) diff --git a/jjb/engine/scripts/deploy.sh b/jjb/engine/scripts/deploy.sh index abd241474..0302e776a 100755 --- a/jjb/engine/scripts/deploy.sh +++ b/jjb/engine/scripts/deploy.sh @@ -31,6 +31,14 @@ if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then cd "$WORKSPACE" fi +# NOTE (fdegir): this variable is set by generate-change-metadata.sh script +# if a dependency to engine repo is determined so we check out that refspec +# to ensure we respect the dependency +if [[ ! -z "${NORDIX_ENGINE_REFSPEC+x}" ]]; then + echo "Info : Fetching dependent change ${NORDIX_ENGINE_REFSPEC} infra/engine" + git fetch "https://gerrit.nordix.org/infra/engine" ${NORDIX_ENGINE_REFSPEC} && git checkout FETCH_HEAD +fi + if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then ENGINE_ROOT_DIR="/opt/engine/offline/git/engine" else diff --git a/jjb/engine/scripts/generate-change-metadata.sh b/jjb/engine/scripts/generate-change-metadata.sh index 219c186a5..ead0a7788 100755 --- a/jjb/engine/scripts/generate-change-metadata.sh +++ b/jjb/engine/scripts/generate-change-metadata.sh @@ -23,14 +23,75 @@ set -o nounset set -o errexit set -o pipefail -# NOTE (fdegir): we need to remove $WORKSPACE and clone infra/engine repository -# as infra/engine is what drives the deployment -if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then - cd "$HOME" && /bin/rm -rf "$WORKSPACE" - echo "Info : Cloning infra/engine repository" - git clone -q https://gerrit.nordix.org/infra/engine.git "$WORKSPACE" - cd "$WORKSPACE" -fi +# NOTE (fdegir): in case if a change depends on another open change in different repo +# we determine it here and override the version and refspec for that repo +function determine_dependency() { + echo "Info : Determining dependency" + echo "Info : Commit message is" + echo "-------------------------------------------------------------------------" + echo "$GERRIT_CHANGE_COMMIT_MESSAGE" + echo "-------------------------------------------------------------------------" + + if ! echo "$GERRIT_CHANGE_COMMIT_MESSAGE" | grep -qs "^Depends-On:"; then + echo "Info : No dependency found" + return + else + echo "Info : Dependency found!" + DEPENDENT_CHANGE_ID=$(echo "$GERRIT_CHANGE_COMMIT_MESSAGE" | grep "^Depends-On:" | cut -d":" -f2 | sed 's/\s*//g') + fi + + echo "Info : Processing change $DEPENDENT_CHANGE_ID" + declare -a DEPENDENT_CHANGE_METADATA + readarray -t DEPENDENT_CHANGE_METADATA < <(curl -s "https://gerrit.nordix.org/changes/?q=$DEPENDENT_CHANGE_ID&o=CURRENT_REVISION" 2>&1 \ + | grep "project\|status\|current_revision\|ref.*," | sed -e 's/\s//g' -e 's/,//g' -e 's/"//g'| cut -d: -f2) + + # NOTE (fdegir): check if the dependeny change exists + # and ignore if it doesn't + if [[ -z "${DEPENDENT_CHANGE_METADATA+x}" ]]; then + echo "Warn : Unable to determine dependent change. Ignoring dependency!" + return + fi + + DEPENDENT_CHANGE_PROJECT="${DEPENDENT_CHANGE_METADATA[0]}" + DEPENDENT_CHANGE_STATUS="${DEPENDENT_CHANGE_METADATA[1]}" + DEPENDENT_CHANGE_REVISION="${DEPENDENT_CHANGE_METADATA[2]}" + DEPENDENT_CHANGE_REFSPEC="${DEPENDENT_CHANGE_METADATA[3]}" + echo "-------------------------------------------------------------------------" + echo " DEPENDENT_CHANGE_PROJECT=$DEPENDENT_CHANGE_PROJECT" + echo " DEPENDENT_CHANGE_STATUS=$DEPENDENT_CHANGE_STATUS" + echo " DEPENDENT_CHANGE_REVISION=$DEPENDENT_CHANGE_REVISION" + echo " DEPENDENT_CHANGE_REFSPEC=$DEPENDENT_CHANGE_REFSPEC" + echo "-------------------------------------------------------------------------" + + # NOTE (fdegir): we only follow the dependency if the change status is NEW + if [[ "$DEPENDENT_CHANGE_STATUS" != "NEW" ]]; then + echo "Warn : Change is not open. Ignoring dependency!" + return + fi + + # NOTE (fdegir): we do not accept dependency of a change that is sent to same project as the dependent change + if [[ "$DEPENDENT_CHANGE_PROJECT" == "$GERRIT_PROJECT" ]]; then + echo "Warn : Change is for the same project. Ignoring dependency!" + return + fi + + DEPENDENT_COMPONENT="${DEPENDENT_CHANGE_PROJECT//*\//}" + + echo "# dependent change" >> "$CHANGE_METADATA_FILE" + + # NOTE (fdegir): if dependent change is on stack repo, we need to treat it + # differently + if [[ "$DEPENDENT_CHANGE_PROJECT" =~ /stack/ ]]; then + # set vars + echo "STACK_TYPE=$DEPENDENT_COMPONENT" >> "$CHANGE_METADATA_FILE" + echo "STACK_VERSION=$DEPENDENT_CHANGE_REVISION" >> "$CHANGE_METADATA_FILE" + echo "STACK_REFSPEC=$DEPENDENT_CHANGE_REFSPEC" >> "$CHANGE_METADATA_FILE" + else + echo "NORDIX_$(echo $DEPENDENT_COMPONENT | tr a-z A-Z)_VERSION=$DEPENDENT_CHANGE_REVISION" >> "$CHANGE_METADATA_FILE" + echo "NORDIX_$(echo $DEPENDENT_COMPONENT | tr a-z A-Z)_REFSPEC=$DEPENDENT_CHANGE_REFSPEC" >> "$CHANGE_METADATA_FILE" + fi + +} # file to record change metadata export CHANGE_METADATA_FILE="$WORKSPACE/change.properties" @@ -49,12 +110,6 @@ if [[ "$GERRIT_PROJECT" =~ /stack/ ]]; then export STACK_VERSION="$GERRIT_PATCHSET_REVISION" export STACK_REFSPEC="$GERRIT_REFSPEC" - # log info to console - echo " STACK_TYPE=$STACK_TYPE" - echo " DEPLOY_SCENARIO=$DEPLOY_SCENARIO" - echo " STACK_VERSION=$GERRIT_PATCHSET_REVISION" - echo " STACK_REFSPEC=$GERRIT_REFSPEC" - # record vars to properties file echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE" echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE" @@ -79,13 +134,6 @@ elif [[ "$GERRIT_PROJECT" =~ /installer/ ]]; then esac export STACK_VERSION="$GERRIT_BRANCH" - # log info to console - echo " STACK_TYPE=$STACK_TYPE" - echo " DEPLOY_SCENARIO=$DEPLOY_SCENARIO" - echo " STACK_VERSION=$STACK_VERSION" - echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION" - echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC" - # record vars to properties file echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE" echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE" @@ -97,13 +145,6 @@ else export STACK_TYPE=kubernetes export STACK_VERSION=$GERRIT_BRANCH - # log info to console - echo " STACK_TYPE=$STACK_TYPE" - echo " DEPLOY_SCENARIO=$DEPLOY_SCENARIO" - echo " STACK_VERSION=$STACK_VERSION" - echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION" - echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC" - # record vars to properties file echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE" echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE" @@ -134,6 +175,10 @@ NORDIX_ARM_REPO=nordix-${STACK_TYPE} NORDIX_ARTIFACT_URL=$NORDIX_ARTIFACT_URL EOF +# NOTE (fdegir): we generated the change metadata file normally until here +# we now determine the dependency as the last step to ensure we override vars +determine_dependency + echo "Info : Change metadata" echo "-------------------------------------------------------------------------" cat "$CHANGE_METADATA_FILE" diff --git a/jjb/engine/scripts/package.sh b/jjb/engine/scripts/package.sh index d63d46b06..bcebb6cde 100755 --- a/jjb/engine/scripts/package.sh +++ b/jjb/engine/scripts/package.sh @@ -22,7 +22,23 @@ set -o nounset set -o errexit set -o pipefail -source "$WORKSPACE/change.properties" +# NOTE (fdegir): we need to remove $WORKSPACE and clone infra/engine repository +# as infra/engine is what drives the deployment +if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then + cd "$HOME" && /bin/rm -rf "$WORKSPACE" + echo "Info : Cloning infra/engine repository" + git clone -q https://gerrit.nordix.org/infra/engine.git "$WORKSPACE" + cd "$WORKSPACE" +fi + +# NOTE (fdegir): this variable is set by generate-change-metadata.sh script +# if a dependency to engine repo is determined so we check out that refspec +# to ensure we respect the dependency +if [[ ! -z "${NORDIX_ENGINE_REFSPEC+x}" ]]; then + echo "Info : Fetching dependent change ${NORDIX_ENGINE_REFSPEC} infra/engine" + git fetch "https://gerrit.nordix.org/infra/engine" ${NORDIX_ENGINE_REFSPEC} && git checkout FETCH_HEAD +fi + echo "Info : Packaging stack $STACK_TYPE" cd "$WORKSPACE/engine" diff --git a/jjb/engine/verify-jobs/engine-verify-tox.yaml b/jjb/engine/verify-jobs/engine-verify-tox.yaml index d679ed906..8fa11abad 100644 --- a/jjb/engine/verify-jobs/engine-verify-tox.yaml +++ b/jjb/engine/verify-jobs/engine-verify-tox.yaml @@ -113,6 +113,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: diff --git a/jjb/engine/verify-jobs/ironic-verify-tox.yaml b/jjb/engine/verify-jobs/ironic-verify-tox.yaml index 0d7aaddce..926df5cb5 100644 --- a/jjb/engine/verify-jobs/ironic-verify-tox.yaml +++ b/jjb/engine/verify-jobs/ironic-verify-tox.yaml @@ -110,6 +110,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: diff --git a/jjb/engine/verify-jobs/kubernetes-verify-tox.yaml b/jjb/engine/verify-jobs/kubernetes-verify-tox.yaml index 0e64c81c7..b7ae50aad 100644 --- a/jjb/engine/verify-jobs/kubernetes-verify-tox.yaml +++ b/jjb/engine/verify-jobs/kubernetes-verify-tox.yaml @@ -122,6 +122,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: diff --git a/jjb/engine/verify-jobs/onap-verify-tox.yaml b/jjb/engine/verify-jobs/onap-verify-tox.yaml index 8d4979a97..bdafd2d08 100644 --- a/jjb/engine/verify-jobs/onap-verify-tox.yaml +++ b/jjb/engine/verify-jobs/onap-verify-tox.yaml @@ -116,6 +116,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: diff --git a/jjb/engine/verify-jobs/openstack-verify-tox.yaml b/jjb/engine/verify-jobs/openstack-verify-tox.yaml index 869123525..6724dcf29 100644 --- a/jjb/engine/verify-jobs/openstack-verify-tox.yaml +++ b/jjb/engine/verify-jobs/openstack-verify-tox.yaml @@ -116,6 +116,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: -- 2.25.1