From: Fatih Degirmenci Date: Mon, 6 Jul 2020 07:14:17 +0000 (+0000) Subject: engine: Implement change dependency handling X-Git-Url: https://gerrit.nordix.org/gitweb?a=commitdiff_plain;ds=inline;h=6cd44a7fea816effd76c19fd95ea7cdcf21c1536;p=infra%2Fcicd.git engine: Implement change dependency handling The new functionality enables specifying dependency between open changes sent for different projects on Gerrit. When this functionality is used, the generate-change-metadata.sh determines the change properties such as change project, revision, and refspec and sets the variables accordingly for the framework to use. In order to use this feature, the dependent change needs to be specified in commit message as below. Depends-On: For example Depends-On: Iea9c21b9898f3733aa5486932ca6d5abffd8b803 Additional notes about the feature are - the format to specify dependent change must follow the above example - the dependent change must be specified on its own line - the dependent change must be specified at the beginning of the line - only single dependency can be specified at the moment but this can be improved further if it is deemed necessary - the specified change must be open (not merged or abandoned) - only the current/latest patchset can be used (as the earlier patchsets are already outdated) - dependent change must not be on same repository as the change under test See the generate-change-metadata.sh determine_dependency function for how things work. Change-Id: I2e5cfa23fb4a46ff49a4bdf137054b11202d39eb --- diff --git a/jjb/engine/scripts/deploy.sh b/jjb/engine/scripts/deploy.sh index abd24147..0302e776 100755 --- a/jjb/engine/scripts/deploy.sh +++ b/jjb/engine/scripts/deploy.sh @@ -31,6 +31,14 @@ if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then cd "$WORKSPACE" fi +# NOTE (fdegir): this variable is set by generate-change-metadata.sh script +# if a dependency to engine repo is determined so we check out that refspec +# to ensure we respect the dependency +if [[ ! -z "${NORDIX_ENGINE_REFSPEC+x}" ]]; then + echo "Info : Fetching dependent change ${NORDIX_ENGINE_REFSPEC} infra/engine" + git fetch "https://gerrit.nordix.org/infra/engine" ${NORDIX_ENGINE_REFSPEC} && git checkout FETCH_HEAD +fi + if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then ENGINE_ROOT_DIR="/opt/engine/offline/git/engine" else diff --git a/jjb/engine/scripts/generate-change-metadata.sh b/jjb/engine/scripts/generate-change-metadata.sh index 219c186a..ead0a778 100755 --- a/jjb/engine/scripts/generate-change-metadata.sh +++ b/jjb/engine/scripts/generate-change-metadata.sh @@ -23,14 +23,75 @@ set -o nounset set -o errexit set -o pipefail -# NOTE (fdegir): we need to remove $WORKSPACE and clone infra/engine repository -# as infra/engine is what drives the deployment -if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then - cd "$HOME" && /bin/rm -rf "$WORKSPACE" - echo "Info : Cloning infra/engine repository" - git clone -q https://gerrit.nordix.org/infra/engine.git "$WORKSPACE" - cd "$WORKSPACE" -fi +# NOTE (fdegir): in case if a change depends on another open change in different repo +# we determine it here and override the version and refspec for that repo +function determine_dependency() { + echo "Info : Determining dependency" + echo "Info : Commit message is" + echo "-------------------------------------------------------------------------" + echo "$GERRIT_CHANGE_COMMIT_MESSAGE" + echo "-------------------------------------------------------------------------" + + if ! echo "$GERRIT_CHANGE_COMMIT_MESSAGE" | grep -qs "^Depends-On:"; then + echo "Info : No dependency found" + return + else + echo "Info : Dependency found!" + DEPENDENT_CHANGE_ID=$(echo "$GERRIT_CHANGE_COMMIT_MESSAGE" | grep "^Depends-On:" | cut -d":" -f2 | sed 's/\s*//g') + fi + + echo "Info : Processing change $DEPENDENT_CHANGE_ID" + declare -a DEPENDENT_CHANGE_METADATA + readarray -t DEPENDENT_CHANGE_METADATA < <(curl -s "https://gerrit.nordix.org/changes/?q=$DEPENDENT_CHANGE_ID&o=CURRENT_REVISION" 2>&1 \ + | grep "project\|status\|current_revision\|ref.*," | sed -e 's/\s//g' -e 's/,//g' -e 's/"//g'| cut -d: -f2) + + # NOTE (fdegir): check if the dependeny change exists + # and ignore if it doesn't + if [[ -z "${DEPENDENT_CHANGE_METADATA+x}" ]]; then + echo "Warn : Unable to determine dependent change. Ignoring dependency!" + return + fi + + DEPENDENT_CHANGE_PROJECT="${DEPENDENT_CHANGE_METADATA[0]}" + DEPENDENT_CHANGE_STATUS="${DEPENDENT_CHANGE_METADATA[1]}" + DEPENDENT_CHANGE_REVISION="${DEPENDENT_CHANGE_METADATA[2]}" + DEPENDENT_CHANGE_REFSPEC="${DEPENDENT_CHANGE_METADATA[3]}" + echo "-------------------------------------------------------------------------" + echo " DEPENDENT_CHANGE_PROJECT=$DEPENDENT_CHANGE_PROJECT" + echo " DEPENDENT_CHANGE_STATUS=$DEPENDENT_CHANGE_STATUS" + echo " DEPENDENT_CHANGE_REVISION=$DEPENDENT_CHANGE_REVISION" + echo " DEPENDENT_CHANGE_REFSPEC=$DEPENDENT_CHANGE_REFSPEC" + echo "-------------------------------------------------------------------------" + + # NOTE (fdegir): we only follow the dependency if the change status is NEW + if [[ "$DEPENDENT_CHANGE_STATUS" != "NEW" ]]; then + echo "Warn : Change is not open. Ignoring dependency!" + return + fi + + # NOTE (fdegir): we do not accept dependency of a change that is sent to same project as the dependent change + if [[ "$DEPENDENT_CHANGE_PROJECT" == "$GERRIT_PROJECT" ]]; then + echo "Warn : Change is for the same project. Ignoring dependency!" + return + fi + + DEPENDENT_COMPONENT="${DEPENDENT_CHANGE_PROJECT//*\//}" + + echo "# dependent change" >> "$CHANGE_METADATA_FILE" + + # NOTE (fdegir): if dependent change is on stack repo, we need to treat it + # differently + if [[ "$DEPENDENT_CHANGE_PROJECT" =~ /stack/ ]]; then + # set vars + echo "STACK_TYPE=$DEPENDENT_COMPONENT" >> "$CHANGE_METADATA_FILE" + echo "STACK_VERSION=$DEPENDENT_CHANGE_REVISION" >> "$CHANGE_METADATA_FILE" + echo "STACK_REFSPEC=$DEPENDENT_CHANGE_REFSPEC" >> "$CHANGE_METADATA_FILE" + else + echo "NORDIX_$(echo $DEPENDENT_COMPONENT | tr a-z A-Z)_VERSION=$DEPENDENT_CHANGE_REVISION" >> "$CHANGE_METADATA_FILE" + echo "NORDIX_$(echo $DEPENDENT_COMPONENT | tr a-z A-Z)_REFSPEC=$DEPENDENT_CHANGE_REFSPEC" >> "$CHANGE_METADATA_FILE" + fi + +} # file to record change metadata export CHANGE_METADATA_FILE="$WORKSPACE/change.properties" @@ -49,12 +110,6 @@ if [[ "$GERRIT_PROJECT" =~ /stack/ ]]; then export STACK_VERSION="$GERRIT_PATCHSET_REVISION" export STACK_REFSPEC="$GERRIT_REFSPEC" - # log info to console - echo " STACK_TYPE=$STACK_TYPE" - echo " DEPLOY_SCENARIO=$DEPLOY_SCENARIO" - echo " STACK_VERSION=$GERRIT_PATCHSET_REVISION" - echo " STACK_REFSPEC=$GERRIT_REFSPEC" - # record vars to properties file echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE" echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE" @@ -79,13 +134,6 @@ elif [[ "$GERRIT_PROJECT" =~ /installer/ ]]; then esac export STACK_VERSION="$GERRIT_BRANCH" - # log info to console - echo " STACK_TYPE=$STACK_TYPE" - echo " DEPLOY_SCENARIO=$DEPLOY_SCENARIO" - echo " STACK_VERSION=$STACK_VERSION" - echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION" - echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC" - # record vars to properties file echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE" echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE" @@ -97,13 +145,6 @@ else export STACK_TYPE=kubernetes export STACK_VERSION=$GERRIT_BRANCH - # log info to console - echo " STACK_TYPE=$STACK_TYPE" - echo " DEPLOY_SCENARIO=$DEPLOY_SCENARIO" - echo " STACK_VERSION=$STACK_VERSION" - echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION" - echo " NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC" - # record vars to properties file echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE" echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE" @@ -134,6 +175,10 @@ NORDIX_ARM_REPO=nordix-${STACK_TYPE} NORDIX_ARTIFACT_URL=$NORDIX_ARTIFACT_URL EOF +# NOTE (fdegir): we generated the change metadata file normally until here +# we now determine the dependency as the last step to ensure we override vars +determine_dependency + echo "Info : Change metadata" echo "-------------------------------------------------------------------------" cat "$CHANGE_METADATA_FILE" diff --git a/jjb/engine/scripts/package.sh b/jjb/engine/scripts/package.sh index d63d46b0..bcebb6cd 100755 --- a/jjb/engine/scripts/package.sh +++ b/jjb/engine/scripts/package.sh @@ -22,7 +22,23 @@ set -o nounset set -o errexit set -o pipefail -source "$WORKSPACE/change.properties" +# NOTE (fdegir): we need to remove $WORKSPACE and clone infra/engine repository +# as infra/engine is what drives the deployment +if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then + cd "$HOME" && /bin/rm -rf "$WORKSPACE" + echo "Info : Cloning infra/engine repository" + git clone -q https://gerrit.nordix.org/infra/engine.git "$WORKSPACE" + cd "$WORKSPACE" +fi + +# NOTE (fdegir): this variable is set by generate-change-metadata.sh script +# if a dependency to engine repo is determined so we check out that refspec +# to ensure we respect the dependency +if [[ ! -z "${NORDIX_ENGINE_REFSPEC+x}" ]]; then + echo "Info : Fetching dependent change ${NORDIX_ENGINE_REFSPEC} infra/engine" + git fetch "https://gerrit.nordix.org/infra/engine" ${NORDIX_ENGINE_REFSPEC} && git checkout FETCH_HEAD +fi + echo "Info : Packaging stack $STACK_TYPE" cd "$WORKSPACE/engine" diff --git a/jjb/engine/verify-jobs/engine-verify-tox.yaml b/jjb/engine/verify-jobs/engine-verify-tox.yaml index d679ed90..8fa11aba 100644 --- a/jjb/engine/verify-jobs/engine-verify-tox.yaml +++ b/jjb/engine/verify-jobs/engine-verify-tox.yaml @@ -113,6 +113,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: diff --git a/jjb/engine/verify-jobs/ironic-verify-tox.yaml b/jjb/engine/verify-jobs/ironic-verify-tox.yaml index 0d7aaddc..926df5cb 100644 --- a/jjb/engine/verify-jobs/ironic-verify-tox.yaml +++ b/jjb/engine/verify-jobs/ironic-verify-tox.yaml @@ -110,6 +110,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: diff --git a/jjb/engine/verify-jobs/kubernetes-verify-tox.yaml b/jjb/engine/verify-jobs/kubernetes-verify-tox.yaml index 0e64c81c..b7ae50aa 100644 --- a/jjb/engine/verify-jobs/kubernetes-verify-tox.yaml +++ b/jjb/engine/verify-jobs/kubernetes-verify-tox.yaml @@ -122,6 +122,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: diff --git a/jjb/engine/verify-jobs/onap-verify-tox.yaml b/jjb/engine/verify-jobs/onap-verify-tox.yaml index 8d4979a9..bdafd2d0 100644 --- a/jjb/engine/verify-jobs/onap-verify-tox.yaml +++ b/jjb/engine/verify-jobs/onap-verify-tox.yaml @@ -116,6 +116,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: diff --git a/jjb/engine/verify-jobs/openstack-verify-tox.yaml b/jjb/engine/verify-jobs/openstack-verify-tox.yaml index 86912352..6724dcf2 100644 --- a/jjb/engine/verify-jobs/openstack-verify-tox.yaml +++ b/jjb/engine/verify-jobs/openstack-verify-tox.yaml @@ -116,6 +116,7 @@ - compare-type: ANT pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' custom-url: '* $JOB_NAME $BUILD_URL' + readable-message: 'true' builders: - multijob: