engine: Implement change dependency handling 83/5283/6
authorFatih Degirmenci <fatih.degirmenci@est.tech>
Mon, 6 Jul 2020 07:14:17 +0000 (07:14 +0000)
committerFatih Degirmenci <fatih.degirmenci@est.tech>
Mon, 6 Jul 2020 09:14:44 +0000 (09:14 +0000)
The new functionality enables specifying dependency between open
changes sent for different projects on Gerrit. When this functionality
is used, the generate-change-metadata.sh determines the change properties
such as change project, revision, and refspec and sets the variables
accordingly for the framework to use.

In order to use this feature, the dependent change needs to be specified
in commit message as below.

Depends-On: <Change-Id of dependent change>

For example

Depends-On: Iea9c21b9898f3733aa5486932ca6d5abffd8b803

Additional notes about the feature are
- the format to specify dependent change must follow the above example
- the dependent change must be specified on its own line
- the dependent change must be specified at the beginning of the line
- only single dependency can be specified at the moment but this can
be improved further if it is deemed necessary
- the specified change must be open (not merged or abandoned)
- only the current/latest patchset can be used (as the earlier patchsets are
already outdated)
- dependent change must not be on same repository as the change under test

See the generate-change-metadata.sh determine_dependency function for
how things work.

Change-Id: I2e5cfa23fb4a46ff49a4bdf137054b11202d39eb

jjb/engine/scripts/deploy.sh
jjb/engine/scripts/generate-change-metadata.sh
jjb/engine/scripts/package.sh
jjb/engine/verify-jobs/engine-verify-tox.yaml
jjb/engine/verify-jobs/ironic-verify-tox.yaml
jjb/engine/verify-jobs/kubernetes-verify-tox.yaml
jjb/engine/verify-jobs/onap-verify-tox.yaml
jjb/engine/verify-jobs/openstack-verify-tox.yaml

index abd2414742b2fef1d4ee96e3919696573355576b..0302e776a54db534f8d2d24b9257af59022c0b3e 100755 (executable)
@@ -31,6 +31,14 @@ if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then
   cd "$WORKSPACE"
 fi
 
+# NOTE (fdegir): this variable is set by generate-change-metadata.sh script
+# if a dependency to engine repo is determined so we check out that refspec
+# to ensure we respect the dependency
+if [[ ! -z "${NORDIX_ENGINE_REFSPEC+x}" ]]; then
+  echo "Info  : Fetching dependent change ${NORDIX_ENGINE_REFSPEC} infra/engine"
+  git fetch "https://gerrit.nordix.org/infra/engine" ${NORDIX_ENGINE_REFSPEC} && git checkout FETCH_HEAD
+fi
+
 if [[ "$EXECUTION_MODE" == "offline-deployment" ]]; then
   ENGINE_ROOT_DIR="/opt/engine/offline/git/engine"
 else
index 219c186a52809c0a8c5ae1b2bc48960033b1e0aa..ead0a77886da01273e8e2e43c8f0597e9f1a2ada 100755 (executable)
@@ -23,14 +23,75 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
-# NOTE (fdegir): we need to remove $WORKSPACE and clone infra/engine repository
-# as infra/engine is what drives the deployment
-if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then
-  cd "$HOME" && /bin/rm -rf "$WORKSPACE"
-  echo "Info : Cloning infra/engine repository"
-  git clone -q https://gerrit.nordix.org/infra/engine.git "$WORKSPACE"
-  cd "$WORKSPACE"
-fi
+# NOTE (fdegir): in case if a change depends on another open change in different repo
+# we determine it here and override the version and refspec for that repo
+function determine_dependency() {
+  echo "Info  : Determining dependency"
+  echo "Info  : Commit message is"
+  echo "-------------------------------------------------------------------------"
+  echo "$GERRIT_CHANGE_COMMIT_MESSAGE"
+  echo "-------------------------------------------------------------------------"
+
+  if ! echo "$GERRIT_CHANGE_COMMIT_MESSAGE" | grep -qs "^Depends-On:"; then
+    echo "Info  : No dependency found"
+    return
+  else
+    echo "Info  : Dependency found!"
+    DEPENDENT_CHANGE_ID=$(echo "$GERRIT_CHANGE_COMMIT_MESSAGE" | grep "^Depends-On:" | cut -d":" -f2 | sed 's/\s*//g')
+  fi
+
+  echo "Info  : Processing change $DEPENDENT_CHANGE_ID"
+  declare -a DEPENDENT_CHANGE_METADATA
+  readarray -t DEPENDENT_CHANGE_METADATA < <(curl -s "https://gerrit.nordix.org/changes/?q=$DEPENDENT_CHANGE_ID&o=CURRENT_REVISION" 2>&1 \
+      | grep "project\|status\|current_revision\|ref.*," | sed -e 's/\s//g' -e 's/,//g' -e 's/"//g'| cut -d: -f2)
+
+  # NOTE (fdegir): check if the dependeny change exists
+  # and ignore if it doesn't
+  if [[ -z "${DEPENDENT_CHANGE_METADATA+x}" ]]; then
+    echo "Warn  : Unable to determine dependent change. Ignoring dependency!"
+    return
+  fi
+
+  DEPENDENT_CHANGE_PROJECT="${DEPENDENT_CHANGE_METADATA[0]}"
+  DEPENDENT_CHANGE_STATUS="${DEPENDENT_CHANGE_METADATA[1]}"
+  DEPENDENT_CHANGE_REVISION="${DEPENDENT_CHANGE_METADATA[2]}"
+  DEPENDENT_CHANGE_REFSPEC="${DEPENDENT_CHANGE_METADATA[3]}"
+  echo "-------------------------------------------------------------------------"
+  echo "        DEPENDENT_CHANGE_PROJECT=$DEPENDENT_CHANGE_PROJECT"
+  echo "        DEPENDENT_CHANGE_STATUS=$DEPENDENT_CHANGE_STATUS"
+  echo "        DEPENDENT_CHANGE_REVISION=$DEPENDENT_CHANGE_REVISION"
+  echo "        DEPENDENT_CHANGE_REFSPEC=$DEPENDENT_CHANGE_REFSPEC"
+  echo "-------------------------------------------------------------------------"
+
+  # NOTE (fdegir): we only follow the dependency if the change status is NEW
+  if [[ "$DEPENDENT_CHANGE_STATUS" != "NEW" ]]; then
+    echo "Warn  : Change is not open. Ignoring dependency!"
+    return
+  fi
+
+  # NOTE (fdegir): we do not accept dependency of a change that is sent to same project as the dependent change
+  if [[ "$DEPENDENT_CHANGE_PROJECT" == "$GERRIT_PROJECT" ]]; then
+    echo "Warn  : Change is for the same project. Ignoring dependency!"
+    return
+  fi
+
+  DEPENDENT_COMPONENT="${DEPENDENT_CHANGE_PROJECT//*\//}"
+
+  echo "# dependent change" >> "$CHANGE_METADATA_FILE"
+
+  # NOTE (fdegir): if dependent change is on stack repo, we need to treat it
+  # differently
+  if [[ "$DEPENDENT_CHANGE_PROJECT" =~ /stack/ ]]; then
+    # set vars
+    echo "STACK_TYPE=$DEPENDENT_COMPONENT" >> "$CHANGE_METADATA_FILE"
+    echo "STACK_VERSION=$DEPENDENT_CHANGE_REVISION" >> "$CHANGE_METADATA_FILE"
+    echo "STACK_REFSPEC=$DEPENDENT_CHANGE_REFSPEC" >> "$CHANGE_METADATA_FILE"
+  else
+    echo "NORDIX_$(echo $DEPENDENT_COMPONENT | tr a-z A-Z)_VERSION=$DEPENDENT_CHANGE_REVISION" >> "$CHANGE_METADATA_FILE"
+    echo "NORDIX_$(echo $DEPENDENT_COMPONENT | tr a-z A-Z)_REFSPEC=$DEPENDENT_CHANGE_REFSPEC" >> "$CHANGE_METADATA_FILE"
+  fi
+
+}
 
 # file to record change metadata
 export CHANGE_METADATA_FILE="$WORKSPACE/change.properties"
@@ -49,12 +110,6 @@ if [[ "$GERRIT_PROJECT" =~ /stack/ ]]; then
   export STACK_VERSION="$GERRIT_PATCHSET_REVISION"
   export STACK_REFSPEC="$GERRIT_REFSPEC"
 
-  # log info to console
-  echo "        STACK_TYPE=$STACK_TYPE"
-  echo "        DEPLOY_SCENARIO=$DEPLOY_SCENARIO"
-  echo "        STACK_VERSION=$GERRIT_PATCHSET_REVISION"
-  echo "        STACK_REFSPEC=$GERRIT_REFSPEC"
-
   # record vars to properties file
   echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE"
   echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE"
@@ -79,13 +134,6 @@ elif [[ "$GERRIT_PROJECT" =~ /installer/ ]]; then
   esac
   export STACK_VERSION="$GERRIT_BRANCH"
 
-  # log info to console
-  echo "        STACK_TYPE=$STACK_TYPE"
-  echo "        DEPLOY_SCENARIO=$DEPLOY_SCENARIO"
-  echo "        STACK_VERSION=$STACK_VERSION"
-  echo "        NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION"
-  echo "        NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC"
-
   # record vars to properties file
   echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE"
   echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE"
@@ -97,13 +145,6 @@ else
   export STACK_TYPE=kubernetes
   export STACK_VERSION=$GERRIT_BRANCH
 
-  # log info to console
-  echo "        STACK_TYPE=$STACK_TYPE"
-  echo "        DEPLOY_SCENARIO=$DEPLOY_SCENARIO"
-  echo "        STACK_VERSION=$STACK_VERSION"
-  echo "        NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_VERSION=$GERRIT_PATCHSET_REVISION"
-  echo "        NORDIX_$(echo $ENGINE_COMPONENT | tr a-z A-Z)_REFSPEC=$GERRIT_REFSPEC"
-
   # record vars to properties file
   echo "STACK_TYPE=$STACK_TYPE" >> "$CHANGE_METADATA_FILE"
   echo "DEPLOY_SCENARIO=$DEPLOY_SCENARIO" >> "$CHANGE_METADATA_FILE"
@@ -134,6 +175,10 @@ NORDIX_ARM_REPO=nordix-${STACK_TYPE}
 NORDIX_ARTIFACT_URL=$NORDIX_ARTIFACT_URL
 EOF
 
+# NOTE (fdegir): we generated the change metadata file normally until here
+# we now determine the dependency as the last step to ensure we override vars
+determine_dependency
+
 echo "Info  : Change metadata"
 echo "-------------------------------------------------------------------------"
 cat "$CHANGE_METADATA_FILE"
index d63d46b0656c2479cbba564b64028d9129c38b52..bcebb6cde7c08135f17eb2b5dbdf1d2a13a70f7b 100755 (executable)
@@ -22,7 +22,23 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
-source "$WORKSPACE/change.properties"
+# NOTE (fdegir): we need to remove $WORKSPACE and clone infra/engine repository
+# as infra/engine is what drives the deployment
+if [[ "$GERRIT_PROJECT" != "infra/engine" ]]; then
+  cd "$HOME" && /bin/rm -rf "$WORKSPACE"
+  echo "Info : Cloning infra/engine repository"
+  git clone -q https://gerrit.nordix.org/infra/engine.git "$WORKSPACE"
+  cd "$WORKSPACE"
+fi
+
+# NOTE (fdegir): this variable is set by generate-change-metadata.sh script
+# if a dependency to engine repo is determined so we check out that refspec
+# to ensure we respect the dependency
+if [[ ! -z "${NORDIX_ENGINE_REFSPEC+x}" ]]; then
+  echo "Info  : Fetching dependent change ${NORDIX_ENGINE_REFSPEC} infra/engine"
+  git fetch "https://gerrit.nordix.org/infra/engine" ${NORDIX_ENGINE_REFSPEC} && git checkout FETCH_HEAD
+fi
+
 echo "Info  : Packaging stack $STACK_TYPE"
 
 cd "$WORKSPACE/engine"
index d679ed9069f72c3d84af85a7e4544f733e03a7bc..8fa11abada00bf9470af82fca72a406f26a186fc 100644 (file)
                 - compare-type: ANT
                   pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py'
           custom-url: '* $JOB_NAME $BUILD_URL'
+          readable-message: 'true'
 
     builders:
       - multijob:
index 0d7aaddce48919ef3a26c80a0c40be7a9f14d5dc..926df5cb52b3b29c1997b63363a2e9e5a1802aeb 100644 (file)
                 - compare-type: ANT
                   pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py'
           custom-url: '* $JOB_NAME $BUILD_URL'
+          readable-message: 'true'
 
     builders:
       - multijob:
index 0e64c81c7649ea454e9a9d9a08f1e1973688454e..b7ae50aadddf5b5034ddf9c69585ece92adb4379 100644 (file)
                 - compare-type: ANT
                   pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py'
           custom-url: '* $JOB_NAME $BUILD_URL'
+          readable-message: 'true'
 
     builders:
       - multijob:
index 8d4979a97327fd287a5d542a35716e92301dc924..bdafd2d08a240b4a4a3d38bfa1c86bd0f7d84dce 100644 (file)
                 - compare-type: ANT
                   pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py'
           custom-url: '* $JOB_NAME $BUILD_URL'
+          readable-message: 'true'
 
     builders:
       - multijob:
index 8691235250720a73bea3262b0e57118366eb29ad..6724dcf29402356d6a604c4ab45faa554bee1165 100644 (file)
                 - compare-type: ANT
                   pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py'
           custom-url: '* $JOB_NAME $BUILD_URL'
+          readable-message: 'true'
 
     builders:
       - multijob: