jjb/geode/apache-geode-test/execute_stressnew.sh

   1 #!/usr/bin/env bash
   2
   3 #
   4 # Licensed to the Apache Software Foundation (ASF) under one or more
   5 # contributor license agreements.  See the NOTICE file distributed with
   6 # this work for additional information regarding copyright ownership.
   7 # The ASF licenses this file to You under the Apache License, Version 2.0
   8 # (the "License"); you may not use this file except in compliance with
   9 # the License.  You may obtain a copy of the License at
  10 #
  11 #      http://www.apache.org/licenses/LICENSE-2.0
  12 #
  13 # Unless required by applicable law or agreed to in writing, software
  14 # distributed under the License is distributed on an "AS IS" BASIS,
  15 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16 # See the License for the specific language governing permissions and
  17 # limitations under the License.
  18 #
  19
  20 set -e
  21
  22 BASE_DIR=$(pwd)
  23
  24 SOURCE="${BASH_SOURCE[0]}"
  25 while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink
  26   SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
  27   SOURCE="$(readlink "$SOURCE")"
  28   [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
  29 done
  30 SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
  31
  32 function changes_for_path() {
  33   pushd geode >> /dev/null
  34     local path="$1" # only expand once in the line below
  35     # Get the last release tag on the branch
  36     local releaseTag=$(git describe --abbrev=0 --tags)
  37     local releaseBase=$(git rev-list -n 1 $releaseTag) || exit $?
  38
  39     if [[ "${releaseBase}" == "" ]]; then
  40       echo "Could not determine release base. Exiting..."
  41       exit 1
  42     fi
  43     git diff --name-only ${releaseBase} -- $path
  44   popd >> /dev/null
  45 }
  46
  47 UNIT_TEST_CHANGES=$(changes_for_path '*/src/test/java') || exit $?
  48 INTEGRATION_TEST_CHANGES=$(changes_for_path '*/src/integrationTest/java') || exit $?
  49 DISTRIBUTED_TEST_CHANGES=$(changes_for_path '*/src/distributedTest/java') || exit $?
  50 ACCEPTANCE_TEST_CHANGES=$(changes_for_path '*/src/acceptanceTest/java') || exit $?
  51 UPGRADE_TEST_CHANGES=$(changes_for_path '*/src/upgradeTest/java') || exit $?
  52
  53 CHANGED_FILES_ARRAY=( $UNIT_TEST_CHANGES $INTEGRATION_TEST_CHANGES $DISTRIBUTED_TEST_CHANGES $ACCEPTANCE_TEST_CHANGES $UPGRADE_TEST_CHANGES )
  54 NUM_CHANGED_FILES=${#CHANGED_FILES_ARRAY[@]}
  55
  56 echo "${NUM_CHANGED_FILES} changed tests"
  57
  58 if [[  "${NUM_CHANGED_FILES}" -eq 0 ]]
  59 then
  60   echo "No changed test files, nothing to test."
  61   exit 0
  62 fi
  63
  64 if [[ "${NUM_CHANGED_FILES}" -gt 25 ]]
  65 then
  66   echo "${NUM_CHANGED_FILES} is too many changed tests to stress test. Allowing this job to pass without stress testing."
  67   exit 0
  68 fi
  69
  70 TEST_TARGETS=""
  71
  72 function append_to_test_targets() {
  73   local target="$1"
  74   local files="$2"
  75   if [[ -n "$files" ]]
  76   then
  77     TEST_TARGETS="$TEST_TARGETS $target"
  78     for FILENAME in $files
  79     do
  80       SHORT_NAME=$(basename $FILENAME)
  81       SHORT_NAME="${SHORT_NAME%.java}"
  82       TEST_TARGETS="$TEST_TARGETS --tests $SHORT_NAME"
  83     done
  84   fi
  85 }
  86
  87 append_to_test_targets "repeatUnitTest" "$UNIT_TEST_CHANGES"
  88 append_to_test_targets "repeatIntegrationTest" "$INTEGRATION_TEST_CHANGES"
  89 append_to_test_targets "repeatDistributedTest" "$DISTRIBUTED_TEST_CHANGES"
  90 append_to_test_targets "repeatUpgradeTest" "$UPGRADE_TEST_CHANGES"
  91
  92 # Acceptance tests cannot currently run in parallel, so do not stress these tests
  93 #append_to_test_targets "repeatAcceptanceTest" "$ACCEPTANCE_TEST_CHANGES"
  94
  95 export GRADLE_TASK="compileTestJava compileIntegrationTestJava compileDistributedTestJava $TEST_TARGETS"
  96 export GRADLE_TASK_OPTIONS="--no-parallel -Prepeat=50 -PfailOnNoMatchingTests=false"
  97
  98 echo "---------------------------------------------------"
  99 echo "GRADLE_TASK_OPTIONS=${GRADLE_TASK_OPTIONS}"
 100 echo "GRADLE_TASK=${GRADLE_TASK}"
 101 echo "---------------------------------------------------"
 102
 103 REPODIR=$(cd geode; git rev-parse --show-toplevel)
 104
 105 if [[ ${PARALLEL_GRADLE:-"true"} == "true" ]]; then
 106   PARALLEL_GRADLE="--parallel"
 107 else
 108   PARALLEL_GRADLE="--no-parallel"
 109 fi
 110 DEFAULT_GRADLE_TASK_OPTIONS="${PARALLEL_GRADLE} --console=plain --no-daemon"
 111 GRADLE_SKIP_TASK_OPTIONS="-x javadoc -x spotlessCheck -x rat"
 112
 113 if [[ -n "${PARALLEL_DUNIT}" && "${PARALLEL_DUNIT}" == "true" ]]; then
 114   PARALLEL_DUNIT="-PparallelDunit"
 115   if [[ -n "${DUNIT_PARALLEL_FORKS}" && "${TEST_USING_DOCKER}" == "false" ]] ; then
 116     DUNIT_PARALLEL_FORKS="--max-workers=${DUNIT_PARALLEL_FORKS} -PtestMaxParallelForks=${DUNIT_PARALLEL_FORKS} -PdunitParallelForks=${DUNIT_PARALLEL_FORKS}"
 117   elif [[ -n "${DUNIT_PARALLEL_FORKS}" && "${TEST_USING_DOCKER}" == "true" ]] ; then
 118     DUNIT_PARALLEL_FORKS="-PdunitParallelForks=${DUNIT_PARALLEL_FORKS}"
 119     DOCKER_TESTING_OPTIONS="-PdunitDockerUser=geode -PdunitDockerImage=\$(docker images --format '{{.Repository}}:{{.Tag}}' | grep test-container)"
 120   fi
 121 else
 122   PARALLEL_DUNIT=""
 123   DUNIT_PARALLEL_FORKS=""
 124   DOCKER_TESTING_OPTIONS=""
 125 fi
 126
 127 case $ARTIFACT_SLUG in
 128   windows*)
 129     echo "Making environment adjustments for windows."
 130     JAVA_BUILD_PATH=C:/java${JAVA_BUILD_VERSION}
 131     JAVA_TEST_PATH=C:/java${JAVA_TEST_VERSION}
 132     TEST_JAVA_8_HOME=C:/java8
 133     TEST_JAVA_11_HOME=C:/java11
 134     TEST_JAVA_17_HOME=C:/java17
 135     GRADLE_SKIP_TASK_OPTIONS="${GRADLE_SKIP_TASK_OPTIONS} -x docker"
 136     SEP=";"
 137     OPTIONAL_KILL_ALL_CMD="${SEP} \
 138       exit_code=\$? ${SEP} \
 139       (taskkill //F //IM java.exe //T) ${SEP} \
 140       exit \${exit_code}
 141       "
 142     ;;
 143   *)
 144     JAVA_BUILD_PATH=/usr/lib/jvm/java-${JAVA_BUILD_VERSION}-openjdk-amd64
 145     JAVA_TEST_PATH=/usr/lib/jvm/java-${JAVA_TEST_VERSION}-openjdk-amd64
 146     TEST_JAVA_8_HOME=/usr/lib/jvm/java-8-openjdk-amd64
 147     TEST_JAVA_11_HOME=/usr/lib/jvm/java-11-openjdk-amd64/
 148     TEST_JAVA_17_HOME=/usr/lib/jvm/java-17-openjdk-amd64/
 149     SEP="&&"
 150     ;;
 151 esac
 152
 153 GRADLE_ARGS=" \
 154     -PcompileJVM=${JAVA_BUILD_PATH} \
 155     -PcompileJVMVer=${JAVA_BUILD_VERSION} \
 156     -PtestJVM=${JAVA_TEST_PATH} \
 157     -PtestJVMVer=${JAVA_TEST_VERSION} \
 158     -PtestJava8Home=${TEST_JAVA_8_HOME} \
 159     -PtestJava11Home=${TEST_JAVA_11_HOME} \
 160     -PtestJava17Home=${TEST_JAVA_17_HOME} \
 161     ${PARALLEL_DUNIT} \
 162     ${DUNIT_PARALLEL_FORKS} \
 163     ${DOCKER_TESTING_OPTIONS} \
 164     ${DEFAULT_GRADLE_TASK_OPTIONS} \
 165     ${GRADLE_SKIP_TASK_OPTIONS} \
 166     ${GRADLE_TASK} \
 167     ${GRADLE_TASK_OPTIONS} \
 168     ${GRADLE_GLOBAL_ARGS}"
 169
 170 EXEC_COMMAND="bash -c 'echo Building with: $SEP \
 171   ${JAVA_BUILD_PATH}/bin/java -version $SEP \
 172   echo Testing with: $SEP \
 173   ${JAVA_TEST_PATH}/bin/java -version $SEP \
 174   cd geode $SEP \
 175   cp gradlew gradlewStrict $SEP \
 176   sed -e 's/JAVA_HOME/GRADLE_JVM/g' -i.bak gradlewStrict $SEP \
 177   GRADLE_JVM=${JAVA_BUILD_PATH} JAVA_TEST_PATH=${JAVA_TEST_PATH} ./gradlewStrict ${GRADLE_ARGS}'"
 178 echo "${EXEC_COMMAND}"
 179 eval "${EXEC_COMMAND}"
 180
 181 # Cleanup
 182 # Make sure all test containers are stopped and removed when tests are finished
 183 DOCKER_CONTAINERS=$(sudo docker ps -a -q 2> /dev/null)
 184 if [ -n "${DOCKER_CONTAINERS}" ]; then
 185   docker stop ${DOCKER_CONTAINERS} > /dev/null
 186   echo -e  "Removing containers with ID:"
 187   docker rm ${DOCKER_CONTAINERS}
 188 fi
 189