From 5fe5de7dbaf93789b5fd852e7e711a388a034f07 Mon Sep 17 00:00:00 2001 From: "robert.tomczyk" Date: Mon, 22 Jul 2019 20:48:58 +0100 Subject: [PATCH] GEODE: Create CI pipeline for Nordx geode releases Change-Id: I3541599e54b729f395e5df6a04f98b7482553da0 --- .../apache-geode-build.yaml | 33 ++++ jjb/geode/apache-geode-build/execute_build.sh | 92 ++++++++++ .../set_geode_build_version.sh | 31 ++++ jjb/geode/apache-geode-macros.yaml | 38 +++++ jjb/geode/apache-geode-test-coordinator.yaml | 120 +++++++++++++ .../apache-geode-acceptance-test.yaml | 35 ++++ .../apache-geode-distributed-test.yaml | 35 ++++ .../apache-geode-integration-test.yaml | 35 ++++ .../apache-geode-stressnew-test.yaml | 35 ++++ .../apache-geode-unit-test.yaml | 37 ++++ .../apache-geode-upgrade-test.yaml | 35 ++++ .../apache-geode-test/execute_stressnew.sh | 159 ++++++++++++++++++ jjb/geode/apache-geode-test/execute_tests.sh | 92 ++++++++++ .../global-template-apache-geode-build.yaml | 79 +++++++++ .../global-template-apache-geode-test.yaml | 83 +++++++++ jjb/global/jjb-macros.yaml | 24 ++- 16 files changed, 962 insertions(+), 1 deletion(-) create mode 100644 jjb/geode/apache-geode-build/apache-geode-build.yaml create mode 100755 jjb/geode/apache-geode-build/execute_build.sh create mode 100755 jjb/geode/apache-geode-build/set_geode_build_version.sh create mode 100644 jjb/geode/apache-geode-macros.yaml create mode 100644 jjb/geode/apache-geode-test-coordinator.yaml create mode 100644 jjb/geode/apache-geode-test/apache-geode-acceptance-test.yaml create mode 100644 jjb/geode/apache-geode-test/apache-geode-distributed-test.yaml create mode 100644 jjb/geode/apache-geode-test/apache-geode-integration-test.yaml create mode 100644 jjb/geode/apache-geode-test/apache-geode-stressnew-test.yaml create mode 100644 jjb/geode/apache-geode-test/apache-geode-unit-test.yaml create mode 100644 jjb/geode/apache-geode-test/apache-geode-upgrade-test.yaml create mode 100755 jjb/geode/apache-geode-test/execute_stressnew.sh create mode 100755 jjb/geode/apache-geode-test/execute_tests.sh create mode 100644 jjb/geode/global-template-apache-geode-build.yaml create mode 100644 jjb/geode/global-template-apache-geode-test.yaml diff --git a/jjb/geode/apache-geode-build/apache-geode-build.yaml b/jjb/geode/apache-geode-build/apache-geode-build.yaml new file mode 100644 index 00000000..1e52b34f --- /dev/null +++ b/jjb/geode/apache-geode-build/apache-geode-build.yaml @@ -0,0 +1,33 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'apache-geode-build' + build-executor: 'apache-geode-execute-build-macro' + + artifact-slug: 'build' + gradle-task: 'build' + java-build-version: '8' + parallel-gradle: 'false' + parallel-dunit: 'false' + dunit-parallel-forks: '0' + + jobs: + - 'apache-geode-build-jdk{java-build-version}' diff --git a/jjb/geode/apache-geode-build/execute_build.sh b/jjb/geode/apache-geode-build/execute_build.sh new file mode 100755 index 00000000..8e3a5b65 --- /dev/null +++ b/jjb/geode/apache-geode-build/execute_build.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +BASE_DIR=$(pwd) + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +source ${BASE_DIR}/geode/ci/scripts/shared_utilities.sh + +if [[ -z "${GRADLE_TASK}" ]]; then + echo "GRADLE_TASK must be set. exiting..." + exit 1 +fi + +ROOT_DIR=$(pwd) +BUILD_DATE=$(date +%s) + +# Precheckin does not get a geode-build-version +if [ -e "${ROOT_DIR}/geode-build-version" ] ; then + GEODE_BUILD_VERSION_FILE=${ROOT_DIR}/geode-build-version/number + GEODE_BUILD_DIR=/tmp/geode-build + GEODE_PULL_REQUEST_ID_FILE=${ROOT_DIR}/geode/.git/resource/version.json + + if [ -e "${GEODE_PULL_REQUEST_ID_FILE}" ]; then + GEODE_PULL_REQUEST_ID=$(cat ${GEODE_PULL_REQUEST_ID_FILE} | jq --raw-output '.["pr"]') + FULL_PRODUCT_VERSION="geode-pr-${GEODE_PULL_REQUEST_ID}" + else + GEODE_VERSION=$(cat ${GEODE_BUILD_VERSION_FILE} | cut -d'-' -f1) + echo -e "\nGEODE VERSION is ${GEODE_VERSION}" + BUILD_ID=$(cat ${GEODE_BUILD_VERSION_FILE} | cut -d'.' -f4) + FULL_PRODUCT_VERSION=$(cat ${GEODE_BUILD_VERSION_FILE}) + echo -e "FULL PRODUCT VERSION is ${FULL_PRODUCT_VERSION}\n" + fi +fi + +if [[ ${PARALLEL_GRADLE:-"true"} == "true" ]]; then + PARALLEL_GRADLE="--parallel" +else + PARALLEL_GRADLE="" +fi +DEFAULT_GRADLE_TASK_OPTIONS="${PARALLEL_GRADLE} --console=plain --no-daemon" +GRADLE_SKIP_TASK_OPTIONS="" + +if [[ -n "${PARALLEL_DUNIT}" && "${PARALLEL_DUNIT}" == "true" ]]; then + PARALLEL_DUNIT="-PparallelDunit -PdunitDockerUser=geode" + if [ -n "${DUNIT_PARALLEL_FORKS}" ]; then + DUNIT_PARALLEL_FORKS="-PdunitParallelForks=${DUNIT_PARALLEL_FORKS}" + fi +else + PARALLEL_DUNIT="" + DUNIT_PARALLEL_FORKS="" +fi + +SET_JAVA_HOME="export JAVA_HOME=/usr/lib/jvm/java-${JAVA_BUILD_VERSION}-openjdk-amd64" + +if [ -z "${FULL_PRODUCT_VERSION}" ] ; then + FULL_PRODUCT_VERSION="0.0.0-UndefinedVersion" +fi + +GRADLE_ARGS="\ + ${DEFAULT_GRADLE_TASK_OPTIONS} \ + ${GRADLE_SKIP_TASK_OPTIONS} \ + ${GRADLE_GLOBAL_ARGS} \ + -Pversion=${FULL_PRODUCT_VERSION} \ + -PbuildId=${BUILD_ID} \ + build install javadoc spotlessCheck rat checkPom resolveDependencies -x test" + +EXEC_COMMAND="mkdir -p tmp && cd geode && ${SET_JAVA_HOME} && ./gradlew ${GRADLE_ARGS}" +echo "${EXEC_COMMAND}" +eval ${EXEC_COMMAND} diff --git a/jjb/geode/apache-geode-build/set_geode_build_version.sh b/jjb/geode/apache-geode-build/set_geode_build_version.sh new file mode 100755 index 00000000..c597b8bf --- /dev/null +++ b/jjb/geode/apache-geode-build/set_geode_build_version.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +set -o errexit +set -o nounset +set -o pipefail + +cd $WORKSPACE/geode + +# Get most recent tag that is reachable from a commit +GEODE_VERSION=$(git describe | sed -r "s#rel/v(.*)#\1#") +if [ ! -d ${WORKSPACE}/geode-build-version ] ; then + mkdir ${WORKSPACE}/geode-build-version +fi +echo "$GEODE_VERSION" > ${WORKSPACE}/geode-build-version/number diff --git a/jjb/geode/apache-geode-macros.yaml b/jjb/geode/apache-geode-macros.yaml new file mode 100644 index 00000000..491a0253 --- /dev/null +++ b/jjb/geode/apache-geode-macros.yaml @@ -0,0 +1,38 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +# +# macros used by all geode job types +- builder: + name: 'apache-geode-execute-tests-macro' + builders: + - shell: + !include-raw: ./apache-geode-test/execute_tests.sh +- builder: + name: 'apache-geode-execute-stressnew-macro' + builders: + - shell: + !include-raw: ./apache-geode-test/execute_stressnew.sh +- builder: + name: 'apache-geode-execute-build-macro' + builders: + - shell: + !include-raw: ./apache-geode-build/set_geode_build_version.sh + - shell: + !include-raw: ./apache-geode-build/execute_build.sh \ No newline at end of file diff --git a/jjb/geode/apache-geode-test-coordinator.yaml b/jjb/geode/apache-geode-test-coordinator.yaml new file mode 100644 index 00000000..01764a66 --- /dev/null +++ b/jjb/geode/apache-geode-test-coordinator.yaml @@ -0,0 +1,120 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +- project: + name: 'apache-geode-coordinator' + project: 'Nordix/geode' + + stream: + - 'release-nordix-1.9.0': + branch: 'release-nordix/1.9.0' + jobs: + - 'apache-geode-verify-{stream}-coordinator' + +- job-template: + name: 'apache-geode-verify-{stream}-coordinator' + project-type: multijob + disabled: false + concurrent: true + node: geode-ubuntu1804 + + parameters: + - string: + name: PROJECT + default: '{project}' + description: JJB configured PROJECT parameter to identify a Nordix GitHub project + - string: + name: BRANCH + default: '{branch}' + description: JJB configured BRANCH parameter for Nordix GitHub project + + properties: + - logrotate + - github: + url: 'https://github.com/Nordix/geode.git/' + - throttle: + max-per-node: 1 + max-total: 3 + option: project + + wrappers: + - openstack: + single-use: True + + scm: + - git-scm-github: + branch: '{branch}' + refspec: '' + wipe_workspace: true + clean_before: false + base_dir: 'geode' + + triggers: + - github + + builders: + - multijob: + name: Test + condition: SUCCESSFUL + projects: + - name: 'apache-geode-UnitTest-jdk8' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true + - name: 'apache-geode-UnitTest-jdk11' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true + - name: 'apache-geode-StressNewTest-jdk11' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true + - name: 'apache-geode-IntegrationTest-jdk11' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true + - name: 'apache-geode-AcceptanceTest-jdk11' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true + - name: 'apache-geode-UpgradeTest-jdk11' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true + - name: 'apache-geode-DistributedTest-jdk11' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true + - multijob: + name: Build&Release + condition: SUCCESSFUL + execution-type: SEQUENTIALLY + projects: + - name: 'apache-geode-build-jdk8' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true \ No newline at end of file diff --git a/jjb/geode/apache-geode-test/apache-geode-acceptance-test.yaml b/jjb/geode/apache-geode-test/apache-geode-acceptance-test.yaml new file mode 100644 index 00000000..55b157f9 --- /dev/null +++ b/jjb/geode/apache-geode-test/apache-geode-acceptance-test.yaml @@ -0,0 +1,35 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'apache-geode-acceptance-test' + test-type: 'Acceptance' + build-executor: 'apache-geode-execute-tests-macro' + + artifact-slug: 'acceptancetestfiles' + gradle-task: 'acceptanceTest' + java-build-version: '8' + java-test-version: '11' + parallel-gradle: 'false' + parallel-dunit: 'false' + dunit-parallel-forks: '0' + + jobs: + - 'apache-geode-{test-type}Test-jdk{java-test-version}' diff --git a/jjb/geode/apache-geode-test/apache-geode-distributed-test.yaml b/jjb/geode/apache-geode-test/apache-geode-distributed-test.yaml new file mode 100644 index 00000000..23f85c2a --- /dev/null +++ b/jjb/geode/apache-geode-test/apache-geode-distributed-test.yaml @@ -0,0 +1,35 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'apache-geode-distributed-test' + test-type: 'Distributed' + build-executor: 'apache-geode-execute-tests-macro' + + artifact-slug: 'distributedtestfiles' + gradle-task: 'distributedTest' + java-build-version: '8' + java-test-version: '11' + parallel-gradle: 'true' + parallel-dunit: 'true' + dunit-parallel-forks: '12' + + jobs: + - 'apache-geode-{test-type}Test-jdk{java-test-version}' diff --git a/jjb/geode/apache-geode-test/apache-geode-integration-test.yaml b/jjb/geode/apache-geode-test/apache-geode-integration-test.yaml new file mode 100644 index 00000000..45b3d6b5 --- /dev/null +++ b/jjb/geode/apache-geode-test/apache-geode-integration-test.yaml @@ -0,0 +1,35 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'apache-geode-integration-test' + test-type: 'Integration' + build-executor: 'apache-geode-execute-tests-macro' + + artifact-slug: 'integrationtestfiles' + gradle-task: 'integrationTest' + java-build-version: '8' + java-test-version: '11' + parallel-gradle: 'true' + parallel-dunit: 'true' + dunit-parallel-forks: '12' + + jobs: + - 'apache-geode-{test-type}Test-jdk{java-test-version}' diff --git a/jjb/geode/apache-geode-test/apache-geode-stressnew-test.yaml b/jjb/geode/apache-geode-test/apache-geode-stressnew-test.yaml new file mode 100644 index 00000000..9be315ed --- /dev/null +++ b/jjb/geode/apache-geode-test/apache-geode-stressnew-test.yaml @@ -0,0 +1,35 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'apache-geode-stressnew-test' + test-type: 'StressNew' + build-executor: 'apache-geode-execute-stressnew-macro' + + artifact-slug: 'stressnewtestfiles' + gradle-task: 'repeatTest' + java-build-version: '8' + java-test-version: '11' + parallel-gradle: 'false' + parallel-dunit: 'true' + dunit-parallel-forks: '12' + + jobs: + - 'apache-geode-{test-type}Test-jdk{java-test-version}' diff --git a/jjb/geode/apache-geode-test/apache-geode-unit-test.yaml b/jjb/geode/apache-geode-test/apache-geode-unit-test.yaml new file mode 100644 index 00000000..80246ebc --- /dev/null +++ b/jjb/geode/apache-geode-test/apache-geode-unit-test.yaml @@ -0,0 +1,37 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'apache-geode-unit-test' + test-type: 'Unit' + build-executor: 'apache-geode-execute-tests-macro' + + artifact-slug: 'unittestfiles' + gradle-task: 'test' + java-build-version: '8' + java-test-version: + - '8' + - '11' + parallel-gradle: 'true' + parallel-dunit: 'false' + dunit-parallel-forks: '0' + + jobs: + - 'apache-geode-{test-type}Test-jdk{java-test-version}' diff --git a/jjb/geode/apache-geode-test/apache-geode-upgrade-test.yaml b/jjb/geode/apache-geode-test/apache-geode-upgrade-test.yaml new file mode 100644 index 00000000..f81df4ba --- /dev/null +++ b/jjb/geode/apache-geode-test/apache-geode-upgrade-test.yaml @@ -0,0 +1,35 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'apache-geode-upgrade-test' + test-type: 'Upgrade' + build-executor: 'apache-geode-execute-tests-macro' + + artifact-slug: 'upgradetestfiles' + gradle-task: 'upgradeTest' + java-build-version: '8' + java-test-version: '11' + parallel-gradle: 'true' + parallel-dunit: 'true' + dunit-parallel-forks: '12' + + jobs: + - 'apache-geode-{test-type}Test-jdk{java-test-version}' diff --git a/jjb/geode/apache-geode-test/execute_stressnew.sh b/jjb/geode/apache-geode-test/execute_stressnew.sh new file mode 100755 index 00000000..2f7b179d --- /dev/null +++ b/jjb/geode/apache-geode-test/execute_stressnew.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e + +BASE_DIR=$(pwd) + +SOURCE="${BASH_SOURCE[0]}" +while [[ -h "$SOURCE" ]]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +function changes_for_path() { + pushd geode >> /dev/null + local path="$1" # only expand once in the line below + # Get the last release tag on the branch + local releaseTag=$(git describe --abbrev=0 --tags) + local releaseBase=$(git rev-list -n 1 $releaseTag) || exit $? + + if [[ "${releaseBase}" == "" ]]; then + echo "Could not determine release base. Exiting..." + exit 1 + fi + git diff --name-only ${releaseBase} -- $path + popd >> /dev/null +} + +UNIT_TEST_CHANGES=$(changes_for_path '*/src/test/java') || exit $? +INTEGRATION_TEST_CHANGES=$(changes_for_path '*/src/integrationTest/java') || exit $? +DISTRIBUTED_TEST_CHANGES=$(changes_for_path '*/src/distributedTest/java') || exit $? +ACCEPTANCE_TEST_CHANGES=$(changes_for_path '*/src/acceptanceTest/java') || exit $? +UPGRADE_TEST_CHANGES=$(changes_for_path '*/src/upgradeTest/java') || exit $? + +CHANGED_FILES_ARRAY=( $UNIT_TEST_CHANGES $INTEGRATION_TEST_CHANGES $DISTRIBUTED_TEST_CHANGES $ACCEPTANCE_TEST_CHANGES $UPGRADE_TEST_CHANGES ) +NUM_CHANGED_FILES=${#CHANGED_FILES_ARRAY[@]} + +echo "${NUM_CHANGED_FILES} changed tests" + +if [[ "${NUM_CHANGED_FILES}" -eq 0 ]] +then + echo "No changed test files, nothing to test." + exit 0 +fi + +if [[ "${NUM_CHANGED_FILES}" -gt 25 ]] +then + echo "${NUM_CHANGED_FILES} is too many changed tests to stress test. Allowing this job to pass without stress testing." + exit 0 +fi + +TEST_TARGETS="" + +function append_to_test_targets() { + local target="$1" + local files="$2" + if [[ -n "$files" ]] + then + TEST_TARGETS="$TEST_TARGETS $target" + for FILENAME in $files + do + SHORT_NAME=$(basename $FILENAME) + SHORT_NAME="${SHORT_NAME%.java}" + TEST_TARGETS="$TEST_TARGETS --tests $SHORT_NAME" + done + fi +} + +append_to_test_targets "repeatUnitTest" "$UNIT_TEST_CHANGES" +append_to_test_targets "repeatIntegrationTest" "$INTEGRATION_TEST_CHANGES" +append_to_test_targets "repeatDistributedTest" "$DISTRIBUTED_TEST_CHANGES" +append_to_test_targets "repeatUpgradeTest" "$UPGRADE_TEST_CHANGES" + +# Acceptance tests cannot currently run in parallel, so do not stress these tests +#append_to_test_targets "repeatAcceptanceTest" "$ACCEPTANCE_TEST_CHANGES" + +export GRADLE_TASK="compileTestJava compileIntegrationTestJava compileDistributedTestJava $TEST_TARGETS" +export GRADLE_TASK_OPTIONS="--no-parallel -Prepeat=50 -PfailOnNoMatchingTests=false" + +echo "---------------------------------------------------" +echo "GRADLE_TASK_OPTIONS=${GRADLE_TASK_OPTIONS}" +echo "GRADLE_TASK=${GRADLE_TASK}" +echo "---------------------------------------------------" + +REPODIR=$(cd geode; git rev-parse --show-toplevel) + +if [[ ${PARALLEL_GRADLE:-"true"} == "true" ]]; then + PARALLEL_GRADLE="--parallel" +else + PARALLEL_GRADLE="--no-parallel" +fi +DEFAULT_GRADLE_TASK_OPTIONS="${PARALLEL_GRADLE} --console=plain --no-daemon" +GRADLE_SKIP_TASK_OPTIONS="-x javadoc -x spotlessCheck -x rat" + +if [[ -n "${PARALLEL_DUNIT}" && "${PARALLEL_DUNIT}" == "true" ]]; then + PARALLEL_DUNIT="-PparallelDunit -PdunitDockerUser=geode" + if [ -n "${DUNIT_PARALLEL_FORKS}" ]; then + DUNIT_PARALLEL_FORKS="-PdunitParallelForks=${DUNIT_PARALLEL_FORKS}" + fi +else + PARALLEL_DUNIT="" + DUNIT_PARALLEL_FORKS="" +fi + +case $ARTIFACT_SLUG in + windows*) + JAVA_BUILD_PATH=C:/java${JAVA_BUILD_VERSION} + JAVA_TEST_PATH=C:/java${JAVA_TEST_VERSION} + SEP=";" + ;; + *) + JAVA_BUILD_PATH=/usr/lib/jvm/java-${JAVA_BUILD_VERSION}-openjdk-amd64 + JAVA_TEST_PATH=/usr/lib/jvm/java-${JAVA_TEST_VERSION}-openjdk-amd64 + SEP="&&" + ;; +esac + +GRADLE_ARGS=" \ + -PcompileJVM=${JAVA_BUILD_PATH} \ + -PcompileJVMVer=${JAVA_BUILD_VERSION} \ + -PtestJVM=${JAVA_TEST_PATH} \ + -PtestJVMVer=${JAVA_TEST_VERSION} \ + ${PARALLEL_DUNIT} \ + ${DUNIT_PARALLEL_FORKS} \ + -PdunitDockerImage=\$(docker images --format '{{.Repository}}:{{.Tag}}' | grep test-container) \ + ${DEFAULT_GRADLE_TASK_OPTIONS} \ + ${GRADLE_SKIP_TASK_OPTIONS} \ + ${GRADLE_TASK} \ + ${GRADLE_TASK_OPTIONS} \ + ${GRADLE_GLOBAL_ARGS}" + +EXEC_COMMAND="bash -c 'echo Building with: $SEP \ + ${JAVA_BUILD_PATH}/bin/java -version $SEP \ + echo Testing with: $SEP \ + ${JAVA_TEST_PATH}/bin/java -version $SEP \ + cd geode $SEP \ + cp gradlew gradlewStrict $SEP \ + sed -e 's/JAVA_HOME/GRADLE_JVM/g' -i.bak gradlewStrict $SEP \ + GRADLE_JVM=${JAVA_BUILD_PATH} ./gradlewStrict ${GRADLE_ARGS}'" +echo "${EXEC_COMMAND}" +eval "${EXEC_COMMAND}" \ No newline at end of file diff --git a/jjb/geode/apache-geode-test/execute_tests.sh b/jjb/geode/apache-geode-test/execute_tests.sh new file mode 100755 index 00000000..d0371d29 --- /dev/null +++ b/jjb/geode/apache-geode-test/execute_tests.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +BASE_DIR=$(pwd) + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTDIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +if [[ -z "${GRADLE_TASK}" ]]; then + echo "GRADLE_TASK must be set. exiting..." + exit 1 +fi + +REPODIR=$(cd geode; git rev-parse --show-toplevel) + +if [[ ${PARALLEL_GRADLE:-"true"} == "true" ]]; then + PARALLEL_GRADLE="--parallel" +else + PARALLEL_GRADLE="--no-parallel" +fi +DEFAULT_GRADLE_TASK_OPTIONS="${PARALLEL_GRADLE} --console=plain --no-daemon" +GRADLE_SKIP_TASK_OPTIONS="-x javadoc -x spotlessCheck -x rat" + +if [[ -n "${PARALLEL_DUNIT}" && "${PARALLEL_DUNIT}" == "true" ]]; then + PARALLEL_DUNIT="-PparallelDunit -PdunitDockerUser=geode" + if [ -n "${DUNIT_PARALLEL_FORKS}" ]; then + DUNIT_PARALLEL_FORKS="-PdunitParallelForks=${DUNIT_PARALLEL_FORKS}" + fi +else + PARALLEL_DUNIT="" + DUNIT_PARALLEL_FORKS="" +fi + +case $ARTIFACT_SLUG in + windows*) + JAVA_BUILD_PATH=C:/java${JAVA_BUILD_VERSION} + JAVA_TEST_PATH=C:/java${JAVA_TEST_VERSION} + SEP=";" + ;; + *) + JAVA_BUILD_PATH=/usr/lib/jvm/java-${JAVA_BUILD_VERSION}-openjdk-amd64 + JAVA_TEST_PATH=/usr/lib/jvm/java-${JAVA_TEST_VERSION}-openjdk-amd64 + SEP="&&" + ;; +esac + +GRADLE_ARGS=" \ + -PcompileJVM=${JAVA_BUILD_PATH} \ + -PcompileJVMVer=${JAVA_BUILD_VERSION} \ + -PtestJVM=${JAVA_TEST_PATH} \ + -PtestJVMVer=${JAVA_TEST_VERSION} \ + ${PARALLEL_DUNIT} \ + ${DUNIT_PARALLEL_FORKS} \ + -PdunitDockerImage=\$(docker images --format '{{.Repository}}:{{.Tag}}' | grep test-container) \ + ${DEFAULT_GRADLE_TASK_OPTIONS} \ + ${GRADLE_SKIP_TASK_OPTIONS} \ + ${GRADLE_TASK} \ + ${GRADLE_TASK_OPTIONS} \ + ${GRADLE_GLOBAL_ARGS}" + +EXEC_COMMAND="bash -c 'echo Building with: $SEP \ + ${JAVA_BUILD_PATH}/bin/java -version $SEP \ + echo Testing with: $SEP \ + ${JAVA_TEST_PATH}/bin/java -version $SEP \ + cd geode $SEP \ + cp gradlew gradlewStrict $SEP \ + sed -e 's/JAVA_HOME/GRADLE_JVM/g' -i.bak gradlewStrict $SEP \ + GRADLE_JVM=${JAVA_BUILD_PATH} ./gradlewStrict ${GRADLE_ARGS}'" +echo "${EXEC_COMMAND}" +eval "${EXEC_COMMAND}" diff --git a/jjb/geode/global-template-apache-geode-build.yaml b/jjb/geode/global-template-apache-geode-build.yaml new file mode 100644 index 00000000..0f559f96 --- /dev/null +++ b/jjb/geode/global-template-apache-geode-build.yaml @@ -0,0 +1,79 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +- job-template: + name: 'apache-geode-build-jdk{java-build-version}' + + node: geode-ubuntu1804 + disabled: false + concurrent: true + + properties: + - logrotate + - github: + url: 'https://github.com/Nordix/geode.git/' + + parameters: + - string: + name: PROJECT + default: 'Nordix/geode' + description: JJB configured PROJECT parameter to identify a Nordix GitHub project + - string: + name: BRANCH + default: 'release-nordix/1.9.0' + description: JJB configured BRANCH parameter for Nordix GitHub project + - string: + name: ARTIFACT_SLUG + default: '{artifact-slug}' + description: 'The geode artifact required' + - string: + name: GRADLE_TASK + default: '{gradle-task}' + description: 'Single atomic piece of work for a build, such as compiling classes or generating javadoc' + - string: + name: PARALLEL_GRADLE + default: '{parallel-gradle}' + description: 'Parallel execution of gradle tasks' + - string: + name: PARALLEL_DUNIT + default: '{parallel-dunit}' + description: 'Parallel test execution using docker images' + - string: + name: DUNIT_PARALLEL_FORKS + default: '{dunit-parallel-forks}' + description: 'Control how many concurrent dunit (using docker) tests will be run' + - string: + name: JAVA_BUILD_VERSION + default: '{java-build-version}' + description: 'Java version use to build geode' + + scm: + - git-scm-github: + branch: '$BRANCH' + refspec: '' + wipe_workspace: true + clean_before: false + base_dir: 'geode' + + builders: + - '{build-executor}' + +# publishers: +# - notify-slack: +# slack-channel: '#geode' diff --git a/jjb/geode/global-template-apache-geode-test.yaml b/jjb/geode/global-template-apache-geode-test.yaml new file mode 100644 index 00000000..1acb6cc7 --- /dev/null +++ b/jjb/geode/global-template-apache-geode-test.yaml @@ -0,0 +1,83 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +- job-template: + name: 'apache-geode-{test-type}Test-jdk{java-test-version}' + + node: geode-ubuntu1804 + disabled: false + concurrent: true + + properties: + - logrotate + - github: + url: 'https://github.com/Nordix/geode.git/' + + parameters: + - string: + name: PROJECT + default: 'Nordix/geode' + description: JJB configured PROJECT parameter to identify a Nordix GitHub project + - string: + name: BRANCH + default: 'release-nordix/1.9.0' + description: JJB configured BRANCH parameter for Nordix GitHub project + - string: + name: ARTIFACT_SLUG + default: '{artifact-slug}' + description: 'The geode artifact required' + - string: + name: GRADLE_TASK + default: '{gradle-task}' + description: 'Single atomic piece of work for a build, such as compiling classes or generating javadoc' + - string: + name: PARALLEL_GRADLE + default: '{parallel-gradle}' + description: 'Parallel execution of gradle tasks' + - string: + name: PARALLEL_DUNIT + default: '{parallel-dunit}' + description: 'Parallel test execution using docker images' + - string: + name: DUNIT_PARALLEL_FORKS + default: '{dunit-parallel-forks}' + description: 'Control how many concurrent dunit (using docker) tests will be run' + - string: + name: JAVA_BUILD_VERSION + default: '{java-build-version}' + description: 'Java version use to build geode' + - string: + name: JAVA_TEST_VERSION + default: '{java-test-version}' + description: 'Java version use during test execution' + + scm: + - git-scm-github: + branch: '$BRANCH' + refspec: '' + wipe_workspace: true + clean_before: false + base_dir: 'geode' + + builders: + - '{build-executor}' + +# publishers: +# - notify-slack: +# slack-channel: '#geode' diff --git a/jjb/global/jjb-macros.yaml b/jjb/global/jjb-macros.yaml index 534e61c8..2669bc8a 100644 --- a/jjb/global/jjb-macros.yaml +++ b/jjb/global/jjb-macros.yaml @@ -116,7 +116,7 @@ artifact-days-to-keep: 60 artifact-num-to-keep: 200 -# defaults for SCM and Gerrit +# defaults for SCM, Gerrit and GitHub - scm: name: git-scm scm: @@ -161,6 +161,28 @@ before: '{clean_before}' choosing-strategy: 'gerrit' +- scm: + name: git-scm-github + scm: + - git: + url: 'https://github.com/$PROJECT.git' + branches: + - '*/{branch}' + refspec: '{refspec}' + timeout: 15 + per-build-tag: false + skip-tag: true + shallow-clone: false + use-author: false + ignore-notify: false + wipe-workspace: '{wipe_workspace}' + prune: false + clean: + after: false + before: '{clean_before}' + choosing-strategy: 'default' + basedir: '{base_dir}' + # triggers for Nordix Gerrit - trigger: name: nordix-gerrit-trigger-patchset-created -- 2.25.1