From: robert.tomczyk Date: Thu, 14 Nov 2019 09:25:33 +0000 (+0000) Subject: GEODE: Add New Integration test for geode-native X-Git-Url: https://gerrit.nordix.org/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F95%2F2895%2F4;p=infra%2Fcicd.git GEODE: Add New Integration test for geode-native - Separate job for New Integration Tests - As the requirement changed the jobs will be executed on Ubuntu and not SLES15 - Executing verification jobs in parallel to save time - Add configuration steps for Ubuntu as part of slave setup Change-Id: I07b07b208fb89260cd29d09083c4da91d1ea049b --- diff --git a/jjb/geode-native/apache-geode-native-macros.yaml b/jjb/geode-native/apache-geode-native-macros.yaml index 54d0da3d..efb2dbb3 100644 --- a/jjb/geode-native/apache-geode-native-macros.yaml +++ b/jjb/geode-native/apache-geode-native-macros.yaml @@ -39,3 +39,8 @@ builders: - shell: !include-raw: ./apache-geode-native-test/integrationtest.sh +- builder: + name: 'apache-geode-native-newintegrationtest-macro' + builders: + - shell: + !include-raw: ./apache-geode-native-test/new_integrationtest.sh diff --git a/jjb/geode-native/apache-geode-native-slave/apache-geode-native-slave-setup.yaml b/jjb/geode-native/apache-geode-native-slave/apache-geode-native-slave-setup.yaml index 98611a46..d9d1bc99 100644 --- a/jjb/geode-native/apache-geode-native-slave/apache-geode-native-slave-setup.yaml +++ b/jjb/geode-native/apache-geode-native-slave/apache-geode-native-slave-setup.yaml @@ -20,7 +20,7 @@ - job: name: 'apache-geode-native-SlaveSetup' - node: geode-sles15 + node: geode-ubuntu1804 disabled: false concurrent: true diff --git a/jjb/geode-native/apache-geode-native-slave/slave_setup.sh b/jjb/geode-native/apache-geode-native-slave/slave_setup.sh index b08fa3fa..3dc2c5d3 100755 --- a/jjb/geode-native/apache-geode-native-slave/slave_setup.sh +++ b/jjb/geode-native/apache-geode-native-slave/slave_setup.sh @@ -23,7 +23,7 @@ set -o pipefail # Set build tools versions base on the: # https://github.com/apache/geode-native/blob/develop/docker/Dockerfile -CLANG_VERSION="6.0" +CLANG_VERSION="6.0.1" GEODE_VERSION="1.10.0" RAT_VERSION="0.13" CMAKE_VERSION="3.12.2" @@ -47,19 +47,19 @@ install_dev_packages() { echo "Info: Upgrade SLES and install all available patches" echo "-------------------------------------------------------------" - sudo zypper --gpg-auto-import-keys refresh ; sudo zypper -q update -y + sudo zypper --gpg-auto-import-keys refresh ; sudo zypper update -y sudo zypper -q patches echo "Info: Install clang development environment" echo "-------------------------------------------------------------" - sudo zypper -q install -y \ + sudo zypper install -y \ libc++-devel \ libc++abi-devel \ libopenssl-devel \ zlib-devel \ - clang${CLANG_VERSION%.*} \ - clang${CLANG_VERSION%.*}-checker \ - clang${CLANG_VERSION%.*}-devel \ + clang${CLANG_VERSION:0:1} \ + clang${CLANG_VERSION:0:1}-checker \ + clang${CLANG_VERSION:0:1}-devel \ gcc \ make \ patch \ @@ -69,10 +69,21 @@ install_dev_packages() { java-1_8_0-openjdk-devel \ wget - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${CLANG_VERSION} 999 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${CLANG_VERSION} 999 - sudo update-alternatives --install /usr/bin/cc cc /usr/bin/clang-${CLANG_VERSION} 999 - sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-${CLANG_VERSION} 999 + if [[ ! -L /etc/alternatives/clang ]] ; then + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${CLANG_VERSION} 999 + fi + if [[ ! -L /etc/alternatives/clang++ ]] ; then + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${CLANG_VERSION} 999 + fi + if [[ ! -L /etc/alternatives/clang-tidy ]] ; then + sudo update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-${CLANG_VERSION} 999 + fi + if [[ ! -L /etc/alternatives/cc ]] ; then + sudo update-alternatives --install /usr/bin/cc cc /usr/bin/clang-${CLANG_VERSION} 999 + fi + if [[ ! -L /etc/alternatives/cc++ ]] ; then + sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-${CLANG_VERSION} 999 + fi } configure_geode_build() { @@ -112,8 +123,22 @@ configure_geode_build() { export RAT_HOME="/opt/apache-rat-${RAT_VERSION}" export PATH="$PATH:$GEODE_HOME/bin" - # Get the latest geode-native build docker image - docker pull ${GEODE_NATIVE_DOCKER_IMAGE} + # Update hosts file with ip6-localhost required by geode-native Integration tests + sudo sed -i '/ipv6-loopback$/s/$/ ip6-localhost/' /etc/hosts +} + +get_geode_build_docker_image() { + echo "Info: Start and enable docker on system" + echo "-------------------------------------------------------------" + sudo systemctl enable docker + sudo systemctl start docker + echo "Info: Remove all previous stopped docker containers and images" + echo "-------------------------------------------------------------" + sudo docker container prune -f + sudo docker image prune -f + echo "Info: Get the latest geode-native build docker image" + echo "-------------------------------------------------------------" + docker pull ${GEODE_NATIVE_DOCKER_IMAGE:-apachegeode/geode-native-build} } # Make sure the build server is SLES15 based @@ -126,10 +151,27 @@ case ${ID,,} in else install_dev_packages configure_geode_build + get_geode_build_docker_image fi ;; + ubuntu) + DPKG_LOCK="/var/lib/dpkg/lock-frontend" + # Wait for other apt process to finish by checking the dpkg lock file. + try=0 + while sudo lsof ${DPKG_LOCK} > /dev/null 2>&1 ; do + echo "DPKG file locked: ${DPKG_LOCK}." + echo " Waiting for another pkg instalaltion process to finish ..." + sleep 10 + if [[ ${try} -gt 60 ]] ; then + echo "ERROR: Max number of re-tries reached, exiting..." + exit 1 + fi + try=$((try + 1)) + done + get_geode_build_docker_image + ;; *) - echo "ERROR: Only SLES15 can be use as a build server for this build" + echo "ERROR: Only SLES15 and Ubuntu OS can be use as a build server geode-native" exit 2 ;; esac diff --git a/jjb/geode-native/apache-geode-native-test/apache-geode-native-new-integration-test.yaml b/jjb/geode-native/apache-geode-native-test/apache-geode-native-new-integration-test.yaml new file mode 100644 index 00000000..d1d0240c --- /dev/null +++ b/jjb/geode-native/apache-geode-native-test/apache-geode-native-new-integration-test.yaml @@ -0,0 +1,27 @@ +--- +# +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'apache-geode-native-NewIntegrationTest' + test-type: 'NewIntegrationTest' + build-executor: 'apache-geode-native-newintegrationtest-macro' + + jobs: + - 'apache-geode-native-{test-type}' diff --git a/jjb/geode-native/apache-geode-native-test/integrationtest.sh b/jjb/geode-native/apache-geode-native-test/integrationtest.sh index 91d8ecf5..d6691c59 100755 --- a/jjb/geode-native/apache-geode-native-test/integrationtest.sh +++ b/jjb/geode-native/apache-geode-native-test/integrationtest.sh @@ -21,15 +21,13 @@ set -o errexit set -o pipefail #set -o xtrace -GEODE_NATIVE_DIR="$(pwd)/geode-native" +GEODE_NATIVE_DIR="${WORKSPACE}/geode-native" SOURCE_DIR="/geode-native" -DOCKER_ARGS="--volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}" +DOCKER_ARGS="--rm --volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}" COMMAND="echo -e '\n*** Build ***\n' && mkdir build && cd build &&\ - cmake ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_CXX_CLANG_TIDY=clang-tidy &&\ - cmake --build . -- -j${JOBS} && echo -e '\n*** Old Integration Tests ***\n' &&\ - cd ${SOURCE_DIR}/build/cppcache/integration-test && ctest --timeout 2000 -L STABLE -C Release -j1 &&\ - echo -e '\n*** New Integration Tests ***\n' &&\ - cd ${SOURCE_DIR}/build/cppcache/integration/test && ctest -j1" + cmake ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_CXX_CLANG_TIDY=clang-tidy &&\ + cmake --build . -- -j${JOBS} && echo -e '\n*** Old Integration Tests ***\n' &&\ + cd /build/cppcache/integration-test && ctest --timeout 2000 -L STABLE -C Release -j1" docker run ${DOCKER_ARGS} "${GEODE_NATIVE_DOCKER_IMAGE}" bash -c "${COMMAND}" diff --git a/jjb/geode-native/apache-geode-native-test/new_integrationtest.sh b/jjb/geode-native/apache-geode-native-test/new_integrationtest.sh new file mode 100755 index 00000000..870cd93b --- /dev/null +++ b/jjb/geode-native/apache-geode-native-test/new_integrationtest.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# ============LICENSE_START======================================================= +# Copyright (C) 2018-2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= +set -o nounset +set -o errexit +set -o pipefail +#set -o xtrace + +GEODE_NATIVE_DIR="${WORKSPACE}/geode-native" +SOURCE_DIR="/geode-native" + +DOCKER_ARGS="--rm --volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}" +COMMAND="echo -e '\n*** Build ***\n' && mkdir build && cd build &&\ + cmake ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=-stdlib=libc++ \\ + -DCMAKE_CXX_CLANG_TIDY=clang-tidy -DWITH_IPV6=ON &&\ + cmake --build . -- -j${JOBS} && echo -e '\n*** New Integration Tests ***\n' &&\ + cd /build/cppcache/integration/test && ctest -j1" + +docker run ${DOCKER_ARGS} "${GEODE_NATIVE_DOCKER_IMAGE}" bash -c "${COMMAND}" diff --git a/jjb/geode-native/apache-geode-native-test/ratcheck.sh b/jjb/geode-native/apache-geode-native-test/ratcheck.sh index 4fd7897d..0f2a0545 100755 --- a/jjb/geode-native/apache-geode-native-test/ratcheck.sh +++ b/jjb/geode-native/apache-geode-native-test/ratcheck.sh @@ -21,10 +21,10 @@ set -o errexit set -o pipefail #set -o xtrace -GEODE_NATIVE_DIR="$(pwd)/geode-native" +GEODE_NATIVE_DIR="${WORKSPACE}/geode-native" SOURCE_DIR="/geode-native" -DOCKER_ARGS="--volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}" +DOCKER_ARGS="--rm --volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}" COMMAND="echo -e '\n*** Rat Check ***\n' && mkdir build && cd build &&\ cmake ${SOURCE_DIR} -DUSE_RAT=ON &&\ cmake --build . --target rat-check" diff --git a/jjb/geode-native/apache-geode-native-test/unittest.sh b/jjb/geode-native/apache-geode-native-test/unittest.sh index fdc2c0f9..b9b42fe8 100755 --- a/jjb/geode-native/apache-geode-native-test/unittest.sh +++ b/jjb/geode-native/apache-geode-native-test/unittest.sh @@ -21,10 +21,10 @@ set -o errexit set -o pipefail #set -o xtrace -GEODE_NATIVE_DIR="$(pwd)/geode-native" +GEODE_NATIVE_DIR="${WORKSPACE}/geode-native" SOURCE_DIR="/geode-native" -DOCKER_ARGS="--volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}" +DOCKER_ARGS="--rm --volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}" COMMAND="echo -e '\n*** Unit Test ***\n' && mkdir build && cd build &&\ cmake ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_CXX_CLANG_TIDY=clang-tidy &&\ cmake --build . -- -j${JOBS} && ./cppcache/test/apache-geode_unittests &&\ diff --git a/jjb/geode-native/apache-geode-native-verify.yaml b/jjb/geode-native/apache-geode-native-verify.yaml index fe4dfbf8..ca0668c0 100644 --- a/jjb/geode-native/apache-geode-native-verify.yaml +++ b/jjb/geode-native/apache-geode-native-verify.yaml @@ -32,7 +32,7 @@ project-type: multijob disabled: false concurrent: true - node: geode-sles15 + node: geode-ubuntu1804 parameters: - string: @@ -84,41 +84,36 @@ abort-all-job: true node-parameters: true - multijob: - name: RatCheck + name: Verification condition: SUCCESSFUL - execution-type: SEQUENTIALLY + execution-type: PARALLEL projects: - name: 'apache-geode-native-RatCheck' current-parameters: true git-revision: true abort-all-job: true node-parameters: true - - multijob: - name: UnitTests - condition: SUCCESSFUL - execution-type: SEQUENTIALLY - projects: - name: 'apache-geode-native-UnitTest' current-parameters: true git-revision: true abort-all-job: true node-parameters: true - - multijob: - name: IntegrationTests - condition: SUCCESSFUL - execution-type: SEQUENTIALLY - projects: - name: 'apache-geode-native-IntegrationTest' current-parameters: true git-revision: true abort-all-job: true node-parameters: true + - name: 'apache-geode-native-NewIntegrationTest' + current-parameters: true + git-revision: true + abort-all-job: true + node-parameters: true # - multijob: -# name: Build&Release +# name: Release # condition: SUCCESSFUL # execution-type: SEQUENTIALLY # projects: -# - name: 'apache-geode-native-build-release' +# - name: 'apache-geode-native-SrcRelease' # current-parameters: true # git-revision: true # abort-all-job: true diff --git a/jjb/geode-native/global-template-apache-geode-native-test.yaml b/jjb/geode-native/global-template-apache-geode-native-test.yaml index 2d4f4343..cedc0dbb 100644 --- a/jjb/geode-native/global-template-apache-geode-native-test.yaml +++ b/jjb/geode-native/global-template-apache-geode-native-test.yaml @@ -20,7 +20,7 @@ - job-template: name: 'apache-geode-native-{test-type}' - node: geode-sles15 + node: geode-ubuntu1804 disabled: false concurrent: true