GEODE: Add New Integration test for geode-native

- Separate job for New Integration Tests
- As the requirement changed the jobs will be executed on Ubuntu and not
SLES15
- Executing verification jobs in parallel to save time
- Add configuration steps for Ubuntu as part of slave setup

Change-Id: I07b07b208fb89260cd29d09083c4da91d1ea049b

diff --git a/jjb/geode-native/apache-geode-native-macros.yaml b/jjb/geode-native/apache-geode-native-macros.yaml
index 54d0da3d00d1726882e4350ae61e609b3fa45bd9..efb2dbb31fdcb658571ba8509128151c80c6d97c 100644 (file)
--- a/jjb/geode-native/apache-geode-native-macros.yaml
+++ b/jjb/geode-native/apache-geode-native-macros.yaml
@@ -39,3 +39,8 @@
     builders:
       - shell:
           !include-raw: ./apache-geode-native-test/integrationtest.sh
+- builder:
+    name: 'apache-geode-native-newintegrationtest-macro'
+    builders:
+      - shell:
+          !include-raw: ./apache-geode-native-test/new_integrationtest.sh

diff --git a/jjb/geode-native/apache-geode-native-slave/apache-geode-native-slave-setup.yaml b/jjb/geode-native/apache-geode-native-slave/apache-geode-native-slave-setup.yaml
index 98611a4606ea4d7bf39ef822b3e9926152bff55c..d9d1bc99c2bd5e738925c546c6c26affb7b88ea9 100644 (file)
--- a/jjb/geode-native/apache-geode-native-slave/apache-geode-native-slave-setup.yaml
+++ b/jjb/geode-native/apache-geode-native-slave/apache-geode-native-slave-setup.yaml
@@ -20,7 +20,7 @@
 - job:
     name: 'apache-geode-native-SlaveSetup'
 
-    node: geode-sles15
+    node: geode-ubuntu1804
     disabled: false
     concurrent: true
 

diff --git a/jjb/geode-native/apache-geode-native-slave/slave_setup.sh b/jjb/geode-native/apache-geode-native-slave/slave_setup.sh
index b08fa3faa4426e1b3b8cacd2036403f22d48c67f..3dc2c5d37fe9c0e4ef836bc26060265f12240ed8 100755 (executable)
--- a/jjb/geode-native/apache-geode-native-slave/slave_setup.sh
+++ b/jjb/geode-native/apache-geode-native-slave/slave_setup.sh
@@ -23,7 +23,7 @@ set -o pipefail
 
 # Set build tools versions base on the:
 # https://github.com/apache/geode-native/blob/develop/docker/Dockerfile
-CLANG_VERSION="6.0"
+CLANG_VERSION="6.0.1"
 GEODE_VERSION="1.10.0"
 RAT_VERSION="0.13"
 CMAKE_VERSION="3.12.2"
@@ -47,19 +47,19 @@ install_dev_packages() {
 
   echo "Info: Upgrade SLES and install all available patches"
   echo "-------------------------------------------------------------"
-  sudo zypper --gpg-auto-import-keys refresh ; sudo zypper -q update -y
+  sudo zypper --gpg-auto-import-keys refresh ; sudo zypper update -y
   sudo zypper -q patches
 
   echo "Info: Install clang development environment"
   echo "-------------------------------------------------------------"
-  sudo zypper -q install -y \
+  sudo zypper install -y \
     libc++-devel \
     libc++abi-devel \
     libopenssl-devel \
     zlib-devel \
-    clang${CLANG_VERSION%.*} \
-    clang${CLANG_VERSION%.*}-checker \
-    clang${CLANG_VERSION%.*}-devel \
+    clang${CLANG_VERSION:0:1} \
+    clang${CLANG_VERSION:0:1}-checker \
+    clang${CLANG_VERSION:0:1}-devel \
     gcc \
     make \
     patch \
@@ -69,10 +69,21 @@ install_dev_packages() {
     java-1_8_0-openjdk-devel \
     wget
 
-  sudo update-alternatives --install /usr/bin/clang    clang    /usr/bin/clang-${CLANG_VERSION} 999
-  sudo update-alternatives --install /usr/bin/clang++  clang++  /usr/bin/clang++-${CLANG_VERSION} 999
-  sudo update-alternatives --install /usr/bin/cc       cc       /usr/bin/clang-${CLANG_VERSION} 999
-  sudo update-alternatives --install /usr/bin/c++      c++      /usr/bin/clang++-${CLANG_VERSION} 999
+  if [[ ! -L /etc/alternatives/clang ]] ; then
+    sudo update-alternatives --install /usr/bin/clang  clang  /usr/bin/clang-${CLANG_VERSION} 999
+  fi
+  if [[ ! -L /etc/alternatives/clang++ ]] ; then
+    sudo update-alternatives --install /usr/bin/clang++  clang++  /usr/bin/clang++-${CLANG_VERSION} 999
+  fi
+  if [[ ! -L /etc/alternatives/clang-tidy ]] ; then
+    sudo update-alternatives --install /usr/bin/clang-tidy  clang-tidy  /usr/bin/clang-tidy-${CLANG_VERSION} 999
+  fi
+  if [[ ! -L /etc/alternatives/cc ]] ; then
+    sudo update-alternatives --install /usr/bin/cc  cc  /usr/bin/clang-${CLANG_VERSION} 999
+  fi
+  if [[ ! -L /etc/alternatives/cc++ ]] ; then
+    sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-${CLANG_VERSION} 999
+  fi
 }
 
 configure_geode_build() {
@@ -112,8 +123,22 @@ configure_geode_build() {
   export RAT_HOME="/opt/apache-rat-${RAT_VERSION}"
   export PATH="$PATH:$GEODE_HOME/bin"
 
-  # Get the latest geode-native build docker image
-  docker pull ${GEODE_NATIVE_DOCKER_IMAGE}
+  # Update hosts file with ip6-localhost required by geode-native Integration tests
+  sudo sed  -i '/ipv6-loopback$/s/$/ ip6-localhost/' /etc/hosts
+}
+
+get_geode_build_docker_image() {
+  echo "Info: Start and enable docker on system"
+  echo "-------------------------------------------------------------"
+  sudo systemctl enable docker
+  sudo systemctl start docker
+  echo "Info: Remove all previous stopped docker containers and images"
+  echo "-------------------------------------------------------------"
+  sudo docker container prune -f
+  sudo docker image prune -f
+  echo "Info: Get the latest geode-native build docker image"
+  echo "-------------------------------------------------------------"
+  docker pull ${GEODE_NATIVE_DOCKER_IMAGE:-apachegeode/geode-native-build}
 }
 
 # Make sure the build server is SLES15 based
@@ -126,10 +151,27 @@ case ${ID,,} in
     else
       install_dev_packages
       configure_geode_build
+      get_geode_build_docker_image
     fi
   ;;
+  ubuntu)
+    DPKG_LOCK="/var/lib/dpkg/lock-frontend"
+    # Wait for other apt process to finish by checking the dpkg lock file.
+    try=0
+    while sudo lsof ${DPKG_LOCK}  > /dev/null 2>&1 ; do
+      echo "DPKG file locked: ${DPKG_LOCK}."
+      echo "   Waiting for another pkg instalaltion process to finish ..."
+      sleep 10
+      if [[ ${try} -gt 60 ]] ; then
+        echo "ERROR: Max number of re-tries reached, exiting..."
+        exit 1
+      fi
+      try=$((try + 1))
+    done
+    get_geode_build_docker_image
+  ;;
   *)
-    echo "ERROR: Only SLES15 can be use as a build server for this build"
+    echo "ERROR: Only SLES15 and Ubuntu OS can be use as a build server geode-native"
     exit 2
   ;;
 esac

diff --git a/jjb/geode-native/apache-geode-native-test/apache-geode-native-new-integration-test.yaml b/jjb/geode-native/apache-geode-native-test/apache-geode-native-new-integration-test.yaml
new file mode 100644 (file)
index 0000000..d1d0240
--- /dev/null
+++ b/jjb/geode-native/apache-geode-native-test/apache-geode-native-new-integration-test.yaml
@@ -0,0 +1,27 @@
+---
+#
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2018-2019 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+- project:
+    name: 'apache-geode-native-NewIntegrationTest'
+    test-type: 'NewIntegrationTest'
+    build-executor: 'apache-geode-native-newintegrationtest-macro'
+
+    jobs:
+      - 'apache-geode-native-{test-type}'

diff --git a/jjb/geode-native/apache-geode-native-test/integrationtest.sh b/jjb/geode-native/apache-geode-native-test/integrationtest.sh
index 91d8ecf585bfc5baad6073f5f4521873ae999098..d6691c5969e329065ef97e1831f0800e1686d898 100755 (executable)
--- a/jjb/geode-native/apache-geode-native-test/integrationtest.sh
+++ b/jjb/geode-native/apache-geode-native-test/integrationtest.sh
@@ -21,15 +21,13 @@ set -o errexit
 set -o pipefail
 #set -o xtrace
 
-GEODE_NATIVE_DIR="$(pwd)/geode-native"
+GEODE_NATIVE_DIR="${WORKSPACE}/geode-native"
 SOURCE_DIR="/geode-native"
 
-DOCKER_ARGS="--volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}"
+DOCKER_ARGS="--rm --volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}"
 COMMAND="echo -e '\n*** Build ***\n' && mkdir build && cd build &&\
-    cmake ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_CXX_CLANG_TIDY=clang-tidy &&\
-    cmake --build . -- -j${JOBS} && echo -e '\n*** Old Integration Tests ***\n' &&\
-    cd ${SOURCE_DIR}/build/cppcache/integration-test && ctest --timeout 2000 -L STABLE -C Release -j1 &&\
-    echo -e '\n*** New Integration Tests ***\n' &&\
-    cd ${SOURCE_DIR}/build/cppcache/integration/test && ctest -j1"
+  cmake ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_CXX_CLANG_TIDY=clang-tidy &&\
+  cmake --build . -- -j${JOBS} && echo -e '\n*** Old Integration Tests ***\n' &&\
+  cd /build/cppcache/integration-test && ctest --timeout 2000 -L STABLE -C Release -j1"
 
 docker run ${DOCKER_ARGS} "${GEODE_NATIVE_DOCKER_IMAGE}" bash -c "${COMMAND}"

diff --git a/jjb/geode-native/apache-geode-native-test/new_integrationtest.sh b/jjb/geode-native/apache-geode-native-test/new_integrationtest.sh
new file mode 100755 (executable)
index 0000000..870cd93
--- /dev/null
+++ b/jjb/geode-native/apache-geode-native-test/new_integrationtest.sh
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2018-2019 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+set -o nounset
+set -o errexit
+set -o pipefail
+#set -o xtrace
+
+GEODE_NATIVE_DIR="${WORKSPACE}/geode-native"
+SOURCE_DIR="/geode-native"
+
+DOCKER_ARGS="--rm --volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}"
+COMMAND="echo -e '\n*** Build ***\n' && mkdir build && cd build &&\
+  cmake ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=-stdlib=libc++ \\
+  -DCMAKE_CXX_CLANG_TIDY=clang-tidy -DWITH_IPV6=ON &&\
+  cmake --build . -- -j${JOBS} && echo -e '\n*** New Integration Tests ***\n' &&\
+  cd /build/cppcache/integration/test && ctest -j1"
+
+docker run ${DOCKER_ARGS} "${GEODE_NATIVE_DOCKER_IMAGE}" bash -c "${COMMAND}"

diff --git a/jjb/geode-native/apache-geode-native-test/ratcheck.sh b/jjb/geode-native/apache-geode-native-test/ratcheck.sh
index 4fd7897defd72a68274b671449fd6008eb79f88c..0f2a054580470103a8ae4a8f476c21f3b4325cb6 100755 (executable)
--- a/jjb/geode-native/apache-geode-native-test/ratcheck.sh
+++ b/jjb/geode-native/apache-geode-native-test/ratcheck.sh
@@ -21,10 +21,10 @@ set -o errexit
 set -o pipefail
 #set -o xtrace
 
-GEODE_NATIVE_DIR="$(pwd)/geode-native"
+GEODE_NATIVE_DIR="${WORKSPACE}/geode-native"
 SOURCE_DIR="/geode-native"
 
-DOCKER_ARGS="--volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}"
+DOCKER_ARGS="--rm --volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}"
 COMMAND="echo -e '\n*** Rat Check ***\n' && mkdir build && cd build &&\
          cmake ${SOURCE_DIR} -DUSE_RAT=ON &&\
          cmake --build . --target rat-check"

diff --git a/jjb/geode-native/apache-geode-native-test/unittest.sh b/jjb/geode-native/apache-geode-native-test/unittest.sh
index fdc2c0f97f2ce9c7de8adc7e0e348b7d615b18b8..b9b42fe8bd24b1729c7036d0c7b23278bc006cf9 100755 (executable)
--- a/jjb/geode-native/apache-geode-native-test/unittest.sh
+++ b/jjb/geode-native/apache-geode-native-test/unittest.sh
@@ -21,10 +21,10 @@ set -o errexit
 set -o pipefail
 #set -o xtrace
 
-GEODE_NATIVE_DIR="$(pwd)/geode-native"
+GEODE_NATIVE_DIR="${WORKSPACE}/geode-native"
 SOURCE_DIR="/geode-native"
 
-DOCKER_ARGS="--volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}"
+DOCKER_ARGS="--rm --volume=${GEODE_NATIVE_DIR}:${SOURCE_DIR}"
 COMMAND="echo -e '\n*** Unit Test ***\n' && mkdir build && cd build &&\
     cmake ${SOURCE_DIR} -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_CXX_CLANG_TIDY=clang-tidy &&\
     cmake --build . -- -j${JOBS} && ./cppcache/test/apache-geode_unittests &&\

diff --git a/jjb/geode-native/apache-geode-native-verify.yaml b/jjb/geode-native/apache-geode-native-verify.yaml
index fe4dfbf816791b3fc54dea918efabd6067b92b8d..ca0668c04e05cee028ee0ad09482922cc96cae57 100644 (file)
--- a/jjb/geode-native/apache-geode-native-verify.yaml
+++ b/jjb/geode-native/apache-geode-native-verify.yaml
@@ -32,7 +32,7 @@
     project-type: multijob
     disabled: false
     concurrent: true
-    node: geode-sles15
+    node: geode-ubuntu1804
 
     parameters:
       - string:
@@ -84,41 +84,36 @@
               abort-all-job: true
               node-parameters: true
       - multijob:
-          name: RatCheck
+          name: Verification
           condition: SUCCESSFUL
-          execution-type: SEQUENTIALLY
+          execution-type: PARALLEL
           projects:
             - name: 'apache-geode-native-RatCheck'
               current-parameters: true
               git-revision: true
               abort-all-job: true
               node-parameters: true
-      - multijob:
-          name: UnitTests
-          condition: SUCCESSFUL
-          execution-type: SEQUENTIALLY
-          projects:
             - name: 'apache-geode-native-UnitTest'
               current-parameters: true
               git-revision: true
               abort-all-job: true
               node-parameters: true
-      - multijob:
-          name: IntegrationTests
-          condition: SUCCESSFUL
-          execution-type: SEQUENTIALLY
-          projects:
             - name: 'apache-geode-native-IntegrationTest'
               current-parameters: true
               git-revision: true
               abort-all-job: true
               node-parameters: true
+            - name: 'apache-geode-native-NewIntegrationTest'
+              current-parameters: true
+              git-revision: true
+              abort-all-job: true
+              node-parameters: true
 #      - multijob:
-#          name: Build&Release
+#          name: Release
 #          condition: SUCCESSFUL
 #          execution-type: SEQUENTIALLY
 #          projects:
-#            - name: 'apache-geode-native-build-release'
+#            - name: 'apache-geode-native-SrcRelease'
 #              current-parameters: true
 #              git-revision: true
 #              abort-all-job: true

diff --git a/jjb/geode-native/global-template-apache-geode-native-test.yaml b/jjb/geode-native/global-template-apache-geode-native-test.yaml
index 2d4f4343fb7d7f677df467512bcf6b2ad70590e3..cedc0dbb5c7d22d31a42557b9ef5a0f885957aa2 100644 (file)
--- a/jjb/geode-native/global-template-apache-geode-native-test.yaml
+++ b/jjb/geode-native/global-template-apache-geode-native-test.yaml
@@ -20,7 +20,7 @@
 - job-template:
     name: 'apache-geode-native-{test-type}'
 
-    node: geode-sles15
+    node: geode-ubuntu1804
     disabled: false
     concurrent: true