engine: Refactor tox jobs

Change-Id: I6776a10b472e9767ecc19896fbd7835880784adc

diff --git a/jjb/engine/engine-macros.yaml b/jjb/engine/engine-macros.yaml
new file mode 100644 (file)
index 0000000..b480a6a
--- /dev/null
+++ b/jjb/engine/engine-macros.yaml
@@ -0,0 +1,28 @@
+---
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2019 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+#-------------------------------------------------------------------------------
+# Macros are used to avoid hitting parameter expansion of curly brackets and
+# the need to using include-raw-escape
+#-------------------------------------------------------------------------------
+- builder:
+    name: 'engine-verify-tox-macro'
+    builders:
+      - shell:
+          !include-raw: ./scripts/engine-verify-tox.sh

diff --git a/jjb/engine/engine-verify-tox.yaml b/jjb/engine/engine-verify-tox.yaml
new file mode 100644 (file)
index 0000000..4ed2c3f
--- /dev/null
+++ b/jjb/engine/engine-verify-tox.yaml
@@ -0,0 +1,217 @@
+---
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2019 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+- project:
+    name: 'engine-tox'
+
+    project: 'engine-tox'
+
+    stream:
+      - 'master':
+          branch: '{stream}'
+
+    lint-type:
+      - ansible-lint:
+          disabled: false
+          repos-to-ignore: 'infra/hwconfig'
+      - docs:
+          disabled: false
+          repos-to-ignore: 'infra/hwconfig|infra/swconfig|infra/test'
+      - shellcheck:
+          disabled: false
+          repos-to-ignore: 'infra/hwconfig'
+      - yamllint:
+          disabled: false
+          repos-to-ignore: ''
+
+    jobs:
+      - 'engine-verify-tox-{stream}'
+      - 'engine-verify-{lint-type}-{stream}'
+
+- job-template:
+    name: 'engine-verify-tox-{stream}'
+
+    project-type: multijob
+
+    concurrent: true
+
+    node: infra-tools-docker-slave-ubuntu1804
+
+    parameters:
+      - project-parameters:
+          project: $GERRIT_PROJECT
+          branch: '{branch}'
+          description: 'The project and branch which the change is proposed for'
+      - string:
+          name: VERBOSITY
+          default: 'false'
+          description: 'Verbosity setting for the tox script'
+      - nordix-gerrit-parameters
+
+    properties:
+      - logrotate
+
+    wrappers:
+      - build-timeout:
+          timeout: 60
+      - fix-workspace-permissions
+
+    scm:
+      - git-scm-gerrit:
+          ssh-credentials-id: nordixinfra-nordix-gerrit-ssh
+          branch: '{branch}'
+          refspec: $GERRIT_REFSPEC
+          wipe_workspace: true
+          clean_before: false
+
+    triggers:
+      - gerrit:
+          server-name: 'Nordix Gerrit'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+            - comment-added-contains-event:
+                comment-contains-value: 'reverify'
+          projects:
+            - project-compare-type: 'REG_EXP'
+              project-pattern: 'infra\/engine|infra\/test|infra\/swconfig|infra\/hwconfig'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              disable-strict-forbidden-file-verification: 'false'
+              forbidden-file-paths:
+                - compare-type: ANT
+                  pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py'
+          custom-url: '* $JOB_NAME $BUILD_URL'
+
+    builders:
+      - multijob:
+          name: ansible-lint
+          condition: ALWAYS
+          projects:
+            - name: 'engine-verify-ansible-lint-{stream}'
+              current-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: false
+              predefined-parameters: |
+                GERRIT_PROJECT=$GERRIT_PROJECT
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                VERBOSITY=$VERBOSITY
+      - multijob:
+          name: shellcheck
+          condition: ALWAYS
+          projects:
+            - name: 'engine-verify-shellcheck-{stream}'
+              current-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: false
+              predefined-parameters: |
+                GERRIT_PROJECT=$GERRIT_PROJECT
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                VERBOSITY=$VERBOSITY
+      - multijob:
+          name: yamllint
+          condition: ALWAYS
+          projects:
+            - name: 'engine-verify-yamllint-{stream}'
+              current-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: false
+              predefined-parameters: |
+                GERRIT_PROJECT=$GERRIT_PROJECT
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                VERBOSITY=$VERBOSITY
+      - multijob:
+          name: docs
+          condition: ALWAYS
+          projects:
+            - name: 'engine-verify-ansible-lint-{stream}'
+              current-parameters: true
+              kill-phase-on: NEVER
+              abort-all-job: false
+              predefined-parameters: |
+                GERRIT_PROJECT=$GERRIT_PROJECT
+                GERRIT_BRANCH=$GERRIT_BRANCH
+                GERRIT_REFSPEC=$GERRIT_REFSPEC
+                GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER
+                GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE
+                VERBOSITY=$VERBOSITY
+
+- job-template:
+    name: 'engine-verify-{lint-type}-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    concurrent: true
+
+    node: infra-tools-docker-slave-ubuntu1804
+
+    parameters:
+      - string:
+          name: PROJECT
+          default: $GERRIT_PROJECT
+          description: 'The project which the change is proposed for'
+      - string:
+          name: LINT_TYPE
+          default: '{lint-type}'
+          description: 'Lint type to run'
+      - string:
+          name: REPOS_TO_IGNORE
+          default: '{repos-to-ignore}'
+          description: 'Repos to ignore for the specific lint type'
+      - string:
+          name: VERBOSITY
+          default: 'false'
+          description: 'Verbosity setting for the tox script'
+      - nordix-gerrit-parameters
+
+    properties:
+      - logrotate
+
+    wrappers:
+      - build-timeout:
+          timeout: 150
+      - fix-workspace-permissions
+
+    scm:
+      - git-scm-gerrit:
+          ssh-credentials-id: nordixinfra-nordix-gerrit-ssh
+          branch: '{branch}'
+          refspec: $GERRIT_REFSPEC
+          wipe_workspace: true
+          clean_before: false
+
+    builders:
+      - 'engine-verify-tox-macro'
+
+# vim: set ts=2 sw=2 expandtab:

diff --git a/jjb/engine/scripts/engine-verify-tox.sh b/jjb/engine/scripts/engine-verify-tox.sh
new file mode 100755 (executable)
index 0000000..b3a64c6
--- /dev/null
+++ b/jjb/engine/scripts/engine-verify-tox.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# ============LICENSE_START=======================================================
+#  Copyright (C) 2019 The Nordix Foundation. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+#-------------------------------------------------------------------------------
+# Various tests are run using tox such as docs, yamllint, ansible-lint, and
+# shellcheck. Some repos do not have certain types of files and those repos
+# are ignored using REPOS_TO_IGNORE variable which is controlled within the
+# jjb per lint type.
+#-------------------------------------------------------------------------------
+run_tox() {
+
+  echo "Info  : Preparing to run tox for the repo $GERRIT_PROJECT"
+
+  # set defaults
+  VERBOSITY=${VERBOSITY:-false}
+  LINT_TYPE="${LINT_TYPE:-ansible-lint}"
+  export VERBOSITY LINT_TYPE
+
+  # check if the change is for a repo we should ignore
+  if [[ "$REPOS_TO_IGNORE" =~ "$GERRIT_PROJECT" ]]; then
+    echo "Info  : Ignoring change as $GERRIT_PROJECT is in ignore list!"
+    echo "Info  : Done!"
+    exit 0
+  fi
+
+  # ensure we are in job build WORKSPACE
+  cd "$WORKSPACE"
+
+  # set DEBIAN_FRONTEND to run apt non-interactively
+  DEBIAN_FRONTEND=noninteractive
+  export DEBIAN_FRONTEND
+
+  # install dependencies
+  echo "Info  : Install python3.6-minimal python3-distutils virtualenv using apt"
+  redirect_cmd sudo apt update
+  redirect_cmd sudo apt install -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew -y python3.6-minimal python3-distutils virtualenv
+
+  # create and activate virtualenv
+  echo "Info  : Create and activate python virtualenv"
+  redirect_cmd virtualenv -p python3.6 .venv
+  set +u
+  redirect_cmd source .venv/bin/activate
+  set -u
+
+  # install test-requirements
+  echo "Info  : Install python packages listed in test-requirements.txt using pip"
+  redirect_cmd pip install --force-reinstall -r test-requirements.txt
+
+  # run tox
+  echo "Info  : Run $LINT_TYPE using tox"
+  echo "----------------------------------------------------"
+  tox -e "$LINT_TYPE"
+  echo "----------------------------------------------------"
+  echo "Info  : Done!"
+
+}
+#-------------------------------------------------------------------------------
+# In some cases, it is useful to see all the output generated by commands so
+# this function makes it possible for users to achieve that by not redirecting
+# output to /dev/null when verbosity is enabled
+#-------------------------------------------------------------------------------
+redirect_cmd() {
+
+  if [[ "$VERBOSITY" == "false" ]]; then
+    "$@" > /dev/null 2>&1
+  else
+    "$@"
+  fi
+
+}
+#-------------------------------------------------------------------------------
+# run tox
+#-------------------------------------------------------------------------------
+
+run_tox "$@"
+
+# vim: set ts=2 sw=2 expandtab: