From 44122b30dd9968ba5e708618ef982c8fb09923f0 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Wed, 11 Mar 2020 13:03:25 +0100 Subject: [PATCH] engine: Refactor tox jobs Change-Id: I6776a10b472e9767ecc19896fbd7835880784adc --- jjb/engine/engine-macros.yaml | 28 +++ jjb/engine/engine-verify-tox.yaml | 217 ++++++++++++++++++++++++ jjb/engine/scripts/engine-verify-tox.sh | 98 +++++++++++ 3 files changed, 343 insertions(+) create mode 100644 jjb/engine/engine-macros.yaml create mode 100644 jjb/engine/engine-verify-tox.yaml create mode 100755 jjb/engine/scripts/engine-verify-tox.sh diff --git a/jjb/engine/engine-macros.yaml b/jjb/engine/engine-macros.yaml new file mode 100644 index 00000000..b480a6ac --- /dev/null +++ b/jjb/engine/engine-macros.yaml @@ -0,0 +1,28 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +#------------------------------------------------------------------------------- +# Macros are used to avoid hitting parameter expansion of curly brackets and +# the need to using include-raw-escape +#------------------------------------------------------------------------------- +- builder: + name: 'engine-verify-tox-macro' + builders: + - shell: + !include-raw: ./scripts/engine-verify-tox.sh diff --git a/jjb/engine/engine-verify-tox.yaml b/jjb/engine/engine-verify-tox.yaml new file mode 100644 index 00000000..4ed2c3f5 --- /dev/null +++ b/jjb/engine/engine-verify-tox.yaml @@ -0,0 +1,217 @@ +--- +# ============LICENSE_START======================================================= +# Copyright (C) 2019 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +- project: + name: 'engine-tox' + + project: 'engine-tox' + + stream: + - 'master': + branch: '{stream}' + + lint-type: + - ansible-lint: + disabled: false + repos-to-ignore: 'infra/hwconfig' + - docs: + disabled: false + repos-to-ignore: 'infra/hwconfig|infra/swconfig|infra/test' + - shellcheck: + disabled: false + repos-to-ignore: 'infra/hwconfig' + - yamllint: + disabled: false + repos-to-ignore: '' + + jobs: + - 'engine-verify-tox-{stream}' + - 'engine-verify-{lint-type}-{stream}' + +- job-template: + name: 'engine-verify-tox-{stream}' + + project-type: multijob + + concurrent: true + + node: infra-tools-docker-slave-ubuntu1804 + + parameters: + - project-parameters: + project: $GERRIT_PROJECT + branch: '{branch}' + description: 'The project and branch which the change is proposed for' + - string: + name: VERBOSITY + default: 'false' + description: 'Verbosity setting for the tox script' + - nordix-gerrit-parameters + + properties: + - logrotate + + wrappers: + - build-timeout: + timeout: 60 + - fix-workspace-permissions + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: '{branch}' + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false + + triggers: + - gerrit: + server-name: 'Nordix Gerrit' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: 'infra\/engine|infra\/test|infra\/swconfig|infra\/hwconfig' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + disable-strict-forbidden-file-verification: 'false' + forbidden-file-paths: + - compare-type: ANT + pattern: '.gitignore|.gitreview|.yamllint|README.md|bindep.txt|setup.cfg|setup.py' + custom-url: '* $JOB_NAME $BUILD_URL' + + builders: + - multijob: + name: ansible-lint + condition: ALWAYS + projects: + - name: 'engine-verify-ansible-lint-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + predefined-parameters: | + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + VERBOSITY=$VERBOSITY + - multijob: + name: shellcheck + condition: ALWAYS + projects: + - name: 'engine-verify-shellcheck-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + predefined-parameters: | + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + VERBOSITY=$VERBOSITY + - multijob: + name: yamllint + condition: ALWAYS + projects: + - name: 'engine-verify-yamllint-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + predefined-parameters: | + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + VERBOSITY=$VERBOSITY + - multijob: + name: docs + condition: ALWAYS + projects: + - name: 'engine-verify-ansible-lint-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + predefined-parameters: | + GERRIT_PROJECT=$GERRIT_PROJECT + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + VERBOSITY=$VERBOSITY + +- job-template: + name: 'engine-verify-{lint-type}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + node: infra-tools-docker-slave-ubuntu1804 + + parameters: + - string: + name: PROJECT + default: $GERRIT_PROJECT + description: 'The project which the change is proposed for' + - string: + name: LINT_TYPE + default: '{lint-type}' + description: 'Lint type to run' + - string: + name: REPOS_TO_IGNORE + default: '{repos-to-ignore}' + description: 'Repos to ignore for the specific lint type' + - string: + name: VERBOSITY + default: 'false' + description: 'Verbosity setting for the tox script' + - nordix-gerrit-parameters + + properties: + - logrotate + + wrappers: + - build-timeout: + timeout: 150 + - fix-workspace-permissions + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: '{branch}' + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false + + builders: + - 'engine-verify-tox-macro' + +# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/engine/scripts/engine-verify-tox.sh b/jjb/engine/scripts/engine-verify-tox.sh new file mode 100755 index 00000000..b3a64c6e --- /dev/null +++ b/jjb/engine/scripts/engine-verify-tox.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# ============LICENSE_START======================================================= +# Copyright (C) 2019 The Nordix Foundation. All rights reserved. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= + +set -o nounset +set -o errexit +set -o pipefail + +#------------------------------------------------------------------------------- +# Various tests are run using tox such as docs, yamllint, ansible-lint, and +# shellcheck. Some repos do not have certain types of files and those repos +# are ignored using REPOS_TO_IGNORE variable which is controlled within the +# jjb per lint type. +#------------------------------------------------------------------------------- +run_tox() { + + echo "Info : Preparing to run tox for the repo $GERRIT_PROJECT" + + # set defaults + VERBOSITY=${VERBOSITY:-false} + LINT_TYPE="${LINT_TYPE:-ansible-lint}" + export VERBOSITY LINT_TYPE + + # check if the change is for a repo we should ignore + if [[ "$REPOS_TO_IGNORE" =~ "$GERRIT_PROJECT" ]]; then + echo "Info : Ignoring change as $GERRIT_PROJECT is in ignore list!" + echo "Info : Done!" + exit 0 + fi + + # ensure we are in job build WORKSPACE + cd "$WORKSPACE" + + # set DEBIAN_FRONTEND to run apt non-interactively + DEBIAN_FRONTEND=noninteractive + export DEBIAN_FRONTEND + + # install dependencies + echo "Info : Install python3.6-minimal python3-distutils virtualenv using apt" + redirect_cmd sudo apt update + redirect_cmd sudo apt install -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew -y python3.6-minimal python3-distutils virtualenv + + # create and activate virtualenv + echo "Info : Create and activate python virtualenv" + redirect_cmd virtualenv -p python3.6 .venv + set +u + redirect_cmd source .venv/bin/activate + set -u + + # install test-requirements + echo "Info : Install python packages listed in test-requirements.txt using pip" + redirect_cmd pip install --force-reinstall -r test-requirements.txt + + # run tox + echo "Info : Run $LINT_TYPE using tox" + echo "----------------------------------------------------" + tox -e "$LINT_TYPE" + echo "----------------------------------------------------" + echo "Info : Done!" + +} +#------------------------------------------------------------------------------- +# In some cases, it is useful to see all the output generated by commands so +# this function makes it possible for users to achieve that by not redirecting +# output to /dev/null when verbosity is enabled +#------------------------------------------------------------------------------- +redirect_cmd() { + + if [[ "$VERBOSITY" == "false" ]]; then + "$@" > /dev/null 2>&1 + else + "$@" + fi + +} +#------------------------------------------------------------------------------- +# run tox +#------------------------------------------------------------------------------- + +run_tox "$@" + +# vim: set ts=2 sw=2 expandtab: -- 2.25.1