From 24834fe75c946094230a51338b123c31b5e9a027 Mon Sep 17 00:00:00 2001 From: Fatih Degirmenci Date: Wed, 8 Jan 2020 18:52:01 +0100 Subject: [PATCH] cloud-infra: Switch engine lint jenkins jobs to tox This change switches to tox for engine jenkins lint jobs to run lint in standardized and easier way. This also helps us to reduce the duplication in jjb. Apart from that, all jobs are set to vote now. But, yamllint errors are suppressed in engine tox.ini so the job will never fail, helping us to see the existing issues for fixing them over time. Once we fix yamllint issues, tox.ini will be updated in order to make yamllint issues to be caught thus block the change from merging. Change-Id: I66692a8db944c6e2d35cd229bf7a23ff4c0bff07 --- .../cloud-infra-engine-verify-lint.yaml | 29 +++----- jjb/cloud-infra/cloud-infra-macros.yaml | 16 +---- .../scripts/cloud-infra-ansible-lint.sh | 71 ------------------- ...nfra-shellcheck.sh => cloud-infra-lint.sh} | 56 +++++++-------- .../scripts/cloud-infra-yaml-lint.sh | 71 ------------------- 5 files changed, 38 insertions(+), 205 deletions(-) delete mode 100644 jjb/cloud-infra/scripts/cloud-infra-ansible-lint.sh rename jjb/cloud-infra/scripts/{cloud-infra-shellcheck.sh => cloud-infra-lint.sh} (53%) delete mode 100644 jjb/cloud-infra/scripts/cloud-infra-yaml-lint.sh diff --git a/jjb/cloud-infra/cloud-infra-engine-verify-lint.yaml b/jjb/cloud-infra/cloud-infra-engine-verify-lint.yaml index 01f3c42ef..e766c0173 100644 --- a/jjb/cloud-infra/cloud-infra-engine-verify-lint.yaml +++ b/jjb/cloud-infra/cloud-infra-engine-verify-lint.yaml @@ -28,21 +28,9 @@ disabled: false lint-type: - - ansible-lint: - successful: false - failed: false - unstable: false - notbuilt: false - - yaml-lint: - successful: true - failed: true - unstable: true - notbuilt: true - - shellcheck: - successful: true - failed: true - unstable: true - notbuilt: true + - ansible-lint + - yamllint + - shellcheck jobs: - 'cloud-infra-verify-engine-{lint-type}-{stream}' @@ -61,6 +49,10 @@ project: '{project}' branch: '{branch}' - nordix-gerrit-parameters + - string: + name: LINT_TYPE + default: '{lint-type}' + description: 'Lint type to run' properties: - logrotate @@ -100,14 +92,9 @@ file-paths: - compare-type: 'ANT' pattern: '**' - skip-vote: - successful: '{obj:successful}' - failed: '{obj:failed}' - unstable: '{obj:unstable}' - notbuilt: '{obj:notbuilt}' custom-url: '* $JOB_NAME $BUILD_URL' builders: - - 'cloud-infra-{lint-type}-macro' + - 'cloud-infra-lint-macro' # vim: set ts=2 sw=2 expandtab: diff --git a/jjb/cloud-infra/cloud-infra-macros.yaml b/jjb/cloud-infra/cloud-infra-macros.yaml index 17a2fb973..34c6772c3 100644 --- a/jjb/cloud-infra/cloud-infra-macros.yaml +++ b/jjb/cloud-infra/cloud-infra-macros.yaml @@ -108,21 +108,11 @@ !include-raw: ./scenario-promote.sh - builder: - name: 'cloud-infra-ansible-lint-macro' + name: 'cloud-infra-lint-macro' builders: - shell: - !include-raw: ./scripts/cloud-infra-ansible-lint.sh - -- builder: - name: 'cloud-infra-yaml-lint-macro' - builders: - - shell: - !include-raw: ./scripts/cloud-infra-yaml-lint.sh - -- builder: - name: 'cloud-infra-shellcheck-macro' - builders: + !include-raw: ./wait-for-pkg-manager.sh - shell: - !include-raw: ./scripts/cloud-infra-shellcheck.sh + !include-raw: ./scripts/cloud-infra-lint.sh # vim: set ts=2 sw=2 expandtab: diff --git a/jjb/cloud-infra/scripts/cloud-infra-ansible-lint.sh b/jjb/cloud-infra/scripts/cloud-infra-ansible-lint.sh deleted file mode 100644 index 03f32ee63..000000000 --- a/jjb/cloud-infra/scripts/cloud-infra-ansible-lint.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# ============LICENSE_START======================================================= -# Copyright (C) 2019 The Nordix Foundation. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# ============LICENSE_END========================================================= - -# source engine-vars.sh so we can install the correct versions of -# pip, ansible, and ansible-lint -source $WORKSPACE/engine/config/engine-vars.sh - -# install dependencies -sudo apt update > /dev/null 2>&1 -sudo apt install -y -q=3 gcc libffi-dev libssl-dev lsb-release libpython3-dev \ - python3-minimal python3-pip python3-yaml virtualenv > /dev/null 2>&1 - -# override ENGINE_ANSIBLE_LINT_VERSION locally while ansible-lint patch is still open -export ENGINE_ANSIBLE_LINT_VERSION=4.1.0 - -cd $WORKSPACE -echo "Info: Install pip==$ENGINE_PIP_VERSION ansible==$ENGINE_ANSIBLE_VERSION and ansible-lint==$ENGINE_ANSIBLE_LINT_VERSION" -echo "----------------------------------------------------" - -# install pip, ansible, and ansible-lint in venv -export ANSIBLE_VENV=$WORKSPACE/.venv -virtualenv -p python3 --quiet --no-site-packages ${ANSIBLE_VENV} -source ${ANSIBLE_VENV}/bin/activate -pip -q install --upgrade pip==$ENGINE_PIP_VERSION -pip -q install --upgrade virtualenv pip setuptools shade \ - ansible==$ENGINE_ANSIBLE_VERSION ansible-lint==$ENGINE_ANSIBLE_LINT_VERSION - -echo "----------------------------------------------------" -echo "Info: Done" -echo "Info: Run ansible-lint on playbooks" -echo "----------------------------------------------------" - -# set exit_code so we exit ourselves rather than ansible-lint bailing out without -# verifying all the playbooks -declare -i exit_code=0 - -# lint the playbooks -cd $WORKSPACE/engine -for playbook in $(find . -type f -name *.yml | sort); do - echo "--> Checking '${playbook}' playbook..." - ansible-lint --nocolor ${playbook} - lint_exit_code=$? - if [[ $lint_exit_code != 0 ]]; then - exit_code=$lint_exit_code - else - echo [OK] - fi - echo -done -echo "----------------------------------------------------" -echo "Info: Done" -exit $exit_code - -# vim: set ts=2 sw=2 expandtab: diff --git a/jjb/cloud-infra/scripts/cloud-infra-shellcheck.sh b/jjb/cloud-infra/scripts/cloud-infra-lint.sh similarity index 53% rename from jjb/cloud-infra/scripts/cloud-infra-shellcheck.sh rename to jjb/cloud-infra/scripts/cloud-infra-lint.sh index 48ea0fcb2..2516b4a55 100644 --- a/jjb/cloud-infra/scripts/cloud-infra-shellcheck.sh +++ b/jjb/cloud-infra/scripts/cloud-infra-lint.sh @@ -18,39 +18,37 @@ # SPDX-License-Identifier: Apache-2.0 # ============LICENSE_END========================================================= +set -o nounset +set -o errexit +set -o pipefail + +# ensure we are in job build WORKSPACE +cd "$WORKSPACE" + # install dependencies -sudo apt update > /dev/null 2>&1 -sudo apt remove -y -q=3 shellcheck > /dev/null 2>&1 +echo "Info: Install virtualenv python3-minimal using apt" +sudo apt update -q=3 > /dev/null 2>&1 +sudo apt install -q=3 -y python3-minimal virtualenv > /dev/null 2>&1 -# apt doesn't have the shellcheck version v0.7.0 so we install -# it from snap -sudo snap install shellcheck -SHELLCHECK_EXECUTABLE=/snap/bin/shellcheck -echo "Info: Using shellcheck version $(${SHELLCHECK_EXECUTABLE} --version)" +# create and activate virtualenv +echo "Info: Create and activate python virtualenv" +virtualenv -p python3 .venv > /dev/null 2>&1 +set +u +source .venv/bin/activate > /dev/null 2>&1 +set -u -cd $WORKSPACE -echo "Info: Run shellcheck on bash scripts" -echo "----------------------------------------------------" +# install test-requirements +echo "Info: Install python packages listed in test-requirements.txt using pip" +pip install --quiet -r test-requirements.txt > /dev/null 2>&1 -# set exit_code so we exit ourselves rather than shellcheck bailing out without -# verifying all the bash scripts -declare -i exit_code=0 - -# lint the bash scripts -for shell_script in $(find ${WORKSPACE}/ -type f -name *.sh); do - echo "--> Checking '${shell_script}' shell script..." - "${SHELLCHECK_EXECUTABLE}" --color=never --source-path="${WORKSPACE}" --external-sources \ - --format=tty ${shell_script} - lint_exit_code=$? - if [[ $lint_exit_code != 0 ]]; then - exit_code=$lint_exit_code - else - echo [OK] - fi - echo -done +# set default lint type +export LINT_TYPE="${LINT_TYPE:-ansible-lint}" + +# run tox +echo "Info: Run $LINT_TYPE using tox" +echo "----------------------------------------------------" +tox -e "$LINT_TYPE" echo "----------------------------------------------------" -echo "Info: Done" -exit $exit_code +echo "Info: Done!" # vim: set ts=2 sw=2 expandtab: diff --git a/jjb/cloud-infra/scripts/cloud-infra-yaml-lint.sh b/jjb/cloud-infra/scripts/cloud-infra-yaml-lint.sh deleted file mode 100644 index c0f2577e4..000000000 --- a/jjb/cloud-infra/scripts/cloud-infra-yaml-lint.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# ============LICENSE_START======================================================= -# Copyright (C) 2019 The Nordix Foundation. All rights reserved. -# ================================================================================ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# ============LICENSE_END========================================================= - -# source engine-vars.sh so we can install the correct versions of -# pip, ansible, and ansible-lint -source $WORKSPACE/engine/config/engine-vars.sh - -# install dependencies -sudo apt update > /dev/null 2>&1 -sudo apt install -y -q=3 gcc libffi-dev libssl-dev lsb-release libpython3-dev \ - python3-minimal python3-pip python3-yaml virtualenv > /dev/null 2>&1 - -# override ENGINE_YAML_LINT_VERSION locally while yamllint patch is still open -export ENGINE_YAML_LINT_VERSION=1.19.0 - -cd $WORKSPACE -echo "Info: Install pip==$ENGINE_PIP_VERSION ansible==$ENGINE_ANSIBLE_VERSION and yamllint==$ENGINE_YAML_LINT_VERSION" -echo "----------------------------------------------------" - -# install pip, ansible, and yamllint in venv -export ANSIBLE_VENV=$WORKSPACE/.venv -virtualenv -p python3 --quiet --no-site-packages ${ANSIBLE_VENV} -source ${ANSIBLE_VENV}/bin/activate -pip -q install --upgrade pip==$ENGINE_PIP_VERSION -pip -q install --upgrade virtualenv pip setuptools shade \ - ansible==$ENGINE_ANSIBLE_VERSION yamllint==$ENGINE_YAML_LINT_VERSION - -echo "----------------------------------------------------" -echo "Info: Done" -echo "Info: Run yamllint on yaml files" -echo "----------------------------------------------------" - -# set exit_code so we exit ourselves rather than ansible-lint bailing out without -# verifying all the yaml files -declare -i exit_code=0 - -# lint the yaml files -cd $WORKSPACE/engine -for yaml_file in $(find . -type f -name *.yml | sort); do - echo "--> Checking '${yaml_file}' yaml file..." - yamllint --format standard --strict ${yaml_file} - lint_exit_code=$? - if [[ $lint_exit_code != 0 ]]; then - exit_code=$lint_exit_code - else - echo [OK] - fi - echo -done -echo "----------------------------------------------------" -echo "Info: Done" -exit $exit_code - -# vim: set ts=2 sw=2 expandtab: -- 2.25.1