cloud-infra: Switch engine lint jenkins jobs to tox
This change switches to tox for engine jenkins lint jobs to run
lint in standardized and easier way. This also helps us to reduce
the duplication in jjb.
Apart from that, all jobs are set to vote now. But, yamllint errors
are suppressed in engine tox.ini so the job will never fail, helping
us to see the existing issues for fixing them over time. Once we
fix yamllint issues, tox.ini will be updated in order to make yamllint
issues to be caught thus block the change from merging.
Change-Id: I66692a8db944c6e2d35cd229bf7a23ff4c0bff07
diff --git a/jjb/cloud-infra/cloud-infra-engine-verify-lint.yaml b/jjb/cloud-infra/cloud-infra-engine-verify-lint.yaml
index 01f3c42..e766c01 100644
--- a/jjb/cloud-infra/cloud-infra-engine-verify-lint.yaml
+++ b/jjb/cloud-infra/cloud-infra-engine-verify-lint.yaml
@@ -28,21 +28,9 @@
disabled: false
lint-type:
- - ansible-lint:
- successful: false
- failed: false
- unstable: false
- notbuilt: false
- - yaml-lint:
- successful: true
- failed: true
- unstable: true
- notbuilt: true
- - shellcheck:
- successful: true
- failed: true
- unstable: true
- notbuilt: true
+ - ansible-lint
+ - yamllint
+ - shellcheck
jobs:
- 'cloud-infra-verify-engine-{lint-type}-{stream}'
@@ -61,6 +49,10 @@
project: '{project}'
branch: '{branch}'
- nordix-gerrit-parameters
+ - string:
+ name: LINT_TYPE
+ default: '{lint-type}'
+ description: 'Lint type to run'
properties:
- logrotate
@@ -100,14 +92,9 @@
file-paths:
- compare-type: 'ANT'
pattern: '**'
- skip-vote:
- successful: '{obj:successful}'
- failed: '{obj:failed}'
- unstable: '{obj:unstable}'
- notbuilt: '{obj:notbuilt}'
custom-url: '* $JOB_NAME $BUILD_URL'
builders:
- - 'cloud-infra-{lint-type}-macro'
+ - 'cloud-infra-lint-macro'
# vim: set ts=2 sw=2 expandtab:
diff --git a/jjb/cloud-infra/cloud-infra-macros.yaml b/jjb/cloud-infra/cloud-infra-macros.yaml
index 17a2fb9..34c6772 100644
--- a/jjb/cloud-infra/cloud-infra-macros.yaml
+++ b/jjb/cloud-infra/cloud-infra-macros.yaml
@@ -108,21 +108,11 @@
!include-raw: ./scenario-promote.sh
- builder:
- name: 'cloud-infra-ansible-lint-macro'
+ name: 'cloud-infra-lint-macro'
builders:
- shell:
- !include-raw: ./scripts/cloud-infra-ansible-lint.sh
-
-- builder:
- name: 'cloud-infra-yaml-lint-macro'
- builders:
+ !include-raw: ./wait-for-pkg-manager.sh
- shell:
- !include-raw: ./scripts/cloud-infra-yaml-lint.sh
-
-- builder:
- name: 'cloud-infra-shellcheck-macro'
- builders:
- - shell:
- !include-raw: ./scripts/cloud-infra-shellcheck.sh
+ !include-raw: ./scripts/cloud-infra-lint.sh
# vim: set ts=2 sw=2 expandtab:
diff --git a/jjb/cloud-infra/scripts/cloud-infra-ansible-lint.sh b/jjb/cloud-infra/scripts/cloud-infra-ansible-lint.sh
deleted file mode 100644
index 03f32ee..0000000
--- a/jjb/cloud-infra/scripts/cloud-infra-ansible-lint.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/bash
-
-# ============LICENSE_START=======================================================
-# Copyright (C) 2019 The Nordix Foundation. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# SPDX-License-Identifier: Apache-2.0
-# ============LICENSE_END=========================================================
-
-# source engine-vars.sh so we can install the correct versions of
-# pip, ansible, and ansible-lint
-source $WORKSPACE/engine/config/engine-vars.sh
-
-# install dependencies
-sudo apt update > /dev/null 2>&1
-sudo apt install -y -q=3 gcc libffi-dev libssl-dev lsb-release libpython3-dev \
- python3-minimal python3-pip python3-yaml virtualenv > /dev/null 2>&1
-
-# override ENGINE_ANSIBLE_LINT_VERSION locally while ansible-lint patch is still open
-export ENGINE_ANSIBLE_LINT_VERSION=4.1.0
-
-cd $WORKSPACE
-echo "Info: Install pip==$ENGINE_PIP_VERSION ansible==$ENGINE_ANSIBLE_VERSION and ansible-lint==$ENGINE_ANSIBLE_LINT_VERSION"
-echo "----------------------------------------------------"
-
-# install pip, ansible, and ansible-lint in venv
-export ANSIBLE_VENV=$WORKSPACE/.venv
-virtualenv -p python3 --quiet --no-site-packages ${ANSIBLE_VENV}
-source ${ANSIBLE_VENV}/bin/activate
-pip -q install --upgrade pip==$ENGINE_PIP_VERSION
-pip -q install --upgrade virtualenv pip setuptools shade \
- ansible==$ENGINE_ANSIBLE_VERSION ansible-lint==$ENGINE_ANSIBLE_LINT_VERSION
-
-echo "----------------------------------------------------"
-echo "Info: Done"
-echo "Info: Run ansible-lint on playbooks"
-echo "----------------------------------------------------"
-
-# set exit_code so we exit ourselves rather than ansible-lint bailing out without
-# verifying all the playbooks
-declare -i exit_code=0
-
-# lint the playbooks
-cd $WORKSPACE/engine
-for playbook in $(find . -type f -name *.yml | sort); do
- echo "--> Checking '${playbook}' playbook..."
- ansible-lint --nocolor ${playbook}
- lint_exit_code=$?
- if [[ $lint_exit_code != 0 ]]; then
- exit_code=$lint_exit_code
- else
- echo [OK]
- fi
- echo
-done
-echo "----------------------------------------------------"
-echo "Info: Done"
-exit $exit_code
-
-# vim: set ts=2 sw=2 expandtab:
diff --git a/jjb/cloud-infra/scripts/cloud-infra-lint.sh b/jjb/cloud-infra/scripts/cloud-infra-lint.sh
new file mode 100644
index 0000000..2516b4a
--- /dev/null
+++ b/jjb/cloud-infra/scripts/cloud-infra-lint.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# ============LICENSE_START=======================================================
+# Copyright (C) 2019 The Nordix Foundation. All rights reserved.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+# ensure we are in job build WORKSPACE
+cd "$WORKSPACE"
+
+# install dependencies
+echo "Info: Install virtualenv python3-minimal using apt"
+sudo apt update -q=3 > /dev/null 2>&1
+sudo apt install -q=3 -y python3-minimal virtualenv > /dev/null 2>&1
+
+# create and activate virtualenv
+echo "Info: Create and activate python virtualenv"
+virtualenv -p python3 .venv > /dev/null 2>&1
+set +u
+source .venv/bin/activate > /dev/null 2>&1
+set -u
+
+# install test-requirements
+echo "Info: Install python packages listed in test-requirements.txt using pip"
+pip install --quiet -r test-requirements.txt > /dev/null 2>&1
+
+# set default lint type
+export LINT_TYPE="${LINT_TYPE:-ansible-lint}"
+
+# run tox
+echo "Info: Run $LINT_TYPE using tox"
+echo "----------------------------------------------------"
+tox -e "$LINT_TYPE"
+echo "----------------------------------------------------"
+echo "Info: Done!"
+
+# vim: set ts=2 sw=2 expandtab:
diff --git a/jjb/cloud-infra/scripts/cloud-infra-shellcheck.sh b/jjb/cloud-infra/scripts/cloud-infra-shellcheck.sh
deleted file mode 100644
index 48ea0fc..0000000
--- a/jjb/cloud-infra/scripts/cloud-infra-shellcheck.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-
-# ============LICENSE_START=======================================================
-# Copyright (C) 2019 The Nordix Foundation. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# SPDX-License-Identifier: Apache-2.0
-# ============LICENSE_END=========================================================
-
-# install dependencies
-sudo apt update > /dev/null 2>&1
-sudo apt remove -y -q=3 shellcheck > /dev/null 2>&1
-
-# apt doesn't have the shellcheck version v0.7.0 so we install
-# it from snap
-sudo snap install shellcheck
-SHELLCHECK_EXECUTABLE=/snap/bin/shellcheck
-echo "Info: Using shellcheck version $(${SHELLCHECK_EXECUTABLE} --version)"
-
-cd $WORKSPACE
-echo "Info: Run shellcheck on bash scripts"
-echo "----------------------------------------------------"
-
-# set exit_code so we exit ourselves rather than shellcheck bailing out without
-# verifying all the bash scripts
-declare -i exit_code=0
-
-# lint the bash scripts
-for shell_script in $(find ${WORKSPACE}/ -type f -name *.sh); do
- echo "--> Checking '${shell_script}' shell script..."
- "${SHELLCHECK_EXECUTABLE}" --color=never --source-path="${WORKSPACE}" --external-sources \
- --format=tty ${shell_script}
- lint_exit_code=$?
- if [[ $lint_exit_code != 0 ]]; then
- exit_code=$lint_exit_code
- else
- echo [OK]
- fi
- echo
-done
-echo "----------------------------------------------------"
-echo "Info: Done"
-exit $exit_code
-
-# vim: set ts=2 sw=2 expandtab:
diff --git a/jjb/cloud-infra/scripts/cloud-infra-yaml-lint.sh b/jjb/cloud-infra/scripts/cloud-infra-yaml-lint.sh
deleted file mode 100644
index c0f2577..0000000
--- a/jjb/cloud-infra/scripts/cloud-infra-yaml-lint.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/bin/bash
-
-# ============LICENSE_START=======================================================
-# Copyright (C) 2019 The Nordix Foundation. All rights reserved.
-# ================================================================================
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# SPDX-License-Identifier: Apache-2.0
-# ============LICENSE_END=========================================================
-
-# source engine-vars.sh so we can install the correct versions of
-# pip, ansible, and ansible-lint
-source $WORKSPACE/engine/config/engine-vars.sh
-
-# install dependencies
-sudo apt update > /dev/null 2>&1
-sudo apt install -y -q=3 gcc libffi-dev libssl-dev lsb-release libpython3-dev \
- python3-minimal python3-pip python3-yaml virtualenv > /dev/null 2>&1
-
-# override ENGINE_YAML_LINT_VERSION locally while yamllint patch is still open
-export ENGINE_YAML_LINT_VERSION=1.19.0
-
-cd $WORKSPACE
-echo "Info: Install pip==$ENGINE_PIP_VERSION ansible==$ENGINE_ANSIBLE_VERSION and yamllint==$ENGINE_YAML_LINT_VERSION"
-echo "----------------------------------------------------"
-
-# install pip, ansible, and yamllint in venv
-export ANSIBLE_VENV=$WORKSPACE/.venv
-virtualenv -p python3 --quiet --no-site-packages ${ANSIBLE_VENV}
-source ${ANSIBLE_VENV}/bin/activate
-pip -q install --upgrade pip==$ENGINE_PIP_VERSION
-pip -q install --upgrade virtualenv pip setuptools shade \
- ansible==$ENGINE_ANSIBLE_VERSION yamllint==$ENGINE_YAML_LINT_VERSION
-
-echo "----------------------------------------------------"
-echo "Info: Done"
-echo "Info: Run yamllint on yaml files"
-echo "----------------------------------------------------"
-
-# set exit_code so we exit ourselves rather than ansible-lint bailing out without
-# verifying all the yaml files
-declare -i exit_code=0
-
-# lint the yaml files
-cd $WORKSPACE/engine
-for yaml_file in $(find . -type f -name *.yml | sort); do
- echo "--> Checking '${yaml_file}' yaml file..."
- yamllint --format standard --strict ${yaml_file}
- lint_exit_code=$?
- if [[ $lint_exit_code != 0 ]]; then
- exit_code=$lint_exit_code
- else
- echo [OK]
- fi
- echo
-done
-echo "----------------------------------------------------"
-echo "Info: Done"
-exit $exit_code
-
-# vim: set ts=2 sw=2 expandtab: