From d643ea7acf4cd3e57ba89f092d39e0874b8b111c Mon Sep 17 00:00:00 2001 From: "robert.tomczyk" Date: Wed, 2 Dec 2020 18:20:02 +0000 Subject: [PATCH] INFRA: Switch to tox for JJB verify and merge jobs - The tox is used for JJB verify and merge jobs - The shellcheck and yamllint jobs are disable for time beeing on JJB verify for Nordix - The existing docker slave server is used: infra-tools-docker-slave-ubuntu1804 Change-Id: Iac236ad8e40943a832aaaeb7ae44b72b29d85e04 --- .yamllint | 32 ++++++++++ jjb/global/jjb-macros.yaml | 17 ------ jjb/infra/jjb-merge-jobs.yaml | 49 +++++++++++---- jjb/infra/jjb-verify-jobs.yaml | 103 ++++++++++++++++++++++++++++---- jjb/infra/jjb-verify.sh | 30 ---------- jjb/infra/lint.sh | 86 ++++++++++++++++++++++++++ jjb/infra/test-requirements.txt | 11 ++++ tox.ini | 49 +++++++++++++++ 8 files changed, 306 insertions(+), 71 deletions(-) create mode 100644 .yamllint delete mode 100755 jjb/infra/jjb-verify.sh create mode 100644 jjb/infra/lint.sh create mode 100644 jjb/infra/test-requirements.txt create mode 100644 tox.ini diff --git a/.yamllint b/.yamllint new file mode 100644 index 00000000..fb924d74 --- /dev/null +++ b/.yamllint @@ -0,0 +1,32 @@ +--- +yaml-files: + - '*.yaml' + - '*.yml' + - '.yamllint' + +rules: + braces: enable + brackets: enable + colons: enable + commas: enable + comments: + level: warning + comments-indentation: + level: warning + document-end: disable + document-start: + level: warning + empty-lines: enable + empty-values: disable + hyphens: enable + indentation: enable + key-duplicates: enable + key-ordering: disable + line-length: disable + new-line-at-end-of-file: enable + new-lines: enable + octal-values: disable + quoted-strings: disable + trailing-spaces: enable + truthy: + level: error diff --git a/jjb/global/jjb-macros.yaml b/jjb/global/jjb-macros.yaml index 8d6428ed..d9a37c50 100644 --- a/jjb/global/jjb-macros.yaml +++ b/jjb/global/jjb-macros.yaml @@ -1,21 +1,4 @@ --- -# these parameters are used for infra related variables -- parameter: - name: infra-parameters - parameters: - - string: - name: JJB_GIT_REPO - default: https://opendev.org/jjb/jenkins-job-builder.git - description: OpenDev JJB repo - - string: - name: JJB_VERSION - default: 2.9.1 - description: OpenDev JJB version to install - matches to LFN JJB version - - string: - name: NORDIX_REGISTRY - default: registry.nordix.org - description: Nordix Container Image Registry - # these parameters are used for setting the Gerrit Project on Nordix - parameter: name: project-parameters diff --git a/jjb/infra/jjb-merge-jobs.yaml b/jjb/infra/jjb-merge-jobs.yaml index e9ced766..4df7d67a 100644 --- a/jjb/infra/jjb-merge-jobs.yaml +++ b/jjb/infra/jjb-merge-jobs.yaml @@ -1,37 +1,61 @@ --- +# ============LICENSE_START======================================================= +# Copyright (C) 2020 Nordix Foundation. +# ================================================================================ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END========================================================= # this job gets triggered when a change is merged in cicd/jjb # the updates to the jobs are deployed to jenkins using jenkins-jobs update command -# result will be send to Gerrit in the form of a comment +# executed from tox. The result will be send to Gerrit in the form of a comment - project: - name: infra-jjb-merge - project: infra/cicd stream: - master: branch: '{stream}' + lint-type: jjbdeploy jobs: - - infra-jjb-merge-{stream} + - infra-merge-cicd-jjb-{stream} - job-template: - name: infra-jjb-merge-{stream} - - node: infra-ubuntu1804 - + name: infra-merge-cicd-jjb-{stream} + node: infra-tools-docker-slave-ubuntu1804 disabled: false - concurrent: false parameters: - - infra-parameters - project-parameters: project: '{project}' branch: '{branch}' + description: 'The project and branch which the change is proposed for' + - string: + name: LINT_TYPE + default: '{lint-type}' + description: 'Lint type to run' + - string: + name: VERBOSITY + default: 'false' + description: 'Verbosity setting for the lint script' - nordix-gerrit-parameters + properties: + - logrotate + scm: - git-scm: ssh-credentials-id: nordixinfra-nordix-gerrit-ssh @@ -48,7 +72,10 @@ wrappers: - nordixinfra-jjb-creds-wrapper + - build-timeout: + timeout: 60 + - fix-workspace-permissions builders: - shell: - !include-raw-escape: ./jjb-merge.sh + !include-raw-escape: ./lint.sh diff --git a/jjb/infra/jjb-verify-jobs.yaml b/jjb/infra/jjb-verify-jobs.yaml index 589954ed..810bfa9a 100644 --- a/jjb/infra/jjb-verify-jobs.yaml +++ b/jjb/infra/jjb-verify-jobs.yaml @@ -1,37 +1,68 @@ --- +# ============LICENSE_START==================================================== +# Copyright (C) 2020 Nordix Foundation. +# ============================================================================= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# ============LICENSE_END====================================================== # this job gets triggered when a change is proposed to cicd/jjb -# the change is verified using jenkins-jobs test command +# the change is verified using jenkins-jobs tox test command # result will be send to Gerrit in the form of Verified +1 or -1 -- project: +- project: name: infra-jjb-verify - project: infra/cicd - stream: - master: branch: '{stream}' + lint-type: + - jjbtest + - shellcheck + - yamllint + jobs: - - infra-jjb-verify-{stream} + - infra-verify-cicd-jjb-{stream} + - infra-verify-cicd-{lint-type}-{stream} - job-template: - name: infra-jjb-verify-{stream} - - node: infra-ubuntu1804 - + name: infra-verify-cicd-jjb-{stream} + project-type: multijob + node: infra-tools-docker-slave-ubuntu1804 disabled: false - concurrent: true parameters: - - infra-parameters - project-parameters: project: '{project}' branch: '{branch}' + description: 'The project and branch which the change is proposed for' + - string: + name: VERBOSITY + default: 'false' + description: 'Verbosity setting for the lint script' - nordix-gerrit-parameters + properties: + - logrotate + + wrappers: + - build-timeout: + timeout: 60 + - fix-workspace-permissions + scm: - git-scm-gerrit: ssh-credentials-id: nordixinfra-nordix-gerrit-ssh @@ -51,9 +82,55 @@ unstable: false notbuilt: false + builders: + - multijob: + name: JJB Static Analysis + condition: ALWAYS + execution-type: PARALLEL + projects: + - name: 'infra-verify-cicd-jjbtest-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + - name: 'infra-verify-cicd-shellcheck-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + - name: 'infra-verify-cicd-yamllint-{stream}' + current-parameters: true + kill-phase-on: NEVER + abort-all-job: false + +- job-template: + name: 'infra-verify-cicd-{lint-type}-{stream}' + node: infra-tools-docker-slave-ubuntu1804 + disabled: '{obj:disabled}' + concurrent: true + + parameters: + - string: + name: LINT_TYPE + default: '{lint-type}' + description: 'Lint type to run' + + properties: + - logrotate + wrappers: - - nordixinfra-jjb-creds-wrapper + - build-timeout: + timeout: 150 + - fix-workspace-permissions + + scm: + - git-scm-gerrit: + ssh-credentials-id: nordixinfra-nordix-gerrit-ssh + branch: refs/heads/$GERRIT_BRANCH + refspec: $GERRIT_REFSPEC + wipe_workspace: true + clean_before: false builders: - shell: - !include-raw-escape: ./jjb-verify.sh + !include-raw-escape: ./lint.sh + +# vim: set ts=2 sw=2 ft=yaml expandtab: diff --git a/jjb/infra/jjb-verify.sh b/jjb/infra/jjb-verify.sh deleted file mode 100755 index 8389ae3d..00000000 --- a/jjb/infra/jjb-verify.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - -# install required packages -sudo apt update > /dev/null 2>&1 -sudo apt install -y -q=3 gcc libffi-dev libssl-dev lsb-release libpython3-dev \ - python3-minimal python3-pip python3-yaml virtualenv - -# workaround for https://github.com/pypa/virtualenv/issues/1029 -export PS1=${PS1:-} - -cd $WORKSPACE -echo "Info: Installing OpenStack Jenkins Job Builder (JJB) version $JJB_VERSION from source in venv" -echo "----------------------------------------------------" -git clone -q $JJB_GIT_REPO -b $JJB_VERSION > /dev/null 2>&1 -virtualenv -p python3 --quiet --no-site-packages .venv -source .venv/bin/activate -cd jenkins-job-builder && pip install -q -r test-requirements.txt -e . -echo "Info: Installed JJB" -jenkins-jobs --version -echo "----------------------------------------------------" -echo "Info: Proceeding with verifying job templates" -echo "----------------------------------------------------" -cd $WORKSPACE -jenkins-jobs --conf jjb/global/jenkins_jobs.ini --user $JJB_USERNAME --password $JJB_USERTOKEN \ - test --recursive jjb/ -echo "----------------------------------------------------" -echo "Info: Done" diff --git a/jjb/infra/lint.sh b/jjb/infra/lint.sh new file mode 100644 index 00000000..6d4e060e --- /dev/null +++ b/jjb/infra/lint.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +#------------------------------------------------------------------------------- +# Various tests are run using tox such as jjbtest, yamllint, and shellehck. +#------------------------------------------------------------------------------- +run_tox() { + + echo "Info : Preparing to run tox for the repo $GERRIT_PROJECT" + + # set and export defaults + LINT_TYPE="${LINT_TYPE:-jjbtest}" + VERBOSITY=${VERBOSITY:-false} + export LINT_TYPE VERBOSITY + + # ensure we are in job build WORKSPACE + cd "$WORKSPACE" + + # set DEBIAN_FRONTEND to run apt non-interactively + DEBIAN_FRONTEND=noninteractive + export DEBIAN_FRONTEND + + # Wait for other apt auto-update process to finish by checking the dpkg lock file. + DPKG_LOCK="/var/lib/dpkg/lock-frontend" + try=0 + while sudo lsof "${DPKG_LOCK}" > /dev/null 2>&1 ; do + echo "DPKG file locked: ${DPKG_LOCK}." + echo " Waiting for another pkg instalaltion process to finish ..." + sleep 10 + if [[ ${try} -gt 120 ]] ; then + echo "ERROR: Max number of re-tries reached, exiting..." + exit 1 + fi + try=$((try + 1)) + done + + # install dependencies + echo "Info : Install python3.7-minimal python3-distutils virtualenv using apt" + redirect_cmd sudo apt update + redirect_cmd sudo apt install -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew -y \ + python3.7-minimal python3-distutils virtualenv + + # create and activate virtualenv + echo "Info : Create and activate python virtualenv" + redirect_cmd virtualenv -p python3 .venv + set +u + redirect_cmd source .venv/bin/activate + set -u + + # install only tox since the rest of the requirements are installed by tox itself + TOX_PACKAGE=$(grep "^tox==" jjb/infra/test-requirements.txt) + echo "Info : Install $TOX_PACKAGE" + redirect_cmd pip install --force-reinstall "$TOX_PACKAGE" + + # run tox + echo "Info : Run $LINT_TYPE using tox" + echo "----------------------------------------------------" + tox -e "$LINT_TYPE" + echo "----------------------------------------------------" + echo "Info : Done!" + +} +#------------------------------------------------------------------------------- +# In some cases, it is useful to see all the output generated by commands so +# this function makes it possible for users to achieve that by not redirecting +# output to /dev/null when verbosity is enabled +#------------------------------------------------------------------------------- +redirect_cmd() { + + if [[ "$VERBOSITY" == "false" ]]; then + "$@" > /dev/null 2>&1 + else + "$@" + fi + +} +#------------------------------------------------------------------------------- +# run tox +#------------------------------------------------------------------------------- + +run_tox "$@" + +# vim: set ts=2 sw=2 ft=bash expandtab: diff --git a/jjb/infra/test-requirements.txt b/jjb/infra/test-requirements.txt new file mode 100644 index 00000000..a2b622da --- /dev/null +++ b/jjb/infra/test-requirements.txt @@ -0,0 +1,11 @@ +#------------------------------------------------------------------------------- +# This file contains cicd repo requirement version pins. +# Changing versions might have side effects! +#------------------------------------------------------------------------------- + +pip==20.0.1 +jenkins-job-builder==3.4.0 +tox==3.14.3 +ansible-lint==4.1.0 +yamllint==1.19.0 +shellcheck-py==0.7.0.1 diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..6f20440b --- /dev/null +++ b/tox.ini @@ -0,0 +1,49 @@ +[tox] +basepython = py36 +minversion = 2.5 +skipsdist = true +envlist = shellcheck, yamllint, jjbtest, jjbdeploy +install_command = pip install --force-reinstall {opts} {packages} +deps = -r{toxinidir}/jjb/infra/test-requirements.txt + +[testenv:jjbtest] +description = invoke jenkins-job-builder to analyse jjb templates +passenv = * +deps = -r{toxinidir}/jjb/infra/test-requirements.txt +whitelist_externals = bash +commands = + bash -c "jenkins-jobs --conf {toxinidir}/jjb/global/jenkins_jobs.ini \ + test --config-xml --recursive {toxinidir}/jjb" + +[testenv:jjbdeploy] +description = invoke jenkins-job-builder to deploy jobs from jjb templates +passenv = * +deps = -r{toxinidir}/jjb/infra/test-requirements.txt +whitelist_externals = bash +commands = + bash -c "jenkins-jobs --conf {toxinidir}jjb/global/jenkins_jobs.ini \ + --user $JJB_USERNAME --password $JJB_USERTOKEN update --recursive \ + --delete-old --workers 4 {toxinidir}/jjb" + +[testenv:yamllint] +description = invoke yamllint to analyse YAML files +deps = -r{toxinidir}/jjb/infra/test-requirements.txt +ignore_outcome=true +whitelist_externals = bash +commands = + bash -c "find {toxinidir} -type f -regex '.*.ya?ml' \ + ! -regex '.*.tox.*\|.*.venv.*' \ + -print0 | xargs -t -n1 -0 yamllint --format standard --strict" + +[testenv:shellcheck] +description = invoke shellcheck to analyse bash shell scripts +deps = -r{toxinidir}/jjb/infra/test-requirements.txt +ignore_outcome=true +whitelist_externals = bash +commands = + bash -c "find {toxinidir} -type f -regex '.*.sh' \ + ! -regex '.*.tox.*\|.*.venv.*' \ + -print0 | xargs -t -n1 -0 shellcheck --color=never --source-path={toxinidir} \ + --external-sources --format=tty" + +# vim: set ts=2 sw=2 ft=ini expandtab: -- 2.25.1