--- /dev/null
+---
+yaml-files:
+ - '*.yaml'
+ - '*.yml'
+ - '.yamllint'
+
+rules:
+ braces: enable
+ brackets: enable
+ colons: enable
+ commas: enable
+ comments:
+ level: warning
+ comments-indentation:
+ level: warning
+ document-end: disable
+ document-start:
+ level: warning
+ empty-lines: enable
+ empty-values: disable
+ hyphens: enable
+ indentation: enable
+ key-duplicates: enable
+ key-ordering: disable
+ line-length: disable
+ new-line-at-end-of-file: enable
+ new-lines: enable
+ octal-values: disable
+ quoted-strings: disable
+ trailing-spaces: enable
+ truthy:
+ level: error
---
-# these parameters are used for infra related variables
-- parameter:
- name: infra-parameters
- parameters:
- - string:
- name: JJB_GIT_REPO
- default: https://opendev.org/jjb/jenkins-job-builder.git
- description: OpenDev JJB repo
- - string:
- name: JJB_VERSION
- default: 2.9.1
- description: OpenDev JJB version to install - matches to LFN JJB version
- - string:
- name: NORDIX_REGISTRY
- default: registry.nordix.org
- description: Nordix Container Image Registry
-
# these parameters are used for setting the Gerrit Project on Nordix
- parameter:
name: project-parameters
---
+# ============LICENSE_START=======================================================
+# Copyright (C) 2020 Nordix Foundation.
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
# this job gets triggered when a change is merged in cicd/jjb
# the updates to the jobs are deployed to jenkins using jenkins-jobs update command
-# result will be send to Gerrit in the form of a comment
+# executed from tox. The result will be send to Gerrit in the form of a comment
- project:
-
name: infra-jjb-merge
-
project: infra/cicd
stream:
- master:
branch: '{stream}'
+ lint-type: jjbdeploy
jobs:
- - infra-jjb-merge-{stream}
+ - infra-merge-cicd-jjb-{stream}
- job-template:
- name: infra-jjb-merge-{stream}
-
- node: infra-ubuntu1804
-
+ name: infra-merge-cicd-jjb-{stream}
+ node: infra-tools-docker-slave-ubuntu1804
disabled: false
-
concurrent: false
parameters:
- - infra-parameters
- project-parameters:
project: '{project}'
branch: '{branch}'
+ description: 'The project and branch which the change is proposed for'
+ - string:
+ name: LINT_TYPE
+ default: '{lint-type}'
+ description: 'Lint type to run'
+ - string:
+ name: VERBOSITY
+ default: 'false'
+ description: 'Verbosity setting for the lint script'
- nordix-gerrit-parameters
+ properties:
+ - logrotate
+
scm:
- git-scm:
ssh-credentials-id: nordixinfra-nordix-gerrit-ssh
wrappers:
- nordixinfra-jjb-creds-wrapper
+ - build-timeout:
+ timeout: 60
+ - fix-workspace-permissions
builders:
- shell:
- !include-raw-escape: ./jjb-merge.sh
+ !include-raw-escape: ./lint.sh
---
+# ============LICENSE_START====================================================
+# Copyright (C) 2020 Nordix Foundation.
+# =============================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END======================================================
# this job gets triggered when a change is proposed to cicd/jjb
-# the change is verified using jenkins-jobs test command
+# the change is verified using jenkins-jobs tox test command
# result will be send to Gerrit in the form of Verified +1 or -1
-- project:
+- project:
name: infra-jjb-verify
-
project: infra/cicd
-
stream:
- master:
branch: '{stream}'
+ lint-type:
+ - jjbtest
+ - shellcheck
+ - yamllint
+
jobs:
- - infra-jjb-verify-{stream}
+ - infra-verify-cicd-jjb-{stream}
+ - infra-verify-cicd-{lint-type}-{stream}
- job-template:
- name: infra-jjb-verify-{stream}
-
- node: infra-ubuntu1804
-
+ name: infra-verify-cicd-jjb-{stream}
+ project-type: multijob
+ node: infra-tools-docker-slave-ubuntu1804
disabled: false
-
concurrent: true
parameters:
- - infra-parameters
- project-parameters:
project: '{project}'
branch: '{branch}'
+ description: 'The project and branch which the change is proposed for'
+ - string:
+ name: VERBOSITY
+ default: 'false'
+ description: 'Verbosity setting for the lint script'
- nordix-gerrit-parameters
+ properties:
+ - logrotate
+
+ wrappers:
+ - build-timeout:
+ timeout: 60
+ - fix-workspace-permissions
+
scm:
- git-scm-gerrit:
ssh-credentials-id: nordixinfra-nordix-gerrit-ssh
unstable: false
notbuilt: false
+ builders:
+ - multijob:
+ name: JJB Static Analysis
+ condition: ALWAYS
+ execution-type: PARALLEL
+ projects:
+ - name: 'infra-verify-cicd-jjbtest-{stream}'
+ current-parameters: true
+ kill-phase-on: NEVER
+ abort-all-job: false
+ - name: 'infra-verify-cicd-shellcheck-{stream}'
+ current-parameters: true
+ kill-phase-on: NEVER
+ abort-all-job: false
+ - name: 'infra-verify-cicd-yamllint-{stream}'
+ current-parameters: true
+ kill-phase-on: NEVER
+ abort-all-job: false
+
+- job-template:
+ name: 'infra-verify-cicd-{lint-type}-{stream}'
+ node: infra-tools-docker-slave-ubuntu1804
+ disabled: '{obj:disabled}'
+ concurrent: true
+
+ parameters:
+ - string:
+ name: LINT_TYPE
+ default: '{lint-type}'
+ description: 'Lint type to run'
+
+ properties:
+ - logrotate
+
wrappers:
- - nordixinfra-jjb-creds-wrapper
+ - build-timeout:
+ timeout: 150
+ - fix-workspace-permissions
+
+ scm:
+ - git-scm-gerrit:
+ ssh-credentials-id: nordixinfra-nordix-gerrit-ssh
+ branch: refs/heads/$GERRIT_BRANCH
+ refspec: $GERRIT_REFSPEC
+ wipe_workspace: true
+ clean_before: false
builders:
- shell:
- !include-raw-escape: ./jjb-verify.sh
+ !include-raw-escape: ./lint.sh
+
+# vim: set ts=2 sw=2 ft=yaml expandtab:
+++ /dev/null
-#!/bin/bash
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# install required packages
-sudo apt update > /dev/null 2>&1
-sudo apt install -y -q=3 gcc libffi-dev libssl-dev lsb-release libpython3-dev \
- python3-minimal python3-pip python3-yaml virtualenv
-
-# workaround for https://github.com/pypa/virtualenv/issues/1029
-export PS1=${PS1:-}
-
-cd $WORKSPACE
-echo "Info: Installing OpenStack Jenkins Job Builder (JJB) version $JJB_VERSION from source in venv"
-echo "----------------------------------------------------"
-git clone -q $JJB_GIT_REPO -b $JJB_VERSION > /dev/null 2>&1
-virtualenv -p python3 --quiet --no-site-packages .venv
-source .venv/bin/activate
-cd jenkins-job-builder && pip install -q -r test-requirements.txt -e .
-echo "Info: Installed JJB"
-jenkins-jobs --version
-echo "----------------------------------------------------"
-echo "Info: Proceeding with verifying job templates"
-echo "----------------------------------------------------"
-cd $WORKSPACE
-jenkins-jobs --conf jjb/global/jenkins_jobs.ini --user $JJB_USERNAME --password $JJB_USERTOKEN \
- test --recursive jjb/
-echo "----------------------------------------------------"
-echo "Info: Done"
--- /dev/null
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+#-------------------------------------------------------------------------------
+# Various tests are run using tox such as jjbtest, yamllint, and shellehck.
+#-------------------------------------------------------------------------------
+run_tox() {
+
+ echo "Info : Preparing to run tox for the repo $GERRIT_PROJECT"
+
+ # set and export defaults
+ LINT_TYPE="${LINT_TYPE:-jjbtest}"
+ VERBOSITY=${VERBOSITY:-false}
+ export LINT_TYPE VERBOSITY
+
+ # ensure we are in job build WORKSPACE
+ cd "$WORKSPACE"
+
+ # set DEBIAN_FRONTEND to run apt non-interactively
+ DEBIAN_FRONTEND=noninteractive
+ export DEBIAN_FRONTEND
+
+ # Wait for other apt auto-update process to finish by checking the dpkg lock file.
+ DPKG_LOCK="/var/lib/dpkg/lock-frontend"
+ try=0
+ while sudo lsof "${DPKG_LOCK}" > /dev/null 2>&1 ; do
+ echo "DPKG file locked: ${DPKG_LOCK}."
+ echo " Waiting for another pkg instalaltion process to finish ..."
+ sleep 10
+ if [[ ${try} -gt 120 ]] ; then
+ echo "ERROR: Max number of re-tries reached, exiting..."
+ exit 1
+ fi
+ try=$((try + 1))
+ done
+
+ # install dependencies
+ echo "Info : Install python3.7-minimal python3-distutils virtualenv using apt"
+ redirect_cmd sudo apt update
+ redirect_cmd sudo apt install -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew -y \
+ python3.7-minimal python3-distutils virtualenv
+
+ # create and activate virtualenv
+ echo "Info : Create and activate python virtualenv"
+ redirect_cmd virtualenv -p python3 .venv
+ set +u
+ redirect_cmd source .venv/bin/activate
+ set -u
+
+ # install only tox since the rest of the requirements are installed by tox itself
+ TOX_PACKAGE=$(grep "^tox==" jjb/infra/test-requirements.txt)
+ echo "Info : Install $TOX_PACKAGE"
+ redirect_cmd pip install --force-reinstall "$TOX_PACKAGE"
+
+ # run tox
+ echo "Info : Run $LINT_TYPE using tox"
+ echo "----------------------------------------------------"
+ tox -e "$LINT_TYPE"
+ echo "----------------------------------------------------"
+ echo "Info : Done!"
+
+}
+#-------------------------------------------------------------------------------
+# In some cases, it is useful to see all the output generated by commands so
+# this function makes it possible for users to achieve that by not redirecting
+# output to /dev/null when verbosity is enabled
+#-------------------------------------------------------------------------------
+redirect_cmd() {
+
+ if [[ "$VERBOSITY" == "false" ]]; then
+ "$@" > /dev/null 2>&1
+ else
+ "$@"
+ fi
+
+}
+#-------------------------------------------------------------------------------
+# run tox
+#-------------------------------------------------------------------------------
+
+run_tox "$@"
+
+# vim: set ts=2 sw=2 ft=bash expandtab:
--- /dev/null
+#-------------------------------------------------------------------------------
+# This file contains cicd repo requirement version pins.
+# Changing versions might have side effects!
+#-------------------------------------------------------------------------------
+
+pip==20.0.1
+jenkins-job-builder==3.4.0
+tox==3.14.3
+ansible-lint==4.1.0
+yamllint==1.19.0
+shellcheck-py==0.7.0.1
--- /dev/null
+[tox]
+basepython = py36
+minversion = 2.5
+skipsdist = true
+envlist = shellcheck, yamllint, jjbtest, jjbdeploy
+install_command = pip install --force-reinstall {opts} {packages}
+deps = -r{toxinidir}/jjb/infra/test-requirements.txt
+
+[testenv:jjbtest]
+description = invoke jenkins-job-builder to analyse jjb templates
+passenv = *
+deps = -r{toxinidir}/jjb/infra/test-requirements.txt
+whitelist_externals = bash
+commands =
+ bash -c "jenkins-jobs --conf {toxinidir}/jjb/global/jenkins_jobs.ini \
+ test --config-xml --recursive {toxinidir}/jjb"
+
+[testenv:jjbdeploy]
+description = invoke jenkins-job-builder to deploy jobs from jjb templates
+passenv = *
+deps = -r{toxinidir}/jjb/infra/test-requirements.txt
+whitelist_externals = bash
+commands =
+ bash -c "jenkins-jobs --conf {toxinidir}jjb/global/jenkins_jobs.ini \
+ --user $JJB_USERNAME --password $JJB_USERTOKEN update --recursive \
+ --delete-old --workers 4 {toxinidir}/jjb"
+
+[testenv:yamllint]
+description = invoke yamllint to analyse YAML files
+deps = -r{toxinidir}/jjb/infra/test-requirements.txt
+ignore_outcome=true
+whitelist_externals = bash
+commands =
+ bash -c "find {toxinidir} -type f -regex '.*.ya?ml' \
+ ! -regex '.*.tox.*\|.*.venv.*' \
+ -print0 | xargs -t -n1 -0 yamllint --format standard --strict"
+
+[testenv:shellcheck]
+description = invoke shellcheck to analyse bash shell scripts
+deps = -r{toxinidir}/jjb/infra/test-requirements.txt
+ignore_outcome=true
+whitelist_externals = bash
+commands =
+ bash -c "find {toxinidir} -type f -regex '.*.sh' \
+ ! -regex '.*.tox.*\|.*.venv.*' \
+ -print0 | xargs -t -n1 -0 shellcheck --color=never --source-path={toxinidir} \
+ --external-sources --format=tty"
+
+# vim: set ts=2 sw=2 ft=ini expandtab:
Code Review / infra / cicd.git / commitdiff