tests: replace pycodestyle with black
Drop pycodestyle for code style checking in favor of black. Black is
much faster, stable PEP8 compliant code style checker offering also
automatic formatting. It aims to be very stable and produce smallest
diffs. It's used by many small and big projects.
Running checkstyle with black takes a few seconds with a terse output.
Thus, test-checkstyle-diff is no longer necessary.
Expand scope of checkstyle to all python files in the repo, replacing
test-checkstyle with checkstyle-python.
Also, fixstyle-python is now available for automatic style formatting.
Note: python virtualenv has been consolidated in test/Makefile,
test/requirements*.txt which will eventually be moved to a central
location. This is required to simply the automated generation of
docker executor images in the CI.
Type: improvement
Change-Id: I022a326603485f58585e879ac0f697fceefbc9c8
Signed-off-by: Klement Sekera <klement.sekera@gmail.com>
Signed-off-by: Dave Wallace <dwallacelf@gmail.com>
diff --git a/extras/scripts/crcchecker.py b/extras/scripts/crcchecker.py
index f3021c3..01cb025 100755
--- a/extras/scripts/crcchecker.py
+++ b/extras/scripts/crcchecker.py
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
-'''
+"""
crcchecker is a tool to used to enforce that .api messages do not change.
API files with a semantic version < 1.0.0 are ignored.
-'''
+"""
import sys
import os
@@ -14,67 +14,75 @@
# pylint: disable=subprocess-run-check
-ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + '/../..'
-APIGENBIN = f'{ROOTDIR}/src/tools/vppapigen/vppapigen.py'
+ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + "/../.."
+APIGENBIN = f"{ROOTDIR}/src/tools/vppapigen/vppapigen.py"
def crc_from_apigen(revision, filename):
- '''Runs vppapigen with crc plugin returning a JSON object with CRCs for
- all APIs in filename'''
+ """Runs vppapigen with crc plugin returning a JSON object with CRCs for
+ all APIs in filename"""
if not revision and not os.path.isfile(filename):
- print(f'skipping: {filename}', file=sys.stderr)
+ print(f"skipping: {filename}", file=sys.stderr)
# Return <class 'set'> instead of <class 'dict'>
return {-1}
if revision:
- apigen = (f'{APIGENBIN} --git-revision {revision} --includedir src '
- f'--input {filename} CRC')
+ apigen = (
+ f"{APIGENBIN} --git-revision {revision} --includedir src "
+ f"--input {filename} CRC"
+ )
else:
- apigen = (f'{APIGENBIN} --includedir src --input {filename} CRC')
+ apigen = f"{APIGENBIN} --includedir src --input {filename} CRC"
returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE)
if returncode.returncode == 2: # No such file
- print(f'skipping: {revision}:{filename} {returncode}', file=sys.stderr)
+ print(f"skipping: {revision}:{filename} {returncode}", file=sys.stderr)
return {}
if returncode.returncode != 0:
- print(f'vppapigen failed for {revision}:{filename} with '
- 'command\n {apigen}\n error: {rv}',
- returncode.stderr.decode('ascii'), file=sys.stderr)
+ print(
+ f"vppapigen failed for {revision}:{filename} with "
+ "command\n {apigen}\n error: {rv}",
+ returncode.stderr.decode("ascii"),
+ file=sys.stderr,
+ )
sys.exit(-2)
return json.loads(returncode.stdout)
def dict_compare(dict1, dict2):
- '''Compare two dictionaries returning added, removed, modified
- and equal entries'''
+ """Compare two dictionaries returning added, removed, modified
+ and equal entries"""
d1_keys = set(dict1.keys())
d2_keys = set(dict2.keys())
intersect_keys = d1_keys.intersection(d2_keys)
added = d1_keys - d2_keys
removed = d2_keys - d1_keys
- modified = {o: (dict1[o], dict2[o]) for o in intersect_keys
- if dict1[o]['crc'] != dict2[o]['crc']}
+ modified = {
+ o: (dict1[o], dict2[o])
+ for o in intersect_keys
+ if dict1[o]["crc"] != dict2[o]["crc"]
+ }
same = set(o for o in intersect_keys if dict1[o] == dict2[o])
return added, removed, modified, same
def filelist_from_git_ls():
- '''Returns a list of all api files in the git repository'''
+ """Returns a list of all api files in the git repository"""
filelist = []
- git_ls = 'git ls-files *.api'
+ git_ls = "git ls-files *.api"
returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE)
if returncode.returncode != 0:
sys.exit(returncode.returncode)
- for line in returncode.stdout.decode('ascii').split('\n'):
+ for line in returncode.stdout.decode("ascii").split("\n"):
if line:
filelist.append(line)
return filelist
def is_uncommitted_changes():
- '''Returns true if there are uncommitted changes in the repo'''
- git_status = 'git status --porcelain -uno'
+ """Returns true if there are uncommitted changes in the repo"""
+ git_status = "git status --porcelain -uno"
returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE)
if returncode.returncode != 0:
sys.exit(returncode.returncode)
@@ -85,27 +93,29 @@
def filelist_from_git_grep(filename):
- '''Returns a list of api files that this <filename> api files imports.'''
+ """Returns a list of api files that this <filename> api files imports."""
filelist = []
try:
- returncode = check_output(f'git grep -e "import .*{filename}"'
- ' -- *.api',
- shell=True)
+ returncode = check_output(
+ f'git grep -e "import .*{filename}"' " -- *.api", shell=True
+ )
except CalledProcessError:
return []
- for line in returncode.decode('ascii').split('\n'):
+ for line in returncode.decode("ascii").split("\n"):
if line:
- filename, _ = line.split(':')
+ filename, _ = line.split(":")
filelist.append(filename)
return filelist
def filelist_from_patchset(pattern):
- '''Returns list of api files in changeset and the list of api
- files they import.'''
+ """Returns list of api files in changeset and the list of api
+ files they import."""
filelist = []
- git_cmd = ('((git diff HEAD~1.. --name-only;git ls-files -m) | '
- 'sort -u | grep "\\.api$")')
+ git_cmd = (
+ "((git diff HEAD~1.. --name-only;git ls-files -m) | "
+ 'sort -u | grep "\\.api$")'
+ )
try:
res = check_output(git_cmd, shell=True)
except CalledProcessError:
@@ -113,7 +123,7 @@
# Check for dependencies (imports)
imported_files = []
- for line in res.decode('ascii').split('\n'):
+ for line in res.decode("ascii").split("\n"):
if not line:
continue
if not re.search(pattern, line):
@@ -126,88 +136,91 @@
def is_deprecated(message):
- '''Given a message, return True if message is deprecated'''
- if 'options' in message:
- if 'deprecated' in message['options']:
+ """Given a message, return True if message is deprecated"""
+ if "options" in message:
+ if "deprecated" in message["options"]:
return True
# recognize the deprecated format
- if 'status' in message['options'] and \
- message['options']['status'] == 'deprecated':
+ if (
+ "status" in message["options"]
+ and message["options"]["status"] == "deprecated"
+ ):
print("WARNING: please use 'option deprecated;'")
return True
return False
def is_in_progress(message):
- '''Given a message, return True if message is marked as in_progress'''
- if 'options' in message:
- if 'in_progress' in message['options']:
+ """Given a message, return True if message is marked as in_progress"""
+ if "options" in message:
+ if "in_progress" in message["options"]:
return True
# recognize the deprecated format
- if 'status' in message['options'] and \
- message['options']['status'] == 'in_progress':
+ if (
+ "status" in message["options"]
+ and message["options"]["status"] == "in_progress"
+ ):
print("WARNING: please use 'option in_progress;'")
return True
return False
def report(new, old):
- '''Given a dictionary of new crcs and old crcs, print all the
+ """Given a dictionary of new crcs and old crcs, print all the
added, removed, modified, in-progress, deprecated messages.
- Return the number of backwards incompatible changes made.'''
+ Return the number of backwards incompatible changes made."""
# pylint: disable=too-many-branches
- new.pop('_version', None)
- old.pop('_version', None)
+ new.pop("_version", None)
+ old.pop("_version", None)
added, removed, modified, _ = dict_compare(new, old)
backwards_incompatible = 0
# print the full list of in-progress messages
# they should eventually either disappear of become supported
for k in new.keys():
- newversion = int(new[k]['version'])
+ newversion = int(new[k]["version"])
if newversion == 0 or is_in_progress(new[k]):
- print(f'in-progress: {k}')
+ print(f"in-progress: {k}")
for k in added:
- print(f'added: {k}')
+ print(f"added: {k}")
for k in removed:
- oldversion = int(old[k]['version'])
- if oldversion > 0 and not is_deprecated(old[k]) and not \
- is_in_progress(old[k]):
+ oldversion = int(old[k]["version"])
+ if oldversion > 0 and not is_deprecated(old[k]) and not is_in_progress(old[k]):
backwards_incompatible += 1
- print(f'removed: ** {k}')
+ print(f"removed: ** {k}")
else:
- print(f'removed: {k}')
+ print(f"removed: {k}")
for k in modified.keys():
- oldversion = int(old[k]['version'])
- newversion = int(new[k]['version'])
+ oldversion = int(old[k]["version"])
+ newversion = int(new[k]["version"])
if oldversion > 0 and not is_in_progress(old[k]):
backwards_incompatible += 1
- print(f'modified: ** {k}')
+ print(f"modified: ** {k}")
else:
- print(f'modified: {k}')
+ print(f"modified: {k}")
# check which messages are still there but were marked for deprecation
for k in new.keys():
- newversion = int(new[k]['version'])
+ newversion = int(new[k]["version"])
if newversion > 0 and is_deprecated(new[k]):
if k in old:
if not is_deprecated(old[k]):
- print(f'deprecated: {k}')
+ print(f"deprecated: {k}")
else:
- print(f'added+deprecated: {k}')
+ print(f"added+deprecated: {k}")
return backwards_incompatible
def check_patchset():
- '''Compare the changes to API messages in this changeset.
+ """Compare the changes to API messages in this changeset.
Ignores API files with version < 1.0.0.
Only considers API files located under the src directory in the repo.
- '''
- files = filelist_from_patchset('^src/')
- revision = 'HEAD~1'
+ """
+ files = filelist_from_patchset("^src/")
+ revision = "HEAD~1"
oldcrcs = {}
newcrcs = {}
@@ -216,7 +229,7 @@
_ = crc_from_apigen(None, filename)
# Ignore removed files
if isinstance(_, set) == 0:
- if isinstance(_, set) == 0 and _['_version']['major'] == '0':
+ if isinstance(_, set) == 0 and _["_version"]["major"] == "0":
continue
newcrcs.update(_)
@@ -225,27 +238,31 @@
backwards_incompatible = report(newcrcs, oldcrcs)
if backwards_incompatible:
# alert on changing production API
- print("crcchecker: Changing production APIs in an incompatible way",
- file=sys.stderr)
+ print(
+ "crcchecker: Changing production APIs in an incompatible way",
+ file=sys.stderr,
+ )
sys.exit(-1)
else:
- print('*' * 67)
- print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
- print('*' * 67)
+ print("*" * 67)
+ print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
+ print("*" * 67)
def main():
- '''Main entry point.'''
- parser = argparse.ArgumentParser(description='VPP CRC checker.')
- parser.add_argument('--git-revision',
- help='Git revision to compare against')
- parser.add_argument('--dump-manifest', action='store_true',
- help='Dump CRC for all messages')
- parser.add_argument('--check-patchset', action='store_true',
- help='Check patchset for backwards incompatbile changes')
- parser.add_argument('files', nargs='*')
- parser.add_argument('--diff', help='Files to compare (on filesystem)',
- nargs=2)
+ """Main entry point."""
+ parser = argparse.ArgumentParser(description="VPP CRC checker.")
+ parser.add_argument("--git-revision", help="Git revision to compare against")
+ parser.add_argument(
+ "--dump-manifest", action="store_true", help="Dump CRC for all messages"
+ )
+ parser.add_argument(
+ "--check-patchset",
+ action="store_true",
+ help="Check patchset for backwards incompatbile changes",
+ )
+ parser.add_argument("files", nargs="*")
+ parser.add_argument("--diff", help="Files to compare (on filesystem)", nargs=2)
args = parser.parse_args()
@@ -267,17 +284,16 @@
for filename in files:
crcs.update(crc_from_apigen(args.git_revision, filename))
for k, value in crcs.items():
- print(f'{k}: {value}')
+ print(f"{k}: {value}")
sys.exit(0)
# Find changes between current patchset and given revision (previous)
if args.check_patchset:
if args.git_revision:
- print('Argument git-revision ignored', file=sys.stderr)
+ print("Argument git-revision ignored", file=sys.stderr)
# Check there are no uncomitted changes
if is_uncommitted_changes():
- print('Please stash or commit changes in workspace',
- file=sys.stderr)
+ print("Please stash or commit changes in workspace", file=sys.stderr)
sys.exit(-1)
check_patchset()
sys.exit(0)
@@ -286,7 +302,7 @@
# Find changes between a given file and a revision
files = args.files if args.files else filelist_from_git_ls()
- revision = args.git_revision if args.git_revision else 'HEAD~1'
+ revision = args.git_revision if args.git_revision else "HEAD~1"
oldcrcs = {}
newcrcs = {}
@@ -299,13 +315,16 @@
if args.check_patchset:
if backwards_incompatible:
# alert on changing production API
- print("crcchecker: Changing production APIs in an incompatible way", file=sys.stderr)
+ print(
+ "crcchecker: Changing production APIs in an incompatible way",
+ file=sys.stderr,
+ )
sys.exit(-1)
else:
- print('*' * 67)
- print('* VPP CHECKAPI SUCCESSFULLY COMPLETED')
- print('*' * 67)
+ print("*" * 67)
+ print("* VPP CHECKAPI SUCCESSFULLY COMPLETED")
+ print("*" * 67)
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/extras/scripts/list_api_changes.py b/extras/scripts/list_api_changes.py
index ac8cf47..7433f54 100755
--- a/extras/scripts/list_api_changes.py
+++ b/extras/scripts/list_api_changes.py
@@ -4,21 +4,21 @@
import os
import subprocess
-starttag = 'v19.08-rc0'
-endtag = 'HEAD'
+starttag = "v19.08-rc0"
+endtag = "HEAD"
emit_md = True
apifiles = []
-for root, dirnames, filenames in os.walk('.'):
- for filename in fnmatch.filter(filenames, '*.api'):
+for root, dirnames, filenames in os.walk("."):
+ for filename in fnmatch.filter(filenames, "*.api"):
apifiles.append(os.path.join(root, filename))
for f in apifiles:
- commits = subprocess.check_output(['git', 'log',
- '--oneline', starttag + '..' + endtag,
- f])
+ commits = subprocess.check_output(
+ ["git", "log", "--oneline", starttag + ".." + endtag, f]
+ )
if commits:
- if f[0:2] == './':
+ if f[0:2] == "./":
f = f[2:]
if emit_md:
print("| @c %s ||" % f)
@@ -27,9 +27,10 @@
parts = line.strip().split()
commit = parts[0]
message = b" ".join(parts[1:]).decode().replace("|", r"\|")
- print("| [%s](https://gerrit.fd.io/r/gitweb?"
- "p=vpp.git;a=commit;h=%s) | %s |" % (
- commit, commit, message))
+ print(
+ "| [%s](https://gerrit.fd.io/r/gitweb?"
+ "p=vpp.git;a=commit;h=%s) | %s |" % (commit, commit, message)
+ )
print()
else:
print(f)