| #!/usr/bin/env python3 |
| |
| """ |
| crcchecker is a tool to used to enforce that .api messages do not change. |
| API files with a semantic version < 1.0.0 are ignored. |
| """ |
| |
| import sys |
| import os |
| import json |
| import argparse |
| import re |
| from subprocess import run, PIPE, check_output, CalledProcessError |
| |
| # pylint: disable=subprocess-run-check |
| |
| ROOTDIR = os.path.dirname(os.path.realpath(__file__)) + "/../.." |
| APIGENBIN = f"{ROOTDIR}/src/tools/vppapigen/vppapigen.py" |
| |
| |
| def crc_from_apigen(revision, filename): |
| """Runs vppapigen with crc plugin returning a JSON object with CRCs for |
| all APIs in filename""" |
| if not revision and not os.path.isfile(filename): |
| print(f"skipping: {filename}", file=sys.stderr) |
| # Return <class 'set'> instead of <class 'dict'> |
| return {-1} |
| |
| if revision: |
| apigen = ( |
| f"{APIGENBIN} --git-revision {revision} --includedir src " |
| f"--input {filename} CRC" |
| ) |
| else: |
| apigen = f"{APIGENBIN} --includedir src --input {filename} CRC" |
| returncode = run(apigen.split(), stdout=PIPE, stderr=PIPE) |
| if returncode.returncode == 2: # No such file |
| print(f"skipping: {revision}:{filename} {returncode}", file=sys.stderr) |
| return {} |
| if returncode.returncode != 0: |
| print( |
| f"vppapigen failed for {revision}:{filename} with " |
| f"command:\n {apigen}\n error: {returncode.returncode}", |
| file=sys.stderr, |
| ) |
| if returncode.stderr: |
| print(f"stderr: {returncode.stderr.decode('ascii')}", file=sys.stderr) |
| if returncode.stdout: |
| print(f"stdout: {returncode.stdout.decode('ascii')}", file=sys.stderr) |
| sys.exit(-2) |
| |
| return json.loads(returncode.stdout) |
| |
| |
| def dict_compare(dict1, dict2): |
| """Compare two dictionaries returning added, removed, modified |
| and equal entries""" |
| d1_keys = set(dict1.keys()) |
| d2_keys = set(dict2.keys()) |
| intersect_keys = d1_keys.intersection(d2_keys) |
| added = d1_keys - d2_keys |
| removed = d2_keys - d1_keys |
| modified = { |
| o: (dict1[o], dict2[o]) |
| for o in intersect_keys |
| if dict1[o]["crc"] != dict2[o]["crc"] |
| } |
| same = set(o for o in intersect_keys if dict1[o] == dict2[o]) |
| return added, removed, modified, same |
| |
| |
| def filelist_from_git_ls(): |
| """Returns a list of all api files in the git repository""" |
| filelist = [] |
| git_ls = "git ls-files *.api" |
| returncode = run(git_ls.split(), stdout=PIPE, stderr=PIPE) |
| if returncode.returncode != 0: |
| sys.exit(returncode.returncode) |
| |
| for line in returncode.stdout.decode("ascii").split("\n"): |
| if line: |
| filelist.append(line) |
| return filelist |
| |
| |
| def is_uncommitted_changes(): |
| """Returns true if there are uncommitted changes in the repo""" |
| # Don't run this check in the Jenkins CI |
| if os.getenv("FDIOTOOLS_IMAGE") is None: |
| git_status = "git status --porcelain -uno" |
| returncode = run(git_status.split(), stdout=PIPE, stderr=PIPE) |
| if returncode.returncode != 0: |
| sys.exit(returncode.returncode) |
| |
| if returncode.stdout: |
| return True |
| return False |
| |
| |
| def filelist_from_git_grep(filename): |
| """Returns a list of api files that this <filename> api files imports.""" |
| filelist = [] |
| try: |
| returncode = check_output( |
| f'git grep -e "import .*{filename}"' " -- *.api", shell=True |
| ) |
| except CalledProcessError: |
| return [] |
| for line in returncode.decode("ascii").split("\n"): |
| if line: |
| filename, _ = line.split(":") |
| filelist.append(filename) |
| return filelist |
| |
| |
| def filelist_from_patchset(pattern): |
| """Returns list of api files in changeset and the list of api |
| files they import.""" |
| filelist = [] |
| git_cmd = ( |
| "((git diff HEAD~1.. --name-only;git ls-files -m) | " |
| 'sort -u | grep "\\.api$")' |
| ) |
| try: |
| res = check_output(git_cmd, shell=True) |
| except CalledProcessError: |
| return [] |
| |
| # Check for dependencies (imports) |
| imported_files = [] |
| for line in res.decode("ascii").split("\n"): |
| if not line: |
| continue |
| if not re.search(pattern, line): |
| continue |
| filelist.append(line) |
| imported_files.extend(filelist_from_git_grep(os.path.basename(line))) |
| |
| filelist.extend(imported_files) |
| return set(filelist) |
| |
| |
| def is_deprecated(message): |
| """Given a message, return True if message is deprecated""" |
| if "options" in message: |
| if "deprecated" in message["options"]: |
| return True |
| # recognize the deprecated format |
| if ( |
| "status" in message["options"] |
| and message["options"]["status"] == "deprecated" |
| ): |
| print("WARNING: please use 'option deprecated;'") |
| return True |
| return False |
| |
| |
| def is_in_progress(message): |
| """Given a message, return True if message is marked as in_progress""" |
| if "options" in message: |
| if "in_progress" in message["options"]: |
| return True |
| # recognize the deprecated format |
| if ( |
| "status" in message["options"] |
| and message["options"]["status"] == "in_progress" |
| ): |
| print("WARNING: please use 'option in_progress;'") |
| return True |
| return False |
| |
| |
| def report(new, old): |
| """Given a dictionary of new crcs and old crcs, print all the |
| added, removed, modified, in-progress, deprecated messages. |
| Return the number of backwards incompatible changes made.""" |
| |
| # pylint: disable=too-many-branches |
| |
| new.pop("_version", None) |
| old.pop("_version", None) |
| added, removed, modified, _ = dict_compare(new, old) |
| backwards_incompatible = 0 |
| |
| # print the full list of in-progress messages |
| # they should eventually either disappear of become supported |
| for k in new.keys(): |
| newversion = int(new[k]["version"]) |
| if newversion == 0 or is_in_progress(new[k]): |
| print(f"in-progress: {k}") |
| for k in added: |
| print(f"added: {k}") |
| for k in removed: |
| oldversion = int(old[k]["version"]) |
| if oldversion > 0 and not is_deprecated(old[k]) and not is_in_progress(old[k]): |
| backwards_incompatible += 1 |
| print(f"removed: ** {k}") |
| else: |
| print(f"removed: {k}") |
| for k in modified.keys(): |
| oldversion = int(old[k]["version"]) |
| newversion = int(new[k]["version"]) |
| if oldversion > 0 and not is_in_progress(old[k]): |
| backwards_incompatible += 1 |
| print(f"modified: ** {k}") |
| else: |
| print(f"modified: {k}") |
| |
| # check which messages are still there but were marked for deprecation |
| for k in new.keys(): |
| newversion = int(new[k]["version"]) |
| if newversion > 0 and is_deprecated(new[k]): |
| if k in old: |
| if not is_deprecated(old[k]): |
| print(f"deprecated: {k}") |
| else: |
| print(f"added+deprecated: {k}") |
| |
| return backwards_incompatible |
| |
| |
| def check_patchset(): |
| """Compare the changes to API messages in this changeset. |
| Ignores API files with version < 1.0.0. |
| Only considers API files located under the src directory in the repo. |
| """ |
| files = filelist_from_patchset("^src/") |
| revision = "HEAD~1" |
| |
| oldcrcs = {} |
| newcrcs = {} |
| for filename in files: |
| # Ignore files that have version < 1.0.0 |
| _ = crc_from_apigen(None, filename) |
| # Ignore removed files |
| if isinstance(_, set) == 0: |
| if isinstance(_, set) == 0 and _["_version"]["major"] == "0": |
| continue |
| newcrcs.update(_) |
| |
| oldcrcs.update(crc_from_apigen(revision, filename)) |
| |
| backwards_incompatible = report(newcrcs, oldcrcs) |
| if backwards_incompatible: |
| # alert on changing production API |
| print( |
| "crcchecker: Changing production APIs in an incompatible way", |
| file=sys.stderr, |
| ) |
| sys.exit(-1) |
| else: |
| print("*" * 67) |
| print("* VPP CHECKAPI SUCCESSFULLY COMPLETED") |
| print("*" * 67) |
| |
| |
| def main(): |
| """Main entry point.""" |
| parser = argparse.ArgumentParser(description="VPP CRC checker.") |
| parser.add_argument("--git-revision", help="Git revision to compare against") |
| parser.add_argument( |
| "--dump-manifest", action="store_true", help="Dump CRC for all messages" |
| ) |
| parser.add_argument( |
| "--check-patchset", |
| action="store_true", |
| help="Check patchset for backwards incompatbile changes", |
| ) |
| parser.add_argument("files", nargs="*") |
| parser.add_argument("--diff", help="Files to compare (on filesystem)", nargs=2) |
| |
| args = parser.parse_args() |
| |
| if args.diff and args.files: |
| parser.print_help() |
| sys.exit(-1) |
| |
| # Diff two files |
| if args.diff: |
| oldcrcs = crc_from_apigen(None, args.diff[0]) |
| newcrcs = crc_from_apigen(None, args.diff[1]) |
| backwards_incompatible = report(newcrcs, oldcrcs) |
| sys.exit(0) |
| |
| # Dump CRC for messages in given files / revision |
| if args.dump_manifest: |
| files = args.files if args.files else filelist_from_git_ls() |
| crcs = {} |
| for filename in files: |
| crcs.update(crc_from_apigen(args.git_revision, filename)) |
| for k, value in crcs.items(): |
| print(f"{k}: {value}") |
| sys.exit(0) |
| |
| # Find changes between current patchset and given revision (previous) |
| if args.check_patchset: |
| if args.git_revision: |
| print("Argument git-revision ignored", file=sys.stderr) |
| # Check there are no uncomitted changes |
| if is_uncommitted_changes(): |
| print("Please stash or commit changes in workspace", file=sys.stderr) |
| sys.exit(-1) |
| check_patchset() |
| sys.exit(0) |
| |
| # Find changes between current workspace and revision |
| # Find changes between a given file and a revision |
| files = args.files if args.files else filelist_from_git_ls() |
| |
| revision = args.git_revision if args.git_revision else "HEAD~1" |
| |
| oldcrcs = {} |
| newcrcs = {} |
| for file in files: |
| newcrcs.update(crc_from_apigen(None, file)) |
| oldcrcs.update(crc_from_apigen(revision, file)) |
| |
| backwards_incompatible = report(newcrcs, oldcrcs) |
| |
| if args.check_patchset: |
| if backwards_incompatible: |
| # alert on changing production API |
| print( |
| "crcchecker: Changing production APIs in an incompatible way", |
| file=sys.stderr, |
| ) |
| sys.exit(-1) |
| else: |
| print("*" * 67) |
| print("* VPP CHECKAPI SUCCESSFULLY COMPLETED") |
| print("*" * 67) |
| |
| |
| if __name__ == "__main__": |
| main() |