vppapigen: make json in parallel
Type: improvement
This patches makes the make json-api-files
run in parallel in the same python runtime.
Default number of workers is 8, and run time
goes from ~20s to ~2s on average.
Change-Id: Id8cff013889db2671f6b6b4af9a019460c656f81
Signed-off-by: Nathan Skrzypczak <nathan.skrzypczak@gmail.com>
diff --git a/src/tools/vppapigen/generate_json.py b/src/tools/vppapigen/generate_json.py
index e8041c5..610f84f 100755
--- a/src/tools/vppapigen/generate_json.py
+++ b/src/tools/vppapigen/generate_json.py
@@ -16,15 +16,15 @@
import argparse
import pathlib
import subprocess
+import vppapigen
+import os
+from multiprocessing import Pool
BASE_DIR = (
subprocess.check_output("git rev-parse --show-toplevel", shell=True)
.strip()
.decode()
)
-vppapigen_bin = pathlib.Path(
- "%s/src/tools/vppapigen/vppapigen.py" % BASE_DIR
-).as_posix()
src_dir_depth = 3
output_path = pathlib.Path(
@@ -55,32 +55,14 @@
return [x for x in api_search_globs(src_dir)]
-def vppapigen(vppapigen_bin, output_path, src_dir, src_file):
- try:
- subprocess.check_output(
- [
- vppapigen_bin,
- "--includedir",
- src_dir.as_posix(),
- "--input",
- src_file.as_posix(),
- "JSON",
- "--output",
- "%s/%s/%s.json"
- % (
- output_path,
- output_dir_map[
- src_file.as_posix().split("/")[
- src_dir_depth + BASE_DIR.count("/") - 1
- ]
- ],
- src_file.name,
- ),
- ]
- )
- except KeyError:
- print("src_file: %s" % src_file)
- raise
+def get_n_parallel(n_parallel):
+ if n_parallel == 0:
+ n_parallel = os.environ.get("MAKE_PARALLEL_JOBS", os.cpu_count())
+ try:
+ n_parallel = int(n_parallel)
+ except ValueError:
+ return os.cpu_count()
+ return n_parallel or os.cpu_count()
def main():
@@ -88,11 +70,15 @@
cliparser.add_argument("--srcdir", action="store", default="%s/src" % BASE_DIR),
cliparser.add_argument("--output", action="store", help="directory to store files"),
cliparser.add_argument(
+ "--parallel", type=int, default=0, help="Number of parallel processes"
+ ),
+ cliparser.add_argument(
"--debug-target",
action="store_true",
default=False,
help="'True' if -debug target",
),
+
args = cliparser.parse_args()
src_dir = pathlib.Path(args.srcdir)
@@ -109,8 +95,29 @@
for f in output_dir.glob("**/*.api.json"):
f.unlink()
- for f in api_files(src_dir):
- vppapigen(vppapigen_bin, output_dir, src_dir, f)
+ with Pool(get_n_parallel(args.parallel)) as p:
+ p.map(
+ vppapigen.run_kw_vppapigen,
+ [
+ {
+ "output": "%s/%s/%s.json"
+ % (
+ output_path,
+ output_dir_map[
+ f.as_posix().split("/")[
+ src_dir_depth + BASE_DIR.count("/") - 1
+ ]
+ ],
+ f.name,
+ ),
+ "input_file": f.as_posix(),
+ "includedir": [src_dir.as_posix()],
+ "output_module": "JSON",
+ }
+ for f in api_files(src_dir)
+ ],
+ )
+
print("json files written to: %s/." % output_dir)