blob: 8b999114e52c7848b6fbadcd2bf015481379e9b8 [file] [log] [blame]
Chris Luke54ccf222016-07-25 16:38:11 -04001#!/usr/bin/env python
2# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at:
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# Looks for preprocessor macros with struct initializers and siphons them
17# off into another file for later parsing; ostensibly to generate
18# documentation from struct initializer data.
19
20import os, sys, re, argparse, json
21
22DEFAULT_OUTPUT = "build-root/docs/siphons"
23DEFAULT_PREFIX = os.getcwd()
24
25ap = argparse.ArgumentParser()
26ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
27 help="Output directory for .siphon files [%s]" % DEFAULT_OUTPUT)
28ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
29 help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX)
30ap.add_argument("input", nargs='+', metavar="input_file",
31 help="Input C source files")
32args = ap.parse_args()
33
34"""Patterns that match the start of code blocks we want to siphon"""
35siphon_patterns = [
36 ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"), "clicmd" ),
37]
38
39"""Matches a siphon comment block start"""
40siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
41
42"""Matches a siphon comment block stop"""
43siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
44
45"""Siphon block directive delimiter"""
46siphon_block_delimiter = "%%"
47
48"""Matches a siphon block directive such as '%clicmd:group_label Debug CLI%'"""
49siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
50 (siphon_block_delimiter, siphon_block_delimiter))
51
52"""Matches the start of an initializer block"""
53siphon_initializer = re.compile("\s*=")
54
55"""
56count open and close braces in str
57return (0, index) when braces were found and count becomes 0.
58index indicates the position at which the last closing brace was
59found.
60return (-1, -1) if a closing brace is found before any opening one.
61return (count, -1) if not all opening braces are closed, count is the
62current depth
63"""
64def count_braces(str, count=0, found=False):
65 for index in range(0, len(str)):
66 if str[index] == '{':
67 count += 1;
68 found = True
69 elif str[index] == '}':
70 if count == 0:
71 # means we never found an open brace
72 return (-1, -1)
73 count -= 1;
74
75 if count == 0 and found:
76 return (count, index)
77
78 return (count, -1)
79
80# Collated output for each siphon
81output = {}
82
Chris Luked4024f52016-09-06 09:32:36 -040083# Build a list of known siphons
84known_siphons = []
85for item in siphon_patterns:
86 siphon = item[1]
87 if siphon not in known_siphons:
88 known_siphons.append(siphon)
89
90# Setup information for siphons we know about
91for siphon in known_siphons:
92 output[siphon] = {
93 "file": "%s/%s.siphon" % (args.output, siphon),
94 "global": {},
95 "items": [],
96 }
97
Chris Luke54ccf222016-07-25 16:38:11 -040098# Pre-process file names in case they indicate a file with
99# a list of files
100files = []
101for filename in args.input:
102 if filename.startswith('@'):
103 with open(filename[1:], 'r') as fp:
104 lines = fp.readlines()
105 for line in lines:
106 files.append(line.strip())
107 lines = None
108 else:
109 files.append(filename)
110
111# Iterate all the input files we've been given
112for filename in files:
113 # Strip the current directory off the start of the
114 # filename for brevity
115 if filename[0:len(args.input_prefix)] == args.input_prefix:
116 filename = filename[len(args.input_prefix):]
117 if filename[0] == "/":
118 filename = filename[1:]
119
120 # Work out the abbreviated directory name
121 directory = os.path.dirname(filename)
122 if directory[0:2] == "./":
123 directory = directory[2:]
124 elif directory[0:len(args.input_prefix)] == args.input_prefix:
125 directory = directory[len(args.input_prefix):]
126 if directory[0] == "/":
127 directory = directory[1:]
128
129 # Open the file and explore its contents...
130 sys.stderr.write("Siphoning from %s...\n" % filename)
131 directives = {}
132 with open(filename) as fd:
133 siphon = None
134 close_siphon = None
135 siphon_block = ""
136 in_block = False
137 line_num = 0
138 siphon_line = 0
139
140 for line in fd:
141 line_num += 1
142 str = line[:-1] # filter \n
143
144 """See if there is a block directive and if so extract it"""
145 def process_block_directive(str, directives):
146 m = siphon_block_directive.search(str)
147 if m is not None:
148 k = m.group(2)
149 v = m.group(3).strip()
150 directives[k] = v
151 # Return only the parts we did not match
152 return str[0:m.start(1)] + str[m.end(4):]
153
154 return str
155
156 def process_block_prefix(str):
157 if str.startswith(" * "):
158 str = str[3:]
159 elif str == " *":
160 str = ""
161 return str
162
163 if not in_block:
164 # See if the line contains the start of a siphon doc block
165 m = siphon_block_start.search(str)
166 if m is not None:
167 in_block = True
168 t = m.group(1)
169
170 # Now check if the block closes on the same line
171 m = siphon_block_stop.search(t)
172 if m is not None:
173 t = m.group(1)
174 in_block = False
175
176 # Check for directives
177 t = process_block_directive(t, directives)
178
179 # Filter for normal comment prefixes
180 t = process_block_prefix(t)
181
182 # Add what is left
183 siphon_block += t
184
185 # Skip to next line
186 continue
187
188 else:
189 # Check to see if we have an end block marker
190 m = siphon_block_stop.search(str)
191 if m is not None:
192 in_block = False
193 t = m.group(1)
194 else:
195 t = str
196
197 # Check for directives
198 t = process_block_directive(t, directives)
199
200 # Filter for normal comment prefixes
201 t = process_block_prefix(t)
202
203 # Add what is left
204 siphon_block += t + "\n"
205
206 # Skip to next line
207 continue
208
209
210 if siphon is None:
211 # Look for blocks we need to siphon
212 for p in siphon_patterns:
213 if p[0].match(str):
214 siphon = [ p[1], str + "\n", 0 ]
215 siphon_line = line_num
216
217 # see if we have an initializer
218 m = siphon_initializer.search(str)
219 if m is not None:
220 # count the braces on this line
221 (count, index) = count_braces(str[m.start():])
222 siphon[2] = count
223 # TODO - it's possible we have the initializer all on the first line
224 # we should check for it, but also account for the possibility that
225 # the open brace is on the next line
226 #if count == 0:
227 # # braces balanced
228 # close_siphon = siphon
229 # siphon = None
230 else:
231 # no initializer: close the siphon right now
232 close_siphon = siphon
233 siphon = None
234 else:
235 # See if we should end the siphon here - do we have balanced
236 # braces?
237 (count, index) = count_braces(str, count=siphon[2], found=True)
238 if count == 0:
239 # braces balanced - add the substring and close the siphon
240 siphon[1] += str[:index+1] + ";\n"
241 close_siphon = siphon
242 siphon = None
243 else:
244 # add the whole string, move on
245 siphon[2] = count
246 siphon[1] += str + "\n"
247
248 if close_siphon is not None:
249 # Write the siphoned contents to the right place
250 siphon_name = close_siphon[0]
Chris Luke54ccf222016-07-25 16:38:11 -0400251
252 # Copy directives for the file
253 details = {}
254 for key in directives:
255 if ":" in key:
256 (sn, label) = key.split(":")
257 if sn == siphon_name:
258 details[label] = directives[key]
259 else:
260 details[key] = directives[key]
261
262 # Copy details for this block
263 details['file'] = filename
264 details['line_start'] = siphon_line
265 details['line_end'] = line_num
266 details['siphon_block'] = siphon_block.strip()
267
268 # Some defaults
269 if "group" not in details:
270 if "group_label" in details:
271 # use the filename since group labels are mostly of file scope
272 details['group'] = details['file']
273 else:
274 details['group'] = directory
275
276 if "group_label" not in details:
277 details['group_label'] = details['group']
278
279 details["block"] = close_siphon[1]
280
281 # Store the item
282 output[siphon_name]['items'].append(details)
283
284 # All done
285 close_siphon = None
286 siphon_block = ""
287
288 # Update globals
289 for key in directives.keys():
290 if ':' not in key:
291 continue
292
293 if filename.endswith("/dir.dox"):
294 # very special! use the parent directory name
295 l = directory
296 else:
297 l = filename
298
299 (sn, label) = key.split(":")
300
301 if sn not in output:
302 output[sn] = {}
303 if 'global' not in output[sn]:
304 output[sn]['global'] = {}
305 if l not in output[sn]['global']:
306 output[sn]['global'][l] = {}
307 if 'file' not in output[sn]:
308 output[sn]['file'] = "%s/%s.siphon" % (args.output, sn)
309 if 'items' not in output[sn]:
310 output[sn]['items'] = []
311
312 output[sn]['global'][l][label] = directives[key]
313
314
315# Write out the data
316for siphon in output.keys():
317 sys.stderr.write("Saving siphon %s...\n" % siphon)
318 s = output[siphon]
319 with open(s['file'], "a") as fp:
320 json.dump(s, fp, separators=(',', ': '), indent=4, sort_keys=True)
321
322# All done