blob: 2c08b90a93beedd5d9a49a71554b7e86836b6926 [file] [log] [blame]
Klement Sekera958b7502017-09-28 06:31:53 +02001#!/usr/bin/env python2
Klement Sekera8f2a4ea2017-05-04 06:15:18 +02002
3import json
4
5
Klement Sekera8f2a4ea2017-05-04 06:15:18 +02006class ParseError (Exception):
7 pass
8
9
10magic_prefix = "vl_api_"
11magic_suffix = "_t"
12
13
14def remove_magic(what):
15 if what.startswith(magic_prefix) and what.endswith(magic_suffix):
16 return what[len(magic_prefix): - len(magic_suffix)]
17 return what
18
19
Klement Sekera958b7502017-09-28 06:31:53 +020020class Field(object):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020021
Klement Sekera2108c0c2018-08-24 11:43:20 +020022 def __init__(self, field_name, field_type, array_len=None,
23 nelem_field=None):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020024 self.name = field_name
25 self.type = field_type
26 self.len = array_len
27 self.nelem_field = nelem_field
28
29 def __str__(self):
30 if self.len is None:
Klement Sekera34a962b2018-09-06 19:31:36 +020031 return "Field(name: %s, type: %s)" % (self.name, self.type)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020032 elif self.len > 0:
Klement Sekera34a962b2018-09-06 19:31:36 +020033 return "Field(name: %s, type: %s, length: %s)" % (self.name,
34 self.type,
35 self.len)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020036 else:
Klement Sekera34a962b2018-09-06 19:31:36 +020037 return (
38 "Field(name: %s, type: %s, variable length stored in: %s)" %
39 (self.name, self.type, self.nelem_field))
40
41 def is_vla(self):
42 return self.nelem_field is not None
43
44 def has_vla(self):
45 return self.is_vla() or self.type.has_vla()
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020046
47
Ole Troan53fffa12018-11-13 12:36:56 +010048class Alias(Field):
49 pass
50
51
Klement Sekera958b7502017-09-28 06:31:53 +020052class Type(object):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020053 def __init__(self, name):
54 self.name = name
55
Klement Sekera32a9d7b2017-12-10 05:15:41 +010056 def __str__(self):
57 return self.name
58
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020059
60class SimpleType (Type):
61
Klement Sekera34a962b2018-09-06 19:31:36 +020062 def has_vla(self):
63 return False
64
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020065
Klement Sekera2108c0c2018-08-24 11:43:20 +020066def get_msg_header_defs(struct_type_class, field_class, json_parser, logger):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020067 return [
68 struct_type_class(['msg_header1_t',
69 ['u16', '_vl_msg_id'],
70 ['u32', 'context'],
71 ],
Klement Sekera2108c0c2018-08-24 11:43:20 +020072 json_parser, field_class, logger
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020073 ),
74 struct_type_class(['msg_header2_t',
75 ['u16', '_vl_msg_id'],
76 ['u32', 'client_index'],
77 ['u32', 'context'],
78 ],
Klement Sekera2108c0c2018-08-24 11:43:20 +020079 json_parser, field_class, logger
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020080 ),
81 ]
82
83
Klement Sekera958b7502017-09-28 06:31:53 +020084class Struct(object):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020085
86 def __init__(self, name, fields):
87 self.name = name
88 self.fields = fields
89 self.field_names = [n.name for n in self.fields]
Klement Sekera2108c0c2018-08-24 11:43:20 +020090 self.depends = [f.type for f in self.fields]
Klement Sekera8f2a4ea2017-05-04 06:15:18 +020091
Klement Sekera32a9d7b2017-12-10 05:15:41 +010092 def __str__(self):
93 return "[%s]" % "], [".join([str(f) for f in self.fields])
94
Klement Sekera34a962b2018-09-06 19:31:36 +020095 def has_vla(self):
96 for f in self.fields:
97 if f.has_vla():
98 return True
99 return False
100
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200101
Klement Sekera2108c0c2018-08-24 11:43:20 +0200102class Enum(SimpleType):
103 def __init__(self, name, value_pairs, enumtype):
104 super(Enum, self).__init__(name)
105 self.type = enumtype
106 self.value_pairs = value_pairs
107
108 def __str__(self):
109 return "Enum(%s, [%s])" % (
110 self.name,
111 "], [" .join(["%s => %s" % (i, j) for i, j in self.value_pairs])
112 )
113
114
115class Union(Type):
116 def __init__(self, name, type_pairs, crc):
117 Type.__init__(self, name)
118 self.crc = crc
119 self.type_pairs = type_pairs
120 self.depends = [t for t, _ in self.type_pairs]
121
122 def __str__(self):
123 return "Union(%s, [%s])" % (
124 self.name,
125 "], [" .join(["%s %s" % (i, j) for i, j in self.type_pairs])
126 )
127
Klement Sekera34a962b2018-09-06 19:31:36 +0200128 def has_vla(self):
129 return False
130
Klement Sekera2108c0c2018-08-24 11:43:20 +0200131
Klement Sekera958b7502017-09-28 06:31:53 +0200132class Message(object):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200133
Klement Sekera2108c0c2018-08-24 11:43:20 +0200134 def __init__(self, logger, definition, json_parser):
135 struct_type_class = json_parser.struct_type_class
136 field_class = json_parser.field_class
Klement Sekeradc15be22017-06-12 06:49:33 +0200137 self.request = None
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200138 self.logger = logger
139 m = definition
140 logger.debug("Parsing message definition `%s'" % m)
141 name = m[0]
142 self.name = name
143 logger.debug("Message name is `%s'" % name)
144 ignore = True
145 self.header = None
Klement Sekera2108c0c2018-08-24 11:43:20 +0200146 self.is_reply = json_parser.is_reply(self.name)
147 self.is_event = json_parser.is_event(self.name)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200148 fields = []
149 for header in get_msg_header_defs(struct_type_class, field_class,
Klement Sekera2108c0c2018-08-24 11:43:20 +0200150 json_parser, logger):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200151 logger.debug("Probing header `%s'" % header.name)
152 if header.is_part_of_def(m[1:]):
153 self.header = header
154 logger.debug("Found header `%s'" % header.name)
155 fields.append(field_class(field_name='header',
156 field_type=self.header))
157 ignore = False
158 break
Klement Sekera2108c0c2018-08-24 11:43:20 +0200159 if ignore and not self.is_event and not self.is_reply:
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200160 raise ParseError("While parsing message `%s': could not find all "
161 "common header fields" % name)
162 for field in m[1:]:
163 if len(field) == 1 and 'crc' in field:
164 self.crc = field['crc']
165 logger.debug("Found CRC `%s'" % self.crc)
166 continue
167 else:
Klement Sekera2108c0c2018-08-24 11:43:20 +0200168 field_type = json_parser.lookup_type_like_id(field[0])
Klement Sekera8b6b5ab2018-05-03 14:27:42 +0200169 logger.debug("Parsing message field `%s'" % field)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200170 if len(field) == 2:
171 if self.header is not None and\
172 self.header.has_field(field[1]):
173 continue
174 p = field_class(field_name=field[1],
175 field_type=field_type)
176 elif len(field) == 3:
177 if field[2] == 0:
178 raise ParseError(
179 "While parsing message `%s': variable length "
180 "array `%s' doesn't have reference to member "
181 "containing the actual length" % (
182 name, field[1]))
183 p = field_class(
184 field_name=field[1],
185 field_type=field_type,
186 array_len=field[2])
187 elif len(field) == 4:
188 nelem_field = None
189 for f in fields:
190 if f.name == field[3]:
191 nelem_field = f
192 if nelem_field is None:
193 raise ParseError(
194 "While parsing message `%s': couldn't find "
195 "variable length array `%s' member containing "
196 "the actual length `%s'" % (
197 name, field[1], field[3]))
198 p = field_class(
199 field_name=field[1],
200 field_type=field_type,
201 array_len=field[2],
202 nelem_field=nelem_field)
203 else:
204 raise Exception("Don't know how to parse message "
205 "definition for message `%s': `%s'" %
206 (m, m[1:]))
207 logger.debug("Parsed field `%s'" % p)
208 fields.append(p)
209 self.fields = fields
Klement Sekera2108c0c2018-08-24 11:43:20 +0200210 self.depends = [f.type for f in self.fields]
Klement Sekera34a962b2018-09-06 19:31:36 +0200211 logger.debug("Parsed message: %s" % self)
212
213 def __str__(self):
214 return "Message(%s, [%s], {crc: %s}" % \
215 (self.name,
216 "], [".join([str(f) for f in self.fields]),
217 self.crc)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200218
219
220class StructType (Type, Struct):
221
Klement Sekera2108c0c2018-08-24 11:43:20 +0200222 def __init__(self, definition, json_parser, field_class, logger):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200223 t = definition
Klement Sekera8b6b5ab2018-05-03 14:27:42 +0200224 logger.debug("Parsing struct definition `%s'" % t)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200225 name = t[0]
226 fields = []
227 for field in t[1:]:
228 if len(field) == 1 and 'crc' in field:
229 self.crc = field['crc']
230 continue
Klement Sekera2108c0c2018-08-24 11:43:20 +0200231 field_type = json_parser.lookup_type_like_id(field[0])
Klement Sekera8b6b5ab2018-05-03 14:27:42 +0200232 logger.debug("Parsing type field `%s'" % field)
Klement Sekera329a1c12018-02-25 18:48:04 +0100233 if len(field) == 2:
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200234 p = field_class(field_name=field[1],
Klement Sekera329a1c12018-02-25 18:48:04 +0100235 field_type=field_type)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200236 elif len(field) == 3:
237 if field[2] == 0:
238 raise ParseError("While parsing type `%s': array `%s' has "
239 "variable length" % (name, field[1]))
240 p = field_class(field_name=field[1],
Klement Sekera329a1c12018-02-25 18:48:04 +0100241 field_type=field_type,
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200242 array_len=field[2])
Klement Sekera8b6b5ab2018-05-03 14:27:42 +0200243 elif len(field) == 4:
244 nelem_field = None
245 for f in fields:
246 if f.name == field[3]:
247 nelem_field = f
248 if nelem_field is None:
249 raise ParseError(
250 "While parsing message `%s': couldn't find "
251 "variable length array `%s' member containing "
252 "the actual length `%s'" % (
253 name, field[1], field[3]))
254 p = field_class(field_name=field[1],
255 field_type=field_type,
256 array_len=field[2],
257 nelem_field=nelem_field)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200258 else:
259 raise ParseError(
Klement Sekera8b6b5ab2018-05-03 14:27:42 +0200260 "Don't know how to parse field `%s' of type definition "
261 "for type `%s'" % (field, t))
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200262 fields.append(p)
263 Type.__init__(self, name)
264 Struct.__init__(self, name, fields)
265
Klement Sekera32a9d7b2017-12-10 05:15:41 +0100266 def __str__(self):
267 return "StructType(%s, %s)" % (Type.__str__(self),
268 Struct.__str__(self))
269
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200270 def has_field(self, name):
271 return name in self.field_names
272
273 def is_part_of_def(self, definition):
274 for idx in range(len(self.fields)):
275 field = definition[idx]
276 p = self.fields[idx]
277 if field[1] != p.name:
278 return False
279 if field[0] != p.type.name:
280 raise ParseError(
281 "Unexpected field type `%s' (should be `%s'), "
282 "while parsing msg/def/field `%s/%s/%s'" %
283 (field[0], p.type, p.name, definition, field))
284 return True
285
286
Klement Sekera958b7502017-09-28 06:31:53 +0200287class JsonParser(object):
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200288 def __init__(self, logger, files, simple_type_class=SimpleType,
Klement Sekera2108c0c2018-08-24 11:43:20 +0200289 enum_class=Enum, union_class=Union,
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200290 struct_type_class=StructType, field_class=Field,
Ole Troan53fffa12018-11-13 12:36:56 +0100291 message_class=Message, alias_class=Alias):
Klement Sekera2108c0c2018-08-24 11:43:20 +0200292 self.services = {}
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200293 self.messages = {}
Klement Sekera2108c0c2018-08-24 11:43:20 +0200294 self.enums = {}
295 self.unions = {}
Ole Troan53fffa12018-11-13 12:36:56 +0100296 self.aliases = {}
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200297 self.types = {
298 x: simple_type_class(x) for x in [
299 'i8', 'i16', 'i32', 'i64',
300 'u8', 'u16', 'u32', 'u64',
301 'f64'
302 ]
303 }
304
Ole Troan003d5da2018-12-18 12:23:13 +0100305 self.types['string'] = simple_type_class('vl_api_string_t')
Klement Sekera2108c0c2018-08-24 11:43:20 +0200306 self.replies = set()
307 self.events = set()
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200308 self.simple_type_class = simple_type_class
Klement Sekera2108c0c2018-08-24 11:43:20 +0200309 self.enum_class = enum_class
310 self.union_class = union_class
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200311 self.struct_type_class = struct_type_class
312 self.field_class = field_class
Ole Troan53fffa12018-11-13 12:36:56 +0100313 self.alias_class = alias_class
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200314 self.message_class = message_class
315
316 self.exceptions = []
317 self.json_files = []
318 self.types_by_json = {}
Klement Sekera2108c0c2018-08-24 11:43:20 +0200319 self.enums_by_json = {}
320 self.unions_by_json = {}
Ole Troan53fffa12018-11-13 12:36:56 +0100321 self.aliases_by_json = {}
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200322 self.messages_by_json = {}
323 self.logger = logger
324 for f in files:
325 self.parse_json_file(f)
326 self.finalize_parsing()
327
328 def parse_json_file(self, path):
329 self.logger.info("Parsing json api file: `%s'" % path)
330 self.json_files.append(path)
Ole Troan52ca7562018-03-06 17:45:32 +0100331 self.types_by_json[path] = []
Klement Sekera2108c0c2018-08-24 11:43:20 +0200332 self.enums_by_json[path] = []
333 self.unions_by_json[path] = []
Ole Troan53fffa12018-11-13 12:36:56 +0100334 self.aliases_by_json[path] = []
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200335 self.messages_by_json[path] = {}
336 with open(path) as f:
337 j = json.load(f)
Klement Sekera2108c0c2018-08-24 11:43:20 +0200338 for k in j['services']:
339 if k in self.services:
340 raise ParseError("Duplicate service `%s'" % k)
341 self.services[k] = j['services'][k]
342 self.replies.add(self.services[k]["reply"])
343 if "events" in self.services[k]:
344 for x in self.services[k]["events"]:
345 self.events.add(x)
346 for e in j['enums']:
347 name = e[0]
348 value_pairs = e[1:-1]
349 enumtype = self.types[e[-1]["enumtype"]]
350 enum = self.enum_class(name, value_pairs, enumtype)
351 self.enums[enum.name] = enum
352 self.logger.debug("Parsed enum: %s" % enum)
353 self.enums_by_json[path].append(enum)
354 exceptions = []
355 progress = 0
356 last_progress = 0
357 while True:
358 for u in j['unions']:
359 name = u[0]
360 if name in self.unions:
361 progress = progress + 1
362 continue
363 try:
364 type_pairs = [[self.lookup_type_like_id(t), n]
365 for t, n in u[1:-1]]
366 crc = u[-1]["crc"]
367 union = self.union_class(name, type_pairs, crc)
368 progress = progress + 1
369 except ParseError as e:
370 exceptions.append(e)
371 continue
372 self.unions[union.name] = union
373 self.logger.debug("Parsed union: %s" % union)
374 self.unions_by_json[path].append(union)
Ole Troan53fffa12018-11-13 12:36:56 +0100375 for name, body in j['aliases'].iteritems():
376 if name in self.aliases:
377 progress = progress + 1
378 continue
379 if 'length' in body:
380 array_len = body['length']
381 else:
382 array_len = None
383 t = self.types[body['type']]
384 alias = self.alias_class(name, t, array_len)
385 self.aliases[name] = alias
386 self.logger.debug("Parsed alias: %s" % alias)
387 self.aliases_by_json[path].append(alias)
Klement Sekera2108c0c2018-08-24 11:43:20 +0200388 for t in j['types']:
389 if t[0] in self.types:
390 progress = progress + 1
391 continue
392 try:
393 type_ = self.struct_type_class(t, self,
394 self.field_class,
395 self.logger)
396 if type_.name in self.types:
397 raise ParseError(
398 "Duplicate type `%s'" % type_.name)
399 progress = progress + 1
400 except ParseError as e:
401 exceptions.append(e)
402 continue
403 self.types[type_.name] = type_
404 self.types_by_json[path].append(type_)
405 self.logger.debug("Parsed type: %s" % type_)
406 if not exceptions:
407 # finished parsing
408 break
409 if progress <= last_progress:
410 # cannot make forward progress
411 self.exceptions.extend(exceptions)
Klement Sekeraab11ec92018-11-26 15:37:28 +0100412 break
Klement Sekera2108c0c2018-08-24 11:43:20 +0200413 exceptions = []
414 last_progress = progress
415 progress = 0
Klement Sekera8b6b5ab2018-05-03 14:27:42 +0200416 prev_length = len(self.messages)
417 processed = []
418 while True:
419 exceptions = []
420 for m in j['messages']:
421 if m in processed:
422 continue
423 try:
Klement Sekera2108c0c2018-08-24 11:43:20 +0200424 msg = self.message_class(self.logger, m, self)
Klement Sekera8b6b5ab2018-05-03 14:27:42 +0200425 if msg.name in self.messages:
426 raise ParseError(
427 "Duplicate message `%s'" % msg.name)
428 except ParseError as e:
429 exceptions.append(e)
430 continue
431 self.messages[msg.name] = msg
432 self.messages_by_json[path][msg.name] = msg
433 processed.append(m)
434 if prev_length == len(self.messages):
435 # cannot make forward progress ...
436 self.exceptions.extend(exceptions)
437 break
438 prev_length = len(self.messages)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200439
Klement Sekera2108c0c2018-08-24 11:43:20 +0200440 def lookup_type_like_id(self, name):
441 mundane_name = remove_magic(name)
442 if name in self.types:
443 return self.types[name]
444 elif name in self.enums:
445 return self.enums[name]
446 elif name in self.unions:
447 return self.unions[name]
Ole Troan53fffa12018-11-13 12:36:56 +0100448 elif name in self.aliases:
449 return self.aliases[name]
Klement Sekera2108c0c2018-08-24 11:43:20 +0200450 elif mundane_name in self.types:
451 return self.types[mundane_name]
452 elif mundane_name in self.enums:
453 return self.enums[mundane_name]
454 elif mundane_name in self.unions:
455 return self.unions[mundane_name]
Ole Troan53fffa12018-11-13 12:36:56 +0100456 elif mundane_name in self.aliases:
457 return self.aliases[mundane_name]
Klement Sekera2108c0c2018-08-24 11:43:20 +0200458 raise ParseError(
459 "Could not find type, enum or union by magic name `%s' nor by "
460 "mundane name `%s'" % (name, mundane_name))
461
462 def is_reply(self, message):
463 return message in self.replies
464
465 def is_event(self, message):
466 return message in self.events
467
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200468 def get_reply(self, message):
Klement Sekera2108c0c2018-08-24 11:43:20 +0200469 return self.messages[self.services[message]['reply']]
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200470
471 def finalize_parsing(self):
472 if len(self.messages) == 0:
473 for e in self.exceptions:
Damjan Marion4c64b6e2018-08-26 18:14:46 +0200474 self.logger.warning(e)
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200475 for jn, j in self.messages_by_json.items():
476 remove = []
477 for n, m in j.items():
478 try:
Klement Sekera2108c0c2018-08-24 11:43:20 +0200479 if not m.is_reply and not m.is_event:
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200480 try:
481 m.reply = self.get_reply(n)
Klement Sekera2108c0c2018-08-24 11:43:20 +0200482 if "stream" in self.services[m.name]:
483 m.reply_is_stream = \
484 self.services[m.name]["stream"]
485 else:
486 m.reply_is_stream = False
Klement Sekeradc15be22017-06-12 06:49:33 +0200487 m.reply.request = m
Klement Sekera8f2a4ea2017-05-04 06:15:18 +0200488 except:
489 raise ParseError(
490 "Cannot find reply to message `%s'" % n)
491 except ParseError as e:
492 self.exceptions.append(e)
493 remove.append(n)
494
495 self.messages_by_json[jn] = {
496 k: v for k, v in j.items() if k not in remove}