Add client-side msg_name_and_crc -> msg_index table
vppapigen now generates per-message crcs. Verified that whitespace
and real changes in message A don't change the crc for message B, etc.
Fixed the sample and flowperpkt plugins to participate. Others need
the same treatment. They don't build due to python/java language binding
build issues.
To use the scheme:
Client connects as usual.
Then call: u32 vl_api_get_msg_index(char * name_and_crc)
name_and_crc is a string like: "flowperpkt_tx_interface_add_del_753301f3",
aka the message name with _%08x <expected crc> appended.
Try these vpp-api-test commands to play with it:
vat# dump_msg_api_table
<snip>
[366]: punt_reply_cca27fbe
[367]: ipsec_spd_dump_5e9ae88e
[368]: ipsec_spd_details_6f7821b0
[369]: sample_macswap_enable_disable_0f2813e2
[370]: sample_macswap_enable_disable_reply_476738e5
[371]: flowperpkt_tx_interface_add_del_753301f3
[372]: flowperpkt_tx_interface_add_del_reply_d47e6e0b
vat# get_msg_id sample_macswap_enable_disable_reply_476738e5
'sample_macswap_enable_disable_reply_476738e5' has message index 370
vat# get_msg_id sample_macswap_enable_disable_reply_476738e3
'sample_macswap_enable_disable_reply_476738e3' not found
CRCs may vary, etc.
vppapigen is used to build a set of JSON representations
of each API file from vpp-api/Makefile.am and that is in
turn used by each language binding (Java, Python, Lua).
Change-Id: I3d64582e779dac5f20cddec79c562c288d8fd9c6
Signed-off-by: Dave Barach <dave@barachs.net>
Signed-off-by: Ole Troan <ot@cisco.com>
diff --git a/vpp-api/Makefile.am b/vpp-api/Makefile.am
index 1812b63..310bd23 100644
--- a/vpp-api/Makefile.am
+++ b/vpp-api/Makefile.am
@@ -1,2 +1,17 @@
AUTOMAKE_OPTIONS = foreign
SUBDIRS = python java
+
+api_json_dir = $(abs_builddir)/vpp-api
+api_srcs:=$(shell find $(prefix)/../ -name '*.api')
+api_json:=$(patsubst %.api,$(api_json_dir)/%.api.json,$(notdir $(api_srcs)))
+
+define define_compile_rules
+$(api_json_dir)/%.api.json: $(1)%.api
+ @echo " + Generating '$$<'"
+ @mkdir -p $$(@D)
+ $(CC) $$(CPPFLAGS) -E -P -C -x c $$< | vppapigen --input - --json $$@
+endef
+
+$(foreach directory,$(dir $(api_srcs)),$(eval $(call define_compile_rules,$(directory))))
+
+BUILT_SOURCES = $(api_json)
diff --git a/vpp-api/configure.ac b/vpp-api/configure.ac
index 0a052cf..3d7bf25 100644
--- a/vpp-api/configure.ac
+++ b/vpp-api/configure.ac
@@ -1,4 +1,4 @@
-AC_INIT(vpp-api, 1.0.0)
+AC_INIT(vpp-api, 1.1.0)
LT_INIT
AC_CONFIG_MACRO_DIR([m4])
AC_CONFIG_SUBDIRS([java])
diff --git a/vpp-api/python/Makefile.am b/vpp-api/python/Makefile.am
index 59b1b92..d2c3fb5 100644
--- a/vpp-api/python/Makefile.am
+++ b/vpp-api/python/Makefile.am
@@ -52,7 +52,8 @@
cd $(srcdir); \
mkdir -p $(prefix)/lib/python2.7/site-packages; \
PYTHONUSERBASE=$(prefix) \
- python setup.py build_ext -L $(prefix)/lib64 install --user
+ python setup.py build_ext -L $(prefix)/lib64 \
+ -I $(prefix)/../vppinfra/include/ install --user
#
# Test client
diff --git a/vpp-api/python/pneum/api-gen.py b/vpp-api/python/pneum/api-gen.py
deleted file mode 100755
index 4456835..0000000
--- a/vpp-api/python/pneum/api-gen.py
+++ /dev/null
@@ -1,351 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) 2016 Cisco and/or its affiliates.
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at:
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import argparse, sys, os, importlib, pprint
-
-parser = argparse.ArgumentParser(description='VPP Python API generator')
-parser.add_argument('-i', action="store", dest="inputfile")
-parser.add_argument('-c', '--cfile', action="store")
-args = parser.parse_args()
-
-sys.path.append(".")
-
-inputfile = args.inputfile.replace('.py', '')
-cfg = importlib.import_module(inputfile, package=None)
-
-# https://docs.python.org/3/library/struct.html
-format_struct = {'u8': 'B',
- 'u16' : 'H',
- 'u32' : 'I',
- 'i32' : 'i',
- 'u64' : 'Q',
- 'f64' : 'd',
- 'vl_api_fib_path_t' : 'IIBBBBBBBBBBBBBBBBBBBBB',
- 'vl_api_ip4_fib_counter_t' : 'IBQQ',
- 'vl_api_ip6_fib_counter_t' : 'QQBQQ',
- 'vl_api_lisp_adjacency_t' : 'B' * 35,
- };
-#
-# NB: If new types are introduced in vpe.api, these must be updated.
-#
-type_size = {'u8': 1,
- 'u16' : 2,
- 'u32' : 4,
- 'i32' : 4,
- 'u64' : 8,
- 'f64' : 8,
- 'vl_api_fib_path_t' : 29,
- 'vl_api_ip4_fib_counter_t' : 21,
- 'vl_api_ip6_fib_counter_t' : 33,
- 'vl_api_lisp_adjacency_t' : 35,
-};
-
-def get_args(t):
- argslist = []
- for i in t:
- if i[1][0] == '_':
- argslist.append(i[1][1:])
- else:
- argslist.append(i[1])
-
- return argslist
-
-def get_pack(t):
- zeroarray = False
- bytecount = 0
- pack = '>'
- tup = u''
- j = -1
- for i in t:
- j += 1
- if len(i) is 3 or len(i) is 4: # TODO: add support for variable length arrays (VPP-162)
- size = type_size[i[0]]
- bytecount += size * int(i[2])
- # Check if we have a zero length array
- if i[2] == '0':
- tup += 'msg[' + str(bytecount) + ':],'
- zeroarray = True
- continue
- if size == 1:
- n = i[2] * size
- pack += str(n) + 's'
- tup += 'tr[' + str(j) + '],'
- continue
- pack += format_struct[i[0]] * int(i[2])
- tup += 'tr[' + str(j) + ':' + str(j + int(i[2])) + '],'
- j += int(i[2]) - 1
- else:
- bytecount += type_size[i[0]]
- pack += format_struct[i[0]]
- tup += 'tr[' + str(j) + '],'
- return pack, bytecount, tup, zeroarray
-
-def get_reply_func(f):
- if f['name']+'_reply' in func_name:
- return func_name[f['name']+'_reply']
- if f['name'].find('_dump') > 0:
- r = f['name'].replace('_dump','_details')
- if r in func_name:
- return func_name[r]
- return None
-
-def get_enums():
- # Read enums from stdin
- enums_by_name = {}
- enums_by_index = {}
- i = 1
- for l in sys.stdin:
- l = l.replace(',\n','')
- print l, "=", i
-
- l = l.replace('VL_API_','').lower()
- enums_by_name[l] = i
- enums_by_index[i] = l
-
- i += 1
- return enums_by_name, enums_by_index
-
-def get_definitions():
- # Pass 1
- func_list = []
- func_name = {}
- i = 1
- for a in cfg.messages:
- pack, packlen, tup, zeroarray = get_pack(a[1:])
- func_name[a[0]] = dict([('name', a[0]), ('pack', pack), ('packlen', packlen), ('tup', tup), ('args', get_args(a[1:])),
- ('zeroarray', zeroarray)])
- func_list.append(func_name[a[0]]) # Indexed by name
- return func_list, func_name
-
-def generate_c_macros(func_list, enums_by_name):
- file = open(args.cfile, 'w+')
- print >>file, "#define foreach_api_msg \\"
- for f in func_list:
- if not f['name'] in enums_by_name:
- continue
- print >>file, "_(" + f['name'].upper() + ", " + f['name'] + ") \\"
- print >>file, '''
-void pneum_set_handlers(void) {
-#define _(N,n) \\
- api_func_table[VL_API_##N] = sizeof(vl_api_##n##_t);
- foreach_api_msg;
-#undef _
-}
- '''
-
-#
-# Print array with a hash of 'decode' and 'multipart'
-# Simplify to do only decode for now. And deduce multipart from _dump?
-#
-def decode_function_print(name, args, pack, packlen, tup):
-
- print(u'def ' + name + u'_decode(msg):')
- print(u" n = namedtuple('" + name + "', '" + ', '.join(args) + "')" +
- '''
- if not n:
- return None
- ''')
- print(u" tr = unpack('" + pack + "', msg[:" + str(packlen) + "])")
- print(u" r = n._make((" + tup + "))" +
- '''
- if not r:
- return None
- return r
- ''')
-
-def function_print(name, id, args, pack, multipart, zeroarray):
- if len(args) < 4:
- print "def", name + "(async = False):"
- else:
- print "def", name + "(" + ', '.join(args[3:]) + ", async = False):"
- print " global waiting_for_reply"
- print " context = get_context(" + id + ")"
-
- print '''
- results[context] = {}
- results[context]['e'] = threading.Event()
- results[context]['e'].clear()
- results[context]['r'] = []
- waiting_for_reply = True
- '''
- if multipart == True:
- print " results[context]['m'] = True"
-
- if zeroarray == True:
- print " vpp_api.write(pack('" + pack + "', " + id + ", 0, context, " + ', '.join(args[3:-1]) + ") + " + args[-1] + ")"
- else:
- print " vpp_api.write(pack('" + pack + "', " + id + ", 0, context, " + ', '.join(args[3:]) + "))"
-
- if multipart == True:
- print " vpp_api.write(pack('>HII', VL_API_CONTROL_PING, 0, context))"
-
- print '''
- if not async:
- results[context]['e'].wait(5)
- return results[context]['r']
- return context
- '''
-
-#
-# Should dynamically create size
-#
-def api_table_print (name, msg_id):
- f = name + '_decode'
- print('api_func_table[' + msg_id + '] = ' + f)
-
-#
-# Generate the main Python file
-#
-
-print '''
-
-#
-# AUTO-GENERATED FILE. PLEASE DO NOT EDIT.
-#
-import sys, time, threading, signal, os, logging
-from struct import *
-from collections import namedtuple
-
-#
-# Import C API shared object
-#
-import vpp_api
-
-context = 0
-results = {}
-waiting_for_reply = False
-
-#
-# XXX: Make this return a unique number
-#
-def get_context(id):
- global context
- context += 1
- return context
-
-def msg_handler(msg):
- global result, context, event_callback, waiting_for_reply
- if not msg:
- logging.warning('vpp_api.read failed')
- return
-
- id = unpack('>H', msg[0:2])
- logging.debug('Received message', id[0])
- if id[0] == VL_API_RX_THREAD_EXIT:
- logging.info("We got told to leave")
- return;
-
- #
- # Decode message and returns a tuple.
- #
- logging.debug('api_func', api_func_table[id[0]])
- r = api_func_table[id[0]](msg)
- if not r:
- logging.warning('Message decode failed', id[0])
- return
-
- if 'context' in r._asdict():
- if r.context > 0:
- context = r.context
-
- #
- # XXX: Call provided callback for event
- # Are we guaranteed to not get an event during processing of other messages?
- # How to differentiate what's a callback message and what not? Context = 0?
- #
- logging.debug('R:', context, r, waiting_for_reply)
- if waiting_for_reply == False:
- event_callback(r)
- return
-
- #
- # Collect results until control ping
- #
- if id[0] == VL_API_CONTROL_PING_REPLY:
- results[context]['e'].set()
- waiting_for_reply = False
- return
- if not context in results:
- logging.warning('Not expecting results for this context', context)
- return
- if 'm' in results[context]:
- results[context]['r'].append(r)
- return
-
- results[context]['r'] = r
- results[context]['e'].set()
- waiting_for_reply = False
-
-def connect(name):
- signal.alarm(3) # 3 second
- rv = vpp_api.connect(name, msg_handler)
- signal.alarm(0)
- logging.info("Connect:", rv)
- return rv
-
-def disconnect():
- rv = vpp_api.disconnect()
- logging.info("Disconnected")
- return rv
-
-def register_event_callback(callback):
- global event_callback
- event_callback = callback
-'''
-
-enums_by_name, enums_by_index = get_enums()
-func_list, func_name = get_definitions()
-
-#
-# Not needed with the new msg_size field.
-# generate_c_macros(func_list, enums_by_name)
-#
-
-pp = pprint.PrettyPrinter(indent=4)
-#print 'enums_by_index =', pp.pprint(enums_by_index)
-#print 'func_name =', pp.pprint(func_name)
-
-# Pass 2
-
-#
-# 1) The VPE API lacks a clear definition of what messages are reply messages
-# 2) Length is missing, and has to be pre-known or in case of variable sized ones calculated per message type
-#
-for f in func_list:
- #if f['name'].find('_reply') > 0 or f['name'].find('_details') > 0:
- decode_function_print(f['name'], f['args'], f['pack'], f['packlen'], f['tup'])
-
- #r = get_reply_func(f)
- #if not r:
- # #
- # # XXX: Functions here are not taken care of. E.g. events
- # #
- # print('Missing function', f)
- # continue
-
- if f['name'].find('_dump') > 0:
- f['multipart'] = True
- else:
- f['multipart'] = False
- msg_id_in = 'VL_API_' + f['name'].upper()
- function_print(f['name'], msg_id_in, f['args'], f['pack'], f['multipart'], f['zeroarray'])
-
-
-print "api_func_table = [0] * 10000"
-for f in func_list:
- # if f['name'].find('_reply') > 0 or f['name'].find('_details') > 0:
- msg_id_in = 'VL_API_' + f['name'].upper()
- api_table_print(f['name'], msg_id_in)
diff --git a/vpp-api/python/pneum/pneum.c b/vpp-api/python/pneum/pneum.c
index ebe47b2..3b54553 100644
--- a/vpp-api/python/pneum/pneum.c
+++ b/vpp-api/python/pneum/pneum.c
@@ -52,7 +52,7 @@
pneum_main_t pneum_main;
-extern int wrap_pneum_callback(char *data, int len);
+pneum_callback_t pneum_callback;
/*
* Satisfy external references when -lvlib is not available.
@@ -62,24 +62,16 @@
clib_warning ("vlib_cli_output called...");
}
-#define vl_api_version(n,v) static u32 vpe_api_version = v;
-#include <vpp-api/vpe.api.h>
-#undef vl_api_version
void
-vl_client_add_api_signatures (vl_api_memclnt_create_t *mp)
+pneum_free (void * msg)
{
- /*
- * Send the main API signature in slot 0. This bit of code must
- * match the checks in ../vpe/api/api.c: vl_msg_api_version_check().
- */
- mp->api_versions[0] = clib_host_to_net_u32 (vpe_api_version);
+ vl_msg_api_free (msg);
}
static void
pneum_api_handler (void *msg)
{
u16 id = ntohs(*((u16 *)msg));
-
if (id == VL_API_RX_THREAD_EXIT) {
pneum_main_t *pm = &pneum_main;
vl_msg_api_free(msg);
@@ -91,8 +83,9 @@
clib_warning("Message ID %d has wrong length: %d\n", id, l);
/* Call Python callback */
- (void)wrap_pneum_callback(msg, l);
- vl_msg_api_free(msg);
+ ASSERT(pneum_callback);
+ (pneum_callback)(msg, l);
+ pneum_free(msg);
}
static void *
@@ -115,8 +108,22 @@
pthread_exit(0);
}
+uword *
+pneum_msg_table_get_hash (void)
+{
+ api_main_t *am = &api_main;
+ return (am->msg_index_by_name_and_crc);
+}
+
int
-pneum_connect (char * name, char * chroot_prefix)
+pneum_msg_table_size(void)
+{
+ api_main_t *am = &api_main;
+ return hash_elts(am->msg_index_by_name_and_crc);
+}
+
+int
+pneum_connect (char * name, char * chroot_prefix, pneum_callback_t cb)
{
int rv = 0;
pneum_main_t *pm = &pneum_main;
@@ -134,12 +141,15 @@
return (-1);
}
- /* Start the rx queue thread */
- rv = pthread_create(&pm->rx_thread_handle, NULL, pneum_rx_thread_fn, 0);
- if (rv) {
- clib_warning("pthread_create returned %d", rv);
- vl_client_api_unmap();
- return (-1);
+ if (cb) {
+ /* Start the rx queue thread */
+ rv = pthread_create(&pm->rx_thread_handle, NULL, pneum_rx_thread_fn, 0);
+ if (rv) {
+ clib_warning("pthread_create returned %d", rv);
+ vl_client_api_unmap();
+ return (-1);
+ }
+ pneum_callback = cb;
}
pm->connected_to_vlib = 1;
@@ -164,6 +174,7 @@
if (pm->connected_to_vlib) {
vl_client_disconnect();
vl_client_api_unmap();
+ pneum_callback = 0;
}
memset (pm, 0, sizeof (*pm));
@@ -175,8 +186,11 @@
{
unix_shared_memory_queue_t *q;
api_main_t *am = &api_main;
+ pneum_main_t *pm = &pneum_main;
uword msg;
+ if (!pm->connected_to_vlib) return -1;
+
*l = 0;
if (am->our_pid == 0) return (-1);
@@ -219,7 +233,9 @@
api_main_t *am = &api_main;
vl_api_header_t *mp = vl_msg_api_alloc(l);
unix_shared_memory_queue_t *q;
+ pneum_main_t *pm = &pneum_main;
+ if (!pm->connected_to_vlib) return -1;
if (!mp) return (-1);
memcpy(mp, p, l);
mp->client_index = pneum_client_index();
@@ -228,7 +244,7 @@
if (rv != 0) {
printf("vpe_api_write fails: %d\n", rv);
/* Clear message */
- vl_msg_api_free(mp);
+ pneum_free(mp);
}
return (rv);
}
diff --git a/vpp-api/python/pneum/pneum.h b/vpp-api/python/pneum/pneum.h
index 75b10f8..00585eb 100644
--- a/vpp-api/python/pneum/pneum.h
+++ b/vpp-api/python/pneum/pneum.h
@@ -15,9 +15,15 @@
#ifndef included_pneum_h
#define included_pneum_h
-int pneum_connect(char * name, char * chroot_prefix);
+#include <vppinfra/types.h>
+
+typedef void (*pneum_callback_t)(unsigned char * data, int len);
+int pneum_connect(char * name, char * chroot_prefix, pneum_callback_t cb);
int pneum_disconnect(void);
int pneum_read(char **data, int *l);
int pneum_write(char *data, int len);
+void pneum_free(void * msg);
+uword * pneum_msg_table_get_hash (void);
+int pneum_msg_table_size(void);
#endif
diff --git a/vpp-api/python/pneum/test_pneum.c b/vpp-api/python/pneum/test_pneum.c
index 20c29a7..4910719 100644
--- a/vpp-api/python/pneum/test_pneum.c
+++ b/vpp-api/python/pneum/test_pneum.c
@@ -76,7 +76,7 @@
vl_api_show_version_t message;
vl_api_show_version_t *mp;
int async = 1;
- int rv = pneum_connect("pneum_client", NULL);
+ int rv = pneum_connect("pneum_client", NULL, NULL);
if (rv != 0) {
printf("Connect failed: %d\n", rv);
diff --git a/vpp-api/python/vpp_papi/pneum_wrap.c b/vpp-api/python/vpp_papi/pneum_wrap.c
index 18c4f23..5763707 100644
--- a/vpp-api/python/vpp_papi/pneum_wrap.c
+++ b/vpp-api/python/vpp_papi/pneum_wrap.c
@@ -15,11 +15,12 @@
#include <Python.h>
#include "../pneum/pneum.h"
+#include <vppinfra/hash.h>
static PyObject *pneum_callback = NULL;
-int
-wrap_pneum_callback (char *data, int len)
+static void
+wrap_pneum_callback (unsigned char * data, int len)
{
PyGILState_STATE gstate;
PyObject *result;//, *arglist;
@@ -38,7 +39,6 @@
PyErr_Print();
PyGILState_Release(gstate);
- return (0);
}
static PyObject *
@@ -46,22 +46,28 @@
{
char * name, * chroot_prefix = NULL;
int rv;
- PyObject * temp;
+ PyObject * temp = NULL;
+ pneum_callback_t cb = NULL;
- if (!PyArg_ParseTuple(args, "sO|s:wrap_connect", &name, &temp, &chroot_prefix))
+ if (!PyArg_ParseTuple(args, "s|Os:wrap_connect",
+ &name, &temp, &chroot_prefix))
return (NULL);
- if (!PyCallable_Check(temp)) {
- PyErr_SetString(PyExc_TypeError, "parameter must be callable");
- return NULL;
- }
+ if (temp)
+ {
+ if (!PyCallable_Check(temp))
+ {
+ PyErr_SetString(PyExc_TypeError, "parameter must be callable");
+ return NULL;
+ }
- Py_XINCREF(temp); /* Add a reference to new callback */
- Py_XDECREF(pneum_callback); /* Dispose of previous callback */
- pneum_callback = temp; /* Remember new callback */
-
+ Py_XINCREF(temp); /* Add a reference to new callback */
+ Py_XDECREF(pneum_callback); /* Dispose of previous callback */
+ pneum_callback = temp; /* Remember new callback */
+ cb = wrap_pneum_callback;
+ }
Py_BEGIN_ALLOW_THREADS
- rv = pneum_connect(name, chroot_prefix);
+ rv = pneum_connect(name, chroot_prefix, cb);
Py_END_ALLOW_THREADS
return PyLong_FromLong(rv);
}
@@ -90,8 +96,6 @@
return PyLong_FromLong(rv);
}
-void vl_msg_api_free(void *);
-
static PyObject *
wrap_read (PyObject *self, PyObject *args)
{
@@ -110,15 +114,44 @@
#endif
if (!ret) { Py_RETURN_NONE; }
- vl_msg_api_free(data);
+ pneum_free(data);
return ret;
}
+static PyObject *
+wrap_msg_table (PyObject *self, PyObject *args)
+{
+ int i = 0, rv = 0;
+ hash_pair_t *hp;
+ uword *h = pneum_msg_table_get_hash();
+ PyObject *ret = PyList_New(pneum_msg_table_size());
+ if (!ret) goto error;
+ hash_foreach_pair (hp, h,
+ ({
+ PyObject *item = PyTuple_New(2);
+ if (!item) goto error;
+ rv = PyTuple_SetItem(item, 0, PyLong_FromLong((u32)hp->value[0]));
+ if (rv) goto error;
+ rv = PyTuple_SetItem(item, 1, PyString_FromString((char *)hp->key));
+ if (rv) goto error;
+ PyList_SetItem(ret, i, item);
+ i++;
+ }));
+
+ return ret;
+
+ error:
+ /* TODO: Raise exception */
+ printf("msg_table failed");
+ Py_RETURN_NONE;
+}
+
static PyMethodDef vpp_api_Methods[] = {
{"connect", wrap_connect, METH_VARARGS, "Connect to the VPP API."},
{"disconnect", wrap_disconnect, METH_VARARGS, "Disconnect from the VPP API."},
{"write", wrap_write, METH_VARARGS, "Write data to the VPP API."},
{"read", wrap_read, METH_VARARGS, "Read data from the VPP API."},
+ {"msg_table", wrap_msg_table, METH_VARARGS, "Get API dictionary."},
{NULL, NULL, 0, NULL} /* Sentinel */
};