stats: support multiple works for error counters
The current code only allowed access to the main thread error counters.
That is not so useful for a multi worker instance.
No return a vector indexed by thread of counter_t values.
Type: fix
Change-Id: Ie322c8889c0c8175e1116e71de04a2cf453b9ed7
Signed-off-by: Ole Troan <ot@cisco.com>
diff --git a/src/vlib/counter.c b/src/vlib/counter.c
index 5b73d5f..faf1069 100644
--- a/src/vlib/counter.c
+++ b/src/vlib/counter.c
@@ -38,6 +38,7 @@
*/
#include <vlib/vlib.h>
+#include <vlib/stat_weak_inlines.h>
void
vlib_clear_simple_counters (vlib_simple_counter_main_t * cm)
@@ -74,19 +75,6 @@
}
}
-void *vlib_stats_push_heap (void *) __attribute__ ((weak));
-void *
-vlib_stats_push_heap (void *unused)
-{
- return 0;
-};
-
-void vlib_stats_pop_heap (void *, void *, u32, int) __attribute__ ((weak));
-void
-vlib_stats_pop_heap (void *notused, void *notused2, u32 i, int type)
-{
-};
-
void
vlib_validate_simple_counter (vlib_simple_counter_main_t * cm, u32 index)
{
diff --git a/src/vlib/error.c b/src/vlib/error.c
index a416649..7c61f31 100644
--- a/src/vlib/error.c
+++ b/src/vlib/error.c
@@ -39,6 +39,7 @@
#include <vlib/vlib.h>
#include <vppinfra/heap.h>
+#include <vlib/stat_weak_inlines.h>
uword
vlib_error_drop_buffers (vlib_main_t * vm,
@@ -109,20 +110,6 @@
return n_buffers;
}
-void vlib_stats_register_error_index (u8 *, u64 *, u64)
- __attribute__ ((weak));
-void
-vlib_stats_register_error_index (u8 * notused, u64 * notused2, u64 notused3)
-{
-};
-
-void vlib_stats_pop_heap2 (void *, u32, void *) __attribute__ ((weak));
-void
-vlib_stats_pop_heap2 (void *notused, u32 notused2, void *notused3)
-{
-};
-
-
/* Reserves given number of error codes for given node. */
void
vlib_register_errors (vlib_main_t * vm,
@@ -132,7 +119,6 @@
vlib_node_t *n = vlib_get_node (vm, node_index);
uword l;
void *oldheap;
- void *vlib_stats_push_heap (void) __attribute__ ((weak));
ASSERT (vlib_get_thread_index () == 0);
@@ -157,7 +143,7 @@
vec_validate (vm->error_elog_event_types, l - 1);
/* Switch to the stats segment ... */
- oldheap = vlib_stats_push_heap ();
+ oldheap = vlib_stats_push_heap (0);
/* Allocate a counter/elog type for each error. */
vec_validate (em->counters, l - 1);
@@ -186,7 +172,7 @@
}
/* (re)register the em->counters base address, switch back to main heap */
- vlib_stats_pop_heap2 (em->counters, vm->thread_index, oldheap);
+ vlib_stats_pop_heap2 (em->counters, vm->thread_index, oldheap, 1);
{
elog_event_type_t t;
diff --git a/src/vlib/stat_weak_inlines.h b/src/vlib/stat_weak_inlines.h
new file mode 100644
index 0000000..5c04610
--- /dev/null
+++ b/src/vlib/stat_weak_inlines.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2019 Cisco and/or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * NOTE: Only include this file from external components that require
+ * a loose coupling to the stats component.
+ */
+
+#ifndef included_stat_weak_inlines_h
+#define included_stat_weak_inlines_h
+void *vlib_stats_push_heap (void *) __attribute__ ((weak));
+void *
+vlib_stats_push_heap (void *unused)
+{
+ return 0;
+};
+
+void vlib_stats_pop_heap (void *, void *, u32, int) __attribute__ ((weak));
+void
+vlib_stats_pop_heap (void *notused, void *notused2, u32 i, int type)
+{
+};
+void vlib_stats_register_error_index (u8 *, u64 *, u64)
+ __attribute__ ((weak));
+void
+vlib_stats_register_error_index (u8 * notused, u64 * notused2, u64 notused3)
+{
+};
+
+void vlib_stats_pop_heap2 (void *, u32, void *, int) __attribute__ ((weak));
+void
+vlib_stats_pop_heap2 (void *notused, u32 notused2, void *notused3,
+ int notused4)
+{
+};
+
+void vlib_stat_segment_lock (void) __attribute__ ((weak));
+void
+vlib_stat_segment_lock (void)
+{
+}
+
+void vlib_stat_segment_unlock (void) __attribute__ ((weak));
+void
+vlib_stat_segment_unlock (void)
+{
+}
+
+#endif
diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 6a23bfd..22fa5f1 100644
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -23,6 +23,8 @@
#include <vlib/threads.h>
#include <vlib/unix/cj.h>
+#include <vlib/stat_weak_inlines.h>
+
DECLARE_CJ_GLOBAL_LOG;
#define FRAME_QUEUE_NELTS 64
@@ -872,8 +874,13 @@
clib_mem_set_heap (oldheap);
vec_add1_aligned (vlib_mains, vm_clone, CLIB_CACHE_LINE_BYTES);
+ /* Switch to the stats segment ... */
+ void *oldheap = vlib_stats_push_heap (0);
vm_clone->error_main.counters = vec_dup_aligned
(vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES);
+ vlib_stats_pop_heap2 (vm_clone->error_main.counters,
+ worker_thread_index, oldheap, 1);
+
vm_clone->error_main.counters_last_clear = vec_dup_aligned
(vlib_mains[0]->error_main.counters_last_clear,
CLIB_CACHE_LINE_BYTES);
@@ -1036,9 +1043,15 @@
clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
sizeof (vm->error_main));
j = vec_len (vm->error_main.counters) - 1;
+
+ /* Switch to the stats segment ... */
+ void *oldheap = vlib_stats_push_heap (0);
vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES);
- vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
vm_clone->error_main.counters = old_counters;
+ vlib_stats_pop_heap2 (vm_clone->error_main.counters, vm_clone->thread_index,
+ oldheap, 0);
+
+ vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
vm_clone->error_main.counters_last_clear = old_counters_all_clear;
nm_clone = &vm_clone->node_main;
@@ -1466,18 +1479,6 @@
}
-void vlib_stat_segment_lock (void) __attribute__ ((weak));
-void
-vlib_stat_segment_lock (void)
-{
-}
-
-void vlib_stat_segment_unlock (void) __attribute__ ((weak));
-void
-vlib_stat_segment_unlock (void)
-{
-}
-
void
vlib_worker_thread_barrier_release (vlib_main_t * vm)
{
diff --git a/src/vnet/interface_api.c b/src/vnet/interface_api.c
index 1b91382..a91936f 100644
--- a/src/vnet/interface_api.c
+++ b/src/vnet/interface_api.c
@@ -392,18 +392,6 @@
REPLY_MACRO (VL_API_SW_INTERFACE_ADD_DEL_ADDRESS_REPLY);
}
-void stats_dslock_with_hint (int hint, int tag) __attribute__ ((weak));
-void
-stats_dslock_with_hint (int hint, int tag)
-{
-}
-
-void stats_dsunlock (void) __attribute__ ((weak));
-void
-stats_dsunlock (void)
-{
-}
-
static void
vl_api_sw_interface_set_table_t_handler (vl_api_sw_interface_set_table_t * mp)
{
@@ -414,15 +402,11 @@
VALIDATE_SW_IF_INDEX (mp);
- stats_dslock_with_hint (1 /* release hint */ , 4 /* tag */ );
-
if (mp->is_ipv6)
rv = ip_table_bind (FIB_PROTOCOL_IP6, sw_if_index, table_id, 1);
else
rv = ip_table_bind (FIB_PROTOCOL_IP4, sw_if_index, table_id, 1);
- stats_dsunlock ();
-
BAD_SW_IF_INDEX_LABEL;
REPLY_MACRO (VL_API_SW_INTERFACE_SET_TABLE_REPLY);
diff --git a/src/vnet/ip/ip_api.c b/src/vnet/ip/ip_api.c
index 5a6053d..f4e5d17 100644
--- a/src/vnet/ip/ip_api.c
+++ b/src/vnet/ip/ip_api.c
@@ -119,9 +119,6 @@
_(IP_PUNT_REDIRECT_DUMP, ip_punt_redirect_dump)
-extern void stats_dslock_with_hint (int hint, int tag);
-extern void stats_dsunlock (void);
-
static vl_api_ip_neighbor_flags_t
ip_neighbor_flags_encode (ip_neighbor_flags_t f)
{
@@ -704,8 +701,6 @@
VALIDATE_SW_IF_INDEX ((&mp->neighbor));
- stats_dslock_with_hint (1 /* release hint */ , 7 /* tag */ );
-
flags = ip_neighbor_flags_decode (mp->neighbor.flags);
type = ip_address_decode (&mp->neighbor.ip_address, &ip);
mac_address_decode (mp->neighbor.mac_address, &mac);
@@ -722,8 +717,6 @@
else
rv = ip_neighbor_del (&ip, type, ntohl (mp->neighbor.sw_if_index));
- stats_dsunlock ();
-
BAD_SW_IF_INDEX_LABEL;
/* *INDENT-OFF* */
@@ -879,8 +872,6 @@
path.frp_flags = path_flags;
- stats_dslock_with_hint (1 /* release hint */ , 2 /* tag */ );
-
if (is_drop || (is_local && (~0 == next_hop_sw_if_index)) ||
is_classify || is_unreach || is_prohibit)
{
@@ -910,7 +901,6 @@
if (pool_is_free_index (cm->tables,
ntohl (classify_table_index)))
{
- stats_dsunlock ();
return VNET_API_ERROR_NO_SUCH_TABLE;
}
@@ -920,7 +910,6 @@
}
else
{
- stats_dsunlock ();
return VNET_API_ERROR_NO_SUCH_TABLE;
}
@@ -964,7 +953,6 @@
}
}
- stats_dsunlock ();
return (0);
}
@@ -1271,8 +1259,6 @@
{
fib_node_index_t mfib_entry_index = ~0;
- stats_dslock_with_hint (1 /* release hint */ , 2 /* tag */ );
-
fib_route_path_t path = {
.frp_sw_if_index = next_hop_sw_if_index,
.frp_proto = nh_proto,
@@ -1308,7 +1294,6 @@
}
done:
- stats_dsunlock ();
return (mfib_entry_index);
}
@@ -2969,8 +2954,6 @@
u32 fib_index;
int rv;
- stats_dslock_with_hint (1 /* release hint */ , 6 /* tag */ );
-
fib_index = fib_table_find (FIB_PROTOCOL_IP4, ntohl (mp->proxy.table_id));
if (~0 == fib_index)
@@ -2985,7 +2968,6 @@
rv = vnet_proxy_arp_add_del (&lo, &hi, fib_index, mp->is_add == 0);
out:
- stats_dsunlock ();
REPLY_MACRO (VL_API_PROXY_ARP_ADD_DEL_REPLY);
}
@@ -3166,8 +3148,6 @@
int rv = VNET_API_ERROR_NO_SUCH_FIB;
u32 target_fib_id = ntohl (mp->vrf_id);
- stats_dslock_with_hint (1 /* release hint */ , 8 /* tag */ );
-
/* *INDENT-OFF* */
pool_foreach (fib_table, im4->fibs,
({
@@ -3215,7 +3195,6 @@
})); /* pool_foreach (fib) */
/* *INDENT-ON* */
- stats_dsunlock ();
return rv;
}
@@ -3233,8 +3212,6 @@
int rv = VNET_API_ERROR_NO_SUCH_FIB;
u32 target_fib_id = ntohl (mp->vrf_id);
- stats_dslock_with_hint (1 /* release hint */ , 9 /* tag */ );
-
/* *INDENT-OFF* */
pool_foreach (fib_table, im6->fibs,
({
@@ -3273,7 +3250,6 @@
})); /* pool_foreach (fib) */
/* *INDENT-ON* */
- stats_dsunlock ();
return rv;
}
diff --git a/src/vnet/mpls/mpls_api.c b/src/vnet/mpls/mpls_api.c
index 440dc9c..52434da 100644
--- a/src/vnet/mpls/mpls_api.c
+++ b/src/vnet/mpls/mpls_api.c
@@ -56,9 +56,6 @@
_(SW_INTERFACE_SET_MPLS_ENABLE, sw_interface_set_mpls_enable) \
_(MPLS_FIB_DUMP, mpls_fib_dump)
-extern void stats_dslock_with_hint (int hint, int tag);
-extern void stats_dsunlock (void);
-
void
mpls_table_delete (u32 table_id, u8 is_api)
{
@@ -318,8 +315,6 @@
clib_memset (&rpath, 0, sizeof (rpath));
- stats_dslock_with_hint (1 /* release hint */ , 5 /* tag */ );
-
if (mp->mt_next_hop_proto_is_ip4)
{
rpath.frp_proto = DPO_PROTO_IP4;
@@ -394,8 +389,6 @@
vec_free (rpaths);
- stats_dsunlock ();
-
out:
/* *INDENT-OFF* */
REPLY_MACRO2(VL_API_MPLS_TUNNEL_ADD_DEL_REPLY,
diff --git a/src/vnet/pg/pg_api.c b/src/vnet/pg/pg_api.c
index 49b661e..7775343 100644
--- a/src/vnet/pg/pg_api.c
+++ b/src/vnet/pg/pg_api.c
@@ -46,9 +46,6 @@
_(PG_CAPTURE, pg_capture) \
_(PG_ENABLE_DISABLE, pg_enable_disable)
-extern void stats_dslock_with_hint (int hint, int tag);
-extern void stats_dsunlock (void);
-
static void
vl_api_pg_create_interface_t_handler (vl_api_pg_create_interface_t * mp)
{
diff --git a/src/vpp-api/client/stat_client.c b/src/vpp-api/client/stat_client.c
index a386eec..8991806 100644
--- a/src/vpp-api/client/stat_client.c
+++ b/src/vpp-api/client/stat_client.c
@@ -231,7 +231,6 @@
int i;
vlib_counter_t **combined_c; /* Combined counter */
counter_t **simple_c; /* Simple counter */
- counter_t *error_base;
uint64_t *offset_vector;
assert (sm->shared_header);
@@ -275,10 +274,16 @@
break;
case STAT_DIR_TYPE_ERROR_INDEX:
- error_base =
- stat_segment_pointer (sm->shared_header,
- sm->shared_header->error_offset);
- result.error_value = error_base[ep->index];
+ /* Gather errors from all threads into a vector */
+ offset_vector = stat_segment_pointer (sm->shared_header,
+ sm->shared_header->error_offset);
+ vec_validate (result.error_vector, vec_len (offset_vector) - 1);
+ for (i = 0; i < vec_len (offset_vector); i++)
+ {
+ counter_t *cb =
+ stat_segment_pointer (sm->shared_header, offset_vector[i]);
+ result.error_vector[i] = cb[ep->index];
+ }
break;
case STAT_DIR_TYPE_NAME_VECTOR:
diff --git a/src/vpp-api/client/stat_client.h b/src/vpp-api/client/stat_client.h
index 1c76a93..901ec32 100644
--- a/src/vpp-api/client/stat_client.h
+++ b/src/vpp-api/client/stat_client.h
@@ -18,7 +18,7 @@
#define included_stat_client_h
#define STAT_VERSION_MAJOR 1
-#define STAT_VERSION_MINOR 1
+#define STAT_VERSION_MINOR 2
#include <stdint.h>
#include <unistd.h>
@@ -46,7 +46,7 @@
union
{
double scalar_value;
- uint64_t error_value;
+ counter_t *error_vector;
counter_t **simple_counter_vec;
vlib_counter_t **combined_counter_vec;
uint8_t **name_vector;
diff --git a/src/vpp-api/python/vpp_papi/vpp_stats.py b/src/vpp-api/python/vpp_papi/vpp_stats.py
index c90aa34..86a80dd 100644
--- a/src/vpp-api/python/vpp_papi/vpp_stats.py
+++ b/src/vpp-api/python/vpp_papi/vpp_stats.py
@@ -40,7 +40,7 @@
union
{
double scalar_value;
- uint64_t error_value;
+ counter_t *error_vector;
counter_t **simple_counter_vec;
vlib_counter_t **combined_counter_vec;
uint8_t **name_vector;
@@ -126,6 +126,12 @@
vec.append(if_per_thread)
return vec
+def error_vec_list(api, e):
+ vec = []
+ for thread in range(api.stat_segment_vec_len(e)):
+ vec.append(e[thread])
+ return vec
+
def name_vec_list(api, e):
return [ffi.string(e[i]).decode('utf-8') for i in range(api.stat_segment_vec_len(e)) if e[i] != ffi.NULL]
@@ -138,7 +144,7 @@
if e.type == 3:
return combined_counter_vec_list(api, e.combined_counter_vec)
if e.type == 4:
- return e.error_value
+ return error_vec_list(api, e.error_vector)
if e.type == 5:
return name_vec_list(api, e.name_vector)
raise NotImplementedError()
@@ -248,6 +254,11 @@
return None
retries += 1
+ def get_err_counter(self, name):
+ """Get an error counter. The errors from each worker thread
+ are summed"""
+ return sum(self.get_counter(name))
+
def disconnect(self):
self.api.stat_segment_disconnect_r(self.client)
self.api.stat_client_free(self.client)
@@ -265,8 +276,8 @@
return None
retries += 1
- return {k: error_counters[k]
- for k in error_counters.keys() if error_counters[k]}
+ return {k: sum(error_counters[k])
+ for k in error_counters.keys() if sum(error_counters[k])}
def set_errors_str(self):
'''Return all errors counters > 0 pretty printed'''
diff --git a/src/vpp/app/vpp_get_stats.c b/src/vpp/app/vpp_get_stats.c
index 1d87802..c00fb83 100644
--- a/src/vpp/app/vpp_get_stats.c
+++ b/src/vpp/app/vpp_get_stats.c
@@ -80,7 +80,9 @@
break;
case STAT_DIR_TYPE_ERROR_INDEX:
- fformat (stdout, "%llu %s\n", res[i].error_value, res[i].name);
+ for (j = 0; j < vec_len (res[i].error_vector); j++)
+ fformat (stdout, "%llu %s\n", res[i].error_vector[j],
+ res[i].name);
break;
case STAT_DIR_TYPE_SCALAR_INDEX:
@@ -213,7 +215,9 @@
break;
case STAT_DIR_TYPE_ERROR_INDEX:
- fformat (stdout, "%llu %s\n", res[i].error_value, res[i].name);
+ for (j = 0; j < vec_len (res[i].error_vector); j++)
+ fformat (stdout, "[@%d] %llu %s\n", j, res[i].error_vector[j],
+ res[i].name);
break;
case STAT_DIR_TYPE_SCALAR_INDEX:
diff --git a/src/vpp/app/vpp_prometheus_export.c b/src/vpp/app/vpp_prometheus_export.c
index e2fdd71..06f1a91 100644
--- a/src/vpp/app/vpp_prometheus_export.c
+++ b/src/vpp/app/vpp_prometheus_export.c
@@ -97,9 +97,13 @@
}
break;
case STAT_DIR_TYPE_ERROR_INDEX:
- fformat (stream, "# TYPE %s counter\n", prom_string (res[i].name));
- fformat (stream, "%s{thread=\"0\"} %lld\n",
- prom_string (res[i].name), res[i].error_value);
+ for (j = 0; j < vec_len (res[i].error_vector); j++)
+ {
+ fformat (stream, "# TYPE %s counter\n",
+ prom_string (res[i].name));
+ fformat (stream, "%s{thread=\"%d\"} %lld\n",
+ prom_string (res[i].name), j, res[i].error_vector[j]);
+ }
break;
case STAT_DIR_TYPE_SCALAR_INDEX:
diff --git a/src/vpp/stats/stat_segment.c b/src/vpp/stats/stat_segment.c
index bb2ffad..4cd00a2 100644
--- a/src/vpp/stats/stat_segment.c
+++ b/src/vpp/stats/stat_segment.c
@@ -22,6 +22,7 @@
#undef HAVE_MEMFD_CREATE
#include <vppinfra/linux/syscall.h>
#include <vpp-api/client/stat_client.h>
+
stat_segment_main_t stat_segment_main;
/*
@@ -200,22 +201,29 @@
}
void
-vlib_stats_pop_heap2 (u64 * error_vector, u32 thread_index, void *oldheap)
+vlib_stats_pop_heap2 (u64 * error_vector, u32 thread_index, void *oldheap,
+ int lock)
{
stat_segment_main_t *sm = &stat_segment_main;
stat_segment_shared_header_t *shared_header = sm->shared_header;
ASSERT (shared_header);
- vlib_stat_segment_lock ();
+ if (lock)
+ vlib_stat_segment_lock ();
/* Reset the client hash table pointer, since it WILL change! */
- shared_header->error_offset =
+ vec_validate (sm->error_vector, thread_index);
+ sm->error_vector[thread_index] =
stat_segment_offset (shared_header, error_vector);
+
+ shared_header->error_offset =
+ stat_segment_offset (shared_header, sm->error_vector);
shared_header->directory_offset =
stat_segment_offset (shared_header, sm->directory_vector);
- vlib_stat_segment_unlock ();
+ if (lock)
+ vlib_stat_segment_unlock ();
clib_mem_set_heap (oldheap);
}
diff --git a/src/vpp/stats/stat_segment.h b/src/vpp/stats/stat_segment.h
index fd7ce79..b9ffedf 100644
--- a/src/vpp/stats/stat_segment.h
+++ b/src/vpp/stats/stat_segment.h
@@ -110,6 +110,7 @@
/* statistics segment */
uword *directory_vector_by_name;
stat_segment_directory_entry_t *directory_vector;
+ u64 *error_vector;
u8 **interfaces;
u8 **nodes;
diff --git a/src/vpp/stats/stats.md b/src/vpp/stats/stats.md
index 6a62ca6..20ca790 100644
--- a/src/vpp/stats/stats.md
+++ b/src/vpp/stats/stats.md
@@ -106,7 +106,8 @@
break;
case STAT_DIR_TYPE_ERROR_INDEX:
- fformat (stdout, "%llu %s\n", res[i].error_value, res[i].name);
+ for (j = 0; j < vec_len (res[i].error_vector); j++)
+ fformat (stdout, "[@%d] %llu %s\n", j, res[i].error_vector[j], res[i].name);
break;
case STAT_DIR_TYPE_SCALAR_INDEX: