blob: 30e5c5053640ee6db82968874370b6e051e75714 [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
Damjan Marione936bbe2016-02-25 23:17:38 +010015
16#include <stdint.h>
17
Ed Warnickecb9cada2015-12-08 15:45:58 -070018#include <vlib/vlib.h>
19#include <vnet/vnet.h>
20#include <vnet/policer/policer.h>
Neale Rannsd91c1db2017-07-31 02:30:50 -070021#include <vnet/policer/police_inlines.h>
Matus Fabian4ac74c92016-05-31 07:33:29 -070022#include <vnet/ip/ip.h>
Matus Fabian70e6a8d2016-06-20 08:10:42 -070023#include <vnet/classify/policer_classify.h>
24#include <vnet/classify/vnet_classify.h>
Neale Ranns3b81a1e2018-09-06 09:50:26 -070025#include <vnet/l2/feat_bitmap.h>
26#include <vnet/l2/l2_input.h>
Matus Fabian4ac74c92016-05-31 07:33:29 -070027
Ed Warnickecb9cada2015-12-08 15:45:58 -070028
29/* Dispatch functions meant to be instantiated elsewhere */
30
Damjan Marion3891cd82016-10-27 10:27:00 +020031typedef struct
32{
Ed Warnickecb9cada2015-12-08 15:45:58 -070033 u32 next_index;
34 u32 sw_if_index;
35 u32 policer_index;
36} vnet_policer_trace_t;
37
38/* packet trace format function */
Damjan Marion3891cd82016-10-27 10:27:00 +020039static u8 *
40format_policer_trace (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -070041{
42 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
43 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
Damjan Marion3891cd82016-10-27 10:27:00 +020044 vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
45
Ed Warnickecb9cada2015-12-08 15:45:58 -070046 s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
Damjan Marion3891cd82016-10-27 10:27:00 +020047 t->sw_if_index, t->policer_index, t->next_index);
Ed Warnickecb9cada2015-12-08 15:45:58 -070048 return s;
49}
50
51#define foreach_vnet_policer_error \
52_(TRANSMIT, "Packets Transmitted") \
53_(DROP, "Packets Dropped")
54
Damjan Marion3891cd82016-10-27 10:27:00 +020055typedef enum
56{
Ed Warnickecb9cada2015-12-08 15:45:58 -070057#define _(sym,str) VNET_POLICER_ERROR_##sym,
58 foreach_vnet_policer_error
59#undef _
Damjan Marion3891cd82016-10-27 10:27:00 +020060 VNET_POLICER_N_ERROR,
Ed Warnickecb9cada2015-12-08 15:45:58 -070061} vnet_policer_error_t;
62
Damjan Marion3891cd82016-10-27 10:27:00 +020063static char *vnet_policer_error_strings[] = {
Ed Warnickecb9cada2015-12-08 15:45:58 -070064#define _(sym,string) string,
65 foreach_vnet_policer_error
66#undef _
67};
68
Damjan Marion3891cd82016-10-27 10:27:00 +020069static inline uword
Brian Russell15c18e32021-02-17 15:45:56 +000070vnet_policer_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
71 vlib_frame_t *frame)
Ed Warnickecb9cada2015-12-08 15:45:58 -070072{
Damjan Marion3891cd82016-10-27 10:27:00 +020073 u32 n_left_from, *from, *to_next;
Ed Warnickecb9cada2015-12-08 15:45:58 -070074 vnet_policer_next_t next_index;
Damjan Marion3891cd82016-10-27 10:27:00 +020075 vnet_policer_main_t *pm = &vnet_policer_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -070076 u64 time_in_policer_periods;
77 u32 transmitted = 0;
78
Damjan Marion3891cd82016-10-27 10:27:00 +020079 time_in_policer_periods =
80 clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
Ed Warnickecb9cada2015-12-08 15:45:58 -070081
82 from = vlib_frame_vector_args (frame);
83 n_left_from = frame->n_vectors;
84 next_index = node->cached_next_index;
85
86 while (n_left_from > 0)
87 {
88 u32 n_left_to_next;
89
Damjan Marion3891cd82016-10-27 10:27:00 +020090 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
Ed Warnickecb9cada2015-12-08 15:45:58 -070091
92 while (n_left_from >= 4 && n_left_to_next >= 2)
93 {
Damjan Marion3891cd82016-10-27 10:27:00 +020094 u32 bi0, bi1;
95 vlib_buffer_t *b0, *b1;
96 u32 next0, next1;
97 u32 sw_if_index0, sw_if_index1;
98 u32 pi0 = 0, pi1 = 0;
99 u8 act0, act1;
100
Ed Warnickecb9cada2015-12-08 15:45:58 -0700101 /* Prefetch next iteration. */
102 {
Damjan Marion3891cd82016-10-27 10:27:00 +0200103 vlib_buffer_t *b2, *b3;
104
Ed Warnickecb9cada2015-12-08 15:45:58 -0700105 b2 = vlib_get_buffer (vm, from[2]);
106 b3 = vlib_get_buffer (vm, from[3]);
Damjan Marion3891cd82016-10-27 10:27:00 +0200107
Ed Warnickecb9cada2015-12-08 15:45:58 -0700108 vlib_prefetch_buffer_header (b2, LOAD);
109 vlib_prefetch_buffer_header (b3, LOAD);
110 }
111
Damjan Marion3891cd82016-10-27 10:27:00 +0200112 /* speculatively enqueue b0 and b1 to the current next frame */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700113 to_next[0] = bi0 = from[0];
114 to_next[1] = bi1 = from[1];
115 from += 2;
116 to_next += 2;
117 n_left_from -= 2;
118 n_left_to_next -= 2;
119
120 b0 = vlib_get_buffer (vm, bi0);
121 b1 = vlib_get_buffer (vm, bi1);
122
Damjan Marion3891cd82016-10-27 10:27:00 +0200123 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
Damjan Marion3891cd82016-10-27 10:27:00 +0200124 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700125
Brian Russell15c18e32021-02-17 15:45:56 +0000126 pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
127 pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700128
Damjan Marion3891cd82016-10-27 10:27:00 +0200129 act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
130 POLICE_CONFORM /* no chaining */ );
Ed Warnickecb9cada2015-12-08 15:45:58 -0700131
Damjan Marion3891cd82016-10-27 10:27:00 +0200132 act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
133 POLICE_CONFORM /* no chaining */ );
Matus Fabian4ac74c92016-05-31 07:33:29 -0700134
Brian Russellc5299ff2021-02-09 10:16:58 +0000135 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP)) /* drop action */
Damjan Marion3891cd82016-10-27 10:27:00 +0200136 {
137 next0 = VNET_POLICER_NEXT_DROP;
138 b0->error = node->errors[VNET_POLICER_ERROR_DROP];
139 }
140 else /* transmit or mark-and-transmit action */
141 {
142 transmitted++;
Brian Russell15c18e32021-02-17 15:45:56 +0000143 vnet_feature_next (&next0, b0);
Damjan Marion3891cd82016-10-27 10:27:00 +0200144 }
Matus Fabian4ac74c92016-05-31 07:33:29 -0700145
Brian Russellc5299ff2021-02-09 10:16:58 +0000146 if (PREDICT_FALSE (act1 == QOS_ACTION_DROP)) /* drop action */
Damjan Marion3891cd82016-10-27 10:27:00 +0200147 {
148 next1 = VNET_POLICER_NEXT_DROP;
149 b1->error = node->errors[VNET_POLICER_ERROR_DROP];
150 }
151 else /* transmit or mark-and-transmit action */
152 {
153 transmitted++;
Brian Russell15c18e32021-02-17 15:45:56 +0000154 vnet_feature_next (&next1, b1);
Damjan Marion3891cd82016-10-27 10:27:00 +0200155 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700156
Damjan Marion3891cd82016-10-27 10:27:00 +0200157 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
158 {
159 if (b0->flags & VLIB_BUFFER_IS_TRACED)
160 {
161 vnet_policer_trace_t *t =
162 vlib_add_trace (vm, node, b0, sizeof (*t));
163 t->sw_if_index = sw_if_index0;
164 t->next_index = next0;
165 }
166 if (b1->flags & VLIB_BUFFER_IS_TRACED)
167 {
168 vnet_policer_trace_t *t =
169 vlib_add_trace (vm, node, b1, sizeof (*t));
170 t->sw_if_index = sw_if_index1;
171 t->next_index = next1;
172 }
173 }
174
175 /* verify speculative enqueues, maybe switch current next frame */
176 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
177 to_next, n_left_to_next,
178 bi0, bi1, next0, next1);
179 }
180
Ed Warnickecb9cada2015-12-08 15:45:58 -0700181 while (n_left_from > 0 && n_left_to_next > 0)
182 {
Damjan Marion3891cd82016-10-27 10:27:00 +0200183 u32 bi0;
184 vlib_buffer_t *b0;
185 u32 next0;
186 u32 sw_if_index0;
187 u32 pi0 = 0;
188 u8 act0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700189
190 bi0 = from[0];
191 to_next[0] = bi0;
192 from += 1;
193 to_next += 1;
194 n_left_from -= 1;
195 n_left_to_next -= 1;
196
197 b0 = vlib_get_buffer (vm, bi0);
198
Damjan Marion3891cd82016-10-27 10:27:00 +0200199 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700200
Brian Russell15c18e32021-02-17 15:45:56 +0000201 pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700202
Damjan Marion3891cd82016-10-27 10:27:00 +0200203 act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
204 POLICE_CONFORM /* no chaining */ );
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700205
Brian Russellc5299ff2021-02-09 10:16:58 +0000206 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP)) /* drop action */
Damjan Marion3891cd82016-10-27 10:27:00 +0200207 {
208 next0 = VNET_POLICER_NEXT_DROP;
209 b0->error = node->errors[VNET_POLICER_ERROR_DROP];
210 }
211 else /* transmit or mark-and-transmit action */
212 {
213 transmitted++;
Brian Russell15c18e32021-02-17 15:45:56 +0000214 vnet_feature_next (&next0, b0);
Damjan Marion3891cd82016-10-27 10:27:00 +0200215 }
216
217 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
218 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
219 {
220 vnet_policer_trace_t *t =
221 vlib_add_trace (vm, node, b0, sizeof (*t));
222 t->sw_if_index = sw_if_index0;
223 t->next_index = next0;
224 t->policer_index = pi0;
225 }
226
227 /* verify speculative enqueue, maybe switch current next frame */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700228 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
229 to_next, n_left_to_next,
230 bi0, next0);
231 }
232
233 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
234 }
235
Damjan Marion3891cd82016-10-27 10:27:00 +0200236 vlib_node_increment_counter (vm, node->node_index,
237 VNET_POLICER_ERROR_TRANSMIT, transmitted);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700238 return frame->n_vectors;
239}
240
Brian Russell15c18e32021-02-17 15:45:56 +0000241VLIB_NODE_FN (policer_input_node)
242(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700243{
Brian Russell15c18e32021-02-17 15:45:56 +0000244 return vnet_policer_inline (vm, node, frame);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700245}
246
Brian Russell15c18e32021-02-17 15:45:56 +0000247VLIB_REGISTER_NODE (policer_input_node) = {
248 .name = "policer-input",
Ed Warnickecb9cada2015-12-08 15:45:58 -0700249 .vector_size = sizeof (u32),
250 .format_trace = format_policer_trace,
251 .type = VLIB_NODE_TYPE_INTERNAL,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700252 .n_errors = ARRAY_LEN(vnet_policer_error_strings),
253 .error_strings = vnet_policer_error_strings,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700254 .n_next_nodes = VNET_POLICER_N_NEXT,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700255 .next_nodes = {
Brian Russell15c18e32021-02-17 15:45:56 +0000256 [VNET_POLICER_NEXT_DROP] = "error-drop",
257 },
Ed Warnickecb9cada2015-12-08 15:45:58 -0700258};
Damjan Marion1c80e832016-05-11 23:07:18 +0200259
Brian Russell15c18e32021-02-17 15:45:56 +0000260VNET_FEATURE_INIT (policer_input_node, static) = {
261 .arc_name = "device-input",
262 .node_name = "policer-input",
263 .runs_before = VNET_FEATURES ("ethernet-input"),
Ed Warnickecb9cada2015-12-08 15:45:58 -0700264};
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700265
Damjan Marion3891cd82016-10-27 10:27:00 +0200266typedef struct
267{
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700268 u32 sw_if_index;
269 u32 next_index;
270 u32 table_index;
271 u32 offset;
272 u32 policer_index;
273} policer_classify_trace_t;
274
275static u8 *
276format_policer_classify_trace (u8 * s, va_list * args)
277{
278 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
279 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
Damjan Marion3891cd82016-10-27 10:27:00 +0200280 policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700281
282 s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
Damjan Marion3891cd82016-10-27 10:27:00 +0200283 " policer_index %d",
284 t->sw_if_index, t->next_index, t->table_index, t->offset,
285 t->policer_index);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700286 return s;
287}
288
289#define foreach_policer_classify_error \
290_(MISS, "Policer classify misses") \
291_(HIT, "Policer classify hits") \
Paul Vinciguerrabdc0e6b2018-09-22 05:32:50 -0700292_(CHAIN_HIT, "Policer classify hits after chain walk") \
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700293_(DROP, "Policer classify action drop")
294
Damjan Marion3891cd82016-10-27 10:27:00 +0200295typedef enum
296{
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700297#define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
298 foreach_policer_classify_error
299#undef _
Damjan Marion3891cd82016-10-27 10:27:00 +0200300 POLICER_CLASSIFY_N_ERROR,
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700301} policer_classify_error_t;
302
Damjan Marion3891cd82016-10-27 10:27:00 +0200303static char *policer_classify_error_strings[] = {
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700304#define _(sym,string) string,
305 foreach_policer_classify_error
306#undef _
307};
308
309static inline uword
310policer_classify_inline (vlib_main_t * vm,
Damjan Marion3891cd82016-10-27 10:27:00 +0200311 vlib_node_runtime_t * node,
312 vlib_frame_t * frame,
313 policer_classify_table_id_t tid)
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700314{
Damjan Marion3891cd82016-10-27 10:27:00 +0200315 u32 n_left_from, *from, *to_next;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700316 policer_classify_next_index_t next_index;
Damjan Marion3891cd82016-10-27 10:27:00 +0200317 policer_classify_main_t *pcm = &policer_classify_main;
318 vnet_classify_main_t *vcm = pcm->vnet_classify_main;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700319 f64 now = vlib_time_now (vm);
320 u32 hits = 0;
321 u32 misses = 0;
322 u32 chain_hits = 0;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700323 u32 n_next_nodes;
324 u64 time_in_policer_periods;
325
326 time_in_policer_periods =
Damjan Marion3891cd82016-10-27 10:27:00 +0200327 clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700328
329 n_next_nodes = node->n_next_nodes;
330
331 from = vlib_frame_vector_args (frame);
332 n_left_from = frame->n_vectors;
333
334 /* First pass: compute hashes */
335 while (n_left_from > 2)
336 {
Damjan Marion3891cd82016-10-27 10:27:00 +0200337 vlib_buffer_t *b0, *b1;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700338 u32 bi0, bi1;
Damjan Marion3891cd82016-10-27 10:27:00 +0200339 u8 *h0, *h1;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700340 u32 sw_if_index0, sw_if_index1;
341 u32 table_index0, table_index1;
Damjan Marion3891cd82016-10-27 10:27:00 +0200342 vnet_classify_table_t *t0, *t1;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700343
344 /* Prefetch next iteration */
345 {
Damjan Marion3891cd82016-10-27 10:27:00 +0200346 vlib_buffer_t *p1, *p2;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700347
Damjan Marion3891cd82016-10-27 10:27:00 +0200348 p1 = vlib_get_buffer (vm, from[1]);
349 p2 = vlib_get_buffer (vm, from[2]);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700350
Damjan Marion3891cd82016-10-27 10:27:00 +0200351 vlib_prefetch_buffer_header (p1, STORE);
352 CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
353 vlib_prefetch_buffer_header (p2, STORE);
354 CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700355 }
356
357 bi0 = from[0];
358 b0 = vlib_get_buffer (vm, bi0);
359 h0 = b0->data;
360
361 bi1 = from[1];
362 b1 = vlib_get_buffer (vm, bi1);
363 h1 = b1->data;
364
365 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
Damjan Marion3891cd82016-10-27 10:27:00 +0200366 table_index0 =
367 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700368
369 sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
Damjan Marion3891cd82016-10-27 10:27:00 +0200370 table_index1 =
371 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700372
373 t0 = pool_elt_at_index (vcm->tables, table_index0);
374
375 t1 = pool_elt_at_index (vcm->tables, table_index1);
376
Damjan Marion3891cd82016-10-27 10:27:00 +0200377 vnet_buffer (b0)->l2_classify.hash =
378 vnet_classify_hash_packet (t0, (u8 *) h0);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700379
Damjan Marion3891cd82016-10-27 10:27:00 +0200380 vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700381
Damjan Marion3891cd82016-10-27 10:27:00 +0200382 vnet_buffer (b1)->l2_classify.hash =
383 vnet_classify_hash_packet (t1, (u8 *) h1);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700384
Damjan Marion3891cd82016-10-27 10:27:00 +0200385 vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700386
Damjan Marion3891cd82016-10-27 10:27:00 +0200387 vnet_buffer (b0)->l2_classify.table_index = table_index0;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700388
Damjan Marion3891cd82016-10-27 10:27:00 +0200389 vnet_buffer (b1)->l2_classify.table_index = table_index1;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700390
391 from += 2;
392 n_left_from -= 2;
393 }
394
395 while (n_left_from > 0)
396 {
Damjan Marion3891cd82016-10-27 10:27:00 +0200397 vlib_buffer_t *b0;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700398 u32 bi0;
Damjan Marion3891cd82016-10-27 10:27:00 +0200399 u8 *h0;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700400 u32 sw_if_index0;
401 u32 table_index0;
Damjan Marion3891cd82016-10-27 10:27:00 +0200402 vnet_classify_table_t *t0;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700403
404 bi0 = from[0];
405 b0 = vlib_get_buffer (vm, bi0);
406 h0 = b0->data;
407
408 sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
Damjan Marion3891cd82016-10-27 10:27:00 +0200409 table_index0 =
410 pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700411
412 t0 = pool_elt_at_index (vcm->tables, table_index0);
Damjan Marion3891cd82016-10-27 10:27:00 +0200413 vnet_buffer (b0)->l2_classify.hash =
414 vnet_classify_hash_packet (t0, (u8 *) h0);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700415
Damjan Marion3891cd82016-10-27 10:27:00 +0200416 vnet_buffer (b0)->l2_classify.table_index = table_index0;
417 vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700418
419 from++;
420 n_left_from--;
421 }
422
423 next_index = node->cached_next_index;
424 from = vlib_frame_vector_args (frame);
425 n_left_from = frame->n_vectors;
426
427 while (n_left_from > 0)
428 {
429 u32 n_left_to_next;
430
431 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
432
433 /* Not enough load/store slots to dual loop... */
434 while (n_left_from > 0 && n_left_to_next > 0)
Damjan Marion3891cd82016-10-27 10:27:00 +0200435 {
436 u32 bi0;
437 vlib_buffer_t *b0;
438 u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
439 u32 table_index0;
440 vnet_classify_table_t *t0;
441 vnet_classify_entry_t *e0;
442 u64 hash0;
443 u8 *h0;
444 u8 act0;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700445
Damjan Marion3891cd82016-10-27 10:27:00 +0200446 /* Stride 3 seems to work best */
447 if (PREDICT_TRUE (n_left_from > 3))
448 {
449 vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
450 vnet_classify_table_t *tp1;
451 u32 table_index1;
452 u64 phash1;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700453
Damjan Marion3891cd82016-10-27 10:27:00 +0200454 table_index1 = vnet_buffer (p1)->l2_classify.table_index;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700455
Damjan Marion3891cd82016-10-27 10:27:00 +0200456 if (PREDICT_TRUE (table_index1 != ~0))
457 {
458 tp1 = pool_elt_at_index (vcm->tables, table_index1);
459 phash1 = vnet_buffer (p1)->l2_classify.hash;
460 vnet_classify_prefetch_entry (tp1, phash1);
461 }
462 }
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700463
Damjan Marion3891cd82016-10-27 10:27:00 +0200464 /* Speculatively enqueue b0 to the current next frame */
465 bi0 = from[0];
466 to_next[0] = bi0;
467 from += 1;
468 to_next += 1;
469 n_left_from -= 1;
470 n_left_to_next -= 1;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700471
Damjan Marion3891cd82016-10-27 10:27:00 +0200472 b0 = vlib_get_buffer (vm, bi0);
473 h0 = b0->data;
474 table_index0 = vnet_buffer (b0)->l2_classify.table_index;
475 e0 = 0;
476 t0 = 0;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700477
Damjan Marion3891cd82016-10-27 10:27:00 +0200478 if (tid == POLICER_CLASSIFY_TABLE_L2)
479 {
John Lobeb0b2e2017-07-22 00:21:36 -0400480 /* Feature bitmap update and determine the next node */
481 next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
482 L2INPUT_FEAT_POLICER_CLAS);
Damjan Marion3891cd82016-10-27 10:27:00 +0200483 }
484 else
485 vnet_get_config_data (pcm->vnet_config_main[tid],
486 &b0->current_config_index, &next0,
487 /* # bytes of config data */ 0);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700488
Damjan Marion3891cd82016-10-27 10:27:00 +0200489 vnet_buffer (b0)->l2_classify.opaque_index = ~0;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700490
Damjan Marion3891cd82016-10-27 10:27:00 +0200491 if (PREDICT_TRUE (table_index0 != ~0))
492 {
493 hash0 = vnet_buffer (b0)->l2_classify.hash;
494 t0 = pool_elt_at_index (vcm->tables, table_index0);
495 e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700496
Damjan Marion3891cd82016-10-27 10:27:00 +0200497 if (e0)
498 {
499 act0 = vnet_policer_police (vm,
500 b0,
501 e0->next_index,
502 time_in_policer_periods,
503 e0->opaque_index);
Brian Russellc5299ff2021-02-09 10:16:58 +0000504 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
Damjan Marion3891cd82016-10-27 10:27:00 +0200505 {
506 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
507 b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
Damjan Marion3891cd82016-10-27 10:27:00 +0200508 }
509 hits++;
510 }
511 else
512 {
513 while (1)
514 {
515 if (PREDICT_TRUE (t0->next_table_index != ~0))
516 {
517 t0 = pool_elt_at_index (vcm->tables,
518 t0->next_table_index);
519 }
520 else
521 {
522 next0 = (t0->miss_next_index < n_next_nodes) ?
523 t0->miss_next_index : next0;
524 misses++;
525 break;
526 }
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700527
Damjan Marion3891cd82016-10-27 10:27:00 +0200528 hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
529 e0 =
530 vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
531 if (e0)
532 {
533 act0 = vnet_policer_police (vm,
534 b0,
535 e0->next_index,
536 time_in_policer_periods,
537 e0->opaque_index);
Brian Russellc5299ff2021-02-09 10:16:58 +0000538 if (PREDICT_FALSE (act0 == QOS_ACTION_DROP))
Damjan Marion3891cd82016-10-27 10:27:00 +0200539 {
540 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
541 b0->error =
542 node->errors[POLICER_CLASSIFY_ERROR_DROP];
Damjan Marion3891cd82016-10-27 10:27:00 +0200543 }
544 hits++;
545 chain_hits++;
546 break;
547 }
548 }
549 }
550 }
551 if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
552 && (b0->flags & VLIB_BUFFER_IS_TRACED)))
553 {
554 policer_classify_trace_t *t =
555 vlib_add_trace (vm, node, b0, sizeof (*t));
556 t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
557 t->next_index = next0;
558 t->table_index = t0 ? t0 - vcm->tables : ~0;
559 t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
560 t->policer_index = e0 ? e0->next_index : ~0;
561 }
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700562
Damjan Marion3891cd82016-10-27 10:27:00 +0200563 /* Verify speculative enqueue, maybe switch current next frame */
564 vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
565 n_left_to_next, bi0, next0);
566 }
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700567
568 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
569 }
570
571 vlib_node_increment_counter (vm, node->node_index,
Damjan Marion3891cd82016-10-27 10:27:00 +0200572 POLICER_CLASSIFY_ERROR_MISS, misses);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700573 vlib_node_increment_counter (vm, node->node_index,
Damjan Marion3891cd82016-10-27 10:27:00 +0200574 POLICER_CLASSIFY_ERROR_HIT, hits);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700575 vlib_node_increment_counter (vm, node->node_index,
Damjan Marion3891cd82016-10-27 10:27:00 +0200576 POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700577
578 return frame->n_vectors;
579}
580
Filip Tehlar30d93482019-03-06 04:28:32 -0800581VLIB_NODE_FN (ip4_policer_classify_node) (vlib_main_t * vm,
582 vlib_node_runtime_t * node,
583 vlib_frame_t * frame)
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700584{
Damjan Marion3891cd82016-10-27 10:27:00 +0200585 return policer_classify_inline (vm, node, frame,
586 POLICER_CLASSIFY_TABLE_IP4);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700587}
588
Damjan Marion3891cd82016-10-27 10:27:00 +0200589/* *INDENT-OFF* */
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700590VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700591 .name = "ip4-policer-classify",
592 .vector_size = sizeof (u32),
593 .format_trace = format_policer_classify_trace,
594 .n_errors = ARRAY_LEN(policer_classify_error_strings),
595 .error_strings = policer_classify_error_strings,
596 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
597 .next_nodes = {
598 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
599 },
600};
Damjan Marion3891cd82016-10-27 10:27:00 +0200601/* *INDENT-ON* */
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700602
Filip Tehlar30d93482019-03-06 04:28:32 -0800603VLIB_NODE_FN (ip6_policer_classify_node) (vlib_main_t * vm,
604 vlib_node_runtime_t * node,
605 vlib_frame_t * frame)
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700606{
Damjan Marion3891cd82016-10-27 10:27:00 +0200607 return policer_classify_inline (vm, node, frame,
608 POLICER_CLASSIFY_TABLE_IP6);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700609}
610
Damjan Marion3891cd82016-10-27 10:27:00 +0200611/* *INDENT-OFF* */
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700612VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700613 .name = "ip6-policer-classify",
614 .vector_size = sizeof (u32),
615 .format_trace = format_policer_classify_trace,
616 .n_errors = ARRAY_LEN(policer_classify_error_strings),
617 .error_strings = policer_classify_error_strings,
618 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
619 .next_nodes = {
620 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
621 },
622};
Damjan Marion3891cd82016-10-27 10:27:00 +0200623/* *INDENT-ON* */
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700624
Filip Tehlar30d93482019-03-06 04:28:32 -0800625VLIB_NODE_FN (l2_policer_classify_node) (vlib_main_t * vm,
626 vlib_node_runtime_t * node,
627 vlib_frame_t * frame)
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700628{
Damjan Marion3891cd82016-10-27 10:27:00 +0200629 return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700630}
631
Krishanpal singhd2fec4a2017-11-27 19:40:56 +0530632/* *INDENT-OFF* */
633VLIB_REGISTER_NODE (l2_policer_classify_node) = {
Krishanpal singhd2fec4a2017-11-27 19:40:56 +0530634 .name = "l2-policer-classify",
635 .vector_size = sizeof (u32),
636 .format_trace = format_policer_classify_trace,
637 .n_errors = ARRAY_LEN (policer_classify_error_strings),
638 .error_strings = policer_classify_error_strings,
639 .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
640 .next_nodes = {
641 [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
642 },
643};
Krishanpal singhd2fec4a2017-11-27 19:40:56 +0530644/* *INDENT-ON* */
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700645
Filip Tehlar30d93482019-03-06 04:28:32 -0800646#ifndef CLIB_MARCH_VARIANT
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700647static clib_error_t *
Damjan Marion3891cd82016-10-27 10:27:00 +0200648policer_classify_init (vlib_main_t * vm)
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700649{
Damjan Marion3891cd82016-10-27 10:27:00 +0200650 policer_classify_main_t *pcm = &policer_classify_main;
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700651
652 pcm->vlib_main = vm;
Damjan Marion3891cd82016-10-27 10:27:00 +0200653 pcm->vnet_main = vnet_get_main ();
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700654 pcm->vnet_classify_main = &vnet_classify_main;
655
656 /* Initialize L2 feature next-node indexes */
Damjan Marion3891cd82016-10-27 10:27:00 +0200657 feat_bitmap_init_next_nodes (vm,
658 l2_policer_classify_node.index,
659 L2INPUT_N_FEAT,
660 l2input_get_feat_names (),
661 pcm->feat_next_node_index);
Matus Fabian70e6a8d2016-06-20 08:10:42 -0700662
663 return 0;
664}
665
666VLIB_INIT_FUNCTION (policer_classify_init);
Filip Tehlar30d93482019-03-06 04:28:32 -0800667#endif /* CLIB_MARCH_VARIANT */
Damjan Marion3891cd82016-10-27 10:27:00 +0200668
669/*
670 * fd.io coding-style-patch-verification: ON
671 *
672 * Local Variables:
673 * eval: (c-set-style "gnu")
674 * End:
675 */