Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | * buffer.h: VLIB buffers |
| 17 | * |
| 18 | * Copyright (c) 2008 Eliot Dresselhaus |
| 19 | * |
| 20 | * Permission is hereby granted, free of charge, to any person obtaining |
| 21 | * a copy of this software and associated documentation files (the |
| 22 | * "Software"), to deal in the Software without restriction, including |
| 23 | * without limitation the rights to use, copy, modify, merge, publish, |
| 24 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 25 | * permit persons to whom the Software is furnished to do so, subject to |
| 26 | * the following conditions: |
| 27 | * |
| 28 | * The above copyright notice and this permission notice shall be |
| 29 | * included in all copies or substantial portions of the Software. |
| 30 | * |
| 31 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 32 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 33 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 34 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 35 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 36 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 37 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 38 | */ |
| 39 | |
| 40 | #ifndef included_vlib_buffer_h |
| 41 | #define included_vlib_buffer_h |
| 42 | |
| 43 | #include <vppinfra/types.h> |
| 44 | #include <vppinfra/cache.h> |
| 45 | #include <vppinfra/serialize.h> |
| 46 | #include <vppinfra/vector.h> |
Damjan Marion | 6b0f589 | 2017-07-27 04:01:24 -0400 | [diff] [blame] | 47 | #include <vppinfra/lock.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 48 | #include <vlib/error.h> /* for vlib_error_t */ |
Damjan Marion | 1901020 | 2016-03-24 17:17:47 +0100 | [diff] [blame] | 49 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 50 | #include <vlib/config.h> /* for __PRE_DATA_SIZE */ |
Damjan Marion | 1901020 | 2016-03-24 17:17:47 +0100 | [diff] [blame] | 51 | #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 52 | |
Damjan Marion | 5de3fec | 2019-02-06 14:22:32 +0100 | [diff] [blame] | 53 | #define VLIB_BUFFER_DEFAULT_DATA_SIZE (2048) |
| 54 | |
Damjan Marion | bd0da97 | 2018-10-31 10:59:02 +0100 | [diff] [blame] | 55 | /* Minimum buffer chain segment size. Does not apply to last buffer in chain. |
| 56 | Dataplane code can safely asume that specified amount of data is not split |
| 57 | into 2 chained buffers */ |
| 58 | #define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE (128) |
| 59 | |
| 60 | /* Amount of head buffer data copied to each replica head buffer */ |
| 61 | #define VLIB_BUFFER_CLONE_HEAD_SIZE (256) |
| 62 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 63 | /** \file |
| 64 | vlib buffer structure definition and a few select |
| 65 | access methods. This structure and the buffer allocation |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 66 | mechanism should perhaps live in vnet, but it would take a lot |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 67 | of typing to make it so. |
| 68 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 69 | |
Damjan Marion | dac0352 | 2018-02-01 15:30:13 +0100 | [diff] [blame] | 70 | /** |
| 71 | * Buffer Flags |
| 72 | */ |
| 73 | #define foreach_vlib_buffer_flag \ |
Damjan Marion | 36eb7c2 | 2019-01-18 20:45:30 +0100 | [diff] [blame] | 74 | _( 0, IS_TRACED, 0) \ |
| 75 | _( 1, NEXT_PRESENT, 0) \ |
| 76 | _( 2, TOTAL_LENGTH_VALID, 0) \ |
| 77 | _( 3, EXT_HDR_VALID, "ext-hdr-valid") |
Damjan Marion | dac0352 | 2018-02-01 15:30:13 +0100 | [diff] [blame] | 78 | |
| 79 | /* NOTE: only buffer generic flags should be defined here, please consider |
| 80 | using user flags. i.e. src/vnet/buffer.h */ |
| 81 | |
| 82 | enum |
| 83 | { |
| 84 | #define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)), |
| 85 | foreach_vlib_buffer_flag |
| 86 | #undef _ |
| 87 | }; |
| 88 | |
| 89 | enum |
| 90 | { |
| 91 | #define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit), |
| 92 | foreach_vlib_buffer_flag |
| 93 | #undef _ |
| 94 | }; |
| 95 | |
| 96 | /* User defined buffer flags. */ |
| 97 | #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n)) |
| 98 | #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n)) |
Damjan Marion | 36eb7c2 | 2019-01-18 20:45:30 +0100 | [diff] [blame] | 99 | #define VLIB_BUFFER_FLAGS_ALL (0x0f) |
Damjan Marion | dac0352 | 2018-02-01 15:30:13 +0100 | [diff] [blame] | 100 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 101 | /** VLIB buffer representation. */ |
| 102 | typedef union |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 103 | { |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 104 | struct |
| 105 | { |
| 106 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame] | 107 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 108 | /** signed offset in data[], pre_data[] that we are currently |
| 109 | * processing. If negative current header points into predata area. */ |
| 110 | i16 current_data; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 111 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 112 | /** Nbytes between current data and the end of this buffer. */ |
| 113 | u16 current_length; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 114 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 115 | /** buffer flags: |
| 116 | <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, |
| 117 | <br> VLIB_BUFFER_IS_TRACED: trace this buffer. |
| 118 | <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer. |
| 119 | <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says |
| 120 | <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header, |
| 121 | set to avoid adding it to a flow report |
| 122 | <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N |
| 123 | */ |
| 124 | u32 flags; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 125 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 126 | /** Generic flow identifier */ |
| 127 | u32 flow_id; |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 128 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 129 | /** Reference count for this buffer. */ |
| 130 | volatile u8 ref_count; |
Dave Barach | b5adaea | 2016-06-17 14:09:56 -0400 | [diff] [blame] | 131 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 132 | /** index of buffer pool this buffer belongs. */ |
| 133 | u8 buffer_pool_index; |
Damjan Marion | e58041f | 2019-01-18 19:56:09 +0100 | [diff] [blame] | 134 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 135 | /** Error code for buffers to be enqueued to error handler. */ |
| 136 | vlib_error_t error; |
Damjan Marion | e58041f | 2019-01-18 19:56:09 +0100 | [diff] [blame] | 137 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 138 | /** Next buffer for this linked-list of buffers. Only valid if |
| 139 | * VLIB_BUFFER_NEXT_PRESENT flag is set. */ |
| 140 | u32 next_buffer; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 141 | |
Neale Ranns | 76b5649 | 2018-09-28 15:16:14 +0000 | [diff] [blame] | 142 | /** The following fields can be in a union because once a packet enters |
| 143 | * the punt path, it is no longer on a feature arc */ |
| 144 | union |
| 145 | { |
| 146 | /** Used by feature subgraph arcs to visit enabled feature nodes */ |
| 147 | u32 current_config_index; |
| 148 | /* the reason the packet once punted */ |
| 149 | u32 punt_reason; |
| 150 | }; |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame] | 151 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 152 | /** Opaque data used by sub-graphs for their own purposes. */ |
| 153 | u32 opaque[10]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 154 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 155 | /** part of buffer metadata which is initialized on alloc ends here. */ |
| 156 | STRUCT_MARK (template_end); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 157 | |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 158 | /** start of 2nd cache line */ |
| 159 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
| 160 | |
Dave Barach | a638c18 | 2019-06-21 18:24:07 -0400 | [diff] [blame^] | 161 | /** Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 162 | * set. */ |
Dave Barach | a638c18 | 2019-06-21 18:24:07 -0400 | [diff] [blame^] | 163 | u32 trace_handle; |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 164 | |
| 165 | /** Only valid for first buffer in chain. Current length plus total length |
| 166 | * given here give total number of bytes in buffer chain. */ |
| 167 | u32 total_length_not_including_first_buffer; |
| 168 | |
| 169 | /**< More opaque data, see ../vnet/vnet/buffer.h */ |
| 170 | u32 opaque2[14]; |
| 171 | |
| 172 | /** start of third cache line */ |
| 173 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline2); |
| 174 | |
| 175 | /** Space for inserting data before buffer start. Packet rewrite string |
| 176 | * will be rewritten backwards and may extend back before |
| 177 | * buffer->data[0]. Must come directly before packet data. */ |
| 178 | u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]; |
| 179 | |
| 180 | /** Packet data */ |
| 181 | u8 data[0]; |
| 182 | }; |
| 183 | #ifdef CLIB_HAVE_VEC128 |
| 184 | u8x16 as_u8x16[4]; |
| 185 | #endif |
| 186 | #ifdef CLIB_HAVE_VEC256 |
Damjan Marion | 22f23ae | 2019-01-24 15:36:57 +0100 | [diff] [blame] | 187 | u8x32 as_u8x32[2]; |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 188 | #endif |
| 189 | #ifdef CLIB_HAVE_VEC512 |
Damjan Marion | 22f23ae | 2019-01-24 15:36:57 +0100 | [diff] [blame] | 190 | u8x64 as_u8x64[1]; |
Damjan Marion | 9a8a12a | 2019-01-23 16:52:10 +0100 | [diff] [blame] | 191 | #endif |
| 192 | } vlib_buffer_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 193 | |
Damjan Marion | 1901020 | 2016-03-24 17:17:47 +0100 | [diff] [blame] | 194 | #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE) |
| 195 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 196 | /** \brief Prefetch buffer metadata. |
| 197 | The first 64 bytes of buffer contains most header information |
| 198 | |
| 199 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 200 | @param type - LOAD, STORE. In most cases, STORE is the right answer |
| 201 | */ |
| 202 | |
| 203 | #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type) |
Damjan Marion | d30bf01 | 2018-11-18 23:48:43 +0100 | [diff] [blame] | 204 | #define vlib_prefetch_buffer_data(b,type) \ |
| 205 | CLIB_PREFETCH (vlib_buffer_get_current(b), CLIB_CACHE_LINE_BYTES, type) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 206 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 207 | always_inline void |
| 208 | vlib_buffer_struct_is_sane (vlib_buffer_t * b) |
| 209 | { |
| 210 | ASSERT (sizeof (b[0]) % 64 == 0); |
| 211 | |
| 212 | /* Rewrite data must be before and contiguous with packet data. */ |
| 213 | ASSERT (b->pre_data + VLIB_BUFFER_PRE_DATA_SIZE == b->data); |
| 214 | } |
| 215 | |
Damjan Marion | 8f49936 | 2018-10-22 13:07:02 +0200 | [diff] [blame] | 216 | always_inline uword |
| 217 | vlib_buffer_get_va (vlib_buffer_t * b) |
| 218 | { |
| 219 | return pointer_to_uword (b->data); |
| 220 | } |
| 221 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 222 | /** \brief Get pointer to current data to process |
| 223 | |
| 224 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 225 | @return - (void *) (b->data + b->current_data) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 226 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 227 | |
| 228 | always_inline void * |
| 229 | vlib_buffer_get_current (vlib_buffer_t * b) |
| 230 | { |
| 231 | /* Check bounds. */ |
| 232 | ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE); |
| 233 | return b->data + b->current_data; |
| 234 | } |
| 235 | |
Damjan Marion | 8f49936 | 2018-10-22 13:07:02 +0200 | [diff] [blame] | 236 | always_inline uword |
| 237 | vlib_buffer_get_current_va (vlib_buffer_t * b) |
| 238 | { |
| 239 | return vlib_buffer_get_va (b) + b->current_data; |
| 240 | } |
| 241 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 242 | /** \brief Advance current data pointer by the supplied (signed!) amount |
| 243 | |
| 244 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 245 | @param l - (word) signed increment |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 246 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 247 | always_inline void |
| 248 | vlib_buffer_advance (vlib_buffer_t * b, word l) |
| 249 | { |
| 250 | ASSERT (b->current_length >= l); |
| 251 | b->current_data += l; |
| 252 | b->current_length -= l; |
Damjan Marion | bd0da97 | 2018-10-31 10:59:02 +0100 | [diff] [blame] | 253 | |
| 254 | ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 || |
| 255 | b->current_length >= VLIB_BUFFER_MIN_CHAIN_SEG_SIZE); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 256 | } |
| 257 | |
Filip Tehlar | 816f437 | 2017-04-26 16:09:06 +0200 | [diff] [blame] | 258 | /** \brief Check if there is enough space in buffer to advance |
| 259 | |
| 260 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 261 | @param l - (word) size to check |
| 262 | @return - 0 if there is less space than 'l' in buffer |
| 263 | */ |
| 264 | always_inline u8 |
| 265 | vlib_buffer_has_space (vlib_buffer_t * b, word l) |
| 266 | { |
| 267 | return b->current_length >= l; |
| 268 | } |
| 269 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 270 | /** \brief Reset current header & length to state they were in when |
| 271 | packet was received. |
| 272 | |
| 273 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 274 | */ |
| 275 | |
| 276 | always_inline void |
| 277 | vlib_buffer_reset (vlib_buffer_t * b) |
| 278 | { |
| 279 | b->current_length += clib_max (b->current_data, 0); |
| 280 | b->current_data = 0; |
| 281 | } |
| 282 | |
| 283 | /** \brief Get pointer to buffer's opaque data array |
| 284 | |
| 285 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 286 | @return - (void *) b->opaque |
| 287 | */ |
| 288 | always_inline void * |
| 289 | vlib_get_buffer_opaque (vlib_buffer_t * b) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 290 | { |
| 291 | return (void *) b->opaque; |
| 292 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 293 | |
| 294 | /** \brief Get pointer to buffer's opaque2 data array |
| 295 | |
| 296 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 297 | @return - (void *) b->opaque2 |
| 298 | */ |
| 299 | always_inline void * |
| 300 | vlib_get_buffer_opaque2 (vlib_buffer_t * b) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 301 | { |
| 302 | return (void *) b->opaque2; |
| 303 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 304 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 305 | /** \brief Get pointer to the end of buffer's data |
| 306 | * @param b pointer to the buffer |
| 307 | * @return pointer to tail of packet's data |
| 308 | */ |
| 309 | always_inline u8 * |
| 310 | vlib_buffer_get_tail (vlib_buffer_t * b) |
| 311 | { |
| 312 | return b->data + b->current_data + b->current_length; |
| 313 | } |
| 314 | |
| 315 | /** \brief Append uninitialized data to buffer |
| 316 | * @param b pointer to the buffer |
| 317 | * @param size number of uninitialized bytes |
| 318 | * @return pointer to beginning of uninitialized data |
| 319 | */ |
| 320 | always_inline void * |
Eyal Bari | d3d4241 | 2018-11-05 13:29:25 +0200 | [diff] [blame] | 321 | vlib_buffer_put_uninit (vlib_buffer_t * b, u16 size) |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 322 | { |
| 323 | void *p = vlib_buffer_get_tail (b); |
| 324 | /* XXX make sure there's enough space */ |
| 325 | b->current_length += size; |
| 326 | return p; |
| 327 | } |
| 328 | |
| 329 | /** \brief Prepend uninitialized data to buffer |
| 330 | * @param b pointer to the buffer |
| 331 | * @param size number of uninitialized bytes |
| 332 | * @return pointer to beginning of uninitialized data |
| 333 | */ |
| 334 | always_inline void * |
| 335 | vlib_buffer_push_uninit (vlib_buffer_t * b, u8 size) |
| 336 | { |
| 337 | ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size); |
| 338 | b->current_data -= size; |
| 339 | b->current_length += size; |
| 340 | |
| 341 | return vlib_buffer_get_current (b); |
| 342 | } |
| 343 | |
| 344 | /** \brief Make head room, typically for packet headers |
| 345 | * @param b pointer to the buffer |
| 346 | * @param size number of head room bytes |
| 347 | * @return pointer to start of buffer (current data) |
| 348 | */ |
| 349 | always_inline void * |
| 350 | vlib_buffer_make_headroom (vlib_buffer_t * b, u8 size) |
| 351 | { |
| 352 | ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size); |
| 353 | b->current_data += size; |
| 354 | return vlib_buffer_get_current (b); |
| 355 | } |
| 356 | |
Dave Barach | a638c18 | 2019-06-21 18:24:07 -0400 | [diff] [blame^] | 357 | /** \brief Construct a trace handle from thread and pool index |
| 358 | * @param thread Thread id |
| 359 | * @param pool_index Pool index |
| 360 | * @return trace handle |
| 361 | */ |
| 362 | always_inline u32 |
| 363 | vlib_buffer_make_trace_handle (u32 thread, u32 pool_index) |
| 364 | { |
| 365 | u32 rv; |
| 366 | ASSERT (thread < 0xff); |
| 367 | ASSERT (pool_index < 0x00FFFFFF); |
| 368 | rv = (thread << 24) | (pool_index & 0x00FFFFFF); |
| 369 | return rv; |
| 370 | } |
| 371 | |
| 372 | /** \brief Extract the thread id from a trace handle |
| 373 | * @param trace_handle the trace handle |
| 374 | * @return the thread id |
| 375 | */ |
| 376 | always_inline u32 |
| 377 | vlib_buffer_get_trace_thread (vlib_buffer_t * b) |
| 378 | { |
| 379 | u32 trace_handle = b->trace_handle; |
| 380 | |
| 381 | return trace_handle >> 24; |
| 382 | } |
| 383 | |
| 384 | /** \brief Extract the trace (pool) index from a trace handle |
| 385 | * @param trace_handle the trace handle |
| 386 | * @return the trace index |
| 387 | */ |
| 388 | always_inline u32 |
| 389 | vlib_buffer_get_trace_index (vlib_buffer_t * b) |
| 390 | { |
| 391 | u32 trace_handle = b->trace_handle; |
| 392 | return trace_handle & 0x00FFFFFF; |
| 393 | } |
| 394 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 395 | /** \brief Retrieve bytes from buffer head |
| 396 | * @param b pointer to the buffer |
| 397 | * @param size number of bytes to pull |
| 398 | * @return pointer to start of buffer (current data) |
| 399 | */ |
| 400 | always_inline void * |
| 401 | vlib_buffer_pull (vlib_buffer_t * b, u8 size) |
| 402 | { |
| 403 | if (b->current_length + VLIB_BUFFER_PRE_DATA_SIZE < size) |
| 404 | return 0; |
| 405 | |
| 406 | void *data = vlib_buffer_get_current (b); |
| 407 | vlib_buffer_advance (b, size); |
| 408 | return data; |
| 409 | } |
| 410 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 411 | /* Forward declaration. */ |
| 412 | struct vlib_main_t; |
| 413 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 414 | typedef struct |
| 415 | { |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 416 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
| 417 | u32 *cached_buffers; |
| 418 | u32 n_alloc; |
| 419 | } vlib_buffer_pool_thread_t; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 420 | typedef struct |
| 421 | { |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 422 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame] | 423 | uword start; |
| 424 | uword size; |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 425 | uword log2_page_size; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 426 | u8 index; |
| 427 | u32 numa_node; |
Damjan Marion | 68b4da6 | 2018-09-30 18:26:20 +0200 | [diff] [blame] | 428 | u32 physmem_map_index; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 429 | u32 data_size; |
| 430 | u32 n_buffers; |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 431 | u32 *buffers; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 432 | u8 *name; |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 433 | clib_spinlock_t lock; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 434 | |
| 435 | /* per-thread data */ |
| 436 | vlib_buffer_pool_thread_t *threads; |
| 437 | |
| 438 | /* buffer metadata template */ |
| 439 | vlib_buffer_t buffer_template; |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame] | 440 | } vlib_buffer_pool_t; |
| 441 | |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 442 | #define VLIB_BUFFER_MAX_NUMA_NODES 32 |
| 443 | |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame] | 444 | typedef struct |
| 445 | { |
| 446 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 447 | /* Virtual memory address and size of buffer memory, used for calculating |
| 448 | buffer index */ |
| 449 | uword buffer_mem_start; |
| 450 | uword buffer_mem_size; |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame] | 451 | vlib_buffer_pool_t *buffer_pools; |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 452 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 453 | /* Hash table mapping buffer index into number |
| 454 | 0 => allocated but free, 1 => allocated and not-free. |
| 455 | If buffer index is not in hash table then this buffer |
| 456 | has never been allocated. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 457 | uword *buffer_known_hash; |
Damjan Marion | 6b0f589 | 2017-07-27 04:01:24 -0400 | [diff] [blame] | 458 | clib_spinlock_t buffer_known_hash_lockp; |
Damjan Marion | b592d1b | 2019-02-28 23:16:11 +0100 | [diff] [blame] | 459 | u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 460 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 461 | /* config */ |
| 462 | u32 buffers_per_numa; |
| 463 | u16 ext_hdr_size; |
Damjan Marion | 5de3fec | 2019-02-06 14:22:32 +0100 | [diff] [blame] | 464 | u32 default_data_size; |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 465 | |
| 466 | /* logging */ |
| 467 | vlib_log_class_t log_default; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 468 | } vlib_buffer_main_t; |
| 469 | |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 470 | clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm); |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 471 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 472 | /* |
| 473 | */ |
| 474 | |
| 475 | /** \brief Compile time buffer trajectory tracing option |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 476 | Turn this on if you run into "bad monkey" contexts, |
| 477 | and you want to know exactly which nodes they've visited... |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 478 | See vlib/main.c... |
| 479 | */ |
| 480 | #define VLIB_BUFFER_TRACE_TRAJECTORY 0 |
| 481 | |
| 482 | #if VLIB_BUFFER_TRACE_TRAJECTORY > 0 |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 483 | extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index); |
| 484 | extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b); |
| 485 | extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b); |
| 486 | #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \ |
| 487 | vlib_buffer_trace_trajectory_init (b); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 488 | #else |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 489 | #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) |
| 490 | #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */ |
| 491 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 492 | extern u16 __vlib_buffer_external_hdr_size; |
| 493 | #define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \ |
| 494 | static void __clib_constructor \ |
| 495 | vnet_buffer_set_ext_hdr_size() \ |
| 496 | { \ |
| 497 | if (__vlib_buffer_external_hdr_size) \ |
| 498 | clib_error ("buffer external header space already set"); \ |
| 499 | __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \ |
| 500 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 501 | |
Damjan Marion | 910d369 | 2019-01-21 11:48:34 +0100 | [diff] [blame] | 502 | #endif /* included_vlib_buffer_h */ |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 503 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 504 | /* |
| 505 | * fd.io coding-style-patch-verification: ON |
| 506 | * |
| 507 | * Local Variables: |
| 508 | * eval: (c-set-style "gnu") |
| 509 | * End: |
| 510 | */ |