Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | * buffer.h: VLIB buffers |
| 17 | * |
| 18 | * Copyright (c) 2008 Eliot Dresselhaus |
| 19 | * |
| 20 | * Permission is hereby granted, free of charge, to any person obtaining |
| 21 | * a copy of this software and associated documentation files (the |
| 22 | * "Software"), to deal in the Software without restriction, including |
| 23 | * without limitation the rights to use, copy, modify, merge, publish, |
| 24 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 25 | * permit persons to whom the Software is furnished to do so, subject to |
| 26 | * the following conditions: |
| 27 | * |
| 28 | * The above copyright notice and this permission notice shall be |
| 29 | * included in all copies or substantial portions of the Software. |
| 30 | * |
| 31 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 32 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 33 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 34 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 35 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 36 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 37 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 38 | */ |
| 39 | |
| 40 | #ifndef included_vlib_buffer_h |
| 41 | #define included_vlib_buffer_h |
| 42 | |
| 43 | #include <vppinfra/types.h> |
| 44 | #include <vppinfra/cache.h> |
| 45 | #include <vppinfra/serialize.h> |
| 46 | #include <vppinfra/vector.h> |
Damjan Marion | 6b0f589 | 2017-07-27 04:01:24 -0400 | [diff] [blame] | 47 | #include <vppinfra/lock.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 48 | #include <vlib/error.h> /* for vlib_error_t */ |
Damjan Marion | 1901020 | 2016-03-24 17:17:47 +0100 | [diff] [blame] | 49 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 50 | #include <vlib/config.h> /* for __PRE_DATA_SIZE */ |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 51 | #define VLIB_BUFFER_DATA_SIZE (2048) |
Damjan Marion | 1901020 | 2016-03-24 17:17:47 +0100 | [diff] [blame] | 52 | #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 53 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 54 | /** \file |
| 55 | vlib buffer structure definition and a few select |
| 56 | access methods. This structure and the buffer allocation |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 57 | mechanism should perhaps live in vnet, but it would take a lot |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 58 | of typing to make it so. |
| 59 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 60 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 61 | /* VLIB buffer representation. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 62 | typedef struct |
| 63 | { |
| 64 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Dave Barach | f869028 | 2017-03-01 11:38:02 -0500 | [diff] [blame] | 65 | STRUCT_MARK (template_start); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 66 | /* Offset within data[] that we are currently processing. |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 67 | If negative current header points into predata area. */ |
| 68 | i16 current_data; /**< signed offset in data[], pre_data[] |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 69 | that we are currently processing. |
| 70 | If negative current header points into predata area. |
| 71 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 72 | u16 current_length; /**< Nbytes between current data and |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 73 | the end of this buffer. |
| 74 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 75 | u32 flags; /**< buffer flags: |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame] | 76 | <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 77 | <br> VLIB_BUFFER_IS_TRACED: trace this buffer. |
| 78 | <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer. |
| 79 | <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says |
| 80 | <br> VLIB_BUFFER_REPL_FAIL: packet replication failure |
Dave Barach | b5adaea | 2016-06-17 14:09:56 -0400 | [diff] [blame] | 81 | <br> VLIB_BUFFER_RECYCLE: as it says |
Dave Barach | ad41b35 | 2016-10-07 08:30:23 -0700 | [diff] [blame] | 82 | <br> VLIB_BUFFER_FLOW_REPORT: buffer is a flow report, |
Damjan Marion | ff54270 | 2017-02-27 11:29:20 +0100 | [diff] [blame] | 83 | <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header, |
Dave Barach | ad41b35 | 2016-10-07 08:30:23 -0700 | [diff] [blame] | 84 | set to avoid adding it to a flow report |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 85 | <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N |
| 86 | */ |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame] | 87 | |
| 88 | /* any change to the following line requres update of |
| 89 | * vlib_buffer_get_free_list_index(...) and |
| 90 | * vlib_buffer_set_free_list_index(...) functions */ |
Dave Wallace | d756b35 | 2017-07-03 13:11:38 -0400 | [diff] [blame] | 91 | #define VLIB_BUFFER_FREE_LIST_INDEX_MASK ((1 << 5) - 1) |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame] | 92 | |
Dave Wallace | d756b35 | 2017-07-03 13:11:38 -0400 | [diff] [blame] | 93 | #define VLIB_BUFFER_IS_TRACED (1 << 5) |
| 94 | #define VLIB_BUFFER_LOG2_NEXT_PRESENT (6) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 95 | #define VLIB_BUFFER_NEXT_PRESENT (1 << VLIB_BUFFER_LOG2_NEXT_PRESENT) |
Dave Wallace | d756b35 | 2017-07-03 13:11:38 -0400 | [diff] [blame] | 96 | #define VLIB_BUFFER_IS_RECYCLED (1 << 7) |
| 97 | #define VLIB_BUFFER_TOTAL_LENGTH_VALID (1 << 8) |
| 98 | #define VLIB_BUFFER_REPL_FAIL (1 << 9) |
| 99 | #define VLIB_BUFFER_RECYCLE (1 << 10) |
| 100 | #define VLIB_BUFFER_FLOW_REPORT (1 << 11) |
| 101 | #define VLIB_BUFFER_EXT_HDR_VALID (1 << 12) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 102 | |
| 103 | /* User defined buffer flags. */ |
| 104 | #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n)) |
| 105 | #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n)) |
| 106 | |
Dave Barach | f869028 | 2017-03-01 11:38:02 -0500 | [diff] [blame] | 107 | STRUCT_MARK (template_end); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 108 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 109 | u32 next_buffer; /**< Next buffer for this linked-list of buffers. |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 110 | Only valid if VLIB_BUFFER_NEXT_PRESENT flag is set. |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 111 | */ |
| 112 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 113 | vlib_error_t error; /**< Error code for buffers to be enqueued |
| 114 | to error handler. |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 115 | */ |
Dave Barach | d653460 | 2016-06-14 18:38:02 -0400 | [diff] [blame] | 116 | u32 current_config_index; /**< Used by feature subgraph arcs to |
| 117 | visit enabled feature nodes |
| 118 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 119 | |
Damjan Marion | 87cd119 | 2016-11-04 11:00:27 +0100 | [diff] [blame] | 120 | u8 feature_arc_index; /**< Used to identify feature arcs by intermediate |
| 121 | feature node |
| 122 | */ |
| 123 | |
Damjan Marion | c47ed03 | 2017-01-25 14:18:03 +0100 | [diff] [blame] | 124 | u8 n_add_refs; /**< Number of additional references to this buffer. */ |
| 125 | |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame^] | 126 | u8 buffer_pool_index; /**< index of buffer pool this buffer belongs. */ |
| 127 | u8 dont_waste_me[1]; /**< Available space in the (precious) |
Damjan Marion | 87cd119 | 2016-11-04 11:00:27 +0100 | [diff] [blame] | 128 | first 32 octets of buffer metadata |
| 129 | Before allocating any of it, discussion required! |
| 130 | */ |
Dave Barach | b5adaea | 2016-06-17 14:09:56 -0400 | [diff] [blame] | 131 | |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame] | 132 | u32 opaque[10]; /**< Opaque data used by sub-graphs for their own purposes. |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 133 | See .../vnet/vnet/buffer.h |
| 134 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 135 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 136 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 137 | u32 trace_index; /**< Specifies index into trace buffer |
| 138 | if VLIB_PACKET_IS_TRACED flag is set. |
Dave Barach | d653460 | 2016-06-14 18:38:02 -0400 | [diff] [blame] | 139 | */ |
Dave Barach | b5adaea | 2016-06-17 14:09:56 -0400 | [diff] [blame] | 140 | u32 recycle_count; /**< Used by L2 path recycle code */ |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame] | 141 | |
| 142 | u32 total_length_not_including_first_buffer; |
| 143 | /**< Only valid for first buffer in chain. Current length plus |
| 144 | total length given here give total number of bytes in buffer chain. |
| 145 | */ |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 146 | u32 align_pad; /**< available */ |
| 147 | u32 opaque2[12]; /**< More opaque data, see ../vnet/vnet/buffer.h */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 148 | |
| 149 | /***** end of second cache line */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 150 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline2); |
| 151 | u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data |
Damjan Marion | 1901020 | 2016-03-24 17:17:47 +0100 | [diff] [blame] | 152 | before buffer start. |
| 153 | Packet rewrite string will be |
| 154 | rewritten backwards and may extend |
| 155 | back before buffer->data[0]. |
| 156 | Must come directly before packet data. |
| 157 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 158 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 159 | u8 data[0]; /**< Packet data. Hardware DMA here */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 160 | } vlib_buffer_t; /* Must be a multiple of 64B. */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 161 | |
Damjan Marion | 1901020 | 2016-03-24 17:17:47 +0100 | [diff] [blame] | 162 | #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE) |
| 163 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 164 | /** \brief Prefetch buffer metadata. |
| 165 | The first 64 bytes of buffer contains most header information |
| 166 | |
| 167 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 168 | @param type - LOAD, STORE. In most cases, STORE is the right answer |
| 169 | */ |
| 170 | |
| 171 | #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type) |
| 172 | |
| 173 | always_inline vlib_buffer_t * |
| 174 | vlib_buffer_next_contiguous (vlib_buffer_t * b, u32 buffer_bytes) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 175 | { |
| 176 | return (void *) (b + 1) + buffer_bytes; |
| 177 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 178 | |
| 179 | always_inline void |
| 180 | vlib_buffer_struct_is_sane (vlib_buffer_t * b) |
| 181 | { |
| 182 | ASSERT (sizeof (b[0]) % 64 == 0); |
| 183 | |
| 184 | /* Rewrite data must be before and contiguous with packet data. */ |
| 185 | ASSERT (b->pre_data + VLIB_BUFFER_PRE_DATA_SIZE == b->data); |
| 186 | } |
| 187 | |
| 188 | /** \brief Get pointer to current data to process |
| 189 | |
| 190 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 191 | @return - (void *) (b->data + b->current_data) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 192 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 193 | |
| 194 | always_inline void * |
| 195 | vlib_buffer_get_current (vlib_buffer_t * b) |
| 196 | { |
| 197 | /* Check bounds. */ |
| 198 | ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE); |
| 199 | return b->data + b->current_data; |
| 200 | } |
| 201 | |
| 202 | /** \brief Advance current data pointer by the supplied (signed!) amount |
| 203 | |
| 204 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 205 | @param l - (word) signed increment |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 206 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 207 | always_inline void |
| 208 | vlib_buffer_advance (vlib_buffer_t * b, word l) |
| 209 | { |
| 210 | ASSERT (b->current_length >= l); |
| 211 | b->current_data += l; |
| 212 | b->current_length -= l; |
| 213 | } |
| 214 | |
Filip Tehlar | 816f437 | 2017-04-26 16:09:06 +0200 | [diff] [blame] | 215 | /** \brief Check if there is enough space in buffer to advance |
| 216 | |
| 217 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 218 | @param l - (word) size to check |
| 219 | @return - 0 if there is less space than 'l' in buffer |
| 220 | */ |
| 221 | always_inline u8 |
| 222 | vlib_buffer_has_space (vlib_buffer_t * b, word l) |
| 223 | { |
| 224 | return b->current_length >= l; |
| 225 | } |
| 226 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 227 | /** \brief Reset current header & length to state they were in when |
| 228 | packet was received. |
| 229 | |
| 230 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 231 | */ |
| 232 | |
| 233 | always_inline void |
| 234 | vlib_buffer_reset (vlib_buffer_t * b) |
| 235 | { |
| 236 | b->current_length += clib_max (b->current_data, 0); |
| 237 | b->current_data = 0; |
| 238 | } |
| 239 | |
| 240 | /** \brief Get pointer to buffer's opaque data array |
| 241 | |
| 242 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 243 | @return - (void *) b->opaque |
| 244 | */ |
| 245 | always_inline void * |
| 246 | vlib_get_buffer_opaque (vlib_buffer_t * b) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 247 | { |
| 248 | return (void *) b->opaque; |
| 249 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 250 | |
| 251 | /** \brief Get pointer to buffer's opaque2 data array |
| 252 | |
| 253 | @param b - (vlib_buffer_t *) pointer to the buffer |
| 254 | @return - (void *) b->opaque2 |
| 255 | */ |
| 256 | always_inline void * |
| 257 | vlib_get_buffer_opaque2 (vlib_buffer_t * b) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 258 | { |
| 259 | return (void *) b->opaque2; |
| 260 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 261 | |
Dave Barach | 68b0fb0 | 2017-02-28 15:15:56 -0500 | [diff] [blame] | 262 | /** \brief Get pointer to the end of buffer's data |
| 263 | * @param b pointer to the buffer |
| 264 | * @return pointer to tail of packet's data |
| 265 | */ |
| 266 | always_inline u8 * |
| 267 | vlib_buffer_get_tail (vlib_buffer_t * b) |
| 268 | { |
| 269 | return b->data + b->current_data + b->current_length; |
| 270 | } |
| 271 | |
| 272 | /** \brief Append uninitialized data to buffer |
| 273 | * @param b pointer to the buffer |
| 274 | * @param size number of uninitialized bytes |
| 275 | * @return pointer to beginning of uninitialized data |
| 276 | */ |
| 277 | always_inline void * |
| 278 | vlib_buffer_put_uninit (vlib_buffer_t * b, u8 size) |
| 279 | { |
| 280 | void *p = vlib_buffer_get_tail (b); |
| 281 | /* XXX make sure there's enough space */ |
| 282 | b->current_length += size; |
| 283 | return p; |
| 284 | } |
| 285 | |
| 286 | /** \brief Prepend uninitialized data to buffer |
| 287 | * @param b pointer to the buffer |
| 288 | * @param size number of uninitialized bytes |
| 289 | * @return pointer to beginning of uninitialized data |
| 290 | */ |
| 291 | always_inline void * |
| 292 | vlib_buffer_push_uninit (vlib_buffer_t * b, u8 size) |
| 293 | { |
| 294 | ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size); |
| 295 | b->current_data -= size; |
| 296 | b->current_length += size; |
| 297 | |
| 298 | return vlib_buffer_get_current (b); |
| 299 | } |
| 300 | |
| 301 | /** \brief Make head room, typically for packet headers |
| 302 | * @param b pointer to the buffer |
| 303 | * @param size number of head room bytes |
| 304 | * @return pointer to start of buffer (current data) |
| 305 | */ |
| 306 | always_inline void * |
| 307 | vlib_buffer_make_headroom (vlib_buffer_t * b, u8 size) |
| 308 | { |
| 309 | ASSERT (b->current_data + VLIB_BUFFER_PRE_DATA_SIZE >= size); |
| 310 | b->current_data += size; |
| 311 | return vlib_buffer_get_current (b); |
| 312 | } |
| 313 | |
| 314 | /** \brief Retrieve bytes from buffer head |
| 315 | * @param b pointer to the buffer |
| 316 | * @param size number of bytes to pull |
| 317 | * @return pointer to start of buffer (current data) |
| 318 | */ |
| 319 | always_inline void * |
| 320 | vlib_buffer_pull (vlib_buffer_t * b, u8 size) |
| 321 | { |
| 322 | if (b->current_length + VLIB_BUFFER_PRE_DATA_SIZE < size) |
| 323 | return 0; |
| 324 | |
| 325 | void *data = vlib_buffer_get_current (b); |
| 326 | vlib_buffer_advance (b, size); |
| 327 | return data; |
| 328 | } |
| 329 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 330 | /* Forward declaration. */ |
| 331 | struct vlib_main_t; |
| 332 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 333 | typedef struct vlib_buffer_free_list_t |
| 334 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 335 | /* Template buffer used to initialize first 16 bytes of buffers |
| 336 | allocated on this free list. */ |
| 337 | vlib_buffer_t buffer_init_template; |
| 338 | |
| 339 | /* Our index into vlib_main_t's buffer_free_list_pool. */ |
| 340 | u32 index; |
| 341 | |
| 342 | /* Number of data bytes for buffers in this free list. */ |
| 343 | u32 n_data_bytes; |
| 344 | |
| 345 | /* Number of buffers to allocate when we need to allocate new buffers |
| 346 | from physmem heap. */ |
| 347 | u32 min_n_buffers_each_physmem_alloc; |
| 348 | |
| 349 | /* Total number of buffers allocated from this free list. */ |
| 350 | u32 n_alloc; |
| 351 | |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 352 | /* Vector of free buffers. Each element is a byte offset into I/O heap. */ |
| 353 | u32 *buffers; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 354 | |
Damjan Marion | b6a8ed7 | 2017-08-29 00:15:35 +0200 | [diff] [blame] | 355 | /* global vector of free buffers, used only on main thread. |
| 356 | Bufers are returned to global buffers only in case when number of |
| 357 | buffers on free buffers list grows about threshold */ |
| 358 | u32 *global_buffers; |
| 359 | clib_spinlock_t global_buffers_lock; |
| 360 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 361 | /* Memory chunks allocated for this free list |
| 362 | recorded here so they can be freed when free list |
| 363 | is deleted. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 364 | void **buffer_memory_allocated; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 365 | |
| 366 | /* Free list name. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 367 | u8 *name; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 368 | |
| 369 | /* Callback functions to initialize newly allocated buffers. |
| 370 | If null buffers are zeroed. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 371 | void (*buffer_init_function) (struct vlib_main_t * vm, |
| 372 | struct vlib_buffer_free_list_t * fl, |
| 373 | u32 * buffers, u32 n_buffers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 374 | |
| 375 | /* Callback function to announce that buffers have been |
| 376 | added to the freelist */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 377 | void (*buffers_added_to_freelist_function) |
| 378 | (struct vlib_main_t * vm, struct vlib_buffer_free_list_t * fl); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 379 | |
| 380 | uword buffer_init_function_opaque; |
| 381 | } __attribute__ ((aligned (16))) vlib_buffer_free_list_t; |
| 382 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 383 | typedef struct |
| 384 | { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 385 | u32 (*vlib_buffer_alloc_cb) (struct vlib_main_t * vm, u32 * buffers, |
| 386 | u32 n_buffers); |
| 387 | u32 (*vlib_buffer_alloc_from_free_list_cb) (struct vlib_main_t * vm, |
| 388 | u32 * buffers, u32 n_buffers, |
| 389 | u32 free_list_index); |
| 390 | void (*vlib_buffer_free_cb) (struct vlib_main_t * vm, u32 * buffers, |
| 391 | u32 n_buffers); |
| 392 | void (*vlib_buffer_free_no_next_cb) (struct vlib_main_t * vm, u32 * buffers, |
| 393 | u32 n_buffers); |
| 394 | void (*vlib_packet_template_init_cb) (struct vlib_main_t * vm, void *t, |
| 395 | void *packet_data, |
| 396 | uword n_packet_data_bytes, |
| 397 | uword |
| 398 | min_n_buffers_each_physmem_alloc, |
| 399 | u8 * name); |
| 400 | void (*vlib_buffer_delete_free_list_cb) (struct vlib_main_t * vm, |
| 401 | u32 free_list_index); |
| 402 | } vlib_buffer_callbacks_t; |
| 403 | |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 404 | extern vlib_buffer_callbacks_t *vlib_buffer_callbacks; |
| 405 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 406 | typedef struct |
| 407 | { |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 408 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame^] | 409 | uword start; |
| 410 | uword size; |
| 411 | vlib_physmem_region_index_t physmem_region; |
| 412 | } vlib_buffer_pool_t; |
| 413 | |
| 414 | typedef struct |
| 415 | { |
| 416 | CLIB_CACHE_LINE_ALIGN_MARK (cacheline0); |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 417 | /* Virtual memory address and size of buffer memory, used for calculating |
| 418 | buffer index */ |
| 419 | uword buffer_mem_start; |
| 420 | uword buffer_mem_size; |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame^] | 421 | vlib_buffer_pool_t *buffer_pools; |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 422 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 423 | /* Buffer free callback, for subversive activities */ |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 424 | u32 (*buffer_free_callback) (struct vlib_main_t * vm, |
| 425 | u32 * buffers, |
| 426 | u32 n_buffers, u32 follow_buffer_next); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 427 | /* Pool of buffer free lists. |
| 428 | Multiple free lists exist for packet generator which uses |
| 429 | separate free lists for each packet stream --- so as to avoid |
| 430 | initializing static data for each packet generated. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 431 | vlib_buffer_free_list_t *buffer_free_list_pool; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 432 | #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0) |
Damjan Marion | 1901020 | 2016-03-24 17:17:47 +0100 | [diff] [blame] | 433 | #define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 434 | |
| 435 | /* Hash table mapping buffer size (rounded to next unit of |
| 436 | sizeof (vlib_buffer_t)) to free list index. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 437 | uword *free_list_by_size; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 438 | |
| 439 | /* Hash table mapping buffer index into number |
| 440 | 0 => allocated but free, 1 => allocated and not-free. |
| 441 | If buffer index is not in hash table then this buffer |
| 442 | has never been allocated. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 443 | uword *buffer_known_hash; |
Damjan Marion | 6b0f589 | 2017-07-27 04:01:24 -0400 | [diff] [blame] | 444 | clib_spinlock_t buffer_known_hash_lockp; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 445 | |
| 446 | /* List of free-lists needing Blue Light Special announcements */ |
| 447 | vlib_buffer_free_list_t **announce_list; |
| 448 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 449 | /* Callbacks */ |
| 450 | vlib_buffer_callbacks_t cb; |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 451 | int callbacks_registered; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 452 | } vlib_buffer_main_t; |
| 453 | |
Damjan Marion | 149ba77 | 2017-10-12 13:09:26 +0200 | [diff] [blame^] | 454 | u8 vlib_buffer_add_physmem_region (struct vlib_main_t *vm, |
| 455 | vlib_physmem_region_index_t region); |
| 456 | |
Damjan Marion | 49d66f1 | 2017-07-20 18:10:35 +0200 | [diff] [blame] | 457 | clib_error_t *vlib_buffer_main_init (struct vlib_main_t *vm); |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 458 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 459 | typedef struct |
| 460 | { |
| 461 | struct vlib_main_t *vlib_main; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 462 | |
| 463 | u32 first_buffer, last_buffer; |
| 464 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 465 | union |
| 466 | { |
| 467 | struct |
| 468 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 469 | /* Total accumulated bytes in chain starting with first_buffer. */ |
| 470 | u32 n_total_data_bytes; |
| 471 | |
| 472 | /* Max number of bytes to accumulate in chain starting with first_buffer. |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 473 | As this limit is reached buffers are enqueued to next node. */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 474 | u32 max_n_data_bytes_per_chain; |
| 475 | |
| 476 | /* Next node to enqueue buffers to relative to current process node. */ |
| 477 | u32 next_index; |
| 478 | |
| 479 | /* Free list to use to allocate new buffers. */ |
| 480 | u32 free_list_index; |
| 481 | } tx; |
| 482 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 483 | struct |
| 484 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 485 | /* CLIB fifo of buffer indices waiting to be unserialized. */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 486 | u32 *buffer_fifo; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 487 | |
| 488 | /* Event type used to signal that RX buffers have been added to fifo. */ |
| 489 | uword ready_one_time_event; |
| 490 | } rx; |
| 491 | }; |
| 492 | } vlib_serialize_buffer_main_t; |
| 493 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 494 | void serialize_open_vlib_buffer (serialize_main_t * m, struct vlib_main_t *vm, |
| 495 | vlib_serialize_buffer_main_t * sm); |
| 496 | void unserialize_open_vlib_buffer (serialize_main_t * m, |
| 497 | struct vlib_main_t *vm, |
| 498 | vlib_serialize_buffer_main_t * sm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 499 | |
| 500 | u32 serialize_close_vlib_buffer (serialize_main_t * m); |
| 501 | void unserialize_close_vlib_buffer (serialize_main_t * m); |
| 502 | void *vlib_set_buffer_free_callback (struct vlib_main_t *vm, void *fp); |
| 503 | |
| 504 | always_inline u32 |
| 505 | serialize_vlib_buffer_n_bytes (serialize_main_t * m) |
| 506 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 507 | serialize_stream_t *s = &m->stream; |
| 508 | vlib_serialize_buffer_main_t *sm |
| 509 | = uword_to_pointer (m->stream.data_function_opaque, |
| 510 | vlib_serialize_buffer_main_t *); |
| 511 | return sm->tx.n_total_data_bytes + s->current_buffer_index + |
| 512 | vec_len (s->overflow_buffer); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 513 | } |
| 514 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 515 | /* |
| 516 | */ |
| 517 | |
| 518 | /** \brief Compile time buffer trajectory tracing option |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 519 | Turn this on if you run into "bad monkey" contexts, |
| 520 | and you want to know exactly which nodes they've visited... |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 521 | See vlib/main.c... |
| 522 | */ |
| 523 | #define VLIB_BUFFER_TRACE_TRAJECTORY 0 |
| 524 | |
| 525 | #if VLIB_BUFFER_TRACE_TRAJECTORY > 0 |
Dave Barach | 7bee773 | 2017-10-18 18:48:11 -0400 | [diff] [blame] | 526 | extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index); |
| 527 | extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b); |
| 528 | extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b); |
| 529 | #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \ |
| 530 | vlib_buffer_trace_trajectory_init (b); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 531 | #else |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 532 | #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) |
| 533 | #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */ |
| 534 | |
| 535 | #endif /* included_vlib_buffer_h */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 536 | |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 537 | #define VLIB_BUFFER_REGISTER_CALLBACKS(x,...) \ |
| 538 | __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks; \ |
| 539 | static void __vlib_add_buffer_callbacks_t_##x (void) \ |
| 540 | __attribute__((__constructor__)) ; \ |
| 541 | static void __vlib_add_buffer_callbacks_t_##x (void) \ |
| 542 | { \ |
| 543 | if (vlib_buffer_callbacks) \ |
| 544 | clib_panic ("vlib buffer callbacks already registered"); \ |
| 545 | vlib_buffer_callbacks = &__##x##_buffer_callbacks; \ |
| 546 | } \ |
| 547 | __VA_ARGS__ vlib_buffer_callbacks_t __##x##_buffer_callbacks |
| 548 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 549 | /* |
| 550 | * fd.io coding-style-patch-verification: ON |
| 551 | * |
| 552 | * Local Variables: |
| 553 | * eval: (c-set-style "gnu") |
| 554 | * End: |
| 555 | */ |