Luis Farias | 9d66fca | 2020-05-28 19:01:58 -0700 | [diff] [blame] | 1 | /****************************************************************************** |
| 2 | * |
| 3 | * Copyright (c) 2019 Intel. |
| 4 | * |
| 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at |
| 8 | * |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | * |
| 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
| 16 | * |
| 17 | *******************************************************************************/ |
| 18 | |
| 19 | #ifndef __WLS_H__ |
| 20 | #define __WLS_H__ |
| 21 | |
| 22 | #ifdef __KERNEL__ |
| 23 | #include <linux/types.h> |
| 24 | #include <linux/module.h> |
| 25 | #include <linux/device.h> |
| 26 | #include <linux/ioctl.h> |
| 27 | #include <linux/cdev.h> |
| 28 | #include <linux/wait.h> |
| 29 | #include <linux/sched.h> |
| 30 | #define MODNAME (KBUILD_MODNAME) |
| 31 | #else /* __KERNEL__ */ |
| 32 | #include <sys/ioctl.h> |
| 33 | #include <stdint.h> |
| 34 | |
| 35 | #ifdef DPDK_WLS |
| 36 | #include <semaphore.h> |
| 37 | #include <rte_common.h> |
| 38 | #include <rte_atomic.h> |
| 39 | #include <rte_memzone.h> |
| 40 | #endif /* DPDK_WLS */ |
| 41 | |
| 42 | #endif |
| 43 | #include "ttypes.h" |
| 44 | #include "syslib.h" |
| 45 | |
| 46 | #define WLS_PRINT(format, args...) printk(format, ##args) |
| 47 | #define WLS_ERROR(format, args...) printk(KERN_ERR "wls err: " format,##args) |
| 48 | |
| 49 | #ifdef _DEBUG_ |
| 50 | #define WLS_DEBUG(format, args...) \ |
| 51 | do { \ |
| 52 | printk(KERN_INFO "wls debug: " format,##args); \ |
| 53 | }while(0) |
| 54 | #else /*_DEBUG_*/ |
| 55 | #define WLS_DEBUG(format, args...) do { } while(0) |
| 56 | #endif /*_DEBUG_*/ |
| 57 | |
| 58 | /****************************************************************************** |
| 59 | * Module error codes * |
| 60 | ******************************************************************************/ |
| 61 | #define WLS_RC_MDMA_ID_ERROR (-1) |
| 62 | #define WLS_RC_MDMA_TASK_ERROR (-2) |
| 63 | #define WLS_RC_ALLOC_DELAY_MEM_ERROR (-3) |
| 64 | #define WLS_RC_ALLOC_BAR_MEM_ERROR (-4) |
| 65 | #define WLS_RC_ALLOC_TAR_MEM_ERROR (-5) |
| 66 | #define WLS_RC_PARAM_SIZE_ERROR (-6) |
| 67 | #define WLS_RC_WLS_HEAP_ALLOC_ERROR (-7) |
| 68 | #define WLS_RC_IRQ_ALLOC_ERROR (-8) |
| 69 | #define WLS_RC_DMA_ALLOC_ERROR (-9) |
| 70 | #define WLS_RC_TRANSACTION_ERROR (-10) |
| 71 | #define WLS_RC_PHY_CTX_ERROR (-11) |
| 72 | #define WLS_RC_KERNEL_HEAP_ALLOC_ERROR (-12) |
| 73 | #define WLS_RC_CONFIGURATION_ERROR (-13) |
| 74 | #define WLS_RC_THREAD_CREATION_ERROR (-14) |
| 75 | |
| 76 | #define WLS_IOC_MAGIC 'W' |
| 77 | #define WLS_IOC_OPEN _IOWR(WLS_IOC_MAGIC, WLS_IOC_OPEN_NO, uint64_t) |
| 78 | #define WLS_IOC_CLOSE _IOWR(WLS_IOC_MAGIC, WLS_IOC_CLOSE_NO, uint64_t) |
| 79 | #define WLS_IOC_PUT _IOWR(WLS_IOC_MAGIC, WLS_IOC_PUT_NO, uint64_t) |
| 80 | #define WLS_IOC_EVENT _IOWR(WLS_IOC_MAGIC, WLS_IOC_EVENT_NO, uint64_t) |
| 81 | #define WLS_IOC_WAIT _IOWR(WLS_IOC_MAGIC, WLS_IOC_WAIT_NO, uint64_t) |
| 82 | #define WLS_IOC_WAKE_UP _IOWR(WLS_IOC_MAGIC, WLS_IOC_WAKE_UP_NO, uint64_t) |
| 83 | #define WLS_IOC_CONNECT _IOWR(WLS_IOC_MAGIC, WLS_IOC_CONNECT_NO, uint64_t) |
| 84 | #define WLS_IOC_FILL _IOWR(WLS_IOC_MAGIC, WLS_IOC_FILL_NO, uint64_t) |
| 85 | |
| 86 | enum { |
| 87 | WLS_IOC_OPEN_NO = 1, |
| 88 | WLS_IOC_CLOSE_NO, |
| 89 | WLS_IOC_PUT_NO, |
| 90 | WLS_IOC_EVENT_NO, |
| 91 | WLS_IOC_WAIT_NO, |
| 92 | WLS_IOC_WAKE_UP_NO, |
| 93 | WLS_IOC_CONNECT_NO, |
| 94 | WLS_IOC_FILL_NO, |
| 95 | WLS_IOC_COUNT, |
| 96 | }; |
| 97 | |
| 98 | enum { |
| 99 | WLS_FILL_PUSH = 1, |
| 100 | WLS_FILL_PULL, |
| 101 | WLS_FILL_MAX, |
| 102 | }; |
| 103 | |
| 104 | |
| 105 | #define WLS_US_CLIENTS_MAX 64 |
| 106 | |
| 107 | #define CACHE_LINE_SIZE 64 /**< Cache line size. */ |
| 108 | #define CACHE_LINE_MASK (CACHE_LINE_SIZE-1) /**< Cache line mask. */ |
| 109 | |
| 110 | #define CACHE_LINE_ROUNDUP(size) \ |
| 111 | (CACHE_LINE_SIZE * ((size + CACHE_LINE_SIZE - 1) / CACHE_LINE_SIZE)) |
| 112 | |
| 113 | #define DMA_ALIGNMENT_SIZE 256L |
| 114 | |
| 115 | // To make DMA we make sure that block starts on 256 bytes boundary |
| 116 | #define DMA_ALIGNMENT_ROUNDUP(size) \ |
| 117 | (DMA_ALIGNMENT_SIZE * ((size + DMA_ALIGNMENT_SIZE - 1) / DMA_ALIGNMENT_SIZE)) |
| 118 | |
| 119 | /**< Return the first cache-aligned value greater or equal to size. */ |
| 120 | |
| 121 | /** |
| 122 | * Force alignment to cache line. |
| 123 | */ |
| 124 | #define __wls_cache_aligned __attribute__((__aligned__(CACHE_LINE_SIZE))) |
| 125 | |
| 126 | #define WLS_HUGE_DEF_PAGE_SIZE 0x40000000LL |
| 127 | #define WLS_IS_ONE_HUGE_PAGE(ptr, size, hp_size) ((((unsigned long long)ptr & (~(hp_size - 1)))\ |
| 128 | == (((unsigned long long)ptr + size - 1) & (~(hp_size - 1)))) ? 1 : 0) |
| 129 | |
| 130 | typedef struct hugepage_tabl_s |
| 131 | { |
| 132 | union { |
| 133 | void *pageVa; |
| 134 | uint64_t padding_pageVa; |
| 135 | }; |
| 136 | uint64_t pagePa; |
| 137 | }hugepage_tabl_t; |
| 138 | |
| 139 | #define DMA_MAP_MAX_BLOCK_SIZE 64*1024 |
| 140 | #define MAX_N_HUGE_PAGES 512 |
| 141 | #define UL_FREE_BLOCK_QUEUE_SIZE 384 |
| 142 | |
| 143 | #define WLS_GET_QUEUE_N_ELEMENTS 384 |
| 144 | #define WLS_PUT_QUEUE_N_ELEMENTS 384 |
| 145 | |
| 146 | #ifdef DPDK_WLS |
| 147 | #define WLS_DEV_SHM_NAME_LEN RTE_MEMZONE_NAMESIZE |
| 148 | #else |
| 149 | #define WLS_DEV_SHM_NAME_LEN 256 |
| 150 | #endif |
| 151 | |
| 152 | #define FIFO_LEN 384 |
| 153 | |
| 154 | typedef struct wls_wait_req_s { |
| 155 | uint64_t wls_us_kernel_va; |
| 156 | uint64_t start_time; |
| 157 | uint64_t ctx; |
| 158 | uint64_t action; |
| 159 | uint64_t nMsg; |
| 160 | }wls_wait_req_t; |
| 161 | |
| 162 | #ifdef DPDK_WLS |
| 163 | typedef struct wls_sema_priv_s |
| 164 | { |
| 165 | sem_t sem; |
| 166 | rte_atomic16_t is_irq; |
| 167 | wls_wait_req_t drv_block[FIFO_LEN]; |
| 168 | volatile unsigned int drv_block_put; |
| 169 | volatile unsigned int drv_block_get; |
| 170 | } wls_sema_priv_t; |
| 171 | |
| 172 | typedef struct wls_us_priv_s |
| 173 | { |
| 174 | wls_sema_priv_t sema; |
| 175 | U8 NeedToWakeUp; |
| 176 | U8 isWait; |
| 177 | volatile V32 pid; |
| 178 | } wls_us_priv_t; |
| 179 | |
| 180 | #endif |
| 181 | |
| 182 | typedef struct wls_us_ctx_s |
| 183 | { |
| 184 | union { |
| 185 | void * wls_us_user_space_va; |
| 186 | uint64_t padding_wls_us_user_space_va; |
| 187 | }; |
| 188 | |
| 189 | uint64_t wls_us_kernel_va; |
| 190 | |
| 191 | uint64_t wls_us_pa; |
| 192 | |
| 193 | uint32_t wls_us_ctx_size; |
| 194 | uint32_t HugePageSize; |
| 195 | |
| 196 | union { |
| 197 | void* alloc_buffer; |
| 198 | uint64_t padding_alloc_buffer; |
| 199 | }; |
| 200 | |
| 201 | hugepage_tabl_t hugepageTbl [MAX_N_HUGE_PAGES]; |
| 202 | |
| 203 | FASTQUEUE ul_free_block_pq; |
| 204 | uint64_t ul_free_block_storage[UL_FREE_BLOCK_QUEUE_SIZE * sizeof(uint64_t)]; |
| 205 | |
| 206 | WLS_MSG_QUEUE get_queue; |
| 207 | WLS_MSG_HANDLE get_storage[WLS_GET_QUEUE_N_ELEMENTS]; |
| 208 | |
| 209 | WLS_MSG_QUEUE put_queue; |
| 210 | WLS_MSG_HANDLE put_storage[WLS_PUT_QUEUE_N_ELEMENTS]; |
| 211 | |
| 212 | uint64_t freePtrList[UL_FREE_BLOCK_QUEUE_SIZE * sizeof(uint64_t)]; |
| 213 | uint32_t freeListIndex; |
| 214 | uint32_t dualMode; |
| 215 | |
| 216 | // dst userspace context address (kernel va) |
| 217 | uint64_t dst_kernel_va; |
| 218 | // dst userspace context address (local user sapce va) |
| 219 | |
| 220 | volatile uint64_t dst_user_va; |
| 221 | // dst userspace context address (local user sapce va) |
| 222 | volatile uint64_t dst_pa; |
| 223 | |
| 224 | uint32_t alloc_size; |
| 225 | #ifdef DPDK_WLS |
| 226 | wls_us_priv_t wls_us_private; |
| 227 | #else |
| 228 | HANDLE wls_us_private; |
| 229 | #endif |
| 230 | uint32_t mode; |
| 231 | uint32_t secmode; |
| 232 | char wls_dev_name[WLS_DEV_SHM_NAME_LEN]; |
| 233 | char wls_shm_name[WLS_DEV_SHM_NAME_LEN]; |
| 234 | }wls_us_ctx_t; |
| 235 | |
| 236 | |
| 237 | |
| 238 | typedef struct wls_fill_req_s { |
| 239 | uint64_t wls_us_kernel_va; |
| 240 | uint64_t ctx; |
| 241 | uint64_t action; |
| 242 | uint64_t nMsg; |
| 243 | }wls_fill_req_t; |
| 244 | |
| 245 | typedef struct wls_connect_req_s { |
| 246 | uint64_t wls_us_kernel_va; |
| 247 | }wls_connect_req_t; |
| 248 | |
| 249 | #ifdef __KERNEL__ |
| 250 | |
| 251 | typedef struct wls_sema_priv_s |
| 252 | { |
| 253 | wait_queue_head_t queue; |
| 254 | atomic_t is_irq; |
| 255 | wls_wait_req_t drv_block[FIFO_LEN]; |
| 256 | volatile unsigned int drv_block_put; |
| 257 | volatile unsigned int drv_block_get; |
| 258 | }wls_sema_priv_t; |
| 259 | |
| 260 | typedef struct wls_drv_ctx_s |
| 261 | { |
| 262 | uint32_t init_mask; |
| 263 | uint32_t us_ctx_cout; |
| 264 | wls_us_ctx_t* p_wls_us_ctx[WLS_US_CLIENTS_MAX]; |
| 265 | wls_us_ctx_t* p_wls_us_pa_ctx[WLS_US_CLIENTS_MAX]; |
| 266 | uint32_t nWlsClients; |
| 267 | }wls_drv_ctx_t; |
| 268 | |
| 269 | #elif defined DPDK_WLS |
| 270 | |
| 271 | typedef struct wls_drv_ctx_s |
| 272 | { |
| 273 | uint32_t init_mask; |
| 274 | uint32_t us_ctx_cout; |
| 275 | wls_us_ctx_t p_wls_us_ctx[WLS_US_CLIENTS_MAX]; |
| 276 | wls_us_ctx_t p_wls_us_pa_ctx[WLS_US_CLIENTS_MAX]; |
| 277 | uint32_t nWlsClients; |
| 278 | pthread_mutex_t mng_mutex; |
| 279 | }wls_drv_ctx_t; |
| 280 | |
| 281 | #endif |
| 282 | typedef struct wls_open_req_s { |
| 283 | uint64_t ctx; |
| 284 | uint64_t ctx_pa; |
| 285 | uint32_t size; |
| 286 | }wls_open_req_t; |
| 287 | |
| 288 | typedef struct wls_close_req_s { |
| 289 | uint64_t ctx; |
| 290 | uint64_t ctx_pa; |
| 291 | uint32_t size; |
| 292 | }wls_close_req_t; |
| 293 | |
| 294 | typedef enum wls_events_num_s { |
| 295 | WLS_EVENT_IA_READY = 0, |
| 296 | WLS_EVENT_IA_STOP, |
| 297 | WLS_EVENT_IA_ERROR, |
| 298 | WLS_EVENT_MAX |
| 299 | }wls_events_num_t; |
| 300 | |
| 301 | typedef struct wls_event_req_s { |
| 302 | uint64_t wls_us_kernel_va; |
| 303 | uint64_t event_to_wls; |
| 304 | uint64_t event_param; |
| 305 | }wls_event_req_t; |
| 306 | |
| 307 | typedef struct wls_put_req_s { |
| 308 | uint64_t wls_us_kernel_va; |
| 309 | }wls_put_req_t; |
| 310 | |
| 311 | typedef struct wls_wake_up_req_s { |
| 312 | uint64_t wls_us_kernel_va; |
| 313 | uint32_t id; |
| 314 | uint64_t ctx; |
| 315 | }wls_wake_up_req_t; |
| 316 | |
| 317 | |
| 318 | #define SYS_CPU_CLOCK (2300000000L) |
| 319 | #define CLOCK_PER_MS (SYS_CPU_CLOCK/1000) |
| 320 | #define CLOCK_PER_US (SYS_CPU_CLOCK/1000000) |
| 321 | |
| 322 | static inline uint64_t |
| 323 | wls_rdtsc(void) |
| 324 | { |
| 325 | union { |
| 326 | uint64_t tsc_64; |
| 327 | struct { |
| 328 | uint32_t lo_32; |
| 329 | uint32_t hi_32; |
| 330 | }; |
| 331 | } tsc; |
| 332 | |
| 333 | asm volatile("rdtsc" : |
| 334 | "=a" (tsc.lo_32), |
| 335 | "=d" (tsc.hi_32)); |
| 336 | return tsc.tsc_64; |
| 337 | } |
| 338 | |
| 339 | static inline uint64_t rdtsc_ticks_diff(unsigned long curr, unsigned long prev) |
| 340 | { |
| 341 | if (curr >= prev) |
| 342 | return (unsigned long)(curr - prev); |
| 343 | else |
| 344 | return (unsigned long)(0xFFFFFFFFFFFFFFFF - prev + curr); |
| 345 | } |
| 346 | |
| 347 | void wls_show_data(void* ptr, unsigned int size); |
| 348 | void *wls_get_sh_ctx(void); |
| 349 | void *wls_get_sh_ctx_pa(void); |
| 350 | |
| 351 | #endif /* __WLS_H__*/ |
| 352 | |