| /* Copyright (c) 2015 Google Inc |
| * Davide Libenzi <dlibenzi@google.com> |
| * See LICENSE for details. |
| * |
| * Static percpu variables: |
| * |
| * The per CPU utility macros allow file local declaration of per CPU variables. |
| * When a struct my_struct needs to have a per CPU instance, one would declare |
| * something like: |
| * |
| * static DEFINE_PERCPU(struct my_struct, my_data); |
| * |
| * The per CPU data can then be accessed with either of those forms: |
| * |
| * struct my_struct *ptr = PERCPU_VARPTR(my_data); |
| * PERCPU_VAR(my_data).field = 17; |
| * |
| * When the per CPU data has complex initialization, it is possible to register |
| * functions which will be called immediately after the per CPU data is created: |
| * |
| * DEFINE_PERCPU_INIT(my_init); |
| * |
| * Then the my_init() function would just: |
| * |
| * static void my_init(void) |
| * { |
| * for (int i = 0; i < num_cores; i++) { |
| * struct my_struct *ptr = _PERCPU_VARPTR(my_data, i); |
| * |
| * // Initialize ptr data |
| * } |
| * } |
| * |
| * |
| * Dynamic percpu variables: |
| * |
| * You can also declare per-cpu variables dynamically, though it's not quite the |
| * same as the static variables. Careful - We return *pointers*, and our users |
| * need to dereference them when using any of the PERCPU_ helpers. |
| * |
| * Example (per core u64s) Note each *use* dereferences 'foos': |
| * |
| * uint64_t *foos = percpu_zalloc(uint64_t, MEM_WAIT); |
| * |
| * // Each core increments |
| * PERCPU_VAR(*foos)++; |
| * |
| * // One core can print them all out |
| * for_each_core(i) |
| * printk("Addr %p, value %lu\n", _PERCPU_VARPTR(*foos, i), |
| * _PERCPU_VAR(*foos, i)); |
| * |
| * // Free, but don't deref here. 'foos' is your handle. |
| * percpu_free(foos); |
| */ |
| |
| #pragma once |
| |
| #include <sys/types.h> |
| #include <arch/topology.h> |
| #include <ros/common.h> |
| |
| #define PERCPU_SECTION __percpu |
| #define PERCPU_SECTION_STR STRINGIFY(PERCPU_SECTION) |
| |
| #define PERCPU_START_VAR PASTE(__start_, PERCPU_SECTION) |
| #define PERCPU_STOP_VAR PASTE(__stop_, PERCPU_SECTION) |
| |
| #define PERCPU_DYN_SIZE 1024 |
| #define PERCPU_STATIC_SIZE (PERCPU_STOP_VAR - PERCPU_START_VAR) |
| #define PERCPU_SIZE (PERCPU_STATIC_SIZE + PERCPU_DYN_SIZE) |
| #define PERCPU_OFFSET(var) ((char *) &(var) - PERCPU_START_VAR) |
| |
| #define __PERCPU_VARPTR(var, cpu) \ |
| ({ \ |
| typeof(var) *__cv; \ |
| if (likely(percpu_base)) \ |
| __cv = (typeof(var) *) (percpu_base + cpu * PERCPU_SIZE + \ |
| PERCPU_OFFSET(var)); \ |
| else \ |
| __cv = &var; \ |
| __cv; \ |
| }) |
| #define _PERCPU_VARPTR(var, cpu) __PERCPU_VARPTR(var, cpu) |
| #define PERCPU_VARPTR(var) __PERCPU_VARPTR(var, core_id()) |
| |
| #define _PERCPU_VAR(var, cpu) (*__PERCPU_VARPTR(var, cpu)) |
| #define PERCPU_VAR(var) (*__PERCPU_VARPTR(var, core_id())) |
| |
| #define DEFINE_PERCPU(type, var) \ |
| __typeof__(type) var __attribute__ ((section (PERCPU_SECTION_STR))) |
| #define DECLARE_PERCPU(type, var) \ |
| extern __typeof__(type) var \ |
| __attribute__ ((section (PERCPU_SECTION_STR))) |
| |
| #define PERCPU_INIT_SECTION __percpu_init |
| #define PERCPU_INIT_SECTION_STR STRINGIFY(PERCPU_INIT_SECTION) |
| |
| #define PERCPU_INIT_START_VAR PASTE(__start_, PERCPU_INIT_SECTION) |
| #define PERCPU_INIT_STOP_VAR PASTE(__stop_, PERCPU_INIT_SECTION) |
| |
| #define PERCPU_INIT_NAME(func) PASTE(__percpu_, func) |
| #define DEFINE_PERCPU_INIT(func) \ |
| static void func(void); \ |
| void (* const PERCPU_INIT_NAME(func))(void) \ |
| __attribute__ ((section (PERCPU_INIT_SECTION_STR))) = (func) |
| |
| extern char __attribute__((weak)) PERCPU_START_VAR[]; |
| extern char __attribute__((weak)) PERCPU_STOP_VAR[]; |
| extern char *percpu_base; |
| |
| void percpu_init(void); |
| |
| #define percpu_alloc(x, flags) __percpu_alloc(sizeof(x), __alignof__(x), flags) |
| #define percpu_zalloc(x, flags) __percpu_zalloc(sizeof(x), __alignof__(x), \ |
| flags) |
| #define percpu_free(x) __percpu_free(x, sizeof(*x)) |
| |
| void *__percpu_alloc(size_t size, size_t align, int flags); |
| void *__percpu_zalloc(size_t size, size_t align, int flags); |
| void __percpu_free(void *base, size_t size); |