blob: 2adf8ef632ec51a8ce0d7977f167ff81a28a2f77 [file] [log] [blame]
#include <parlib/arch/arch.h>
#include <stdbool.h>
#include <errno.h>
#include <parlib/vcore.h>
#include <parlib/mcs.h>
#include <sys/param.h>
#include <parlib/parlib.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <stdio.h>
#include <parlib/event.h>
#include <parlib/uthread.h>
#include <parlib/ucq.h>
#include <ros/arch/membar.h>
#include <parlib/printf-ext.h>
#include <parlib/poke.h>
#include <parlib/assert.h>
#include <parlib/stdio.h>
__thread int __vcoreid = 0;
__thread bool __vcore_context = FALSE;
__thread struct syscall __vcore_one_sysc = {.flags = (atomic_t)SC_DONE, 0};
/* Per vcore entery function used when reentering at the top of a vcore's stack */
static __thread void (*__vcore_reentry_func)(void) = NULL;
/* The default user vcore_entry function. */
void __attribute__((noreturn)) __vcore_entry(void)
{
extern void uthread_vcore_entry(void);
uthread_vcore_entry();
fprintf(stderr, "vcore_entry() should never return!\n");
abort();
__builtin_unreachable();
}
void vcore_entry(void) __attribute__((weak, alias ("__vcore_entry")));
/* TODO: probably don't want to dealloc. Considering caching */
static void free_transition_tls(int id)
{
if (get_vcpd_tls_desc(id)) {
/* Note we briefly have no TLS desc in VCPD. This is fine so
* long as that vcore doesn't get started fresh before we put in
* a new desc */
free_tls(get_vcpd_tls_desc(id));
set_vcpd_tls_desc(id, NULL);
}
}
static int allocate_transition_tls(int id)
{
/* Libc function to initialize TLS-based locale info for ctype
* functions. */
extern void __ctype_init(void);
/* We want to free and then reallocate the tls rather than simply
* reinitializing it because its size may have changed. TODO: not sure
* if this is right. 0-ing is one thing, but freeing and reallocating
* can be expensive, esp if syscalls are involved. Check out glibc's
* allocatestack.c for what might work. */
free_transition_tls(id);
void *tcb = allocate_tls();
if (!tcb) {
errno = ENOMEM;
return -1;
}
/* Setup some intitial TLS data for the newly allocated transition tls.
*/
void *temp_tcb = get_tls_desc();
set_tls_desc(tcb);
begin_safe_access_tls_vars();
__vcoreid = id;
__vcore_context = TRUE;
__ctype_init();
end_safe_access_tls_vars();
set_tls_desc(temp_tcb);
/* Install the new tls into the vcpd. */
set_vcpd_tls_desc(id, tcb);
return 0;
}
static void free_vcore_stack(int id)
{
// don't actually free stacks
}
static int allocate_vcore_stack(int id)
{
struct preempt_data *vcpd = vcpd_of(id);
if (vcpd->vcore_stack)
return 0; // reuse old stack
void* stackbot = mmap(0, TRANSITION_STACK_SIZE,
PROT_READ | PROT_WRITE | PROT_EXEC,
MAP_POPULATE | MAP_ANONYMOUS | MAP_PRIVATE, -1,
0);
if (stackbot == MAP_FAILED)
return -1; // errno set by mmap
vcpd->vcore_stack = (uintptr_t)stackbot + TRANSITION_STACK_SIZE;
return 0;
}
/* Helper: prepares a vcore for use. Takes a block of pages for the UCQs.
*
* Vcores need certain things, such as a stack and TLS. These are determined by
* userspace. Every vcore needs these set up before we drop into vcore context
* on that vcore. This means we need to prep before asking the kernel for those
* vcores.
*
* We could have this function do its own mmap, at the expense of O(n) syscalls
* when we prepare the extra vcores. */
static void __prep_vcore(int vcoreid, uintptr_t mmap_block)
{
struct preempt_data *vcpd = vcpd_of(vcoreid);
int ret;
ret = allocate_vcore_stack(vcoreid);
assert(!ret);
ret = allocate_transition_tls(vcoreid);
assert(!ret);
vcpd->ev_mbox_public.type = EV_MBOX_UCQ;
ucq_init_raw(&vcpd->ev_mbox_public.ucq,
mmap_block + 0 * PGSIZE,
mmap_block + 1 * PGSIZE);
vcpd->ev_mbox_private.type = EV_MBOX_UCQ;
ucq_init_raw(&vcpd->ev_mbox_private.ucq,
mmap_block + 2 * PGSIZE,
mmap_block + 3 * PGSIZE);
/* Set the lowest level entry point for each vcore. */
vcpd->vcore_entry = (uintptr_t)__kernel_vcore_entry;
}
static void prep_vcore_0(void)
{
uintptr_t mmap_block;
mmap_block = (uintptr_t)mmap(0, PGSIZE * 4,
PROT_WRITE | PROT_READ,
MAP_POPULATE | MAP_ANONYMOUS | MAP_PRIVATE,
-1, 0);
assert((void*)mmap_block != MAP_FAILED);
__prep_vcore(0, mmap_block);
}
static void prep_remaining_vcores(void)
{
uintptr_t mmap_block;
mmap_block = (uintptr_t)mmap(0, PGSIZE * 4 * (max_vcores() - 1),
PROT_WRITE | PROT_READ,
MAP_POPULATE | MAP_ANONYMOUS | MAP_PRIVATE,
-1, 0);
assert((void*)mmap_block != MAP_FAILED);
for (int i = 1; i < max_vcores(); i++)
__prep_vcore(i, mmap_block + 4 * (i - 1) * PGSIZE);
}
/* Run libc specific early setup code. */
static void vcore_libc_init(void)
{
register_printf_specifier('r', printf_errstr, printf_errstr_info);
/* TODO: register for other kevents/signals and whatnot (can probably
* reuse the simple ev_q). Could also do this via explicit functions
* from the program. */
}
/* We need to separate the guts of vcore_lib_ctor() into a separate function,
* since the uthread ctor depends on this ctor running first.
*
* Also note that if you make a global ctor (not static, like this used to be),
* any shared objects that you load when the binary is built with -rdynamic will
* run the global ctor from the binary, not the one from the .so. */
void vcore_lib_init(void)
{
/* Note this is racy, but okay. The first time through, we are _S.
* Also, this is the "lowest" level constructor for now, so we don't
* need to call any other init functions after our run_once() call. This
* may change in the future. */
parlib_init_once_racy(return);
/* Need to alloc vcore0's transition stuff here (technically, just the
* TLS) so that schedulers can use vcore0's transition TLS before it
* comes up in vcore_entry() */
prep_vcore_0();
assert(!in_vcore_context());
vcore_libc_init();
}
static void __attribute__((constructor)) vcore_lib_ctor(void)
{
if (__in_fake_parlib())
return;
vcore_lib_init();
}
/* Helper functions used to reenter at the top of a vcore's stack for an
* arbitrary function */
static void __attribute__((noinline, noreturn)) __vcore_reenter()
{
__vcore_reentry_func();
assert(0);
}
void vcore_reenter(void (*entry_func)(void))
{
assert(in_vcore_context());
struct preempt_data *vcpd = vcpd_of(vcore_id());
__vcore_reentry_func = entry_func;
set_stack_pointer((void*)vcpd->vcore_stack);
cmb();
__vcore_reenter();
}
/* Helper, picks some sane defaults and changes the process into an MCP */
void vcore_change_to_m(void)
{
int ret;
prep_remaining_vcores();
__procdata.res_req[RES_CORES].amt_wanted = 1;
__procdata.res_req[RES_CORES].amt_wanted_min = 1; /* whatever */
assert(!in_multi_mode());
assert(!in_vcore_context());
ret = sys_change_to_m();
assert(!ret);
assert(in_multi_mode());
assert(!in_vcore_context());
}
static void __vc_req_poke(void *nr_vc_wanted)
{
long nr_vcores_wanted = *(long*)nr_vc_wanted;
/* We init'd up to max_vcores() VCs during init. This assumes the
* kernel doesn't magically change that value (which it should not do).
* */
nr_vcores_wanted = MIN(nr_vcores_wanted, max_vcores());
if (nr_vcores_wanted > __procdata.res_req[RES_CORES].amt_wanted)
__procdata.res_req[RES_CORES].amt_wanted = nr_vcores_wanted;
if (nr_vcores_wanted > num_vcores())
sys_poke_ksched(0, RES_CORES); /* 0 -> poke for ourselves */
}
static struct poke_tracker vc_req_poke = POKE_INITIALIZER(__vc_req_poke);
/* Requests the kernel that we have a total of nr_vcores_wanted.
*
* This is callable by multiple threads/vcores concurrently. Exactly one of
* them will actually run __vc_req_poke. The others will just return.
*
* This means that two threads could ask for differing amounts, and only one of
* them will succeed. This is no different than a racy write to a shared
* variable. The poke provides a single-threaded environment, so that we don't
* worry about racing on VCPDs or hitting the kernel with excessive SYS_pokes.
*
* Since we're using the post-and-poke style, we can do a 'last write wins'
* policy for the value used in the poke (and subsequent pokes). */
void vcore_request_total(long nr_vcores_wanted)
{
static long nr_vc_wanted;
if (parlib_never_vc_request || !parlib_wants_to_be_mcp)
return;
if (nr_vcores_wanted == __procdata.res_req[RES_CORES].amt_wanted)
return;
/* We race to "post our work" here. Whoever handles the poke will get
* the latest value written here. */
nr_vc_wanted = nr_vcores_wanted;
poke(&vc_req_poke, &nr_vc_wanted);
}
/* This tries to get "more vcores", based on the number we currently have.
*
* What happens is we can have a bunch of threads trying to get "another vcore",
* which currently means more than num_vcores(). If you have someone ask for
* two more, and then someone else ask for one more, how many you ultimately ask
* for depends on if the kernel heard you and adjusted num_vcores in between the
* two calls. Or maybe your amt_wanted already was num_vcores + 5, so neither
* call is telling the kernel anything new. It comes down to "one more than I
* have" vs "one more than I've already asked for".
*
* So for now, this will keep the older behavior (one more than I have). This
* is all quite racy, so we can just guess and request a total number of vcores.
*/
void vcore_request_more(long nr_new_vcores)
{
vcore_request_total(nr_new_vcores + num_vcores());
}
/* This can return, if you failed to yield due to a concurrent event. Note
* we're atomicly setting the CAN_RCV flag, and aren't bothering with CASing
* (either with the kernel or uthread's handle_indirs()). We don't particularly
* care what other code does - we intend to set those flags no matter what. */
void vcore_yield(bool preempt_pending)
{
unsigned long old_nr;
uint32_t vcoreid = vcore_id();
struct preempt_data *vcpd = vcpd_of(vcoreid);
if (!preempt_pending && parlib_never_yield)
return;
__sync_fetch_and_and(&vcpd->flags, ~VC_CAN_RCV_MSG);
/* no wrmb() necessary, handle_events() has an mb() if it is checking */
/* Clears notif pending and tries to handle events. This is an
* optimization to avoid the yield syscall if we have an event pending.
* If there is one, we want to unwind and return to the 2LS loop, where
* we may not want to yield anymore.
*
* Note that the kernel only cares about CAN_RCV_MSG for the desired
* vcore; when spamming, it relies on membership of lists within the
* kernel. Look at spam_list_member() for more info (k/s/event.c). */
if (handle_events(vcoreid)) {
__sync_fetch_and_or(&vcpd->flags, VC_CAN_RCV_MSG);
return;
}
/* If we are yielding since we don't want the core, tell the kernel we
* want one less vcore (vc_yield assumes a dumb 2LS).
*
* If yield fails (slight race), we may end up having more vcores than
* amt_wanted for a while, and might lose one later on (after a
* preempt/timeslicing) - the 2LS will have to notice eventually if it
* actually needs more vcores (which it already needs to do).
* amt_wanted could even be 0.
*
* In general, any time userspace decrements or sets to 0, it could get
* preempted, so the kernel will still give us at least one, until the
* last vcore properly yields without missing a message (and becomes a
* WAITING proc, which the ksched will not give cores to).
*
* I think it's possible for userspace to do this (lock, read
* amt_wanted, check all message queues for all vcores, subtract
* amt_wanted (not set to 0), unlock) so long as every event handler +1s
* the amt wanted, but that's a huge pain, and we already have event
* handling code making sure a process can't sleep (transition to
* WAITING) if a message arrives (can't yield if notif_pending, can't go
* WAITING without yielding, and the event posting the notif_pending
* will find the online VC or be delayed by spinlock til the proc is
* WAITING). */
if (!preempt_pending) {
do {
old_nr = __procdata.res_req[RES_CORES].amt_wanted;
if (old_nr == 0)
break;
} while (!__sync_bool_compare_and_swap(
&__procdata.res_req[RES_CORES].amt_wanted,
old_nr, old_nr - 1));
}
/* We can probably yield. This may pop back up if notif_pending became
* set by the kernel after we cleared it and we lost the race. */
sys_yield(preempt_pending);
__sync_fetch_and_or(&vcpd->flags, VC_CAN_RCV_MSG);
}
/* Enables notifs, and deals with missed notifs by self notifying. This should
* be rare, so the syscall overhead isn't a big deal. The other alternative
* would be to uthread_yield(), which would require us to revert some uthread
* interface changes. */
void enable_notifs(uint32_t vcoreid)
{
__enable_notifs(vcoreid);
wrmb(); /* need to read after the write that enabled notifs */
/* Note we could get migrated before executing this. If that happens,
* our vcore had gone into vcore context (which is what we wanted), and
* this self_notify to our old vcore is spurious and harmless. */
if (vcpd_of(vcoreid)->notif_pending)
sys_self_notify(vcoreid, EV_NONE, 0, TRUE);
}
/* Helper to disable notifs. It simply checks to make sure we disabled uthread
* migration, which is a common mistake. */
void disable_notifs(uint32_t vcoreid)
{
if (!in_vcore_context() && current_uthread)
assert(current_uthread->flags & UTHREAD_DONT_MIGRATE);
__disable_notifs(vcoreid);
}
/* Like smp_idle(), this will put the core in a state that it can only be woken
* up by an IPI. For now, this is a halt. Maybe an mwait in the future.
*
* This will return if an event was pending (could be the one you were waiting
* for) or if the halt failed for some reason, such as a concurrent RKM. If
* successful, this will not return at all, and the vcore will restart from the
* top next time it wakes. Any sort of IRQ will wake the core.
*
* Alternatively, I might make this so it never returns, if that's easier to
* work with (similar issues with yield). */
void vcore_idle(void)
{
uint32_t vcoreid = vcore_id();
/* Once we enable notifs, the calling context will be treated like a
* uthread (saved into the uth slot). We don't want to ever run it
* again, so we need to make sure there's no cur_uth. */
assert(!current_uthread);
/* This clears notif_pending (check, signal, check again pattern). */
if (handle_events(vcoreid))
return;
/* This enables notifs, but also checks notif pending. At this point,
* any new notifs will restart the vcore from the top. */
enable_notifs(vcoreid);
/* From now, til we get into the kernel, any notifs will permanently
* destroy this context and start the VC from the top.
*
* Once we're in the kernel, any messages (__notify, __preempt), will be
* RKMs. halt will need to check for those atomically. Checking for
* notif_pending in the kernel (sleep only if not set) is not enough,
* since not all reasons for the kernel to stay awak set notif_pending
* (e.g., __preempts and __death).
*
* At this point, we're out of VC ctx, so anyone who sets notif_pending
* should also send an IPI / __notify */
sys_halt_core(0);
/* in case halt returns without actually restarting the VC ctx. */
disable_notifs(vcoreid);
}
/* Helper, that actually makes sure a vcore is running. Call this is you really
* want vcoreid. More often, you'll want to call the regular version. */
static void __ensure_vcore_runs(uint32_t vcoreid)
{
if (vcore_is_preempted(vcoreid)) {
printd("[vcore]: VC %d changing to VC %d\n", vcore_id(),
vcoreid);
/* Note that at this moment, the vcore could still be mapped
* (we're racing with __preempt. If that happens, we'll just
* fail the sys_change_vcore(), and next time __ensure runs
* we'll get it. */
/* We want to recover them from preemption. Since we know they
* have notifs disabled, they will need to be directly
* restarted, so we can skip the other logic and cut straight to
* the sys_change_vcore() */
sys_change_vcore(vcoreid, FALSE);
}
}
/* Helper, looks for any preempted vcores, making sure each of them runs at some
* point. This is pretty heavy-weight, and should be used to help get out of
* weird deadlocks (spinning in vcore context, waiting on another vcore). If
* you might know which vcore you are waiting on, use ensure_vc_runs. */
static void __ensure_all_run(void)
{
for (int i = 0; i < max_vcores(); i++)
__ensure_vcore_runs(i);
}
/* Makes sure a vcore is running. If it is preempted, we'll switch to
* it. This will return, either immediately if the vcore is running, or later
* when someone preempt-recovers us.
*
* If you pass in your own vcoreid, this will make sure all other preempted
* vcores run. */
void ensure_vcore_runs(uint32_t vcoreid)
{
/* if the vcoreid is ourselves, make sure everyone else is running */
if (vcoreid == vcore_id()) {
__ensure_all_run();
return;
}
__ensure_vcore_runs(vcoreid);
}
#define NR_RELAX_SPINS 1000
/* If you are spinning and waiting on another vcore, call this. Pass in the
* vcoreid of the core you are waiting on, or your own vcoreid if you don't
* know. It will spin for a bit before firing up the potentially expensive
* __ensure_all_run(). */
void cpu_relax_vc(uint32_t other_vcoreid)
{
static __thread unsigned int __vc_relax_spun = 0;
/* Uthreads with notifs enabled can just spin normally. This actually
* depends on the 2LS preemption policy. Currently, we receive notifs
* whenever another core is preempted, so we don't need to poll. */
if (notif_is_enabled(vcore_id())) {
cpu_relax();
return;
}
if (__vc_relax_spun++ >= NR_RELAX_SPINS) {
/* if other_vcoreid == vcore_id(), this might be expensive */
ensure_vcore_runs(other_vcoreid);
__vc_relax_spun = 0;
}
cpu_relax();
}
/* Check with the kernel to determine what vcore we are. Normally, you should
* never call this, since your vcoreid is stored in your TLS. Also, if you call
* it from a uthread, you could get migrated, so you should drop into some form
* of vcore context (DONT_MIGRATE on) */
uint32_t get_vcoreid(void)
{
if (!in_vcore_context()) {
assert(current_uthread);
assert(current_uthread->flags & UTHREAD_DONT_MIGRATE);
}
return __get_vcoreid();
}
/* Debugging helper. Pass in the string you want printed if your vcoreid is
* wrong, and pass in what vcoreid you think you are. Don't call from uthread
* context unless migrations are disabled. Will print some stuff and return
* FALSE if you were wrong. */
bool check_vcoreid(const char *str, uint32_t vcoreid)
{
uint32_t kvcoreid = get_vcoreid();
if (vcoreid != kvcoreid) {
printf("%s: VC %d thought it was VC %d\n", str, kvcoreid,
vcoreid);
return FALSE;
}
return TRUE;
}
/* Helper. Yields the vcore, or restarts it from scratch. */
void __attribute__((noreturn)) vcore_yield_or_restart(void)
{
struct preempt_data *vcpd = vcpd_of(vcore_id());
vcore_yield(FALSE);
/* If vcore_yield returns, we have an event. Just restart vcore
* context. */
set_stack_pointer((void*)vcpd->vcore_stack);
vcore_entry();
}
void vcore_wake(uint32_t vcoreid, bool force_ipi)
{
struct preempt_data *vcpd = vcpd_of(vcoreid);
vcpd->notif_pending = true;
if (vcoreid == vcore_id())
return;
if (force_ipi || !arch_has_mwait())
sys_self_notify(vcoreid, EV_NONE, 0, true);
}