| /* See COPYRIGHT for copyright information. |
| * The two TRAP* macros (minus the .data parts) are from the JOS project. |
| * Everything else: |
| * Copyright (c) 2009, 2013 The Regents of the University of California |
| * Barret Rhoden <brho@cs.berkeley.edu> |
| * See LICENSE for details. |
| */ |
| #include <arch/mmu.h> |
| #include <arch/trap.h> |
| #include <arch/x86.h> |
| #include <ros/memlayout.h> |
| |
| ################################################################### |
| # exceptions/interrupts |
| ################################################################### |
| |
| /* The TRAPHANDLER macro defines a globally-visible function for handling |
| * a trap. It pushes a trap number onto the stack, then jumps to _alltraps. |
| * It also builds this traps portion of the trap_tbl. |
| * Use TRAPHANDLER for traps where the CPU automatically pushes an error code. |
| */ |
| #define TRAPHANDLER(name, num) \ |
| .text; \ |
| .globl name; /* define global symbol for 'name' */ \ |
| .type name, @function; /* symbol type is function */ \ |
| .align 2; /* align function definition */ \ |
| name: /* function starts here */ \ |
| pushq $(num); \ |
| jmp _alltraps; \ |
| .data; \ |
| .quad name; \ |
| .long num |
| |
| /* Use TRAPHANDLER_NOEC for traps where the CPU doesn't push an error code. |
| * It pushes a 0 in place of the error code, so the trap frame has the same |
| * format in either case. */ |
| #define TRAPHANDLER_NOEC(name, num) \ |
| .text; \ |
| .globl name; \ |
| .type name, @function; \ |
| .align 2; \ |
| name: \ |
| pushq $0; \ |
| pushq $(num); \ |
| jmp _alltraps; \ |
| .data; \ |
| .quad name; \ |
| .long num |
| |
| /* Same as NOEC, but for IRQs instead. num is the ISR number it is mapped to */ |
| #define IRQ_HANDLER(name, num) \ |
| .text; \ |
| .globl name; \ |
| .type name, @function; \ |
| .align 2; \ |
| name: \ |
| pushq $0; \ |
| pushq $(num); \ |
| jmp _allirqs; \ |
| .data; \ |
| .quad name; \ |
| .long num |
| |
| #define NMI_HANDLER(name, num) \ |
| .text; \ |
| .globl name; \ |
| .type name, @function; \ |
| .align 2; \ |
| name: \ |
| pushq $0; \ |
| pushq $(num); \ |
| jmp _nmi_entry; \ |
| .data; \ |
| .quad name; \ |
| .long num |
| |
| #define DOUBLEFAULT_HANDLER(name, num) \ |
| .text; \ |
| .globl name; \ |
| .type name, @function; \ |
| .align 2; \ |
| name: \ |
| pushq $(num); \ |
| jmp _dblf_entry; \ |
| .data; \ |
| .quad name; \ |
| .long num |
| |
| /* Bare minimum IRQ handler: send a LAPIC_EOI and immediately iret. */ |
| #define POKE_HANDLER(name, num) \ |
| .text; \ |
| .globl name; \ |
| .type name, @function; \ |
| .align 2; \ |
| name:; \ |
| pushq %rax; \ |
| pushq %rcx; \ |
| pushq %rdx; \ |
| movq $0, %rax; \ |
| movq $0, %rdx; \ |
| movq $(MSR_LAPIC_EOI), %rcx; \ |
| wrmsr; \ |
| popq %rdx; \ |
| popq %rcx; \ |
| popq %rax; \ |
| iretq; \ |
| .data; \ |
| .quad name; \ |
| .long num |
| |
| # Set data for the trap_tbl symbol. Set text for the entry_points. The various |
| # trap/irq handler macros will toggle data on and off as needed. |
| .data |
| .globl trap_tbl |
| trap_tbl: |
| .text |
| .globl __asm_entry_points_start |
| __asm_entry_points_start: |
| |
| /* Generate entry points for the different traps. Note that all of these bounce |
| * off the corresponding trap.c function, such as handle_irqs, and that the name |
| * e.g. ISR_divide_error is soley for the little stup that jumps to something |
| * like _alltraps. |
| * |
| * Technically, these HANDLER entries do not need to be in numeric order. |
| * trap.c will do a 'foreach (up to last-1), set the IDT for the number to point |
| * to the func' in the order in which they appear in the trap tbl, so the 'last |
| * one wins'. */ |
| TRAPHANDLER_NOEC(ISR_divide_error, T_DIVIDE) |
| TRAPHANDLER_NOEC(ISR_debug_exceptions, T_DEBUG) |
| NMI_HANDLER(ISR_NMI, T_NMI) |
| TRAPHANDLER_NOEC(ISR_breakpoint, T_BRKPT) |
| TRAPHANDLER_NOEC(ISR_overflow, T_OFLOW) |
| TRAPHANDLER_NOEC(ISR_bounds_check, T_BOUND) |
| TRAPHANDLER_NOEC(ISR_invalid_opcode, T_ILLOP) |
| TRAPHANDLER_NOEC(ISR_device_not_available, T_DEVICE) |
| DOUBLEFAULT_HANDLER(ISR_double_fault, T_DBLFLT) |
| /* 9 reserved */ |
| TRAPHANDLER(ISR_invalid_TSS, T_TSS) |
| TRAPHANDLER(ISR_segment_not_present, T_SEGNP) |
| TRAPHANDLER(ISR_stack_exception, T_STACK) |
| TRAPHANDLER(ISR_general_protection_fault, T_GPFLT) |
| TRAPHANDLER(ISR_page_fault, T_PGFLT) |
| /* 15 reserved */ |
| TRAPHANDLER_NOEC(ISR_floating_point_error, T_FPERR) |
| TRAPHANDLER(ISR_alignment_check, T_ALIGN) |
| TRAPHANDLER_NOEC(ISR_machine_check, T_MCHK) |
| TRAPHANDLER_NOEC(ISR_simd_error, T_SIMDERR) |
| /* 20 - 31 reserved */ |
| /* 32-47 are PIC/8259 IRQ vectors */ |
| IRQ_HANDLER(IRQ0, 32) |
| IRQ_HANDLER(IRQ1, 33) |
| IRQ_HANDLER(IRQ2, 34) |
| IRQ_HANDLER(IRQ3, 35) |
| IRQ_HANDLER(IRQ4, 36) |
| IRQ_HANDLER(IRQ5, 37) |
| IRQ_HANDLER(IRQ6, 38) |
| IRQ_HANDLER(IRQ7, 39) |
| IRQ_HANDLER(IRQ8, 40) |
| IRQ_HANDLER(IRQ9, 41) |
| IRQ_HANDLER(IRQ10, 42) |
| IRQ_HANDLER(IRQ11, 43) |
| IRQ_HANDLER(IRQ12, 44) |
| IRQ_HANDLER(IRQ13, 45) |
| IRQ_HANDLER(IRQ14, 46) |
| IRQ_HANDLER(IRQ15, 47) |
| TRAPHANDLER_NOEC(ISR_syscall, T_SYSCALL) |
| /* 49-223 are IOAPIC routing vectors (from IOAPIC to LAPIC) */ |
| IRQ_HANDLER(IRQ17, 49) |
| IRQ_HANDLER(IRQ18, 50) |
| IRQ_HANDLER(IRQ19, 51) |
| IRQ_HANDLER(IRQ20, 52) |
| IRQ_HANDLER(IRQ21, 53) |
| IRQ_HANDLER(IRQ22, 54) |
| IRQ_HANDLER(IRQ23, 55) |
| IRQ_HANDLER(IRQ24, 56) |
| IRQ_HANDLER(IRQ25, 57) |
| IRQ_HANDLER(IRQ26, 58) |
| IRQ_HANDLER(IRQ27, 59) |
| IRQ_HANDLER(IRQ28, 60) |
| IRQ_HANDLER(IRQ29, 61) |
| IRQ_HANDLER(IRQ30, 62) |
| IRQ_HANDLER(IRQ31, 63) |
| IRQ_HANDLER(IRQ32, 64) |
| IRQ_HANDLER(IRQ33, 65) |
| IRQ_HANDLER(IRQ34, 66) |
| IRQ_HANDLER(IRQ35, 67) |
| IRQ_HANDLER(IRQ36, 68) |
| IRQ_HANDLER(IRQ37, 69) |
| IRQ_HANDLER(IRQ38, 70) |
| IRQ_HANDLER(IRQ39, 71) |
| IRQ_HANDLER(IRQ40, 72) |
| IRQ_HANDLER(IRQ41, 73) |
| IRQ_HANDLER(IRQ42, 74) |
| IRQ_HANDLER(IRQ43, 75) |
| IRQ_HANDLER(IRQ44, 76) |
| IRQ_HANDLER(IRQ45, 77) |
| IRQ_HANDLER(IRQ46, 78) |
| IRQ_HANDLER(IRQ47, 79) |
| IRQ_HANDLER(IRQ48, 80) |
| IRQ_HANDLER(IRQ49, 81) |
| IRQ_HANDLER(IRQ50, 82) |
| IRQ_HANDLER(IRQ51, 83) |
| IRQ_HANDLER(IRQ52, 84) |
| IRQ_HANDLER(IRQ53, 85) |
| IRQ_HANDLER(IRQ54, 86) |
| IRQ_HANDLER(IRQ55, 87) |
| IRQ_HANDLER(IRQ56, 88) |
| IRQ_HANDLER(IRQ57, 89) |
| IRQ_HANDLER(IRQ58, 90) |
| IRQ_HANDLER(IRQ59, 91) |
| IRQ_HANDLER(IRQ60, 92) |
| IRQ_HANDLER(IRQ61, 93) |
| IRQ_HANDLER(IRQ62, 94) |
| IRQ_HANDLER(IRQ63, 95) |
| IRQ_HANDLER(IRQ64, 96) |
| IRQ_HANDLER(IRQ65, 97) |
| IRQ_HANDLER(IRQ66, 98) |
| IRQ_HANDLER(IRQ67, 99) |
| IRQ_HANDLER(IRQ68, 100) |
| IRQ_HANDLER(IRQ69, 101) |
| IRQ_HANDLER(IRQ70, 102) |
| IRQ_HANDLER(IRQ71, 103) |
| IRQ_HANDLER(IRQ72, 104) |
| IRQ_HANDLER(IRQ73, 105) |
| IRQ_HANDLER(IRQ74, 106) |
| IRQ_HANDLER(IRQ75, 107) |
| IRQ_HANDLER(IRQ76, 108) |
| IRQ_HANDLER(IRQ77, 109) |
| IRQ_HANDLER(IRQ78, 110) |
| IRQ_HANDLER(IRQ79, 111) |
| IRQ_HANDLER(IRQ80, 112) |
| IRQ_HANDLER(IRQ81, 113) |
| IRQ_HANDLER(IRQ82, 114) |
| IRQ_HANDLER(IRQ83, 115) |
| IRQ_HANDLER(IRQ84, 116) |
| IRQ_HANDLER(IRQ85, 117) |
| IRQ_HANDLER(IRQ86, 118) |
| IRQ_HANDLER(IRQ87, 119) |
| IRQ_HANDLER(IRQ88, 120) |
| IRQ_HANDLER(IRQ89, 121) |
| IRQ_HANDLER(IRQ90, 122) |
| IRQ_HANDLER(IRQ91, 123) |
| IRQ_HANDLER(IRQ92, 124) |
| IRQ_HANDLER(IRQ93, 125) |
| IRQ_HANDLER(IRQ94, 126) |
| IRQ_HANDLER(IRQ95, 127) |
| IRQ_HANDLER(IRQ96, 128) |
| IRQ_HANDLER(IRQ97, 129) |
| IRQ_HANDLER(IRQ98, 130) |
| IRQ_HANDLER(IRQ99, 131) |
| IRQ_HANDLER(IRQ100, 132) |
| IRQ_HANDLER(IRQ101, 133) |
| IRQ_HANDLER(IRQ102, 134) |
| IRQ_HANDLER(IRQ103, 135) |
| IRQ_HANDLER(IRQ104, 136) |
| IRQ_HANDLER(IRQ105, 137) |
| IRQ_HANDLER(IRQ106, 138) |
| IRQ_HANDLER(IRQ107, 139) |
| IRQ_HANDLER(IRQ108, 140) |
| IRQ_HANDLER(IRQ109, 141) |
| IRQ_HANDLER(IRQ110, 142) |
| IRQ_HANDLER(IRQ111, 143) |
| IRQ_HANDLER(IRQ112, 144) |
| IRQ_HANDLER(IRQ113, 145) |
| IRQ_HANDLER(IRQ114, 146) |
| IRQ_HANDLER(IRQ115, 147) |
| IRQ_HANDLER(IRQ116, 148) |
| IRQ_HANDLER(IRQ117, 149) |
| IRQ_HANDLER(IRQ118, 150) |
| IRQ_HANDLER(IRQ119, 151) |
| IRQ_HANDLER(IRQ120, 152) |
| IRQ_HANDLER(IRQ121, 153) |
| IRQ_HANDLER(IRQ122, 154) |
| IRQ_HANDLER(IRQ123, 155) |
| IRQ_HANDLER(IRQ124, 156) |
| IRQ_HANDLER(IRQ125, 157) |
| IRQ_HANDLER(IRQ126, 158) |
| IRQ_HANDLER(IRQ127, 159) |
| IRQ_HANDLER(IRQ128, 160) |
| IRQ_HANDLER(IRQ129, 161) |
| IRQ_HANDLER(IRQ130, 162) |
| IRQ_HANDLER(IRQ131, 163) |
| IRQ_HANDLER(IRQ132, 164) |
| IRQ_HANDLER(IRQ133, 165) |
| IRQ_HANDLER(IRQ134, 166) |
| IRQ_HANDLER(IRQ135, 167) |
| IRQ_HANDLER(IRQ136, 168) |
| IRQ_HANDLER(IRQ137, 169) |
| IRQ_HANDLER(IRQ138, 170) |
| IRQ_HANDLER(IRQ139, 171) |
| IRQ_HANDLER(IRQ140, 172) |
| IRQ_HANDLER(IRQ141, 173) |
| IRQ_HANDLER(IRQ142, 174) |
| IRQ_HANDLER(IRQ143, 175) |
| IRQ_HANDLER(IRQ144, 176) |
| IRQ_HANDLER(IRQ145, 177) |
| IRQ_HANDLER(IRQ146, 178) |
| IRQ_HANDLER(IRQ147, 179) |
| IRQ_HANDLER(IRQ148, 180) |
| IRQ_HANDLER(IRQ149, 181) |
| IRQ_HANDLER(IRQ150, 182) |
| IRQ_HANDLER(IRQ151, 183) |
| IRQ_HANDLER(IRQ152, 184) |
| IRQ_HANDLER(IRQ153, 185) |
| IRQ_HANDLER(IRQ154, 186) |
| IRQ_HANDLER(IRQ155, 187) |
| IRQ_HANDLER(IRQ156, 188) |
| IRQ_HANDLER(IRQ157, 189) |
| IRQ_HANDLER(IRQ158, 190) |
| IRQ_HANDLER(IRQ159, 191) |
| IRQ_HANDLER(IRQ160, 192) |
| IRQ_HANDLER(IRQ161, 193) |
| IRQ_HANDLER(IRQ162, 194) |
| IRQ_HANDLER(IRQ163, 195) |
| IRQ_HANDLER(IRQ164, 196) |
| IRQ_HANDLER(IRQ165, 197) |
| IRQ_HANDLER(IRQ166, 198) |
| IRQ_HANDLER(IRQ167, 199) |
| IRQ_HANDLER(IRQ168, 200) |
| IRQ_HANDLER(IRQ169, 201) |
| IRQ_HANDLER(IRQ170, 202) |
| IRQ_HANDLER(IRQ171, 203) |
| IRQ_HANDLER(IRQ172, 204) |
| IRQ_HANDLER(IRQ173, 205) |
| IRQ_HANDLER(IRQ174, 206) |
| IRQ_HANDLER(IRQ175, 207) |
| IRQ_HANDLER(IRQ176, 208) |
| IRQ_HANDLER(IRQ177, 209) |
| IRQ_HANDLER(IRQ178, 210) |
| IRQ_HANDLER(IRQ179, 211) |
| IRQ_HANDLER(IRQ180, 212) |
| IRQ_HANDLER(IRQ181, 213) |
| IRQ_HANDLER(IRQ182, 214) |
| IRQ_HANDLER(IRQ183, 215) |
| IRQ_HANDLER(IRQ184, 216) |
| IRQ_HANDLER(IRQ185, 217) |
| IRQ_HANDLER(IRQ186, 218) |
| IRQ_HANDLER(IRQ187, 219) |
| IRQ_HANDLER(IRQ188, 220) |
| IRQ_HANDLER(IRQ189, 221) |
| IRQ_HANDLER(IRQ190, 222) |
| IRQ_HANDLER(IRQ191, 223) |
| /* 224-239 are OS IPI vectors (0xe0-0xef) */ |
| IRQ_HANDLER(IRQ192, I_SMP_CALL0) |
| IRQ_HANDLER(IRQ193, I_SMP_CALL1) |
| IRQ_HANDLER(IRQ194, I_SMP_CALL2) |
| IRQ_HANDLER(IRQ195, I_SMP_CALL3) |
| IRQ_HANDLER(IRQ196, I_SMP_CALL4) |
| IRQ_HANDLER(IRQ197, 229) |
| IRQ_HANDLER(IRQ198, 230) |
| IRQ_HANDLER(IRQ199, 231) |
| IRQ_HANDLER(IRQ200, 232) |
| IRQ_HANDLER(IRQ201, 233) |
| IRQ_HANDLER(IRQ202, 234) |
| IRQ_HANDLER(IRQ203, 235) |
| IRQ_HANDLER(IRQ204, I_TESTING) |
| POKE_HANDLER(IRQ205, I_POKE_GUEST) |
| POKE_HANDLER(IRQ206, I_POKE_CORE) |
| IRQ_HANDLER(IRQ207, I_KERNEL_MSG) |
| /* 240-255 are LAPIC vectors (0xf0-0xff), hightest priority class */ |
| IRQ_HANDLER(IRQ208, 240) |
| IRQ_HANDLER(IRQ209, 241) |
| IRQ_HANDLER(IRQ210, 242) |
| IRQ_HANDLER(IRQ211, 243) |
| IRQ_HANDLER(IRQ212, 244) |
| IRQ_HANDLER(IRQ213, 245) |
| IRQ_HANDLER(IRQ214, 246) |
| IRQ_HANDLER(IRQ215, 247) |
| IRQ_HANDLER(IRQ216, 248) |
| IRQ_HANDLER(IRQ217, 249) |
| IRQ_HANDLER(IRQ218, 250) |
| IRQ_HANDLER(IRQ219, 251) |
| IRQ_HANDLER(IRQ220, 252) |
| IRQ_HANDLER(IRQ221, 253) |
| IRQ_HANDLER(IRQ222, 254) |
| IRQ_HANDLER(IRQ223, 255) |
| /* But make sure default is last!! */ |
| TRAPHANDLER_NOEC(ISR_default, T_DEFAULT) |
| |
| .data |
| .globl trap_tbl_end |
| trap_tbl_end: |
| |
| .text |
| _alltraps: |
| cld |
| pushq %r15 |
| pushq %r14 |
| pushq %r13 |
| pushq %r12 |
| pushq %r11 |
| pushq %r10 |
| pushq %r9 |
| pushq %r8 |
| pushq %rdi |
| pushq %rsi |
| pushq %rbp |
| pushq %rdx |
| pushq %rcx |
| pushq %rbx |
| pushq %rax |
| cmpw $GD_KT, 0x90(%rsp) # 0x90 - diff btw tf_cs and tf_rax |
| je trap_all_tf |
| # this is a user TF. we need to swapgs to get the kernel's gs and mark |
| # the context as partial |
| swapgs # user's GS is now in MSR_KERNEL_GS_BASE |
| movl $0x1, 0xac(%rsp) # 0xac - diff btw tf_padding0 and tf_rax |
| trap_all_tf: |
| pushq $0 # fsbase space |
| pushq $0 # gsbase space |
| movq $0, %rbp # so we can backtrace to this point |
| movq %rsp, %rdi |
| call trap |
| # the return paths are only used by the kernel |
| addq $0x10, %rsp # skip fs/gs base |
| popq %rax |
| popq %rbx |
| popq %rcx |
| popq %rdx |
| popq %rbp |
| popq %rsi |
| popq %rdi |
| popq %r8 |
| popq %r9 |
| popq %r10 |
| popq %r11 |
| popq %r12 |
| popq %r13 |
| popq %r14 |
| popq %r15 |
| addq $0x10, %rsp # skip trapno and err |
| iretq |
| |
| # might merge this with _alltraps |
| _allirqs: |
| cld |
| pushq %r15 |
| pushq %r14 |
| pushq %r13 |
| pushq %r12 |
| pushq %r11 |
| pushq %r10 |
| pushq %r9 |
| pushq %r8 |
| pushq %rdi |
| pushq %rsi |
| pushq %rbp |
| pushq %rdx |
| pushq %rcx |
| pushq %rbx |
| pushq %rax |
| cmpw $GD_KT, 0x90(%rsp) # 0x90 - diff btw tf_cs and tf_rax |
| je irq_all_tf |
| # this is a user TF. we need to swapgs to get the kernel's gs and mark |
| # the context as partial |
| swapgs # user's GS is now in MSR_KERNEL_GS_BASE |
| movl $0x1, 0xac(%rsp) # 0xac - diff btw tf_padding0 and tf_rax |
| irq_all_tf: |
| pushq $0 # fsbase space |
| pushq $0 # gsbase space |
| movq $0, %rbp # so we can backtrace to this point |
| movq %rsp, %rdi |
| call handle_irq |
| # the return paths are only used by the kernel |
| addq $0x10, %rsp # skip fs/gs base |
| popq %rax |
| popq %rbx |
| popq %rcx |
| popq %rdx |
| popq %rbp |
| popq %rsi |
| popq %rdi |
| popq %r8 |
| popq %r9 |
| popq %r10 |
| popq %r11 |
| popq %r12 |
| popq %r13 |
| popq %r14 |
| popq %r15 |
| addq $0x10, %rsp # skip trapno and err |
| iretq |
| |
| # Similar to the NMI handler, we come in on a special stack, but we can trust |
| # the contents of GS for kernel contexts. This is mostly true. If we double |
| # faulted in this file, for instance, then GS could be garbage. But that should |
| # never happen. Most double faults will be for running into a guard page on a |
| # stack. |
| # |
| # Since the double fault is so bad, we just want to get into the kernel's C code |
| # where we can run some diagnostics. We (currently) won't try to recover, since |
| # it's probably a kernel bug. |
| _dblf_entry: |
| cld |
| pushq %r15 |
| pushq %r14 |
| pushq %r13 |
| pushq %r12 |
| pushq %r11 |
| pushq %r10 |
| pushq %r9 |
| pushq %r8 |
| pushq %rdi |
| pushq %rsi |
| pushq %rbp |
| pushq %rdx |
| pushq %rcx |
| pushq %rbx |
| pushq %rax |
| cmpw $GD_KT, 0x90(%rsp) # 0x90 - diff btw tf_cs and tf_rax |
| je dblf_all_tf |
| # this is a user TF. we need to swapgs to get the kernel's gs and mark |
| # the context as partial |
| swapgs # user's GS is now in MSR_KERNEL_GS_BASE |
| movl $0x1, 0xac(%rsp) # 0xac - diff btw tf_padding0 and tf_rax |
| dblf_all_tf: |
| pushq $0 # fsbase space |
| pushq $0 # gsbase space |
| movq $0, %rbp # so we can backtrace to this point |
| movq %rsp, %rdi |
| call handle_double_fault |
| dblf_spin: |
| jmp dblf_spin |
| |
| # Unlike normal trap and IRQ handlers, both user and kernel TFs are handled the |
| # similarly. Both come in here and return from here. We cannot do anything |
| # fancy like proc_restartcore() from NMI context. |
| # |
| # All NMIs will come in fresh on the same stack (IST1, per-core). We don't need |
| # to find a stackpointer, but we do need to find GS. |
| # |
| # Regardless of whether or not the interrupted context was in the kernel, we |
| # can't trust GS. Basically we can never tell from looking at GS and KERN_GS |
| # whether swapgs occurred, since the user could have put kernel addrs in their |
| # GS. But we can use the stack for storage to bootstrap GS. %rsp at entry |
| # points to pcpui *. |
| # |
| # We can also tell if GS was set correctly or not and avoid the wrmsr calls. |
| # This isn't a huge deal. But note that we don't know whether or not the kernel |
| # actually set the gsbase. We just know if it was correct or not. The user |
| # could have set it and the kernel hadn't had a chance to swapgs yet. The NMI |
| # handler doesn't care, since all TFs come in and go out via this asm. |
| # |
| # If we want to be paranoid, we can completely ignore user TFs and just save and |
| # restore GS with the same mechanism we use for the kernel. But since we came |
| # in on a user TF, we can use swapgs. Note that even for nested NMI handlers, |
| # we have a user TF for only the first time, and only the last exit will swapgs |
| # back to the way it was initially. |
| _nmi_entry: |
| cld |
| pushq %r15 |
| pushq %r14 |
| pushq %r13 |
| pushq %r12 |
| pushq %r11 |
| pushq %r10 |
| pushq %r9 |
| pushq %r8 |
| pushq %rdi |
| pushq %rsi |
| pushq %rbp |
| pushq %rdx |
| pushq %rcx |
| pushq %rbx |
| pushq %rax |
| cmpw $GD_KT, 0x90(%rsp) # 0x90 - diff btw tf_cs and tf_rax |
| je nmi_kern_tf |
| # this is a user TF. we need to swapgs to get the kernel's gs and mark |
| # the context as partial |
| swapgs # user's GS is now in MSR_KERNEL_GS_BASE |
| movl $0x1, 0xac(%rsp) # 0xac - diff btw tf_padding0 and tf_rax |
| pushq $0 # fsbase space |
| pushq $0 # gsbase space |
| jmp nmi_all_tf |
| nmi_kern_tf: |
| # this is a kernel TF. but we don't know if they set up gs yet, so |
| # we'll save and restore whatever they had loaded and use our own |
| pushq $0 # fsbase space |
| # Get the current GS base into rax |
| movl $MSR_GS_BASE, %ecx |
| rdmsr |
| shlq $32, %rdx |
| orq %rdx, %rax |
| # Get the real GS base from the top of the stack. This was set in |
| # smp_boot, and our rsp pointed to it when we entered the kernel. |
| movq 0xb8(%rsp), %rdx # 0xb8 from fs_base to the top |
| # Compare them. If they are the same, we can just push 0 for gsbase |
| # (which later will mean "no need to restore GS". |
| cmpq %rdx, %rax |
| je nmi_gs_ok |
| # They weren't the same. Save the old one and set the new one. |
| pushq %rax # gsbase space |
| movq %rdx, %rax |
| shrq $32, %rdx |
| andl $0xffffffff, %eax |
| wrmsr |
| jmp nmi_all_tf |
| nmi_gs_ok: |
| pushq $0 # gsbase space |
| nmi_all_tf: |
| # At this point, GS is set correctly, either due to swapgs (user TF), |
| # wrmsr (kern TF with bad GS), or it was already fine (and gsbase in the |
| # TF = 0). |
| movq $0, %rbp # so we can backtrace to this point |
| movq %rsp, %rdi |
| call handle_nmi |
| # Unlike in normal IRQs/Traps, both user and kernel contexts return via |
| # this path. |
| cmpw $GD_KT, 0xa0(%rsp) # 0xa0 - diff btw tf_cs and tf_gsbase |
| je nmi_kern_restore_gs |
| # User TF. Restore whatever was there with swapgs. We don't care what |
| # it was, nor do we care what was in the TF. |
| swapgs # user's GS is now in MSR_GS_BASE |
| addq $0x10, %rsp # skip gs/fs base |
| jmp nmi_popal |
| nmi_kern_restore_gs: |
| popq %rax # fetch saved gsbase |
| addq $0x08, %rsp # skip fs base |
| cmpq $0, %rax |
| je nmi_popal |
| # gsbase in the TF != 0, which means we need to restore that gsbase |
| movl $MSR_GS_BASE, %ecx |
| movq %rax, %rdx |
| shrq $32, %rdx |
| andl $0xffffffff, %eax |
| wrmsr |
| nmi_popal: |
| popq %rax |
| popq %rbx |
| popq %rcx |
| popq %rdx |
| popq %rbp |
| popq %rsi |
| popq %rdi |
| popq %r8 |
| popq %r9 |
| popq %r10 |
| popq %r11 |
| popq %r12 |
| popq %r13 |
| popq %r14 |
| popq %r15 |
| addq $0x10, %rsp # skip trapno and err |
| iretq |
| |
| .globl __nmi_pop_ok_start; |
| .globl __nmi_pop_ok_end; |
| .globl __nmi_pop_fail_start; |
| .globl __nmi_pop_fail_end; |
| |
| # extern void nmi_try_to_pop(struct hw_trapframe *tf, int *status, |
| # int old_val, int new_val); |
| # |
| # __nmi_bottom_half calls this to atomically pop a hw_tf (%rdi) and set |
| # &pcpui->nmi_status (%rsi) with compare and swap to NMI_NORMAL_OPN (%ecx) given |
| # that it was NMI_IN_PROGRESS (%edx) |
| # |
| # (Careful, nmi_status is an int, not a long.) |
| # |
| # If the real NMI handler interrupts us, it'll move us to the fail section of |
| # the code. That code is identical to 'ok', up until ok's final statement. |
| # |
| # In that event, we'll need a little help returning: specifically to bootstrap |
| # pcpui and our current stackpointer. pcpui is already saved near the top of |
| # stack. We'll save rsp ourselves. |
| .globl nmi_try_to_pop; |
| .type nmi_try_to_pop, @function; |
| nmi_try_to_pop: |
| __nmi_pop_ok_start: |
| # careful only to use caller-saved or argument registers before saving |
| movl %edx, %eax # load old_val into eax for the CAS |
| cmpxchgl %ecx, (%rsi) # no need for LOCK, since an NMI would serialize |
| jz nmi_ok_cas_worked # ZF = 1 on successful CAS |
| ret |
| nmi_ok_cas_worked: |
| # save callee-saved regs (the pops below clobber them, and we might |
| # return) |
| pushq %rbp |
| pushq %rbx |
| pushq %r12 |
| pushq %r13 |
| pushq %r14 |
| pushq %r15 |
| # We need to save the current rsp into the scratch space at the top of |
| # the stack. This assumes we're within the top page of our stack, which |
| # should always be true. Careful not to use rdi, which still has an |
| # argument. |
| movq %rsp, %rbx |
| # Want to round rbx up to PGSIZE, then subtract 8, to get our slot. |
| movq $0xfff, %rax |
| notq %rax # rax = 0xfffffffffffff000 |
| andq %rax, %rbx # round down rbx |
| addq $0x1000, %rbx # add PGSIZE, assuming rsp was not page aligned |
| subq $0x8, %rbx # point to the scratch space |
| movq %rsp, (%rbx) # save rsp in the scratch space |
| # We jump our rsp to the base of the HW_TF. This is still on the same |
| # stack, just farther back than where our caller is. We need to be |
| # careful to not clobber the stack. Otherwise we'll have chaos. |
| movq %rdi, %rsp |
| # From here down is the same as the normal NMI exit path, but with 'ok' |
| # in the symbol names. |
| cmpw $GD_KT, 0xa0(%rsp) # 0xa0 - diff btw tf_cs and tf_gsbase |
| je nmi_ok_kern_restore_gs |
| # User TF. Restore whatever was there with swapgs. We don't care what |
| # it was, nor do we care what was in the TF. |
| swapgs # user's GS is now in MSR_GS_BASE |
| addq $0x10, %rsp # skip gs/fs base |
| jmp nmi_ok_popal |
| nmi_ok_kern_restore_gs: |
| popq %rax # fetch saved gsbase |
| addq $0x08, %rsp # skip fs base |
| cmpq $0, %rax |
| je nmi_ok_popal |
| # gsbase in the TF != 0, which means we need to restore that gsbase |
| movl $MSR_GS_BASE, %ecx |
| movq %rax, %rdx |
| shrq $32, %rdx |
| andl $0xffffffff, %eax |
| wrmsr |
| nmi_ok_popal: |
| popq %rax |
| popq %rbx |
| popq %rcx |
| popq %rdx |
| popq %rbp |
| popq %rsi |
| popq %rdi |
| popq %r8 |
| popq %r9 |
| popq %r10 |
| popq %r11 |
| popq %r12 |
| popq %r13 |
| popq %r14 |
| popq %r15 |
| addq $0x10, %rsp # skip trapno and err |
| iretq |
| __nmi_pop_ok_end: |
| |
| # This is the 'fail' case. It is identical to the 'ok' case, up until the |
| # iretq, other than 'ok' replaced with 'fail'. In place of iretq, we undo the |
| # entire operation. |
| __nmi_pop_fail_start: |
| # careful only to use caller-saved or argument registers before saving |
| movl %edx, %eax # load old_val into eax for the CAS |
| cmpxchgl %ecx, (%rsi) # no need for LOCK, since an NMI would serialize |
| jz nmi_fail_cas_worked # ZF = 1 on successful CAS |
| ret |
| nmi_fail_cas_worked: |
| # save callee-saved regs (the pops below clobber them, and we might |
| # return) |
| pushq %rbp |
| pushq %rbx |
| pushq %r12 |
| pushq %r13 |
| pushq %r14 |
| pushq %r15 |
| # We need to save the current rsp into the scratch space at the top of |
| # the stack. This assumes we're within the top page of our stack, which |
| # should always be true. Careful not to use rdi, which still has an |
| # argument. |
| movq %rsp, %rbx |
| # Want to round rbx up to PGSIZE, then subtract 8, to get our slot. |
| movq $0xfff, %rax |
| notq %rax # rax = 0xfffffffffffff000 |
| andq %rax, %rbx # round down rbx |
| addq $0x1000, %rbx # add PGSIZE, assuming rsp was not page aligned |
| subq $0x8, %rbx # point to the scratch space |
| movq %rsp, (%rbx) # save rsp in the scratch space |
| # We jump our rsp to the base of the HW_TF. This is still on the same |
| # stack, just farther back than where our caller is. We need to be |
| # careful to not clobber the stack. Otherwise we'll have chaos. |
| movq %rdi, %rsp |
| # From here down is the same as the normal NMI exit path and the ok |
| # path, but with 'fail' in the symbol names. |
| cmpw $GD_KT, 0xa0(%rsp) # 0xa0 - diff btw tf_cs and tf_gsbase |
| je nmi_fail_kern_restore_gs |
| # User TF. Restore whatever was there with swapgs. We don't care what |
| # it was, nor do we care what was in the TF. |
| swapgs # user's GS is now in MSR_GS_BASE |
| addq $0x10, %rsp # skip gs/fs base |
| jmp nmi_fail_popal |
| nmi_fail_kern_restore_gs: |
| popq %rax # fetch saved gsbase |
| addq $0x08, %rsp # skip fs base |
| cmpq $0, %rax |
| je nmi_fail_popal |
| # gsbase in the TF != 0, which means we need to restore that gsbase |
| movl $MSR_GS_BASE, %ecx |
| movq %rax, %rdx |
| shrq $32, %rdx |
| andl $0xffffffff, %eax |
| wrmsr |
| nmi_fail_popal: |
| popq %rax |
| popq %rbx |
| popq %rcx |
| popq %rdx |
| popq %rbp |
| popq %rsi |
| popq %rdi |
| popq %r8 |
| popq %r9 |
| popq %r10 |
| popq %r11 |
| popq %r12 |
| popq %r13 |
| popq %r14 |
| popq %r15 |
| addq $0x10, %rsp # skip trapno and err |
| # Here's is where we differ from OK. Time to undo everything and return |
| # rsp currently is pointing at tf->tf_rip. Remember that we don't want |
| # to write anything to the stack - everything in the TF is still the way |
| # it was when we started to pop. |
| # |
| # First off, let's get the stack addr of the pcpui pointer loaded |
| movq %rsp, %rbx |
| movq $0xfff, %rax |
| notq %rax # rax = 0xfffffffffffff000 |
| andq %rax, %rbx # round down rbx |
| addq $0x1000, %rbx # add PGSIZE, assuming rsp was not page aligned |
| subq $0x10, %rbx # point to the pcpui pointer |
| # Now let's start to unwind |
| subq $0x98, %rsp # jump from rip to tf_gsbase (top of hw_tf) |
| # Need to restore gs, just like on an NMI entry |
| cmpw $GD_KT, 0xa0(%rsp) # 0xa0 - diff btw tf_cs and tf_gsbase |
| je nmi_pop_fail_kern_tf |
| # This is a user TF. We need to swapgs to get the kernel's gs |
| # We don't need to mark the context as partial (we never do for NMIs, |
| # actually), and in general, we don't want to write anything on the |
| # stack. |
| swapgs # user's GS is now in MSR_KERNEL_GS_BASE |
| jmp nmi_pop_fail_all_tf |
| nmi_pop_fail_kern_tf: |
| # Kernel TF. We basically need to do the same thing on entry, since we |
| # might have restored some weird GS base. We can tell based on |
| # tf_gsbase 0 for gsbase means we didn't need to change GS |
| cmpq $0, (%rsp) |
| je nmi_pop_fail_gs_fine |
| # rbx points to where pcpui* is stored |
| mov (%rbx), %rdx |
| movl $MSR_GS_BASE, %ecx |
| movq %rdx, %rax |
| shrq $32, %rdx |
| andl $0xffffffff, %eax |
| wrmsr |
| nmi_pop_fail_gs_fine: |
| nmi_pop_fail_all_tf: |
| addq $0x8, %rbx # move to the scratch slot, holding rsp |
| mov (%rbx), %rsp |
| # restore callee-saved regs |
| popq %r15 |
| popq %r14 |
| popq %r13 |
| popq %r12 |
| popq %rbx |
| popq %rbp |
| ret |
| # sweet jeebus. |
| __nmi_pop_fail_end: |
| |
| |
| .globl sysenter_handler; |
| .type sysenter_handler, @function; |
| |
| sysenter_handler: |
| #ifndef CONFIG_NOFASTCALL_FSBASE |
| # Do a quick TLS / FS base change, never changing stacks. |
| # When rdi has the magic number, rsi has the new base |
| movabs $FASTCALL_SETFSBASE, %rax |
| cmp %rax, %rdi |
| jne normal_syscall # could profile this and jump differently |
| # need to check rsi, make sure it is canonical (will enfore below ULIM). |
| # need to either do this check, or handle the kernel GP fault on wrmsr. |
| movq %rsi, %rdi |
| shrq $47, %rdi |
| cmp $0, %rdi |
| jne fastcall_pop |
| # need to use cx, dx, and ax for the wrmsr. dx and ax are free. |
| movq %rcx, %rdi # save rcx, the retaddr |
| movq %rsi, %rdx |
| movq %rsi, %rax |
| shrq $32, %rdx |
| andl $0xffffffff, %eax |
| movl $MSR_FS_BASE, %ecx |
| wrmsr |
| movq %rdi, %rcx # restore retaddr |
| fastcall_pop: |
| rex.w sysret |
| normal_syscall: |
| #endif |
| # cld is handled by the SFMASK |
| swapgs # user's GS is now in MSR_KERNEL_GS_BASE |
| movq %gs:0, %rsp |
| # Saving the FPU callee-saved state for now. Might be able to have the |
| # preempt handler deal with it. |
| pushq $0 # space for mxcsr, fpucw, and padding0 |
| movw $0x1, 0x6(%rsp) # tf_padding0 = 1, partial context |
| fnstcw 0x4(%rsp) |
| stmxcsr (%rsp) |
| pushq %rdx # rsp, saved by userspace |
| pushq %rcx # rip, saved by hardware |
| pushq %r15 |
| pushq %r14 |
| pushq %r13 |
| pushq %r12 |
| pushq %rbp |
| pushq %rbx |
| pushq $0 # fsbase space |
| pushq $0 # gsbase space |
| movq $0, %rbp # so we can backtrace to this point |
| movq %rsp, %rdx |
| # arg0, rdi: struct sysc*. arg1, rsi: count. arg2, rdx: sw_tf |
| call sysenter_callwrapper |
| # return via pop_tf, never this path |
| sysenter_spin: |
| jmp sysenter_spin |
| |
| .globl vmexit_handler; |
| .type vmexit_handler, @function; |
| vmexit_handler: |
| # rflags has all flags = 0, so cli and cld already. |
| # HOST_GS_BASE and RSP is set by the hardware |
| # Set default values. Most of these will be set in C later. |
| pushq $0 # guest_pa |
| pushq $0 # guest_va |
| pushq $0 # intrinfo2 and 1 |
| pushq $0 # exit_qual + exit_reason |
| pushq $0 # pad + trap_inject |
| pushq $0 # flags + guest_pcorid |
| pushq $0 # cr3 |
| pushq $0 # cr2 |
| pushq $0 # rsp |
| pushq $0 # rflags |
| pushq $0 # rip |
| # Save register state |
| pushq %r15 |
| pushq %r14 |
| pushq %r13 |
| pushq %r12 |
| pushq %r11 |
| pushq %r10 |
| pushq %r9 |
| pushq %r8 |
| pushq %rdi |
| pushq %rsi |
| pushq %rbp |
| pushq %rdx |
| pushq %rcx |
| pushq %rbx |
| pushq %rax |
| movq $0, %rbp # so we can backtrace to this point |
| movq %rsp, %rdi |
| call handle_vmexit |
| vmexit_spin: |
| jmp vmexit_spin |
| |
| .globl __asm_entry_points_end |
| __asm_entry_points_end: |
