i86xpv/sys/machprivregs.h

	machprivregs.h revision 843e19887f64dde75055cf8842fc4db2171eff45
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _SYS_MACHPRIVREGS_H
#define _SYS_MACHPRIVREGS_H

#pragma ident   "%Z%%M% %I% %E% SMI"

#include <sys/hypervisor.h>

/*
 * Platform dependent instruction sequences for manipulating
 * privileged state
 */

#ifdef __cplusplus
extern "C" {
#endif

/*
 * CLI and STI are quite complex to virtualize!
 */

#if defined(__amd64)

#define CURVCPU(r)                  \
    movq    %gs:CPU_VCPU_INFO, r

#define CURTHREAD(r)                    \
    movq    %gs:CPU_THREAD, r

#elif defined(__i386)

#define CURVCPU(r)                  \
    movl    %gs:CPU_VCPU_INFO, r

#define CURTHREAD(r)                    \
    movl    %gs:CPU_THREAD, r

#endif  /* __i386 */

#define XEN_TEST_EVENT_PENDING(r)           \
    testb   $0xff, VCPU_INFO_EVTCHN_UPCALL_PENDING(r)

#define XEN_SET_UPCALL_MASK(r)              \
    movb    $1, VCPU_INFO_EVTCHN_UPCALL_MASK(r)

#define XEN_GET_UPCALL_MASK(r, mask)            \
    movb    VCPU_INFO_EVTCHN_UPCALL_MASK(r), mask

#define XEN_TEST_UPCALL_MASK(r)             \
    testb   $1, VCPU_INFO_EVTCHN_UPCALL_MASK(r)

#define XEN_CLEAR_UPCALL_MASK(r)            \
    ASSERT_UPCALL_MASK_IS_SET;          \
    movb    $0, VCPU_INFO_EVTCHN_UPCALL_MASK(r)

#ifdef DEBUG

/*
 * Much logic depends on the upcall mask being set at
 * various points in the code; use this macro to validate.
 *
 * Need to use CURVCPU(r) to establish the vcpu pointer.
 */
#if defined(__amd64)

#define ASSERT_UPCALL_MASK_IS_SET           \
    pushq   %r11;                   \
    CURVCPU(%r11);                  \
    XEN_TEST_UPCALL_MASK(%r11);         \
    jne 6f;                 \
    cmpl    $0, stistipanic(%rip);          \
    jle 6f;                 \
    movl    $-1, stistipanic(%rip);         \
    movq    stistimsg(%rip), %rdi;          \
    xorl    %eax, %eax;             \
    call    panic;                  \
6:  pushq   %rax;                   \
    pushq   %rbx;                   \
    movl    %gs:CPU_ID, %eax;           \
    leaq    .+0(%rip), %r11;            \
    leaq    laststi(%rip), %rbx;            \
    movq    %r11, (%rbx, %rax, 8);          \
    popq    %rbx;                   \
    popq    %rax;                   \
    popq    %r11

#define SAVE_CLI_LOCATION               \
    pushq   %rax;                   \
    pushq   %rbx;                   \
    pushq   %rcx;                   \
    movl    %gs:CPU_ID, %eax;           \
    leaq    .+0(%rip), %rcx;            \
    leaq    lastcli, %rbx;              \
    movq    %rcx, (%rbx, %rax, 8);          \
    popq    %rcx;                   \
    popq    %rbx;                   \
    popq    %rax;                   \

#elif defined(__i386)

#define ASSERT_UPCALL_MASK_IS_SET           \
    pushl   %ecx;                   \
    CURVCPU(%ecx);                  \
    XEN_TEST_UPCALL_MASK(%ecx);         \
    jne 6f;                 \
    cmpl    $0, stistipanic;            \
    jle 6f;                 \
    movl    $-1, stistipanic;           \
    movl    stistimsg, %ecx;            \
    pushl   %ecx;                   \
    call    panic;                  \
6:  pushl   %eax;                   \
    pushl   %ebx;                   \
    movl    %gs:CPU_ID, %eax;           \
    leal    .+0, %ecx;              \
    leal    laststi, %ebx;              \
    movl    %ecx, (%ebx, %eax, 4);          \
    popl    %ebx;                   \
    popl    %eax;                   \
    popl    %ecx

#define SAVE_CLI_LOCATION               \
    pushl   %eax;                   \
    pushl   %ebx;                   \
    pushl   %ecx;                   \
    movl    %gs:CPU_ID, %eax;           \
    leal    .+0, %ecx;              \
    leal    lastcli, %ebx;              \
    movl    %ecx, (%ebx, %eax, 4);          \
    popl    %ecx;                   \
    popl    %ebx;                   \
    popl    %eax;                   \

#endif  /* __i386 */

#else   /* DEBUG */

#define ASSERT_UPCALL_MASK_IS_SET   /* empty */
#define SAVE_CLI_LOCATION       /* empty */

#endif  /* DEBUG */

#define KPREEMPT_DISABLE(t)             \
    addb    $1, T_PREEMPT(t)

#define KPREEMPT_ENABLE_NOKP(t)             \
    subb    $1, T_PREEMPT(t)

#define CLI(r)                      \
    CURTHREAD(r);                   \
    KPREEMPT_DISABLE(r);                \
    CURVCPU(r);                 \
    XEN_SET_UPCALL_MASK(r);             \
    SAVE_CLI_LOCATION;              \
    CURTHREAD(r);                   \
    KPREEMPT_ENABLE_NOKP(r)

#define CLIRET(r, ret)                  \
    CURTHREAD(r);                   \
    KPREEMPT_DISABLE(r);                \
    CURVCPU(r);                 \
    XEN_GET_UPCALL_MASK(r, ret);            \
    XEN_SET_UPCALL_MASK(r);             \
    SAVE_CLI_LOCATION;              \
    CURTHREAD(r);                   \
    KPREEMPT_ENABLE_NOKP(r)

/*
 * We use the fact that HYPERVISOR_block will clear the upcall mask
 * for us and then give us an upcall if there is a pending event
 * to achieve getting a callback on this cpu without the danger of
 * being preempted and migrating to another cpu between the upcall
 * enable and the callback delivery.
 */
#if defined(__amd64)

#define STI_CLOBBER     /* clobbers %rax, %rdi, %r11 */     \
    CURVCPU(%r11);                          \
    ASSERT_UPCALL_MASK_IS_SET;                  \
    movw    $0x100, %ax;    /* assume mask set, pending clear */    \
    movw    $0, %di;    /* clear mask and pending */        \
    lock;                               \
    cmpxchgw %di, VCPU_INFO_EVTCHN_UPCALL_PENDING(%r11);        \
    jz  7f;     /* xchg worked, we're done */       \
    movl    $__HYPERVISOR_sched_op, %eax; /* have pending upcall */ \
    movl    $SCHEDOP_block, %edi;                   \
    pushq   %rsi;   /* hypercall clobbers C param regs plus r10 */  \
    pushq   %rcx;                           \
    pushq   %rdx;                           \
    pushq   %r8;                            \
    pushq   %r9;                            \
    pushq   %r10;                           \
    TRAP_INSTR; /* clear upcall mask, force upcall */       \
    popq    %r10;                           \
    popq    %r9;                            \
    popq    %r8;                            \
    popq    %rdx;                           \
    popq    %rcx;                           \
    popq    %rsi;                           \
7:

#define STI                             \
    pushq   %r11;                           \
    pushq   %rdi;                           \
    pushq   %rax;                           \
    STI_CLOBBER;    /* clobbers %r11, %rax, %rdi */         \
    popq    %rax;                           \
    popq    %rdi;                           \
    popq    %r11

#elif defined(__i386)

#define STI_CLOBBER     /* clobbers %eax, %ebx, %ecx */     \
    CURVCPU(%ecx);                          \
    ASSERT_UPCALL_MASK_IS_SET;                  \
    movw    $0x100, %ax;    /* assume mask set, pending clear */    \
    movw    $0, %bx;    /* clear mask and pending */        \
    lock;                               \
    cmpxchgw %bx, VCPU_INFO_EVTCHN_UPCALL_PENDING(%ecx);        \
    jz  7f;     /* xchg worked, we're done */       \
    movl    $__HYPERVISOR_sched_op, %eax; /* have pending upcall */ \
    movl    $SCHEDOP_block, %ebx;                   \
    TRAP_INSTR;     /* clear upcall mask, force upcall */   \
7:

#define STI                     \
    pushl   %eax;                   \
    pushl   %ebx;                   \
    pushl   %ecx;                   \
    STI_CLOBBER;    /* clobbers %eax, %ebx, %ecx */ \
    popl    %ecx;                   \
    popl    %ebx;                   \
    popl    %eax

#endif  /* __i386 */

/*
 * Map the PS_IE bit to the hypervisor's event mask bit
 * To -set- the event mask, we have to do a CLI
 * To -clear- the event mask, we have to do a STI
 * (with all the accompanying pre-emption and callbacks, ick)
 *
 * And vice versa.
 */

#if defined(__amd64)

#define IE_TO_EVENT_MASK(rtmp, rfl)     \
    testq   $PS_IE, rfl;            \
    jnz 4f;             \
    CLI(rtmp);              \
    jmp 5f;             \
4:  STI;                    \
5:

#define EVENT_MASK_TO_IE(rtmp, rfl)     \
    andq    $_BITNOT(PS_IE), rfl;       \
    CURVCPU(rtmp);              \
    XEN_TEST_UPCALL_MASK(rtmp);     \
    jnz 1f;             \
    orq $PS_IE, rfl;            \
1:

#elif defined(__i386)

#define IE_TO_EVENT_MASK(rtmp, rfl)     \
    testl   $PS_IE, rfl;            \
    jnz 4f;             \
    CLI(rtmp);              \
    jmp 5f;             \
4:  STI;                    \
5:

#define EVENT_MASK_TO_IE(rtmp, rfl)     \
    andl    $_BITNOT(PS_IE), rfl;       \
    CURVCPU(rtmp);              \
    XEN_TEST_UPCALL_MASK(rtmp);     \
    jnz 1f;             \
    orl $PS_IE, rfl;            \
1:

#endif  /* __i386 */

/*
 * Used to re-enable interrupts in the body of exception handlers
 */

#if defined(__amd64)

#define ENABLE_INTR_FLAGS       \
    pushq   $F_ON;          \
    popfq;              \
    STI

#elif defined(__i386)

#define ENABLE_INTR_FLAGS       \
    pushl   $F_ON;          \
    popfl;              \
    STI

#endif  /* __i386 */

/*
 * Virtualize IRET and SYSRET
 */

#if defined(__amd64)

#if defined(DEBUG)

/*
 * Die nastily with a #ud trap if we are about to switch to user
 * mode in HYPERVISOR_IRET and RUPDATE_PENDING is set.
 */
#define __ASSERT_NO_RUPDATE_PENDING         \
    pushq   %r15;                   \
    cmpw    $KCS_SEL, 0x10(%rsp);           \
    je  1f;                 \
    movq    %gs:CPU_THREAD, %r15;           \
    movq    T_LWP(%r15), %r15;          \
    testb   $0x1, PCB_RUPDATE(%r15);        \
    je  1f;                 \
    ud2;                        \
1:  popq    %r15

#else   /* DEBUG */

#define __ASSERT_NO_RUPDATE_PENDING

#endif  /* DEBUG */

/*
 * Switching from guest kernel to user mode.
 * flag == VGCF_IN_SYSCALL => return via sysret
 * flag == 0 => return via iretq
 *
 * See definition in public/arch-x86_64.h. Stack going in must be:
 * rax, r11, rcx, flags, rip, cs, rflags, rsp, ss.
 */
#define HYPERVISOR_IRET(flag)           \
    __ASSERT_NO_RUPDATE_PENDING;        \
    pushq   $flag;              \
    pushq   %rcx;               \
    pushq   %r11;               \
    pushq   %rax;               \
    movl    $__HYPERVISOR_iret, %eax;   \
    syscall;                \
    ud2 /* die nastily if we return! */

#define IRET    HYPERVISOR_IRET(0)
#define SYSRETQ HYPERVISOR_IRET(VGCF_IN_SYSCALL)
#define SYSRETL ud2     /* 32-bit syscall/sysret not supported */
#define SWAPGS  /* empty - handled in hypervisor */

#elif defined(__i386)

/*
 * Switching from guest kernel to user mode.
 * See definition in public/arch-x86_32.h. Stack going in must be:
 * eax, flags, eip, cs, eflags, esp, ss.
 */
#define HYPERVISOR_IRET             \
    pushl   %eax;               \
    movl    $__HYPERVISOR_iret, %eax;   \
    int $0x82;              \
    ud2 /* die nastily if we return! */

#define IRET    HYPERVISOR_IRET
#define SYSRET  ud2     /* 32-bit syscall/sysret not supported */

#endif  /* __i386 */


/*
 * Xen 3.x wedges the current value of upcall_mask into unused byte of
 * saved %cs on stack at the time of passing through a trap or interrupt
 * gate.  Since Xen also updates PS_IE in %[e,r]lags as well, we always
 * mask off the saved upcall mask so the kernel and/or tools like debuggers
 * will not be confused about bits set in reserved portions of %cs slot.
 *
 * See xen/include/public/arch-x86_[32,64].h:cpu_user_regs_t for details.
 */
#if defined(__amd64)

#define CLEAN_CS    movb    $0, REGOFF_CS+4(%rsp)

#elif defined(__i386)

#define CLEAN_CS    movb    $0, REGOFF_CS+2(%esp)

#endif  /* __i386 */

/*
 * All exceptions for amd64 have %r11 and %rcx on the stack.
 * Just pop them back into their appropriate registers and
 * let it get saved as is running native.
 */
#if defined(__amd64)

#define XPV_TRAP_POP    \
    popq    %rcx;   \
    popq    %r11

#define XPV_TRAP_PUSH   \
    pushq   %r11;   \
    pushq   %rcx

#endif  /* __amd64 */


/*
 * Macros for saving the original segment registers and restoring them
 * for fast traps.
 */
#if defined(__amd64)

/*
 * Smaller versions of INTR_PUSH and INTR_POP for fast traps.
 * The following registers have been pushed onto the stack by
 * hardware at this point:
 *
 *  greg_t  r_rip;
 *  greg_t  r_cs;
 *  greg_t  r_rfl;
 *  greg_t  r_rsp;
 *  greg_t  r_ss;
 *
 * This handler is executed both by 32-bit and 64-bit applications.
 * 64-bit applications allow us to treat the set (%rdi, %rsi, %rdx,
 * %rcx, %r8, %r9, %r10, %r11, %rax) as volatile across function calls.
 * However, 32-bit applications only expect (%eax, %edx, %ecx) to be volatile
 * across a function call -- in particular, %esi and %edi MUST be saved!
 *
 * We could do this differently by making a FAST_INTR_PUSH32 for 32-bit
 * programs, and FAST_INTR_PUSH for 64-bit programs, but it doesn't seem
 * particularly worth it.
 *
 */
#define FAST_INTR_PUSH          \
    INTGATE_INIT_KERNEL_FLAGS;  \
    popq    %rcx;           \
    popq    %r11;           \
    subq    $REGOFF_RIP, %rsp;  \
    movq    %rsi, REGOFF_RSI(%rsp); \
    movq    %rdi, REGOFF_RDI(%rsp); \
    CLEAN_CS

#define FAST_INTR_POP           \
    movq    REGOFF_RSI(%rsp), %rsi; \
    movq    REGOFF_RDI(%rsp), %rdi; \
    addq    $REGOFF_RIP, %rsp

#define FAST_INTR_RETURN        \
    ASSERT_UPCALL_MASK_IS_SET;  \
    HYPERVISOR_IRET(0)

#elif defined(__i386)

#define FAST_INTR_PUSH          \
    cld;                \
    __SEGREGS_PUSH          \
    __SEGREGS_LOAD_KERNEL       \

#define FAST_INTR_POP           \
    __SEGREGS_POP

#define FAST_INTR_RETURN        \
    IRET

#endif  /* __i386 */

/*
 * Handling the CR0.TS bit for floating point handling.
 *
 * When the TS bit is *set*, attempts to touch the floating
 * point hardware will result in a #nm trap.
 */
#if defined(__amd64)

#define STTS(rtmp)              \
    pushq   %rdi;               \
    movl    $1, %edi;           \
    call    HYPERVISOR_fpu_taskswitch;  \
    popq    %rdi

#define CLTS                    \
    pushq   %rdi;               \
    xorl    %edi, %edi;         \
    call    HYPERVISOR_fpu_taskswitch;  \
    popq    %rdi

#elif defined(__i386)

#define STTS(r)                 \
    pushl   $1;             \
    call    HYPERVISOR_fpu_taskswitch;  \
    addl    $4, %esp

#define CLTS                    \
    pushl   $0;             \
    call    HYPERVISOR_fpu_taskswitch;  \
    addl    $4, %esp

#endif  /* __i386 */

#ifdef __cplusplus
}
#endif

#endif  /* _SYS_MACHPRIVREGS_H */