SVM rework

git-svn-id: svn://svn.savannah.nongnu.org/qemu/trunk@4605 c046a42c-6fe2-441c-8c8c-71466251a162

7 files changed, 383 insertions, 477 deletions

diff --git a/target-i386/TODO b/target-i386/TODO
index 9fd1bbeb3b..b671e24b4e 100644
--- a/target-i386/TODO
+++ b/target-i386/TODO
@@ -1,9 +1,7 @@
 Correctness issues:
 
 - some eflags manipulation incorrectly reset the bit 0x2.
-- SVM: rework the implementation: simplify code, move most intercept
-  tests as dynamic, correct segment access, verify exception safety,
-  cpu save/restore, SMM save/restore. 
+- SVM: test, cpu save/restore, SMM save/restore. 
 - x86_64: lcall/ljmp intel/amd differences ?
 - better code fetch (different exception handling + CS.limit support)
 - user/kernel PUSHL/POPL in helper.c
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 364485e008..a2b7ba54b4 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -149,6 +149,8 @@
 #define HF_GIF_SHIFT        20 /* if set CPU takes interrupts */
 #define HF_HIF_SHIFT        21 /* shadow copy of IF_MASK when in SVM */
 #define HF_NMI_SHIFT        22 /* CPU serving NMI */
+#define HF_SVME_SHIFT       23 /* SVME enabled (copy of EFER.SVME */
+#define HF_SVMI_SHIFT       24 /* SVM intercepts are active */
 
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
 #define HF_SOFTMMU_MASK      (1 << HF_SOFTMMU_SHIFT)
@@ -169,6 +171,8 @@
 #define HF_GIF_MASK          (1 << HF_GIF_SHIFT)
 #define HF_HIF_MASK          (1 << HF_HIF_SHIFT)
 #define HF_NMI_MASK          (1 << HF_NMI_SHIFT)
+#define HF_SVME_MASK         (1 << HF_SVME_SHIFT)
+#define HF_SVMI_MASK         (1 << HF_SVMI_SHIFT)
 
 #define CR0_PE_MASK  (1 << 0)
 #define CR0_MP_MASK  (1 << 1)
@@ -242,6 +246,7 @@
 #define MSR_EFER_LME   (1 << 8)
 #define MSR_EFER_LMA   (1 << 10)
 #define MSR_EFER_NXE   (1 << 11)
+#define MSR_EFER_SVME  (1 << 12)
 #define MSR_EFER_FFXSR (1 << 14)
 
 #define MSR_STAR                        0xc0000081
@@ -322,6 +327,7 @@
 #define CPUID_EXT3_3DNOWPREFETCH (1 << 8)
 #define CPUID_EXT3_OSVW    (1 << 9)
 #define CPUID_EXT3_IBS     (1 << 10)
+#define CPUID_EXT3_SKINIT  (1 << 12)
 
 #define EXCP00_DIVZ	0
 #define EXCP01_SSTP	1
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 133754ddc1..7ce3de32c0 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -1096,16 +1096,15 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
         (env->efer & MSR_EFER_NXE) &&
         (env->cr[4] & CR4_PAE_MASK))
         error_code |= PG_ERROR_I_D_MASK;
-    if (INTERCEPTEDl(_exceptions, 1 << EXCP0E_PAGE)) {
-        stq_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2), addr);
+    if (env->intercept_exceptions & (1 << EXCP0E_PAGE)) {
+        /* cr2 is not modified in case of exceptions */
+        stq_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2), 
+                 addr);
     } else {
         env->cr[2] = addr;
     }
     env->error_code = error_code;
     env->exception_index = EXCP0E_PAGE;
-    /* the VMM will handle this */
-    if (INTERCEPTEDl(_exceptions, 1 << EXCP0E_PAGE))
-        return 2;
     return 1;
 }
 
diff --git a/target-i386/helper.h b/target-i386/helper.h
index 8e8f70b55b..3fd99d2673 100644
--- a/target-i386/helper.h
+++ b/target-i386/helper.h
@@ -43,7 +43,8 @@ DEF_HELPER(void, helper_lcall_protected, (int new_cs, target_ulong new_eip,
 DEF_HELPER(void, helper_iret_real, (int shift))
 DEF_HELPER(void, helper_iret_protected, (int shift, int next_eip))
 DEF_HELPER(void, helper_lret_protected, (int shift, int addend))
-DEF_HELPER(void, helper_movl_crN_T0, (int reg, target_ulong t0))
+DEF_HELPER(target_ulong, helper_read_crN, (int reg))
+DEF_HELPER(void, helper_write_crN, (int reg, target_ulong t0))
 DEF_HELPER(void, helper_lmsw, (target_ulong t0))
 DEF_HELPER(void, helper_clts, (void))
 #if !defined(CONFIG_USER_ONLY)
diff --git a/target-i386/op_helper.c b/target-i386/op_helper.c
index cab085ad53..b5f04f524d 100644
--- a/target-i386/op_helper.c
+++ b/target-i386/op_helper.c
@@ -625,18 +625,7 @@ static void do_interrupt_protected(int intno, int is_int, int error_code,
     int has_error_code, new_stack, shift;
     uint32_t e1, e2, offset, ss, esp, ss_e1, ss_e2;
     uint32_t old_eip, sp_mask;
-    int svm_should_check = 1;
 
-    if ((env->intercept & INTERCEPT_SVM_MASK) && !is_int && next_eip==-1) {
-        next_eip = EIP;
-        svm_should_check = 0;
-    }
-
-    if (svm_should_check
-        && (INTERCEPTEDl(_exceptions, 1 << intno)
-        && !is_int)) {
-        raise_interrupt(intno, is_int, error_code, 0);
-    }
     has_error_code = 0;
     if (!is_int && !is_hw) {
         switch(intno) {
@@ -872,17 +861,7 @@ static void do_interrupt64(int intno, int is_int, int error_code,
     int has_error_code, new_stack;
     uint32_t e1, e2, e3, ss;
     target_ulong old_eip, esp, offset;
-    int svm_should_check = 1;
 
-    if ((env->intercept & INTERCEPT_SVM_MASK) && !is_int && next_eip==-1) {
-        next_eip = EIP;
-        svm_should_check = 0;
-    }
-    if (svm_should_check
-        && INTERCEPTEDl(_exceptions, 1 << intno)
-        && !is_int) {
-        raise_interrupt(intno, is_int, error_code, 0);
-    }
     has_error_code = 0;
     if (!is_int && !is_hw) {
         switch(intno) {
@@ -1139,17 +1118,7 @@ static void do_interrupt_real(int intno, int is_int, int error_code,
     int selector;
     uint32_t offset, esp;
     uint32_t old_cs, old_eip;
-    int svm_should_check = 1;
 
-    if ((env->intercept & INTERCEPT_SVM_MASK) && !is_int && next_eip==-1) {
-        next_eip = EIP;
-        svm_should_check = 0;
-    }
-    if (svm_should_check
-        && INTERCEPTEDl(_exceptions, 1 << intno)
-        && !is_int) {
-        raise_interrupt(intno, is_int, error_code, 0);
-    }
     /* real mode (simpler !) */
     dt = &env->idt;
     if (intno * 4 + 3 > dt->limit)
@@ -1307,6 +1276,8 @@ void raise_interrupt(int intno, int is_int, int error_code,
     if (!is_int) {
         helper_svm_check_intercept_param(SVM_EXIT_EXCP_BASE + intno, error_code);
         intno = check_exception(intno, &error_code);
+    } else {
+        helper_svm_check_intercept_param(SVM_EXIT_SWINT, 0);
     }
 
     env->exception_index = intno;
@@ -1316,18 +1287,6 @@ void raise_interrupt(int intno, int is_int, int error_code,
     cpu_loop_exit();
 }
 
-/* same as raise_exception_err, but do not restore global registers */
-static void raise_exception_err_norestore(int exception_index, int error_code)
-{
-    exception_index = check_exception(exception_index, &error_code);
-
-    env->exception_index = exception_index;
-    env->error_code = error_code;
-    env->exception_is_int = 0;
-    env->exception_next_eip = 0;
-    longjmp(env->jmp_env, 1);
-}
-
 /* shortcuts to generate exceptions */
 
 void (raise_exception_err)(int exception_index, int error_code)
@@ -1921,8 +1880,10 @@ void helper_single_step(void)
 void helper_cpuid(void)
 {
     uint32_t index;
-    index = (uint32_t)EAX;
 
+    helper_svm_check_intercept_param(SVM_EXIT_CPUID, 0);
+    
+    index = (uint32_t)EAX;
     /* test if maximum index reached */
     if (index & 0x80000000) {
         if (index > env->cpuid_xlevel)
@@ -2957,9 +2918,35 @@ void helper_sysexit(void)
 #endif
 }
 
-void helper_movl_crN_T0(int reg, target_ulong t0)
+#if defined(CONFIG_USER_ONLY)
+target_ulong helper_read_crN(int reg)
 {
-#if !defined(CONFIG_USER_ONLY)
+    return 0;
+}
+
+void helper_write_crN(int reg, target_ulong t0)
+{
+}
+#else
+target_ulong helper_read_crN(int reg)
+{
+    target_ulong val;
+
+    helper_svm_check_intercept_param(SVM_EXIT_READ_CR0 + reg, 0);
+    switch(reg) {
+    default:
+        val = env->cr[reg];
+        break;
+    case 8:
+        val = cpu_get_apic_tpr(env);
+        break;
+    }
+    return val;
+}
+
+void helper_write_crN(int reg, target_ulong t0)
+{
+    helper_svm_check_intercept_param(SVM_EXIT_WRITE_CR0 + reg, 0);
     switch(reg) {
     case 0:
         cpu_x86_update_cr0(env, t0);
@@ -2978,15 +2965,15 @@ void helper_movl_crN_T0(int reg, target_ulong t0)
         env->cr[reg] = t0;
         break;
     }
-#endif
 }
+#endif
 
 void helper_lmsw(target_ulong t0)
 {
     /* only 4 lower bits of CR0 are modified. PE cannot be set to zero
        if already set to one. */
     t0 = (env->cr[0] & ~0xe) | (t0 & 0xf);
-    helper_movl_crN_T0(0, t0);
+    helper_write_crN(0, t0);
 }
 
 void helper_clts(void)
@@ -3010,6 +2997,7 @@ void helper_movl_drN_T0(int reg, target_ulong t0)
 
 void helper_invlpg(target_ulong addr)
 {
+    helper_svm_check_intercept_param(SVM_EXIT_INVLPG, 0);
     cpu_x86_flush_tlb(env, addr);
 }
 
@@ -3020,6 +3008,8 @@ void helper_rdtsc(void)
     if ((env->cr[4] & CR4_TSD_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
         raise_exception(EXCP0D_GPF);
     }
+    helper_svm_check_intercept_param(SVM_EXIT_RDTSC, 0);
+
     val = cpu_get_tsc(env);
     EAX = (uint32_t)(val);
     EDX = (uint32_t)(val >> 32);
@@ -3030,7 +3020,6 @@ void helper_rdpmc(void)
     if ((env->cr[4] & CR4_PCE_MASK) && ((env->hflags & HF_CPL_MASK) != 0)) {
         raise_exception(EXCP0D_GPF);
     }
-
     helper_svm_check_intercept_param(SVM_EXIT_RDPMC, 0);
     
     /* currently unimplemented */
@@ -3050,6 +3039,8 @@ void helper_wrmsr(void)
 {
     uint64_t val;
 
+    helper_svm_check_intercept_param(SVM_EXIT_MSR, 1);
+
     val = ((uint32_t)EAX) | ((uint64_t)((uint32_t)EDX) << 32);
 
     switch((uint32_t)ECX) {
@@ -3119,6 +3110,9 @@ void helper_wrmsr(void)
 void helper_rdmsr(void)
 {
     uint64_t val;
+
+    helper_svm_check_intercept_param(SVM_EXIT_MSR, 0);
+
     switch((uint32_t)ECX) {
     case MSR_IA32_SYSENTER_CS:
         val = env->sysenter_cs;
@@ -4549,6 +4543,8 @@ void helper_idivq_EAX(target_ulong t0)
 
 void helper_hlt(void)
 {
+    helper_svm_check_intercept_param(SVM_EXIT_HLT, 0);
+    
     env->hflags &= ~HF_INHIBIT_IRQ_MASK; /* needed if sti is just before */
     env->hflags |= HF_HALTED_MASK;
     env->exception_index = EXCP_HLT;
@@ -4560,12 +4556,14 @@ void helper_monitor(target_ulong ptr)
     if ((uint32_t)ECX != 0)
         raise_exception(EXCP0D_GPF);
     /* XXX: store address ? */
+    helper_svm_check_intercept_param(SVM_EXIT_MONITOR, 0);
 }
 
 void helper_mwait(void)
 {
     if ((uint32_t)ECX != 0)
         raise_exception(EXCP0D_GPF);
+    helper_svm_check_intercept_param(SVM_EXIT_MWAIT, 0);
     /* XXX: not complete but not completely erroneous */
     if (env->cpu_index != 0 || env->next_cpu != NULL) {
         /* more than one CPU: do not sleep because another CPU may
@@ -4706,10 +4704,7 @@ void tlb_fill(target_ulong addr, int is_write, int mmu_idx, void *retaddr)
                 cpu_restore_state(tb, env, pc, NULL);
             }
         }
-        if (retaddr)
-            raise_exception_err(env->exception_index, env->error_code);
-        else
-            raise_exception_err_norestore(env->exception_index, env->error_code);
+        raise_exception_err(env->exception_index, env->error_code);
     }
     env = saved_env;
 }
@@ -4717,16 +4712,6 @@ void tlb_fill(target_ulong addr, int is_write, int mmu_idx, void *retaddr)
 
 /* Secure Virtual Machine helpers */
 
-void helper_stgi(void)
-{
-    env->hflags |= HF_GIF_MASK;
-}
-
-void helper_clgi(void)
-{
-    env->hflags &= ~HF_GIF_MASK;
-}
-
 #if defined(CONFIG_USER_ONLY)
 
 void helper_vmrun(void) 
@@ -4741,6 +4726,12 @@ void helper_vmload(void)
 void helper_vmsave(void) 
 { 
 }
+void helper_stgi(void)
+{
+}
+void helper_clgi(void)
+{
+}
 void helper_skinit(void) 
 { 
 }
@@ -4760,20 +4751,37 @@ void helper_svm_check_io(uint32_t port, uint32_t param,
 }
 #else
 
-static inline uint32_t
-vmcb2cpu_attrib(uint16_t vmcb_attrib, uint32_t vmcb_base, uint32_t vmcb_limit)
+static inline void svm_save_seg(target_phys_addr_t addr,
+                                const SegmentCache *sc)
 {
-    return    ((vmcb_attrib & 0x00ff) << 8)          /* Type, S, DPL, P */
-	    | ((vmcb_attrib & 0x0f00) << 12)         /* AVL, L, DB, G */
-	    | ((vmcb_base >> 16) & 0xff)             /* Base 23-16 */
-	    | (vmcb_base & 0xff000000)               /* Base 31-24 */
-	    | (vmcb_limit & 0xf0000);                /* Limit 19-16 */
+    stw_phys(addr + offsetof(struct vmcb_seg, selector), 
+             sc->selector);
+    stq_phys(addr + offsetof(struct vmcb_seg, base), 
+             sc->base);
+    stl_phys(addr + offsetof(struct vmcb_seg, limit), 
+             sc->limit);
+    stw_phys(addr + offsetof(struct vmcb_seg, attrib), 
+             (sc->flags >> 8) | ((sc->flags >> 12) & 0x0f00));
+}
+                                
+static inline void svm_load_seg(target_phys_addr_t addr, SegmentCache *sc)
+{
+    unsigned int flags;
+
+    sc->selector = lduw_phys(addr + offsetof(struct vmcb_seg, selector));
+    sc->base = ldq_phys(addr + offsetof(struct vmcb_seg, base));
+    sc->limit = ldl_phys(addr + offsetof(struct vmcb_seg, limit));
+    flags = lduw_phys(addr + offsetof(struct vmcb_seg, attrib));
+    sc->flags = ((flags & 0xff) << 8) | ((flags & 0x0f00) << 12);
 }
 
-static inline uint16_t cpu2vmcb_attrib(uint32_t cpu_attrib)
+static inline void svm_load_seg_cache(target_phys_addr_t addr, 
+                                      CPUState *env, int seg_reg)
 {
-    return    ((cpu_attrib >> 8) & 0xff)             /* Type, S, DPL, P */
-	    | ((cpu_attrib & 0xf00000) >> 12);       /* AVL, L, DB, G */
+    SegmentCache sc1, *sc = &sc1;
+    svm_load_seg(addr, sc);
+    cpu_x86_load_seg_cache(env, seg_reg, sc->selector,
+                           sc->base, sc->limit, sc->flags);
 }
 
 void helper_vmrun(void)
@@ -4782,6 +4790,8 @@ void helper_vmrun(void)
     uint32_t event_inj;
     uint32_t int_ctl;
 
+    helper_svm_check_intercept_param(SVM_EXIT_VMRUN, 0);
+
     addr = EAX;
     if (loglevel & CPU_LOG_TB_IN_ASM)
         fprintf(logfile,"vmrun! " TARGET_FMT_lx "\n", addr);
@@ -4806,10 +4816,14 @@ void helper_vmrun(void)
     stq_phys(env->vm_hsave + offsetof(struct vmcb, save.efer), env->efer);
     stq_phys(env->vm_hsave + offsetof(struct vmcb, save.rflags), compute_eflags());
 
-    SVM_SAVE_SEG(env->vm_hsave, segs[R_ES], es);
-    SVM_SAVE_SEG(env->vm_hsave, segs[R_CS], cs);
-    SVM_SAVE_SEG(env->vm_hsave, segs[R_SS], ss);
-    SVM_SAVE_SEG(env->vm_hsave, segs[R_DS], ds);
+    svm_save_seg(env->vm_hsave + offsetof(struct vmcb, save.es), 
+                  &env->segs[R_ES]);
+    svm_save_seg(env->vm_hsave + offsetof(struct vmcb, save.cs), 
+                 &env->segs[R_CS]);
+    svm_save_seg(env->vm_hsave + offsetof(struct vmcb, save.ss), 
+                 &env->segs[R_SS]);
+    svm_save_seg(env->vm_hsave + offsetof(struct vmcb, save.ds), 
+                 &env->segs[R_DS]);
 
     stq_phys(env->vm_hsave + offsetof(struct vmcb, save.rip), EIP);
     stq_phys(env->vm_hsave + offsetof(struct vmcb, save.rsp), ESP);
@@ -4817,15 +4831,16 @@ void helper_vmrun(void)
 
     /* load the interception bitmaps so we do not need to access the
        vmcb in svm mode */
-    /* We shift all the intercept bits so we can OR them with the TB
-       flags later on */
-    env->intercept            = (ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept)) << INTERCEPT_INTR) | INTERCEPT_SVM_MASK;
+    env->intercept            = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept));
     env->intercept_cr_read    = lduw_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_cr_read));
     env->intercept_cr_write   = lduw_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_cr_write));
     env->intercept_dr_read    = lduw_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_dr_read));
     env->intercept_dr_write   = lduw_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_dr_write));
     env->intercept_exceptions = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.intercept_exceptions));
 
+    /* enable intercepts */
+    env->hflags |= HF_SVMI_MASK;
+
     env->gdt.base  = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.gdtr.base));
     env->gdt.limit = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, save.gdtr.limit));
 
@@ -4857,12 +4872,15 @@ void helper_vmrun(void)
     load_eflags(ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rflags)),
                 ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK));
     CC_OP = CC_OP_EFLAGS;
-    CC_DST = 0xffffffff;
 
-    SVM_LOAD_SEG(env->vm_vmcb, ES, es);
-    SVM_LOAD_SEG(env->vm_vmcb, CS, cs);
-    SVM_LOAD_SEG(env->vm_vmcb, SS, ss);
-    SVM_LOAD_SEG(env->vm_vmcb, DS, ds);
+    svm_load_seg_cache(env->vm_vmcb + offsetof(struct vmcb, save.es),
+                       env, R_ES);
+    svm_load_seg_cache(env->vm_vmcb + offsetof(struct vmcb, save.cs),
+                       env, R_CS);
+    svm_load_seg_cache(env->vm_vmcb + offsetof(struct vmcb, save.ss),
+                       env, R_SS);
+    svm_load_seg_cache(env->vm_vmcb + offsetof(struct vmcb, save.ds),
+                       env, R_DS);
 
     EIP = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, save.rip));
     env->eip = EIP;
@@ -4933,7 +4951,8 @@ void helper_vmrun(void)
         if (loglevel & CPU_LOG_TB_IN_ASM)
             fprintf(logfile, " %#x %#x\n", env->exception_index, env->error_code);
     }
-    if ((int_ctl & V_IRQ_MASK) || (env->intercept & INTERCEPT_VINTR)) {
+    if ((int_ctl & V_IRQ_MASK) || 
+        (env->intercept & (1ULL << (SVM_EXIT_INTR - SVM_EXIT_INTR)))) {
         env->interrupt_request |= CPU_INTERRUPT_VIRQ;
     }
 
@@ -4942,23 +4961,30 @@ void helper_vmrun(void)
 
 void helper_vmmcall(void)
 {
-    if (loglevel & CPU_LOG_TB_IN_ASM)
-        fprintf(logfile,"vmmcall!\n");
+    helper_svm_check_intercept_param(SVM_EXIT_VMMCALL, 0);
+    raise_exception(EXCP06_ILLOP);
 }
 
 void helper_vmload(void)
 {
     target_ulong addr;
+    helper_svm_check_intercept_param(SVM_EXIT_VMLOAD, 0);
+
+    /* XXX: invalid in 32 bit */
     addr = EAX;
     if (loglevel & CPU_LOG_TB_IN_ASM)
         fprintf(logfile,"vmload! " TARGET_FMT_lx "\nFS: %016" PRIx64 " | " TARGET_FMT_lx "\n",
                 addr, ldq_phys(addr + offsetof(struct vmcb, save.fs.base)),
                 env->segs[R_FS].base);
 
-    SVM_LOAD_SEG2(addr, segs[R_FS], fs);
-    SVM_LOAD_SEG2(addr, segs[R_GS], gs);
-    SVM_LOAD_SEG2(addr, tr, tr);
-    SVM_LOAD_SEG2(addr, ldt, ldtr);
+    svm_load_seg_cache(addr + offsetof(struct vmcb, save.fs),
+                       env, R_FS);
+    svm_load_seg_cache(addr + offsetof(struct vmcb, save.gs),
+                       env, R_GS);
+    svm_load_seg(addr + offsetof(struct vmcb, save.tr),
+                 &env->tr);
+    svm_load_seg(addr + offsetof(struct vmcb, save.ldtr),
+                 &env->ldt);
 
 #ifdef TARGET_X86_64
     env->kernelgsbase = ldq_phys(addr + offsetof(struct vmcb, save.kernel_gs_base));
@@ -4975,16 +5001,21 @@ void helper_vmload(void)
 void helper_vmsave(void)
 {
     target_ulong addr;
+    helper_svm_check_intercept_param(SVM_EXIT_VMSAVE, 0);
     addr = EAX;
     if (loglevel & CPU_LOG_TB_IN_ASM)
         fprintf(logfile,"vmsave! " TARGET_FMT_lx "\nFS: %016" PRIx64 " | " TARGET_FMT_lx "\n",
                 addr, ldq_phys(addr + offsetof(struct vmcb, save.fs.base)),
                 env->segs[R_FS].base);
 
-    SVM_SAVE_SEG(addr, segs[R_FS], fs);
-    SVM_SAVE_SEG(addr, segs[R_GS], gs);
-    SVM_SAVE_SEG(addr, tr, tr);
-    SVM_SAVE_SEG(addr, ldt, ldtr);
+    svm_save_seg(addr + offsetof(struct vmcb, save.fs), 
+                 &env->segs[R_FS]);
+    svm_save_seg(addr + offsetof(struct vmcb, save.gs), 
+                 &env->segs[R_GS]);
+    svm_save_seg(addr + offsetof(struct vmcb, save.tr), 
+                 &env->tr);
+    svm_save_seg(addr + offsetof(struct vmcb, save.ldtr), 
+                 &env->ldt);
 
 #ifdef TARGET_X86_64
     stq_phys(addr + offsetof(struct vmcb, save.kernel_gs_base), env->kernelgsbase);
@@ -4998,50 +5029,65 @@ void helper_vmsave(void)
     stq_phys(addr + offsetof(struct vmcb, save.sysenter_eip), env->sysenter_eip);
 }
 
+void helper_stgi(void)
+{
+    helper_svm_check_intercept_param(SVM_EXIT_STGI, 0);
+    env->hflags |= HF_GIF_MASK;
+}
+
+void helper_clgi(void)
+{
+    helper_svm_check_intercept_param(SVM_EXIT_CLGI, 0);
+    env->hflags &= ~HF_GIF_MASK;
+}
+
 void helper_skinit(void)
 {
+    helper_svm_check_intercept_param(SVM_EXIT_SKINIT, 0);
+    /* XXX: not implemented */
     if (loglevel & CPU_LOG_TB_IN_ASM)
         fprintf(logfile,"skinit!\n");
+    raise_exception(EXCP06_ILLOP);
 }
 
 void helper_invlpga(void)
 {
+    helper_svm_check_intercept_param(SVM_EXIT_INVLPGA, 0);
     tlb_flush(env, 0);
 }
 
 void helper_svm_check_intercept_param(uint32_t type, uint64_t param)
 {
+    if (likely(!(env->hflags & HF_SVMI_MASK)))
+        return;
     switch(type) {
     case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR0 + 8:
-        if (INTERCEPTEDw(_cr_read, (1 << (type - SVM_EXIT_READ_CR0)))) {
+        if (env->intercept_cr_read & (1 << (type - SVM_EXIT_READ_CR0))) {
             helper_vmexit(type, param);
         }
         break;
-    case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR0 + 8:
-        if (INTERCEPTEDw(_dr_read, (1 << (type - SVM_EXIT_READ_DR0)))) {
+    case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR0 + 8:
+        if (env->intercept_cr_write & (1 << (type - SVM_EXIT_WRITE_CR0))) {
             helper_vmexit(type, param);
         }
         break;
-    case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR0 + 8:
-        if (INTERCEPTEDw(_cr_write, (1 << (type - SVM_EXIT_WRITE_CR0)))) {
+    case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR0 + 7:
+        if (env->intercept_dr_read & (1 << (type - SVM_EXIT_READ_DR0))) {
             helper_vmexit(type, param);
         }
         break;
-    case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR0 + 8:
-        if (INTERCEPTEDw(_dr_write, (1 << (type - SVM_EXIT_WRITE_DR0)))) {
+    case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR0 + 7:
+        if (env->intercept_dr_write & (1 << (type - SVM_EXIT_WRITE_DR0))) {
             helper_vmexit(type, param);
         }
         break;
-    case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 16:
-        if (INTERCEPTEDl(_exceptions, (1 << (type - SVM_EXIT_EXCP_BASE)))) {
+    case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 31:
+        if (env->intercept_exceptions & (1 << (type - SVM_EXIT_EXCP_BASE))) {
             helper_vmexit(type, param);
         }
         break;
-    case SVM_EXIT_IOIO:
-        break;
-
     case SVM_EXIT_MSR:
-        if (INTERCEPTED(1ULL << INTERCEPT_MSR_PROT)) {
+        if (env->intercept & (1ULL << (SVM_EXIT_MSR - SVM_EXIT_INTR))) {
             /* FIXME: this should be read in at vmrun (faster this way?) */
             uint64_t addr = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.msrpm_base_pa));
             uint32_t t0, t1;
@@ -5071,7 +5117,7 @@ void helper_svm_check_intercept_param(uint32_t type, uint64_t param)
         }
         break;
     default:
-        if (INTERCEPTED((1ULL << ((type - SVM_EXIT_INTR) + INTERCEPT_INTR)))) {
+        if (env->intercept & (1ULL << (type - SVM_EXIT_INTR))) {
             helper_vmexit(type, param);
         }
         break;
@@ -5081,7 +5127,7 @@ void helper_svm_check_intercept_param(uint32_t type, uint64_t param)
 void helper_svm_check_io(uint32_t port, uint32_t param, 
                          uint32_t next_eip_addend)
 {
-    if (INTERCEPTED(1ULL << INTERCEPT_IOIO_PROT)) {
+    if (env->intercept & (1ULL << (SVM_EXIT_IOIO - SVM_EXIT_INTR))) {
         /* FIXME: this should be read in at vmrun (faster this way?) */
         uint64_t addr = ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.iopm_base_pa));
         uint16_t mask = (1 << ((param >> 4) & 7)) - 1;
@@ -5113,10 +5159,14 @@ void helper_vmexit(uint32_t exit_code, uint64_t exit_info_1)
     }
 
     /* Save the VM state in the vmcb */
-    SVM_SAVE_SEG(env->vm_vmcb, segs[R_ES], es);
-    SVM_SAVE_SEG(env->vm_vmcb, segs[R_CS], cs);
-    SVM_SAVE_SEG(env->vm_vmcb, segs[R_SS], ss);
-    SVM_SAVE_SEG(env->vm_vmcb, segs[R_DS], ds);
+    svm_save_seg(env->vm_vmcb + offsetof(struct vmcb, save.es), 
+                 &env->segs[R_ES]);
+    svm_save_seg(env->vm_vmcb + offsetof(struct vmcb, save.cs), 
+                 &env->segs[R_CS]);
+    svm_save_seg(env->vm_vmcb + offsetof(struct vmcb, save.ss), 
+                 &env->segs[R_SS]);
+    svm_save_seg(env->vm_vmcb + offsetof(struct vmcb, save.ds), 
+                 &env->segs[R_DS]);
 
     stq_phys(env->vm_vmcb + offsetof(struct vmcb, save.gdtr.base), env->gdt.base);
     stl_phys(env->vm_vmcb + offsetof(struct vmcb, save.gdtr.limit), env->gdt.limit);
@@ -5146,6 +5196,7 @@ void helper_vmexit(uint32_t exit_code, uint64_t exit_info_1)
 
     /* Reload the host state from vm_hsave */
     env->hflags &= ~HF_HIF_MASK;
+    env->hflags &= ~HF_SVMI_MASK;
     env->intercept = 0;
     env->intercept_exceptions = 0;
     env->interrupt_request &= ~CPU_INTERRUPT_VIRQ;
@@ -5169,6 +5220,14 @@ void helper_vmexit(uint32_t exit_code, uint64_t exit_info_1)
     env->hflags &= ~HF_LMA_MASK;
     if (env->efer & MSR_EFER_LMA)
        env->hflags |= HF_LMA_MASK;
+    /* XXX: should also emulate the VM_CR MSR */
+    env->hflags &= ~HF_SVME_MASK;
+    if (env->cpuid_ext3_features & CPUID_EXT3_SVM) {
+        if (env->efer & MSR_EFER_SVME)
+            env->hflags |= HF_SVME_MASK;
+    } else {
+        env->efer &= ~MSR_EFER_SVME;
+    }
 #endif
 
     env->eflags = 0;
@@ -5176,10 +5235,14 @@ void helper_vmexit(uint32_t exit_code, uint64_t exit_info_1)
                 ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK));
     CC_OP = CC_OP_EFLAGS;
 
-    SVM_LOAD_SEG(env->vm_hsave, ES, es);
-    SVM_LOAD_SEG(env->vm_hsave, CS, cs);
-    SVM_LOAD_SEG(env->vm_hsave, SS, ss);
-    SVM_LOAD_SEG(env->vm_hsave, DS, ds);
+    svm_load_seg_cache(env->vm_hsave + offsetof(struct vmcb, save.es),
+                       env, R_ES);
+    svm_load_seg_cache(env->vm_hsave + offsetof(struct vmcb, save.cs),
+                       env, R_CS);
+    svm_load_seg_cache(env->vm_hsave + offsetof(struct vmcb, save.ss),
+                       env, R_SS);
+    svm_load_seg_cache(env->vm_hsave + offsetof(struct vmcb, save.ds),
+                       env, R_DS);
 
     EIP = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.rip));
     ESP = ldq_phys(env->vm_hsave + offsetof(struct vmcb, save.rsp));
diff --git a/target-i386/svm.h b/target-i386/svm.h
index add52fc986..6d86890818 100644
--- a/target-i386/svm.h
+++ b/target-i386/svm.h
@@ -1,91 +1,6 @@
 #ifndef __SVM_H
 #define __SVM_H
 
-enum {
-        /* We shift all the intercept bits so we can OR them with the
-           TB flags later on */
-	INTERCEPT_INTR = HF_HIF_SHIFT,
-	INTERCEPT_NMI,
-	INTERCEPT_SMI,
-	INTERCEPT_INIT,
-	INTERCEPT_VINTR,
-	INTERCEPT_SELECTIVE_CR0,
-	INTERCEPT_STORE_IDTR,
-	INTERCEPT_STORE_GDTR,
-	INTERCEPT_STORE_LDTR,
-	INTERCEPT_STORE_TR,
-	INTERCEPT_LOAD_IDTR,
-	INTERCEPT_LOAD_GDTR,
-	INTERCEPT_LOAD_LDTR,
-	INTERCEPT_LOAD_TR,
-	INTERCEPT_RDTSC,
-	INTERCEPT_RDPMC,
-	INTERCEPT_PUSHF,
-	INTERCEPT_POPF,
-	INTERCEPT_CPUID,
-	INTERCEPT_RSM,
-	INTERCEPT_IRET,
-	INTERCEPT_INTn,
-	INTERCEPT_INVD,
-	INTERCEPT_PAUSE,
-	INTERCEPT_HLT,
-	INTERCEPT_INVLPG,
-	INTERCEPT_INVLPGA,
-	INTERCEPT_IOIO_PROT,
-	INTERCEPT_MSR_PROT,
-	INTERCEPT_TASK_SWITCH,
-	INTERCEPT_FERR_FREEZE,
-	INTERCEPT_SHUTDOWN,
-	INTERCEPT_VMRUN,
-	INTERCEPT_VMMCALL,
-	INTERCEPT_VMLOAD,
-	INTERCEPT_VMSAVE,
-	INTERCEPT_STGI,
-	INTERCEPT_CLGI,
-	INTERCEPT_SKINIT,
-	INTERCEPT_RDTSCP,
-	INTERCEPT_ICEBP,
-	INTERCEPT_WBINVD,
-};
-/* This is not really an intercept but rather a placeholder to
-   show that we are in an SVM (just like a hidden flag, but keeps the
-   TBs clean) */
-#define INTERCEPT_SVM 63
-#define INTERCEPT_SVM_MASK (1ULL << INTERCEPT_SVM)
-
-struct __attribute__ ((__packed__)) vmcb_control_area {
-	uint16_t intercept_cr_read;
-	uint16_t intercept_cr_write;
-	uint16_t intercept_dr_read;
-	uint16_t intercept_dr_write;
-	uint32_t intercept_exceptions;
-	uint64_t intercept;
-	uint8_t reserved_1[44];
-	uint64_t iopm_base_pa;
-	uint64_t msrpm_base_pa;
-	uint64_t tsc_offset;
-	uint32_t asid;
-	uint8_t tlb_ctl;
-	uint8_t reserved_2[3];
-	uint32_t int_ctl;
-	uint32_t int_vector;
-	uint32_t int_state;
-	uint8_t reserved_3[4];
-	uint64_t exit_code;
-	uint64_t exit_info_1;
-	uint64_t exit_info_2;
-	uint32_t exit_int_info;
-	uint32_t exit_int_info_err;
-	uint64_t nested_ctl;
-	uint8_t reserved_4[16];
-	uint32_t event_inj;
-	uint32_t event_inj_err;
-	uint64_t nested_cr3;
-	uint64_t lbr_ctl;
-	uint8_t reserved_5[832];
-};
-
-
 #define TLB_CONTROL_DO_NOTHING 0
 #define TLB_CONTROL_FLUSH_ALL_ASID 1
 
@@ -116,104 +31,6 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 #define SVM_IOIO_SIZE_MASK (7 << SVM_IOIO_SIZE_SHIFT)
 #define SVM_IOIO_ASIZE_MASK (7 << SVM_IOIO_ASIZE_SHIFT)
 
-struct __attribute__ ((__packed__)) vmcb_seg {
-	uint16_t selector;
-	uint16_t attrib;
-	uint32_t limit;
-	uint64_t base;
-};
-
-struct __attribute__ ((__packed__)) vmcb_save_area {
-	struct vmcb_seg es;
-	struct vmcb_seg cs;
-	struct vmcb_seg ss;
-	struct vmcb_seg ds;
-	struct vmcb_seg fs;
-	struct vmcb_seg gs;
-	struct vmcb_seg gdtr;
-	struct vmcb_seg ldtr;
-	struct vmcb_seg idtr;
-	struct vmcb_seg tr;
-	uint8_t reserved_1[43];
-	uint8_t cpl;
-	uint8_t reserved_2[4];
-	uint64_t efer;
-	uint8_t reserved_3[112];
-	uint64_t cr4;
-	uint64_t cr3;
-	uint64_t cr0;
-	uint64_t dr7;
-	uint64_t dr6;
-	uint64_t rflags;
-	uint64_t rip;
-	uint8_t reserved_4[88];
-	uint64_t rsp;
-	uint8_t reserved_5[24];
-	uint64_t rax;
-	uint64_t star;
-	uint64_t lstar;
-	uint64_t cstar;
-	uint64_t sfmask;
-	uint64_t kernel_gs_base;
-	uint64_t sysenter_cs;
-	uint64_t sysenter_esp;
-	uint64_t sysenter_eip;
-	uint64_t cr2;
-	/* qemu: cr8 added to reuse this as hsave */
-	uint64_t cr8;
-	uint8_t reserved_6[32 - 8]; /* originally 32 */
-	uint64_t g_pat;
-	uint64_t dbgctl;
-	uint64_t br_from;
-	uint64_t br_to;
-	uint64_t last_excp_from;
-	uint64_t last_excp_to;
-};
-
-struct __attribute__ ((__packed__)) vmcb {
-	struct vmcb_control_area control;
-	struct vmcb_save_area save;
-};
-
-#define SVM_CPUID_FEATURE_SHIFT 2
-#define SVM_CPUID_FUNC 0x8000000a
-
-#define MSR_EFER_SVME_MASK (1ULL << 12)
-
-#define SVM_SELECTOR_S_SHIFT 4
-#define SVM_SELECTOR_DPL_SHIFT 5
-#define SVM_SELECTOR_P_SHIFT 7
-#define SVM_SELECTOR_AVL_SHIFT 8
-#define SVM_SELECTOR_L_SHIFT 9
-#define SVM_SELECTOR_DB_SHIFT 10
-#define SVM_SELECTOR_G_SHIFT 11
-
-#define SVM_SELECTOR_TYPE_MASK (0xf)
-#define SVM_SELECTOR_S_MASK (1 << SVM_SELECTOR_S_SHIFT)
-#define SVM_SELECTOR_DPL_MASK (3 << SVM_SELECTOR_DPL_SHIFT)
-#define SVM_SELECTOR_P_MASK (1 << SVM_SELECTOR_P_SHIFT)
-#define SVM_SELECTOR_AVL_MASK (1 << SVM_SELECTOR_AVL_SHIFT)
-#define SVM_SELECTOR_L_MASK (1 << SVM_SELECTOR_L_SHIFT)
-#define SVM_SELECTOR_DB_MASK (1 << SVM_SELECTOR_DB_SHIFT)
-#define SVM_SELECTOR_G_MASK (1 << SVM_SELECTOR_G_SHIFT)
-
-#define SVM_SELECTOR_WRITE_MASK (1 << 1)
-#define SVM_SELECTOR_READ_MASK SVM_SELECTOR_WRITE_MASK
-#define SVM_SELECTOR_CODE_MASK (1 << 3)
-
-#define INTERCEPT_CR0_MASK 1
-#define INTERCEPT_CR3_MASK (1 << 3)
-#define INTERCEPT_CR4_MASK (1 << 4)
-
-#define INTERCEPT_DR0_MASK 1
-#define INTERCEPT_DR1_MASK (1 << 1)
-#define INTERCEPT_DR2_MASK (1 << 2)
-#define INTERCEPT_DR3_MASK (1 << 3)
-#define INTERCEPT_DR4_MASK (1 << 4)
-#define INTERCEPT_DR5_MASK (1 << 5)
-#define INTERCEPT_DR6_MASK (1 << 6)
-#define INTERCEPT_DR7_MASK (1 << 7)
-
 #define SVM_EVTINJ_VEC_MASK 0xff
 
 #define SVM_EVTINJ_TYPE_SHIFT 8
@@ -313,37 +130,95 @@ struct __attribute__ ((__packed__)) vmcb {
 
 #define SVM_CR0_SELECTIVE_MASK (1 << 3 | 1) /* TS and MP */
 
-#define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda"
-#define SVM_VMRUN  ".byte 0x0f, 0x01, 0xd8"
-#define SVM_VMSAVE ".byte 0x0f, 0x01, 0xdb"
-#define SVM_CLGI   ".byte 0x0f, 0x01, 0xdd"
-#define SVM_STGI   ".byte 0x0f, 0x01, 0xdc"
-#define SVM_INVLPGA ".byte 0x0f, 0x01, 0xdf"
-
-/* function references */
-
-#define INTERCEPTED(mask) (env->intercept & mask)
-#define INTERCEPTEDw(var, mask) (env->intercept ## var & mask)
-#define INTERCEPTEDl(var, mask) (env->intercept ## var & mask)
+struct __attribute__ ((__packed__)) vmcb_control_area {
+	uint16_t intercept_cr_read;
+	uint16_t intercept_cr_write;
+	uint16_t intercept_dr_read;
+	uint16_t intercept_dr_write;
+	uint32_t intercept_exceptions;
+	uint64_t intercept;
+	uint8_t reserved_1[44];
+	uint64_t iopm_base_pa;
+	uint64_t msrpm_base_pa;
+	uint64_t tsc_offset;
+	uint32_t asid;
+	uint8_t tlb_ctl;
+	uint8_t reserved_2[3];
+	uint32_t int_ctl;
+	uint32_t int_vector;
+	uint32_t int_state;
+	uint8_t reserved_3[4];
+	uint64_t exit_code;
+	uint64_t exit_info_1;
+	uint64_t exit_info_2;
+	uint32_t exit_int_info;
+	uint32_t exit_int_info_err;
+	uint64_t nested_ctl;
+	uint8_t reserved_4[16];
+	uint32_t event_inj;
+	uint32_t event_inj_err;
+	uint64_t nested_cr3;
+	uint64_t lbr_ctl;
+	uint8_t reserved_5[832];
+};
 
-#define SVM_LOAD_SEG(addr, seg_index, seg) \
-    cpu_x86_load_seg_cache(env, \
-                    R_##seg_index, \
-                    lduw_phys(addr + offsetof(struct vmcb, save.seg.selector)),\
-                    ldq_phys(addr + offsetof(struct vmcb, save.seg.base)),\
-                    ldl_phys(addr + offsetof(struct vmcb, save.seg.limit)),\
-                    vmcb2cpu_attrib(lduw_phys(addr + offsetof(struct vmcb, save.seg.attrib)), ldq_phys(addr + offsetof(struct vmcb, save.seg.base)), ldl_phys(addr + offsetof(struct vmcb, save.seg.limit))))
+struct __attribute__ ((__packed__)) vmcb_seg {
+	uint16_t selector;
+	uint16_t attrib;
+	uint32_t limit;
+	uint64_t base;
+};
 
-#define SVM_LOAD_SEG2(addr, seg_qemu, seg_vmcb) \
-    env->seg_qemu.selector  = lduw_phys(addr + offsetof(struct vmcb, save.seg_vmcb.selector)); \
-    env->seg_qemu.base      = ldq_phys(addr + offsetof(struct vmcb, save.seg_vmcb.base)); \
-    env->seg_qemu.limit     = ldl_phys(addr + offsetof(struct vmcb, save.seg_vmcb.limit)); \
-    env->seg_qemu.flags     = vmcb2cpu_attrib(lduw_phys(addr + offsetof(struct vmcb, save.seg_vmcb.attrib)), env->seg_qemu.base, env->seg_qemu.limit)
+struct __attribute__ ((__packed__)) vmcb_save_area {
+	struct vmcb_seg es;
+	struct vmcb_seg cs;
+	struct vmcb_seg ss;
+	struct vmcb_seg ds;
+	struct vmcb_seg fs;
+	struct vmcb_seg gs;
+	struct vmcb_seg gdtr;
+	struct vmcb_seg ldtr;
+	struct vmcb_seg idtr;
+	struct vmcb_seg tr;
+	uint8_t reserved_1[43];
+	uint8_t cpl;
+	uint8_t reserved_2[4];
+	uint64_t efer;
+	uint8_t reserved_3[112];
+	uint64_t cr4;
+	uint64_t cr3;
+	uint64_t cr0;
+	uint64_t dr7;
+	uint64_t dr6;
+	uint64_t rflags;
+	uint64_t rip;
+	uint8_t reserved_4[88];
+	uint64_t rsp;
+	uint8_t reserved_5[24];
+	uint64_t rax;
+	uint64_t star;
+	uint64_t lstar;
+	uint64_t cstar;
+	uint64_t sfmask;
+	uint64_t kernel_gs_base;
+	uint64_t sysenter_cs;
+	uint64_t sysenter_esp;
+	uint64_t sysenter_eip;
+	uint64_t cr2;
+	/* qemu: cr8 added to reuse this as hsave */
+	uint64_t cr8;
+	uint8_t reserved_6[32 - 8]; /* originally 32 */
+	uint64_t g_pat;
+	uint64_t dbgctl;
+	uint64_t br_from;
+	uint64_t br_to;
+	uint64_t last_excp_from;
+	uint64_t last_excp_to;
+};
 
-#define SVM_SAVE_SEG(addr, seg_qemu, seg_vmcb) \
-    stw_phys(addr + offsetof(struct vmcb, save.seg_vmcb.selector), env->seg_qemu.selector); \
-    stq_phys(addr + offsetof(struct vmcb, save.seg_vmcb.base), env->seg_qemu.base); \
-    stl_phys(addr + offsetof(struct vmcb, save.seg_vmcb.limit), env->seg_qemu.limit); \
-    stw_phys(addr + offsetof(struct vmcb, save.seg_vmcb.attrib), cpu2vmcb_attrib(env->seg_qemu.flags))
+struct __attribute__ ((__packed__)) vmcb {
+	struct vmcb_control_area control;
+	struct vmcb_save_area save;
+};
 
 #endif
diff --git a/target-i386/translate.c b/target-i386/translate.c
index bba9b567b5..ba6284f760 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -733,7 +733,7 @@ static void gen_check_io(DisasContext *s, int ot, target_ulong cur_eip,
         tcg_gen_helper_0_1(gen_check_io_func[ot],
                            cpu_tmp2_i32);
     }
-    if(s->flags & (1ULL << INTERCEPT_IOIO_PROT)) {
+    if(s->flags & HF_SVMI_MASK) {
         if (!state_saved) {
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
@@ -2322,59 +2322,24 @@ static inline int svm_is_rep(int prefixes)
     return ((prefixes & (PREFIX_REPZ | PREFIX_REPNZ)) ? 8 : 0);
 }
 
-static inline int
+static inline void
 gen_svm_check_intercept_param(DisasContext *s, target_ulong pc_start,
                               uint32_t type, uint64_t param)
 {
-    if(!(s->flags & (INTERCEPT_SVM_MASK)))
-	/* no SVM activated */
-        return 0;
-    switch(type) {
-        /* CRx and DRx reads/writes */
-        case SVM_EXIT_READ_CR0 ... SVM_EXIT_EXCP_BASE - 1:
-            if (s->cc_op != CC_OP_DYNAMIC) {
-                gen_op_set_cc_op(s->cc_op);
-            }
-            gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_2(helper_svm_check_intercept_param, 
-                               tcg_const_i32(type), tcg_const_i64(param));
-            /* this is a special case as we do not know if the interception occurs
-               so we assume there was none */
-            return 0;
-        case SVM_EXIT_MSR:
-            if(s->flags & (1ULL << INTERCEPT_MSR_PROT)) {
-                if (s->cc_op != CC_OP_DYNAMIC) {
-                    gen_op_set_cc_op(s->cc_op);
-                }
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_2(helper_svm_check_intercept_param,
-                                   tcg_const_i32(type), tcg_const_i64(param));
-                /* this is a special case as we do not know if the interception occurs
-                   so we assume there was none */
-                return 0;
-            }
-            break;
-        default:
-            if(s->flags & (1ULL << ((type - SVM_EXIT_INTR) + INTERCEPT_INTR))) {
-                if (s->cc_op != CC_OP_DYNAMIC) {
-                    gen_op_set_cc_op(s->cc_op);
-                }
-                gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_2(helper_vmexit,
-                                   tcg_const_i32(type), tcg_const_i64(param));
-                /* we can optimize this one so TBs don't get longer
-                   than up to vmexit */
-                gen_eob(s);
-                return 1;
-            }
-    }
-    return 0;
+    /* no SVM activated; fast case */
+    if (likely(!(s->flags & HF_SVMI_MASK)))
+        return;
+    if (s->cc_op != CC_OP_DYNAMIC)
+        gen_op_set_cc_op(s->cc_op);
+    gen_jmp_im(pc_start - s->cs_base);
+    tcg_gen_helper_0_2(helper_svm_check_intercept_param, 
+                       tcg_const_i32(type), tcg_const_i64(param));
 }
 
-static inline int
+static inline void
 gen_svm_check_intercept(DisasContext *s, target_ulong pc_start, uint64_t type)
 {
-    return gen_svm_check_intercept_param(s, pc_start, type, 0);
+    gen_svm_check_intercept_param(s, pc_start, type, 0);
 }
 
 static inline void gen_stack_update(DisasContext *s, int addend)
@@ -5743,8 +5708,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         val = 0;
         goto do_lret;
     case 0xcf: /* iret */
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_IRET))
-            break;
+        gen_svm_check_intercept(s, pc_start, SVM_EXIT_IRET);
         if (!s->pe) {
             /* real mode */
             tcg_gen_helper_0_1(helper_iret_real, tcg_const_i32(s->dflag));
@@ -5890,8 +5854,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         /************************/
         /* flags */
     case 0x9c: /* pushf */
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_PUSHF))
-            break;
+        gen_svm_check_intercept(s, pc_start, SVM_EXIT_PUSHF);
         if (s->vm86 && s->iopl != 3) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
@@ -5902,8 +5865,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x9d: /* popf */
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_POPF))
-            break;
+        gen_svm_check_intercept(s, pc_start, SVM_EXIT_POPF);
         if (s->vm86 && s->iopl != 3) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
@@ -6187,14 +6149,10 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0xcc: /* int3 */
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_SWINT))
-            break;
         gen_interrupt(s, EXCP03_INT3, pc_start - s->cs_base, s->pc - s->cs_base);
         break;
     case 0xcd: /* int N */
         val = ldub_code(s->pc++);
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_SWINT))
-            break;
         if (s->vm86 && s->iopl != 3) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
@@ -6204,16 +6162,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xce: /* into */
         if (CODE64(s))
             goto illegal_op;
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_SWINT))
-            break;
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
         gen_jmp_im(pc_start - s->cs_base);
         tcg_gen_helper_0_1(helper_into, tcg_const_i32(s->pc - pc_start));
         break;
     case 0xf1: /* icebp (undocumented, exits to external debugger) */
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_ICEBP))
-            break;
+        gen_svm_check_intercept(s, pc_start, SVM_EXIT_ICEBP);
 #if 1
         gen_debug(s, pc_start - s->cs_base);
 #else
@@ -6371,25 +6326,25 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (s->cpl != 0) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            int retval = 0;
+            if (s->cc_op != CC_OP_DYNAMIC)
+                gen_op_set_cc_op(s->cc_op);
+            gen_jmp_im(pc_start - s->cs_base);
             if (b & 2) {
-                retval = gen_svm_check_intercept_param(s, pc_start, SVM_EXIT_MSR, 0);
                 tcg_gen_helper_0_0(helper_rdmsr);
             } else {
-                retval = gen_svm_check_intercept_param(s, pc_start, SVM_EXIT_MSR, 1);
                 tcg_gen_helper_0_0(helper_wrmsr);
             }
-            if(retval)
-                gen_eob(s);
         }
         break;
     case 0x131: /* rdtsc */
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_RDTSC))
-            break;
+        if (s->cc_op != CC_OP_DYNAMIC)
+            gen_op_set_cc_op(s->cc_op);
         gen_jmp_im(pc_start - s->cs_base);
         tcg_gen_helper_0_0(helper_rdtsc);
         break;
     case 0x133: /* rdpmc */
+        if (s->cc_op != CC_OP_DYNAMIC)
+            gen_op_set_cc_op(s->cc_op);
         gen_jmp_im(pc_start - s->cs_base);
         tcg_gen_helper_0_0(helper_rdpmc);
         break;
@@ -6452,16 +6407,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         break;
 #endif
     case 0x1a2: /* cpuid */
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_CPUID))
-            break;
         tcg_gen_helper_0_0(helper_cpuid);
         break;
     case 0xf4: /* hlt */
         if (s->cpl != 0) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_HLT))
-                break;
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(s->pc - s->cs_base);
@@ -6477,8 +6428,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         case 0: /* sldt */
             if (!s->pe || s->vm86)
                 goto illegal_op;
-            if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_READ))
-                break;
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_READ);
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,ldt.selector));
             ot = OT_WORD;
             if (mod == 3)
@@ -6491,8 +6441,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
-                if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE))
-                    break;
+                gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE);
                 gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -6502,8 +6451,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         case 1: /* str */
             if (!s->pe || s->vm86)
                 goto illegal_op;
-            if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_READ))
-                break;
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_READ);
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,tr.selector));
             ot = OT_WORD;
             if (mod == 3)
@@ -6516,8 +6464,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
-                if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE))
-                    break;
+                gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE);
                 gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -6550,8 +6497,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         case 0: /* sgdt */
             if (mod == 3)
                 goto illegal_op;
-            if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_READ))
-                break;
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_READ);
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.limit));
             gen_op_st_T0_A0(OT_WORD + s->mem_index);
@@ -6568,8 +6514,6 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     if (!(s->cpuid_ext_features & CPUID_EXT_MONITOR) ||
                         s->cpl != 0)
                         goto illegal_op;
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_MONITOR))
-                        break;
                     gen_jmp_im(pc_start - s->cs_base);
 #ifdef TARGET_X86_64
                     if (s->aflag == 2) {
@@ -6592,8 +6536,6 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                         gen_op_set_cc_op(s->cc_op);
                         s->cc_op = CC_OP_DYNAMIC;
                     }
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_MWAIT))
-                        break;
                     gen_jmp_im(s->pc - s->cs_base);
                     tcg_gen_helper_0_0(helper_mwait);
                     gen_eob(s);
@@ -6602,8 +6544,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     goto illegal_op;
                 }
             } else { /* sidt */
-                if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ))
-                    break;
+                gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
                 gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.limit));
                 gen_op_st_T0_A0(OT_WORD + s->mem_index);
@@ -6617,52 +6558,85 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         case 2: /* lgdt */
         case 3: /* lidt */
             if (mod == 3) {
+                if (s->cc_op != CC_OP_DYNAMIC)
+                    gen_op_set_cc_op(s->cc_op);
+                gen_jmp_im(pc_start - s->cs_base);
                 switch(rm) {
                 case 0: /* VMRUN */
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_VMRUN))
+                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
+                        goto illegal_op;
+                    if (s->cpl != 0) {
+                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
                         break;
-                    if (s->cc_op != CC_OP_DYNAMIC)
-                        gen_op_set_cc_op(s->cc_op);
-                    gen_jmp_im(s->pc - s->cs_base);
-                    tcg_gen_helper_0_0(helper_vmrun);
-                    s->cc_op = CC_OP_EFLAGS;
-                    gen_eob(s);
+                    } else {
+                        tcg_gen_helper_0_0(helper_vmrun);
+                        s->cc_op = CC_OP_EFLAGS;
+                        gen_eob(s);
+                    }
                     break;
                 case 1: /* VMMCALL */
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_VMMCALL))
-                         break;
-                    /* FIXME: cause #UD if hflags & SVM */
+                    if (!(s->flags & HF_SVME_MASK))
+                        goto illegal_op;
                     tcg_gen_helper_0_0(helper_vmmcall);
                     break;
                 case 2: /* VMLOAD */
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_VMLOAD))
-                         break;
-                    tcg_gen_helper_0_0(helper_vmload);
+                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
+                        goto illegal_op;
+                    if (s->cpl != 0) {
+                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                        break;
+                    } else {
+                        tcg_gen_helper_0_0(helper_vmload);
+                    }
                     break;
                 case 3: /* VMSAVE */
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_VMSAVE))
-                         break;
-                    tcg_gen_helper_0_0(helper_vmsave);
+                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
+                        goto illegal_op;
+                    if (s->cpl != 0) {
+                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                        break;
+                    } else {
+                        tcg_gen_helper_0_0(helper_vmsave);
+                    }
                     break;
                 case 4: /* STGI */
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_STGI))
-                         break;
-                    tcg_gen_helper_0_0(helper_stgi);
+                    if ((!(s->flags & HF_SVME_MASK) &&
+                         !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || 
+                        !s->pe)
+                        goto illegal_op;
+                    if (s->cpl != 0) {
+                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                        break;
+                    } else {
+                        tcg_gen_helper_0_0(helper_stgi);
+                    }
                     break;
                 case 5: /* CLGI */
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_CLGI))
-                         break;
-                    tcg_gen_helper_0_0(helper_clgi);
+                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
+                        goto illegal_op;
+                    if (s->cpl != 0) {
+                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                        break;
+                    } else {
+                        tcg_gen_helper_0_0(helper_clgi);
+                    }
                     break;
                 case 6: /* SKINIT */
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_SKINIT))
-                         break;
+                    if ((!(s->flags & HF_SVME_MASK) && 
+                         !(s->cpuid_ext3_features & CPUID_EXT3_SKINIT)) || 
+                        !s->pe)
+                        goto illegal_op;
                     tcg_gen_helper_0_0(helper_skinit);
                     break;
                 case 7: /* INVLPGA */
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_INVLPGA))
-                         break;
-                    tcg_gen_helper_0_0(helper_invlpga);
+                    if (!(s->flags & HF_SVME_MASK) || !s->pe)
+                        goto illegal_op;
+                    if (s->cpl != 0) {
+                        gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+                        break;
+                    } else {
+                        tcg_gen_helper_0_0(helper_invlpga);
+                    }
                     break;
                 default:
                     goto illegal_op;
@@ -6670,9 +6644,8 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             } else if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
-                if (gen_svm_check_intercept(s, pc_start,
-                                            op==2 ? SVM_EXIT_GDTR_WRITE : SVM_EXIT_IDTR_WRITE))
-                    break;
+                gen_svm_check_intercept(s, pc_start,
+                                        op==2 ? SVM_EXIT_GDTR_WRITE : SVM_EXIT_IDTR_WRITE);
                 gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_T1_A0(OT_WORD + s->mem_index);
                 gen_add_A0_im(s, 2);
@@ -6689,8 +6662,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             }
             break;
         case 4: /* smsw */
-            if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0))
-                break;
+            gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0);
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]));
             gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 1);
             break;
@@ -6698,8 +6670,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
-                if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0))
-                    break;
+                gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
                 gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
                 tcg_gen_helper_0_1(helper_lmsw, cpu_T[0]);
                 gen_jmp_im(s->pc - s->cs_base);
@@ -6724,8 +6695,6 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                         goto illegal_op;
                     }
                 } else {
-                    if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_INVLPG))
-                        break;
                     gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
                     tcg_gen_helper_0_1(helper_invlpg, cpu_A0);
                     gen_jmp_im(s->pc - s->cs_base);
@@ -6742,8 +6711,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (s->cpl != 0) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            if (gen_svm_check_intercept(s, pc_start, (b & 2) ? SVM_EXIT_INVD : SVM_EXIT_WBINVD))
-                break;
+            gen_svm_check_intercept(s, pc_start, (b & 2) ? SVM_EXIT_INVD : SVM_EXIT_WBINVD);
             /* nothing to do */
         }
         break;
@@ -6892,21 +6860,18 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             case 3:
             case 4:
             case 8:
+                if (s->cc_op != CC_OP_DYNAMIC)
+                    gen_op_set_cc_op(s->cc_op);
+                gen_jmp_im(pc_start - s->cs_base);
                 if (b & 2) {
-                    gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0 + reg);
                     gen_op_mov_TN_reg(ot, 0, rm);
-                    tcg_gen_helper_0_2(helper_movl_crN_T0, 
+                    tcg_gen_helper_0_2(helper_write_crN, 
                                        tcg_const_i32(reg), cpu_T[0]);
                     gen_jmp_im(s->pc - s->cs_base);
                     gen_eob(s);
                 } else {
-                    gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0 + reg);
-#if !defined(CONFIG_USER_ONLY)
-                    if (reg == 8)
-                        tcg_gen_helper_1_0(helper_movtl_T0_cr8, cpu_T[0]);
-                    else
-#endif
-                        tcg_gen_ld_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[reg]));
+                    tcg_gen_helper_1_1(helper_read_crN, 
+                                       cpu_T[0], tcg_const_i32(reg));
                     gen_op_mov_reg_T0(ot, rm);
                 }
                 break;
@@ -7054,8 +7019,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         /* ignore for now */
         break;
     case 0x1aa: /* rsm */
-        if (gen_svm_check_intercept(s, pc_start, SVM_EXIT_RSM))
-            break;
+        gen_svm_check_intercept(s, pc_start, SVM_EXIT_RSM);
         if (!(s->flags & HF_SMM_MASK))
             goto illegal_op;
         if (s->cc_op != CC_OP_DYNAMIC) {