/usr/src/sys/arch/amd64/amd64/vmm

Bug Summary

File:	arch/amd64/amd64/vmm_machdep.c
Warning:	line 6581, column 8 Access to field 'spc_schedflags' results in a dereference of a null pointer (loaded from variable 'spc')
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name vmm_machdep.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/arch/amd64/amd64/vmm_machdep.c
1/* $OpenBSD: vmm_machdep.c,v 1.14 2024/01/10 04:13:59 dv Exp $ */
2/*
* Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

18#include <sys/param.h>
19#include <sys/systm.h>
20#include <sys/signalvar.h>
21#include <sys/malloc.h>
22#include <sys/device.h>
23#include <sys/pool.h>
24#include <sys/proc.h>
25#include <sys/user.h>
26#include <sys/ioctl.h>
27#include <sys/queue.h>
28#include <sys/refcnt.h>
29#include <sys/rwlock.h>
30#include <sys/pledge.h>
31#include <sys/memrange.h>
32#include <sys/tracepoint.h>

34#include <uvm/uvm_extern.h>

36#include <machine/fpu.h>
37#include <machine/pmap.h>
38#include <machine/biosvar.h>
39#include <machine/segments.h>
40#include <machine/cpufunc.h>
41#include <machine/vmmvar.h>

43#include <dev/isa/isareg.h>
44#include <dev/pv/pvreg.h>

46#include <dev/vmm/vmm.h>

48#ifdef MP_LOCKDEBUG
49#include <ddb/db_output.h>
50extern int __mp_lock_spinout;
51#endif /* MP_LOCKDEBUG */

53void *l1tf_flush_region;

55#define DEVNAME(s)((s)->sc_dev.dv_xname)  ((s)->sc_dev.dv_xname)

57#define CTRL_DUMP(x,y,z)printf("     %s: Can set:%s Can clear:%s\n", "z" , vcpu_vmx_check_cap
(x, IA32_VMX_y_CTLS, IA32_VMX_z, 1) ? "Yes" : "No", vcpu_vmx_check_cap
(x, IA32_VMX_y_CTLS, IA32_VMX_z, 0) ? "Yes" : "No"); printf("     %s: Can set:%s Can clear:%s\n", #z , \
		vcpu_vmx_check_cap(x, IA32_VMX_##y ##_CTLS, \
		IA32_VMX_##z, 1) ? "Yes" : "No", \
		vcpu_vmx_check_cap(x, IA32_VMX_##y ##_CTLS, \
		IA32_VMX_##z, 0) ? "Yes" : "No");

63#define VMX_EXIT_INFO_HAVE_RIP0x1		0x1
64#define VMX_EXIT_INFO_HAVE_REASON0x2	0x2
65#define VMX_EXIT_INFO_COMPLETE(0x1 | 0x2)				\
  (VMX_EXIT_INFO_HAVE_RIP0x1 | VMX_EXIT_INFO_HAVE_REASON0x2)

68void vmx_dump_vmcs_field(uint16_t, const char *);
69int vmm_enabled(void);
70void vmm_activate_machdep(struct device *, int);
71int vmmioctl_machdep(dev_t, u_long, caddr_t, int, struct proc *);
72int vmm_quiesce_vmx(void);
73int vm_run(struct vm_run_params *);
74int vm_intr_pending(struct vm_intr_params *);
75int vm_rwregs(struct vm_rwregs_params *, int);
76int vm_mprotect_ept(struct vm_mprotect_ept_params *);
77int vm_rwvmparams(struct vm_rwvmparams_params *, int);
78int vcpu_readregs_vmx(struct vcpu *, uint64_t, int, struct vcpu_reg_state *);
79int vcpu_readregs_svm(struct vcpu *, uint64_t, struct vcpu_reg_state *);
80int vcpu_writeregs_vmx(struct vcpu *, uint64_t, int, struct vcpu_reg_state *);
81int vcpu_writeregs_svm(struct vcpu *, uint64_t, struct vcpu_reg_state *);
82int vcpu_reset_regs(struct vcpu *, struct vcpu_reg_state *);
83int vcpu_reset_regs_vmx(struct vcpu *, struct vcpu_reg_state *);
84int vcpu_reset_regs_svm(struct vcpu *, struct vcpu_reg_state *);
85int vcpu_reload_vmcs_vmx(struct vcpu *);
86int vcpu_init(struct vcpu *);
87int vcpu_init_vmx(struct vcpu *);
88int vcpu_init_svm(struct vcpu *);
89int vcpu_run_vmx(struct vcpu *, struct vm_run_params *);
90int vcpu_run_svm(struct vcpu *, struct vm_run_params *);
91void vcpu_deinit(struct vcpu *);
92void vcpu_deinit_svm(struct vcpu *);
93void vcpu_deinit_vmx(struct vcpu *);
94int vm_impl_init(struct vm *, struct proc *);
95int vm_impl_init_vmx(struct vm *, struct proc *);
96int vm_impl_init_svm(struct vm *, struct proc *);
97void vm_impl_deinit(struct vm *);
98int vcpu_vmx_check_cap(struct vcpu *, uint32_t, uint32_t, int);
99int vcpu_vmx_compute_ctrl(uint64_t, uint16_t, uint32_t, uint32_t, uint32_t *);
100int vmx_get_exit_info(uint64_t *, uint64_t *);
101int vmx_load_pdptes(struct vcpu *);
102int vmx_handle_exit(struct vcpu *);
103int svm_handle_exit(struct vcpu *);
104int svm_handle_msr(struct vcpu *);
105int vmm_handle_xsetbv(struct vcpu *, uint64_t *);
106int vmx_handle_xsetbv(struct vcpu *);
107int svm_handle_xsetbv(struct vcpu *);
108int vmm_handle_cpuid(struct vcpu *);
109int vmx_handle_rdmsr(struct vcpu *);
110int vmx_handle_wrmsr(struct vcpu *);
111int vmx_handle_cr0_write(struct vcpu *, uint64_t);
112int vmx_handle_cr4_write(struct vcpu *, uint64_t);
113int vmx_handle_cr(struct vcpu *);
114int svm_handle_inout(struct vcpu *);
115int vmx_handle_inout(struct vcpu *);
116int svm_handle_hlt(struct vcpu *);
117int vmx_handle_hlt(struct vcpu *);
118int vmm_inject_ud(struct vcpu *);
119int vmm_inject_gp(struct vcpu *);
120int vmm_inject_db(struct vcpu *);
121void vmx_handle_intr(struct vcpu *);
122void vmx_handle_intwin(struct vcpu *);
123void vmx_handle_misc_enable_msr(struct vcpu *);
124int vmm_get_guest_memtype(struct vm *, paddr_t);
125int vmx_get_guest_faulttype(void);
126int svm_get_guest_faulttype(struct vmcb *);
127int vmx_get_exit_qualification(uint64_t *);
128int vmm_get_guest_cpu_cpl(struct vcpu *);
129int vmm_get_guest_cpu_mode(struct vcpu *);
130int svm_fault_page(struct vcpu *, paddr_t);
131int vmx_fault_page(struct vcpu *, paddr_t);
132int vmx_handle_np_fault(struct vcpu *);
133int svm_handle_np_fault(struct vcpu *);
134int vmx_mprotect_ept(vm_map_t, paddr_t, paddr_t, int);
135pt_entry_t *vmx_pmap_find_pte_ept(pmap_t, paddr_t);
136int vmm_alloc_vpid(uint16_t *);
137void vmm_free_vpid(uint16_t);
138const char *vcpu_state_decode(u_int);
139const char *vmx_exit_reason_decode(uint32_t);
140const char *svm_exit_reason_decode(uint32_t);
141const char *vmx_instruction_error_decode(uint32_t);
142void svm_setmsrbr(struct vcpu *, uint32_t);
143void svm_setmsrbw(struct vcpu *, uint32_t);
144void svm_setmsrbrw(struct vcpu *, uint32_t);
145void vmx_setmsrbr(struct vcpu *, uint32_t);
146void vmx_setmsrbw(struct vcpu *, uint32_t);
147void vmx_setmsrbrw(struct vcpu *, uint32_t);
148void svm_set_clean(struct vcpu *, uint32_t);
149void svm_set_dirty(struct vcpu *, uint32_t);

151int vmm_gpa_is_valid(struct vcpu *vcpu, paddr_t gpa, size_t obj_size);
152void vmm_init_pvclock(struct vcpu *, paddr_t);
153int vmm_update_pvclock(struct vcpu *);
154int vmm_pat_is_valid(uint64_t);

156#ifdef MULTIPROCESSOR1
157static int vmx_remote_vmclear(struct cpu_info*, struct vcpu *);
158#endif

160#ifdef VMM_DEBUG
161void dump_vcpu(struct vcpu *);
162void vmx_vcpu_dump_regs(struct vcpu *);
163void vmx_dump_vmcs(struct vcpu *);
164const char *msr_name_decode(uint32_t);
165void vmm_segment_desc_decode(uint64_t);
166void vmm_decode_cr0(uint64_t);
167void vmm_decode_cr3(uint64_t);
168void vmm_decode_cr4(uint64_t);
169void vmm_decode_msr_value(uint64_t, uint64_t);
170void vmm_decode_apicbase_msr_value(uint64_t);
171void vmm_decode_ia32_fc_value(uint64_t);
172void vmm_decode_mtrrcap_value(uint64_t);
173void vmm_decode_perf_status_value(uint64_t);
174void vmm_decode_perf_ctl_value(uint64_t);
175void vmm_decode_mtrrdeftype_value(uint64_t);
176void vmm_decode_efer_value(uint64_t);
177void vmm_decode_rflags(uint64_t);
178void vmm_decode_misc_enable_value(uint64_t);
179const char *vmm_decode_cpu_mode(struct vcpu *);

181extern int mtrr2mrt(int);

183struct vmm_reg_debug_info {
uint64_t	vrdi_bit;
const char	*vrdi_present;
const char	*vrdi_absent;
187};
188#endif /* VMM_DEBUG */

190extern uint64_t tsc_frequency;
191extern int tsc_is_invariant;

193const char *vmm_hv_signature = VMM_HV_SIGNATURE"OpenBSDVMM58";

195const struct kmem_pa_mode vmm_kp_contig = {
.kp_constraint = &no_constraint,
.kp_maxseg = 1,
.kp_align = 4096,
.kp_zero = 1,
200};

202extern struct cfdriver vmm_cd;
203extern const struct cfattach vmm_ca;

205/*
* Helper struct to easily get the VMCS field IDs needed in vmread/vmwrite
* to access the individual fields of the guest segment registers. This
* struct is indexed by VCPU_REGS_* id.
*/
210const struct {
uint64_t selid;
uint64_t limitid;
uint64_t arid;
uint64_t baseid;
215} vmm_vmx_sreg_vmcs_fields[] = {
{ VMCS_GUEST_IA32_ES_SEL0x0800, VMCS_GUEST_IA32_ES_LIMIT0x4800,
 VMCS_GUEST_IA32_ES_AR0x4814, VMCS_GUEST_IA32_ES_BASE0x6806 },
{ VMCS_GUEST_IA32_CS_SEL0x0802, VMCS_GUEST_IA32_CS_LIMIT0x4802,
 VMCS_GUEST_IA32_CS_AR0x4816, VMCS_GUEST_IA32_CS_BASE0x6808 },
{ VMCS_GUEST_IA32_SS_SEL0x0804, VMCS_GUEST_IA32_SS_LIMIT0x4804,
 VMCS_GUEST_IA32_SS_AR0x4818, VMCS_GUEST_IA32_SS_BASE0x680A },
{ VMCS_GUEST_IA32_DS_SEL0x0806, VMCS_GUEST_IA32_DS_LIMIT0x4806,
 VMCS_GUEST_IA32_DS_AR0x481A, VMCS_GUEST_IA32_DS_BASE0x680C },
{ VMCS_GUEST_IA32_FS_SEL0x0808, VMCS_GUEST_IA32_FS_LIMIT0x4808,
 VMCS_GUEST_IA32_FS_AR0x481C, VMCS_GUEST_IA32_FS_BASE0x680E },
{ VMCS_GUEST_IA32_GS_SEL0x080A, VMCS_GUEST_IA32_GS_LIMIT0x480A,
 VMCS_GUEST_IA32_GS_AR0x481E, VMCS_GUEST_IA32_GS_BASE0x6810 },
{ VMCS_GUEST_IA32_LDTR_SEL0x080C, VMCS_GUEST_IA32_LDTR_LIMIT0x480C,
 VMCS_GUEST_IA32_LDTR_AR0x4820, VMCS_GUEST_IA32_LDTR_BASE0x6812 },
{ VMCS_GUEST_IA32_TR_SEL0x080E, VMCS_GUEST_IA32_TR_LIMIT0x480E,
 VMCS_GUEST_IA32_TR_AR0x4822, VMCS_GUEST_IA32_TR_BASE0x6814 }
232};

234/* Pools for VMs and VCPUs */
235extern struct pool vm_pool;
236extern struct pool vcpu_pool;

238extern struct vmm_softc *vmm_softc;

240/* IDT information used when populating host state area */
241extern vaddr_t idt_vaddr;
242extern struct gate_descriptor *idt;

244/* Constants used in "CR access exit" */
245#define CR_WRITE0	0
246#define CR_READ1		1
247#define CR_CLTS2		2
248#define CR_LMSW3		3

250/*
* vmm_enabled
*
* Checks if we have at least one CPU with either VMX or SVM.
* Returns 1 if we have at least one of either type, but not both, 0 otherwise.
*/
256int
257vmm_enabled(void)
258{
struct cpu_info *ci;
CPU_INFO_ITERATORint cii;
int found_vmx = 0, found_svm = 0;

/* Check if we have at least one CPU with either VMX or SVM */
CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
if (ci->ci_vmm_flags & CI_VMM_VMX(1 << 0))
	found_vmx = 1;
if (ci->ci_vmm_flags & CI_VMM_SVM(1 << 1))
	found_svm = 1;
}

/* Don't support both SVM and VMX at the same time */
if (found_vmx && found_svm)
return (0);

if (found_vmx || found_svm)
return 1;

return 0;
279}

281void
282vmm_attach_machdep(struct device *parent, struct device *self, void *aux)
283{
struct vmm_softc *sc = (struct vmm_softc *)self;
struct cpu_info *ci;
CPU_INFO_ITERATORint cii;

sc->sc_md.nr_rvi_cpus = 0;
sc->sc_md.nr_ept_cpus = 0;

/* Calculate CPU features */
CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
if (ci->ci_vmm_flags & CI_VMM_RVI(1 << 2))
	sc->sc_md.nr_rvi_cpus++;
if (ci->ci_vmm_flags & CI_VMM_EPT(1 << 3))
	sc->sc_md.nr_ept_cpus++;
}

sc->sc_md.pkru_enabled = 0;
if (rcr4() & CR4_PKE0x00400000)
sc->sc_md.pkru_enabled = 1;

if (sc->sc_md.nr_ept_cpus) {
printf(": VMX/EPT");
sc->mode = VMM_MODE_EPT;
} else if (sc->sc_md.nr_rvi_cpus) {
printf(": SVM/RVI");
sc->mode = VMM_MODE_RVI;
} else {
printf(": unknown");
sc->mode = VMM_MODE_UNKNOWN;
}

if (sc->mode == VMM_MODE_EPT) {
if (!(curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_has_l1_flush_msr)) {
	l1tf_flush_region = km_alloc(VMX_L1D_FLUSH_SIZE(64 * 1024),
	    &kv_any, &vmm_kp_contig, &kd_waitok);
	if (!l1tf_flush_region) {
		printf(" (failing, no memory)");
		sc->mode = VMM_MODE_UNKNOWN;
	} else {
		printf(" (using slow L1TF mitigation)");
		memset(l1tf_flush_region, 0xcc,__builtin_memset((l1tf_flush_region), (0xcc), ((64 * 1024)))
		    VMX_L1D_FLUSH_SIZE)__builtin_memset((l1tf_flush_region), (0xcc), ((64 * 1024)));
	}
}
}

if (sc->mode == VMM_MODE_RVI) {
sc->max_vpid = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_svm.svm_max_asid;
} else {
sc->max_vpid = 0xFFF;
}

bzero(&sc->vpids, sizeof(sc->vpids))__builtin_bzero((&sc->vpids), (sizeof(sc->vpids)));
rw_init(&sc->vpid_lock, "vpid")_rw_init_flags(&sc->vpid_lock, "vpid", 0, ((void *)0));
337}

339/*
* vmm_quiesce_vmx
*
* Prepare the host for suspend by flushing all VMCS states.
*/
344int
345vmm_quiesce_vmx(void)
346{
struct vm		*vm;
struct vcpu		*vcpu;
int			 err;

/*
* We should be only called from a quiescing device state so we
* don't expect to sleep here. If we can't get all our locks,
* something is wrong.
*/
if ((err = rw_enter(&vmm_softc->vm_lock, RW_WRITE0x0001UL | RW_NOSLEEP0x0040UL)))
return (err);

/* Iterate over each vm... */
SLIST_FOREACH(vm, &vmm_softc->vm_list, vm_link)for((vm) = ((&vmm_softc->vm_list)->slh_first); (vm)
 != ((void *)0); (vm) = ((vm)->vm_link.sle_next)) {
/* Iterate over each vcpu... */
SLIST_FOREACH(vcpu, &vm->vm_vcpu_list, vc_vcpu_link)for((vcpu) = ((&vm->vm_vcpu_list)->slh_first); (vcpu
) != ((void *)0); (vcpu) = ((vcpu)->vc_vcpu_link.sle_next)
) {
	err = rw_enter(&vcpu->vc_lock, RW_WRITE0x0001UL | RW_NOSLEEP0x0040UL);
	if (err)
		break;

	/* We can skip unlaunched VMCS. Nothing to flush. */
	if (atomic_load_int(&vcpu->vc_vmx_vmcs_state)
	    != VMCS_LAUNCHED1) {
		DPRINTF("%s: skipping vcpu %d for vm %d\n",
		    __func__, vcpu->vc_id, vm->vm_id);
		rw_exit_write(&vcpu->vc_lock);
		continue;
	}

376#ifdef MULTIPROCESSOR1
	if (vcpu->vc_last_pcpu != curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})) {
		/* Remote cpu vmclear via ipi. */
		err = vmx_remote_vmclear(vcpu->vc_last_pcpu,
		    vcpu);
		if (err)
			printf("%s: failed to remote vmclear "
			    "vcpu %d of vm %d\n", __func__,
			    vcpu->vc_id, vm->vm_id);
	} else
386#endif
	{
		/* Local cpu vmclear instruction. */
		if ((err = vmclear(&vcpu->vc_control_pa)))
			printf("%s: failed to locally vmclear "
			    "vcpu %d of vm %d\n", __func__,
			    vcpu->vc_id, vm->vm_id);
		atomic_swap_uint(&vcpu->vc_vmx_vmcs_state,_atomic_swap_uint((&vcpu->vc_vmx_vmcs_state), (0))
		    VMCS_CLEARED)_atomic_swap_uint((&vcpu->vc_vmx_vmcs_state), (0));
	}

	rw_exit_write(&vcpu->vc_lock);
	if (err)
		break;
	DPRINTF("%s: cleared vcpu %d for vm %d\n", __func__,
	    vcpu->vc_id, vm->vm_id);
}
if (err)
	break;
}
rw_exit_write(&vmm_softc->vm_lock);

if (err)
return (err);
return (0);
411}

413void
414vmm_activate_machdep(struct device *self, int act)
415{
struct cpu_info		*ci = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});

switch (act) {
case DVACT_QUIESCE2:
/* If we're not in vmm mode, nothing to do. */
if ((ci->ci_flags & CPUF_VMM0x20000) == 0)
	break;

/* Intel systems need extra steps to sync vcpu state. */
if (vmm_softc->mode == VMM_MODE_EPT)
	if (vmm_quiesce_vmx())
		DPRINTF("%s: vmx quiesce failed\n", __func__);

/* Stop virtualization mode on all cpus. */
vmm_stop();
break;

case DVACT_WAKEUP5:
/* Restart virtualization mode on all cpu's. */
if (vmm_softc->vm_ct > 0)
	vmm_start();
break;
}
439}

441int
442vmmioctl_machdep(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
443{
int ret;

switch (cmd) {
case VMM_IOC_INTR((unsigned long)0x80000000 | ((sizeof(struct vm_intr_params) &
 0x1fff) << 16) | ((('V')) << 8) | ((6))):
ret = vm_intr_pending((struct vm_intr_params *)data);
break;
case VMM_IOC_MPROTECT_EPT((unsigned long)0x80000000 | ((sizeof(struct vm_mprotect_ept_params
) & 0x1fff) << 16) | ((('V')) << 8) | ((11))):
ret = vm_mprotect_ept((struct vm_mprotect_ept_params *)data);
break;
default:
DPRINTF("%s: unknown ioctl code 0x%lx\n", __func__, cmd);
ret = ENOTTY25;
}

return (ret);
459}

461int
462pledge_ioctl_vmm_machdep(struct proc *p, long com)
463{
switch (com) {
case VMM_IOC_INTR((unsigned long)0x80000000 | ((sizeof(struct vm_intr_params) &
 0x1fff) << 16) | ((('V')) << 8) | ((6))):
case VMM_IOC_MPROTECT_EPT((unsigned long)0x80000000 | ((sizeof(struct vm_mprotect_ept_params
) & 0x1fff) << 16) | ((('V')) << 8) | ((11))):
return (0);
}

return (EPERM1);
471}

473/*
* vm_intr_pending
*
* IOCTL handler routine for VMM_IOC_INTR messages, sent from vmd when an
* interrupt is pending and needs acknowledgment
*
* Parameters:
*  vip: Describes the vm/vcpu for which the interrupt is pending
*
* Return values:
*  0: if successful
*  ENOENT: if the VM/VCPU defined by 'vip' cannot be found
*/
486int
487vm_intr_pending(struct vm_intr_params *vip)
488{
struct vm *vm;
struct vcpu *vcpu;
491#ifdef MULTIPROCESSOR1
struct cpu_info *ci;
493#endif
int error, ret = 0;

/* Find the desired VM */
error = vm_find(vip->vip_vm_id, &vm);

/* Not found? exit. */
if (error != 0)
return (error);

vcpu = vm_find_vcpu(vm, vip->vip_vcpu_id);

if (vcpu == NULL((void *)0)) {
ret = ENOENT2;
goto out;
}

vcpu->vc_intr = vip->vip_intr;
511#ifdef MULTIPROCESSOR1
ci = READ_ONCE(vcpu->vc_curcpu)({ typeof(vcpu->vc_curcpu) __tmp = *(volatile typeof(vcpu->
vc_curcpu) *)&(vcpu->vc_curcpu); membar_datadep_consumer
(); __tmp; });
if (ci != NULL((void *)0))
x86_send_ipi(ci, X86_IPI_NOP0x00000002);
515#endif

517out:
refcnt_rele_wake(&vm->vm_refcnt);
return (ret);
520}

522/*
* vm_rwvmparams
*
* IOCTL handler to read/write the current vmm params like pvclock gpa, pvclock
* version, etc.
*
* Parameters:
*   vrwp: Describes the VM and VCPU to get/set the params from
*   dir: 0 for reading, 1 for writing
*
* Return values:
*  0: if successful
*  ENOENT: if the VM/VCPU defined by 'vpp' cannot be found
*  EINVAL: if an error occurred reading the registers of the guest
*/
537int
538vm_rwvmparams(struct vm_rwvmparams_params *vpp, int dir)
539{
struct vm *vm;
struct vcpu *vcpu;
int error, ret = 0;

/* Find the desired VM */
error = vm_find(vpp->vpp_vm_id, &vm);

/* Not found? exit. */
if (error != 0)
return (error);

vcpu = vm_find_vcpu(vm, vpp->vpp_vcpu_id);

if (vcpu == NULL((void *)0)) {
ret = ENOENT2;
goto out;
}

if (dir == 0) {
if (vpp->vpp_mask & VM_RWVMPARAMS_PVCLOCK_VERSION0x2)
	vpp->vpp_pvclock_version = vcpu->vc_pvclock_version;
if (vpp->vpp_mask & VM_RWVMPARAMS_PVCLOCK_SYSTEM_GPA0x1)
	vpp->vpp_pvclock_system_gpa = \
	    vcpu->vc_pvclock_system_gpa;
} else {
if (vpp->vpp_mask & VM_RWVMPARAMS_PVCLOCK_VERSION0x2)
	vcpu->vc_pvclock_version = vpp->vpp_pvclock_version;
if (vpp->vpp_mask & VM_RWVMPARAMS_PVCLOCK_SYSTEM_GPA0x1) {
	vmm_init_pvclock(vcpu, vpp->vpp_pvclock_system_gpa);
}
}
571out:
refcnt_rele_wake(&vm->vm_refcnt);
return (ret);
574}

576/*
* vm_readregs
*
* IOCTL handler to read/write the current register values of a guest VCPU.
* The VCPU must not be running.
*
* Parameters:
*   vrwp: Describes the VM and VCPU to get/set the registers from. The
*    register values are returned here as well.
*   dir: 0 for reading, 1 for writing
*
* Return values:
*  0: if successful
*  ENOENT: if the VM/VCPU defined by 'vrwp' cannot be found
*  EINVAL: if an error occurred accessing the registers of the guest
*  EPERM: if the vm cannot be accessed from the calling process
*/
593int
594vm_rwregs(struct vm_rwregs_params *vrwp, int dir)
595{
struct vm *vm;
struct vcpu *vcpu;
struct vcpu_reg_state *vrs = &vrwp->vrwp_regs;
int error, ret = 0;

/* Find the desired VM */
error = vm_find(vrwp->vrwp_vm_id, &vm);

/* Not found? exit. */
if (error != 0)
return (error);

vcpu = vm_find_vcpu(vm, vrwp->vrwp_vcpu_id);

if (vcpu == NULL((void *)0)) {
ret = ENOENT2;
goto out;
}

rw_enter_write(&vcpu->vc_lock);
if (vmm_softc->mode == VMM_MODE_EPT)
ret = (dir == 0) ?
    vcpu_readregs_vmx(vcpu, vrwp->vrwp_mask, 1, vrs) :
    vcpu_writeregs_vmx(vcpu, vrwp->vrwp_mask, 1, vrs);
else if (vmm_softc->mode == VMM_MODE_RVI)
ret = (dir == 0) ?
    vcpu_readregs_svm(vcpu, vrwp->vrwp_mask, vrs) :
    vcpu_writeregs_svm(vcpu, vrwp->vrwp_mask, vrs);
else {
DPRINTF("%s: unknown vmm mode", __func__);
ret = EINVAL22;
}
rw_exit_write(&vcpu->vc_lock);
629out:
refcnt_rele_wake(&vm->vm_refcnt);
return (ret);
632}

634/*
* vm_mprotect_ept
*
* IOCTL handler to sets the access protections of the ept
*
* Parameters:
*   vmep: describes the memory for which the protect will be applied..
*
* Return values:
*  0: if successful
*  ENOENT: if the VM defined by 'vmep' cannot be found
*  EINVAL: if the sgpa or size is not page aligned, the prot is invalid,
*          size is too large (512GB), there is wraparound
*          (like start = 512GB-1 and end = 512GB-2),
*          the address specified is not within the vm's mem range
*          or the address lies inside reserved (MMIO) memory
*/
651int
652vm_mprotect_ept(struct vm_mprotect_ept_params *vmep)
653{
struct vm *vm;
struct vcpu *vcpu;
vaddr_t sgpa;
size_t size;
vm_prot_t prot;
uint64_t msr;
int ret = 0, memtype;

/* If not EPT or RVI, nothing to do here */
if (!(vmm_softc->mode == VMM_MODE_EPT
   || vmm_softc->mode == VMM_MODE_RVI))
return (0);

/* Find the desired VM */
ret = vm_find(vmep->vmep_vm_id, &vm);

/* Not found? exit. */
if (ret != 0) {
DPRINTF("%s: vm id %u not found\n", __func__,
    vmep->vmep_vm_id);
return (ret);
}

vcpu = vm_find_vcpu(vm, vmep->vmep_vcpu_id);

if (vcpu == NULL((void *)0)) {
DPRINTF("%s: vcpu id %u of vm %u not found\n", __func__,
    vmep->vmep_vcpu_id, vmep->vmep_vm_id);
ret = ENOENT2;
goto out_nolock;
}

rw_enter_write(&vcpu->vc_lock);

if (vcpu->vc_state != VCPU_STATE_STOPPED) {
DPRINTF("%s: mprotect_ept %u on vm %u attempted "
    "while vcpu was in state %u (%s)\n", __func__,
    vmep->vmep_vcpu_id, vmep->vmep_vm_id, vcpu->vc_state,
    vcpu_state_decode(vcpu->vc_state));
ret = EBUSY16;
goto out;
}

/* Only proceed if the pmap is in the correct mode */
KASSERT((vmm_softc->mode == VMM_MODE_EPT &&(((vmm_softc->mode == VMM_MODE_EPT && vm->vm_map
->pmap->pm_type == 2) || (vmm_softc->mode == VMM_MODE_RVI
 && vm->vm_map->pmap->pm_type == 3)) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c"
, 701, "(vmm_softc->mode == VMM_MODE_EPT && vm->vm_map->pmap->pm_type == PMAP_TYPE_EPT) || (vmm_softc->mode == VMM_MODE_RVI && vm->vm_map->pmap->pm_type == PMAP_TYPE_RVI)"
))
   vm->vm_map->pmap->pm_type == PMAP_TYPE_EPT) ||(((vmm_softc->mode == VMM_MODE_EPT && vm->vm_map
->pmap->pm_type == 2) || (vmm_softc->mode == VMM_MODE_RVI
 && vm->vm_map->pmap->pm_type == 3)) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c"
, 701, "(vmm_softc->mode == VMM_MODE_EPT && vm->vm_map->pmap->pm_type == PMAP_TYPE_EPT) || (vmm_softc->mode == VMM_MODE_RVI && vm->vm_map->pmap->pm_type == PMAP_TYPE_RVI)"
))
   (vmm_softc->mode == VMM_MODE_RVI &&(((vmm_softc->mode == VMM_MODE_EPT && vm->vm_map
->pmap->pm_type == 2) || (vmm_softc->mode == VMM_MODE_RVI
 && vm->vm_map->pmap->pm_type == 3)) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c"
, 701, "(vmm_softc->mode == VMM_MODE_EPT && vm->vm_map->pmap->pm_type == PMAP_TYPE_EPT) || (vmm_softc->mode == VMM_MODE_RVI && vm->vm_map->pmap->pm_type == PMAP_TYPE_RVI)"
))
    vm->vm_map->pmap->pm_type == PMAP_TYPE_RVI))(((vmm_softc->mode == VMM_MODE_EPT && vm->vm_map
->pmap->pm_type == 2) || (vmm_softc->mode == VMM_MODE_RVI
 && vm->vm_map->pmap->pm_type == 3)) ? (void
)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c"
, 701, "(vmm_softc->mode == VMM_MODE_EPT && vm->vm_map->pmap->pm_type == PMAP_TYPE_EPT) || (vmm_softc->mode == VMM_MODE_RVI && vm->vm_map->pmap->pm_type == PMAP_TYPE_RVI)"
));

sgpa = vmep->vmep_sgpa;
size = vmep->vmep_size;
prot = vmep->vmep_prot;

/* No W^X permissions */
if ((prot & PROT_MASK(0x01 | 0x02 | 0x04)) != prot &&
   (prot & (PROT_WRITE0x02 | PROT_EXEC0x04)) == (PROT_WRITE0x02 | PROT_EXEC0x04)) {
DPRINTF("%s: W+X permission requested\n", __func__);
ret = EINVAL22;
goto out;
}

/* No Write only permissions */
if ((prot & (PROT_READ0x01 | PROT_WRITE0x02 | PROT_EXEC0x04)) == PROT_WRITE0x02) {
DPRINTF("%s: No Write only permissions\n", __func__);
ret = EINVAL22;
goto out;
}

/* No empty permissions */
if (prot == 0) {
DPRINTF("%s: No empty permissions\n", __func__);
ret = EINVAL22;
goto out;
}

/* No execute only on EPT CPUs that don't have that capability */
if (vmm_softc->mode == VMM_MODE_EPT) {
msr = rdmsr(IA32_VMX_EPT_VPID_CAP0x48C);
if (prot == PROT_EXEC0x04 &&
    (msr & IA32_EPT_VPID_CAP_XO_TRANSLATIONS(1ULL << 0)) == 0) {
	DPRINTF("%s: Execute only permissions unsupported,"
	   " adding read permission\n", __func__);

	prot |= PROT_READ0x01;
}
}

/* Must be page aligned */
if ((sgpa & PAGE_MASK((1 << 12) - 1)) || (size & PAGE_MASK((1 << 12) - 1)) || size == 0) {
ret = EINVAL22;
goto out;
}

/* size must be less then 512GB */
if (size >= NBPD_L4(1ULL << 39)) {
ret = EINVAL22;
goto out;
}

/* no wraparound */
if (sgpa + size < sgpa) {
ret = EINVAL22;
goto out;
}

/*
* Specifying addresses within the PCI MMIO space is forbidden.
* Disallow addresses that start inside the MMIO space:
* [VMM_PCI_MMIO_BAR_BASE .. VMM_PCI_MMIO_BAR_END]
*/
if (sgpa >= VMM_PCI_MMIO_BAR_BASE0xF0000000ULL && sgpa <= VMM_PCI_MMIO_BAR_END0xFFDFFFFFULL) {
ret = EINVAL22;
goto out;
}

/*
* ... and disallow addresses that end inside the MMIO space:
* (VMM_PCI_MMIO_BAR_BASE .. VMM_PCI_MMIO_BAR_END]
*/
if (sgpa + size > VMM_PCI_MMIO_BAR_BASE0xF0000000ULL &&
   sgpa + size <= VMM_PCI_MMIO_BAR_END0xFFDFFFFFULL) {
ret = EINVAL22;
goto out;
}

memtype = vmm_get_guest_memtype(vm, sgpa);
if (memtype == VMM_MEM_TYPE_UNKNOWN) {
ret = EINVAL22;
goto out;
}

if (vmm_softc->mode == VMM_MODE_EPT)
ret = vmx_mprotect_ept(vm->vm_map, sgpa, sgpa + size, prot);
else if (vmm_softc->mode == VMM_MODE_RVI) {
pmap_write_protect(vm->vm_map->pmap, sgpa, sgpa + size, prot);
/* XXX requires a invlpga */
ret = 0;
} else
ret = EINVAL22;
793out:
if (vcpu != NULL((void *)0))
rw_exit_write(&vcpu->vc_lock);
796out_nolock:
refcnt_rele_wake(&vm->vm_refcnt);
return (ret);
799}

801/*
* vmx_mprotect_ept
*
* apply the ept protections to the requested pages, faulting in the page if
* required.
*/
807int
808vmx_mprotect_ept(vm_map_t vm_map, paddr_t sgpa, paddr_t egpa, int prot)
809{
struct vmx_invept_descriptor vid;
pmap_t pmap;
pt_entry_t *pte;
paddr_t addr;
int ret = 0;

pmap = vm_map->pmap;

KERNEL_LOCK()_kernel_lock();

for (addr = sgpa; addr < egpa; addr += PAGE_SIZE(1 << 12)) {
pte = vmx_pmap_find_pte_ept(pmap, addr);
if (pte == NULL((void *)0)) {
	ret = uvm_fault(vm_map, addr, VM_FAULT_WIRE((vm_fault_t) 0x2),
	    PROT_READ0x01 | PROT_WRITE0x02 | PROT_EXEC0x04);
	if (ret)
		printf("%s: uvm_fault returns %d, GPA=0x%llx\n",
		    __func__, ret, (uint64_t)addr);

	pte = vmx_pmap_find_pte_ept(pmap, addr);
	if (pte == NULL((void *)0)) {
		KERNEL_UNLOCK()_kernel_unlock();
		return EFAULT14;
	}
}

if (prot & PROT_READ0x01)
	*pte |= EPT_R(1ULL << 0);
else
	*pte &= ~EPT_R(1ULL << 0);

if (prot & PROT_WRITE0x02)
	*pte |= EPT_W(1ULL << 1);
else
	*pte &= ~EPT_W(1ULL << 1);

if (prot & PROT_EXEC0x04)
	*pte |= EPT_X(1ULL << 2);
else
	*pte &= ~EPT_X(1ULL << 2);
}

/*
* SDM 3C: 28.3.3.4 Guidelines for Use of the INVEPT Instruction
* the first bullet point seems to say we should call invept.
*
* Software should use the INVEPT instruction with the “single-context”
* INVEPT type after making any of the following changes to an EPT
* paging-structure entry (the INVEPT descriptor should contain an
* EPTP value that references — directly or indirectly
* — the modified EPT paging structure):
* —   Changing any of the privilege bits 2:0 from 1 to 0.
* */
if (pmap->eptp != 0) {
memset(&vid, 0, sizeof(vid))__builtin_memset((&vid), (0), (sizeof(vid)));
vid.vid_eptp = pmap->eptp;
DPRINTF("%s: flushing EPT TLB for EPTP 0x%llx\n", __func__,
    vid.vid_eptp);
invept(IA32_VMX_INVEPT_SINGLE_CTX0x1, &vid);
}

KERNEL_UNLOCK()_kernel_unlock();

return ret;
874}

876/*
* vmx_pmap_find_pte_ept
*
* find the page table entry specified by addr in the pmap supplied.
*/
881pt_entry_t *
882vmx_pmap_find_pte_ept(pmap_t pmap, paddr_t addr)
883{
int l4idx, l3idx, l2idx, l1idx;
pd_entry_t *pd;
paddr_t pdppa;
pt_entry_t *ptes, *pte;

l4idx = (addr & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39; /* PML4E idx */
l3idx = (addr & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30; /* PDPTE idx */
l2idx = (addr & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21; /* PDE idx */
l1idx = (addr & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12; /* PTE idx */

pd = (pd_entry_t *)pmap->pm_pdir;
if (pd == NULL((void *)0))
return NULL((void *)0);

/*
* l4idx should always be 0 since we don't support more than 512GB
* guest physical memory.
*/
if (l4idx > 0)
return NULL((void *)0);

/*
* l3idx should always be < MAXDSIZ/1GB because we don't support more
* than MAXDSIZ guest phys mem.
*/
if (l3idx >= MAXDSIZ((paddr_t)128*1024*1024*1024) / ((paddr_t)1024 * 1024 * 1024))
return NULL((void *)0);

pdppa = pd[l4idx] & PG_FRAME0x000ffffffffff000UL;
if (pdppa == 0)
return NULL((void *)0);

ptes = (pt_entry_t *)PMAP_DIRECT_MAP(pdppa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pdppa));

pdppa = ptes[l3idx] & PG_FRAME0x000ffffffffff000UL;
if (pdppa == 0)
return NULL((void *)0);

ptes = (pt_entry_t *)PMAP_DIRECT_MAP(pdppa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pdppa));

pdppa = ptes[l2idx] & PG_FRAME0x000ffffffffff000UL;
if (pdppa == 0)
return NULL((void *)0);

ptes = (pt_entry_t *)PMAP_DIRECT_MAP(pdppa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pdppa));

pte = &ptes[l1idx];
if (*pte == 0)
return NULL((void *)0);

return pte;
935}

937/*
* vmm_start
*
* Starts VMM mode on the system
*/
942int
943vmm_start(void)
944{
struct cpu_info *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
946#ifdef MULTIPROCESSOR1
struct cpu_info *ci;
CPU_INFO_ITERATORint cii;
949#ifdef MP_LOCKDEBUG
int nticks;
951#endif /* MP_LOCKDEBUG */
952#endif /* MULTIPROCESSOR */

/* VMM is already running */
if (self->ci_flags & CPUF_VMM0x20000)
return (0);

/* Start VMM on this CPU */
start_vmm_on_cpu(self);
if (!(self->ci_flags & CPUF_VMM0x20000)) {
printf("%s: failed to enter VMM mode\n",
	self->ci_dev->dv_xname);
return (EIO5);
}

966#ifdef MULTIPROCESSOR1
/* Broadcast start VMM IPI */
x86_broadcast_ipi(X86_IPI_START_VMM0x00000100);

CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
if (ci == self)
	continue;
973#ifdef MP_LOCKDEBUG
nticks = __mp_lock_spinout;
975#endif /* MP_LOCKDEBUG */
while (!(ci->ci_flags & CPUF_VMM0x20000)) {
	CPU_BUSY_CYCLE()__asm volatile("pause": : : "memory");
978#ifdef MP_LOCKDEBUG
	if (--nticks <= 0) {
		db_printf("%s: spun out", __func__);
		db_enter();
		nticks = __mp_lock_spinout;
	}
984#endif /* MP_LOCKDEBUG */
}
}
987#endif /* MULTIPROCESSOR */

return (0);
990}

992/*
* vmm_stop
*
* Stops VMM mode on the system
*/
997int
998vmm_stop(void)
999{
struct cpu_info *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
1001#ifdef MULTIPROCESSOR1
struct cpu_info *ci;
CPU_INFO_ITERATORint cii;
1004#ifdef MP_LOCKDEBUG
int nticks;
1006#endif /* MP_LOCKDEBUG */
1007#endif /* MULTIPROCESSOR */

/* VMM is not running */
if (!(self->ci_flags & CPUF_VMM0x20000))
return (0);

/* Stop VMM on this CPU */
stop_vmm_on_cpu(self);
if (self->ci_flags & CPUF_VMM0x20000) {
printf("%s: failed to exit VMM mode\n",
	self->ci_dev->dv_xname);
return (EIO5);
}

1021#ifdef MULTIPROCESSOR1
/* Stop VMM on other CPUs */
x86_broadcast_ipi(X86_IPI_STOP_VMM0x00000200);

CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
if (ci == self)
	continue;
1028#ifdef MP_LOCKDEBUG
nticks = __mp_lock_spinout;
1030#endif /* MP_LOCKDEBUG */
while ((ci->ci_flags & CPUF_VMM0x20000)) {
	CPU_BUSY_CYCLE()__asm volatile("pause": : : "memory");
1033#ifdef MP_LOCKDEBUG
	if (--nticks <= 0) {
		db_printf("%s: spunout", __func__);
		db_enter();
		nticks = __mp_lock_spinout;
	}
1039#endif /* MP_LOCKDEBUG */
}
}
1042#endif /* MULTIPROCESSOR */

return (0);
1045}

1047/*
* start_vmm_on_cpu
*
* Starts VMM mode on 'ci' by executing the appropriate CPU-specific insn
* sequence to enter VMM mode (eg, VMXON)
*/
1053void
1054start_vmm_on_cpu(struct cpu_info *ci)
1055{
uint64_t msr;
uint32_t cr4;
struct vmx_invept_descriptor vid;

/* No VMM mode? exit. */
if ((ci->ci_vmm_flags & CI_VMM_VMX(1 << 0)) == 0 &&
   (ci->ci_vmm_flags & CI_VMM_SVM(1 << 1)) == 0)
return;

/*
* AMD SVM
*/
if (ci->ci_vmm_flags & CI_VMM_SVM(1 << 1)) {
msr = rdmsr(MSR_EFER0xc0000080);
msr |= EFER_SVME0x00001000;
wrmsr(MSR_EFER0xc0000080, msr);
}

/*
* Intel VMX
*/
if (ci->ci_vmm_flags & CI_VMM_VMX(1 << 0)) {
if (ci->ci_vmxon_region == 0)
	return;
else {
	bzero(ci->ci_vmxon_region, PAGE_SIZE)__builtin_bzero((ci->ci_vmxon_region), ((1 << 12)));
	ci->ci_vmxon_region->vr_revision =
	    ci->ci_vmm_cap.vcc_vmx.vmx_vmxon_revision;

	/* Enable VMX */
	msr = rdmsr(MSR_IA32_FEATURE_CONTROL0x03a);
	if (msr & IA32_FEATURE_CONTROL_LOCK0x01) {
		if (!(msr & IA32_FEATURE_CONTROL_VMX_EN0x04))
			return;
	} else {
		msr |= IA32_FEATURE_CONTROL_VMX_EN0x04 |
		    IA32_FEATURE_CONTROL_LOCK0x01;
		wrmsr(MSR_IA32_FEATURE_CONTROL0x03a, msr);
	}

	/* Set CR4.VMXE */
	cr4 = rcr4();
	cr4 |= CR4_VMXE0x00002000;
	lcr4(cr4);

	/* Enter VMX mode and clear EPTs on this cpu */
	if (vmxon((uint64_t *)&ci->ci_vmxon_region_pa))
		panic("vmxon failed");

	memset(&vid, 0, sizeof(vid))__builtin_memset((&vid), (0), (sizeof(vid)));
	if (invept(IA32_VMX_INVEPT_GLOBAL_CTX0x2, &vid))
		panic("invept failed");
}
}

atomic_setbits_intx86_atomic_setbits_u32(&ci->ci_flags, CPUF_VMM0x20000);
1112}

1114/*
* stop_vmm_on_cpu
*
* Stops VMM mode on 'ci' by executing the appropriate CPU-specific insn
* sequence to exit VMM mode (eg, VMXOFF)
*/
1120void
1121stop_vmm_on_cpu(struct cpu_info *ci)
1122{
uint64_t msr;
uint32_t cr4;

if (!(ci->ci_flags & CPUF_VMM0x20000))
return;

/*
* AMD SVM
*/
if (ci->ci_vmm_flags & CI_VMM_SVM(1 << 1)) {
msr = rdmsr(MSR_EFER0xc0000080);
msr &= ~EFER_SVME0x00001000;
wrmsr(MSR_EFER0xc0000080, msr);
}

/*
* Intel VMX
*/
if (ci->ci_vmm_flags & CI_VMM_VMX(1 << 0)) {
if (vmxoff())
	panic("VMXOFF failed");

cr4 = rcr4();
cr4 &= ~CR4_VMXE0x00002000;
lcr4(cr4);
}

atomic_clearbits_intx86_atomic_clearbits_u32(&ci->ci_flags, CPUF_VMM0x20000);
1151}

1153/*
* vmclear_on_cpu
*
* Flush and clear VMCS on 'ci' by executing vmclear.
*
*/
1159void
1160vmclear_on_cpu(struct cpu_info *ci)
1161{
if ((ci->ci_flags & CPUF_VMM0x20000) && (ci->ci_vmm_flags & CI_VMM_VMX(1 << 0))) {
if (vmclear(&ci->ci_vmcs_pa))
	panic("VMCLEAR ipi failed");
atomic_swap_ulong(&ci->ci_vmcs_pa, VMX_VMCS_PA_CLEAR)_atomic_swap_ulong((&ci->ci_vmcs_pa), (0xFFFFFFFFFFFFFFFFUL
));
}
1167}

1169#ifdef MULTIPROCESSOR1
1170static int
1171vmx_remote_vmclear(struct cpu_info *ci, struct vcpu *vcpu)
1172{
1173#ifdef MP_LOCKDEBUG
int nticks = __mp_lock_spinout;
1175#endif /* MP_LOCKDEBUG */

rw_enter_write(&ci->ci_vmcs_lock);
atomic_swap_ulong(&ci->ci_vmcs_pa, vcpu->vc_control_pa)_atomic_swap_ulong((&ci->ci_vmcs_pa), (vcpu->vc_control_pa
));
x86_send_ipi(ci, X86_IPI_VMCLEAR_VMM0x00000004);

while (ci->ci_vmcs_pa != VMX_VMCS_PA_CLEAR0xFFFFFFFFFFFFFFFFUL) {
CPU_BUSY_CYCLE()__asm volatile("pause": : : "memory");
1183#ifdef MP_LOCKDEBUG
if (--nticks <= 0) {
	db_printf("%s: spun out\n", __func__);
	db_enter();
	nticks = __mp_lock_spinout;
}
1189#endif /* MP_LOCKDEBUG */
}
atomic_swap_uint(&vcpu->vc_vmx_vmcs_state, VMCS_CLEARED)_atomic_swap_uint((&vcpu->vc_vmx_vmcs_state), (0));
rw_exit_write(&ci->ci_vmcs_lock);

return (0);
1195}
1196#endif /* MULTIPROCESSOR */

1198/*
* vm_impl_init_vmx
*
* Intel VMX specific VM initialization routine
*
* Parameters:
*  vm: the VM being initialized
*   p: vmd process owning the VM
*
* Return values:
*  0: the initialization was successful
*  ENOMEM: the initialization failed (lack of resources)
*/
1211int
1212vm_impl_init_vmx(struct vm *vm, struct proc *p)
1213{
int i, ret;
vaddr_t mingpa, maxgpa;
struct vm_mem_range *vmr;

/* If not EPT, nothing to do here */
if (vmm_softc->mode != VMM_MODE_EPT)
return (0);

vmr = &vm->vm_memranges[0];
mingpa = vmr->vmr_gpa;
vmr = &vm->vm_memranges[vm->vm_nmemranges - 1];
maxgpa = vmr->vmr_gpa + vmr->vmr_size;

/*
* uvmspace_alloc (currently) always returns a valid vmspace
*/
vm->vm_vmspace = uvmspace_alloc(mingpa, maxgpa, TRUE1, FALSE0);
vm->vm_map = &vm->vm_vmspace->vm_map;

/* Map the new map with an anon */
DPRINTF("%s: created vm_map @ %p\n", __func__, vm->vm_map);
for (i = 0; i < vm->vm_nmemranges; i++) {
vmr = &vm->vm_memranges[i];
ret = uvm_share(vm->vm_map, vmr->vmr_gpa,
    PROT_READ0x01 | PROT_WRITE0x02 | PROT_EXEC0x04,
    &p->p_vmspace->vm_map, vmr->vmr_va, vmr->vmr_size);
if (ret) {
	printf("%s: uvm_share failed (%d)\n", __func__, ret);
	/* uvmspace_free calls pmap_destroy for us */
	uvmspace_free(vm->vm_vmspace);
	vm->vm_vmspace = NULL((void *)0);
	return (ENOMEM12);
}
}

pmap_convert(vm->vm_map->pmap, PMAP_TYPE_EPT2);

return (0);
1252}

1254/*
* vm_impl_init_svm
*
* AMD SVM specific VM initialization routine
*
* Parameters:
*  vm: the VM being initialized
*   p: vmd process owning the VM
*
* Return values:
*  0: the initialization was successful
*  ENOMEM: the initialization failed (lack of resources)
*/
1267int
1268vm_impl_init_svm(struct vm *vm, struct proc *p)
1269{
int i, ret;
vaddr_t mingpa, maxgpa;
struct vm_mem_range *vmr;

/* If not RVI, nothing to do here */
if (vmm_softc->mode != VMM_MODE_RVI)
return (0);

vmr = &vm->vm_memranges[0];
mingpa = vmr->vmr_gpa;
vmr = &vm->vm_memranges[vm->vm_nmemranges - 1];
maxgpa = vmr->vmr_gpa + vmr->vmr_size;

/*
* uvmspace_alloc (currently) always returns a valid vmspace
*/
vm->vm_vmspace = uvmspace_alloc(mingpa, maxgpa, TRUE1, FALSE0);
vm->vm_map = &vm->vm_vmspace->vm_map;

/* Map the new map with an anon */
DPRINTF("%s: created vm_map @ %p\n", __func__, vm->vm_map);
for (i = 0; i < vm->vm_nmemranges; i++) {
vmr = &vm->vm_memranges[i];
ret = uvm_share(vm->vm_map, vmr->vmr_gpa,
    PROT_READ0x01 | PROT_WRITE0x02 | PROT_EXEC0x04,
    &p->p_vmspace->vm_map, vmr->vmr_va, vmr->vmr_size);
if (ret) {
	printf("%s: uvm_share failed (%d)\n", __func__, ret);
	/* uvmspace_free calls pmap_destroy for us */
	uvmspace_free(vm->vm_vmspace);
	vm->vm_vmspace = NULL((void *)0);
	return (ENOMEM12);
}
}

/* Convert pmap to RVI */
pmap_convert(vm->vm_map->pmap, PMAP_TYPE_RVI3);

return (0);
1309}

1311/*
* vm_impl_init
*
* Calls the architecture-specific VM init routine
*
* Parameters:
*  vm: the VM being initialized
*   p: vmd process owning the VM
*
* Return values (from architecture-specific init routines):
*  0: the initialization was successful
*  ENOMEM: the initialization failed (lack of resources)
*/
1324int
1325vm_impl_init(struct vm *vm, struct proc *p)
1326{
int ret;

KERNEL_LOCK()_kernel_lock();
if (vmm_softc->mode == VMM_MODE_EPT)
ret = vm_impl_init_vmx(vm, p);
else if	(vmm_softc->mode == VMM_MODE_RVI)
ret = vm_impl_init_svm(vm, p);
else
panic("%s: unknown vmm mode: %d", __func__, vmm_softc->mode);
KERNEL_UNLOCK()_kernel_unlock();

return (ret);
1339}

1341void
1342vm_impl_deinit(struct vm *vm)
1343{
/* unused */
1345}

1347/*
* vcpu_reload_vmcs_vmx
*
* (Re)load the VMCS on the current cpu. Must be called with the VMCS write
* lock acquired. If the VMCS is determined to be loaded on a remote cpu, an
* ipi will be used to remotely flush it before loading the VMCS locally.
*
* Parameters:
*  vcpu: Pointer to the vcpu needing its VMCS
*
* Return values:
*  0: if successful
*  EINVAL: an error occurred during flush or reload
*/
1361int
1362vcpu_reload_vmcs_vmx(struct vcpu *vcpu)
1363{
struct cpu_info *ci, *last_ci;

rw_assert_wrlock(&vcpu->vc_lock);

ci = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
last_ci = vcpu->vc_last_pcpu;

if (last_ci == NULL((void *)0)) {
/* First launch */
if (vmclear(&vcpu->vc_control_pa))
		return (EINVAL22);
atomic_swap_uint(&vcpu->vc_vmx_vmcs_state, VMCS_CLEARED)_atomic_swap_uint((&vcpu->vc_vmx_vmcs_state), (0));
1376#ifdef MULTIPROCESSOR1
} else if (last_ci != ci) {
/* We've moved CPUs at some point, so remote VMCLEAR */
if (vmx_remote_vmclear(last_ci, vcpu))
	return (EINVAL22);
KASSERT(vcpu->vc_vmx_vmcs_state == VMCS_CLEARED)((vcpu->vc_vmx_vmcs_state == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c", 1381, "vcpu->vc_vmx_vmcs_state == VMCS_CLEARED"
));
1382#endif /* MULTIPROCESSOR */
}

if (vmptrld(&vcpu->vc_control_pa)) {
printf("%s: vmptrld\n", __func__);
return (EINVAL22);
}

return (0);
1391}

1393/*
* vcpu_readregs_vmx
*
* Reads 'vcpu's registers
*
* Parameters:
*  vcpu: the vcpu to read register values from
*  regmask: the types of registers to read
*  loadvmcs: bit to indicate whether the VMCS has to be loaded first
*  vrs: output parameter where register values are stored
*
* Return values:
*  0: if successful
*  EINVAL: an error reading registers occurred
*/
1408int
1409vcpu_readregs_vmx(struct vcpu *vcpu, uint64_t regmask, int loadvmcs,
  struct vcpu_reg_state *vrs)
1411{
int i, ret = 0;
uint64_t sel, limit, ar;
uint64_t *gprs = vrs->vrs_gprs;
uint64_t *crs = vrs->vrs_crs;
uint64_t *msrs = vrs->vrs_msrs;
uint64_t *drs = vrs->vrs_drs;
struct vcpu_segment_info *sregs = vrs->vrs_sregs;
struct vmx_msr_store *msr_store;

if (loadvmcs) {
if (vcpu_reload_vmcs_vmx(vcpu))
	return (EINVAL22);
}

1426#ifdef VMM_DEBUG
/* VMCS should be loaded... */
paddr_t pa = 0ULL;
if (vmptrst(&pa))
panic("%s: vmptrst", __func__);
KASSERT(pa == vcpu->vc_control_pa)((pa == vcpu->vc_control_pa) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c", 1431, "pa == vcpu->vc_control_pa"
));
1432#endif /* VMM_DEBUG */

if (regmask & VM_RWREGS_GPRS0x1) {
gprs[VCPU_REGS_RAX0] = vcpu->vc_gueststate.vg_rax;
gprs[VCPU_REGS_RBX3] = vcpu->vc_gueststate.vg_rbx;
gprs[VCPU_REGS_RCX1] = vcpu->vc_gueststate.vg_rcx;
gprs[VCPU_REGS_RDX2] = vcpu->vc_gueststate.vg_rdx;
gprs[VCPU_REGS_RSI6] = vcpu->vc_gueststate.vg_rsi;
gprs[VCPU_REGS_RDI7] = vcpu->vc_gueststate.vg_rdi;
gprs[VCPU_REGS_R88] = vcpu->vc_gueststate.vg_r8;
gprs[VCPU_REGS_R99] = vcpu->vc_gueststate.vg_r9;
gprs[VCPU_REGS_R1010] = vcpu->vc_gueststate.vg_r10;
gprs[VCPU_REGS_R1111] = vcpu->vc_gueststate.vg_r11;
gprs[VCPU_REGS_R1212] = vcpu->vc_gueststate.vg_r12;
gprs[VCPU_REGS_R1313] = vcpu->vc_gueststate.vg_r13;
gprs[VCPU_REGS_R1414] = vcpu->vc_gueststate.vg_r14;
gprs[VCPU_REGS_R1515] = vcpu->vc_gueststate.vg_r15;
gprs[VCPU_REGS_RBP5] = vcpu->vc_gueststate.vg_rbp;
gprs[VCPU_REGS_RIP16] = vcpu->vc_gueststate.vg_rip;
if (vmread(VMCS_GUEST_IA32_RSP0x681C, &gprs[VCPU_REGS_RSP4]))
	goto errout;
              if (vmread(VMCS_GUEST_IA32_RFLAGS0x6820, &gprs[VCPU_REGS_RFLAGS17]))
	goto errout;
      }

if (regmask & VM_RWREGS_SREGS0x2) {
for (i = 0; i < nitems(vmm_vmx_sreg_vmcs_fields)(sizeof((vmm_vmx_sreg_vmcs_fields)) / sizeof((vmm_vmx_sreg_vmcs_fields
)[0])); i++) {
	if (vmread(vmm_vmx_sreg_vmcs_fields[i].selid, &sel))
		goto errout;
	if (vmread(vmm_vmx_sreg_vmcs_fields[i].limitid, &limit))
		goto errout;
	if (vmread(vmm_vmx_sreg_vmcs_fields[i].arid, &ar))
		goto errout;
	if (vmread(vmm_vmx_sreg_vmcs_fields[i].baseid,
	   &sregs[i].vsi_base))
		goto errout;

	sregs[i].vsi_sel = sel;
	sregs[i].vsi_limit = limit;
	sregs[i].vsi_ar = ar;
}

if (vmread(VMCS_GUEST_IA32_GDTR_LIMIT0x4810, &limit))
	goto errout;
if (vmread(VMCS_GUEST_IA32_GDTR_BASE0x6816,
    &vrs->vrs_gdtr.vsi_base))
	goto errout;
vrs->vrs_gdtr.vsi_limit = limit;

if (vmread(VMCS_GUEST_IA32_IDTR_LIMIT0x4812, &limit))
	goto errout;
if (vmread(VMCS_GUEST_IA32_IDTR_BASE0x6818,
    &vrs->vrs_idtr.vsi_base))
	goto errout;
vrs->vrs_idtr.vsi_limit = limit;
}

if (regmask & VM_RWREGS_CRS0x4) {
crs[VCPU_REGS_CR21] = vcpu->vc_gueststate.vg_cr2;
crs[VCPU_REGS_XCR05] = vcpu->vc_gueststate.vg_xcr0;
if (vmread(VMCS_GUEST_IA32_CR00x6800, &crs[VCPU_REGS_CR00]))
	goto errout;
if (vmread(VMCS_GUEST_IA32_CR30x6802, &crs[VCPU_REGS_CR32]))
	goto errout;
if (vmread(VMCS_GUEST_IA32_CR40x6804, &crs[VCPU_REGS_CR43]))
	goto errout;
if (vmread(VMCS_GUEST_PDPTE00x280A, &crs[VCPU_REGS_PDPTE06]))
	goto errout;
if (vmread(VMCS_GUEST_PDPTE10x280C, &crs[VCPU_REGS_PDPTE17]))
	goto errout;
if (vmread(VMCS_GUEST_PDPTE20x280E, &crs[VCPU_REGS_PDPTE28]))
	goto errout;
if (vmread(VMCS_GUEST_PDPTE30x2810, &crs[VCPU_REGS_PDPTE39]))
	goto errout;
}

msr_store = (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_save_va;

if (regmask & VM_RWREGS_MSRS0x8) {
for (i = 0; i < VCPU_REGS_NMSRS(6 + 1); i++) {
	msrs[i] = msr_store[i].vms_data;
}
}

if (regmask & VM_RWREGS_DRS0x10) {
drs[VCPU_REGS_DR00] = vcpu->vc_gueststate.vg_dr0;
drs[VCPU_REGS_DR11] = vcpu->vc_gueststate.vg_dr1;
drs[VCPU_REGS_DR22] = vcpu->vc_gueststate.vg_dr2;
drs[VCPU_REGS_DR33] = vcpu->vc_gueststate.vg_dr3;
drs[VCPU_REGS_DR64] = vcpu->vc_gueststate.vg_dr6;
if (vmread(VMCS_GUEST_IA32_DR70x681A, &drs[VCPU_REGS_DR75]))
	goto errout;
}

goto out;

1528errout:
ret = EINVAL22;
1530out:
return (ret);
1532}

1534/*
* vcpu_readregs_svm
*
* Reads 'vcpu's registers
*
* Parameters:
*  vcpu: the vcpu to read register values from
*  regmask: the types of registers to read
*  vrs: output parameter where register values are stored
*
* Return values:
*  0: if successful
*/
1547int
1548vcpu_readregs_svm(struct vcpu *vcpu, uint64_t regmask,
  struct vcpu_reg_state *vrs)
1550{
uint64_t *gprs = vrs->vrs_gprs;
uint64_t *crs = vrs->vrs_crs;
uint64_t *msrs = vrs->vrs_msrs;
uint64_t *drs = vrs->vrs_drs;
uint32_t attr;
struct vcpu_segment_info *sregs = vrs->vrs_sregs;
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;

if (regmask & VM_RWREGS_GPRS0x1) {
gprs[VCPU_REGS_RAX0] = vmcb->v_rax;
gprs[VCPU_REGS_RBX3] = vcpu->vc_gueststate.vg_rbx;
gprs[VCPU_REGS_RCX1] = vcpu->vc_gueststate.vg_rcx;
gprs[VCPU_REGS_RDX2] = vcpu->vc_gueststate.vg_rdx;
gprs[VCPU_REGS_RSI6] = vcpu->vc_gueststate.vg_rsi;
gprs[VCPU_REGS_RDI7] = vcpu->vc_gueststate.vg_rdi;
gprs[VCPU_REGS_R88] = vcpu->vc_gueststate.vg_r8;
gprs[VCPU_REGS_R99] = vcpu->vc_gueststate.vg_r9;
gprs[VCPU_REGS_R1010] = vcpu->vc_gueststate.vg_r10;
gprs[VCPU_REGS_R1111] = vcpu->vc_gueststate.vg_r11;
gprs[VCPU_REGS_R1212] = vcpu->vc_gueststate.vg_r12;
gprs[VCPU_REGS_R1313] = vcpu->vc_gueststate.vg_r13;
gprs[VCPU_REGS_R1414] = vcpu->vc_gueststate.vg_r14;
gprs[VCPU_REGS_R1515] = vcpu->vc_gueststate.vg_r15;
gprs[VCPU_REGS_RBP5] = vcpu->vc_gueststate.vg_rbp;
gprs[VCPU_REGS_RIP16] = vmcb->v_rip;
gprs[VCPU_REGS_RSP4] = vmcb->v_rsp;
gprs[VCPU_REGS_RFLAGS17] = vmcb->v_rflags;
}

if (regmask & VM_RWREGS_SREGS0x2) {
sregs[VCPU_REGS_CS1].vsi_sel = vmcb->v_cs.vs_sel;
sregs[VCPU_REGS_CS1].vsi_limit = vmcb->v_cs.vs_lim;
attr = vmcb->v_cs.vs_attr;
sregs[VCPU_REGS_CS1].vsi_ar = (attr & 0xff) | ((attr << 4) &
    0xf000);
sregs[VCPU_REGS_CS1].vsi_base = vmcb->v_cs.vs_base;

sregs[VCPU_REGS_DS3].vsi_sel = vmcb->v_ds.vs_sel;
sregs[VCPU_REGS_DS3].vsi_limit = vmcb->v_ds.vs_lim;
attr = vmcb->v_ds.vs_attr;
sregs[VCPU_REGS_DS3].vsi_ar = (attr & 0xff) | ((attr << 4) &
    0xf000);
sregs[VCPU_REGS_DS3].vsi_base = vmcb->v_ds.vs_base;

sregs[VCPU_REGS_ES0].vsi_sel = vmcb->v_es.vs_sel;
sregs[VCPU_REGS_ES0].vsi_limit = vmcb->v_es.vs_lim;
attr = vmcb->v_es.vs_attr;
sregs[VCPU_REGS_ES0].vsi_ar = (attr & 0xff) | ((attr << 4) &
    0xf000);
sregs[VCPU_REGS_ES0].vsi_base = vmcb->v_es.vs_base;

sregs[VCPU_REGS_FS4].vsi_sel = vmcb->v_fs.vs_sel;
sregs[VCPU_REGS_FS4].vsi_limit = vmcb->v_fs.vs_lim;
attr = vmcb->v_fs.vs_attr;
sregs[VCPU_REGS_FS4].vsi_ar = (attr & 0xff) | ((attr << 4) &
    0xf000);
sregs[VCPU_REGS_FS4].vsi_base = vmcb->v_fs.vs_base;

sregs[VCPU_REGS_GS5].vsi_sel = vmcb->v_gs.vs_sel;
sregs[VCPU_REGS_GS5].vsi_limit = vmcb->v_gs.vs_lim;
attr = vmcb->v_gs.vs_attr;
sregs[VCPU_REGS_GS5].vsi_ar = (attr & 0xff) | ((attr << 4) &
    0xf000);
sregs[VCPU_REGS_GS5].vsi_base = vmcb->v_gs.vs_base;

sregs[VCPU_REGS_SS2].vsi_sel = vmcb->v_ss.vs_sel;
sregs[VCPU_REGS_SS2].vsi_limit = vmcb->v_ss.vs_lim;
attr = vmcb->v_ss.vs_attr;
sregs[VCPU_REGS_SS2].vsi_ar = (attr & 0xff) | ((attr << 4) &
    0xf000);
sregs[VCPU_REGS_SS2].vsi_base = vmcb->v_ss.vs_base;

sregs[VCPU_REGS_LDTR6].vsi_sel = vmcb->v_ldtr.vs_sel;
sregs[VCPU_REGS_LDTR6].vsi_limit = vmcb->v_ldtr.vs_lim;
attr = vmcb->v_ldtr.vs_attr;
sregs[VCPU_REGS_LDTR6].vsi_ar = (attr & 0xff) | ((attr << 4)
    & 0xf000);
sregs[VCPU_REGS_LDTR6].vsi_base = vmcb->v_ldtr.vs_base;

sregs[VCPU_REGS_TR7].vsi_sel = vmcb->v_tr.vs_sel;
sregs[VCPU_REGS_TR7].vsi_limit = vmcb->v_tr.vs_lim;
attr = vmcb->v_tr.vs_attr;
sregs[VCPU_REGS_TR7].vsi_ar = (attr & 0xff) | ((attr << 4) &
    0xf000);
sregs[VCPU_REGS_TR7].vsi_base = vmcb->v_tr.vs_base;

vrs->vrs_gdtr.vsi_limit = vmcb->v_gdtr.vs_lim;
vrs->vrs_gdtr.vsi_base = vmcb->v_gdtr.vs_base;
vrs->vrs_idtr.vsi_limit = vmcb->v_idtr.vs_lim;
vrs->vrs_idtr.vsi_base = vmcb->v_idtr.vs_base;
}

if (regmask & VM_RWREGS_CRS0x4) {
crs[VCPU_REGS_CR00] = vmcb->v_cr0;
crs[VCPU_REGS_CR32] = vmcb->v_cr3;
crs[VCPU_REGS_CR43] = vmcb->v_cr4;
crs[VCPU_REGS_CR21] = vcpu->vc_gueststate.vg_cr2;
crs[VCPU_REGS_XCR05] = vcpu->vc_gueststate.vg_xcr0;
}

if (regmask & VM_RWREGS_MSRS0x8) {
 msrs[VCPU_REGS_EFER0] = vmcb->v_efer;
 msrs[VCPU_REGS_STAR1] = vmcb->v_star;
 msrs[VCPU_REGS_LSTAR2] = vmcb->v_lstar;
 msrs[VCPU_REGS_CSTAR3] = vmcb->v_cstar;
 msrs[VCPU_REGS_SFMASK4] = vmcb->v_sfmask;
 msrs[VCPU_REGS_KGSBASE5] = vmcb->v_kgsbase;
}

if (regmask & VM_RWREGS_DRS0x10) {
drs[VCPU_REGS_DR00] = vcpu->vc_gueststate.vg_dr0;
drs[VCPU_REGS_DR11] = vcpu->vc_gueststate.vg_dr1;
drs[VCPU_REGS_DR22] = vcpu->vc_gueststate.vg_dr2;
drs[VCPU_REGS_DR33] = vcpu->vc_gueststate.vg_dr3;
drs[VCPU_REGS_DR64] = vmcb->v_dr6;
drs[VCPU_REGS_DR75] = vmcb->v_dr7;
}

return (0);
1670}

1672/*
* vcpu_writeregs_vmx
*
* Writes VCPU registers
*
* Parameters:
*  vcpu: the vcpu that has to get its registers written to
*  regmask: the types of registers to write
*  loadvmcs: bit to indicate whether the VMCS has to be loaded first
*  vrs: the register values to write
*
* Return values:
*  0: if successful
*  EINVAL an error writing registers occurred
*/
1687int
1688vcpu_writeregs_vmx(struct vcpu *vcpu, uint64_t regmask, int loadvmcs,
  struct vcpu_reg_state *vrs)
1690{
int i, ret = 0;
uint16_t sel;
uint64_t limit, ar;
uint64_t *gprs = vrs->vrs_gprs;
uint64_t *crs = vrs->vrs_crs;
uint64_t *msrs = vrs->vrs_msrs;
uint64_t *drs = vrs->vrs_drs;
struct vcpu_segment_info *sregs = vrs->vrs_sregs;
struct vmx_msr_store *msr_store;

if (loadvmcs) {
if (vcpu_reload_vmcs_vmx(vcpu))
	return (EINVAL22);
}

1706#ifdef VMM_DEBUG
/* VMCS should be loaded... */
paddr_t pa = 0ULL;
if (vmptrst(&pa))
panic("%s: vmptrst", __func__);
KASSERT(pa == vcpu->vc_control_pa)((pa == vcpu->vc_control_pa) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c", 1711, "pa == vcpu->vc_control_pa"
));
1712#endif /* VMM_DEBUG */

if (regmask & VM_RWREGS_GPRS0x1) {
vcpu->vc_gueststate.vg_rax = gprs[VCPU_REGS_RAX0];
vcpu->vc_gueststate.vg_rbx = gprs[VCPU_REGS_RBX3];
vcpu->vc_gueststate.vg_rcx = gprs[VCPU_REGS_RCX1];
vcpu->vc_gueststate.vg_rdx = gprs[VCPU_REGS_RDX2];
vcpu->vc_gueststate.vg_rsi = gprs[VCPU_REGS_RSI6];
vcpu->vc_gueststate.vg_rdi = gprs[VCPU_REGS_RDI7];
vcpu->vc_gueststate.vg_r8 = gprs[VCPU_REGS_R88];
vcpu->vc_gueststate.vg_r9 = gprs[VCPU_REGS_R99];
vcpu->vc_gueststate.vg_r10 = gprs[VCPU_REGS_R1010];
vcpu->vc_gueststate.vg_r11 = gprs[VCPU_REGS_R1111];
vcpu->vc_gueststate.vg_r12 = gprs[VCPU_REGS_R1212];
vcpu->vc_gueststate.vg_r13 = gprs[VCPU_REGS_R1313];
vcpu->vc_gueststate.vg_r14 = gprs[VCPU_REGS_R1414];
vcpu->vc_gueststate.vg_r15 = gprs[VCPU_REGS_R1515];
vcpu->vc_gueststate.vg_rbp = gprs[VCPU_REGS_RBP5];
vcpu->vc_gueststate.vg_rip = gprs[VCPU_REGS_RIP16];
if (vmwrite(VMCS_GUEST_IA32_RIP0x681E, gprs[VCPU_REGS_RIP16]))
	goto errout;
if (vmwrite(VMCS_GUEST_IA32_RSP0x681C, gprs[VCPU_REGS_RSP4]))
	goto errout;
              if (vmwrite(VMCS_GUEST_IA32_RFLAGS0x6820, gprs[VCPU_REGS_RFLAGS17]))
	goto errout;
}

if (regmask & VM_RWREGS_SREGS0x2) {
for (i = 0; i < nitems(vmm_vmx_sreg_vmcs_fields)(sizeof((vmm_vmx_sreg_vmcs_fields)) / sizeof((vmm_vmx_sreg_vmcs_fields
)[0])); i++) {
	sel = sregs[i].vsi_sel;
	limit = sregs[i].vsi_limit;
	ar = sregs[i].vsi_ar;

	if (vmwrite(vmm_vmx_sreg_vmcs_fields[i].selid, sel))
		goto errout;
	if (vmwrite(vmm_vmx_sreg_vmcs_fields[i].limitid, limit))
		goto errout;
	if (vmwrite(vmm_vmx_sreg_vmcs_fields[i].arid, ar))
		goto errout;
	if (vmwrite(vmm_vmx_sreg_vmcs_fields[i].baseid,
	    sregs[i].vsi_base))
		goto errout;
}

if (vmwrite(VMCS_GUEST_IA32_GDTR_LIMIT0x4810,
    vrs->vrs_gdtr.vsi_limit))
	goto errout;
if (vmwrite(VMCS_GUEST_IA32_GDTR_BASE0x6816,
    vrs->vrs_gdtr.vsi_base))
	goto errout;
if (vmwrite(VMCS_GUEST_IA32_IDTR_LIMIT0x4812,
    vrs->vrs_idtr.vsi_limit))
	goto errout;
if (vmwrite(VMCS_GUEST_IA32_IDTR_BASE0x6818,
    vrs->vrs_idtr.vsi_base))
	goto errout;
}

if (regmask & VM_RWREGS_CRS0x4) {
vcpu->vc_gueststate.vg_xcr0 = crs[VCPU_REGS_XCR05];
if (vmwrite(VMCS_GUEST_IA32_CR00x6800, crs[VCPU_REGS_CR00]))
	goto errout;
if (vmwrite(VMCS_GUEST_IA32_CR30x6802, crs[VCPU_REGS_CR32]))
	goto errout;
if (vmwrite(VMCS_GUEST_IA32_CR40x6804, crs[VCPU_REGS_CR43]))
	goto errout;
if (vmwrite(VMCS_GUEST_PDPTE00x280A, crs[VCPU_REGS_PDPTE06]))
	goto errout;
if (vmwrite(VMCS_GUEST_PDPTE10x280C, crs[VCPU_REGS_PDPTE17]))
	goto errout;
if (vmwrite(VMCS_GUEST_PDPTE20x280E, crs[VCPU_REGS_PDPTE28]))
	goto errout;
if (vmwrite(VMCS_GUEST_PDPTE30x2810, crs[VCPU_REGS_PDPTE39]))
	goto errout;
}

msr_store = (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_save_va;

if (regmask & VM_RWREGS_MSRS0x8) {
for (i = 0; i < VCPU_REGS_NMSRS(6 + 1); i++) {
	msr_store[i].vms_data = msrs[i];
}
}

if (regmask & VM_RWREGS_DRS0x10) {
vcpu->vc_gueststate.vg_dr0 = drs[VCPU_REGS_DR00];
vcpu->vc_gueststate.vg_dr1 = drs[VCPU_REGS_DR11];
vcpu->vc_gueststate.vg_dr2 = drs[VCPU_REGS_DR22];
vcpu->vc_gueststate.vg_dr3 = drs[VCPU_REGS_DR33];
vcpu->vc_gueststate.vg_dr6 = drs[VCPU_REGS_DR64];
if (vmwrite(VMCS_GUEST_IA32_DR70x681A, drs[VCPU_REGS_DR75]))
	goto errout;
}

goto out;

1808errout:
ret = EINVAL22;
1810out:
if (loadvmcs) {
if (vmclear(&vcpu->vc_control_pa))
	ret = EINVAL22;
atomic_swap_uint(&vcpu->vc_vmx_vmcs_state, VMCS_CLEARED)_atomic_swap_uint((&vcpu->vc_vmx_vmcs_state), (0));
}
return (ret);
1817}

1819/*
* vcpu_writeregs_svm
*
* Writes 'vcpu's registers
*
* Parameters:
*  vcpu: the vcpu that has to get its registers written to
*  regmask: the types of registers to write
*  vrs: the register values to write
*
* Return values:
*  0: if successful
*  EINVAL an error writing registers occurred
*/
1833int
1834vcpu_writeregs_svm(struct vcpu *vcpu, uint64_t regmask,
  struct vcpu_reg_state *vrs)
1836{
uint64_t *gprs = vrs->vrs_gprs;
uint64_t *crs = vrs->vrs_crs;
uint16_t attr;
uint64_t *msrs = vrs->vrs_msrs;
uint64_t *drs = vrs->vrs_drs;
struct vcpu_segment_info *sregs = vrs->vrs_sregs;
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;

if (regmask & VM_RWREGS_GPRS0x1) {
vcpu->vc_gueststate.vg_rax = gprs[VCPU_REGS_RAX0];
vcpu->vc_gueststate.vg_rbx = gprs[VCPU_REGS_RBX3];
vcpu->vc_gueststate.vg_rcx = gprs[VCPU_REGS_RCX1];
vcpu->vc_gueststate.vg_rdx = gprs[VCPU_REGS_RDX2];
vcpu->vc_gueststate.vg_rsi = gprs[VCPU_REGS_RSI6];
vcpu->vc_gueststate.vg_rdi = gprs[VCPU_REGS_RDI7];
vcpu->vc_gueststate.vg_r8 = gprs[VCPU_REGS_R88];
vcpu->vc_gueststate.vg_r9 = gprs[VCPU_REGS_R99];
vcpu->vc_gueststate.vg_r10 = gprs[VCPU_REGS_R1010];
vcpu->vc_gueststate.vg_r11 = gprs[VCPU_REGS_R1111];
vcpu->vc_gueststate.vg_r12 = gprs[VCPU_REGS_R1212];
vcpu->vc_gueststate.vg_r13 = gprs[VCPU_REGS_R1313];
vcpu->vc_gueststate.vg_r14 = gprs[VCPU_REGS_R1414];
vcpu->vc_gueststate.vg_r15 = gprs[VCPU_REGS_R1515];
vcpu->vc_gueststate.vg_rbp = gprs[VCPU_REGS_RBP5];
vcpu->vc_gueststate.vg_rip = gprs[VCPU_REGS_RIP16];

vmcb->v_rax = gprs[VCPU_REGS_RAX0];
vmcb->v_rip = gprs[VCPU_REGS_RIP16];
vmcb->v_rsp = gprs[VCPU_REGS_RSP4];
vmcb->v_rflags = gprs[VCPU_REGS_RFLAGS17];
}

if (regmask & VM_RWREGS_SREGS0x2) {
vmcb->v_cs.vs_sel = sregs[VCPU_REGS_CS1].vsi_sel;
vmcb->v_cs.vs_lim = sregs[VCPU_REGS_CS1].vsi_limit;
attr = sregs[VCPU_REGS_CS1].vsi_ar;
vmcb->v_cs.vs_attr = (attr & 0xff) | ((attr >> 4) & 0xf00);
vmcb->v_cs.vs_base = sregs[VCPU_REGS_CS1].vsi_base;
vmcb->v_ds.vs_sel = sregs[VCPU_REGS_DS3].vsi_sel;
vmcb->v_ds.vs_lim = sregs[VCPU_REGS_DS3].vsi_limit;
attr = sregs[VCPU_REGS_DS3].vsi_ar;
vmcb->v_ds.vs_attr = (attr & 0xff) | ((attr >> 4) & 0xf00);
vmcb->v_ds.vs_base = sregs[VCPU_REGS_DS3].vsi_base;
vmcb->v_es.vs_sel = sregs[VCPU_REGS_ES0].vsi_sel;
vmcb->v_es.vs_lim = sregs[VCPU_REGS_ES0].vsi_limit;
attr = sregs[VCPU_REGS_ES0].vsi_ar;
vmcb->v_es.vs_attr = (attr & 0xff) | ((attr >> 4) & 0xf00);
vmcb->v_es.vs_base = sregs[VCPU_REGS_ES0].vsi_base;
vmcb->v_fs.vs_sel = sregs[VCPU_REGS_FS4].vsi_sel;
vmcb->v_fs.vs_lim = sregs[VCPU_REGS_FS4].vsi_limit;
attr = sregs[VCPU_REGS_FS4].vsi_ar;
vmcb->v_fs.vs_attr = (attr & 0xff) | ((attr >> 4) & 0xf00);
vmcb->v_fs.vs_base = sregs[VCPU_REGS_FS4].vsi_base;
vmcb->v_gs.vs_sel = sregs[VCPU_REGS_GS5].vsi_sel;
vmcb->v_gs.vs_lim = sregs[VCPU_REGS_GS5].vsi_limit;
attr = sregs[VCPU_REGS_GS5].vsi_ar;
vmcb->v_gs.vs_attr = (attr & 0xff) | ((attr >> 4) & 0xf00);
vmcb->v_gs.vs_base = sregs[VCPU_REGS_GS5].vsi_base;
vmcb->v_ss.vs_sel = sregs[VCPU_REGS_SS2].vsi_sel;
vmcb->v_ss.vs_lim = sregs[VCPU_REGS_SS2].vsi_limit;
attr = sregs[VCPU_REGS_SS2].vsi_ar;
vmcb->v_ss.vs_attr = (attr & 0xff) | ((attr >> 4) & 0xf00);
vmcb->v_ss.vs_base = sregs[VCPU_REGS_SS2].vsi_base;
vmcb->v_ldtr.vs_sel = sregs[VCPU_REGS_LDTR6].vsi_sel;
vmcb->v_ldtr.vs_lim = sregs[VCPU_REGS_LDTR6].vsi_limit;
attr = sregs[VCPU_REGS_LDTR6].vsi_ar;
vmcb->v_ldtr.vs_attr = (attr & 0xff) | ((attr >> 4) & 0xf00);
vmcb->v_ldtr.vs_base = sregs[VCPU_REGS_LDTR6].vsi_base;
vmcb->v_tr.vs_sel = sregs[VCPU_REGS_TR7].vsi_sel;
vmcb->v_tr.vs_lim = sregs[VCPU_REGS_TR7].vsi_limit;
attr = sregs[VCPU_REGS_TR7].vsi_ar;
vmcb->v_tr.vs_attr = (attr & 0xff) | ((attr >> 4) & 0xf00);
vmcb->v_tr.vs_base = sregs[VCPU_REGS_TR7].vsi_base;
vmcb->v_gdtr.vs_lim = vrs->vrs_gdtr.vsi_limit;
vmcb->v_gdtr.vs_base = vrs->vrs_gdtr.vsi_base;
vmcb->v_idtr.vs_lim = vrs->vrs_idtr.vsi_limit;
vmcb->v_idtr.vs_base = vrs->vrs_idtr.vsi_base;
}

if (regmask & VM_RWREGS_CRS0x4) {
vmcb->v_cr0 = crs[VCPU_REGS_CR00];
vmcb->v_cr3 = crs[VCPU_REGS_CR32];
vmcb->v_cr4 = crs[VCPU_REGS_CR43];
vcpu->vc_gueststate.vg_cr2 = crs[VCPU_REGS_CR21];
vcpu->vc_gueststate.vg_xcr0 = crs[VCPU_REGS_XCR05];
}

if (regmask & VM_RWREGS_MSRS0x8) {
vmcb->v_efer |= msrs[VCPU_REGS_EFER0];
vmcb->v_star = msrs[VCPU_REGS_STAR1];
vmcb->v_lstar = msrs[VCPU_REGS_LSTAR2];
vmcb->v_cstar = msrs[VCPU_REGS_CSTAR3];
vmcb->v_sfmask = msrs[VCPU_REGS_SFMASK4];
vmcb->v_kgsbase = msrs[VCPU_REGS_KGSBASE5];
}

if (regmask & VM_RWREGS_DRS0x10) {
vcpu->vc_gueststate.vg_dr0 = drs[VCPU_REGS_DR00];
vcpu->vc_gueststate.vg_dr1 = drs[VCPU_REGS_DR11];
vcpu->vc_gueststate.vg_dr2 = drs[VCPU_REGS_DR22];
vcpu->vc_gueststate.vg_dr3 = drs[VCPU_REGS_DR33];
vmcb->v_dr6 = drs[VCPU_REGS_DR64];
vmcb->v_dr7 = drs[VCPU_REGS_DR75];
}

return (0);
1943}

1945/*
* vcpu_reset_regs_svm
*
* Initializes 'vcpu's registers to supplied state
*
* Parameters:
*  vcpu: the vcpu whose register state is to be initialized
*  vrs: the register state to set
*
* Return values:
*  0: registers init'ed successfully
*  EINVAL: an error occurred setting register state
*/
1958int
1959vcpu_reset_regs_svm(struct vcpu *vcpu, struct vcpu_reg_state *vrs)
1960{
struct vmcb *vmcb;
int ret;
uint16_t asid;

vmcb = (struct vmcb *)vcpu->vc_control_va;

/*
* Intercept controls
*
* External Interrupt exiting (SVM_INTERCEPT_INTR)
* External NMI exiting (SVM_INTERCEPT_NMI)
* CPUID instruction (SVM_INTERCEPT_CPUID)
* HLT instruction (SVM_INTERCEPT_HLT)
* I/O instructions (SVM_INTERCEPT_INOUT)
* MSR access (SVM_INTERCEPT_MSR)
* shutdown events (SVM_INTERCEPT_SHUTDOWN)
*
* VMRUN instruction (SVM_INTERCEPT_VMRUN)
* VMMCALL instruction (SVM_INTERCEPT_VMMCALL)
* VMLOAD instruction (SVM_INTERCEPT_VMLOAD)
* VMSAVE instruction (SVM_INTERCEPT_VMSAVE)
* STGI instruction (SVM_INTERCEPT_STGI)
* CLGI instruction (SVM_INTERCEPT_CLGI)
* SKINIT instruction (SVM_INTERCEPT_SKINIT)
* ICEBP instruction (SVM_INTERCEPT_ICEBP)
* MWAIT instruction (SVM_INTERCEPT_MWAIT_UNCOND)
* MWAIT instruction (SVM_INTERCEPT_MWAIT_COND)
* MONITOR instruction (SVM_INTERCEPT_MONITOR)
* RDTSCP instruction (SVM_INTERCEPT_RDTSCP)
* INVLPGA instruction (SVM_INTERCEPT_INVLPGA)
* XSETBV instruction (SVM_INTERCEPT_XSETBV) (if available)
*/
vmcb->v_intercept1 = SVM_INTERCEPT_INTR(1UL << 0) | SVM_INTERCEPT_NMI(1UL << 1) |
   SVM_INTERCEPT_CPUID(1UL << 18) | SVM_INTERCEPT_HLT(1UL << 24) | SVM_INTERCEPT_INOUT(1UL << 27) |
   SVM_INTERCEPT_MSR(1UL << 28) | SVM_INTERCEPT_SHUTDOWN(1UL << 31);

vmcb->v_intercept2 = SVM_INTERCEPT_VMRUN(1UL << 0) | SVM_INTERCEPT_VMMCALL(1UL << 1) |
   SVM_INTERCEPT_VMLOAD(1UL << 2) | SVM_INTERCEPT_VMSAVE(1UL << 3) | SVM_INTERCEPT_STGI(1UL << 4) |
   SVM_INTERCEPT_CLGI(1UL << 5) | SVM_INTERCEPT_SKINIT(1UL << 6) | SVM_INTERCEPT_ICEBP(1UL << 8) |
   SVM_INTERCEPT_MWAIT_UNCOND(1UL << 11) | SVM_INTERCEPT_MONITOR(1UL << 10) |
   SVM_INTERCEPT_MWAIT_COND(1UL << 12) | SVM_INTERCEPT_RDTSCP(1UL << 7) |
   SVM_INTERCEPT_INVLPGA(1UL << 26);

if (xsave_mask)
vmcb->v_intercept2 |= SVM_INTERCEPT_XSETBV(1UL << 13);

/* Setup I/O bitmap */
memset((uint8_t *)vcpu->vc_svm_ioio_va, 0xFF, 3 * PAGE_SIZE)__builtin_memset(((uint8_t *)vcpu->vc_svm_ioio_va), (0xFF)
, (3 * (1 << 12)));
vmcb->v_iopm_pa = (uint64_t)(vcpu->vc_svm_ioio_pa);

/* Setup MSR bitmap */
memset((uint8_t *)vcpu->vc_msr_bitmap_va, 0xFF, 2 * PAGE_SIZE)__builtin_memset(((uint8_t *)vcpu->vc_msr_bitmap_va), (0xFF
), (2 * (1 << 12)));
vmcb->v_msrpm_pa = (uint64_t)(vcpu->vc_msr_bitmap_pa);
svm_setmsrbrw(vcpu, MSR_IA32_FEATURE_CONTROL0x03a);
svm_setmsrbrw(vcpu, MSR_SYSENTER_CS0x174);
svm_setmsrbrw(vcpu, MSR_SYSENTER_ESP0x175);
svm_setmsrbrw(vcpu, MSR_SYSENTER_EIP0x176);
svm_setmsrbrw(vcpu, MSR_STAR0xc0000081);
svm_setmsrbrw(vcpu, MSR_LSTAR0xc0000082);
svm_setmsrbrw(vcpu, MSR_CSTAR0xc0000083);
svm_setmsrbrw(vcpu, MSR_SFMASK0xc0000084);
svm_setmsrbrw(vcpu, MSR_FSBASE0xc0000100);
svm_setmsrbrw(vcpu, MSR_GSBASE0xc0000101);
svm_setmsrbrw(vcpu, MSR_KERNELGSBASE0xc0000102);

/* EFER is R/O so we can ensure the guest always has SVME */
svm_setmsrbr(vcpu, MSR_EFER0xc0000080);

/* allow reading TSC */
svm_setmsrbr(vcpu, MSR_TSC0x010);

/* allow reading HWCR and PSTATEDEF to determine TSC frequency */
svm_setmsrbr(vcpu, MSR_HWCR0xc0010015);
svm_setmsrbr(vcpu, MSR_PSTATEDEF(0)(0xc0010064 + (0)));

/* Guest VCPU ASID */
if (vmm_alloc_vpid(&asid)) {
DPRINTF("%s: could not allocate asid\n", __func__);
ret = EINVAL22;
goto exit;
}

vmcb->v_asid = asid;
vcpu->vc_vpid = asid;

/* TLB Control - First time in, flush all*/
vmcb->v_tlb_control = SVM_TLB_CONTROL_FLUSH_ALL1;

/* INTR masking */
vmcb->v_intr_masking = 1;

/* PAT */
vmcb->v_g_pat = PATENTRY(0, PAT_WB)(0x6UL << ((0) * 8)) | PATENTRY(1, PAT_WC)(0x1UL << ((1) * 8)) |
          PATENTRY(2, PAT_UCMINUS)(0x7UL << ((2) * 8)) | PATENTRY(3, PAT_UC)(0x0UL << ((3) * 8)) |
          PATENTRY(4, PAT_WB)(0x6UL << ((4) * 8)) | PATENTRY(5, PAT_WC)(0x1UL << ((5) * 8)) |
          PATENTRY(6, PAT_UCMINUS)(0x7UL << ((6) * 8)) | PATENTRY(7, PAT_UC)(0x0UL << ((7) * 8));

/* NPT */
if (vmm_softc->mode == VMM_MODE_RVI) {
vmcb->v_np_enable = 1;
vmcb->v_n_cr3 = vcpu->vc_parent->vm_map->pmap->pm_pdirpa;
}

/* Enable SVME in EFER (must always be set) */
vmcb->v_efer |= EFER_SVME0x00001000;

ret = vcpu_writeregs_svm(vcpu, VM_RWREGS_ALL(0x1 | 0x2 | 0x4 | 0x8 | 0x10), vrs);

/* xcr0 power on default sets bit 0 (x87 state) */
vcpu->vc_gueststate.vg_xcr0 = XFEATURE_X870x00000001 & xsave_mask;

vcpu->vc_parent->vm_map->pmap->eptp = 0;

2074exit:
return ret;
2076}

2078/*
* svm_setmsrbr
*
* Allow read access to the specified msr on the supplied vcpu.
*
* Parameters:
*  vcpu: the VCPU to allow access
*  msr: the MSR number to allow access to
*/
2087void
2088svm_setmsrbr(struct vcpu *vcpu, uint32_t msr)
2089{
uint8_t *msrs;
uint16_t idx;

msrs = (uint8_t *)vcpu->vc_msr_bitmap_va;

/*
* MSR Read bitmap layout:
* Pentium MSRs (0x0 - 0x1fff) @ 0x0
* Gen6 and Syscall MSRs (0xc0000000 - 0xc0001fff) @ 0x800
* Gen7 and Gen8 MSRs (0xc0010000 - 0xc0011fff) @ 0x1000
*
* Read enable bit is low order bit of 2-bit pair
* per MSR (eg, MSR 0x0 write bit is at bit 0 @ 0x0)
*/
if (msr <= 0x1fff) {
idx = SVM_MSRIDX(msr)((msr) / 4);
msrs[idx] &= ~(SVM_MSRBIT_R(msr)(1 << (((msr) % 4) * 2)));
} else if (msr >= 0xc0000000 && msr <= 0xc0001fff) {
idx = SVM_MSRIDX(msr - 0xc0000000)((msr - 0xc0000000) / 4) + 0x800;
msrs[idx] &= ~(SVM_MSRBIT_R(msr - 0xc0000000)(1 << (((msr - 0xc0000000) % 4) * 2)));
} else if (msr >= 0xc0010000 && msr <= 0xc0011fff) {
idx = SVM_MSRIDX(msr - 0xc0010000)((msr - 0xc0010000) / 4) + 0x1000;
msrs[idx] &= ~(SVM_MSRBIT_R(msr - 0xc0010000)(1 << (((msr - 0xc0010000) % 4) * 2)));
} else {
printf("%s: invalid msr 0x%x\n", __func__, msr);
return;
}
2117}

2119/*
* svm_setmsrbw
*
* Allow write access to the specified msr on the supplied vcpu
*
* Parameters:
*  vcpu: the VCPU to allow access
*  msr: the MSR number to allow access to
*/
2128void
2129svm_setmsrbw(struct vcpu *vcpu, uint32_t msr)
2130{
uint8_t *msrs;
uint16_t idx;

msrs = (uint8_t *)vcpu->vc_msr_bitmap_va;

/*
* MSR Write bitmap layout:
* Pentium MSRs (0x0 - 0x1fff) @ 0x0
* Gen6 and Syscall MSRs (0xc0000000 - 0xc0001fff) @ 0x800
* Gen7 and Gen8 MSRs (0xc0010000 - 0xc0011fff) @ 0x1000
*
* Write enable bit is high order bit of 2-bit pair
* per MSR (eg, MSR 0x0 write bit is at bit 1 @ 0x0)
*/
if (msr <= 0x1fff) {
idx = SVM_MSRIDX(msr)((msr) / 4);
msrs[idx] &= ~(SVM_MSRBIT_W(msr)(1 << (((msr) % 4) * 2 + 1)));
} else if (msr >= 0xc0000000 && msr <= 0xc0001fff) {
idx = SVM_MSRIDX(msr - 0xc0000000)((msr - 0xc0000000) / 4) + 0x800;
msrs[idx] &= ~(SVM_MSRBIT_W(msr - 0xc0000000)(1 << (((msr - 0xc0000000) % 4) * 2 + 1)));
} else if (msr >= 0xc0010000 && msr <= 0xc0011fff) {
idx = SVM_MSRIDX(msr - 0xc0010000)((msr - 0xc0010000) / 4) + 0x1000;
msrs[idx] &= ~(SVM_MSRBIT_W(msr - 0xc0010000)(1 << (((msr - 0xc0010000) % 4) * 2 + 1)));
} else {
printf("%s: invalid msr 0x%x\n", __func__, msr);
return;
}
2158}

2160/*
* svm_setmsrbrw
*
* Allow read/write access to the specified msr on the supplied vcpu
*
* Parameters:
*  vcpu: the VCPU to allow access
*  msr: the MSR number to allow access to
*/
2169void
2170svm_setmsrbrw(struct vcpu *vcpu, uint32_t msr)
2171{
svm_setmsrbr(vcpu, msr);
svm_setmsrbw(vcpu, msr);
2174}

2176/*
* vmx_setmsrbr
*
* Allow read access to the specified msr on the supplied vcpu.
*
* Parameters:
*  vcpu: the VCPU to allow access
*  msr: the MSR number to allow access to
*/
2185void
2186vmx_setmsrbr(struct vcpu *vcpu, uint32_t msr)
2187{
uint8_t *msrs;
uint16_t idx;

msrs = (uint8_t *)vcpu->vc_msr_bitmap_va;

/*
* MSR Read bitmap layout:
* "Low" MSRs (0x0 - 0x1fff) @ 0x0
* "High" MSRs (0xc0000000 - 0xc0001fff) @ 0x400
*/
if (msr <= 0x1fff) {
idx = VMX_MSRIDX(msr)((msr) / 8);
msrs[idx] &= ~(VMX_MSRBIT(msr)(1 << (msr) % 8));
} else if (msr >= 0xc0000000 && msr <= 0xc0001fff) {
idx = VMX_MSRIDX(msr - 0xc0000000)((msr - 0xc0000000) / 8) + 0x400;
msrs[idx] &= ~(VMX_MSRBIT(msr - 0xc0000000)(1 << (msr - 0xc0000000) % 8));
} else
printf("%s: invalid msr 0x%x\n", __func__, msr);
2206}

2208/*
* vmx_setmsrbw
*
* Allow write access to the specified msr on the supplied vcpu
*
* Parameters:
*  vcpu: the VCPU to allow access
*  msr: the MSR number to allow access to
*/
2217void
2218vmx_setmsrbw(struct vcpu *vcpu, uint32_t msr)
2219{
uint8_t *msrs;
uint16_t idx;

msrs = (uint8_t *)vcpu->vc_msr_bitmap_va;

/*
* MSR Write bitmap layout:
* "Low" MSRs (0x0 - 0x1fff) @ 0x800
* "High" MSRs (0xc0000000 - 0xc0001fff) @ 0xc00
*/
if (msr <= 0x1fff) {
idx = VMX_MSRIDX(msr)((msr) / 8) + 0x800;
msrs[idx] &= ~(VMX_MSRBIT(msr)(1 << (msr) % 8));
} else if (msr >= 0xc0000000 && msr <= 0xc0001fff) {
idx = VMX_MSRIDX(msr - 0xc0000000)((msr - 0xc0000000) / 8) + 0xc00;
msrs[idx] &= ~(VMX_MSRBIT(msr - 0xc0000000)(1 << (msr - 0xc0000000) % 8));
} else
printf("%s: invalid msr 0x%x\n", __func__, msr);
2238}

2240/*
* vmx_setmsrbrw
*
* Allow read/write access to the specified msr on the supplied vcpu
*
* Parameters:
*  vcpu: the VCPU to allow access
*  msr: the MSR number to allow access to
*/
2249void
2250vmx_setmsrbrw(struct vcpu *vcpu, uint32_t msr)
2251{
vmx_setmsrbr(vcpu, msr);
vmx_setmsrbw(vcpu, msr);
2254}

2256/*
* svm_set_clean
*
* Sets (mark as unmodified) the VMCB clean bit set in 'value'.
* For example, to set the clean bit for the VMCB intercepts (bit position 0),
* the caller provides 'SVM_CLEANBITS_I' (0x1) for the 'value' argument.
* Multiple cleanbits can be provided in 'value' at the same time (eg,
* "SVM_CLEANBITS_I | SVM_CLEANBITS_TPR").
*
* Note that this function does not clear any bits; to clear bits in the
* vmcb cleanbits bitfield, use 'svm_set_dirty'.
*
* Parameters:
*  vmcs: the VCPU whose VMCB clean value should be set
*  value: the value(s) to enable in the cleanbits mask
*/
2272void
2273svm_set_clean(struct vcpu *vcpu, uint32_t value)
2274{
struct vmcb *vmcb;

/* If no cleanbits support, do nothing */
if (!curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_svm.svm_vmcb_clean)
return;

vmcb = (struct vmcb *)vcpu->vc_control_va;

vmcb->v_vmcb_clean_bits |= value;
2284}

2286/*
* svm_set_dirty
*
* Clears (mark as modified) the VMCB clean bit set in 'value'.
* For example, to clear the bit for the VMCB intercepts (bit position 0)
* the caller provides 'SVM_CLEANBITS_I' (0x1) for the 'value' argument.
* Multiple dirty bits can be provided in 'value' at the same time (eg,
* "SVM_CLEANBITS_I | SVM_CLEANBITS_TPR").
*
* Parameters:
*  vmcs: the VCPU whose VMCB dirty value should be set
*  value: the value(s) to dirty in the cleanbits mask
*/
2299void
2300svm_set_dirty(struct vcpu *vcpu, uint32_t value)
2301{
struct vmcb *vmcb;

/* If no cleanbits support, do nothing */
if (!curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_svm.svm_vmcb_clean)
return;

vmcb = (struct vmcb *)vcpu->vc_control_va;

vmcb->v_vmcb_clean_bits &= ~value;
2311}

2313/*
* vcpu_reset_regs_vmx
*
* Initializes 'vcpu's registers to supplied state
*
* Parameters:
*  vcpu: the vcpu whose register state is to be initialized
*  vrs: the register state to set
*
* Return values:
*  0: registers init'ed successfully
*  EINVAL: an error occurred setting register state
*/
2326int
2327vcpu_reset_regs_vmx(struct vcpu *vcpu, struct vcpu_reg_state *vrs)
2328{
int ret = 0, ug = 0;
uint32_t cr0, cr4;
uint32_t pinbased, procbased, procbased2, exit, entry;
uint32_t want1, want0;
uint64_t ctrlval, cr3;
uint16_t ctrl, vpid;
struct vmx_msr_store *msr_store;

rw_assert_wrlock(&vcpu->vc_lock);

cr0 = vrs->vrs_crs[VCPU_REGS_CR00];

if (vcpu_reload_vmcs_vmx(vcpu)) {
DPRINTF("%s: error reloading VMCS\n", __func__);
ret = EINVAL22;
goto exit;
}

2347#ifdef VMM_DEBUG
/* VMCS should be loaded... */
paddr_t pa = 0ULL;
if (vmptrst(&pa))
panic("%s: vmptrst", __func__);
KASSERT(pa == vcpu->vc_control_pa)((pa == vcpu->vc_control_pa) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c", 2352, "pa == vcpu->vc_control_pa"
));
2353#endif /* VMM_DEBUG */

/* Compute Basic Entry / Exit Controls */
vcpu->vc_vmx_basic = rdmsr(IA32_VMX_BASIC0x480);
vcpu->vc_vmx_entry_ctls = rdmsr(IA32_VMX_ENTRY_CTLS0x484);
vcpu->vc_vmx_exit_ctls = rdmsr(IA32_VMX_EXIT_CTLS0x483);
vcpu->vc_vmx_pinbased_ctls = rdmsr(IA32_VMX_PINBASED_CTLS0x481);
vcpu->vc_vmx_procbased_ctls = rdmsr(IA32_VMX_PROCBASED_CTLS0x482);

/* Compute True Entry / Exit Controls (if applicable) */
if (vcpu->vc_vmx_basic & IA32_VMX_TRUE_CTLS_AVAIL(1ULL << 55)) {
vcpu->vc_vmx_true_entry_ctls = rdmsr(IA32_VMX_TRUE_ENTRY_CTLS0x490);
vcpu->vc_vmx_true_exit_ctls = rdmsr(IA32_VMX_TRUE_EXIT_CTLS0x48F);
vcpu->vc_vmx_true_pinbased_ctls =
    rdmsr(IA32_VMX_TRUE_PINBASED_CTLS0x48D);
vcpu->vc_vmx_true_procbased_ctls =
    rdmsr(IA32_VMX_TRUE_PROCBASED_CTLS0x48E);
}

/* Compute Secondary Procbased Controls (if applicable) */
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
   IA32_VMX_ACTIVATE_SECONDARY_CONTROLS(1ULL << 31), 1))
vcpu->vc_vmx_procbased2_ctls = rdmsr(IA32_VMX_PROCBASED2_CTLS0x48B);

/*
* Pinbased ctrls
*
* We must be able to set the following:
* IA32_VMX_EXTERNAL_INT_EXITING - exit on host interrupt
* IA32_VMX_NMI_EXITING - exit on host NMI
*/
want1 = IA32_VMX_EXTERNAL_INT_EXITING(1ULL << 0) |
   IA32_VMX_NMI_EXITING(1ULL << 3);
want0 = 0;

if (vcpu->vc_vmx_basic & IA32_VMX_TRUE_CTLS_AVAIL(1ULL << 55)) {
ctrl = IA32_VMX_TRUE_PINBASED_CTLS0x48D;
ctrlval = vcpu->vc_vmx_true_pinbased_ctls;
} else {
ctrl = IA32_VMX_PINBASED_CTLS0x481;
ctrlval = vcpu->vc_vmx_pinbased_ctls;
}

if (vcpu_vmx_compute_ctrl(ctrlval, ctrl, want1, want0, &pinbased)) {
DPRINTF("%s: error computing pinbased controls\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_PINBASED_CTLS0x4000, pinbased)) {
DPRINTF("%s: error setting pinbased controls\n", __func__);
ret = EINVAL22;
goto exit;
}

/*
* Procbased ctrls
*
* We must be able to set the following:
* IA32_VMX_HLT_EXITING - exit on HLT instruction
* IA32_VMX_MWAIT_EXITING - exit on MWAIT instruction
* IA32_VMX_UNCONDITIONAL_IO_EXITING - exit on I/O instructions
* IA32_VMX_USE_MSR_BITMAPS - exit on various MSR accesses
* IA32_VMX_CR8_LOAD_EXITING - guest TPR access
* IA32_VMX_CR8_STORE_EXITING - guest TPR access
* IA32_VMX_USE_TPR_SHADOW - guest TPR access (shadow)
* IA32_VMX_MONITOR_EXITING - exit on MONITOR instruction
*
* If we have EPT, we must be able to clear the following
* IA32_VMX_CR3_LOAD_EXITING - don't care about guest CR3 accesses
* IA32_VMX_CR3_STORE_EXITING - don't care about guest CR3 accesses
*/
want1 = IA32_VMX_HLT_EXITING(1ULL << 7) |
   IA32_VMX_MWAIT_EXITING(1ULL << 10) |
   IA32_VMX_UNCONDITIONAL_IO_EXITING(1ULL << 24) |
   IA32_VMX_USE_MSR_BITMAPS(1ULL << 28) |
   IA32_VMX_CR8_LOAD_EXITING(1ULL << 19) |
   IA32_VMX_CR8_STORE_EXITING(1ULL << 20) |
   IA32_VMX_MONITOR_EXITING(1ULL << 29) |
   IA32_VMX_USE_TPR_SHADOW(1ULL << 21);
want0 = 0;

if (vmm_softc->mode == VMM_MODE_EPT) {
want1 |= IA32_VMX_ACTIVATE_SECONDARY_CONTROLS(1ULL << 31);
want0 |= IA32_VMX_CR3_LOAD_EXITING(1ULL << 15) |
    IA32_VMX_CR3_STORE_EXITING(1ULL << 16);
}

if (vcpu->vc_vmx_basic & IA32_VMX_TRUE_CTLS_AVAIL(1ULL << 55)) {
ctrl = IA32_VMX_TRUE_PROCBASED_CTLS0x48E;
ctrlval = vcpu->vc_vmx_true_procbased_ctls;
} else {
ctrl = IA32_VMX_PROCBASED_CTLS0x482;
ctrlval = vcpu->vc_vmx_procbased_ctls;
}

if (vcpu_vmx_compute_ctrl(ctrlval, ctrl, want1, want0, &procbased)) {
DPRINTF("%s: error computing procbased controls\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_PROCBASED_CTLS0x4002, procbased)) {
DPRINTF("%s: error setting procbased controls\n", __func__);
ret = EINVAL22;
goto exit;
}

/*
* Secondary Procbased ctrls
*
* We want to be able to set the following, if available:
* IA32_VMX_ENABLE_VPID - use VPIDs where available
*
* If we have EPT, we must be able to set the following:
* IA32_VMX_ENABLE_EPT - enable EPT
*
* If we have unrestricted guest capability, we must be able to set
* the following:
* IA32_VMX_UNRESTRICTED_GUEST - enable unrestricted guest (if caller
*     specified CR0_PG | CR0_PE in %cr0 in the 'vrs' parameter)
*/
want1 = 0;

/* XXX checking for 2ndary controls can be combined here */
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
   IA32_VMX_ACTIVATE_SECONDARY_CONTROLS(1ULL << 31), 1)) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_VPID(1ULL << 5), 1)) {
	want1 |= IA32_VMX_ENABLE_VPID(1ULL << 5);
	vcpu->vc_vmx_vpid_enabled = 1;
}
}

if (vmm_softc->mode == VMM_MODE_EPT)
want1 |= IA32_VMX_ENABLE_EPT(1ULL << 1);

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
   IA32_VMX_ACTIVATE_SECONDARY_CONTROLS(1ULL << 31), 1)) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_UNRESTRICTED_GUEST(1ULL << 7), 1)) {
	if ((cr0 & (CR0_PE0x00000001 | CR0_PG0x80000000)) == 0) {
		want1 |= IA32_VMX_UNRESTRICTED_GUEST(1ULL << 7);
		ug = 1;
	}
}
}

want0 = ~want1;
ctrlval = vcpu->vc_vmx_procbased2_ctls;
ctrl = IA32_VMX_PROCBASED2_CTLS0x48B;

if (vcpu_vmx_compute_ctrl(ctrlval, ctrl, want1, want0, &procbased2)) {
DPRINTF("%s: error computing secondary procbased controls\n",
     __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_PROCBASED2_CTLS0x401E, procbased2)) {
DPRINTF("%s: error setting secondary procbased controls\n",
     __func__);
ret = EINVAL22;
goto exit;
}

/*
* Exit ctrls
*
* We must be able to set the following:
* IA32_VMX_SAVE_DEBUG_CONTROLS
* IA32_VMX_HOST_SPACE_ADDRESS_SIZE - exit to long mode
* IA32_VMX_ACKNOWLEDGE_INTERRUPT_ON_EXIT - ack interrupt on exit
*/
want1 = IA32_VMX_HOST_SPACE_ADDRESS_SIZE(1ULL << 9) |
   IA32_VMX_ACKNOWLEDGE_INTERRUPT_ON_EXIT(1ULL << 15) |
   IA32_VMX_SAVE_DEBUG_CONTROLS(1ULL << 2);
want0 = 0;

if (vcpu->vc_vmx_basic & IA32_VMX_TRUE_CTLS_AVAIL(1ULL << 55)) {
ctrl = IA32_VMX_TRUE_EXIT_CTLS0x48F;
ctrlval = vcpu->vc_vmx_true_exit_ctls;
} else {
ctrl = IA32_VMX_EXIT_CTLS0x483;
ctrlval = vcpu->vc_vmx_exit_ctls;
}

if (rcr4() & CR4_CET0x00800000)
want1 |= IA32_VMX_LOAD_HOST_CET_STATE(1ULL << 28);
else
want0 |= IA32_VMX_LOAD_HOST_CET_STATE(1ULL << 28);

if (vcpu_vmx_compute_ctrl(ctrlval, ctrl, want1, want0, &exit)) {
DPRINTF("%s: error computing exit controls\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_EXIT_CTLS0x400C, exit)) {
DPRINTF("%s: error setting exit controls\n", __func__);
ret = EINVAL22;
goto exit;
}

/*
* Entry ctrls
*
* We must be able to set the following:
* IA32_VMX_IA32E_MODE_GUEST (if no unrestricted guest)
* IA32_VMX_LOAD_DEBUG_CONTROLS
* We must be able to clear the following:
* IA32_VMX_ENTRY_TO_SMM - enter to SMM
* IA32_VMX_DEACTIVATE_DUAL_MONITOR_TREATMENT
* IA32_VMX_LOAD_IA32_PERF_GLOBAL_CTRL_ON_ENTRY
*/
want1 = IA32_VMX_LOAD_DEBUG_CONTROLS(1ULL << 2);
if (vrs->vrs_msrs[VCPU_REGS_EFER0] & EFER_LMA0x00000400)
want1 |= IA32_VMX_IA32E_MODE_GUEST(1ULL << 9);

want0 = IA32_VMX_ENTRY_TO_SMM(1ULL << 10) |
   IA32_VMX_DEACTIVATE_DUAL_MONITOR_TREATMENT(1ULL << 11) |
   IA32_VMX_LOAD_IA32_PERF_GLOBAL_CTRL_ON_ENTRY(1ULL << 13);

if (vcpu->vc_vmx_basic & IA32_VMX_TRUE_CTLS_AVAIL(1ULL << 55)) {
ctrl = IA32_VMX_TRUE_ENTRY_CTLS0x490;
ctrlval = vcpu->vc_vmx_true_entry_ctls;
} else {
ctrl = IA32_VMX_ENTRY_CTLS0x484;
ctrlval = vcpu->vc_vmx_entry_ctls;
}

if (rcr4() & CR4_CET0x00800000)
want1 |= IA32_VMX_LOAD_GUEST_CET_STATE(1ULL << 20);
else
want0 |= IA32_VMX_LOAD_GUEST_CET_STATE(1ULL << 20);

if (vcpu_vmx_compute_ctrl(ctrlval, ctrl, want1, want0, &entry)) {
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_ENTRY_CTLS0x4012, entry)) {
ret = EINVAL22;
goto exit;
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
   IA32_VMX_ACTIVATE_SECONDARY_CONTROLS(1ULL << 31), 1)) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_VPID(1ULL << 5), 1)) {

	/* We may sleep during allocation, so reload VMCS. */
	vcpu->vc_last_pcpu = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
	ret = vmm_alloc_vpid(&vpid);
	if (vcpu_reload_vmcs_vmx(vcpu)) {
		printf("%s: failed to reload vmcs\n", __func__);
		ret = EINVAL22;
		goto exit;
	}
	if (ret) {
		DPRINTF("%s: could not allocate VPID\n",
		    __func__);
		ret = EINVAL22;
		goto exit;
	}

	if (vmwrite(VMCS_GUEST_VPID0x0000, vpid)) {
		DPRINTF("%s: error setting guest VPID\n",
		    __func__);
		ret = EINVAL22;
		goto exit;
	}

	vcpu->vc_vpid = vpid;
}
}

/*
* Determine which bits in CR0 have to be set to a fixed
* value as per Intel SDM A.7.
* CR0 bits in the vrs parameter must match these.
*/
want1 = (curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr0_fixed0) &
   (curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr0_fixed1);
want0 = ~(curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr0_fixed0) &
   ~(curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr0_fixed1);

/*
* CR0_FIXED0 and CR0_FIXED1 may report the CR0_PG and CR0_PE bits as
* fixed to 1 even if the CPU supports the unrestricted guest
* feature. Update want1 and want0 accordingly to allow
* any value for CR0_PG and CR0_PE in vrs->vrs_crs[VCPU_REGS_CR0] if
* the CPU has the unrestricted guest capability.
*/
if (ug) {
want1 &= ~(CR0_PG0x80000000 | CR0_PE0x00000001);
want0 &= ~(CR0_PG0x80000000 | CR0_PE0x00000001);
}

/*
* VMX may require some bits to be set that userland should not have
* to care about. Set those here.
*/
if (want1 & CR0_NE0x00000020)
cr0 |= CR0_NE0x00000020;

if ((cr0 & want1) != want1) {
ret = EINVAL22;
goto exit;
}

if ((~cr0 & want0) != want0) {
ret = EINVAL22;
goto exit;
}

vcpu->vc_vmx_cr0_fixed1 = want1;
vcpu->vc_vmx_cr0_fixed0 = want0;
/*
* Determine which bits in CR4 have to be set to a fixed
* value as per Intel SDM A.8.
* CR4 bits in the vrs parameter must match these, except
* CR4_VMXE - we add that here since it must always be set.
*/
want1 = (curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed0) &
   (curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed1);
want0 = ~(curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed0) &
   ~(curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed1);

cr4 = vrs->vrs_crs[VCPU_REGS_CR43] | CR4_VMXE0x00002000;

if ((cr4 & want1) != want1) {
ret = EINVAL22;
goto exit;
}

if ((~cr4 & want0) != want0) {
ret = EINVAL22;
goto exit;
}

cr3 = vrs->vrs_crs[VCPU_REGS_CR32];

/* Restore PDPTEs if 32-bit PAE paging is being used */
if (cr3 && (cr4 & CR4_PAE0x00000020) &&
   !(vrs->vrs_msrs[VCPU_REGS_EFER0] & EFER_LMA0x00000400)) {
if (vmwrite(VMCS_GUEST_PDPTE00x280A,
    vrs->vrs_crs[VCPU_REGS_PDPTE06])) {
	ret = EINVAL22;
	goto exit;
}

if (vmwrite(VMCS_GUEST_PDPTE10x280C,
    vrs->vrs_crs[VCPU_REGS_PDPTE17])) {
	ret = EINVAL22;
	goto exit;
}

if (vmwrite(VMCS_GUEST_PDPTE20x280E,
    vrs->vrs_crs[VCPU_REGS_PDPTE28])) {
	ret = EINVAL22;
	goto exit;
}

if (vmwrite(VMCS_GUEST_PDPTE30x2810,
    vrs->vrs_crs[VCPU_REGS_PDPTE39])) {
	ret = EINVAL22;
	goto exit;
}
}

vrs->vrs_crs[VCPU_REGS_CR00] = cr0;
vrs->vrs_crs[VCPU_REGS_CR43] = cr4;

/*
* Select host MSRs to be loaded on exit
*/
msr_store = (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_load_va;
msr_store[0].vms_index = MSR_EFER0xc0000080;
msr_store[0].vms_data = rdmsr(MSR_EFER0xc0000080);
msr_store[1].vms_index = MSR_STAR0xc0000081;
msr_store[1].vms_data = rdmsr(MSR_STAR0xc0000081);
msr_store[2].vms_index = MSR_LSTAR0xc0000082;
msr_store[2].vms_data = rdmsr(MSR_LSTAR0xc0000082);
msr_store[3].vms_index = MSR_CSTAR0xc0000083;
msr_store[3].vms_data = rdmsr(MSR_CSTAR0xc0000083);
msr_store[4].vms_index = MSR_SFMASK0xc0000084;
msr_store[4].vms_data = rdmsr(MSR_SFMASK0xc0000084);
msr_store[5].vms_index = MSR_KERNELGSBASE0xc0000102;
msr_store[5].vms_data = rdmsr(MSR_KERNELGSBASE0xc0000102);
msr_store[6].vms_index = MSR_MISC_ENABLE0x1a0;
msr_store[6].vms_data = rdmsr(MSR_MISC_ENABLE0x1a0);

/*
* Select guest MSRs to be loaded on entry / saved on exit
*/
msr_store = (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_save_va;

msr_store[VCPU_REGS_EFER0].vms_index = MSR_EFER0xc0000080;
msr_store[VCPU_REGS_STAR1].vms_index = MSR_STAR0xc0000081;
msr_store[VCPU_REGS_LSTAR2].vms_index = MSR_LSTAR0xc0000082;
msr_store[VCPU_REGS_CSTAR3].vms_index = MSR_CSTAR0xc0000083;
msr_store[VCPU_REGS_SFMASK4].vms_index = MSR_SFMASK0xc0000084;
msr_store[VCPU_REGS_KGSBASE5].vms_index = MSR_KERNELGSBASE0xc0000102;
msr_store[VCPU_REGS_MISC_ENABLE6].vms_index = MSR_MISC_ENABLE0x1a0;

/*
* Initialize MSR_MISC_ENABLE as it can't be read and populated from vmd
* and some of the content is based on the host.
*/
msr_store[VCPU_REGS_MISC_ENABLE6].vms_data = rdmsr(MSR_MISC_ENABLE0x1a0);
msr_store[VCPU_REGS_MISC_ENABLE6].vms_data &=
   ~(MISC_ENABLE_TCC(1 << 3) | MISC_ENABLE_PERF_MON_AVAILABLE(1 << 7) |
     MISC_ENABLE_EIST_ENABLED(1 << 16) | MISC_ENABLE_ENABLE_MONITOR_FSM(1 << 18) |
     MISC_ENABLE_xTPR_MESSAGE_DISABLE(1 << 23));
msr_store[VCPU_REGS_MISC_ENABLE6].vms_data |=
     MISC_ENABLE_BTS_UNAVAILABLE(1 << 11) | MISC_ENABLE_PEBS_UNAVAILABLE(1 << 12);

/*
* Currently we have the same count of entry/exit MSRs loads/stores
* but this is not an architectural requirement.
*/
if (vmwrite(VMCS_EXIT_MSR_STORE_COUNT0x400E, VMX_NUM_MSR_STORE7)) {
DPRINTF("%s: error setting guest MSR exit store count\n",
    __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_EXIT_MSR_LOAD_COUNT0x4010, VMX_NUM_MSR_STORE7)) {
DPRINTF("%s: error setting guest MSR exit load count\n",
    __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_ENTRY_MSR_LOAD_COUNT0x4014, VMX_NUM_MSR_STORE7)) {
DPRINTF("%s: error setting guest MSR entry load count\n",
    __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_EXIT_STORE_MSR_ADDRESS0x2006,
   vcpu->vc_vmx_msr_exit_save_pa)) {
DPRINTF("%s: error setting guest MSR exit store address\n",
    __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_EXIT_LOAD_MSR_ADDRESS0x2008,
   vcpu->vc_vmx_msr_exit_load_pa)) {
DPRINTF("%s: error setting guest MSR exit load address\n",
    __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_ENTRY_LOAD_MSR_ADDRESS0x200A,
   vcpu->vc_vmx_msr_exit_save_pa)) {
DPRINTF("%s: error setting guest MSR entry load address\n",
    __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_MSR_BITMAP_ADDRESS0x2004,
   vcpu->vc_msr_bitmap_pa)) {
DPRINTF("%s: error setting guest MSR bitmap address\n",
    __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_CR4_MASK0x6002, CR4_VMXE0x00002000)) {
DPRINTF("%s: error setting guest CR4 mask\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_CR0_MASK0x6000, CR0_NE0x00000020)) {
DPRINTF("%s: error setting guest CR0 mask\n", __func__);
ret = EINVAL22;
goto exit;
}

/*
* Set up the VMCS for the register state we want during VCPU start.
* This matches what the CPU state would be after a bootloader
* transition to 'start'.
*/
ret = vcpu_writeregs_vmx(vcpu, VM_RWREGS_ALL(0x1 | 0x2 | 0x4 | 0x8 | 0x10), 0, vrs);

/*
* Set up the MSR bitmap
*/
memset((uint8_t *)vcpu->vc_msr_bitmap_va, 0xFF, PAGE_SIZE)__builtin_memset(((uint8_t *)vcpu->vc_msr_bitmap_va), (0xFF
), ((1 << 12)));
vmx_setmsrbrw(vcpu, MSR_IA32_FEATURE_CONTROL0x03a);
vmx_setmsrbrw(vcpu, MSR_SYSENTER_CS0x174);
vmx_setmsrbrw(vcpu, MSR_SYSENTER_ESP0x175);
vmx_setmsrbrw(vcpu, MSR_SYSENTER_EIP0x176);
vmx_setmsrbrw(vcpu, MSR_EFER0xc0000080);
vmx_setmsrbrw(vcpu, MSR_STAR0xc0000081);
vmx_setmsrbrw(vcpu, MSR_LSTAR0xc0000082);
vmx_setmsrbrw(vcpu, MSR_CSTAR0xc0000083);
vmx_setmsrbrw(vcpu, MSR_SFMASK0xc0000084);
vmx_setmsrbrw(vcpu, MSR_FSBASE0xc0000100);
vmx_setmsrbrw(vcpu, MSR_GSBASE0xc0000101);
vmx_setmsrbrw(vcpu, MSR_KERNELGSBASE0xc0000102);

vmx_setmsrbr(vcpu, MSR_MISC_ENABLE0x1a0);
vmx_setmsrbr(vcpu, MSR_TSC0x010);

/* If host supports CET, pass through access to the guest. */
if (rcr4() & CR4_CET0x00800000)
vmx_setmsrbrw(vcpu, MSR_S_CET0x6a2);

/* XXX CR0 shadow */
/* XXX CR4 shadow */

/* xcr0 power on default sets bit 0 (x87 state) */
vcpu->vc_gueststate.vg_xcr0 = XFEATURE_X870x00000001 & xsave_mask;

/* XXX PAT shadow */
vcpu->vc_shadow_pat = rdmsr(MSR_CR_PAT0x277);

/* Flush the VMCS */
if (vmclear(&vcpu->vc_control_pa)) {
DPRINTF("%s: vmclear failed\n", __func__);
ret = EINVAL22;
}
atomic_swap_uint(&vcpu->vc_vmx_vmcs_state, VMCS_CLEARED)_atomic_swap_uint((&vcpu->vc_vmx_vmcs_state), (0));

2887exit:
return (ret);
2889}

2891/*
* vcpu_init_vmx
*
* Intel VMX specific VCPU initialization routine.
*
* This function allocates various per-VCPU memory regions, sets up initial
* VCPU VMCS controls, and sets initial register values.
*
* Parameters:
*  vcpu: the VCPU structure being initialized
*
* Return values:
*  0: the VCPU was initialized successfully
*  ENOMEM: insufficient resources
*  EINVAL: an error occurred during VCPU initialization
*/
2907int
2908vcpu_init_vmx(struct vcpu *vcpu)
2909{
struct vmcs *vmcs;
uint64_t msr, eptp;
uint32_t cr0, cr4;
int ret = 0;

/* Allocate VMCS VA */
vcpu->vc_control_va = (vaddr_t)km_alloc(PAGE_SIZE(1 << 12), &kv_page, &kp_zero,
   &kd_waitok);
vcpu->vc_vmx_vmcs_state = VMCS_CLEARED0;

if (!vcpu->vc_control_va)
return (ENOMEM12);

/* Compute VMCS PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_control_va,
   (paddr_t *)&vcpu->vc_control_pa)) {
ret = ENOMEM12;
goto exit;
}

/* Allocate MSR bitmap VA */
vcpu->vc_msr_bitmap_va = (vaddr_t)km_alloc(PAGE_SIZE(1 << 12), &kv_page, &kp_zero,
   &kd_waitok);

if (!vcpu->vc_msr_bitmap_va) {
ret = ENOMEM12;
goto exit;
}

/* Compute MSR bitmap PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_msr_bitmap_va,
   (paddr_t *)&vcpu->vc_msr_bitmap_pa)) {
ret = ENOMEM12;
goto exit;
}

/* Allocate MSR exit load area VA */
vcpu->vc_vmx_msr_exit_load_va = (vaddr_t)km_alloc(PAGE_SIZE(1 << 12), &kv_page,
  &kp_zero, &kd_waitok);

if (!vcpu->vc_vmx_msr_exit_load_va) {
ret = ENOMEM12;
goto exit;
}

/* Compute MSR exit load area PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_vmx_msr_exit_load_va,
   &vcpu->vc_vmx_msr_exit_load_pa)) {
ret = ENOMEM12;
goto exit;
}

/* Allocate MSR exit save area VA */
vcpu->vc_vmx_msr_exit_save_va = (vaddr_t)km_alloc(PAGE_SIZE(1 << 12), &kv_page,
  &kp_zero, &kd_waitok);

if (!vcpu->vc_vmx_msr_exit_save_va) {
ret = ENOMEM12;
goto exit;
}

/* Compute MSR exit save area PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_vmx_msr_exit_save_va,
   &vcpu->vc_vmx_msr_exit_save_pa)) {
ret = ENOMEM12;
goto exit;
}

/* Allocate MSR entry load area VA */
vcpu->vc_vmx_msr_entry_load_va = (vaddr_t)km_alloc(PAGE_SIZE(1 << 12), &kv_page,
  &kp_zero, &kd_waitok);

if (!vcpu->vc_vmx_msr_entry_load_va) {
ret = ENOMEM12;
goto exit;
}

/* Compute MSR entry load area PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_vmx_msr_entry_load_va,
   &vcpu->vc_vmx_msr_entry_load_pa)) {
ret = ENOMEM12;
goto exit;
}

vmcs = (struct vmcs *)vcpu->vc_control_va;
vmcs->vmcs_revision = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_vmxon_revision;

/*
* Load the VMCS onto this PCPU so we can write registers
*/
if (vmptrld(&vcpu->vc_control_pa)) {
ret = EINVAL22;
goto exit;
}

/* Configure EPT Pointer */
eptp = vcpu->vc_parent->vm_map->pmap->pm_pdirpa;
msr = rdmsr(IA32_VMX_EPT_VPID_CAP0x48C);
if (msr & IA32_EPT_VPID_CAP_PAGE_WALK_4(1ULL << 6)) {
/* Page walk length 4 supported */
eptp |= ((IA32_EPT_PAGE_WALK_LENGTH0x4 - 1) << 3);
} else {
DPRINTF("EPT page walk length 4 not supported\n");
ret = EINVAL22;
goto exit;
}

if (msr & IA32_EPT_VPID_CAP_WB(1ULL << 14)) {
/* WB cache type supported */
eptp |= IA32_EPT_PAGING_CACHE_TYPE_WB0x6;
} else
DPRINTF("%s: no WB cache type available, guest VM will run "
    "uncached\n", __func__);

DPRINTF("Guest EPTP = 0x%llx\n", eptp);
if (vmwrite(VMCS_GUEST_IA32_EPTP0x201A, eptp)) {
DPRINTF("%s: error setting guest EPTP\n", __func__);
ret = EINVAL22;
goto exit;
}

vcpu->vc_parent->vm_map->pmap->eptp = eptp;

/* Host CR0 */
cr0 = rcr0() & ~CR0_TS0x00000008;
if (vmwrite(VMCS_HOST_IA32_CR00x6C00, cr0)) {
DPRINTF("%s: error writing host CR0\n", __func__);
ret = EINVAL22;
goto exit;
}

/* Host CR4 */
cr4 = rcr4();
if (vmwrite(VMCS_HOST_IA32_CR40x6C04, cr4)) {
DPRINTF("%s: error writing host CR4\n", __func__);
ret = EINVAL22;
goto exit;
}

/* Host Segment Selectors */
if (vmwrite(VMCS_HOST_IA32_CS_SEL0x0C02, GSEL(GCODE_SEL, SEL_KPL)(((1) << 3) | 0))) {
DPRINTF("%s: error writing host CS selector\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_HOST_IA32_DS_SEL0x0C06, GSEL(GDATA_SEL, SEL_KPL)(((2) << 3) | 0))) {
DPRINTF("%s: error writing host DS selector\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_HOST_IA32_ES_SEL0x0C00, GSEL(GDATA_SEL, SEL_KPL)(((2) << 3) | 0))) {
DPRINTF("%s: error writing host ES selector\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_HOST_IA32_FS_SEL0x0C08, GSEL(GDATA_SEL, SEL_KPL)(((2) << 3) | 0))) {
DPRINTF("%s: error writing host FS selector\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_HOST_IA32_GS_SEL0x0C0A, GSEL(GDATA_SEL, SEL_KPL)(((2) << 3) | 0))) {
DPRINTF("%s: error writing host GS selector\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_HOST_IA32_SS_SEL0x0C04, GSEL(GDATA_SEL, SEL_KPL)(((2) << 3) | 0))) {
DPRINTF("%s: error writing host SS selector\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_HOST_IA32_TR_SEL0x0C0C, GSYSSEL(GPROC0_SEL, SEL_KPL)((((0) << 4) + (6 << 3)) | 0))) {
DPRINTF("%s: error writing host TR selector\n", __func__);
ret = EINVAL22;
goto exit;
}

/* Host IDTR base */
if (vmwrite(VMCS_HOST_IA32_IDTR_BASE0x6C0E, idt_vaddr)) {
DPRINTF("%s: error writing host IDTR base\n", __func__);
ret = EINVAL22;
goto exit;
}

/* VMCS link */
if (vmwrite(VMCS_LINK_POINTER0x2800, VMX_VMCS_PA_CLEAR0xFFFFFFFFFFFFFFFFUL)) {
DPRINTF("%s: error writing VMCS link pointer\n", __func__);
ret = EINVAL22;
goto exit;
}

/* Flush the initial VMCS */
if (vmclear(&vcpu->vc_control_pa)) {
DPRINTF("%s: vmclear failed\n", __func__);
ret = EINVAL22;
}

3112exit:
if (ret)
vcpu_deinit_vmx(vcpu);

return (ret);
3117}

3119/*
* vcpu_reset_regs
*
* Resets a vcpu's registers to the provided state
*
* Parameters:
*  vcpu: the vcpu whose registers shall be reset
*  vrs: the desired register state
*
* Return values:
*  0: the vcpu's registers were successfully reset
*  !0: the vcpu's registers could not be reset (see arch-specific reset
*      function for various values that can be returned here)
*/
3133int
3134vcpu_reset_regs(struct vcpu *vcpu, struct vcpu_reg_state *vrs)
3135{
int ret;

if (vmm_softc->mode == VMM_MODE_EPT)
ret = vcpu_reset_regs_vmx(vcpu, vrs);
else if (vmm_softc->mode == VMM_MODE_RVI)
ret = vcpu_reset_regs_svm(vcpu, vrs);
else
panic("%s: unknown vmm mode: %d", __func__, vmm_softc->mode);

return (ret);
3146}

3148/*
* vcpu_init_svm
*
* AMD SVM specific VCPU initialization routine.
*
* This function allocates various per-VCPU memory regions, sets up initial
* VCPU VMCB controls, and sets initial register values.
*
* Parameters:
*  vcpu: the VCPU structure being initialized
*
* Return values:
*  0: the VCPU was initialized successfully
*  ENOMEM: insufficient resources
*  EINVAL: an error occurred during VCPU initialization
*/
3164int
3165vcpu_init_svm(struct vcpu *vcpu)
3166{
int ret = 0;

/* Allocate VMCB VA */
vcpu->vc_control_va = (vaddr_t)km_alloc(PAGE_SIZE(1 << 12), &kv_page, &kp_zero,
   &kd_waitok);

if (!vcpu->vc_control_va)
return (ENOMEM12);

/* Compute VMCB PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_control_va,
   (paddr_t *)&vcpu->vc_control_pa)) {
ret = ENOMEM12;
goto exit;
}

DPRINTF("%s: VMCB va @ 0x%llx, pa @ 0x%llx\n", __func__,
   (uint64_t)vcpu->vc_control_va,
   (uint64_t)vcpu->vc_control_pa);


/* Allocate MSR bitmap VA (2 pages) */
vcpu->vc_msr_bitmap_va = (vaddr_t)km_alloc(2 * PAGE_SIZE(1 << 12), &kv_any,
   &vmm_kp_contig, &kd_waitok);

if (!vcpu->vc_msr_bitmap_va) {
ret = ENOMEM12;
goto exit;
}

/* Compute MSR bitmap PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_msr_bitmap_va,
   (paddr_t *)&vcpu->vc_msr_bitmap_pa)) {
ret = ENOMEM12;
goto exit;
}

DPRINTF("%s: MSR bitmap va @ 0x%llx, pa @ 0x%llx\n", __func__,
   (uint64_t)vcpu->vc_msr_bitmap_va,
   (uint64_t)vcpu->vc_msr_bitmap_pa);

/* Allocate host state area VA */
vcpu->vc_svm_hsa_va = (vaddr_t)km_alloc(PAGE_SIZE(1 << 12), &kv_page,
  &kp_zero, &kd_waitok);

if (!vcpu->vc_svm_hsa_va) {
ret = ENOMEM12;
goto exit;
}

/* Compute host state area PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_svm_hsa_va,
   &vcpu->vc_svm_hsa_pa)) {
ret = ENOMEM12;
goto exit;
}

DPRINTF("%s: HSA va @ 0x%llx, pa @ 0x%llx\n", __func__,
   (uint64_t)vcpu->vc_svm_hsa_va,
   (uint64_t)vcpu->vc_svm_hsa_pa);

/* Allocate IOIO area VA (3 pages) */
vcpu->vc_svm_ioio_va = (vaddr_t)km_alloc(3 * PAGE_SIZE(1 << 12), &kv_any,
  &vmm_kp_contig, &kd_waitok);

if (!vcpu->vc_svm_ioio_va) {
ret = ENOMEM12;
goto exit;
}

/* Compute IOIO area PA */
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), vcpu->vc_svm_ioio_va,
   &vcpu->vc_svm_ioio_pa)) {
ret = ENOMEM12;
goto exit;
}

DPRINTF("%s: IOIO va @ 0x%llx, pa @ 0x%llx\n", __func__,
   (uint64_t)vcpu->vc_svm_ioio_va,
   (uint64_t)vcpu->vc_svm_ioio_pa);

3248exit:
if (ret)
vcpu_deinit_svm(vcpu);

return (ret);
3253}

3255/*
* vcpu_init
*
* Calls the architecture-specific VCPU init routine
*/
3260int
3261vcpu_init(struct vcpu *vcpu)
3262{
int ret = 0;

vcpu->vc_virt_mode = vmm_softc->mode;
vcpu->vc_state = VCPU_STATE_STOPPED;
vcpu->vc_vpid = 0;
vcpu->vc_pvclock_system_gpa = 0;
vcpu->vc_last_pcpu = NULL((void *)0);

rw_init(&vcpu->vc_lock, "vcpu")_rw_init_flags(&vcpu->vc_lock, "vcpu", 0, ((void *)0));

/* Shadow PAT MSR, starting with host's value. */
vcpu->vc_shadow_pat = rdmsr(MSR_CR_PAT0x277);

if (vmm_softc->mode == VMM_MODE_EPT)
ret = vcpu_init_vmx(vcpu);
else if (vmm_softc->mode == VMM_MODE_RVI)
ret = vcpu_init_svm(vcpu);
else
panic("%s: unknown vmm mode: %d", __func__, vmm_softc->mode);

return (ret);
3284}

3286/*
* vcpu_deinit_vmx
*
* Deinitializes the vcpu described by 'vcpu'
*
* Parameters:
*  vcpu: the vcpu to be deinited
*/
3294void
3295vcpu_deinit_vmx(struct vcpu *vcpu)
3296{
if (vcpu->vc_control_va) {
km_free((void *)vcpu->vc_control_va, PAGE_SIZE(1 << 12),
    &kv_page, &kp_zero);
vcpu->vc_control_va = 0;
}
if (vcpu->vc_vmx_msr_exit_save_va) {
km_free((void *)vcpu->vc_vmx_msr_exit_save_va,
    PAGE_SIZE(1 << 12), &kv_page, &kp_zero);
vcpu->vc_vmx_msr_exit_save_va = 0;
}
if (vcpu->vc_vmx_msr_exit_load_va) {
km_free((void *)vcpu->vc_vmx_msr_exit_load_va,
    PAGE_SIZE(1 << 12), &kv_page, &kp_zero);
vcpu->vc_vmx_msr_exit_load_va = 0;
}
if (vcpu->vc_vmx_msr_entry_load_va) {
km_free((void *)vcpu->vc_vmx_msr_entry_load_va,
    PAGE_SIZE(1 << 12), &kv_page, &kp_zero);
vcpu->vc_vmx_msr_entry_load_va = 0;
}

if (vcpu->vc_vmx_vpid_enabled)
vmm_free_vpid(vcpu->vc_vpid);
3320}

3322/*
* vcpu_deinit_svm
*
* Deinitializes the vcpu described by 'vcpu'
*
* Parameters:
*  vcpu: the vcpu to be deinited
*/
3330void
3331vcpu_deinit_svm(struct vcpu *vcpu)
3332{
if (vcpu->vc_control_va) {
km_free((void *)vcpu->vc_control_va, PAGE_SIZE(1 << 12), &kv_page,
    &kp_zero);
vcpu->vc_control_va = 0;
}
if (vcpu->vc_msr_bitmap_va) {
km_free((void *)vcpu->vc_msr_bitmap_va, 2 * PAGE_SIZE(1 << 12), &kv_any,
    &vmm_kp_contig);
vcpu->vc_msr_bitmap_va = 0;
}
if (vcpu->vc_svm_hsa_va) {
km_free((void *)vcpu->vc_svm_hsa_va, PAGE_SIZE(1 << 12), &kv_page,
    &kp_zero);
vcpu->vc_svm_hsa_va = 0;
}
if (vcpu->vc_svm_ioio_va) {
km_free((void *)vcpu->vc_svm_ioio_va, 3 * PAGE_SIZE(1 << 12), &kv_any,
    &vmm_kp_contig);
vcpu->vc_svm_ioio_va = 0;
}

vmm_free_vpid(vcpu->vc_vpid);
3355}

3357/*
* vcpu_deinit
*
* Calls the architecture-specific VCPU deinit routine
*
* Parameters:
*  vcpu: the vcpu to be deinited
*/
3365void
3366vcpu_deinit(struct vcpu *vcpu)
3367{
if (vmm_softc->mode == VMM_MODE_EPT)
vcpu_deinit_vmx(vcpu);
else if	(vmm_softc->mode == VMM_MODE_RVI)
vcpu_deinit_svm(vcpu);
else
panic("%s: unknown vmm mode: %d", __func__, vmm_softc->mode);
3374}

3376/*
* vcpu_vmx_check_cap
*
* Checks if the 'cap' bit in the 'msr' MSR can be set or cleared (set = 1
* or set = 0, respectively).
*
* When considering 'msr', we check to see if true controls are available,
* and use those if so.
*
* Returns 1 of 'cap' can be set/cleared as requested, 0 otherwise.
*/
3387int
3388vcpu_vmx_check_cap(struct vcpu *vcpu, uint32_t msr, uint32_t cap, int set)
3389{
uint64_t ctl;

if (vcpu->vc_vmx_basic & IA32_VMX_TRUE_CTLS_AVAIL(1ULL << 55)) {
switch (msr) {
case IA32_VMX_PINBASED_CTLS0x481:
	ctl = vcpu->vc_vmx_true_pinbased_ctls;
	break;
case IA32_VMX_PROCBASED_CTLS0x482:
	ctl = vcpu->vc_vmx_true_procbased_ctls;
	break;
case IA32_VMX_PROCBASED2_CTLS0x48B:
	ctl = vcpu->vc_vmx_procbased2_ctls;
	break;
case IA32_VMX_ENTRY_CTLS0x484:
	ctl = vcpu->vc_vmx_true_entry_ctls;
	break;
case IA32_VMX_EXIT_CTLS0x483:
	ctl = vcpu->vc_vmx_true_exit_ctls;
	break;
default:
	return (0);
}
} else {
switch (msr) {
case IA32_VMX_PINBASED_CTLS0x481:
	ctl = vcpu->vc_vmx_pinbased_ctls;
	break;
case IA32_VMX_PROCBASED_CTLS0x482:
	ctl = vcpu->vc_vmx_procbased_ctls;
	break;
case IA32_VMX_PROCBASED2_CTLS0x48B:
	ctl = vcpu->vc_vmx_procbased2_ctls;
	break;
case IA32_VMX_ENTRY_CTLS0x484:
	ctl = vcpu->vc_vmx_entry_ctls;
	break;
case IA32_VMX_EXIT_CTLS0x483:
	ctl = vcpu->vc_vmx_exit_ctls;
	break;
default:
	return (0);
}
}

if (set) {
/* Check bit 'cap << 32', must be !0 */
return (ctl & ((uint64_t)cap << 32)) != 0;
} else {
/* Check bit 'cap', must be 0 */
return (ctl & cap) == 0;
}
3441}

3443/*
* vcpu_vmx_compute_ctrl
*
* Computes the appropriate control value, given the supplied parameters
* and CPU capabilities.
*
* Intel has made somewhat of a mess of this computation - it is described
* using no fewer than three different approaches, spread across many
* pages of the SDM. Further compounding the problem is the fact that now
* we have "true controls" for each type of "control", and each needs to
* be examined to get the calculation right, but only if "true" controls
* are present on the CPU we're on.
*
* Parameters:
*  ctrlval: the control value, as read from the CPU MSR
*  ctrl: which control is being set (eg, pinbased, procbased, etc)
*  want0: the set of desired 0 bits
*  want1: the set of desired 1 bits
*  out: (out) the correct value to write into the VMCS for this VCPU,
*      for the 'ctrl' desired.
*
* Returns 0 if successful, or EINVAL if the supplied parameters define
*     an unworkable control setup.
*/
3467int
3468vcpu_vmx_compute_ctrl(uint64_t ctrlval, uint16_t ctrl, uint32_t want1,
uint32_t want0, uint32_t *out)
3470{
int i, set, clear;

*out = 0;

/*
* The Intel SDM gives three formulae for determining which bits to
* set/clear for a given control and desired functionality. Formula
* 1 is the simplest but disallows use of newer features that are
* enabled by functionality in later CPUs.
*
* Formulas 2 and 3 allow such extra functionality. We use formula
* 2 - this requires us to know the identity of controls in the
* "default1" class for each control register, but allows us to not
* have to pass along and/or query both sets of capability MSRs for
* each control lookup. This makes the code slightly longer,
* however.
*/
for (i = 0; i < 32; i++) {
/* Figure out if we can set and / or clear this bit */
set = (ctrlval & (1ULL << (i + 32))) != 0;
clear = ((1ULL << i) & ((uint64_t)ctrlval)) == 0;

/* If the bit can't be set nor cleared, something's wrong */
if (!set && !clear)
	return (EINVAL22);

/*
 * Formula 2.c.i - "If the relevant VMX capability MSR
 * reports that a control has a single setting, use that
 * setting."
 */
if (set && !clear) {
	if (want0 & (1ULL << i))
		return (EINVAL22);
	else
		*out |= (1ULL << i);
} else if (clear && !set) {
	if (want1 & (1ULL << i))
		return (EINVAL22);
	else
		*out &= ~(1ULL << i);
} else {
	/*
	 * 2.c.ii - "If the relevant VMX capability MSR
	 * reports that a control can be set to 0 or 1
	 * and that control's meaning is known to the VMM,
	 * set the control based on the functionality desired."
	 */
	if (want1 & (1ULL << i))
		*out |= (1ULL << i);
	else if (want0 & (1 << i))
		*out &= ~(1ULL << i);
	else {
		/*
		 * ... assuming the control's meaning is not
		 * known to the VMM ...
		 *
		 * 2.c.iii - "If the relevant VMX capability
		 * MSR reports that a control can be set to 0
	 	 * or 1 and the control is not in the default1
		 * class, set the control to 0."
		 *
		 * 2.c.iv - "If the relevant VMX capability
		 * MSR reports that a control can be set to 0
		 * or 1 and the control is in the default1
		 * class, set the control to 1."
		 */
		switch (ctrl) {
		case IA32_VMX_PINBASED_CTLS0x481:
		case IA32_VMX_TRUE_PINBASED_CTLS0x48D:
			/*
			 * A.3.1 - default1 class of pinbased
			 * controls comprises bits 1,2,4
			 */
			switch (i) {
				case 1:
				case 2:
				case 4:
					*out |= (1ULL << i);
					break;
				default:
					*out &= ~(1ULL << i);
					break;
			}
			break;
		case IA32_VMX_PROCBASED_CTLS0x482:
		case IA32_VMX_TRUE_PROCBASED_CTLS0x48E:
			/*
			 * A.3.2 - default1 class of procbased
			 * controls comprises bits 1, 4-6, 8,
			 * 13-16, 26
			 */
			switch (i) {
				case 1:
				case 4 ... 6:
				case 8:
				case 13 ... 16:
				case 26:
					*out |= (1ULL << i);
					break;
				default:
					*out &= ~(1ULL << i);
					break;
			}
			break;
			/*
			 * Unknown secondary procbased controls
			 * can always be set to 0
			 */
		case IA32_VMX_PROCBASED2_CTLS0x48B:
			*out &= ~(1ULL << i);
			break;
		case IA32_VMX_EXIT_CTLS0x483:
		case IA32_VMX_TRUE_EXIT_CTLS0x48F:
			/*
			 * A.4 - default1 class of exit
			 * controls comprises bits 0-8, 10,
			 * 11, 13, 14, 16, 17
			 */
			switch (i) {
				case 0 ... 8:
				case 10 ... 11:
				case 13 ... 14:
				case 16 ... 17:
					*out |= (1ULL << i);
					break;
				default:
					*out &= ~(1ULL << i);
					break;
			}
			break;
		case IA32_VMX_ENTRY_CTLS0x484:
		case IA32_VMX_TRUE_ENTRY_CTLS0x490:
			/*
			 * A.5 - default1 class of entry
			 * controls comprises bits 0-8, 12
			 */
			switch (i) {
				case 0 ... 8:
				case 12:
					*out |= (1ULL << i);
					break;
				default:
					*out &= ~(1ULL << i);
					break;
			}
			break;
		}
	}
}
}

return (0);
3624}

3626/*
* vm_run
*
* Run the vm / vcpu specified by 'vrp'
*
* Parameters:
*  vrp: structure defining the VM to run
*
* Return value:
*  ENOENT: the VM defined in 'vrp' could not be located
*  EBUSY: the VM defined in 'vrp' is already running
*  EFAULT: error copying data from userspace (vmd) on return from previous
*      exit.
*  EAGAIN: help is needed from vmd(8) (device I/O or exit vmm(4) cannot
*      handle in-kernel.)
*  0: the run loop exited and no help is needed from vmd(8)
*/
3643int
3644vm_run(struct vm_run_params *vrp)
3645{
struct vm *vm;
struct vcpu *vcpu;
int ret = 0;
u_int old, next;

/*
* Find desired VM
*/
ret = vm_find(vrp->vrp_vm_id, &vm);
if (ret)
1
Assuming 'ret' is 0→
2
←
Taking false branch→
return (ret);

vcpu = vm_find_vcpu(vm, vrp->vrp_vcpu_id);
if (vcpu == NULL((void *)0)) {
3
←
Assuming 'vcpu' is not equal to NULL→
4
←
Taking false branch→
ret = ENOENT2;
goto out;
}

/*
* Attempt to transition from VCPU_STATE_STOPPED -> VCPU_STATE_RUNNING.
* Failure to make the transition indicates the VCPU is busy.
*/
rw_enter_write(&vcpu->vc_lock);
old = VCPU_STATE_STOPPED;
next = VCPU_STATE_RUNNING;
if (atomic_cas_uint(&vcpu->vc_state, old, next)_atomic_cas_uint((&vcpu->vc_state), (old), (next)) != old) {
5
←
Assuming the condition is false→
6
←
Taking false branch→
ret = EBUSY16;
goto out_unlock;
}

/*
* We may be returning from userland helping us from the last exit.
* If so (vrp_continue == 1), copy in the exit data from vmd. The
* exit data will be consumed before the next entry (this typically
* comprises VCPU register changes as the result of vmd(8)'s actions).
*/
if (vrp->vrp_continue) {
7
←
Assuming field 'vrp_continue' is 0→
8
←
Taking false branch→
if (copyin(vrp->vrp_exit, &vcpu->vc_exit,
    sizeof(struct vm_exit)) == EFAULT14) {
	ret = EFAULT14;
	goto out_unlock;
}
}

WRITE_ONCE(vcpu->vc_curcpu, curcpu())({ typeof(vcpu->vc_curcpu) __tmp = (({struct cpu_info *__ci
; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})); *(volatile typeof(vcpu
->vc_curcpu) *)&(vcpu->vc_curcpu) = __tmp; __tmp; }
);
/* Run the VCPU specified in vrp */
if (vcpu->vc_virt_mode == VMM_MODE_EPT) {
9
←
Assuming field 'vc_virt_mode' is not equal to VMM_MODE_EPT→
10
←
Taking false branch→
ret = vcpu_run_vmx(vcpu, vrp);
} else if (vcpu->vc_virt_mode == VMM_MODE_RVI) {
11
←
Assuming field 'vc_virt_mode' is equal to VMM_MODE_RVI→
12
←
Taking true branch→
ret = vcpu_run_svm(vcpu, vrp);
13
←
Calling 'vcpu_run_svm'→
}
WRITE_ONCE(vcpu->vc_curcpu, NULL)({ typeof(vcpu->vc_curcpu) __tmp = (((void *)0)); *(volatile
 typeof(vcpu->vc_curcpu) *)&(vcpu->vc_curcpu) = __tmp
; __tmp; });

if (ret == 0 || ret == EAGAIN35) {
/* If we are exiting, populate exit data so vmd can help. */
vrp->vrp_exit_reason = (ret == 0) ? VM_EXIT_NONE0xFFFF
    : vcpu->vc_gueststate.vg_exit_reason;
vrp->vrp_irqready = vcpu->vc_irqready;
vcpu->vc_state = VCPU_STATE_STOPPED;

if (copyout(&vcpu->vc_exit, vrp->vrp_exit,
    sizeof(struct vm_exit)) == EFAULT14) {
	ret = EFAULT14;
} else
	ret = 0;
} else {
vrp->vrp_exit_reason = VM_EXIT_TERMINATED0xFFFE;
vcpu->vc_state = VCPU_STATE_TERMINATED;
}
3715out_unlock:
rw_exit_write(&vcpu->vc_lock);
3717out:
refcnt_rele_wake(&vm->vm_refcnt);
return (ret);
3720}

3722/*
* vmm_fpurestore
*
* Restore the guest's FPU state, saving the existing userland thread's
* FPU context if necessary.  Must be called with interrupts disabled.
*/
3728int
3729vmm_fpurestore(struct vcpu *vcpu)
3730{
struct cpu_info *ci = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});

rw_assert_wrlock(&vcpu->vc_lock);

/* save vmm's FPU state if we haven't already */
if (ci->ci_pflags & CPUPF_USERXSTATE0x02) {
30
←
Assuming the condition is false→
31
←
Taking false branch→
ci->ci_pflags &= ~CPUPF_USERXSTATE0x02;
fpusavereset(&curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_addr->u_pcb.pcb_savefpu);
}

if (vcpu->vc_fpuinited)
32
←
Assuming field 'vc_fpuinited' is 0→
33
←
Taking false branch→
xrstor_kern(&vcpu->vc_g_fpu, xsave_mask);

if (xsave_mask) {
34
←
Assuming 'xsave_mask' is 0→
35
←
Taking false branch→
/* Restore guest %xcr0 */
if (xsetbv_user(0, vcpu->vc_gueststate.vg_xcr0)) {
	DPRINTF("%s: guest attempted to set invalid bits in "
	    "xcr0 (guest %%xcr0=0x%llx, host %%xcr0=0x%llx)\n",
	    __func__, vcpu->vc_gueststate.vg_xcr0, xsave_mask);
	return EINVAL22;
}
}

return 0;
3755}

3757/*
* vmm_fpusave
*
* Save the guest's FPU state.  Must be called with interrupts disabled.
*/
3762void
3763vmm_fpusave(struct vcpu *vcpu)
3764{
rw_assert_wrlock(&vcpu->vc_lock);

if (xsave_mask) {
/* Save guest %xcr0 */
vcpu->vc_gueststate.vg_xcr0 = xgetbv(0);

/* Restore host %xcr0 */
xsetbv(0, xsave_mask & XFEATURE_XCR0_MASK(0x00000001 | 0x00000002 | 0x00000004 | (0x00000008 | 0x00000010
) | (0x00000020 | 0x00000040 | 0x00000080) | 0x00000200 | (0x00040000
 | 0x00040000)));
}

/*
* Save full copy of FPU state - guest content is always
* a subset of host's save area (see xsetbv exit handler)
*/
fpusavereset(&vcpu->vc_g_fpu);
vcpu->vc_fpuinited = 1;
3781}

3783/*
* vmm_translate_gva
*
* Translates a guest virtual address to a guest physical address by walking
* the currently active page table (if needed).
*
* Note - this function can possibly alter the supplied VCPU state.
*  Specifically, it may inject exceptions depending on the current VCPU
*  configuration, and may alter %cr2 on #PF. Consequently, this function
*  should only be used as part of instruction emulation.
*
* Parameters:
*  vcpu: The VCPU this translation should be performed for (guest MMU settings
*   are gathered from this VCPU)
*  va: virtual address to translate
*  pa: pointer to paddr_t variable that will receive the translated physical
*   address. 'pa' is unchanged on error.
*  mode: one of PROT_READ, PROT_WRITE, PROT_EXEC indicating the mode in which
*   the address should be translated
*
* Return values:
*  0: the address was successfully translated - 'pa' contains the physical
*     address currently mapped by 'va'.
*  EFAULT: the PTE for 'VA' is unmapped. A #PF will be injected in this case
*     and %cr2 set in the vcpu structure.
*  EINVAL: an error occurred reading paging table structures
*/
3810int
3811vmm_translate_gva(struct vcpu *vcpu, uint64_t va, uint64_t *pa, int mode)
3812{
int level, shift, pdidx;
uint64_t pte, pt_paddr, pte_paddr, mask, low_mask, high_mask;
uint64_t shift_width, pte_size, *hva;
paddr_t hpa;
struct vcpu_reg_state vrs;

level = 0;

if (vmm_softc->mode == VMM_MODE_EPT) {
if (vcpu_readregs_vmx(vcpu, VM_RWREGS_ALL(0x1 | 0x2 | 0x4 | 0x8 | 0x10), 1, &vrs))
	return (EINVAL22);
} else if (vmm_softc->mode == VMM_MODE_RVI) {
if (vcpu_readregs_svm(vcpu, VM_RWREGS_ALL(0x1 | 0x2 | 0x4 | 0x8 | 0x10), &vrs))
	return (EINVAL22);
} else {
printf("%s: unknown vmm mode", __func__);
return (EINVAL22);
}

DPRINTF("%s: guest %%cr0=0x%llx, %%cr3=0x%llx\n", __func__,
   vrs.vrs_crs[VCPU_REGS_CR0], vrs.vrs_crs[VCPU_REGS_CR3]);

if (!(vrs.vrs_crs[VCPU_REGS_CR00] & CR0_PG0x80000000)) {
DPRINTF("%s: unpaged, va=pa=0x%llx\n", __func__,
    va);
*pa = va;
return (0);
}

pt_paddr = vrs.vrs_crs[VCPU_REGS_CR32];

if (vrs.vrs_crs[VCPU_REGS_CR00] & CR0_PE0x00000001) {
if (vrs.vrs_crs[VCPU_REGS_CR43] & CR4_PAE0x00000020) {
	pte_size = sizeof(uint64_t);
	shift_width = 9;

	if (vrs.vrs_msrs[VCPU_REGS_EFER0] & EFER_LMA0x00000400) {
		level = 4;
		mask = L4_MASK0x0000ff8000000000UL;
		shift = L4_SHIFT39;
	} else {
		level = 3;
		mask = L3_MASK0x0000007fc0000000UL;
		shift = L3_SHIFT30;
	}
} else {
	level = 2;
	shift_width = 10;
	mask = 0xFFC00000;
	shift = 22;
	pte_size = sizeof(uint32_t);
}
} else {
return (EINVAL22);
}

DPRINTF("%s: pte size=%lld level=%d mask=0x%llx, shift=%d, "
   "shift_width=%lld\n", __func__, pte_size, level, mask, shift,
   shift_width);

/* XXX: Check for R bit in segment selector and set A bit */

for (;level > 0; level--) {
pdidx = (va & mask) >> shift;
pte_paddr = (pt_paddr) + (pdidx * pte_size);

DPRINTF("%s: read pte level %d @ GPA 0x%llx\n", __func__,
    level, pte_paddr);
if (!pmap_extract(vcpu->vc_parent->vm_map->pmap, pte_paddr,
    &hpa)) {
	DPRINTF("%s: cannot extract HPA for GPA 0x%llx\n",
	    __func__, pte_paddr);
	return (EINVAL22);
}

hpa = hpa | (pte_paddr & 0xFFF);
hva = (uint64_t *)PMAP_DIRECT_MAP(hpa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (hpa));
DPRINTF("%s: GPA 0x%llx -> HPA 0x%llx -> HVA 0x%llx\n",
    __func__, pte_paddr, (uint64_t)hpa, (uint64_t)hva);
if (pte_size == 8)
	pte = *hva;
else
	pte = *(uint32_t *)hva;

DPRINTF("%s: PTE @ 0x%llx = 0x%llx\n", __func__, pte_paddr,
    pte);

/* XXX: Set CR2  */
if (!(pte & PG_V0x0000000000000001UL))
	return (EFAULT14);

/* XXX: Check for SMAP */
if ((mode == PROT_WRITE0x02) && !(pte & PG_RW0x0000000000000002UL))
	return (EPERM1);

if ((vcpu->vc_exit.cpl > 0) && !(pte & PG_u0x0000000000000004UL))
	return (EPERM1);

pte = pte | PG_U0x0000000000000020UL;
if (mode == PROT_WRITE0x02)
	pte = pte | PG_M0x0000000000000040UL;
*hva = pte;

/* XXX: EINVAL if in 32bit and PG_PS is 1 but CR4.PSE is 0 */
if (pte & PG_PS0x0000000000000080UL)
	break;

if (level > 1) {
	pt_paddr = pte & PG_FRAME0x000ffffffffff000UL;
	shift -= shift_width;
	mask = mask >> shift_width;
}
}

low_mask = ((uint64_t)1ULL << shift) - 1;
high_mask = (((uint64_t)1ULL << ((pte_size * 8) - 1)) - 1) ^ low_mask;
*pa = (pte & high_mask) | (va & low_mask);

DPRINTF("%s: final GPA for GVA 0x%llx = 0x%llx\n", __func__,
   va, *pa);

return (0);
3935}


3938/*
* vcpu_run_vmx
*
* VMX main loop used to run a VCPU.
*
* Parameters:
*  vcpu: The VCPU to run
*  vrp: run parameters
*
* Return values:
*  0: The run loop exited and no help is needed from vmd
*  EAGAIN: The run loop exited and help from vmd is needed
*  EINVAL: an error occurred
*/
3952int
3953vcpu_run_vmx(struct vcpu *vcpu, struct vm_run_params *vrp)
3954{
int ret = 0, exitinfo;
struct region_descriptor gdt;
struct cpu_info *ci = NULL((void *)0);
uint64_t exit_reason, cr3, insn_error;
struct schedstate_percpu *spc;
struct vmx_invvpid_descriptor vid;
uint64_t eii, procbased, int_st;
uint16_t irq, ldt_sel;
u_long s;
struct region_descriptor idtr;

rw_assert_wrlock(&vcpu->vc_lock);

if (vcpu_reload_vmcs_vmx(vcpu)) {
printf("%s: failed (re)loading vmcs\n", __func__);
return (EINVAL22);
}

/*
* If we are returning from userspace (vmd) because we exited
* last time, fix up any needed vcpu state first. Which state
* needs to be fixed up depends on what vmd populated in the
* exit data structure.
*/
irq = vrp->vrp_irq;

if (vrp->vrp_intr_pending)
vcpu->vc_intr = 1;
else
vcpu->vc_intr = 0;

if (vrp->vrp_continue) {
switch (vcpu->vc_gueststate.vg_exit_reason) {
case VMX_EXIT_IO30:
	if (vcpu->vc_exit.vei.vei_dir == VEI_DIR_IN)
		vcpu->vc_gueststate.vg_rax =
		    vcpu->vc_exit.vei.vei_data;
	vcpu->vc_gueststate.vg_rip =
	    vcpu->vc_exit.vrs.vrs_gprs[VCPU_REGS_RIP16];
	if (vmwrite(VMCS_GUEST_IA32_RIP0x681E,
	    vcpu->vc_gueststate.vg_rip)) {
		printf("%s: failed to update rip\n", __func__);
		return (EINVAL22);
	}
	break;
case VMX_EXIT_EPT_VIOLATION48:
	ret = vcpu_writeregs_vmx(vcpu, VM_RWREGS_GPRS0x1, 0,
	    &vcpu->vc_exit.vrs);
	if (ret) {
		printf("%s: vm %d vcpu %d failed to update "
		    "registers\n", __func__,
		    vcpu->vc_parent->vm_id, vcpu->vc_id);
		return (EINVAL22);
	}
	break;
case VM_EXIT_NONE0xFFFF:
case VMX_EXIT_HLT12:
case VMX_EXIT_INT_WINDOW7:
case VMX_EXIT_EXTINT1:
case VMX_EXIT_CPUID10:
case VMX_EXIT_XSETBV55:
	break;
4017#ifdef VMM_DEBUG
case VMX_EXIT_TRIPLE_FAULT2:
	DPRINTF("%s: vm %d vcpu %d triple fault\n",
	    __func__, vcpu->vc_parent->vm_id,
	    vcpu->vc_id);
	vmx_vcpu_dump_regs(vcpu);
	dump_vcpu(vcpu);
	vmx_dump_vmcs(vcpu);
	break;
case VMX_EXIT_ENTRY_FAILED_GUEST_STATE33:
	DPRINTF("%s: vm %d vcpu %d failed entry "
	    "due to invalid guest state\n",
	    __func__, vcpu->vc_parent->vm_id,
	    vcpu->vc_id);
	vmx_vcpu_dump_regs(vcpu);
	dump_vcpu(vcpu);
	return (EINVAL22);
default:
	DPRINTF("%s: unimplemented exit type %d (%s)\n",
	    __func__,
	    vcpu->vc_gueststate.vg_exit_reason,
	    vmx_exit_reason_decode(
		vcpu->vc_gueststate.vg_exit_reason));
	vmx_vcpu_dump_regs(vcpu);
	dump_vcpu(vcpu);
	break;
4043#endif /* VMM_DEBUG */
}
memset(&vcpu->vc_exit, 0, sizeof(vcpu->vc_exit))__builtin_memset((&vcpu->vc_exit), (0), (sizeof(vcpu->
vc_exit)));
}

/* Host CR3 */
cr3 = rcr3();
if (vmwrite(VMCS_HOST_IA32_CR30x6C02, cr3)) {
printf("%s: vmwrite(0x%04X, 0x%llx)\n", __func__,
    VMCS_HOST_IA32_CR30x6C02, cr3);
return (EINVAL22);
}

/* Handle vmd(8) injected interrupts */
/* Is there an interrupt pending injection? */
if (irq != 0xFFFF) {
if (vmread(VMCS_GUEST_INTERRUPTIBILITY_ST0x4824, &int_st)) {
	printf("%s: can't get interruptibility state\n",
	    __func__);
	return (EINVAL22);
}

/* Interruptibility state 0x3 covers NMIs and STI */
if (!(int_st & 0x3) && vcpu->vc_irqready) {
	eii = (irq & 0xFF);
	eii |= (1ULL << 31);	/* Valid */
	eii |= (0ULL << 8);	/* Hardware Interrupt */
	if (vmwrite(VMCS_ENTRY_INTERRUPTION_INFO0x4016, eii)) {
		printf("vcpu_run_vmx: can't vector "
		    "interrupt to guest\n");
		return (EINVAL22);
	}

	irq = 0xFFFF;
}
} else if (!vcpu->vc_intr) {
/*
 * Disable window exiting
 */
if (vmread(VMCS_PROCBASED_CTLS0x4002, &procbased)) {
	printf("%s: can't read procbased ctls on exit\n",
	    __func__);
	return (EINVAL22);
} else {
	procbased &= ~IA32_VMX_INTERRUPT_WINDOW_EXITING(1ULL << 2);
	if (vmwrite(VMCS_PROCBASED_CTLS0x4002, procbased)) {
		printf("%s: can't write procbased ctls "
		    "on exit\n", __func__);
		return (EINVAL22);
	}
}
}

while (ret == 0) {
4097#ifdef VMM_DEBUG
paddr_t pa = 0ULL;
vmptrst(&pa);
KASSERT(pa == vcpu->vc_control_pa)((pa == vcpu->vc_control_pa) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c", 4100, "pa == vcpu->vc_control_pa"
));
4101#endif /* VMM_DEBUG */

vmm_update_pvclock(vcpu);

if (ci != curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})) {
	ci = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
	vcpu->vc_last_pcpu = ci;

	setregion(&gdt, ci->ci_gdt, GDT_SIZE((6 << 3) + (1 << 4)) - 1);
	if (gdt.rd_base == 0) {
		printf("%s: setregion\n", __func__);
		return (EINVAL22);
	}

	/* Host GDTR base */
	if (vmwrite(VMCS_HOST_IA32_GDTR_BASE0x6C0C, gdt.rd_base)) {
		printf("%s: vmwrite(0x%04X, 0x%llx)\n",
		    __func__, VMCS_HOST_IA32_GDTR_BASE0x6C0C,
		    gdt.rd_base);
		return (EINVAL22);
	}

	/* Host TR base */
	if (vmwrite(VMCS_HOST_IA32_TR_BASE0x6C0A,
	    (uint64_t)ci->ci_tss)) {
		printf("%s: vmwrite(0x%04X, 0x%llx)\n",
		    __func__, VMCS_HOST_IA32_TR_BASE0x6C0A,
		    (uint64_t)ci->ci_tss);
		return (EINVAL22);
	}
}

/* Inject event if present */
if (vcpu->vc_event != 0) {
	eii = (vcpu->vc_event & 0xFF);
	eii |= (1ULL << 31);	/* Valid */

	/* Set the "Send error code" flag for certain vectors */
	switch (vcpu->vc_event & 0xFF) {
		case VMM_EX_DF8:
		case VMM_EX_TS10:
		case VMM_EX_NP11:
		case VMM_EX_SS12:
		case VMM_EX_GP13:
		case VMM_EX_PF14:
		case VMM_EX_AC17:
			eii |= (1ULL << 11);
	}

	eii |= (3ULL << 8);	/* Hardware Exception */
	if (vmwrite(VMCS_ENTRY_INTERRUPTION_INFO0x4016, eii)) {
		printf("%s: can't vector event to guest\n",
		    __func__);
		ret = EINVAL22;
		break;
	}

	if (vmwrite(VMCS_ENTRY_EXCEPTION_ERROR_CODE0x4018, 0)) {
		printf("%s: can't write error code to guest\n",
		    __func__);
		ret = EINVAL22;
		break;
	}

	vcpu->vc_event = 0;
}

if (vcpu->vc_vmx_vpid_enabled) {
	/* Invalidate old TLB mappings */
	vid.vid_vpid = vcpu->vc_vpid;
	vid.vid_addr = 0;
	invvpid(IA32_VMX_INVVPID_SINGLE_CTX_GLB0x3, &vid);
}

/* Start / resume the VCPU */

/* Disable interrupts and save the current host FPU state. */
s = intr_disable();
if ((ret = vmm_fpurestore(vcpu))) {
	intr_restore(s);
	break;
}

sidt(&idtr);
sldt(&ldt_sel);

TRACEPOINT(vmm, guest_enter, vcpu, vrp)do { extern struct dt_probe (dt_static_vmm_guest_enter); struct
 dt_probe *dtp = &(dt_static_vmm_guest_enter); if (__builtin_expect
(((dt_tracing) != 0), 0) && __builtin_expect(((dtp->
dtp_recording) != 0), 0)) { struct dt_provider *dtpv = dtp->
dtp_prov; dtpv->dtpv_enter(dtpv, dtp, vcpu, vrp); } } while
 (0);

/* Restore any guest PKRU state. */
if (vmm_softc->sc_md.pkru_enabled)
	wrpkru(vcpu->vc_pkru);

ret = vmx_enter_guest(&vcpu->vc_control_pa,
    &vcpu->vc_gueststate,
    (vcpu->vc_vmx_vmcs_state == VMCS_LAUNCHED1),
    ci->ci_vmm_cap.vcc_vmx.vmx_has_l1_flush_msr);

/* Restore host PKRU state. */
if (vmm_softc->sc_md.pkru_enabled) {
	vcpu->vc_pkru = rdpkru(0);
	wrpkru(PGK_VALUE0xfffffffc);
}

lidt(&idtr);
lldt(ldt_sel);

/*
 * On exit, interrupts are disabled, and we are running with
 * the guest FPU state still possibly on the CPU. Save the FPU
 * state before re-enabling interrupts.
 */
vmm_fpusave(vcpu);
intr_restore(s);

atomic_swap_uint(&vcpu->vc_vmx_vmcs_state, VMCS_LAUNCHED)_atomic_swap_uint((&vcpu->vc_vmx_vmcs_state), (1));
exit_reason = VM_EXIT_NONE0xFFFF;

/* If we exited successfully ... */
if (ret == 0) {
	exitinfo = vmx_get_exit_info(
	    &vcpu->vc_gueststate.vg_rip, &exit_reason);
	if (!(exitinfo & VMX_EXIT_INFO_HAVE_RIP0x1)) {
		printf("%s: cannot read guest rip\n", __func__);
		ret = EINVAL22;
		break;
	}
	if (!(exitinfo & VMX_EXIT_INFO_HAVE_REASON0x2)) {
		printf("%s: cant read exit reason\n", __func__);
		ret = EINVAL22;
		break;
	}
	vcpu->vc_gueststate.vg_exit_reason = exit_reason;
	TRACEPOINT(vmm, guest_exit, vcpu, vrp, exit_reason)do { extern struct dt_probe (dt_static_vmm_guest_exit); struct
 dt_probe *dtp = &(dt_static_vmm_guest_exit); if (__builtin_expect
(((dt_tracing) != 0), 0) && __builtin_expect(((dtp->
dtp_recording) != 0), 0)) { struct dt_provider *dtpv = dtp->
dtp_prov; dtpv->dtpv_enter(dtpv, dtp, vcpu, vrp, exit_reason
); } } while (0);

	/* Update our state */
	if (vmread(VMCS_GUEST_IA32_RFLAGS0x6820,
	    &vcpu->vc_gueststate.vg_rflags)) {
		printf("%s: can't read guest rflags during "
		    "exit\n", __func__);
		ret = EINVAL22;
		break;
                      }

	/*
	 * Handle the exit. This will alter "ret" to EAGAIN if
	 * the exit handler determines help from vmd is needed.
	 */
	ret = vmx_handle_exit(vcpu);

	if (vcpu->vc_gueststate.vg_rflags & PSL_I0x00000200)
		vcpu->vc_irqready = 1;
	else
		vcpu->vc_irqready = 0;

	/*
	 * If not ready for interrupts, but interrupts pending,
	 * enable interrupt window exiting.
	 */
	if (vcpu->vc_irqready == 0 && vcpu->vc_intr) {
		if (vmread(VMCS_PROCBASED_CTLS0x4002, &procbased)) {
			printf("%s: can't read procbased ctls "
			    "on intwin exit\n", __func__);
			ret = EINVAL22;
			break;
		}

		procbased |= IA32_VMX_INTERRUPT_WINDOW_EXITING(1ULL << 2);
		if (vmwrite(VMCS_PROCBASED_CTLS0x4002, procbased)) {
			printf("%s: can't write procbased ctls "
			    "on intwin exit\n", __func__);
			ret = EINVAL22;
			break;
		}
	}

	/*
	 * Exit to vmd if we are terminating, failed to enter,
	 * or need help (device I/O)
	 */
	if (ret || vcpu_must_stop(vcpu))
		break;

	if (vcpu->vc_intr && vcpu->vc_irqready) {
		ret = EAGAIN35;
		break;
	}

	/* Check if we should yield - don't hog the {p,v}pu */
	spc = &ci->ci_schedstate;
	if (spc->spc_schedflags & SPCF_SHOULDYIELD0x0002)
		break;

} else {
	/*
	 * We failed vmresume or vmlaunch for some reason,
	 * typically due to invalid vmcs state or other
	 * reasons documented in SDM Vol 3C 30.4.
	 */
	switch (ret) {
	case VMX_FAIL_LAUNCH_INVALID_VMCS2:
		printf("%s: failed %s with invalid vmcs\n",
		    __func__,
		    (vcpu->vc_vmx_vmcs_state == VMCS_LAUNCHED1
			? "vmresume" : "vmlaunch"));
		break;
	case VMX_FAIL_LAUNCH_VALID_VMCS3:
		printf("%s: failed %s with valid vmcs\n",
		    __func__,
		    (vcpu->vc_vmx_vmcs_state == VMCS_LAUNCHED1
			? "vmresume" : "vmlaunch"));
		break;
	default:
		printf("%s: failed %s for unknown reason\n",
		    __func__,
		    (vcpu->vc_vmx_vmcs_state == VMCS_LAUNCHED1
			? "vmresume" : "vmlaunch"));
	}

	ret = EINVAL22;

	/* Try to translate a vmfail error code, if possible. */
	if (vmread(VMCS_INSTRUCTION_ERROR0x4400, &insn_error)) {
		printf("%s: can't read insn error field\n",
		    __func__);
	} else
		printf("%s: error code = %lld, %s\n", __func__,
		    insn_error,
		    vmx_instruction_error_decode(insn_error));
4329#ifdef VMM_DEBUG
	vmx_vcpu_dump_regs(vcpu);
	dump_vcpu(vcpu);
4332#endif /* VMM_DEBUG */
}
}

vcpu->vc_last_pcpu = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});

/* Copy the VCPU register state to the exit structure */
if (vcpu_readregs_vmx(vcpu, VM_RWREGS_ALL(0x1 | 0x2 | 0x4 | 0x8 | 0x10), 0, &vcpu->vc_exit.vrs))
ret = EINVAL22;
vcpu->vc_exit.cpl = vmm_get_guest_cpu_cpl(vcpu);

return (ret);
4344}

4346/*
* vmx_handle_intr
*
* Handle host (external) interrupts. We read which interrupt fired by
* extracting the vector from the VMCS and dispatch the interrupt directly
* to the host using vmm_dispatch_intr.
*/
4353void
4354vmx_handle_intr(struct vcpu *vcpu)
4355{
uint8_t vec;
uint64_t eii;
struct gate_descriptor *idte;
vaddr_t handler;

if (vmread(VMCS_EXIT_INTERRUPTION_INFO0x4404, &eii)) {
printf("%s: can't obtain intr info\n", __func__);
return;
}

vec = eii & 0xFF;

/* XXX check "error valid" code in eii, abort if 0 */
idte=&idt[vec];
handler = idte->gd_looffset + ((uint64_t)idte->gd_hioffset << 16);
vmm_dispatch_intr(handler);
4372}

4374/*
* svm_handle_hlt
*
* Handle HLT exits
*
* Parameters
*  vcpu: The VCPU that executed the HLT instruction
*
* Return Values:
*  EIO: The guest halted with interrupts disabled
*  EAGAIN: Normal return to vmd - vmd should halt scheduling this VCPU
*   until a virtual interrupt is ready to inject
*/
4387int
4388svm_handle_hlt(struct vcpu *vcpu)
4389{
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;
uint64_t rflags = vmcb->v_rflags;

/* All HLT insns are 1 byte */
vcpu->vc_gueststate.vg_rip += 1;

if (!(rflags & PSL_I0x00000200)) {
DPRINTF("%s: guest halted with interrupts disabled\n",
    __func__);
return (EIO5);
}

return (EAGAIN35);
4403}

4405/*
* vmx_handle_hlt
*
* Handle HLT exits. HLTing the CPU with interrupts disabled will terminate
* the guest (no NMIs handled) by returning EIO to vmd.
*
* Parameters:
*  vcpu: The VCPU that executed the HLT instruction
*
* Return Values:
*  EINVAL: An error occurred extracting information from the VMCS, or an
*   invalid HLT instruction was encountered
*  EIO: The guest halted with interrupts disabled
*  EAGAIN: Normal return to vmd - vmd should halt scheduling this VCPU
*   until a virtual interrupt is ready to inject
*
*/
4422int
4423vmx_handle_hlt(struct vcpu *vcpu)
4424{
uint64_t insn_length, rflags;

if (vmread(VMCS_INSTRUCTION_LENGTH0x440C, &insn_length)) {
printf("%s: can't obtain instruction length\n", __func__);
return (EINVAL22);
}

if (vmread(VMCS_GUEST_IA32_RFLAGS0x6820, &rflags)) {
printf("%s: can't obtain guest rflags\n", __func__);
return (EINVAL22);
}

if (insn_length != 1) {
DPRINTF("%s: HLT with instruction length %lld not supported\n",
    __func__, insn_length);
return (EINVAL22);
}

if (!(rflags & PSL_I0x00000200)) {
DPRINTF("%s: guest halted with interrupts disabled\n",
    __func__);
return (EIO5);
}

vcpu->vc_gueststate.vg_rip += insn_length;
return (EAGAIN35);
4451}

4453/*
* vmx_get_exit_info
*
* Returns exit information containing the current guest RIP and exit reason
* in rip and exit_reason. The return value is a bitmask indicating whether
* reading the RIP and exit reason was successful.
*/
4460int
4461vmx_get_exit_info(uint64_t *rip, uint64_t *exit_reason)
4462{
int rv = 0;

if (vmread(VMCS_GUEST_IA32_RIP0x681E, rip) == 0) {
rv |= VMX_EXIT_INFO_HAVE_RIP0x1;
if (vmread(VMCS_EXIT_REASON0x4402, exit_reason) == 0)
	rv |= VMX_EXIT_INFO_HAVE_REASON0x2;
}
return (rv);
4471}

4473/*
* svm_handle_exit
*
* Handle exits from the VM by decoding the exit reason and calling various
* subhandlers as needed.
*/
4479int
4480svm_handle_exit(struct vcpu *vcpu)
4481{
uint64_t exit_reason, rflags;
int update_rip, ret = 0;
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;

update_rip = 0;
exit_reason = vcpu->vc_gueststate.vg_exit_reason;
rflags = vcpu->vc_gueststate.vg_rflags;

switch (exit_reason) {
50
←
Control jumps to 'case 139:'  at line 4540→
case SVM_VMEXIT_VINTR0x64:
if (!(rflags & PSL_I0x00000200)) {
	DPRINTF("%s: impossible interrupt window exit "
	    "config\n", __func__);
	ret = EINVAL22;
	break;
}

/*
 * Guest is now ready for interrupts, so disable interrupt
 * window exiting.
 */
vmcb->v_irq = 0;
vmcb->v_intr_vector = 0;
vmcb->v_intercept1 &= ~SVM_INTERCEPT_VINTR(1UL << 4);
svm_set_dirty(vcpu, SVM_CLEANBITS_TPR(1 << 3) | SVM_CLEANBITS_I(1 << 0));

update_rip = 0;
break;
case SVM_VMEXIT_INTR0x60:
update_rip = 0;
break;
case SVM_VMEXIT_SHUTDOWN0x7F:
update_rip = 0;
ret = EAGAIN35;
break;
case SVM_VMEXIT_NPF0x400:
ret = svm_handle_np_fault(vcpu);
break;
case SVM_VMEXIT_CPUID0x72:
ret = vmm_handle_cpuid(vcpu);
update_rip = 1;
break;
case SVM_VMEXIT_MSR0x7C:
ret = svm_handle_msr(vcpu);
update_rip = 1;
break;
case SVM_VMEXIT_XSETBV0x8D:
ret = svm_handle_xsetbv(vcpu);
update_rip = 1;
break;
case SVM_VMEXIT_IOIO0x7B:
if (svm_handle_inout(vcpu) == 0)
	ret = EAGAIN35;
break;
case SVM_VMEXIT_HLT0x78:
ret = svm_handle_hlt(vcpu);
update_rip = 1;
break;
case SVM_VMEXIT_MWAIT0x8B:
case SVM_VMEXIT_MWAIT_CONDITIONAL0x8C:
case SVM_VMEXIT_MONITOR0x8A:
case SVM_VMEXIT_VMRUN0x80:
case SVM_VMEXIT_VMMCALL0x81:
case SVM_VMEXIT_VMLOAD0x82:
case SVM_VMEXIT_VMSAVE0x83:
case SVM_VMEXIT_STGI0x84:
case SVM_VMEXIT_CLGI0x85:
case SVM_VMEXIT_SKINIT0x86:
case SVM_VMEXIT_RDTSCP0x87:
case SVM_VMEXIT_ICEBP0x88:
case SVM_VMEXIT_INVLPGA0x7A:
ret = vmm_inject_ud(vcpu);
51
←
Calling 'vmm_inject_ud'→
53
←
Returning from 'vmm_inject_ud'→
update_rip = 0;
break;
default:
DPRINTF("%s: unhandled exit 0x%llx (pa=0x%llx)\n", __func__,
    exit_reason, (uint64_t)vcpu->vc_control_pa);
return (EINVAL22);
}

if (update_rip54.1
'update_rip' is 0
) {
54
←
 Execution continues on line 4562→
55
←
Taking false branch→
vmcb->v_rip = vcpu->vc_gueststate.vg_rip;

if (rflags & PSL_T0x00000100) {
	if (vmm_inject_db(vcpu)) {
		printf("%s: can't inject #DB exception to "
		    "guest", __func__);
		return (EINVAL22);
	}
}
}

/* Enable SVME in EFER (must always be set) */
vmcb->v_efer |= EFER_SVME0x00001000;
svm_set_dirty(vcpu, SVM_CLEANBITS_CR(1 << 5));

return (ret);
56
←
Returning zero (loaded from 'ret'), which participates in a condition later→
4579}

4581/*
* vmx_handle_exit
*
* Handle exits from the VM by decoding the exit reason and calling various
* subhandlers as needed.
*/
4587int
4588vmx_handle_exit(struct vcpu *vcpu)
4589{
uint64_t exit_reason, rflags, istate;
int update_rip, ret = 0;

update_rip = 0;
exit_reason = vcpu->vc_gueststate.vg_exit_reason;
rflags = vcpu->vc_gueststate.vg_rflags;

switch (exit_reason) {
case VMX_EXIT_INT_WINDOW7:
if (!(rflags & PSL_I0x00000200)) {
	DPRINTF("%s: impossible interrupt window exit "
	    "config\n", __func__);
	ret = EINVAL22;
	break;
}

ret = EAGAIN35;
update_rip = 0;
break;
case VMX_EXIT_EPT_VIOLATION48:
ret = vmx_handle_np_fault(vcpu);
break;
case VMX_EXIT_CPUID10:
ret = vmm_handle_cpuid(vcpu);
update_rip = 1;
break;
case VMX_EXIT_IO30:
if (vmx_handle_inout(vcpu) == 0)
	ret = EAGAIN35;
break;
case VMX_EXIT_EXTINT1:
vmx_handle_intr(vcpu);
update_rip = 0;
break;
case VMX_EXIT_CR_ACCESS28:
ret = vmx_handle_cr(vcpu);
update_rip = 1;
break;
case VMX_EXIT_HLT12:
ret = vmx_handle_hlt(vcpu);
update_rip = 1;
break;
case VMX_EXIT_RDMSR31:
ret = vmx_handle_rdmsr(vcpu);
update_rip = 1;
break;
case VMX_EXIT_WRMSR32:
ret = vmx_handle_wrmsr(vcpu);
update_rip = 1;
break;
case VMX_EXIT_XSETBV55:
ret = vmx_handle_xsetbv(vcpu);
update_rip = 1;
break;
case VMX_EXIT_MWAIT36:
case VMX_EXIT_MONITOR39:
case VMX_EXIT_VMXON27:
case VMX_EXIT_VMWRITE25:
case VMX_EXIT_VMREAD23:
case VMX_EXIT_VMLAUNCH20:
case VMX_EXIT_VMRESUME24:
case VMX_EXIT_VMPTRLD21:
case VMX_EXIT_VMPTRST22:
case VMX_EXIT_VMCLEAR19:
case VMX_EXIT_VMCALL18:
case VMX_EXIT_VMFUNC59:
case VMX_EXIT_VMXOFF26:
case VMX_EXIT_INVVPID53:
case VMX_EXIT_INVEPT50:
ret = vmm_inject_ud(vcpu);
update_rip = 0;
break;
case VMX_EXIT_TRIPLE_FAULT2:
4663#ifdef VMM_DEBUG
DPRINTF("%s: vm %d vcpu %d triple fault\n", __func__,
    vcpu->vc_parent->vm_id, vcpu->vc_id);
vmx_vcpu_dump_regs(vcpu);
dump_vcpu(vcpu);
vmx_dump_vmcs(vcpu);
4669#endif /* VMM_DEBUG */
ret = EAGAIN35;
update_rip = 0;
break;
default:
4674#ifdef VMM_DEBUG
DPRINTF("%s: unhandled exit 0x%llx (%s)\n", __func__,
    exit_reason, vmx_exit_reason_decode(exit_reason));
4677#endif /* VMM_DEBUG */
return (EINVAL22);
}

if (update_rip) {
if (vmwrite(VMCS_GUEST_IA32_RIP0x681E,
    vcpu->vc_gueststate.vg_rip)) {
	printf("%s: can't advance rip\n", __func__);
	return (EINVAL22);
}

if (vmread(VMCS_GUEST_INTERRUPTIBILITY_ST0x4824,
    &istate)) {
	printf("%s: can't read interruptibility state\n",
	    __func__);
	return (EINVAL22);
}

/* Interruptibility state 0x3 covers NMIs and STI */
istate &= ~0x3;

if (vmwrite(VMCS_GUEST_INTERRUPTIBILITY_ST0x4824,
    istate)) {
	printf("%s: can't write interruptibility state\n",
	    __func__);
	return (EINVAL22);
}

if (rflags & PSL_T0x00000100) {
	if (vmm_inject_db(vcpu)) {
		printf("%s: can't inject #DB exception to "
		    "guest", __func__);
		return (EINVAL22);
	}
}
}

return (ret);
4715}

4717/*
* vmm_inject_gp
*
* Injects an #GP exception into the guest VCPU.
*
* Parameters:
*  vcpu: vcpu to inject into
*
* Return values:
*  Always 0
*/
4728int
4729vmm_inject_gp(struct vcpu *vcpu)
4730{
DPRINTF("%s: injecting #GP at guest %%rip 0x%llx\n", __func__,
   vcpu->vc_gueststate.vg_rip);
vcpu->vc_event = VMM_EX_GP13;

return (0);
4736}

4738/*
* vmm_inject_ud
*
* Injects an #UD exception into the guest VCPU.
*
* Parameters:
*  vcpu: vcpu to inject into
*
* Return values:
*  Always 0
*/
4749int
4750vmm_inject_ud(struct vcpu *vcpu)
4751{
DPRINTF("%s: injecting #UD at guest %%rip 0x%llx\n", __func__,
   vcpu->vc_gueststate.vg_rip);
vcpu->vc_event = VMM_EX_UD6;

return (0);
52
←
Returning zero, which participates in a condition later→
4757}

4759/*
* vmm_inject_db
*
* Injects a #DB exception into the guest VCPU.
*
* Parameters:
*  vcpu: vcpu to inject into
*
* Return values:
*  Always 0
*/
4770int
4771vmm_inject_db(struct vcpu *vcpu)
4772{
DPRINTF("%s: injecting #DB at guest %%rip 0x%llx\n", __func__,
   vcpu->vc_gueststate.vg_rip);
vcpu->vc_event = VMM_EX_DB1;

return (0);
4778}

4780/*
* vmm_get_guest_memtype
*
* Returns the type of memory 'gpa' refers to in the context of vm 'vm'
*/
4785int
4786vmm_get_guest_memtype(struct vm *vm, paddr_t gpa)
4787{
int i;
struct vm_mem_range *vmr;

/* XXX Use binary search? */
for (i = 0; i < vm->vm_nmemranges; i++) {
vmr = &vm->vm_memranges[i];

/*
 * vm_memranges are ascending. gpa can no longer be in one of
 * the memranges
 */
if (gpa < vmr->vmr_gpa)
	break;

if (gpa < vmr->vmr_gpa + vmr->vmr_size) {
	if (vmr->vmr_type == VM_MEM_MMIO2)
		return (VMM_MEM_TYPE_MMIO);
	return (VMM_MEM_TYPE_REGULAR);
}
}

DPRINTF("guest memtype @ 0x%llx unknown\n", (uint64_t)gpa);
return (VMM_MEM_TYPE_UNKNOWN);
4811}

4813/*
* vmx_get_exit_qualification
*
* Return the current VMCS' exit qualification information
*/
4818int
4819vmx_get_exit_qualification(uint64_t *exit_qualification)
4820{
if (vmread(VMCS_GUEST_EXIT_QUALIFICATION0x6400, exit_qualification)) {
printf("%s: can't extract exit qual\n", __func__);
return (EINVAL22);
}

return (0);
4827}

4829/*
* vmx_get_guest_faulttype
*
* Determines the type (R/W/X) of the last fault on the VCPU last run on
* this PCPU.
*/
4835int
4836vmx_get_guest_faulttype(void)
4837{
uint64_t exit_qual;
uint64_t presentmask = IA32_VMX_EPT_FAULT_WAS_READABLE(1ULL << 3) |
   IA32_VMX_EPT_FAULT_WAS_WRITABLE(1ULL << 4) | IA32_VMX_EPT_FAULT_WAS_EXECABLE(1ULL << 5);
vm_prot_t prot, was_prot;

if (vmx_get_exit_qualification(&exit_qual))
return (-1);

if ((exit_qual & presentmask) == 0)
return VM_FAULT_INVALID((vm_fault_t) 0x0);

was_prot = 0;
if (exit_qual & IA32_VMX_EPT_FAULT_WAS_READABLE(1ULL << 3))
was_prot |= PROT_READ0x01;
if (exit_qual & IA32_VMX_EPT_FAULT_WAS_WRITABLE(1ULL << 4))
was_prot |= PROT_WRITE0x02;
if (exit_qual & IA32_VMX_EPT_FAULT_WAS_EXECABLE(1ULL << 5))
was_prot |= PROT_EXEC0x04;

prot = 0;
if (exit_qual & IA32_VMX_EPT_FAULT_READ(1ULL << 0))
prot = PROT_READ0x01;
else if (exit_qual & IA32_VMX_EPT_FAULT_WRITE(1ULL << 1))
prot = PROT_WRITE0x02;
else if (exit_qual & IA32_VMX_EPT_FAULT_EXEC(1ULL << 2))
prot = PROT_EXEC0x04;

if ((was_prot & prot) == 0)
return VM_FAULT_PROTECT((vm_fault_t) 0x1);

return (-1);
4869}

4871/*
* svm_get_guest_faulttype
*
* Determines the type (R/W/X) of the last fault on the VCPU last run on
* this PCPU.
*/
4877int
4878svm_get_guest_faulttype(struct vmcb *vmcb)
4879{
if (!(vmcb->v_exitinfo1 & 0x1))
return VM_FAULT_INVALID((vm_fault_t) 0x0);
return VM_FAULT_PROTECT((vm_fault_t) 0x1);
4883}

4885/*
* svm_fault_page
*
* Request a new page to be faulted into the UVM map of the VM owning 'vcpu'
* at address 'gpa'.
*/
4891int
4892svm_fault_page(struct vcpu *vcpu, paddr_t gpa)
4893{
int ret;

ret = uvm_fault(vcpu->vc_parent->vm_map, gpa, VM_FAULT_WIRE((vm_fault_t) 0x2),
   PROT_READ0x01 | PROT_WRITE0x02 | PROT_EXEC0x04);
if (ret)
printf("%s: uvm_fault returns %d, GPA=0x%llx, rip=0x%llx\n",
    __func__, ret, (uint64_t)gpa, vcpu->vc_gueststate.vg_rip);

return (ret);
4903}

4905/*
* svm_handle_np_fault
*
* High level nested paging handler for SVM. Verifies that a fault is for a
* valid memory region, then faults a page, or aborts otherwise.
*/
4911int
4912svm_handle_np_fault(struct vcpu *vcpu)
4913{
uint64_t gpa;
int gpa_memtype, ret = 0;
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;
struct vm_exit_eptviolation *vee = &vcpu->vc_exit.vee;
struct cpu_info *ci = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});

memset(vee, 0, sizeof(*vee))__builtin_memset((vee), (0), (sizeof(*vee)));

gpa = vmcb->v_exitinfo2;

gpa_memtype = vmm_get_guest_memtype(vcpu->vc_parent, gpa);
switch (gpa_memtype) {
case VMM_MEM_TYPE_REGULAR:
vee->vee_fault_type = VEE_FAULT_HANDLED;
ret = svm_fault_page(vcpu, gpa);
break;
case VMM_MEM_TYPE_MMIO:
vee->vee_fault_type = VEE_FAULT_MMIO_ASSIST;
if (ci->ci_vmm_cap.vcc_svm.svm_decode_assist) {
	vee->vee_insn_len = vmcb->v_n_bytes_fetched;
	memcpy(&vee->vee_insn_bytes, vmcb->v_guest_ins_bytes,__builtin_memcpy((&vee->vee_insn_bytes), (vmcb->v_guest_ins_bytes
), (sizeof(vee->vee_insn_bytes)))
	    sizeof(vee->vee_insn_bytes))__builtin_memcpy((&vee->vee_insn_bytes), (vmcb->v_guest_ins_bytes
), (sizeof(vee->vee_insn_bytes)));
	vee->vee_insn_info |= VEE_BYTES_VALID0x2;
}
ret = EAGAIN35;
break;
default:
printf("%s: unknown memory type %d for GPA 0x%llx\n",
    __func__, gpa_memtype, gpa);
return (EINVAL22);
}

return (ret);
4947}

4949/*
* vmx_fault_page
*
* Request a new page to be faulted into the UVM map of the VM owning 'vcpu'
* at address 'gpa'.
*
* Parameters:
*  vcpu: guest VCPU requiring the page to be faulted into the UVM map
*  gpa: guest physical address that triggered the fault
*
* Return Values:
*  0: if successful
*  EINVAL: if fault type could not be determined or VMCS reload fails
*  EAGAIN: if a protection fault occurred, ie writing to a read-only page
*  errno: if uvm_fault(9) fails to wire in the page
*/
4965int
4966vmx_fault_page(struct vcpu *vcpu, paddr_t gpa)
4967{
int fault_type, ret;

fault_type = vmx_get_guest_faulttype();
switch (fault_type) {
case -1:
printf("%s: invalid fault type\n", __func__);
return (EINVAL22);
case VM_FAULT_PROTECT((vm_fault_t) 0x1):
vcpu->vc_exit.vee.vee_fault_type = VEE_FAULT_PROTECT;
return (EAGAIN35);
default:
vcpu->vc_exit.vee.vee_fault_type = VEE_FAULT_HANDLED;
break;
}

/* We may sleep during uvm_fault(9), so reload VMCS. */
vcpu->vc_last_pcpu = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
ret = uvm_fault(vcpu->vc_parent->vm_map, gpa, VM_FAULT_WIRE((vm_fault_t) 0x2),
   PROT_READ0x01 | PROT_WRITE0x02 | PROT_EXEC0x04);
if (vcpu_reload_vmcs_vmx(vcpu)) {
printf("%s: failed to reload vmcs\n", __func__);
return (EINVAL22);
}

if (ret)
printf("%s: uvm_fault returns %d, GPA=0x%llx, rip=0x%llx\n",
    __func__, ret, (uint64_t)gpa, vcpu->vc_gueststate.vg_rip);

return (ret);
4997}

4999/*
* vmx_handle_np_fault
*
* High level nested paging handler for VMX. Verifies that a fault is for a
* valid memory region, then faults a page, or aborts otherwise.
*/
5005int
5006vmx_handle_np_fault(struct vcpu *vcpu)
5007{
uint64_t insn_len = 0, gpa;
int gpa_memtype, ret = 0;
struct vm_exit_eptviolation *vee = &vcpu->vc_exit.vee;

memset(vee, 0, sizeof(*vee))__builtin_memset((vee), (0), (sizeof(*vee)));

if (vmread(VMCS_GUEST_PHYSICAL_ADDRESS0x2400, &gpa)) {
printf("%s: cannot extract faulting pa\n", __func__);
return (EINVAL22);
}

gpa_memtype = vmm_get_guest_memtype(vcpu->vc_parent, gpa);
switch (gpa_memtype) {
case VMM_MEM_TYPE_REGULAR:
vee->vee_fault_type = VEE_FAULT_HANDLED;
ret = vmx_fault_page(vcpu, gpa);
break;
case VMM_MEM_TYPE_MMIO:
vee->vee_fault_type = VEE_FAULT_MMIO_ASSIST;
if (vmread(VMCS_INSTRUCTION_LENGTH0x440C, &insn_len) ||
    insn_len == 0 || insn_len > 15) {
	printf("%s: failed to extract instruction length\n",
	    __func__);
	ret = EINVAL22;
} else {
	vee->vee_insn_len = (uint32_t)insn_len;
	vee->vee_insn_info |= VEE_LEN_VALID0x1;
	ret = EAGAIN35;
}
break;
default:
printf("%s: unknown memory type %d for GPA 0x%llx\n",
    __func__, gpa_memtype, gpa);
return (EINVAL22);
}

return (ret);
5045}

5047/*
* vmm_get_guest_cpu_cpl
*
* Determines current CPL of 'vcpu'. On VMX/Intel, this is gathered from the
* VMCS field for the DPL of SS (this seems odd, but is documented that way
* in the SDM). For SVM/AMD, this is gathered directly from the VMCB's 'cpl'
* field, as per the APM.
*
* Parameters:
*  vcpu: guest VCPU for which CPL is to be checked
*
* Return Values:
*  -1: the CPL could not be determined
*  0-3 indicating the current CPL. For real mode operation, 0 is returned.
*/
5062int
5063vmm_get_guest_cpu_cpl(struct vcpu *vcpu)
5064{
int mode;
struct vmcb *vmcb;
uint64_t ss_ar;

mode = vmm_get_guest_cpu_mode(vcpu);

if (mode == VMM_CPU_MODE_UNKNOWN)
return (-1);

if (mode == VMM_CPU_MODE_REAL)
return (0);

if (vmm_softc->mode == VMM_MODE_RVI) {
vmcb = (struct vmcb *)vcpu->vc_control_va;
return (vmcb->v_cpl);
} else if (vmm_softc->mode == VMM_MODE_EPT) {
if (vmread(VMCS_GUEST_IA32_SS_AR0x4818, &ss_ar))
	return (-1);
return ((ss_ar & 0x60) >> 5);
} else
return (-1);
5086}

5088/*
* vmm_get_guest_cpu_mode
*
* Determines current CPU mode of 'vcpu'.
*
* Parameters:
*  vcpu: guest VCPU for which mode is to be checked
*
* Return Values:
*  One of VMM_CPU_MODE_*, or VMM_CPU_MODE_UNKNOWN if the mode could not be
*   ascertained.
*/
5100int
5101vmm_get_guest_cpu_mode(struct vcpu *vcpu)
5102{
uint64_t cr0, efer, cs_ar;
uint8_t l, dib;
struct vmcb *vmcb;
struct vmx_msr_store *msr_store;

if (vmm_softc->mode == VMM_MODE_RVI) {
vmcb = (struct vmcb *)vcpu->vc_control_va;
cr0 = vmcb->v_cr0;
efer = vmcb->v_efer;
cs_ar = vmcb->v_cs.vs_attr;
cs_ar = (cs_ar & 0xff) | ((cs_ar << 4) & 0xf000);
} else if (vmm_softc->mode == VMM_MODE_EPT) {
if (vmread(VMCS_GUEST_IA32_CR00x6800, &cr0))
	return (VMM_CPU_MODE_UNKNOWN);
if (vmread(VMCS_GUEST_IA32_CS_AR0x4816, &cs_ar))
	return (VMM_CPU_MODE_UNKNOWN);
msr_store =
    (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_save_va;
efer = msr_store[VCPU_REGS_EFER0].vms_data;
} else
return (VMM_CPU_MODE_UNKNOWN);

l = (cs_ar & 0x2000) >> 13;
dib = (cs_ar & 0x4000) >> 14;

/* Check CR0.PE */
if (!(cr0 & CR0_PE0x00000001))
return (VMM_CPU_MODE_REAL);

/* Check EFER */
if (efer & EFER_LMA0x00000400) {
/* Could be compat or long mode, check CS.L */
if (l)
	return (VMM_CPU_MODE_LONG);
else
	return (VMM_CPU_MODE_COMPAT);
}

/* Check prot vs prot32 */
if (dib)
return (VMM_CPU_MODE_PROT32);
else
return (VMM_CPU_MODE_PROT);
5146}

5148/*
* svm_handle_inout
*
* Exit handler for IN/OUT instructions.
*
* Parameters:
*  vcpu: The VCPU where the IN/OUT instruction occurred
*
* Return values:
*  0: if successful
*  EINVAL: an invalid IN/OUT instruction was encountered
*/
5160int
5161svm_handle_inout(struct vcpu *vcpu)
5162{
uint64_t insn_length, exit_qual;
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;

insn_length = vmcb->v_exitinfo2 - vmcb->v_rip;
exit_qual = vmcb->v_exitinfo1;

/* Bit 0 - direction */
if (exit_qual & 0x1)
vcpu->vc_exit.vei.vei_dir = VEI_DIR_IN;
else
vcpu->vc_exit.vei.vei_dir = VEI_DIR_OUT;
/* Bit 2 - string instruction? */
vcpu->vc_exit.vei.vei_string = (exit_qual & 0x4) >> 2;
/* Bit 3 - REP prefix? */
vcpu->vc_exit.vei.vei_rep = (exit_qual & 0x8) >> 3;

/* Bits 4:6 - size of exit */
if (exit_qual & 0x10)
vcpu->vc_exit.vei.vei_size = 1;
else if (exit_qual & 0x20)
vcpu->vc_exit.vei.vei_size = 2;
else if (exit_qual & 0x40)
vcpu->vc_exit.vei.vei_size = 4;

/* Bit 16:31 - port */
vcpu->vc_exit.vei.vei_port = (exit_qual & 0xFFFF0000) >> 16;
/* Data */
vcpu->vc_exit.vei.vei_data = vmcb->v_rax;

vcpu->vc_exit.vei.vei_insn_len = (uint8_t)insn_length;

TRACEPOINT(vmm, inout, vcpu, vcpu->vc_exit.vei.vei_port,do { extern struct dt_probe (dt_static_vmm_inout); struct dt_probe
 *dtp = &(dt_static_vmm_inout); if (__builtin_expect(((dt_tracing
) != 0), 0) && __builtin_expect(((dtp->dtp_recording
) != 0), 0)) { struct dt_provider *dtpv = dtp->dtp_prov; dtpv
->dtpv_enter(dtpv, dtp, vcpu, vcpu->vc_exit.vei.vei_port
, vcpu->vc_exit.vei.vei_dir, vcpu->vc_exit.vei.vei_data
); } } while (0)
   vcpu->vc_exit.vei.vei_dir, vcpu->vc_exit.vei.vei_data)do { extern struct dt_probe (dt_static_vmm_inout); struct dt_probe
 *dtp = &(dt_static_vmm_inout); if (__builtin_expect(((dt_tracing
) != 0), 0) && __builtin_expect(((dtp->dtp_recording
) != 0), 0)) { struct dt_provider *dtpv = dtp->dtp_prov; dtpv
->dtpv_enter(dtpv, dtp, vcpu, vcpu->vc_exit.vei.vei_port
, vcpu->vc_exit.vei.vei_dir, vcpu->vc_exit.vei.vei_data
); } } while (0);

return (0);
5198}

5200/*
* vmx_handle_inout
*
* Exit handler for IN/OUT instructions.
*
* Parameters:
*  vcpu: The VCPU where the IN/OUT instruction occurred
*
* Return values:
*  0: if successful
*  EINVAL: invalid IN/OUT instruction or vmread failures occurred
*/
5212int
5213vmx_handle_inout(struct vcpu *vcpu)
5214{
uint64_t insn_length, exit_qual;

if (vmread(VMCS_INSTRUCTION_LENGTH0x440C, &insn_length)) {
printf("%s: can't obtain instruction length\n", __func__);
return (EINVAL22);
}

if (vmx_get_exit_qualification(&exit_qual)) {
printf("%s: can't get exit qual\n", __func__);
return (EINVAL22);
}

/* Bits 0:2 - size of exit */
vcpu->vc_exit.vei.vei_size = (exit_qual & 0x7) + 1;
/* Bit 3 - direction */
if ((exit_qual & 0x8) >> 3)
vcpu->vc_exit.vei.vei_dir = VEI_DIR_IN;
else
vcpu->vc_exit.vei.vei_dir = VEI_DIR_OUT;
/* Bit 4 - string instruction? */
vcpu->vc_exit.vei.vei_string = (exit_qual & 0x10) >> 4;
/* Bit 5 - REP prefix? */
vcpu->vc_exit.vei.vei_rep = (exit_qual & 0x20) >> 5;
/* Bit 6 - Operand encoding */
vcpu->vc_exit.vei.vei_encoding = (exit_qual & 0x40) >> 6;
/* Bit 16:31 - port */
vcpu->vc_exit.vei.vei_port = (exit_qual & 0xFFFF0000) >> 16;
/* Data */
vcpu->vc_exit.vei.vei_data = (uint32_t)vcpu->vc_gueststate.vg_rax;

vcpu->vc_exit.vei.vei_insn_len = (uint8_t)insn_length;

TRACEPOINT(vmm, inout, vcpu, vcpu->vc_exit.vei.vei_port,do { extern struct dt_probe (dt_static_vmm_inout); struct dt_probe
 *dtp = &(dt_static_vmm_inout); if (__builtin_expect(((dt_tracing
) != 0), 0) && __builtin_expect(((dtp->dtp_recording
) != 0), 0)) { struct dt_provider *dtpv = dtp->dtp_prov; dtpv
->dtpv_enter(dtpv, dtp, vcpu, vcpu->vc_exit.vei.vei_port
, vcpu->vc_exit.vei.vei_dir, vcpu->vc_exit.vei.vei_data
); } } while (0)
   vcpu->vc_exit.vei.vei_dir, vcpu->vc_exit.vei.vei_data)do { extern struct dt_probe (dt_static_vmm_inout); struct dt_probe
 *dtp = &(dt_static_vmm_inout); if (__builtin_expect(((dt_tracing
) != 0), 0) && __builtin_expect(((dtp->dtp_recording
) != 0), 0)) { struct dt_provider *dtpv = dtp->dtp_prov; dtpv
->dtpv_enter(dtpv, dtp, vcpu, vcpu->vc_exit.vei.vei_port
, vcpu->vc_exit.vei.vei_dir, vcpu->vc_exit.vei.vei_data
); } } while (0);

return (0);
5251}

5253/*
* vmx_load_pdptes
*
* Update the PDPTEs in the VMCS with the values currently indicated by the
* guest CR3. This is used for 32-bit PAE guests when enabling paging.
*
* Parameters
*  vcpu: The vcpu whose PDPTEs should be loaded
*
* Return values:
*  0: if successful
*  EINVAL: if the PDPTEs could not be loaded
*  ENOMEM: memory allocation failure
*/
5267int
5268vmx_load_pdptes(struct vcpu *vcpu)
5269{
uint64_t cr3, cr3_host_phys;
vaddr_t cr3_host_virt;
pd_entry_t *pdptes;
int ret;

if (vmread(VMCS_GUEST_IA32_CR30x6802, &cr3)) {
printf("%s: can't read guest cr3\n", __func__);
return (EINVAL22);
}

if (!pmap_extract(vcpu->vc_parent->vm_map->pmap, (vaddr_t)cr3,
   (paddr_t *)&cr3_host_phys)) {
DPRINTF("%s: nonmapped guest CR3, setting PDPTEs to 0\n",
    __func__);
if (vmwrite(VMCS_GUEST_PDPTE00x280A, 0)) {
	printf("%s: can't write guest PDPTE0\n", __func__);
	return (EINVAL22);
}

if (vmwrite(VMCS_GUEST_PDPTE10x280C, 0)) {
	printf("%s: can't write guest PDPTE1\n", __func__);
	return (EINVAL22);
}

if (vmwrite(VMCS_GUEST_PDPTE20x280E, 0)) {
	printf("%s: can't write guest PDPTE2\n", __func__);
	return (EINVAL22);
}

if (vmwrite(VMCS_GUEST_PDPTE30x2810, 0)) {
	printf("%s: can't write guest PDPTE3\n", __func__);
	return (EINVAL22);
}
return (0);
}

ret = 0;

/* We may sleep during km_alloc(9), so reload VMCS. */
vcpu->vc_last_pcpu = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
cr3_host_virt = (vaddr_t)km_alloc(PAGE_SIZE(1 << 12), &kv_any, &kp_none,
   &kd_waitok);
if (vcpu_reload_vmcs_vmx(vcpu)) {
printf("%s: failed to reload vmcs\n", __func__);
ret = EINVAL22;
goto exit;
}

if (!cr3_host_virt) {
printf("%s: can't allocate address for guest CR3 mapping\n",
    __func__);
return (ENOMEM12);
}

pmap_kenter_pa(cr3_host_virt, cr3_host_phys, PROT_READ0x01);

pdptes = (pd_entry_t *)cr3_host_virt;
if (vmwrite(VMCS_GUEST_PDPTE00x280A, pdptes[0])) {
printf("%s: can't write guest PDPTE0\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_GUEST_PDPTE10x280C, pdptes[1])) {
printf("%s: can't write guest PDPTE1\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_GUEST_PDPTE20x280E, pdptes[2])) {
printf("%s: can't write guest PDPTE2\n", __func__);
ret = EINVAL22;
goto exit;
}

if (vmwrite(VMCS_GUEST_PDPTE30x2810, pdptes[3])) {
printf("%s: can't write guest PDPTE3\n", __func__);
ret = EINVAL22;
goto exit;
}

5351exit:
pmap_kremove(cr3_host_virt, PAGE_SIZE(1 << 12));

/* km_free(9) might sleep, so we need to reload VMCS. */
vcpu->vc_last_pcpu = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
km_free((void *)cr3_host_virt, PAGE_SIZE(1 << 12), &kv_any, &kp_none);
if (vcpu_reload_vmcs_vmx(vcpu)) {
printf("%s: failed to reload vmcs after km_free\n", __func__);
ret = EINVAL22;
}

return (ret);
5363}

5365/*
* vmx_handle_cr0_write
*
* Write handler for CR0. This function ensures valid values are written into
* CR0 for the cpu/vmm mode in use (cr0 must-be-0 and must-be-1 bits, etc).
*
* Parameters
*  vcpu: The vcpu taking the cr0 write exit
*     r: The guest's desired (incoming) cr0 value
*
* Return values:
*  0: if successful
*  EINVAL: if an error occurred
*/
5379int
5380vmx_handle_cr0_write(struct vcpu *vcpu, uint64_t r)
5381{
struct vmx_msr_store *msr_store;
struct vmx_invvpid_descriptor vid;
uint64_t ectls, oldcr0, cr4, mask;
int ret;

/* Check must-be-0 bits */
mask = vcpu->vc_vmx_cr0_fixed1;
if (~r & mask) {
/* Inject #GP, let the guest handle it */
DPRINTF("%s: guest set invalid bits in %%cr0. Zeros "
    "mask=0x%llx, data=0x%llx\n", __func__,
    vcpu->vc_vmx_cr0_fixed1, r);
vmm_inject_gp(vcpu);
return (0);
}

/* Check must-be-1 bits */
mask = vcpu->vc_vmx_cr0_fixed0;
if ((r & mask) != mask) {
/* Inject #GP, let the guest handle it */
DPRINTF("%s: guest set invalid bits in %%cr0. Ones "
    "mask=0x%llx, data=0x%llx\n", __func__,
    vcpu->vc_vmx_cr0_fixed0, r);
vmm_inject_gp(vcpu);
return (0);
}

if (r & 0xFFFFFFFF00000000ULL) {
DPRINTF("%s: setting bits 63:32 of %%cr0 is invalid,"
    " inject #GP, cr0=0x%llx\n", __func__, r);
vmm_inject_gp(vcpu);
return (0);
}

if ((r & CR0_PG0x80000000) && (r & CR0_PE0x00000001) == 0) {
DPRINTF("%s: PG flag set when the PE flag is clear,"
    " inject #GP, cr0=0x%llx\n", __func__, r);
vmm_inject_gp(vcpu);
return (0);
}

if ((r & CR0_NW0x20000000) && (r & CR0_CD0x40000000) == 0) {
DPRINTF("%s: NW flag set when the CD flag is clear,"
    " inject #GP, cr0=0x%llx\n", __func__, r);
vmm_inject_gp(vcpu);
return (0);
}

if (vmread(VMCS_GUEST_IA32_CR00x6800, &oldcr0)) {
printf("%s: can't read guest cr0\n", __func__);
return (EINVAL22);
}

/* CR0 must always have NE set */
r |= CR0_NE0x00000020;

if (vmwrite(VMCS_GUEST_IA32_CR00x6800, r)) {
printf("%s: can't write guest cr0\n", __func__);
return (EINVAL22);
}

/* If the guest hasn't enabled paging ... */
if (!(r & CR0_PG0x80000000) && (oldcr0 & CR0_PG0x80000000)) {
/* Paging was disabled (prev. enabled) - Flush TLB */
if (vmm_softc->mode == VMM_MODE_EPT &&
    vcpu->vc_vmx_vpid_enabled) {
	vid.vid_vpid = vcpu->vc_vpid;
	vid.vid_addr = 0;
	invvpid(IA32_VMX_INVVPID_SINGLE_CTX_GLB0x3, &vid);
}
} else if (!(oldcr0 & CR0_PG0x80000000) && (r & CR0_PG0x80000000)) {
/*
 * Since the guest has enabled paging, then the IA32_VMX_IA32E_MODE_GUEST
 * control must be set to the same as EFER_LME.
 */
msr_store = (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_save_va;

if (vmread(VMCS_ENTRY_CTLS0x4012, &ectls)) {
	printf("%s: can't read entry controls", __func__);
	return (EINVAL22);
}

if (msr_store[VCPU_REGS_EFER0].vms_data & EFER_LME0x00000100)
	ectls |= IA32_VMX_IA32E_MODE_GUEST(1ULL << 9);
else
	ectls &= ~IA32_VMX_IA32E_MODE_GUEST(1ULL << 9);

if (vmwrite(VMCS_ENTRY_CTLS0x4012, ectls)) {
	printf("%s: can't write entry controls", __func__);
	return (EINVAL22);
}

if (vmread(VMCS_GUEST_IA32_CR40x6804, &cr4)) {
	printf("%s: can't read guest cr4\n", __func__);
	return (EINVAL22);
}

/* Load PDPTEs if PAE guest enabling paging */
if (cr4 & CR4_PAE0x00000020) {
	ret = vmx_load_pdptes(vcpu);

	if (ret) {
		printf("%s: updating PDPTEs failed\n", __func__);
		return (ret);
	}
}
}

return (0);
5491}

5493/*
* vmx_handle_cr4_write
*
* Write handler for CR4. This function ensures valid values are written into
* CR4 for the cpu/vmm mode in use (cr4 must-be-0 and must-be-1 bits, etc).
*
* Parameters
*  vcpu: The vcpu taking the cr4 write exit
*     r: The guest's desired (incoming) cr4 value
*
* Return values:
*  0: if successful
*  EINVAL: if an error occurred
*/
5507int
5508vmx_handle_cr4_write(struct vcpu *vcpu, uint64_t r)
5509{
uint64_t mask;

/* Check must-be-0 bits */
mask = ~(curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed1);
if (r & mask) {
/* Inject #GP, let the guest handle it */
DPRINTF("%s: guest set invalid bits in %%cr4. Zeros "
    "mask=0x%llx, data=0x%llx\n", __func__,
    curcpu()->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed1,
    r);
vmm_inject_gp(vcpu);
return (0);
}

/* Check must-be-1 bits */
mask = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed0;
if ((r & mask) != mask) {
/* Inject #GP, let the guest handle it */
DPRINTF("%s: guest set invalid bits in %%cr4. Ones "
    "mask=0x%llx, data=0x%llx\n", __func__,
    curcpu()->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed0,
    r);
vmm_inject_gp(vcpu);
return (0);
}

/* CR4_VMXE must always be enabled */
r |= CR4_VMXE0x00002000;

if (vmwrite(VMCS_GUEST_IA32_CR40x6804, r)) {
printf("%s: can't write guest cr4\n", __func__);
return (EINVAL22);
}

return (0);
5545}

5547/*
* vmx_handle_cr
*
* Handle reads/writes to control registers (except CR3)
*/
5552int
5553vmx_handle_cr(struct vcpu *vcpu)
5554{
uint64_t insn_length, exit_qual, r;
uint8_t crnum, dir, reg;

if (vmread(VMCS_INSTRUCTION_LENGTH0x440C, &insn_length)) {
printf("%s: can't obtain instruction length\n", __func__);
return (EINVAL22);
}

if (vmx_get_exit_qualification(&exit_qual)) {
printf("%s: can't get exit qual\n", __func__);
return (EINVAL22);
}

/* Low 4 bits of exit_qual represent the CR number */
crnum = exit_qual & 0xf;

/*
* Bits 5:4 indicate the direction of operation (or special CR-modifying
* instruction)
*/
dir = (exit_qual & 0x30) >> 4;

/* Bits 11:8 encode the source/target register */
reg = (exit_qual & 0xf00) >> 8;

switch (dir) {
case CR_WRITE0:
if (crnum == 0 || crnum == 4) {
	switch (reg) {
	case 0: r = vcpu->vc_gueststate.vg_rax; break;
	case 1: r = vcpu->vc_gueststate.vg_rcx; break;
	case 2: r = vcpu->vc_gueststate.vg_rdx; break;
	case 3: r = vcpu->vc_gueststate.vg_rbx; break;
	case 4: if (vmread(VMCS_GUEST_IA32_RSP0x681C, &r)) {
			printf("%s: unable to read guest "
			    "RSP\n", __func__);
			return (EINVAL22);
		}
		break;
	case 5: r = vcpu->vc_gueststate.vg_rbp; break;
	case 6: r = vcpu->vc_gueststate.vg_rsi; break;
	case 7: r = vcpu->vc_gueststate.vg_rdi; break;
	case 8: r = vcpu->vc_gueststate.vg_r8; break;
	case 9: r = vcpu->vc_gueststate.vg_r9; break;
	case 10: r = vcpu->vc_gueststate.vg_r10; break;
	case 11: r = vcpu->vc_gueststate.vg_r11; break;
	case 12: r = vcpu->vc_gueststate.vg_r12; break;
	case 13: r = vcpu->vc_gueststate.vg_r13; break;
	case 14: r = vcpu->vc_gueststate.vg_r14; break;
	case 15: r = vcpu->vc_gueststate.vg_r15; break;
	}
	DPRINTF("%s: mov to cr%d @ %llx, data=0x%llx\n",
	    __func__, crnum, vcpu->vc_gueststate.vg_rip, r);
}

if (crnum == 0)
	vmx_handle_cr0_write(vcpu, r);

if (crnum == 4)
	vmx_handle_cr4_write(vcpu, r);

break;
case CR_READ1:
DPRINTF("%s: mov from cr%d @ %llx\n", __func__, crnum,
    vcpu->vc_gueststate.vg_rip);
break;
case CR_CLTS2:
DPRINTF("%s: clts instruction @ %llx\n", __func__,
    vcpu->vc_gueststate.vg_rip);
break;
case CR_LMSW3:
DPRINTF("%s: lmsw instruction @ %llx\n", __func__,
    vcpu->vc_gueststate.vg_rip);
break;
default:
DPRINTF("%s: unknown cr access @ %llx\n", __func__,
    vcpu->vc_gueststate.vg_rip);
}

vcpu->vc_gueststate.vg_rip += insn_length;

return (0);
5637}

5639/*
* vmx_handle_rdmsr
*
* Handler for rdmsr instructions. Bitmap MSRs are allowed implicit access
* and won't end up here. This handler is primarily intended to catch otherwise
* unknown MSR access for possible later inclusion in the bitmap list. For
* each MSR access that ends up here, we log the access (when VMM_DEBUG is
* enabled)
*
* Parameters:
*  vcpu: vcpu structure containing instruction info causing the exit
*
* Return value:
*  0: The operation was successful
*  EINVAL: An error occurred
*/
5655int
5656vmx_handle_rdmsr(struct vcpu *vcpu)
5657{
uint64_t insn_length;
uint64_t *rax, *rdx;
uint64_t *rcx;
int ret;

if (vmread(VMCS_INSTRUCTION_LENGTH0x440C, &insn_length)) {
printf("%s: can't obtain instruction length\n", __func__);
return (EINVAL22);
}

if (insn_length != 2) {
DPRINTF("%s: RDMSR with instruction length %lld not "
    "supported\n", __func__, insn_length);
return (EINVAL22);
}

rax = &vcpu->vc_gueststate.vg_rax;
rcx = &vcpu->vc_gueststate.vg_rcx;
rdx = &vcpu->vc_gueststate.vg_rdx;

switch (*rcx) {
case MSR_BIOS_SIGN0x08b:
case MSR_PLATFORM_ID0x017:
/* Ignored */
*rax = 0;
*rdx = 0;
break;
case MSR_CR_PAT0x277:
*rax = (vcpu->vc_shadow_pat & 0xFFFFFFFFULL);
*rdx = (vcpu->vc_shadow_pat >> 32);
break;
default:
/* Unsupported MSRs causes #GP exception, don't advance %rip */
DPRINTF("%s: unsupported rdmsr (msr=0x%llx), injecting #GP\n",
    __func__, *rcx);
ret = vmm_inject_gp(vcpu);
return (ret);
}

vcpu->vc_gueststate.vg_rip += insn_length;

return (0);
5700}

5702/*
* vmx_handle_xsetbv
*
* VMX-specific part of the xsetbv instruction exit handler
*
* Parameters:
*  vcpu: vcpu structure containing instruction info causing the exit
*
* Return value:
*  0: The operation was successful
*  EINVAL: An error occurred
*/
5714int
5715vmx_handle_xsetbv(struct vcpu *vcpu)
5716{
uint64_t insn_length, *rax;
int ret;

if (vmread(VMCS_INSTRUCTION_LENGTH0x440C, &insn_length)) {
printf("%s: can't obtain instruction length\n", __func__);
return (EINVAL22);
}

/* All XSETBV instructions are 3 bytes */
if (insn_length != 3) {
DPRINTF("%s: XSETBV with instruction length %lld not "
    "supported\n", __func__, insn_length);
return (EINVAL22);
}

rax = &vcpu->vc_gueststate.vg_rax;

ret = vmm_handle_xsetbv(vcpu, rax);

vcpu->vc_gueststate.vg_rip += insn_length;

return ret;
5739}

5741/*
* svm_handle_xsetbv
*
* SVM-specific part of the xsetbv instruction exit handler
*
* Parameters:
*  vcpu: vcpu structure containing instruction info causing the exit
*
* Return value:
*  0: The operation was successful
*  EINVAL: An error occurred
*/
5753int
5754svm_handle_xsetbv(struct vcpu *vcpu)
5755{
uint64_t insn_length, *rax;
int ret;
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;

/* All XSETBV instructions are 3 bytes */
insn_length = 3;

rax = &vmcb->v_rax;

ret = vmm_handle_xsetbv(vcpu, rax);

vcpu->vc_gueststate.vg_rip += insn_length;

return ret;
5770}

5772/*
* vmm_handle_xsetbv
*
* Handler for xsetbv instructions. We allow the guest VM to set xcr0 values
* limited to the xsave_mask in use in the host.
*
* Parameters:
*  vcpu: vcpu structure containing instruction info causing the exit
*  rax: pointer to guest %rax
*
* Return value:
*  0: The operation was successful
*  EINVAL: An error occurred
*/
5786int
5787vmm_handle_xsetbv(struct vcpu *vcpu, uint64_t *rax)
5788{
uint64_t *rdx, *rcx, val;

rcx = &vcpu->vc_gueststate.vg_rcx;
rdx = &vcpu->vc_gueststate.vg_rdx;

if (vmm_get_guest_cpu_cpl(vcpu) != 0) {
DPRINTF("%s: guest cpl not zero\n", __func__);
return (vmm_inject_gp(vcpu));
}

if (*rcx != 0) {
DPRINTF("%s: guest specified invalid xcr register number "
    "%lld\n", __func__, *rcx);
return (vmm_inject_gp(vcpu));
}

val = *rax + (*rdx << 32);
if (val & ~xsave_mask) {
DPRINTF("%s: guest specified xcr0 outside xsave_mask %lld\n",
    __func__, val);
return (vmm_inject_gp(vcpu));
}

vcpu->vc_gueststate.vg_xcr0 = val;

return (0);
5815}

5817/*
* vmx_handle_misc_enable_msr
*
* Handler for writes to the MSR_MISC_ENABLE (0x1a0) MSR on Intel CPUs. We
* limit what the guest can write to this MSR (certain hardware-related
* settings like speedstep, etc).
*
* Parameters:
*  vcpu: vcpu structure containing information about the wrmsr causing this
*   exit
*/
5828void
5829vmx_handle_misc_enable_msr(struct vcpu *vcpu)
5830{
uint64_t *rax, *rdx;
struct vmx_msr_store *msr_store;

rax = &vcpu->vc_gueststate.vg_rax;
rdx = &vcpu->vc_gueststate.vg_rdx;
msr_store = (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_save_va;

/* Filter out guest writes to TCC, EIST, and xTPR */
*rax &= ~(MISC_ENABLE_TCC(1 << 3) | MISC_ENABLE_EIST_ENABLED(1 << 16) |
   MISC_ENABLE_xTPR_MESSAGE_DISABLE(1 << 23));

msr_store[VCPU_REGS_MISC_ENABLE6].vms_data = *rax | (*rdx << 32);
5843}

5845/*
* vmx_handle_wrmsr
*
* Handler for wrmsr instructions. This handler logs the access, and discards
* the written data (when VMM_DEBUG is enabled). Any valid wrmsr will not end
* up here (it will be whitelisted in the MSR bitmap).
*
* Parameters:
*  vcpu: vcpu structure containing instruction info causing the exit
*
* Return value:
*  0: The operation was successful
*  EINVAL: An error occurred
*/
5859int
5860vmx_handle_wrmsr(struct vcpu *vcpu)
5861{
uint64_t insn_length, val;
uint64_t *rax, *rdx, *rcx;
int ret;

if (vmread(VMCS_INSTRUCTION_LENGTH0x440C, &insn_length)) {
printf("%s: can't obtain instruction length\n", __func__);
return (EINVAL22);
}

if (insn_length != 2) {
DPRINTF("%s: WRMSR with instruction length %lld not "
    "supported\n", __func__, insn_length);
return (EINVAL22);
}

rax = &vcpu->vc_gueststate.vg_rax;
rcx = &vcpu->vc_gueststate.vg_rcx;
rdx = &vcpu->vc_gueststate.vg_rdx;
val = (*rdx << 32) | (*rax & 0xFFFFFFFFULL);

switch (*rcx) {
case MSR_CR_PAT0x277:
if (!vmm_pat_is_valid(val)) {
	ret = vmm_inject_gp(vcpu);
	return (ret);
}
vcpu->vc_shadow_pat = val;
break;
case MSR_MISC_ENABLE0x1a0:
vmx_handle_misc_enable_msr(vcpu);
break;
case MSR_SMM_MONITOR_CTL0x09b:
/*
 * 34.15.5 - Enabling dual monitor treatment
 *
 * Unsupported, so inject #GP and return without
 * advancing %rip.
 */
ret = vmm_inject_gp(vcpu);
return (ret);
case KVM_MSR_SYSTEM_TIME0x4b564d01:
vmm_init_pvclock(vcpu,
    (*rax & 0xFFFFFFFFULL) | (*rdx  << 32));
break;
5906#ifdef VMM_DEBUG
default:
/*
 * Log the access, to be able to identify unknown MSRs
 */
DPRINTF("%s: wrmsr exit, msr=0x%llx, discarding data "
    "written from guest=0x%llx:0x%llx\n", __func__,
    *rcx, *rdx, *rax);
5914#endif /* VMM_DEBUG */
}

vcpu->vc_gueststate.vg_rip += insn_length;

return (0);
5920}

5922/*
* svm_handle_msr
*
* Handler for MSR instructions.
*
* Parameters:
*  vcpu: vcpu structure containing instruction info causing the exit
*
* Return value:
*  Always 0 (successful)
*/
5933int
5934svm_handle_msr(struct vcpu *vcpu)
5935{
uint64_t insn_length, val;
uint64_t *rax, *rcx, *rdx;
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;
int ret;

/* XXX: Validate RDMSR / WRMSR insn_length */
insn_length = 2;

rax = &vmcb->v_rax;
rcx = &vcpu->vc_gueststate.vg_rcx;
rdx = &vcpu->vc_gueststate.vg_rdx;

if (vmcb->v_exitinfo1 == 1) {
/* WRMSR */
val = (*rdx << 32) | (*rax & 0xFFFFFFFFULL);

switch (*rcx) {
case MSR_CR_PAT0x277:
	if (!vmm_pat_is_valid(val)) {
		ret = vmm_inject_gp(vcpu);
		return (ret);
	}
	vcpu->vc_shadow_pat = val;
	break;
case MSR_EFER0xc0000080:
	vmcb->v_efer = *rax | EFER_SVME0x00001000;
	break;
case KVM_MSR_SYSTEM_TIME0x4b564d01:
	vmm_init_pvclock(vcpu,
	    (*rax & 0xFFFFFFFFULL) | (*rdx  << 32));
	break;
default:
	/* Log the access, to be able to identify unknown MSRs */
	DPRINTF("%s: wrmsr exit, msr=0x%llx, discarding data "
	    "written from guest=0x%llx:0x%llx\n", __func__,
	    *rcx, *rdx, *rax);
}
} else {
/* RDMSR */
switch (*rcx) {
case MSR_BIOS_SIGN0x08b:
case MSR_INT_PEN_MSG0xc0010055:
case MSR_PLATFORM_ID0x017:
	/* Ignored */
	*rax = 0;
	*rdx = 0;
	break;
case MSR_CR_PAT0x277:
	*rax = (vcpu->vc_shadow_pat & 0xFFFFFFFFULL);
	*rdx = (vcpu->vc_shadow_pat >> 32);
	break;
case MSR_DE_CFG0xc0011029:
	/* LFENCE serializing bit is set by host */
	*rax = DE_CFG_SERIALIZE_LFENCE(1 << 1);
	*rdx = 0;
	break;
default:
	/*
	 * Unsupported MSRs causes #GP exception, don't advance
	 * %rip
	 */
	DPRINTF("%s: unsupported rdmsr (msr=0x%llx), "
	    "injecting #GP\n", __func__, *rcx);
	ret = vmm_inject_gp(vcpu);
	return (ret);
}
}

vcpu->vc_gueststate.vg_rip += insn_length;

return (0);
6007}

6009/*
* vmm_handle_cpuid
*
* Exit handler for CPUID instruction
*
* Parameters:
*  vcpu: vcpu causing the CPUID exit
*
* Return value:
*  0: the exit was processed successfully
*  EINVAL: error occurred validating the CPUID instruction arguments
*/
6021int
6022vmm_handle_cpuid(struct vcpu *vcpu)
6023{
uint64_t insn_length, cr4;
uint64_t *rax, *rbx, *rcx, *rdx;
struct vmcb *vmcb;
uint32_t leaf, subleaf, eax, ebx, ecx, edx;
struct vmx_msr_store *msr_store;
int vmm_cpuid_level;

/* what's the cpuid level we support/advertise? */
vmm_cpuid_level = cpuid_level;
if (vmm_cpuid_level < 0x15 && tsc_is_invariant)
vmm_cpuid_level = 0x15;

if (vmm_softc->mode == VMM_MODE_EPT) {
if (vmread(VMCS_INSTRUCTION_LENGTH0x440C, &insn_length)) {
	DPRINTF("%s: can't obtain instruction length\n",
	    __func__);
	return (EINVAL22);
}

if (vmread(VMCS_GUEST_IA32_CR40x6804, &cr4)) {
	DPRINTF("%s: can't obtain cr4\n", __func__);
	return (EINVAL22);
}

rax = &vcpu->vc_gueststate.vg_rax;

/*
 * "CPUID leaves above 02H and below 80000000H are only
 * visible when IA32_MISC_ENABLE MSR has bit 22 set to its
 * default value 0"
 */
msr_store =
    (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_save_va;
if (msr_store[VCPU_REGS_MISC_ENABLE6].vms_data &
    MISC_ENABLE_LIMIT_CPUID_MAXVAL(1 << 22))
	vmm_cpuid_level = 0x02;
} else {
/* XXX: validate insn_length 2 */
insn_length = 2;
vmcb = (struct vmcb *)vcpu->vc_control_va;
rax = &vmcb->v_rax;
cr4 = vmcb->v_cr4;
}

rbx = &vcpu->vc_gueststate.vg_rbx;
rcx = &vcpu->vc_gueststate.vg_rcx;
rdx = &vcpu->vc_gueststate.vg_rdx;
vcpu->vc_gueststate.vg_rip += insn_length;

leaf = *rax;
subleaf = *rcx;

/*
* "If a value entered for CPUID.EAX is higher than the maximum input
*  value for basic or extended function for that processor then the
*  data for the highest basic information leaf is returned."
*
* "When CPUID returns the highest basic leaf information as a result
*  of an invalid input EAX value, any dependence on input ECX value
*  in the basic leaf is honored."
*
* This means if leaf is between vmm_cpuid_level and 0x40000000 (the start
* of the hypervisor info leaves), clamp to vmm_cpuid_level, but without
* altering subleaf.  Also, if leaf is greater than the extended function
* info, clamp also to vmm_cpuid_level.
*/
if ((leaf > vmm_cpuid_level && leaf < 0x40000000) ||
   (leaf > curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_pnfeatset)) {
DPRINTF("%s: invalid cpuid input leaf 0x%x, guest rip="
    "0x%llx - resetting to 0x%x\n", __func__, leaf,
    vcpu->vc_gueststate.vg_rip - insn_length,
    vmm_cpuid_level);
leaf = vmm_cpuid_level;
}

/* we fake up values in the range (cpuid_level, vmm_cpuid_level] */
if (leaf <= cpuid_level || leaf > 0x80000000)
CPUID_LEAF(leaf, subleaf, eax, ebx, ecx, edx)__asm volatile("cpuid" : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d"
 (edx) : "a" (leaf), "c" (subleaf));
else
eax = ebx = ecx = edx = 0;

switch (leaf) {
case 0x00:	/* Max level and vendor ID */
*rax = vmm_cpuid_level;
*rbx = *((uint32_t *)&cpu_vendor);
*rdx = *((uint32_t *)&cpu_vendor + 1);
*rcx = *((uint32_t *)&cpu_vendor + 2);
break;
case 0x01:	/* Version, brand, feature info */
*rax = cpu_id;
/* mask off host's APIC ID, reset to vcpu id */
*rbx = cpu_ebxfeature & 0x0000FFFF;
*rbx |= (vcpu->vc_id & 0xFF) << 24;
*rcx = (cpu_ecxfeature | CPUIDECX_HV0x80000000) & VMM_CPUIDECX_MASK~(0x00000080 | 0x00000100 | 0x00000008 | 0x00008000 | 0x00000020
 | 0x00000004 | 0x00000010 | 0x00000040 | 0x00000400 | 0x00000800
 | 0x00004000 | 0x00020000 | 0x00040000 | 0x00200000 | 0x01000000
);

/* Guest CR4.OSXSAVE determines presence of CPUIDECX_OSXSAVE */
if (cr4 & CR4_OSXSAVE0x00040000)
	*rcx |= CPUIDECX_OSXSAVE0x08000000;
else
	*rcx &= ~CPUIDECX_OSXSAVE0x08000000;

*rdx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_feature_flags & VMM_CPUIDEDX_MASK~(0x00400000 | 0x20000000 | 0x10000000 | 0x00200000 | 0x00000200
 | 0x00040000 | 0x08000000 | 0x80000000 | 0x00001000 | 0x00000080
 | 0x00004000);
break;
case 0x02:	/* Cache and TLB information */
*rax = eax;
*rbx = ebx;
*rcx = ecx;
*rdx = edx;
break;
case 0x03:	/* Processor serial number (not supported) */
DPRINTF("%s: function 0x03 (processor serial number) not "
"supported\n", __func__);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x04: 	/* Deterministic cache info */
*rax = eax & VMM_CPUID4_CACHE_TOPOLOGY_MASK0x3FF;
*rbx = ebx;
*rcx = ecx;
*rdx = edx;
break;
case 0x05:	/* MONITOR/MWAIT (not supported) */
DPRINTF("%s: function 0x05 (monitor/mwait) not supported\n",
    __func__);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x06:	/* Thermal / Power management (not supported) */
DPRINTF("%s: function 0x06 (thermal/power mgt) not supported\n",
    __func__);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x07:	/* SEFF */
if (subleaf == 0) {
	*rax = 0;	/* Highest subleaf supported */
	*rbx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_feature_sefflags_ebx & VMM_SEFF0EBX_MASK~(0x00000002 | 0x00000004 | 0x00000010 | 0x00000400 | 0x00000800
 | 0x00001000 | 0x00004000 | 0x00400000 | 0x02000000 | 0x00010000
 | 0x00020000 | 0x00200000 | 0x04000000 | 0x08000000 | 0x10000000
 | 0x40000000 | 0x80000000);
	*rcx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_feature_sefflags_ecx & VMM_SEFF0ECX_MASK(0x00000004);
	*rdx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_feature_sefflags_edx & VMM_SEFF0EDX_MASK(0x00000400);
	/*
	 * Only expose PKU support if we've detected it in use
	 * on the host.
	 */
	if (vmm_softc->sc_md.pkru_enabled)
		*rcx |= SEFF0ECX_PKU0x00000008;
	else
		*rcx &= ~SEFF0ECX_PKU0x00000008;

	/* Expose IBT bit if we've enabled CET on the host. */
	if (rcr4() & CR4_CET0x00800000)
		*rdx |= SEFF0EDX_IBT0x00100000;
	else
		*rdx &= ~SEFF0EDX_IBT0x00100000;

} else {
	/* Unsupported subleaf */
	DPRINTF("%s: function 0x07 (SEFF) unsupported subleaf "
	    "0x%x not supported\n", __func__, subleaf);
	*rax = 0;
	*rbx = 0;
	*rcx = 0;
	*rdx = 0;
}
break;
case 0x09:	/* Direct Cache Access (not supported) */
DPRINTF("%s: function 0x09 (direct cache access) not "
    "supported\n", __func__);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x0a:	/* Architectural perf monitoring (not supported) */
DPRINTF("%s: function 0x0a (arch. perf mon) not supported\n",
    __func__);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x0b:	/* Extended topology enumeration (not supported) */
DPRINTF("%s: function 0x0b (topology enumeration) not "
    "supported\n", __func__);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x0d:	/* Processor ext. state information */
if (subleaf == 0) {
	*rax = xsave_mask;
	*rbx = ebx;
	*rcx = ecx;
	*rdx = edx;
} else if (subleaf == 1) {
	*rax = 0;
	*rbx = 0;
	*rcx = 0;
	*rdx = 0;
} else {
	*rax = eax;
	*rbx = ebx;
	*rcx = ecx;
	*rdx = edx;
}
break;
case 0x0f:	/* QoS info (not supported) */
DPRINTF("%s: function 0x0f (QoS info) not supported\n",
    __func__);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x14:	/* Processor Trace info (not supported) */
DPRINTF("%s: function 0x14 (processor trace info) not "
    "supported\n", __func__);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x15:
if (cpuid_level >= 0x15) {
	*rax = eax;
	*rbx = ebx;
	*rcx = ecx;
	*rdx = edx;
} else {
	KASSERT(tsc_is_invariant)((tsc_is_invariant) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c"
, 6259, "tsc_is_invariant"));
	*rax = 1;
	*rbx = 100;
	*rcx = tsc_frequency / 100;
	*rdx = 0;
}
break;
case 0x16:	/* Processor frequency info */
*rax = eax;
*rbx = ebx;
*rcx = ecx;
*rdx = edx;
break;
case 0x40000000:	/* Hypervisor information */
*rax = 0;
*rbx = *((uint32_t *)&vmm_hv_signature[0]);
*rcx = *((uint32_t *)&vmm_hv_signature[4]);
*rdx = *((uint32_t *)&vmm_hv_signature[8]);
break;
case 0x40000001:	/* KVM hypervisor features */
*rax = (1 << KVM_FEATURE_CLOCKSOURCE23) |
    (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT24);
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x80000000:	/* Extended function level */
*rax = 0x80000008; /* curcpu()->ci_pnfeatset */
*rbx = 0;
*rcx = 0;
*rdx = 0;
break;
case 0x80000001: 	/* Extended function info */
*rax = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_efeature_eax;
*rbx = 0;	/* Reserved */
*rcx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_efeature_ecx & VMM_ECPUIDECX_MASK~(0x00000004 | 0x20000000);
*rdx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_feature_eflags & VMM_FEAT_EFLAGS_MASK~(0x08000000);
break;
case 0x80000002:	/* Brand string */
*rax = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[0];
*rbx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[1];
*rcx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[2];
*rdx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[3];
break;
case 0x80000003:	/* Brand string */
*rax = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[4];
*rbx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[5];
*rcx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[6];
*rdx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[7];
break;
case 0x80000004:	/* Brand string */
*rax = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[8];
*rbx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[9];
*rcx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[10];
*rdx = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_brand[11];
break;
case 0x80000005:	/* Reserved (Intel), cacheinfo (AMD) */
*rax = eax;
*rbx = ebx;
*rcx = ecx;
*rdx = edx;
break;
case 0x80000006:	/* ext. cache info */
*rax = eax;
*rbx = ebx;
*rcx = ecx;
*rdx = edx;
break;
case 0x80000007:	/* apmi */
*rax = eax;
*rbx = ebx;
*rcx = ecx;
*rdx = edx & VMM_APMI_EDX_INCLUDE_MASK((1 << 8));
break;
case 0x80000008:	/* Phys bits info and topology (AMD) */
*rax = eax;
*rbx = ebx & VMM_AMDSPEC_EBX_MASK~((1ULL << 12) | (1ULL << 14) | (1ULL << 15
) | (1ULL << 16) | (1ULL << 17) | (1ULL << 18
) | (1ULL << 24) | (1ULL << 25) | (1ULL << 26
));
/* Reset %rcx (topology) */
*rcx = 0;
*rdx = edx;
break;
case 0x8000001d:	/* cache topology (AMD) */
*rax = eax;
*rbx = ebx;
*rcx = ecx;
*rdx = edx;
break;
default:
DPRINTF("%s: unsupported rax=0x%llx\n", __func__, *rax);
*rax = 0;
*rbx = 0;
*rcx = 0;
*rdx = 0;
}


if (vmm_softc->mode == VMM_MODE_RVI) {
/*
 * update %rax. the rest of the registers get updated in
 * svm_enter_guest
	 */
vmcb->v_rax = *rax;
}

return (0);
6364}

6366/*
* vcpu_run_svm
*
* SVM main loop used to run a VCPU.
*
* Parameters:
*  vcpu: The VCPU to run
*  vrp: run parameters
*
* Return values:
*  0: The run loop exited and no help is needed from vmd
*  EAGAIN: The run loop exited and help from vmd is needed
*  EINVAL: an error occurred
*/
6380int
6381vcpu_run_svm(struct vcpu *vcpu, struct vm_run_params *vrp)
6382{
int ret = 0;
struct region_descriptor gdt;
struct cpu_info *ci = NULL((void *)0);
14
←
'ci' initialized to a null pointer value→
uint64_t exit_reason;
struct schedstate_percpu *spc;
uint16_t irq;
struct vmcb *vmcb = (struct vmcb *)vcpu->vc_control_va;

irq = vrp->vrp_irq;

if (vrp->vrp_intr_pending)
15
←
Assuming field 'vrp_intr_pending' is 0→
16
←
Taking false branch→
vcpu->vc_intr = 1;
else
vcpu->vc_intr = 0;

/*
* If we are returning from userspace (vmd) because we exited
* last time, fix up any needed vcpu state first. Which state
* needs to be fixed up depends on what vmd populated in the
* exit data structure.
*/
if (vrp->vrp_continue16.1
Field 'vrp_continue' is 0
) {
17
←
Taking false branch→
switch (vcpu->vc_gueststate.vg_exit_reason) {
case SVM_VMEXIT_IOIO0x7B:
	if (vcpu->vc_exit.vei.vei_dir == VEI_DIR_IN) {
		vcpu->vc_gueststate.vg_rax =
		    vcpu->vc_exit.vei.vei_data;
		vmcb->v_rax = vcpu->vc_gueststate.vg_rax;
	}
	vcpu->vc_gueststate.vg_rip =
	    vcpu->vc_exit.vrs.vrs_gprs[VCPU_REGS_RIP16];
	vmcb->v_rip = vcpu->vc_gueststate.vg_rip;
	break;
case SVM_VMEXIT_NPF0x400:
	ret = vcpu_writeregs_svm(vcpu, VM_RWREGS_GPRS0x1,
	    &vcpu->vc_exit.vrs);
	if (ret) {
		printf("%s: vm %d vcpu %d failed to update "
		    "registers\n", __func__,
		    vcpu->vc_parent->vm_id, vcpu->vc_id);
		return (EINVAL22);
	}
	break;
}
memset(&vcpu->vc_exit, 0, sizeof(vcpu->vc_exit))__builtin_memset((&vcpu->vc_exit), (0), (sizeof(vcpu->
vc_exit)));
}

while (ret == 0) {
18
←
Loop condition is true.  Entering loop body→
vmm_update_pvclock(vcpu);
19
←
Calling 'vmm_update_pvclock'→
22
←
Returning from 'vmm_update_pvclock'→
if (ci != curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})) {
23
←
Assuming the condition is false→
	/*
	 * We are launching for the first time, or we are
	 * resuming from a different pcpu, so we need to
	 * reset certain pcpu-specific values.
	 */
	ci = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
	setregion(&gdt, ci->ci_gdt, GDT_SIZE((6 << 3) + (1 << 4)) - 1);

	if (ci != vcpu->vc_last_pcpu) {
		/*
		 * Flush TLB by guest ASID if feature
		 * available, flush entire TLB if not.
		 */
		if (ci->ci_vmm_cap.vcc_svm.svm_flush_by_asid)
			vmcb->v_tlb_control =
			    SVM_TLB_CONTROL_FLUSH_ASID3;
		else
			vmcb->v_tlb_control =
			    SVM_TLB_CONTROL_FLUSH_ALL1;

		svm_set_dirty(vcpu, SVM_CLEANBITS_ALL((1 << 0) | (1 << 1) | (1 << 2) | (1 <<
 3) | (1 << 4) | (1 << 5) | (1 << 6) | (1 <<
 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 <<
 11) ));
	}

	vcpu->vc_last_pcpu = ci;

	if (gdt.rd_base == 0) {
		ret = EINVAL22;
		break;
	}
}

/* Handle vmd(8) injected interrupts */
/* Is there an interrupt pending injection? */
if (irq != 0xFFFF && vcpu->vc_irqready) {
24
←
Assuming 'irq' is equal to 65535→
	vmcb->v_eventinj = (irq & 0xFF) | (1U << 31);
	irq = 0xFFFF;
}

/* Inject event if present */
if (vcpu->vc_event != 0) {
25
←
Assuming field 'vc_event' is equal to 0→
26
←
Taking false branch→
	DPRINTF("%s: inject event %d\n", __func__,
	    vcpu->vc_event);
	vmcb->v_eventinj = 0;
	/* Set the "Event Valid" flag for certain vectors */
	switch (vcpu->vc_event & 0xFF) {
		case VMM_EX_DF8:
		case VMM_EX_TS10:
		case VMM_EX_NP11:
		case VMM_EX_SS12:
		case VMM_EX_GP13:
		case VMM_EX_PF14:
		case VMM_EX_AC17:
			vmcb->v_eventinj |= (1ULL << 11);
	}
	vmcb->v_eventinj |= (vcpu->vc_event) | (1U << 31);
	vmcb->v_eventinj |= (3ULL << 8); /* Exception */
	vcpu->vc_event = 0;
}

TRACEPOINT(vmm, guest_enter, vcpu, vrp)do { extern struct dt_probe (dt_static_vmm_guest_enter); struct
 dt_probe *dtp = &(dt_static_vmm_guest_enter); if (__builtin_expect
(((dt_tracing) != 0), 0) && __builtin_expect(((dtp->
dtp_recording) != 0), 0)) { struct dt_provider *dtpv = dtp->
dtp_prov; dtpv->dtpv_enter(dtpv, dtp, vcpu, vrp); } } while
 (0);
27
←
Assuming 'dt_tracing' is equal to 0→
28
←
Loop condition is false.  Exiting loop→

/* Start / resume the VCPU */
/* Disable interrupts and save the current host FPU state. */
clgi();
if ((ret = vmm_fpurestore(vcpu))) {
29
←
Calling 'vmm_fpurestore'→
36
←
Returning from 'vmm_fpurestore'→
37
←
Assuming 'ret' is 0→
38
←
Taking false branch→
	stgi();
	break;
}

/* Restore any guest PKRU state. */
if (vmm_softc->sc_md.pkru_enabled)
39
←
Assuming field 'pkru_enabled' is 0→
	wrpkru(vcpu->vc_pkru);

KASSERT(vmcb->v_intercept1 & SVM_INTERCEPT_INTR)((vmcb->v_intercept1 & (1UL << 0)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/arch/amd64/amd64/vmm_machdep.c"
, 6506, "vmcb->v_intercept1 & SVM_INTERCEPT_INTR"));
40
←
Taking false branch→
41
←
Assuming the condition is true→
42
←
'?' condition is true→
wrmsr(MSR_AMD_VM_HSAVE_PA0xc0010117, vcpu->vc_svm_hsa_pa);

ret = svm_enter_guest(vcpu->vc_control_pa,
    &vcpu->vc_gueststate, &gdt);

/* Restore host PKRU state. */
if (vmm_softc->sc_md.pkru_enabled) {
43
←
Assuming field 'pkru_enabled' is 0→
44
←
Taking false branch→
	vcpu->vc_pkru = rdpkru(0);
	wrpkru(PGK_VALUE0xfffffffc);
}

/*
 * On exit, interrupts are disabled, and we are running with
 * the guest FPU state still possibly on the CPU. Save the FPU
 * state before re-enabling interrupts.
 */
vmm_fpusave(vcpu);

/*
 * Enable interrupts now. Note that if the exit was due to INTR
 * (external interrupt), the interrupt will be processed now.
 */
stgi();

vcpu->vc_gueststate.vg_rip = vmcb->v_rip;
vmcb->v_tlb_control = SVM_TLB_CONTROL_FLUSH_NONE0;
svm_set_clean(vcpu, SVM_CLEANBITS_ALL((1 << 0) | (1 << 1) | (1 << 2) | (1 <<
 3) | (1 << 4) | (1 << 5) | (1 << 6) | (1 <<
 7) | (1 << 8) | (1 << 9) | (1 << 10) | (1 <<
 11) ));

/* If we exited successfully ... */
if (ret == 0) {
45
←
Assuming 'ret' is equal to 0→
46
←
Taking true branch→
	exit_reason = vmcb->v_exitcode;
	vcpu->vc_gueststate.vg_exit_reason = exit_reason;
	TRACEPOINT(vmm, guest_exit, vcpu, vrp, exit_reason)do { extern struct dt_probe (dt_static_vmm_guest_exit); struct
 dt_probe *dtp = &(dt_static_vmm_guest_exit); if (__builtin_expect
(((dt_tracing) != 0), 0) && __builtin_expect(((dtp->
dtp_recording) != 0), 0)) { struct dt_provider *dtpv = dtp->
dtp_prov; dtpv->dtpv_enter(dtpv, dtp, vcpu, vrp, exit_reason
); } } while (0);
47
←
Assuming 'dt_tracing' is equal to 0→
48
←
Loop condition is false.  Exiting loop→

	vcpu->vc_gueststate.vg_rflags = vmcb->v_rflags;

	/*
	 * Handle the exit. This will alter "ret" to EAGAIN if
	 * the exit handler determines help from vmd is needed.
	 */
	ret = svm_handle_exit(vcpu);
49
←
Calling 'svm_handle_exit'→
57
←
Returning from 'svm_handle_exit'→

	if (vcpu->vc_gueststate.vg_rflags & PSL_I0x00000200)
58
←
Assuming the condition is true→
59
←
Taking true branch→
		vcpu->vc_irqready = 1;
	else
		vcpu->vc_irqready = 0;

	/*
	 * If not ready for interrupts, but interrupts pending,
	 * enable interrupt window exiting.
	 */
	if (vcpu->vc_irqready59.1
Field 'vc_irqready' is not equal to 0
 == 0 && vcpu->vc_intr) {
		vmcb->v_intercept1 |= SVM_INTERCEPT_VINTR(1UL << 4);
		vmcb->v_irq = 1;
		vmcb->v_intr_misc = SVM_INTR_MISC_V_IGN_TPR0x10;
		vmcb->v_intr_vector = 0;
		svm_set_dirty(vcpu, SVM_CLEANBITS_TPR(1 << 3) |
		    SVM_CLEANBITS_I(1 << 0));
	}

	/*
	 * Exit to vmd if we are terminating, failed to enter,
	 * or need help (device I/O)
	 */
	if (ret59.2
'ret' is 0
 || vcpu_must_stop(vcpu))
60
←
Assuming the condition is false→
		break;

	if (vcpu->vc_intr && vcpu->vc_irqready) {
61
←
Assuming field 'vc_intr' is 0→
		ret = EAGAIN35;
		break;
	}

	/* Check if we should yield - don't hog the cpu */
	spc = &ci->ci_schedstate;
62
←
Null pointer value stored to 'spc'→
	if (spc->spc_schedflags & SPCF_SHOULDYIELD0x0002)
63
←
Access to field 'spc_schedflags' results in a dereference of a null pointer (loaded from variable 'spc')
		break;
}
}

/*
* We are heading back to userspace (vmd), either because we need help
* handling an exit, a guest interrupt is pending, or we failed in some
* way to enter the guest. Copy the guest registers to the exit struct
* and return to vmd.
*/
if (vcpu_readregs_svm(vcpu, VM_RWREGS_ALL(0x1 | 0x2 | 0x4 | 0x8 | 0x10), &vcpu->vc_exit.vrs))
ret = EINVAL22;

return (ret);
6596}

6598/*
* vmm_alloc_vpid
*
* Sets the memory location pointed to by "vpid" to the next available VPID
* or ASID.
*
* Parameters:
*  vpid: Pointer to location to receive the next VPID/ASID
*
* Return Values:
*  0: The operation completed successfully
*  ENOMEM: No VPIDs/ASIDs were available. Content of 'vpid' is unchanged.
*/
6611int
6612vmm_alloc_vpid(uint16_t *vpid)
6613{
uint16_t i;
uint8_t idx, bit;
struct vmm_softc *sc = vmm_softc;

rw_enter_write(&vmm_softc->vpid_lock);
for (i = 1; i <= sc->max_vpid; i++) {
idx = i / 8;
bit = i - (idx * 8);

if (!(sc->vpids[idx] & (1 << bit))) {
	sc->vpids[idx] |= (1 << bit);
	*vpid = i;
	DPRINTF("%s: allocated VPID/ASID %d\n", __func__,
	    i);
	rw_exit_write(&vmm_softc->vpid_lock);
	return 0;
}
}

printf("%s: no available %ss\n", __func__,
   (sc->mode == VMM_MODE_EPT) ? "VPID" :
   "ASID");

rw_exit_write(&vmm_softc->vpid_lock);
return ENOMEM12;
6639}

6641/*
* vmm_free_vpid
*
* Frees the VPID/ASID id supplied in "vpid".
*
* Parameters:
*  vpid: VPID/ASID to free.
*/
6649void
6650vmm_free_vpid(uint16_t vpid)
6651{
uint8_t idx, bit;
struct vmm_softc *sc = vmm_softc;

rw_enter_write(&vmm_softc->vpid_lock);
idx = vpid / 8;
bit = vpid - (idx * 8);
sc->vpids[idx] &= ~(1 << bit);

DPRINTF("%s: freed VPID/ASID %d\n", __func__, vpid);
rw_exit_write(&vmm_softc->vpid_lock);
6662}


6665/* vmm_gpa_is_valid
*
* Check if the given gpa is within guest memory space.
*
* Parameters:
* 	vcpu: The virtual cpu we are running on.
* 	gpa: The address to check.
* 	obj_size: The size of the object assigned to gpa
*
* Return values:
* 	1: gpa is within the memory ranges allocated for the vcpu
* 	0: otherwise
*/
6678int
6679vmm_gpa_is_valid(struct vcpu *vcpu, paddr_t gpa, size_t obj_size)
6680{
struct vm *vm = vcpu->vc_parent;
struct vm_mem_range *vmr;
size_t i;

for (i = 0; i < vm->vm_nmemranges; ++i) {
vmr = &vm->vm_memranges[i];
if (vmr->vmr_size >= obj_size &&
    vmr->vmr_gpa <= gpa &&
    gpa < (vmr->vmr_gpa + vmr->vmr_size - obj_size)) {
    return 1;
}
}
return 0;
6694}

6696void
6697vmm_init_pvclock(struct vcpu *vcpu, paddr_t gpa)
6698{
paddr_t pvclock_gpa = gpa & 0xFFFFFFFFFFFFFFF0;
if (!vmm_gpa_is_valid(vcpu, pvclock_gpa,
       sizeof(struct pvclock_time_info))) {
/* XXX: Kill guest? */
vmm_inject_gp(vcpu);
return;
}

/* XXX: handle case when this struct goes over page boundaries */
if ((pvclock_gpa & PAGE_MASK((1 << 12) - 1)) + sizeof(struct pvclock_time_info) >
   PAGE_SIZE(1 << 12)) {
vmm_inject_gp(vcpu);
return;
}

vcpu->vc_pvclock_system_gpa = gpa;
if (tsc_frequency > 0)
vcpu->vc_pvclock_system_tsc_mul =
    (int) ((1000000000L << 20) / tsc_frequency);
else
vcpu->vc_pvclock_system_tsc_mul = 0;
vmm_update_pvclock(vcpu);
6721}

6723int
6724vmm_update_pvclock(struct vcpu *vcpu)
6725{
struct pvclock_time_info *pvclock_ti;
struct timespec tv;
struct vm *vm = vcpu->vc_parent;
paddr_t pvclock_hpa, pvclock_gpa;

if (vcpu->vc_pvclock_system_gpa & PVCLOCK_SYSTEM_TIME_ENABLE0x01) {
20
←
Assuming the condition is false→
21
←
Taking false branch→
pvclock_gpa = vcpu->vc_pvclock_system_gpa & 0xFFFFFFFFFFFFFFF0;
if (!pmap_extract(vm->vm_map->pmap, pvclock_gpa, &pvclock_hpa))
	return (EINVAL22);
pvclock_ti = (void*) PMAP_DIRECT_MAP(pvclock_hpa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pvclock_hpa));

/* START next cycle (must be odd) */
pvclock_ti->ti_version =
    (++vcpu->vc_pvclock_version << 1) | 0x1;

pvclock_ti->ti_tsc_timestamp = rdtsc();
nanotime(&tv);
pvclock_ti->ti_system_time =
    tv.tv_sec * 1000000000L + tv.tv_nsec;
pvclock_ti->ti_tsc_shift = 12;
pvclock_ti->ti_tsc_to_system_mul =
    vcpu->vc_pvclock_system_tsc_mul;
pvclock_ti->ti_flags = PVCLOCK_FLAG_TSC_STABLE0x01;

/* END (must be even) */
pvclock_ti->ti_version &= ~0x1;
}
return (0);
6754}

6756int
6757vmm_pat_is_valid(uint64_t pat)
6758{
int i;
uint8_t *byte = (uint8_t *)&pat;

/* Intel SDM Vol 3A, 11.12.2: 0x02, 0x03, and 0x08-0xFF result in #GP */
for (i = 0; i < 8; i++) {
if (byte[i] == 0x02 || byte[i] == 0x03 || byte[i] > 0x07) {
	DPRINTF("%s: invalid pat %llx\n", __func__, pat);
	return 0;
}
}

return 1;
6771}

6773/*
* vmx_exit_reason_decode
*
* Returns a human readable string describing exit type 'code'
*/
6778const char *
6779vmx_exit_reason_decode(uint32_t code)
6780{
switch (code) {
case VMX_EXIT_NMI0: return "NMI";
case VMX_EXIT_EXTINT1: return "External interrupt";
case VMX_EXIT_TRIPLE_FAULT2: return "Triple fault";
case VMX_EXIT_INIT3: return "INIT signal";
case VMX_EXIT_SIPI4: return "SIPI signal";
case VMX_EXIT_IO_SMI5: return "I/O SMI";
case VMX_EXIT_OTHER_SMI6: return "other SMI";
case VMX_EXIT_INT_WINDOW7: return "Interrupt window";
case VMX_EXIT_NMI_WINDOW8: return "NMI window";
case VMX_EXIT_TASK_SWITCH9: return "Task switch";
case VMX_EXIT_CPUID10: return "CPUID instruction";
case VMX_EXIT_GETSEC11: return "GETSEC instruction";
case VMX_EXIT_HLT12: return "HLT instruction";
case VMX_EXIT_INVD13: return "INVD instruction";
case VMX_EXIT_INVLPG14: return "INVLPG instruction";
case VMX_EXIT_RDPMC15: return "RDPMC instruction";
case VMX_EXIT_RDTSC16: return "RDTSC instruction";
case VMX_EXIT_RSM17: return "RSM instruction";
case VMX_EXIT_VMCALL18: return "VMCALL instruction";
case VMX_EXIT_VMCLEAR19: return "VMCLEAR instruction";
case VMX_EXIT_VMLAUNCH20: return "VMLAUNCH instruction";
case VMX_EXIT_VMPTRLD21: return "VMPTRLD instruction";
case VMX_EXIT_VMPTRST22: return "VMPTRST instruction";
case VMX_EXIT_VMREAD23: return "VMREAD instruction";
case VMX_EXIT_VMRESUME24: return "VMRESUME instruction";
case VMX_EXIT_VMWRITE25: return "VMWRITE instruction";
case VMX_EXIT_VMXOFF26: return "VMXOFF instruction";
case VMX_EXIT_VMXON27: return "VMXON instruction";
case VMX_EXIT_CR_ACCESS28: return "CR access";
case VMX_EXIT_MOV_DR29: return "MOV DR instruction";
case VMX_EXIT_IO30: return "I/O instruction";
case VMX_EXIT_RDMSR31: return "RDMSR instruction";
case VMX_EXIT_WRMSR32: return "WRMSR instruction";
case VMX_EXIT_ENTRY_FAILED_GUEST_STATE33: return "guest state invalid";
case VMX_EXIT_ENTRY_FAILED_MSR_LOAD34: return "MSR load failed";
case VMX_EXIT_MWAIT36: return "MWAIT instruction";
case VMX_EXIT_MTF37: return "monitor trap flag";
case VMX_EXIT_MONITOR39: return "MONITOR instruction";
case VMX_EXIT_PAUSE40: return "PAUSE instruction";
case VMX_EXIT_ENTRY_FAILED_MCE41: return "MCE during entry";
case VMX_EXIT_TPR_BELOW_THRESHOLD43: return "TPR below threshold";
case VMX_EXIT_APIC_ACCESS44: return "APIC access";
case VMX_EXIT_VIRTUALIZED_EOI45: return "virtualized EOI";
case VMX_EXIT_GDTR_IDTR46: return "GDTR/IDTR access";
case VMX_EXIT_LDTR_TR47: return "LDTR/TR access";
case VMX_EXIT_EPT_VIOLATION48: return "EPT violation";
case VMX_EXIT_EPT_MISCONFIGURATION49: return "EPT misconfiguration";
case VMX_EXIT_INVEPT50: return "INVEPT instruction";
case VMX_EXIT_RDTSCP51: return "RDTSCP instruction";
case VMX_EXIT_VMX_PREEMPTION_TIMER_EXPIRED52:
   return "preemption timer expired";
case VMX_EXIT_INVVPID53: return "INVVPID instruction";
case VMX_EXIT_WBINVD54: return "WBINVD instruction";
case VMX_EXIT_XSETBV55: return "XSETBV instruction";
case VMX_EXIT_APIC_WRITE56: return "APIC write";
case VMX_EXIT_RDRAND57: return "RDRAND instruction";
case VMX_EXIT_INVPCID58: return "INVPCID instruction";
case VMX_EXIT_VMFUNC59: return "VMFUNC instruction";
case VMX_EXIT_RDSEED61: return "RDSEED instruction";
case VMX_EXIT_XSAVES63: return "XSAVES instruction";
case VMX_EXIT_XRSTORS64: return "XRSTORS instruction";
default: return "unknown";
}
6845}

6847/*
* svm_exit_reason_decode
*
* Returns a human readable string describing exit type 'code'
*/
6852const char *
6853svm_exit_reason_decode(uint32_t code)
6854{
switch (code) {
case SVM_VMEXIT_CR0_READ0x00: return "CR0 read";		/* 0x00 */
case SVM_VMEXIT_CR1_READ0x01: return "CR1 read";		/* 0x01 */
case SVM_VMEXIT_CR2_READ0x02: return "CR2 read";		/* 0x02 */
case SVM_VMEXIT_CR3_READ0x03: return "CR3 read";		/* 0x03 */
case SVM_VMEXIT_CR4_READ0x04: return "CR4 read";		/* 0x04 */
case SVM_VMEXIT_CR5_READ0x05: return "CR5 read";		/* 0x05 */
case SVM_VMEXIT_CR6_READ0x06: return "CR6 read";		/* 0x06 */
case SVM_VMEXIT_CR7_READ0x07: return "CR7 read";		/* 0x07 */
case SVM_VMEXIT_CR8_READ0x08: return "CR8 read";		/* 0x08 */
case SVM_VMEXIT_CR9_READ0x09: return "CR9 read";		/* 0x09 */
case SVM_VMEXIT_CR10_READ0x0A: return "CR10 read";		/* 0x0A */
case SVM_VMEXIT_CR11_READ0x0B: return "CR11 read";		/* 0x0B */
case SVM_VMEXIT_CR12_READ0x0C: return "CR12 read";		/* 0x0C */
case SVM_VMEXIT_CR13_READ0x0D: return "CR13 read";		/* 0x0D */
case SVM_VMEXIT_CR14_READ0x0E: return "CR14 read";		/* 0x0E */
case SVM_VMEXIT_CR15_READ0x0F: return "CR15 read";		/* 0x0F */
case SVM_VMEXIT_CR0_WRITE0x10: return "CR0 write";		/* 0x10 */
case SVM_VMEXIT_CR1_WRITE0x11: return "CR1 write";		/* 0x11 */
case SVM_VMEXIT_CR2_WRITE0x12: return "CR2 write";		/* 0x12 */
case SVM_VMEXIT_CR3_WRITE0x13: return "CR3 write";		/* 0x13 */
case SVM_VMEXIT_CR4_WRITE0x14: return "CR4 write";		/* 0x14 */
case SVM_VMEXIT_CR5_WRITE0x15: return "CR5 write";		/* 0x15 */
case SVM_VMEXIT_CR6_WRITE0x16: return "CR6 write";		/* 0x16 */
case SVM_VMEXIT_CR7_WRITE0x17: return "CR7 write";		/* 0x17 */
case SVM_VMEXIT_CR8_WRITE0x18: return "CR8 write";		/* 0x18 */
case SVM_VMEXIT_CR9_WRITE0x19: return "CR9 write";		/* 0x19 */
case SVM_VMEXIT_CR10_WRITE0x1A: return "CR10 write";	/* 0x1A */
case SVM_VMEXIT_CR11_WRITE0x1B: return "CR11 write";	/* 0x1B */
case SVM_VMEXIT_CR12_WRITE0x1C: return "CR12 write";	/* 0x1C */
case SVM_VMEXIT_CR13_WRITE0x1D: return "CR13 write";	/* 0x1D */
case SVM_VMEXIT_CR14_WRITE0x1E: return "CR14 write";	/* 0x1E */
case SVM_VMEXIT_CR15_WRITE0x1F: return "CR15 write";	/* 0x1F */
case SVM_VMEXIT_DR0_READ0x20: return "DR0 read";	 	/* 0x20 */
case SVM_VMEXIT_DR1_READ0x21: return "DR1 read";		/* 0x21 */
case SVM_VMEXIT_DR2_READ0x22: return "DR2 read";		/* 0x22 */
case SVM_VMEXIT_DR3_READ0x23: return "DR3 read";		/* 0x23 */
case SVM_VMEXIT_DR4_READ0x24: return "DR4 read";		/* 0x24 */
case SVM_VMEXIT_DR5_READ0x25: return "DR5 read";		/* 0x25 */
case SVM_VMEXIT_DR6_READ0x26: return "DR6 read";		/* 0x26 */
case SVM_VMEXIT_DR7_READ0x27: return "DR7 read";		/* 0x27 */
case SVM_VMEXIT_DR8_READ0x28: return "DR8 read";		/* 0x28 */
case SVM_VMEXIT_DR9_READ0x29: return "DR9 read";		/* 0x29 */
case SVM_VMEXIT_DR10_READ0x2A: return "DR10 read";		/* 0x2A */
case SVM_VMEXIT_DR11_READ0x2B: return "DR11 read";		/* 0x2B */
case SVM_VMEXIT_DR12_READ0x2C: return "DR12 read";		/* 0x2C */
case SVM_VMEXIT_DR13_READ0x2D: return "DR13 read";		/* 0x2D */
case SVM_VMEXIT_DR14_READ0x2E: return "DR14 read";		/* 0x2E */
case SVM_VMEXIT_DR15_READ0x2F: return "DR15 read";		/* 0x2F */
case SVM_VMEXIT_DR0_WRITE0x30: return "DR0 write";		/* 0x30 */
case SVM_VMEXIT_DR1_WRITE0x31: return "DR1 write";		/* 0x31 */
case SVM_VMEXIT_DR2_WRITE0x32: return "DR2 write";		/* 0x32 */
case SVM_VMEXIT_DR3_WRITE0x33: return "DR3 write";		/* 0x33 */
case SVM_VMEXIT_DR4_WRITE0x34: return "DR4 write";		/* 0x34 */
case SVM_VMEXIT_DR5_WRITE0x35: return "DR5 write";		/* 0x35 */
case SVM_VMEXIT_DR6_WRITE0x36: return "DR6 write";		/* 0x36 */
case SVM_VMEXIT_DR7_WRITE0x37: return "DR7 write";		/* 0x37 */
case SVM_VMEXIT_DR8_WRITE0x38: return "DR8 write";		/* 0x38 */
case SVM_VMEXIT_DR9_WRITE0x39: return "DR9 write";		/* 0x39 */
case SVM_VMEXIT_DR10_WRITE0x3A: return "DR10 write";	/* 0x3A */
case SVM_VMEXIT_DR11_WRITE0x3B: return "DR11 write";	/* 0x3B */
case SVM_VMEXIT_DR12_WRITE0x3C: return "DR12 write";	/* 0x3C */
case SVM_VMEXIT_DR13_WRITE0x3D: return "DR13 write";	/* 0x3D */
case SVM_VMEXIT_DR14_WRITE0x3E: return "DR14 write";	/* 0x3E */
case SVM_VMEXIT_DR15_WRITE0x3F: return "DR15 write";	/* 0x3F */
case SVM_VMEXIT_EXCP00x40: return "Exception 0x00";		/* 0x40 */
case SVM_VMEXIT_EXCP10x41: return "Exception 0x01";		/* 0x41 */
case SVM_VMEXIT_EXCP20x42: return "Exception 0x02";		/* 0x42 */
case SVM_VMEXIT_EXCP30x43: return "Exception 0x03";		/* 0x43 */
case SVM_VMEXIT_EXCP40x44: return "Exception 0x04";		/* 0x44 */
case SVM_VMEXIT_EXCP50x45: return "Exception 0x05";		/* 0x45 */
case SVM_VMEXIT_EXCP60x46: return "Exception 0x06";		/* 0x46 */
case SVM_VMEXIT_EXCP70x47: return "Exception 0x07";		/* 0x47 */
case SVM_VMEXIT_EXCP80x48: return "Exception 0x08";		/* 0x48 */
case SVM_VMEXIT_EXCP90x49: return "Exception 0x09";		/* 0x49 */
case SVM_VMEXIT_EXCP100x4A: return "Exception 0x0A";	/* 0x4A */
case SVM_VMEXIT_EXCP110x4B: return "Exception 0x0B";	/* 0x4B */
case SVM_VMEXIT_EXCP120x4C: return "Exception 0x0C";	/* 0x4C */
case SVM_VMEXIT_EXCP130x4D: return "Exception 0x0D";	/* 0x4D */
case SVM_VMEXIT_EXCP140x4E: return "Exception 0x0E";	/* 0x4E */
case SVM_VMEXIT_EXCP150x4F: return "Exception 0x0F";	/* 0x4F */
case SVM_VMEXIT_EXCP160x50: return "Exception 0x10";	/* 0x50 */
case SVM_VMEXIT_EXCP170x51: return "Exception 0x11";	/* 0x51 */
case SVM_VMEXIT_EXCP180x52: return "Exception 0x12";	/* 0x52 */
case SVM_VMEXIT_EXCP190x53: return "Exception 0x13";	/* 0x53 */
case SVM_VMEXIT_EXCP200x54: return "Exception 0x14";	/* 0x54 */
case SVM_VMEXIT_EXCP210x55: return "Exception 0x15";	/* 0x55 */
case SVM_VMEXIT_EXCP220x56: return "Exception 0x16";	/* 0x56 */
case SVM_VMEXIT_EXCP230x57: return "Exception 0x17";	/* 0x57 */
case SVM_VMEXIT_EXCP240x58: return "Exception 0x18";	/* 0x58 */
case SVM_VMEXIT_EXCP250x59: return "Exception 0x19";	/* 0x59 */
case SVM_VMEXIT_EXCP260x5A: return "Exception 0x1A";	/* 0x5A */
case SVM_VMEXIT_EXCP270x5B: return "Exception 0x1B";	/* 0x5B */
case SVM_VMEXIT_EXCP280x5C: return "Exception 0x1C";	/* 0x5C */
case SVM_VMEXIT_EXCP290x5D: return "Exception 0x1D";	/* 0x5D */
case SVM_VMEXIT_EXCP300x5E: return "Exception 0x1E";	/* 0x5E */
case SVM_VMEXIT_EXCP310x5F: return "Exception 0x1F";	/* 0x5F */
case SVM_VMEXIT_INTR0x60: return "External interrupt";	/* 0x60 */
case SVM_VMEXIT_NMI0x61: return "NMI";			/* 0x61 */
case SVM_VMEXIT_SMI0x62: return "SMI";			/* 0x62 */
case SVM_VMEXIT_INIT0x63: return "INIT";			/* 0x63 */
case SVM_VMEXIT_VINTR0x64: return "Interrupt window";	/* 0x64 */
case SVM_VMEXIT_CR0_SEL_WRITE0x65: return "Sel CR0 write";	/* 0x65 */
case SVM_VMEXIT_IDTR_READ0x66: return "IDTR read";		/* 0x66 */
case SVM_VMEXIT_GDTR_READ0x67: return "GDTR read";		/* 0x67 */
case SVM_VMEXIT_LDTR_READ0x68: return "LDTR read";		/* 0x68 */
case SVM_VMEXIT_TR_READ0x69: return "TR read";		/* 0x69 */
case SVM_VMEXIT_IDTR_WRITE0x6A: return "IDTR write";	/* 0x6A */
case SVM_VMEXIT_GDTR_WRITE0x6B: return "GDTR write";	/* 0x6B */
case SVM_VMEXIT_LDTR_WRITE0x6C: return "LDTR write";	/* 0x6C */
case SVM_VMEXIT_TR_WRITE0x6D: return "TR write";		/* 0x6D */
case SVM_VMEXIT_RDTSC0x6E: return "RDTSC instruction";	/* 0x6E */
case SVM_VMEXIT_RDPMC0x6F: return "RDPMC instruction";	/* 0x6F */
case SVM_VMEXIT_PUSHF0x70: return "PUSHF instruction";	/* 0x70 */
case SVM_VMEXIT_POPF0x71: return "POPF instruction";	/* 0x71 */
case SVM_VMEXIT_CPUID0x72: return "CPUID instruction";	/* 0x72 */
case SVM_VMEXIT_RSM0x73: return "RSM instruction";		/* 0x73 */
case SVM_VMEXIT_IRET0x74: return "IRET instruction";	/* 0x74 */
case SVM_VMEXIT_SWINT0x75: return "SWINT instruction";	/* 0x75 */
case SVM_VMEXIT_INVD0x76: return "INVD instruction";	/* 0x76 */
case SVM_VMEXIT_PAUSE0x77: return "PAUSE instruction";	/* 0x77 */
case SVM_VMEXIT_HLT0x78: return "HLT instruction";		/* 0x78 */
case SVM_VMEXIT_INVLPG0x79: return "INVLPG instruction";	/* 0x79 */
case SVM_VMEXIT_INVLPGA0x7A: return "INVLPGA instruction";	/* 0x7A */
case SVM_VMEXIT_IOIO0x7B: return "I/O instruction";		/* 0x7B */
case SVM_VMEXIT_MSR0x7C: return "RDMSR/WRMSR instruction";	/* 0x7C */
case SVM_VMEXIT_TASK_SWITCH0x7D: return "Task switch";	/* 0x7D */
case SVM_VMEXIT_FERR_FREEZE0x7E: return "FERR_FREEZE";	/* 0x7E */
case SVM_VMEXIT_SHUTDOWN0x7F: return "Triple fault";	/* 0x7F */
case SVM_VMEXIT_VMRUN0x80: return "VMRUN instruction";	/* 0x80 */
case SVM_VMEXIT_VMMCALL0x81: return "VMMCALL instruction";	/* 0x81 */
case SVM_VMEXIT_VMLOAD0x82: return "VMLOAD instruction";	/* 0x82 */
case SVM_VMEXIT_VMSAVE0x83: return "VMSAVE instruction";	/* 0x83 */
case SVM_VMEXIT_STGI0x84: return "STGI instruction";	/* 0x84 */
case SVM_VMEXIT_CLGI0x85: return "CLGI instruction";	/* 0x85 */
case SVM_VMEXIT_SKINIT0x86: return "SKINIT instruction";	/* 0x86 */
case SVM_VMEXIT_RDTSCP0x87: return "RDTSCP instruction";	/* 0x87 */
case SVM_VMEXIT_ICEBP0x88: return "ICEBP instruction";	/* 0x88 */
case SVM_VMEXIT_WBINVD0x89: return "WBINVD instruction";	/* 0x89 */
case SVM_VMEXIT_MONITOR0x8A: return "MONITOR instruction";	/* 0x8A */
case SVM_VMEXIT_MWAIT0x8B: return "MWAIT instruction";	/* 0x8B */
case SVM_VMEXIT_MWAIT_CONDITIONAL0x8C: return "Cond MWAIT";	/* 0x8C */
case SVM_VMEXIT_NPF0x400: return "NPT violation";		/* 0x400 */
default: return "unknown";
}
7000}

7002/*
* vmx_instruction_error_decode
*
* Returns a human readable string describing the instruction error in 'code'
*/
7007const char *
7008vmx_instruction_error_decode(uint32_t code)
7009{
switch (code) {
case 1: return "VMCALL: unsupported in VMX root";
case 2: return "VMCLEAR: invalid paddr";
case 3: return "VMCLEAR: VMXON pointer";
case 4: return "VMLAUNCH: non-clear VMCS";
case 5: return "VMRESUME: non-launched VMCS";
case 6: return "VMRESUME: executed after VMXOFF";
case 7: return "VM entry: invalid control field(s)";
case 8: return "VM entry: invalid host state field(s)";
case 9: return "VMPTRLD: invalid paddr";
case 10: return "VMPTRLD: VMXON pointer";
case 11: return "VMPTRLD: incorrect VMCS revid";
case 12: return "VMREAD/VMWRITE: unsupported VMCS field";
case 13: return "VMWRITE: RO VMCS field";
case 15: return "VMXON: unsupported in VMX root";
case 20: return "VMCALL: invalid VM exit control fields";
case 26: return "VM entry: blocked by MOV SS";
case 28: return "Invalid operand to INVEPT/INVVPID";
case 0x80000021: return "VM entry: invalid guest state";
case 0x80000022: return "VM entry: failure due to MSR loading";
case 0x80000029: return "VM entry: machine-check event";
default: return "unknown";
}
7033}

7035/*
* vcpu_state_decode
*
* Returns a human readable string describing the vcpu state in 'state'.
*/
7040const char *
7041vcpu_state_decode(u_int state)
7042{
switch (state) {
case VCPU_STATE_STOPPED: return "stopped";
case VCPU_STATE_RUNNING: return "running";
case VCPU_STATE_REQTERM: return "requesting termination";
case VCPU_STATE_TERMINATED: return "terminated";
case VCPU_STATE_UNKNOWN: return "unknown";
default: return "invalid";
}
7051}

7053#ifdef VMM_DEBUG
7054/*
* dump_vcpu
*
* Dumps the VMX capabilities of vcpu 'vcpu'
*/
7059void
7060dump_vcpu(struct vcpu *vcpu)
7061{
printf("vcpu @ %p\n", vcpu);
printf("    parent vm @ %p\n", vcpu->vc_parent);
printf("    mode: ");
if (vcpu->vc_virt_mode == VMM_MODE_EPT) {
printf("VMX\n");
printf("    pinbased ctls: 0x%llx\n",
    vcpu->vc_vmx_pinbased_ctls);
printf("    true pinbased ctls: 0x%llx\n",
    vcpu->vc_vmx_true_pinbased_ctls);
CTRL_DUMP(vcpu, PINBASED, EXTERNAL_INT_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "EXTERNAL_INT_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x481, (1ULL << 0), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x481, (1ULL << 0), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PINBASED, NMI_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "NMI_EXITING" , vcpu_vmx_check_cap
(vcpu, 0x481, (1ULL << 3), 1) ? "Yes" : "No", vcpu_vmx_check_cap
(vcpu, 0x481, (1ULL << 3), 0) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PINBASED, VIRTUAL_NMIS)printf("     %s: Can set:%s Can clear:%s\n", "VIRTUAL_NMIS" ,
 vcpu_vmx_check_cap(vcpu, 0x481, (1ULL << 5), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x481, (1ULL << 5), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PINBASED, ACTIVATE_VMX_PREEMPTION_TIMER)printf("     %s: Can set:%s Can clear:%s\n", "ACTIVATE_VMX_PREEMPTION_TIMER"
 , vcpu_vmx_check_cap(vcpu, 0x481, (1ULL << 6), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x481, (1ULL << 6), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PINBASED, PROCESS_POSTED_INTERRUPTS)printf("     %s: Can set:%s Can clear:%s\n", "PROCESS_POSTED_INTERRUPTS"
 , vcpu_vmx_check_cap(vcpu, 0x481, (1ULL << 7), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x481, (1ULL << 7), 0
) ? "Yes" : "No");;
printf("    procbased ctls: 0x%llx\n",
    vcpu->vc_vmx_procbased_ctls);
printf("    true procbased ctls: 0x%llx\n",
    vcpu->vc_vmx_true_procbased_ctls);
CTRL_DUMP(vcpu, PROCBASED, INTERRUPT_WINDOW_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "INTERRUPT_WINDOW_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 2), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 2), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, USE_TSC_OFFSETTING)printf("     %s: Can set:%s Can clear:%s\n", "USE_TSC_OFFSETTING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 3), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 3), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, HLT_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "HLT_EXITING" , vcpu_vmx_check_cap
(vcpu, 0x482, (1ULL << 7), 1) ? "Yes" : "No", vcpu_vmx_check_cap
(vcpu, 0x482, (1ULL << 7), 0) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, INVLPG_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "INVLPG_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 9), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 9), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, MWAIT_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "MWAIT_EXITING" ,
 vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 10), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 10), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, RDPMC_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "RDPMC_EXITING" ,
 vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 11), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 11), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, RDTSC_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "RDTSC_EXITING" ,
 vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 12), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 12), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, CR3_LOAD_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "CR3_LOAD_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 15), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 15), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, CR3_STORE_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "CR3_STORE_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 16), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 16), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, CR8_LOAD_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "CR8_LOAD_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 19), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 19), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, CR8_STORE_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "CR8_STORE_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 20), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 20), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, USE_TPR_SHADOW)printf("     %s: Can set:%s Can clear:%s\n", "USE_TPR_SHADOW"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 21), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 21), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, NMI_WINDOW_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "NMI_WINDOW_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 22), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 22), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, MOV_DR_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "MOV_DR_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 23), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 23), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, UNCONDITIONAL_IO_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "UNCONDITIONAL_IO_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 24), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 24), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, USE_IO_BITMAPS)printf("     %s: Can set:%s Can clear:%s\n", "USE_IO_BITMAPS"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 25), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 25), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, MONITOR_TRAP_FLAG)printf("     %s: Can set:%s Can clear:%s\n", "MONITOR_TRAP_FLAG"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 27), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 27), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, USE_MSR_BITMAPS)printf("     %s: Can set:%s Can clear:%s\n", "USE_MSR_BITMAPS"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 28), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 28), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, MONITOR_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "MONITOR_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 29), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 29), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, PROCBASED, PAUSE_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "PAUSE_EXITING" ,
 vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 30), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x482, (1ULL << 30), 0
) ? "Yes" : "No");;
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
    IA32_VMX_ACTIVATE_SECONDARY_CONTROLS(1ULL << 31), 1)) {
	printf("    procbased2 ctls: 0x%llx\n",
	    vcpu->vc_vmx_procbased2_ctls);
	CTRL_DUMP(vcpu, PROCBASED2, VIRTUALIZE_APIC)printf("     %s: Can set:%s Can clear:%s\n", "VIRTUALIZE_APIC"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 0), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 0), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_EPT)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_EPT" , vcpu_vmx_check_cap
(vcpu, 0x48B, (1ULL << 1), 1) ? "Yes" : "No", vcpu_vmx_check_cap
(vcpu, 0x48B, (1ULL << 1), 0) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, DESCRIPTOR_TABLE_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "DESCRIPTOR_TABLE_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 2), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 2), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_RDTSCP)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_RDTSCP" ,
 vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 3), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 3), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, VIRTUALIZE_X2APIC_MODE)printf("     %s: Can set:%s Can clear:%s\n", "VIRTUALIZE_X2APIC_MODE"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 4), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 4), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_VPID)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_VPID" , vcpu_vmx_check_cap
(vcpu, 0x48B, (1ULL << 5), 1) ? "Yes" : "No", vcpu_vmx_check_cap
(vcpu, 0x48B, (1ULL << 5), 0) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, WBINVD_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "WBINVD_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 6), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 6), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, UNRESTRICTED_GUEST)printf("     %s: Can set:%s Can clear:%s\n", "UNRESTRICTED_GUEST"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 7), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 7), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2,printf("     %s: Can set:%s Can clear:%s\n", "APIC_REGISTER_VIRTUALIZATION"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 8), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 8), 0
) ? "Yes" : "No");
	    APIC_REGISTER_VIRTUALIZATION)printf("     %s: Can set:%s Can clear:%s\n", "APIC_REGISTER_VIRTUALIZATION"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 8), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 8), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2,printf("     %s: Can set:%s Can clear:%s\n", "VIRTUAL_INTERRUPT_DELIVERY"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 9), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 9), 0
) ? "Yes" : "No");
	    VIRTUAL_INTERRUPT_DELIVERY)printf("     %s: Can set:%s Can clear:%s\n", "VIRTUAL_INTERRUPT_DELIVERY"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 9), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 9), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, PAUSE_LOOP_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "PAUSE_LOOP_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 10), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 10), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, RDRAND_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "RDRAND_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 11), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 11), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_INVPCID)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_INVPCID"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 12), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 12), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_VM_FUNCTIONS)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_VM_FUNCTIONS"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 13), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 13), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, VMCS_SHADOWING)printf("     %s: Can set:%s Can clear:%s\n", "VMCS_SHADOWING"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 14), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 14), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_ENCLS_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_ENCLS_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 15), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 15), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, RDSEED_EXITING)printf("     %s: Can set:%s Can clear:%s\n", "RDSEED_EXITING"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 16), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 16), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_PML)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_PML" , vcpu_vmx_check_cap
(vcpu, 0x48B, (1ULL << 17), 1) ? "Yes" : "No", vcpu_vmx_check_cap
(vcpu, 0x48B, (1ULL << 17), 0) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, EPT_VIOLATION_VE)printf("     %s: Can set:%s Can clear:%s\n", "EPT_VIOLATION_VE"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 18), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 18), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, CONCEAL_VMX_FROM_PT)printf("     %s: Can set:%s Can clear:%s\n", "CONCEAL_VMX_FROM_PT"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 19), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 19), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_XSAVES_XRSTORS)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_XSAVES_XRSTORS"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 20), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 20), 0
) ? "Yes" : "No");;
	CTRL_DUMP(vcpu, PROCBASED2, ENABLE_TSC_SCALING)printf("     %s: Can set:%s Can clear:%s\n", "ENABLE_TSC_SCALING"
 , vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 25), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x48B, (1ULL << 25), 0
) ? "Yes" : "No");;
}
printf("    entry ctls: 0x%llx\n",
    vcpu->vc_vmx_entry_ctls);
printf("    true entry ctls: 0x%llx\n",
    vcpu->vc_vmx_true_entry_ctls);
CTRL_DUMP(vcpu, ENTRY, LOAD_DEBUG_CONTROLS)printf("     %s: Can set:%s Can clear:%s\n", "LOAD_DEBUG_CONTROLS"
 , vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 2), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 2), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, ENTRY, IA32E_MODE_GUEST)printf("     %s: Can set:%s Can clear:%s\n", "IA32E_MODE_GUEST"
 , vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 9), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 9), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, ENTRY, ENTRY_TO_SMM)printf("     %s: Can set:%s Can clear:%s\n", "ENTRY_TO_SMM" ,
 vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 10), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 10), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, ENTRY, DEACTIVATE_DUAL_MONITOR_TREATMENT)printf("     %s: Can set:%s Can clear:%s\n", "DEACTIVATE_DUAL_MONITOR_TREATMENT"
 , vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 11), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 11), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, ENTRY, LOAD_IA32_PERF_GLOBAL_CTRL_ON_ENTRY)printf("     %s: Can set:%s Can clear:%s\n", "LOAD_IA32_PERF_GLOBAL_CTRL_ON_ENTRY"
 , vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 13), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 13), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, ENTRY, LOAD_IA32_PAT_ON_ENTRY)printf("     %s: Can set:%s Can clear:%s\n", "LOAD_IA32_PAT_ON_ENTRY"
 , vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 14), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 14), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, ENTRY, LOAD_IA32_EFER_ON_ENTRY)printf("     %s: Can set:%s Can clear:%s\n", "LOAD_IA32_EFER_ON_ENTRY"
 , vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 15), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 15), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, ENTRY, LOAD_IA32_BNDCFGS_ON_ENTRY)printf("     %s: Can set:%s Can clear:%s\n", "LOAD_IA32_BNDCFGS_ON_ENTRY"
 , vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 16), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 16), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, ENTRY, CONCEAL_VM_ENTRIES_FROM_PT)printf("     %s: Can set:%s Can clear:%s\n", "CONCEAL_VM_ENTRIES_FROM_PT"
 , vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 17), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x484, (1ULL << 17), 0
) ? "Yes" : "No");;
printf("    exit ctls: 0x%llx\n",
    vcpu->vc_vmx_exit_ctls);
printf("    true exit ctls: 0x%llx\n",
    vcpu->vc_vmx_true_exit_ctls);
CTRL_DUMP(vcpu, EXIT, SAVE_DEBUG_CONTROLS)printf("     %s: Can set:%s Can clear:%s\n", "SAVE_DEBUG_CONTROLS"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 2), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 2), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, HOST_SPACE_ADDRESS_SIZE)printf("     %s: Can set:%s Can clear:%s\n", "HOST_SPACE_ADDRESS_SIZE"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 9), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 9), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, LOAD_IA32_PERF_GLOBAL_CTRL_ON_EXIT)printf("     %s: Can set:%s Can clear:%s\n", "LOAD_IA32_PERF_GLOBAL_CTRL_ON_EXIT"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 12), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 12), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, ACKNOWLEDGE_INTERRUPT_ON_EXIT)printf("     %s: Can set:%s Can clear:%s\n", "ACKNOWLEDGE_INTERRUPT_ON_EXIT"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 15), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 15), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, SAVE_IA32_PAT_ON_EXIT)printf("     %s: Can set:%s Can clear:%s\n", "SAVE_IA32_PAT_ON_EXIT"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 18), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 18), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, LOAD_IA32_PAT_ON_EXIT)printf("     %s: Can set:%s Can clear:%s\n", "LOAD_IA32_PAT_ON_EXIT"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 19), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 19), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, SAVE_IA32_EFER_ON_EXIT)printf("     %s: Can set:%s Can clear:%s\n", "SAVE_IA32_EFER_ON_EXIT"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 20), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 20), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, LOAD_IA32_EFER_ON_EXIT)printf("     %s: Can set:%s Can clear:%s\n", "LOAD_IA32_EFER_ON_EXIT"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 21), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 21), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, SAVE_VMX_PREEMPTION_TIMER)printf("     %s: Can set:%s Can clear:%s\n", "SAVE_VMX_PREEMPTION_TIMER"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 22), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 22), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, CLEAR_IA32_BNDCFGS_ON_EXIT)printf("     %s: Can set:%s Can clear:%s\n", "CLEAR_IA32_BNDCFGS_ON_EXIT"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 23), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 23), 0
) ? "Yes" : "No");;
CTRL_DUMP(vcpu, EXIT, CONCEAL_VM_EXITS_FROM_PT)printf("     %s: Can set:%s Can clear:%s\n", "CONCEAL_VM_EXITS_FROM_PT"
 , vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 24), 1) ? "Yes"
 : "No", vcpu_vmx_check_cap(vcpu, 0x483, (1ULL << 24), 0
) ? "Yes" : "No");;
}
7158}

7160/*
* vmx_dump_vmcs_field
*
* Debug function to dump the contents of a single VMCS field
*
* Parameters:
*  fieldid: VMCS Field ID
*  msg: string to display
*/
7169void
7170vmx_dump_vmcs_field(uint16_t fieldid, const char *msg)
7171{
uint8_t width;
uint64_t val;


DPRINTF("%s (0x%04x): ", msg, fieldid);
if (vmread(fieldid, &val))
DPRINTF("???? ");
else {
/*
 * Field width encoding : bits 13:14
 *
 * 0: 16-bit
 * 1: 64-bit
 * 2: 32-bit
 * 3: natural width
 */
width = (fieldid >> 13) & 0x3;
switch (width) {
	case 0: DPRINTF("0x%04llx ", val); break;
	case 1:
	case 3: DPRINTF("0x%016llx ", val); break;
	case 2: DPRINTF("0x%08llx ", val);
}
}
7196}

7198/*
* vmx_dump_vmcs
*
* Debug function to dump the contents of the current VMCS.
*/
7203void
7204vmx_dump_vmcs(struct vcpu *vcpu)
7205{
int has_sec, i;
uint32_t cr3_tgt_ct;

/* XXX save and load new vmcs, restore at end */

DPRINTF("--CURRENT VMCS STATE--\n");
printf("VMCS launched: %s\n",
   (vcpu->vc_vmx_vmcs_state == VMCS_LAUNCHED1) ? "Yes" : "No");
DPRINTF("VMXON revision : 0x%x\n",
   curcpu()->ci_vmm_cap.vcc_vmx.vmx_vmxon_revision);
DPRINTF("CR0 fixed0: 0x%llx\n",
   curcpu()->ci_vmm_cap.vcc_vmx.vmx_cr0_fixed0);
DPRINTF("CR0 fixed1: 0x%llx\n",
   curcpu()->ci_vmm_cap.vcc_vmx.vmx_cr0_fixed1);
DPRINTF("CR4 fixed0: 0x%llx\n",
   curcpu()->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed0);
DPRINTF("CR4 fixed1: 0x%llx\n",
   curcpu()->ci_vmm_cap.vcc_vmx.vmx_cr4_fixed1);
DPRINTF("MSR table size: 0x%x\n",
   512 * (curcpu()->ci_vmm_cap.vcc_vmx.vmx_msr_table_size + 1));

has_sec = vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
   IA32_VMX_ACTIVATE_SECONDARY_CONTROLS(1ULL << 31), 1);

if (has_sec) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_VPID(1ULL << 5), 1)) {
	vmx_dump_vmcs_field(VMCS_GUEST_VPID0x0000, "VPID");
}
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PINBASED_CTLS0x481,
   IA32_VMX_PROCESS_POSTED_INTERRUPTS(1ULL << 7), 1)) {
vmx_dump_vmcs_field(VMCS_POSTED_INT_NOTIF_VECTOR0x0002,
    "Posted Int Notif Vec");
}

if (has_sec) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_EPT_VIOLATION_VE(1ULL << 18), 1)) {
	vmx_dump_vmcs_field(VMCS_EPTP_INDEX0x0004, "EPTP idx");
}
}

DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_ES_SEL0x0800, "G.ES");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_CS_SEL0x0802, "G.CS");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_SS_SEL0x0804, "G.SS");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_DS_SEL0x0806, "G.DS");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_FS_SEL0x0808, "G.FS");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_GS_SEL0x080A, "G.GS");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_LDTR_SEL0x080C, "LDTR");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_TR_SEL0x080E, "G.TR");

if (has_sec) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_VIRTUAL_INTERRUPT_DELIVERY(1ULL << 9), 1)) {
	vmx_dump_vmcs_field(VMCS_GUEST_INTERRUPT_STATUS0x0810,
	    "Int sts");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_PML(1ULL << 17), 1)) {
	vmx_dump_vmcs_field(VMCS_GUEST_PML_INDEX0x0812, "PML Idx");
}
}

DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_ES_SEL0x0C00, "H.ES");
vmx_dump_vmcs_field(VMCS_HOST_IA32_CS_SEL0x0C02, "H.CS");
vmx_dump_vmcs_field(VMCS_HOST_IA32_SS_SEL0x0C04, "H.SS");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_DS_SEL0x0C06, "H.DS");
vmx_dump_vmcs_field(VMCS_HOST_IA32_FS_SEL0x0C08, "H.FS");
vmx_dump_vmcs_field(VMCS_HOST_IA32_GS_SEL0x0C0A, "H.GS");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_IO_BITMAP_A0x2000, "I/O Bitmap A");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_IO_BITMAP_B0x2002, "I/O Bitmap B");
DPRINTF("\n");

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
   IA32_VMX_USE_MSR_BITMAPS(1ULL << 28), 1)) {
vmx_dump_vmcs_field(VMCS_MSR_BITMAP_ADDRESS0x2004, "MSR Bitmap");
DPRINTF("\n");
}

vmx_dump_vmcs_field(VMCS_EXIT_STORE_MSR_ADDRESS0x2006, "Exit Store MSRs");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_EXIT_LOAD_MSR_ADDRESS0x2008, "Exit Load MSRs");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_ENTRY_LOAD_MSR_ADDRESS0x200A, "Entry Load MSRs");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_EXECUTIVE_VMCS_POINTER0x200C, "Exec VMCS Ptr");
DPRINTF("\n");

if (has_sec) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_PML(1ULL << 17), 1)) {
	vmx_dump_vmcs_field(VMCS_PML_ADDRESS0x200E, "PML Addr");
	DPRINTF("\n");
}
}

vmx_dump_vmcs_field(VMCS_TSC_OFFSET0x2010, "TSC Offset");
DPRINTF("\n");

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
   IA32_VMX_USE_TPR_SHADOW(1ULL << 21), 1)) {
vmx_dump_vmcs_field(VMCS_VIRTUAL_APIC_ADDRESS0x2012,
    "Virtual APIC Addr");
DPRINTF("\n");
}

if (has_sec) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_VIRTUALIZE_APIC(1ULL << 0), 1)) {
	vmx_dump_vmcs_field(VMCS_APIC_ACCESS_ADDRESS0x2014,
	    "APIC Access Addr");
	DPRINTF("\n");
}
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PINBASED_CTLS0x481,
   IA32_VMX_PROCESS_POSTED_INTERRUPTS(1ULL << 7), 1)) {
vmx_dump_vmcs_field(VMCS_POSTED_INTERRUPT_DESC0x2016,
    "Posted Int Desc Addr");
DPRINTF("\n");
}

if (has_sec) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_VM_FUNCTIONS(1ULL << 13), 1)) {
	vmx_dump_vmcs_field(VMCS_VM_FUNCTION_CONTROLS0x2018,
	    "VM Function Controls");
	DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_EPT(1ULL << 1), 1)) {
	vmx_dump_vmcs_field(VMCS_GUEST_IA32_EPTP0x201A,
	    "EPT Pointer");
	DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_VIRTUAL_INTERRUPT_DELIVERY(1ULL << 9), 1)) {
	vmx_dump_vmcs_field(VMCS_EOI_EXIT_BITMAP_00x201C,
	    "EOI Exit Bitmap 0");
	DPRINTF("\n");
	vmx_dump_vmcs_field(VMCS_EOI_EXIT_BITMAP_10x201E,
	    "EOI Exit Bitmap 1");
	DPRINTF("\n");
	vmx_dump_vmcs_field(VMCS_EOI_EXIT_BITMAP_20x2020,
	    "EOI Exit Bitmap 2");
	DPRINTF("\n");
	vmx_dump_vmcs_field(VMCS_EOI_EXIT_BITMAP_30x2022,
	    "EOI Exit Bitmap 3");
	DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_VM_FUNCTIONS(1ULL << 13), 1)) {
	/* We assume all CPUs have the same VMFUNC caps */
	if (curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_vm_func & 0x1) {
		vmx_dump_vmcs_field(VMCS_EPTP_LIST_ADDRESS0x2024,
		    "EPTP List Addr");
		DPRINTF("\n");
	}
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_VMCS_SHADOWING(1ULL << 14), 1)) {
	vmx_dump_vmcs_field(VMCS_VMREAD_BITMAP_ADDRESS0x2026,
	    "VMREAD Bitmap Addr");
	DPRINTF("\n");
	vmx_dump_vmcs_field(VMCS_VMWRITE_BITMAP_ADDRESS0x2028,
	    "VMWRITE Bitmap Addr");
	DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_EPT_VIOLATION_VE(1ULL << 18), 1)) {
	vmx_dump_vmcs_field(VMCS_VIRTUALIZATION_EXC_ADDRESS0x202A,
	    "#VE Addr");
	DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_XSAVES_XRSTORS(1ULL << 20), 1)) {
	vmx_dump_vmcs_field(VMCS_XSS_EXITING_BITMAP0x202C,
	    "XSS exiting bitmap addr");
	DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_ENCLS_EXITING(1ULL << 15), 1)) {
	vmx_dump_vmcs_field(VMCS_ENCLS_EXITING_BITMAP0x202E,
	    "Encls exiting bitmap addr");
	DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_TSC_SCALING(1ULL << 25), 1)) {
	vmx_dump_vmcs_field(VMCS_TSC_MULTIPLIER0x2032,
	    "TSC scaling factor");
	DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_EPT(1ULL << 1), 1)) {
	vmx_dump_vmcs_field(VMCS_GUEST_PHYSICAL_ADDRESS0x2400,
	    "Guest PA");
	DPRINTF("\n");
}
}

vmx_dump_vmcs_field(VMCS_LINK_POINTER0x2800, "VMCS Link Pointer");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_DEBUGCTL0x2802, "Guest DEBUGCTL");
DPRINTF("\n");

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_ENTRY_CTLS0x484,
   IA32_VMX_LOAD_IA32_PAT_ON_ENTRY(1ULL << 14), 1) ||
   vcpu_vmx_check_cap(vcpu, IA32_VMX_EXIT_CTLS0x483,
   IA32_VMX_SAVE_IA32_PAT_ON_EXIT(1ULL << 18), 1)) {
vmx_dump_vmcs_field(VMCS_GUEST_IA32_PAT0x2804,
    "Guest PAT");
DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_ENTRY_CTLS0x484,
   IA32_VMX_LOAD_IA32_EFER_ON_ENTRY(1ULL << 15), 1) ||
   vcpu_vmx_check_cap(vcpu, IA32_VMX_EXIT_CTLS0x483,
   IA32_VMX_SAVE_IA32_EFER_ON_EXIT(1ULL << 20), 1)) {
vmx_dump_vmcs_field(VMCS_GUEST_IA32_EFER0x2806,
    "Guest EFER");
DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_ENTRY_CTLS0x484,
   IA32_VMX_LOAD_IA32_PERF_GLOBAL_CTRL_ON_ENTRY(1ULL << 13), 1)) {
vmx_dump_vmcs_field(VMCS_GUEST_IA32_PERF_GBL_CTRL0x2808,
    "Guest Perf Global Ctrl");
DPRINTF("\n");
}

if (has_sec) {
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_ENABLE_EPT(1ULL << 1), 1)) {
	vmx_dump_vmcs_field(VMCS_GUEST_PDPTE00x280A, "Guest PDPTE0");
	DPRINTF("\n");
	vmx_dump_vmcs_field(VMCS_GUEST_PDPTE10x280C, "Guest PDPTE1");
	DPRINTF("\n");
	vmx_dump_vmcs_field(VMCS_GUEST_PDPTE20x280E, "Guest PDPTE2");
	DPRINTF("\n");
	vmx_dump_vmcs_field(VMCS_GUEST_PDPTE30x2810, "Guest PDPTE3");
	DPRINTF("\n");
}
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_ENTRY_CTLS0x484,
   IA32_VMX_LOAD_IA32_BNDCFGS_ON_ENTRY(1ULL << 16), 1) ||
   vcpu_vmx_check_cap(vcpu, IA32_VMX_EXIT_CTLS0x483,
   IA32_VMX_CLEAR_IA32_BNDCFGS_ON_EXIT(1ULL << 23), 1)) {
vmx_dump_vmcs_field(VMCS_GUEST_IA32_BNDCFGS0x2812,
    "Guest BNDCFGS");
DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_EXIT_CTLS0x483,
   IA32_VMX_LOAD_IA32_PAT_ON_EXIT(1ULL << 19), 1)) {
vmx_dump_vmcs_field(VMCS_HOST_IA32_PAT0x2C00,
    "Host PAT");
DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_EXIT_CTLS0x483,
   IA32_VMX_LOAD_IA32_EFER_ON_EXIT(1ULL << 21), 1)) {
vmx_dump_vmcs_field(VMCS_HOST_IA32_EFER0x2C02,
    "Host EFER");
DPRINTF("\n");
}

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_EXIT_CTLS0x483,
   IA32_VMX_LOAD_IA32_PERF_GLOBAL_CTRL_ON_EXIT(1ULL << 12), 1)) {
vmx_dump_vmcs_field(VMCS_HOST_IA32_PERF_GBL_CTRL0x2C04,
    "Host Perf Global Ctrl");
DPRINTF("\n");
}

vmx_dump_vmcs_field(VMCS_PINBASED_CTLS0x4000, "Pinbased Ctrls");
vmx_dump_vmcs_field(VMCS_PROCBASED_CTLS0x4002, "Procbased Ctrls");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_EXCEPTION_BITMAP0x4004, "Exception Bitmap");
vmx_dump_vmcs_field(VMCS_PF_ERROR_CODE_MASK0x4006, "#PF Err Code Mask");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_PF_ERROR_CODE_MATCH0x4008, "#PF Err Code Match");
vmx_dump_vmcs_field(VMCS_CR3_TARGET_COUNT0x400A, "CR3 Tgt Count");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_EXIT_CTLS0x400C, "Exit Ctrls");
vmx_dump_vmcs_field(VMCS_EXIT_MSR_STORE_COUNT0x400E, "Exit MSR Store Ct");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_EXIT_MSR_LOAD_COUNT0x4010, "Exit MSR Load Ct");
vmx_dump_vmcs_field(VMCS_ENTRY_CTLS0x4012, "Entry Ctrls");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_ENTRY_MSR_LOAD_COUNT0x4014, "Entry MSR Load Ct");
vmx_dump_vmcs_field(VMCS_ENTRY_INTERRUPTION_INFO0x4016, "Entry Int. Info");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_ENTRY_EXCEPTION_ERROR_CODE0x4018,
   "Entry Ex. Err Code");
vmx_dump_vmcs_field(VMCS_ENTRY_INSTRUCTION_LENGTH0x401A, "Entry Insn Len");
DPRINTF("\n");

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED_CTLS0x482,
   IA32_VMX_USE_TPR_SHADOW(1ULL << 21), 1)) {
vmx_dump_vmcs_field(VMCS_TPR_THRESHOLD0x401C, "TPR Threshold");
DPRINTF("\n");
}

if (has_sec) {
vmx_dump_vmcs_field(VMCS_PROCBASED2_CTLS0x401E, "2ndary Ctrls");
DPRINTF("\n");
if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PROCBASED2_CTLS0x48B,
    IA32_VMX_PAUSE_LOOP_EXITING(1ULL << 10), 1)) {
	vmx_dump_vmcs_field(VMCS_PLE_GAP0x4020, "PLE Gap");
	vmx_dump_vmcs_field(VMCS_PLE_WINDOW0x4022, "PLE Window");
}
DPRINTF("\n");
}

vmx_dump_vmcs_field(VMCS_INSTRUCTION_ERROR0x4400, "Insn Error");
vmx_dump_vmcs_field(VMCS_EXIT_REASON0x4402, "Exit Reason");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_EXIT_INTERRUPTION_INFO0x4404, "Exit Int. Info");
vmx_dump_vmcs_field(VMCS_EXIT_INTERRUPTION_ERR_CODE0x4406,
   "Exit Int. Err Code");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_IDT_VECTORING_INFO0x4408, "IDT vect info");
vmx_dump_vmcs_field(VMCS_IDT_VECTORING_ERROR_CODE0x440A,
   "IDT vect err code");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_INSTRUCTION_LENGTH0x440C, "Insn Len");
vmx_dump_vmcs_field(VMCS_EXIT_INSTRUCTION_INFO0x440E, "Exit Insn Info");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_ES_LIMIT0x4800, "G. ES Lim");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_CS_LIMIT0x4802, "G. CS Lim");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_SS_LIMIT0x4804, "G. SS Lim");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_DS_LIMIT0x4806, "G. DS Lim");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_FS_LIMIT0x4808, "G. FS Lim");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_GS_LIMIT0x480A, "G. GS Lim");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_LDTR_LIMIT0x480C, "G. LDTR Lim");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_TR_LIMIT0x480E, "G. TR Lim");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_GDTR_LIMIT0x4810, "G. GDTR Lim");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_IDTR_LIMIT0x4812, "G. IDTR Lim");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_ES_AR0x4814, "G. ES AR");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_CS_AR0x4816, "G. CS AR");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_SS_AR0x4818, "G. SS AR");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_DS_AR0x481A, "G. DS AR");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_FS_AR0x481C, "G. FS AR");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_GS_AR0x481E, "G. GS AR");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_IA32_LDTR_AR0x4820, "G. LDTR AR");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_TR_AR0x4822, "G. TR AR");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_INTERRUPTIBILITY_ST0x4824, "G. Int St.");
vmx_dump_vmcs_field(VMCS_GUEST_ACTIVITY_STATE0x4826, "G. Act St.");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_GUEST_SMBASE0x4828, "G. SMBASE");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_SYSENTER_CS0x482A, "G. SYSENTER CS");
DPRINTF("\n");

if (vcpu_vmx_check_cap(vcpu, IA32_VMX_PINBASED_CTLS0x481,
   IA32_VMX_ACTIVATE_VMX_PREEMPTION_TIMER(1ULL << 6), 1)) {
vmx_dump_vmcs_field(VMCS_VMX_PREEMPTION_TIMER_VAL0x482E,
    "VMX Preempt Timer");
DPRINTF("\n");
}

vmx_dump_vmcs_field(VMCS_HOST_IA32_SYSENTER_CS0x4C00, "H. SYSENTER CS");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_CR0_MASK0x6000, "CR0 Mask");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_CR4_MASK0x6002, "CR4 Mask");
DPRINTF("\n");

vmx_dump_vmcs_field(VMCS_CR0_READ_SHADOW0x6004, "CR0 RD Shadow");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_CR4_READ_SHADOW0x6006, "CR4 RD Shadow");
DPRINTF("\n");

/* We assume all CPUs have the same max CR3 target ct */
cr3_tgt_ct = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_vmm_cap.vcc_vmx.vmx_cr3_tgt_count;
DPRINTF("Max CR3 target count: 0x%x\n", cr3_tgt_ct);
if (cr3_tgt_ct <= VMX_MAX_CR3_TARGETS256) {
for (i = 0 ; i < cr3_tgt_ct; i++) {
	vmx_dump_vmcs_field(VMCS_CR3_TARGET_00x6008 + (2 * i),
	    "CR3 Target");
	DPRINTF("\n");
}
} else {
DPRINTF("(Bogus CR3 Target Count > %d", VMX_MAX_CR3_TARGETS);
}

vmx_dump_vmcs_field(VMCS_GUEST_EXIT_QUALIFICATION0x6400, "G. Exit Qual");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_IO_RCX0x6402, "I/O RCX");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_IO_RSI0x6404, "I/O RSI");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_IO_RDI0x6406, "I/O RDI");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_IO_RIP0x6408, "I/O RIP");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_LINEAR_ADDRESS0x640A, "G. Lin Addr");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_CR00x6800, "G. CR0");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_CR30x6802, "G. CR3");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_CR40x6804, "G. CR4");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_ES_BASE0x6806, "G. ES Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_CS_BASE0x6808, "G. CS Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_SS_BASE0x680A, "G. SS Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_DS_BASE0x680C, "G. DS Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_FS_BASE0x680E, "G. FS Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_GS_BASE0x6810, "G. GS Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_LDTR_BASE0x6812, "G. LDTR Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_TR_BASE0x6814, "G. TR Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_GDTR_BASE0x6816, "G. GDTR Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_IDTR_BASE0x6818, "G. IDTR Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_DR70x681A, "G. DR7");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_RSP0x681C, "G. RSP");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_RIP0x681E, "G. RIP");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_RFLAGS0x6820, "G. RFLAGS");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_PENDING_DBG_EXC0x6822, "G. Pend Dbg Exc");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_SYSENTER_ESP0x6824, "G. SYSENTER ESP");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_GUEST_IA32_SYSENTER_EIP0x6826, "G. SYSENTER EIP");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_CR00x6C00, "H. CR0");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_CR30x6C02, "H. CR3");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_CR40x6C04, "H. CR4");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_FS_BASE0x6C06, "H. FS Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_GS_BASE0x6C08, "H. GS Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_TR_BASE0x6C0A, "H. TR Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_GDTR_BASE0x6C0C, "H. GDTR Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_IDTR_BASE0x6C0E, "H. IDTR Base");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_SYSENTER_ESP0x6C10, "H. SYSENTER ESP");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_SYSENTER_EIP0x6C12, "H. SYSENTER EIP");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_RSP0x6C14, "H. RSP");
DPRINTF("\n");
vmx_dump_vmcs_field(VMCS_HOST_IA32_RIP0x6C16, "H. RIP");
DPRINTF("\n");
7711}

7713/*
* vmx_vcpu_dump_regs
*
* Debug function to print vcpu regs from the current vcpu
*  note - vmcs for 'vcpu' must be on this pcpu.
*
* Parameters:
*  vcpu - vcpu whose registers should be dumped
*/
7722void
7723vmx_vcpu_dump_regs(struct vcpu *vcpu)
7724{
uint64_t r;
int i;
struct vmx_msr_store *msr_store;

/* XXX reformat this for 32 bit guest as needed */
DPRINTF("vcpu @ %p in %s mode\n", vcpu, vmm_decode_cpu_mode(vcpu));
i = vmm_get_guest_cpu_cpl(vcpu);
if (i == -1)
DPRINTF(" CPL=unknown\n");
else
DPRINTF(" CPL=%d\n", i);
DPRINTF(" rax=0x%016llx rbx=0x%016llx rcx=0x%016llx\n",
   vcpu->vc_gueststate.vg_rax, vcpu->vc_gueststate.vg_rbx,
   vcpu->vc_gueststate.vg_rcx);
DPRINTF(" rdx=0x%016llx rbp=0x%016llx rdi=0x%016llx\n",
   vcpu->vc_gueststate.vg_rdx, vcpu->vc_gueststate.vg_rbp,
   vcpu->vc_gueststate.vg_rdi);
DPRINTF(" rsi=0x%016llx  r8=0x%016llx  r9=0x%016llx\n",
   vcpu->vc_gueststate.vg_rsi, vcpu->vc_gueststate.vg_r8,
   vcpu->vc_gueststate.vg_r9);
DPRINTF(" r10=0x%016llx r11=0x%016llx r12=0x%016llx\n",
   vcpu->vc_gueststate.vg_r10, vcpu->vc_gueststate.vg_r11,
   vcpu->vc_gueststate.vg_r12);
DPRINTF(" r13=0x%016llx r14=0x%016llx r15=0x%016llx\n",
   vcpu->vc_gueststate.vg_r13, vcpu->vc_gueststate.vg_r14,
   vcpu->vc_gueststate.vg_r15);

DPRINTF(" rip=0x%016llx rsp=", vcpu->vc_gueststate.vg_rip);
if (vmread(VMCS_GUEST_IA32_RSP0x681C, &r))
DPRINTF("(error reading)\n");
else
DPRINTF("0x%016llx\n", r);

DPRINTF(" rflags=");
if (vmread(VMCS_GUEST_IA32_RFLAGS0x6820, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%016llx ", r);
vmm_decode_rflags(r);
}

DPRINTF(" cr0=");
if (vmread(VMCS_GUEST_IA32_CR00x6800, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%016llx ", r);
vmm_decode_cr0(r);
}

DPRINTF(" cr2=0x%016llx\n", vcpu->vc_gueststate.vg_cr2);

DPRINTF(" cr3=");
if (vmread(VMCS_GUEST_IA32_CR30x6802, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%016llx ", r);
vmm_decode_cr3(r);
}

DPRINTF(" cr4=");
if (vmread(VMCS_GUEST_IA32_CR40x6804, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%016llx ", r);
vmm_decode_cr4(r);
}

DPRINTF(" --Guest Segment Info--\n");

DPRINTF(" cs=");
if (vmread(VMCS_GUEST_IA32_CS_SEL0x0802, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%04llx rpl=%lld", r, r & 0x3);

DPRINTF(" base=");
if (vmread(VMCS_GUEST_IA32_CS_BASE0x6808, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_CS_LIMIT0x4802, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" a/r=");
if (vmread(VMCS_GUEST_IA32_CS_AR0x4816, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%04llx\n  ", r);
vmm_segment_desc_decode(r);
}

DPRINTF(" ds=");
if (vmread(VMCS_GUEST_IA32_DS_SEL0x0806, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%04llx rpl=%lld", r, r & 0x3);

DPRINTF(" base=");
if (vmread(VMCS_GUEST_IA32_DS_BASE0x680C, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_DS_LIMIT0x4806, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" a/r=");
if (vmread(VMCS_GUEST_IA32_DS_AR0x481A, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%04llx\n  ", r);
vmm_segment_desc_decode(r);
}

DPRINTF(" es=");
if (vmread(VMCS_GUEST_IA32_ES_SEL0x0800, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%04llx rpl=%lld", r, r & 0x3);

DPRINTF(" base=");
if (vmread(VMCS_GUEST_IA32_ES_BASE0x6806, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_ES_LIMIT0x4800, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" a/r=");
if (vmread(VMCS_GUEST_IA32_ES_AR0x4814, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%04llx\n  ", r);
vmm_segment_desc_decode(r);
}

DPRINTF(" fs=");
if (vmread(VMCS_GUEST_IA32_FS_SEL0x0808, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%04llx rpl=%lld", r, r & 0x3);

DPRINTF(" base=");
if (vmread(VMCS_GUEST_IA32_FS_BASE0x680E, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_FS_LIMIT0x4808, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" a/r=");
if (vmread(VMCS_GUEST_IA32_FS_AR0x481C, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%04llx\n  ", r);
vmm_segment_desc_decode(r);
}

DPRINTF(" gs=");
if (vmread(VMCS_GUEST_IA32_GS_SEL0x080A, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%04llx rpl=%lld", r, r & 0x3);

DPRINTF(" base=");
if (vmread(VMCS_GUEST_IA32_GS_BASE0x6810, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_GS_LIMIT0x480A, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" a/r=");
if (vmread(VMCS_GUEST_IA32_GS_AR0x481E, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%04llx\n  ", r);
vmm_segment_desc_decode(r);
}

DPRINTF(" ss=");
if (vmread(VMCS_GUEST_IA32_SS_SEL0x0804, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%04llx rpl=%lld", r, r & 0x3);

DPRINTF(" base=");
if (vmread(VMCS_GUEST_IA32_SS_BASE0x680A, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_SS_LIMIT0x4804, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" a/r=");
if (vmread(VMCS_GUEST_IA32_SS_AR0x4818, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%04llx\n  ", r);
vmm_segment_desc_decode(r);
}

DPRINTF(" tr=");
if (vmread(VMCS_GUEST_IA32_TR_SEL0x080E, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%04llx", r);

DPRINTF(" base=");
if (vmread(VMCS_GUEST_IA32_TR_BASE0x6814, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_TR_LIMIT0x480E, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" a/r=");
if (vmread(VMCS_GUEST_IA32_TR_AR0x4822, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%04llx\n  ", r);
vmm_segment_desc_decode(r);
}

DPRINTF(" gdtr base=");
if (vmread(VMCS_GUEST_IA32_GDTR_BASE0x6816, &r))
DPRINTF("(error reading)   ");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_GDTR_LIMIT0x4810, &r))
DPRINTF("(error reading)\n");
else
DPRINTF("0x%016llx\n", r);

DPRINTF(" idtr base=");
if (vmread(VMCS_GUEST_IA32_IDTR_BASE0x6818, &r))
DPRINTF("(error reading)   ");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_IDTR_LIMIT0x4812, &r))
DPRINTF("(error reading)\n");
else
DPRINTF("0x%016llx\n", r);

DPRINTF(" ldtr=");
if (vmread(VMCS_GUEST_IA32_LDTR_SEL0x080C, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%04llx", r);

DPRINTF(" base=");
if (vmread(VMCS_GUEST_IA32_LDTR_BASE0x6812, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" limit=");
if (vmread(VMCS_GUEST_IA32_LDTR_LIMIT0x480C, &r))
DPRINTF("(error reading)");
else
DPRINTF("0x%016llx", r);

DPRINTF(" a/r=");
if (vmread(VMCS_GUEST_IA32_LDTR_AR0x4820, &r))
DPRINTF("(error reading)\n");
else {
DPRINTF("0x%04llx\n  ", r);
vmm_segment_desc_decode(r);
}

DPRINTF(" --Guest MSRs @ 0x%016llx (paddr: 0x%016llx)--\n",
   (uint64_t)vcpu->vc_vmx_msr_exit_save_va,
   (uint64_t)vcpu->vc_vmx_msr_exit_save_pa);

msr_store = (struct vmx_msr_store *)vcpu->vc_vmx_msr_exit_save_va;

for (i = 0; i < VMX_NUM_MSR_STORE7; i++) {
DPRINTF("  MSR %d @ %p : 0x%08llx (%s), "
    "value=0x%016llx ",
    i, &msr_store[i], msr_store[i].vms_index,
    msr_name_decode(msr_store[i].vms_index),
    msr_store[i].vms_data);
vmm_decode_msr_value(msr_store[i].vms_index,
    msr_store[i].vms_data);
}
8041}

8043/*
* msr_name_decode
*
* Returns a human-readable name for the MSR supplied in 'msr'.
*
* Parameters:
*  msr - The MSR to decode
*
* Return value:
*  NULL-terminated character string containing the name of the MSR requested
*/
8054const char *
8055msr_name_decode(uint32_t msr)
8056{
/*
* Add as needed. Also consider adding a decode function when
* adding to this table.
*/

switch (msr) {
case MSR_TSC0x010: return "TSC";
case MSR_APICBASE0x01b: return "APIC base";
case MSR_IA32_FEATURE_CONTROL0x03a: return "IA32 feature control";
case MSR_PERFCTR00x0c1: return "perf counter 0";
case MSR_PERFCTR10x0c2: return "perf counter 1";
case MSR_TEMPERATURE_TARGET0x1a2: return "temperature target";
case MSR_MTRRcap0x0fe: return "MTRR cap";
case MSR_PERF_STATUS0x198: return "perf status";
case MSR_PERF_CTL0x199: return "perf control";
case MSR_MTRRvarBase0x200: return "MTRR variable base";
case MSR_MTRRfix64K_000000x250: return "MTRR fixed 64K";
case MSR_MTRRfix16K_800000x258: return "MTRR fixed 16K";
case MSR_MTRRfix4K_C00000x268: return "MTRR fixed 4K";
case MSR_CR_PAT0x277: return "PAT";
case MSR_MTRRdefType0x2ff: return "MTRR default type";
case MSR_EFER0xc0000080: return "EFER";
case MSR_STAR0xc0000081: return "STAR";
case MSR_LSTAR0xc0000082: return "LSTAR";
case MSR_CSTAR0xc0000083: return "CSTAR";
case MSR_SFMASK0xc0000084: return "SFMASK";
case MSR_FSBASE0xc0000100: return "FSBASE";
case MSR_GSBASE0xc0000101: return "GSBASE";
case MSR_KERNELGSBASE0xc0000102: return "KGSBASE";
case MSR_MISC_ENABLE0x1a0: return "Misc Enable";
default: return "Unknown MSR";
}
8089}

8091/*
* vmm_segment_desc_decode
*
* Debug function to print segment information for supplied descriptor
*
* Parameters:
*  val - The A/R bytes for the segment descriptor to decode
*/
8099void
8100vmm_segment_desc_decode(uint64_t val)
8101{
uint16_t ar;
uint8_t g, type, s, dpl, p, dib, l;
uint32_t unusable;

/* Exit early on unusable descriptors */
unusable = val & 0x10000;
if (unusable) {
DPRINTF("(unusable)\n");
return;
}

ar = (uint16_t)val;

g = (ar & 0x8000) >> 15;
dib = (ar & 0x4000) >> 14;
l = (ar & 0x2000) >> 13;
p = (ar & 0x80) >> 7;
dpl = (ar & 0x60) >> 5;
s = (ar & 0x10) >> 4;
type = (ar & 0xf);

DPRINTF("granularity=%d dib=%d l(64 bit)=%d present=%d sys=%d ",
   g, dib, l, p, s);

DPRINTF("type=");
if (!s) {
switch (type) {
case SDT_SYSLDT2: DPRINTF("ldt\n"); break;
case SDT_SYS386TSS9: DPRINTF("tss (available)\n"); break;
case SDT_SYS386BSY11: DPRINTF("tss (busy)\n"); break;
case SDT_SYS386CGT12: DPRINTF("call gate\n"); break;
case SDT_SYS386IGT14: DPRINTF("interrupt gate\n"); break;
case SDT_SYS386TGT15: DPRINTF("trap gate\n"); break;
/* XXX handle 32 bit segment types by inspecting mode */
default: DPRINTF("unknown");
}
} else {
switch (type + 16) {
case SDT_MEMRO16: DPRINTF("data, r/o\n"); break;
case SDT_MEMROA17: DPRINTF("data, r/o, accessed\n"); break;
case SDT_MEMRW18: DPRINTF("data, r/w\n"); break;
case SDT_MEMRWA19: DPRINTF("data, r/w, accessed\n"); break;
case SDT_MEMROD20: DPRINTF("data, r/o, expand down\n"); break;
case SDT_MEMRODA21: DPRINTF("data, r/o, expand down, "
    "accessed\n");
	break;
case SDT_MEMRWD22: DPRINTF("data, r/w, expand down\n"); break;
case SDT_MEMRWDA23: DPRINTF("data, r/w, expand down, "
    "accessed\n");
	break;
case SDT_MEME24: DPRINTF("code, x only\n"); break;
case SDT_MEMEA25: DPRINTF("code, x only, accessed\n");
case SDT_MEMER26: DPRINTF("code, r/x\n"); break;
case SDT_MEMERA27: DPRINTF("code, r/x, accessed\n"); break;
case SDT_MEMEC28: DPRINTF("code, x only, conforming\n"); break;
case SDT_MEMEAC29: DPRINTF("code, x only, conforming, "
    "accessed\n");
	break;
case SDT_MEMERC30: DPRINTF("code, r/x, conforming\n"); break;
case SDT_MEMERAC31: DPRINTF("code, r/x, conforming, accessed\n");
	break;
}
}
8165}

8167void
8168vmm_decode_cr0(uint64_t cr0)
8169{
struct vmm_reg_debug_info cr0_info[11] = {
{ CR0_PG0x80000000, "PG ", "pg " },
{ CR0_CD0x40000000, "CD ", "cd " },
{ CR0_NW0x20000000, "NW ", "nw " },
{ CR0_AM0x00040000, "AM ", "am " },
{ CR0_WP0x00010000, "WP ", "wp " },
{ CR0_NE0x00000020, "NE ", "ne " },
{ CR0_ET0x00000010, "ET ", "et " },
{ CR0_TS0x00000008, "TS ", "ts " },
{ CR0_EM0x00000004, "EM ", "em " },
{ CR0_MP0x00000002, "MP ", "mp " },
{ CR0_PE0x00000001, "PE", "pe" }
};

uint8_t i;

DPRINTF("(");
for (i = 0; i < nitems(cr0_info)(sizeof((cr0_info)) / sizeof((cr0_info)[0])); i++)
if (cr0 & cr0_info[i].vrdi_bit)
	DPRINTF("%s", cr0_info[i].vrdi_present);
else
	DPRINTF("%s", cr0_info[i].vrdi_absent);

DPRINTF(")\n");
8194}

8196void
8197vmm_decode_cr3(uint64_t cr3)
8198{
struct vmm_reg_debug_info cr3_info[2] = {
{ CR3_PWT(1ULL << 3), "PWT ", "pwt "},
{ CR3_PCD(1ULL << 4), "PCD", "pcd"}
};

uint64_t cr4;
uint8_t i;

if (vmread(VMCS_GUEST_IA32_CR40x6804, &cr4)) {
DPRINTF("(error)\n");
return;
}

/* If CR4.PCIDE = 0, interpret CR3.PWT and CR3.PCD */
if ((cr4 & CR4_PCIDE0x00020000) == 0) {
DPRINTF("(");
for (i = 0 ; i < nitems(cr3_info)(sizeof((cr3_info)) / sizeof((cr3_info)[0])) ; i++)
	if (cr3 & cr3_info[i].vrdi_bit)
		DPRINTF("%s", cr3_info[i].vrdi_present);
	else
		DPRINTF("%s", cr3_info[i].vrdi_absent);

DPRINTF(")\n");
} else {
DPRINTF("(pcid=0x%llx)\n", cr3 & 0xFFF);
}
8225}

8227void
8228vmm_decode_cr4(uint64_t cr4)
8229{
struct vmm_reg_debug_info cr4_info[19] = {
{ CR4_PKE0x00400000, "PKE ", "pke "},
{ CR4_SMAP0x00200000, "SMAP ", "smap "},
{ CR4_SMEP0x00100000, "SMEP ", "smep "},
{ CR4_OSXSAVE0x00040000, "OSXSAVE ", "osxsave "},
{ CR4_PCIDE0x00020000, "PCIDE ", "pcide "},
{ CR4_FSGSBASE0x00010000, "FSGSBASE ", "fsgsbase "},
{ CR4_SMXE0x00004000, "SMXE ", "smxe "},
{ CR4_VMXE0x00002000, "VMXE ", "vmxe "},
{ CR4_OSXMMEXCPT0x00000400, "OSXMMEXCPT ", "osxmmexcpt "},
{ CR4_OSFXSR0x00000200, "OSFXSR ", "osfxsr "},
{ CR4_PCE0x00000100, "PCE ", "pce "},
{ CR4_PGE0x00000080, "PGE ", "pge "},
{ CR4_MCE0x00000040, "MCE ", "mce "},
{ CR4_PAE0x00000020, "PAE ", "pae "},
{ CR4_PSE0x00000010, "PSE ", "pse "},
{ CR4_DE0x00000008, "DE ", "de "},
{ CR4_TSD0x00000004, "TSD ", "tsd "},
{ CR4_PVI0x00000002, "PVI ", "pvi "},
{ CR4_VME0x00000001, "VME", "vme"}
};

uint8_t i;

DPRINTF("(");
for (i = 0; i < nitems(cr4_info)(sizeof((cr4_info)) / sizeof((cr4_info)[0])); i++)
if (cr4 & cr4_info[i].vrdi_bit)
	DPRINTF("%s", cr4_info[i].vrdi_present);
else
	DPRINTF("%s", cr4_info[i].vrdi_absent);

DPRINTF(")\n");
8262}

8264void
8265vmm_decode_apicbase_msr_value(uint64_t apicbase)
8266{
struct vmm_reg_debug_info apicbase_info[3] = {
{ APICBASE_BSP0x100, "BSP ", "bsp "},
{ APICBASE_ENABLE_X2APIC0x400, "X2APIC ", "x2apic "},
{ APICBASE_GLOBAL_ENABLE0x800, "GLB_EN", "glb_en"}
};

uint8_t i;

DPRINTF("(");
for (i = 0; i < nitems(apicbase_info)(sizeof((apicbase_info)) / sizeof((apicbase_info)[0])); i++)
if (apicbase & apicbase_info[i].vrdi_bit)
	DPRINTF("%s", apicbase_info[i].vrdi_present);
else
	DPRINTF("%s", apicbase_info[i].vrdi_absent);

DPRINTF(")\n");
8283}

8285void
8286vmm_decode_ia32_fc_value(uint64_t fcr)
8287{
struct vmm_reg_debug_info fcr_info[4] = {
{ IA32_FEATURE_CONTROL_LOCK0x01, "LOCK ", "lock "},
{ IA32_FEATURE_CONTROL_SMX_EN0x02, "SMX ", "smx "},
{ IA32_FEATURE_CONTROL_VMX_EN0x04, "VMX ", "vmx "},
{ IA32_FEATURE_CONTROL_SENTER_EN(1ULL << 15), "SENTER ", "senter "}
};

uint8_t i;

DPRINTF("(");
for (i = 0; i < nitems(fcr_info)(sizeof((fcr_info)) / sizeof((fcr_info)[0])); i++)
if (fcr & fcr_info[i].vrdi_bit)
	DPRINTF("%s", fcr_info[i].vrdi_present);
else
	DPRINTF("%s", fcr_info[i].vrdi_absent);

if (fcr & IA32_FEATURE_CONTROL_SENTER_EN(1ULL << 15))
DPRINTF(" [SENTER param = 0x%llx]",
    (fcr & IA32_FEATURE_CONTROL_SENTER_PARAM_MASK) >> 8);

DPRINTF(")\n");
8309}

8311void
8312vmm_decode_mtrrcap_value(uint64_t val)
8313{
struct vmm_reg_debug_info mtrrcap_info[3] = {
{ MTRRcap_FIXED0x100, "FIXED ", "fixed "},
{ MTRRcap_WC0x400, "WC ", "wc "},
{ MTRRcap_SMRR0x800, "SMRR ", "smrr "}
};

uint8_t i;

DPRINTF("(");
for (i = 0; i < nitems(mtrrcap_info)(sizeof((mtrrcap_info)) / sizeof((mtrrcap_info)[0])); i++)
if (val & mtrrcap_info[i].vrdi_bit)
	DPRINTF("%s", mtrrcap_info[i].vrdi_present);
else
	DPRINTF("%s", mtrrcap_info[i].vrdi_absent);

if (val & MTRRcap_FIXED0x100)
DPRINTF(" [nr fixed ranges = 0x%llx]",
    (val & 0xff));

DPRINTF(")\n");
8334}

8336void
8337vmm_decode_perf_status_value(uint64_t val)
8338{
DPRINTF("(pstate ratio = 0x%llx)\n", (val & 0xffff));
8340}

8342void vmm_decode_perf_ctl_value(uint64_t val)
8343{
DPRINTF("(%s ", (val & PERF_CTL_TURBO) ? "TURBO" : "turbo");
DPRINTF("pstate req = 0x%llx)\n", (val & 0xfffF));
8346}

8348void
8349vmm_decode_mtrrdeftype_value(uint64_t mtrrdeftype)
8350{
struct vmm_reg_debug_info mtrrdeftype_info[2] = {
{ MTRRdefType_FIXED_ENABLE0x400, "FIXED ", "fixed "},
{ MTRRdefType_ENABLE0x800, "ENABLED ", "enabled "},
};

uint8_t i;
int type;

DPRINTF("(");
for (i = 0; i < nitems(mtrrdeftype_info)(sizeof((mtrrdeftype_info)) / sizeof((mtrrdeftype_info)[0])); i++)
if (mtrrdeftype & mtrrdeftype_info[i].vrdi_bit)
	DPRINTF("%s", mtrrdeftype_info[i].vrdi_present);
else
	DPRINTF("%s", mtrrdeftype_info[i].vrdi_absent);

DPRINTF("type = ");
type = mtrr2mrt(mtrrdeftype & 0xff);
switch (type) {
case MDF_UNCACHEABLE(1<<0): DPRINTF("UC"); break;
case MDF_WRITECOMBINE(1<<1): DPRINTF("WC"); break;
case MDF_WRITETHROUGH(1<<2): DPRINTF("WT"); break;
case MDF_WRITEPROTECT(1<<4): DPRINTF("RO"); break;
case MDF_WRITEBACK(1<<3): DPRINTF("WB"); break;
case MDF_UNKNOWN(1<<5):
default:
DPRINTF("??");
break;
}

DPRINTF(")\n");
8381}

8383void
8384vmm_decode_efer_value(uint64_t efer)
8385{
struct vmm_reg_debug_info efer_info[4] = {
{ EFER_SCE0x00000001, "SCE ", "sce "},
{ EFER_LME0x00000100, "LME ", "lme "},
{ EFER_LMA0x00000400, "LMA ", "lma "},
{ EFER_NXE0x00000800, "NXE", "nxe"},
};

uint8_t i;

DPRINTF("(");
for (i = 0; i < nitems(efer_info)(sizeof((efer_info)) / sizeof((efer_info)[0])); i++)
if (efer & efer_info[i].vrdi_bit)
	DPRINTF("%s", efer_info[i].vrdi_present);
else
	DPRINTF("%s", efer_info[i].vrdi_absent);

DPRINTF(")\n");
8403}

8405void
8406vmm_decode_msr_value(uint64_t msr, uint64_t val)
8407{
switch (msr) {
case MSR_APICBASE0x01b: vmm_decode_apicbase_msr_value(val); break;
case MSR_IA32_FEATURE_CONTROL0x03a: vmm_decode_ia32_fc_value(val); break;
case MSR_MTRRcap0x0fe: vmm_decode_mtrrcap_value(val); break;
case MSR_PERF_STATUS0x198: vmm_decode_perf_status_value(val); break;
case MSR_PERF_CTL0x199: vmm_decode_perf_ctl_value(val); break;
case MSR_MTRRdefType0x2ff: vmm_decode_mtrrdeftype_value(val); break;
case MSR_EFER0xc0000080: vmm_decode_efer_value(val); break;
case MSR_MISC_ENABLE0x1a0: vmm_decode_misc_enable_value(val); break;
default: DPRINTF("\n");
}
8419}

8421void
8422vmm_decode_rflags(uint64_t rflags)
8423{
struct vmm_reg_debug_info rflags_info[16] = {
{ PSL_C0x00000001,   "CF ", "cf "},
{ PSL_PF0x00000004,  "PF ", "pf "},
{ PSL_AF0x00000010,  "AF ", "af "},
{ PSL_Z0x00000040,   "ZF ", "zf "},
{ PSL_N0x00000080,   "SF ", "sf "},	/* sign flag */
{ PSL_T0x00000100,   "TF ", "tf "},
{ PSL_I0x00000200,   "IF ", "if "},
{ PSL_D0x00000400,   "DF ", "df "},
{ PSL_V0x00000800,   "OF ", "of "},	/* overflow flag */
{ PSL_NT0x00004000,  "NT ", "nt "},
{ PSL_RF0x00010000,  "RF ", "rf "},
{ PSL_VM0x00020000,  "VM ", "vm "},
{ PSL_AC0x00040000,  "AC ", "ac "},
{ PSL_VIF0x00080000, "VIF ", "vif "},
{ PSL_VIP0x00100000, "VIP ", "vip "},
{ PSL_ID0x00200000,  "ID ", "id "},
};

uint8_t i, iopl;

DPRINTF("(");
for (i = 0; i < nitems(rflags_info)(sizeof((rflags_info)) / sizeof((rflags_info)[0])); i++)
if (rflags & rflags_info[i].vrdi_bit)
	DPRINTF("%s", rflags_info[i].vrdi_present);
else
	DPRINTF("%s", rflags_info[i].vrdi_absent);

iopl = (rflags & PSL_IOPL0x00003000) >> 12;
DPRINTF("IOPL=%d", iopl);

DPRINTF(")\n");
8456}

8458void
8459vmm_decode_misc_enable_value(uint64_t misc)
8460{
struct vmm_reg_debug_info misc_info[10] = {
{ MISC_ENABLE_FAST_STRINGS(1 << 0),		"FSE ", "fse "},
{ MISC_ENABLE_TCC(1 << 3),			"TCC ", "tcc "},
{ MISC_ENABLE_PERF_MON_AVAILABLE(1 << 7),	"PERF ", "perf "},
{ MISC_ENABLE_BTS_UNAVAILABLE(1 << 11),		"BTSU ", "btsu "},
{ MISC_ENABLE_PEBS_UNAVAILABLE(1 << 12),		"PEBSU ", "pebsu "},
{ MISC_ENABLE_EIST_ENABLED(1 << 16),		"EIST ", "eist "},
{ MISC_ENABLE_ENABLE_MONITOR_FSM(1 << 18),	"MFSM ", "mfsm "},
{ MISC_ENABLE_LIMIT_CPUID_MAXVAL(1 << 22),	"CMAX ", "cmax "},
{ MISC_ENABLE_xTPR_MESSAGE_DISABLE(1 << 23),	"xTPRD ", "xtprd "},
{ MISC_ENABLE_XD_BIT_DISABLE(1 << 2),		"NXD", "nxd"},
};

uint8_t i;

DPRINTF("(");
for (i = 0; i < nitems(misc_info)(sizeof((misc_info)) / sizeof((misc_info)[0])); i++)
if (misc & misc_info[i].vrdi_bit)
	DPRINTF("%s", misc_info[i].vrdi_present);
else
	DPRINTF("%s", misc_info[i].vrdi_absent);

DPRINTF(")\n");
8484}

8486const char *
8487vmm_decode_cpu_mode(struct vcpu *vcpu)
8488{
int mode = vmm_get_guest_cpu_mode(vcpu);

switch (mode) {
case VMM_CPU_MODE_REAL: return "real";
case VMM_CPU_MODE_PROT: return "16 bit protected";
case VMM_CPU_MODE_PROT32: return "32 bit protected";
case VMM_CPU_MODE_COMPAT: return "compatibility";
case VMM_CPU_MODE_LONG: return "long";
default: return "unknown";
}
8499}
8500#endif /* VMM_DEBUG */