/usr/src/sys/arch/amd64/amd64/pmap.c

Bug Summary

File:	arch/amd64/amd64/pmap.c
Warning:	line 992, column 19 The left operand of '\|' is a garbage value
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple amd64-unknown-openbsd7.4 -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name pmap.c -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -ffp-contract=on -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -target-feature +retpoline-external-thunk -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/llvm16/lib/clang/16 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/legacy-dpm -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu13 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/inc -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/inc/pmfw_if -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D SUSPEND -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fcf-protection=branch -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /home/ben/Projects/scan/2024-01-11-110808-61670-1 -x c /usr/src/sys/arch/amd64/amd64/pmap.c
1/*	$OpenBSD: pmap.c,v 1.165 2023/12/29 13:23:27 jca Exp $	*/
2/*	$NetBSD: pmap.c,v 1.3 2003/05/08 18:13:13 thorpej Exp $	*/

4/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

29/*
* Copyright 2001 (c) Wasabi Systems, Inc.
* All rights reserved.
*
* Written by Frank van der Linden for Wasabi Systems, Inc.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
*    notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
*    notice, this list of conditions and the following disclaimer in the
*    documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
*    must display the following acknowledgement:
*      This product includes software developed for the NetBSD Project by
*      Wasabi Systems, Inc.
* 4. The name of Wasabi Systems, Inc. may not be used to endorse
*    or promote products derived from this software without specific prior
*    written permission.
*
* THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/

64/*
* This is the i386 pmap modified and generalized to support x86-64
* as well. The idea is to hide the upper N levels of the page tables
* inside pmap_get_ptp, pmap_free_ptp and pmap_growkernel. The rest
* is mostly untouched, except that it uses some more generalized
* macros and interfaces.
*
* This pmap has been tested on the i386 as well, and it can be easily
* adapted to PAE.
*
* fvdl@wasabisystems.com 18-Jun-2001
*/

77/*
* pmap.c: i386 pmap module rewrite
* Chuck Cranor <chuck@ccrc.wustl.edu>
* 11-Aug-97
*
* history of this pmap module: in addition to my own input, i used
*    the following references for this rewrite of the i386 pmap:
*
* [1] the NetBSD i386 pmap.   this pmap appears to be based on the
*     BSD hp300 pmap done by Mike Hibler at University of Utah.
*     it was then ported to the i386 by William Jolitz of UUNET
*     Technologies, Inc.   Then Charles M. Hannum of the NetBSD
*     project fixed some bugs and provided some speed ups.
*
* [2] the FreeBSD i386 pmap.   this pmap seems to be the
*     Hibler/Jolitz pmap, as modified for FreeBSD by John S. Dyson
*     and David Greenman.
*
* [3] the Mach pmap.   this pmap, from CMU, seems to have migrated
*     between several processors.   the VAX version was done by
*     Avadis Tevanian, Jr., and Michael Wayne Young.    the i386
*     version was done by Lance Berc, Mike Kupfer, Bob Baron,
*     David Golub, and Richard Draves.    the alpha version was
*     done by Alessandro Forin (CMU/Mach) and Chris Demetriou
*     (NetBSD/alpha).
*/

104#include <sys/param.h>
105#include <sys/systm.h>
106#include <sys/atomic.h>
107#include <sys/proc.h>
108#include <sys/pool.h>
109#include <sys/user.h>
110#include <sys/mutex.h>

112#include <uvm/uvm.h>

114#include <machine/cpu.h>
115#ifdef MULTIPROCESSOR1
116#include <machine/i82489reg.h>
117#include <machine/i82489var.h>
118#endif

120#include "vmm.h"

122#if NVMM1 > 0
123#include <machine/vmmvar.h>
124#endif /* NVMM > 0 */

126#include "acpi.h"

128/* #define PMAP_DEBUG */

130#ifdef PMAP_DEBUG
131#define DPRINTF(x...)   do { printf(x); } while(0)
132#else
133#define DPRINTF(x...)
134#endif /* PMAP_DEBUG */


137/*
* general info:
*
*  - for an explanation of how the i386 MMU hardware works see
*    the comments in <machine/pte.h>.
*
*  - for an explanation of the general memory structure used by
*    this pmap (including the recursive mapping), see the comments
*    in <machine/pmap.h>.
*
* this file contains the code for the "pmap module."   the module's
* job is to manage the hardware's virtual to physical address mappings.
* note that there are two levels of mapping in the VM system:
*
*  [1] the upper layer of the VM system uses vm_map's and vm_map_entry's
*      to map ranges of virtual address space to objects/files.  for
*      example, the vm_map may say: "map VA 0x1000 to 0x22000 read-only
*      to the file /bin/ls starting at offset zero."   note that
*      the upper layer mapping is not concerned with how individual
*      vm_pages are mapped.
*
*  [2] the lower layer of the VM system (the pmap) maintains the mappings
*      from virtual addresses.   it is concerned with which vm_page is
*      mapped where.   for example, when you run /bin/ls and start
*      at page 0x1000 the fault routine may lookup the correct page
*      of the /bin/ls file and then ask the pmap layer to establish
*      a mapping for it.
*
* note that information in the lower layer of the VM system can be
* thrown away since it can easily be reconstructed from the info
* in the upper layer.
*
* data structures we use include:
*  - struct pmap: describes the address space of one process
*  - struct pv_entry: describes one <PMAP,VA> mapping of a PA
*  - struct pg_to_free: a list of virtual addresses whose mappings
*	have been changed.   used for TLB flushing.
*/

176/*
* memory allocation
*
*  - there are three data structures that we must dynamically allocate:
*
* [A] new process' page directory page (PDP)
*	- plan 1: done at pmap_create() we use
*	  pool_get(&pmap_pmap_pool, PR_WAITOK) to do this allocation.
*
* if we are low in free physical memory then we sleep in
* pool_get() -- in this case this is ok since we are creating
* a new pmap and should not be holding any locks.
*
* XXX: the fork code currently has no way to return an "out of
* memory, try again" error code since uvm_fork [fka vm_fork]
* is a void function.
*
* [B] new page tables pages (PTP)
*	call uvm_pagealloc()
*		=> success: zero page, add to pm_pdir
*		=> failure: we are out of free vm_pages, let pmap_enter()
*		   tell UVM about it.
*
* note: for kernel PTPs, we start with NKPTP of them.   as we map
* kernel memory (at uvm_map time) we check to see if we've grown
* the kernel pmap.   if so, we call the optional function
* pmap_growkernel() to grow the kernel PTPs in advance.
*
* [C] pv_entry structures
*	- try to allocate one from the pool.
*	If we fail, we simply let pmap_enter() tell UVM about it.
*/

209long nkptp[] = NKPTP_INITIALIZER{ 0, 0, 0, 0 };

211const vaddr_t ptp_masks[] = PTP_MASK_INITIALIZER{ (((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)|0x00000000001ff000UL), ((0x0000ff8000000000UL|0x0000007fc0000000UL
)|0x000000003fe00000UL), (0x0000ff8000000000UL|0x0000007fc0000000UL
), 0x0000ff8000000000UL };
212const int ptp_shifts[] = PTP_SHIFT_INITIALIZER{ 12, 21, 30, 39 };
213const long nkptpmax[] = NKPTPMAX_INITIALIZER{ (unsigned long)((unsigned long)((unsigned long)((unsigned long
)1 * 512) * 512) * 512), (unsigned long)((unsigned long)((unsigned
 long)1 * 512) * 512), (unsigned long)((unsigned long)1 * 512
), (unsigned long)1 };
214const long nbpd[] = NBPD_INITIALIZER{ (1ULL << 12), (1ULL << 21), (1ULL << 30),
 (1ULL << 39) };
215pd_entry_t *const normal_pdes[] = PDES_INITIALIZER{ ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
 39))) + 255 * (1ULL << 30))), ((pd_entry_t *)((char *)
((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21
))), ((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t
 *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255
 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (
1ULL << 12))) };

217#define pmap_pte_set(p, n)_atomic_swap_64((p), (n))		atomic_swap_64(p, n)_atomic_swap_64((p), (n))
218#define pmap_pte_clearbits(p, b)x86_atomic_clearbits_u64(p, b)	x86_atomic_clearbits_u64(p, b)
219#define pmap_pte_setbits(p, b)x86_atomic_setbits_u64(p, b)		x86_atomic_setbits_u64(p, b)

221/*
* global data structures
*/

225struct pmap kernel_pmap_store;	/* the kernel's pmap (proc0) */

227/*
* pg_nx: NX PTE bit (if CPU supports)
* pg_g_kern: PG_G if global pages should be used in kernel mappings,
*	0 otherwise (for insecure CPUs)
*/
232pt_entry_t pg_nx = 0;
233pt_entry_t pg_g_kern = 0;

235/* pg_xo: XO PTE bits, set to PKU key1 (if cpu supports PKU) */
236pt_entry_t pg_xo;

238/*
* pmap_pg_wc: if our processor supports PAT then we set this
* to be the pte bits for Write Combining. Else we fall back to
* UC- so mtrrs can override the cacheability;
*/
243int pmap_pg_wc = PG_UCMINUS(0x0000000000000010UL);

245/*
* pmap_use_pcid: nonzero if PCID use is enabled (currently we require INVPCID)
*
* The next three are zero unless and until PCID support is enabled so code
* can just 'or' them in as needed without tests.
* cr3_pcid: CR3_REUSE_PCID
* cr3_pcid_proc and cr3_pcid_temp: PCID_PROC and PCID_TEMP
*/
253#if PCID_KERN0 != 0
254# error "pmap.c assumes PCID_KERN is zero"
255#endif
256int pmap_use_pcid;
257static u_int cr3_pcid_proc;
258static u_int cr3_pcid_temp;
259/* these two are accessed from locore.o */
260paddr_t cr3_reuse_pcid;
261paddr_t cr3_pcid_proc_intel;

263/*
* other data structures
*/

267pt_entry_t protection_codes[8];     /* maps MI prot to i386 prot code */
268int pmap_initialized = 0;	    /* pmap_init done yet? */

270/*
* pv management structures.
*/
273struct pool pmap_pv_pool;

275/*
* linked list of all non-kernel pmaps
*/

279struct pmap_head pmaps;
280struct mutex pmaps_lock = MUTEX_INITIALIZER(IPL_VM){ ((void *)0), ((((0xa)) > 0x0 && ((0xa)) < 0x9
) ? 0x9 : ((0xa))), 0x0 };

282/*
* pool that pmap structures are allocated from
*/

286struct pool pmap_pmap_pool;

288/*
* When we're freeing a ptp, we need to delay the freeing until all
* tlb shootdown has been done. This is the list of the to-be-freed pages.
*/
292TAILQ_HEAD(pg_to_free, vm_page)struct pg_to_free { struct vm_page *tqh_first; struct vm_page
 **tqh_last; };

294/*
* pool that PDPs are allocated from
*/

298struct pool pmap_pdp_pool;
299void pmap_pdp_ctor(pd_entry_t *);
300void pmap_pdp_ctor_intel(pd_entry_t *);

302extern vaddr_t msgbuf_vaddr;
303extern paddr_t msgbuf_paddr;

305extern vaddr_t idt_vaddr;			/* we allocate IDT early */
306extern paddr_t idt_paddr;

308extern vaddr_t lo32_vaddr;
309extern vaddr_t lo32_paddr;

311vaddr_t virtual_avail;
312extern int end;

314/*
* local prototypes
*/

318void pmap_enter_pv(struct vm_page *, struct pv_entry *, struct pmap *,
  vaddr_t, struct vm_page *);
320struct vm_page *pmap_get_ptp(struct pmap *, vaddr_t);
321struct vm_page *pmap_find_ptp(struct pmap *, vaddr_t, paddr_t, int);
322int pmap_find_pte_direct(struct pmap *pm, vaddr_t va, pt_entry_t **pd, int *offs);
323void pmap_free_ptp(struct pmap *, struct vm_page *,
  vaddr_t, struct pg_to_free *);
325void pmap_freepage(struct pmap *, struct vm_page *, int, struct pg_to_free *);
326#ifdef MULTIPROCESSOR1
327static int pmap_is_active(struct pmap *, struct cpu_info *);
328#endif
329paddr_t pmap_map_ptes(struct pmap *);
330struct pv_entry *pmap_remove_pv(struct vm_page *, struct pmap *, vaddr_t);
331void pmap_do_remove(struct pmap *, vaddr_t, vaddr_t, int);
332void pmap_remove_ept(struct pmap *, vaddr_t, vaddr_t);
333void pmap_do_remove_ept(struct pmap *, vaddr_t);
334int pmap_enter_ept(struct pmap *, vaddr_t, paddr_t, vm_prot_t);
335int pmap_remove_pte(struct pmap *, struct vm_page *, pt_entry_t *,
  vaddr_t, int, struct pv_entry **);
337void pmap_remove_ptes(struct pmap *, struct vm_page *, vaddr_t,
  vaddr_t, vaddr_t, int, struct pv_entry **);
339#define PMAP_REMOVE_ALL0		0	/* remove all mappings */
340#define PMAP_REMOVE_SKIPWIRED1	1	/* skip wired mappings */

342void pmap_unmap_ptes(struct pmap *, paddr_t);
343int pmap_get_physpage(vaddr_t, int, paddr_t *);
344int pmap_pdes_valid(vaddr_t, pd_entry_t *);
345void pmap_alloc_level(vaddr_t, int, long *);

347static inline
348void pmap_sync_flags_pte(struct vm_page *, u_long);

350void pmap_tlb_shootpage(struct pmap *, vaddr_t, int);
351void pmap_tlb_shootrange(struct pmap *, vaddr_t, vaddr_t, int);
352void pmap_tlb_shoottlb(struct pmap *, int);
353#ifdef MULTIPROCESSOR1
354void pmap_tlb_shootwait(void);
355#else
356#define	pmap_tlb_shootwait()		do { } while (0)
357#endif

359/*
* p m a p   i n l i n e   h e l p e r   f u n c t i o n s
*/

363/*
* pmap_is_curpmap: is this pmap the one currently loaded [in %cr3]?
*		of course the kernel is always loaded
*/

368static inline int
369pmap_is_curpmap(struct pmap *pmap)
370{
return((pmap == pmap_kernel()(&kernel_pmap_store)) ||
      (pmap->pm_pdirpa == (rcr3() & CR3_PADDR0x7ffffffffffff000ULL)));
373}

375/*
* pmap_is_active: is this pmap loaded into the specified processor's %cr3?
*/

379#ifdef MULTIPROCESSOR1
380static inline int
381pmap_is_active(struct pmap *pmap, struct cpu_info *ci)
382{
return pmap == pmap_kernel()(&kernel_pmap_store) || pmap == ci->ci_proc_pmap;
384}
385#endif

387static inline u_int
388pmap_pte2flags(u_long pte)
389{
return (((pte & PG_U0x0000000000000020UL) ? PG_PMAP_REF0x02000000 : 0) |
   ((pte & PG_M0x0000000000000040UL) ? PG_PMAP_MOD0x01000000 : 0));
392}

394static inline void
395pmap_sync_flags_pte(struct vm_page *pg, u_long pte)
396{
if (pte & (PG_U0x0000000000000020UL|PG_M0x0000000000000040UL)) {
atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, pmap_pte2flags(pte));
}
400}

402/*
* pmap_map_ptes: map a pmap's PTEs into KVM
*
* This should not be done for EPT pmaps
*/
407paddr_t
408pmap_map_ptes(struct pmap *pmap)
409{
paddr_t cr3;

KASSERT(pmap->pm_type != PMAP_TYPE_EPT)((pmap->pm_type != 2) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c"
, 412, "pmap->pm_type != PMAP_TYPE_EPT"));

/* the kernel's pmap is always accessible */
if (pmap == pmap_kernel()(&kernel_pmap_store))
return 0;

/*
* Lock the target map before switching to its page tables to
* guarantee other CPUs have finished changing the tables before
* we potentially start caching table and TLB entries.
*/
mtx_enter(&pmap->pm_mtx);

cr3 = rcr3();
KASSERT((cr3 & CR3_PCID) == PCID_KERN ||(((cr3 & 0xfffULL) == 0 || (cr3 & 0xfffULL) == 1) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c"
, 427, "(cr3 & CR3_PCID) == PCID_KERN || (cr3 & CR3_PCID) == PCID_PROC"
))
(cr3 & CR3_PCID) == PCID_PROC)(((cr3 & 0xfffULL) == 0 || (cr3 & 0xfffULL) == 1) ? (
void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c"
, 427, "(cr3 & CR3_PCID) == PCID_KERN || (cr3 & CR3_PCID) == PCID_PROC"
));
if (pmap->pm_pdirpa == (cr3 & CR3_PADDR0x7ffffffffffff000ULL))
cr3 = 0;
else {
cr3 |= cr3_reuse_pcid;
lcr3(pmap->pm_pdirpa | cr3_pcid_temp);
}

return cr3;
436}

438void
439pmap_unmap_ptes(struct pmap *pmap, paddr_t save_cr3)
440{
if (pmap != pmap_kernel()(&kernel_pmap_store))
mtx_leave(&pmap->pm_mtx);

if (save_cr3 != 0)
lcr3(save_cr3);
446}

448int
449pmap_find_pte_direct(struct pmap *pm, vaddr_t va, pt_entry_t **pd, int *offs)
450{
u_long mask, shift;
pd_entry_t pde;
paddr_t pdpa;
int lev;

pdpa = pm->pm_pdirpa;
shift = L4_SHIFT39;
mask = L4_MASK0x0000ff8000000000UL;
for (lev = PTP_LEVELS4; lev > 0; lev--) {
*pd = (pd_entry_t *)PMAP_DIRECT_MAP(pdpa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pdpa));
*offs = (VA_SIGN_POS(va)((va) & ~0xffff000000000000) & mask) >> shift;
pde = (*pd)[*offs];

/* Large pages are different, break early if we run into one. */
if ((pde & (PG_PS0x0000000000000080UL|PG_V0x0000000000000001UL)) != PG_V0x0000000000000001UL)
	return (lev - 1);

pdpa = ((*pd)[*offs] & PG_FRAME0x000ffffffffff000UL);
/* 4096/8 == 512 == 2^9 entries per level */
shift -= 9;
mask >>= 9;
}

return (0);
475}

477/*
* p m a p   k e n t e r   f u n c t i o n s
*
* functions to quickly enter/remove pages from the kernel address
* space.   pmap_kremove is exported to MI kernel.  we make use of
* the recursive PTE mappings.
*/

485/*
* pmap_kenter_pa: enter a kernel mapping without R/M (pv_entry) tracking
*
* => no need to lock anything, assume va is already allocated
* => should be faster than normal pmap enter function
*/

492void
493pmap_kenter_pa(vaddr_t va, paddr_t pa, vm_prot_t prot)
494{
pt_entry_t *pte, opte, npte;

pte = kvtopte(va);

npte = (pa & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1))) | ((prot & PROT_WRITE0x02) ? PG_RW0x0000000000000002UL : PG_RO0x0000000000000000UL) |
   ((pa & PMAP_NOCACHE0x1) ? PG_N0x0000000000000010UL : 0) |
   ((pa & PMAP_WC0x2) ? pmap_pg_wc : 0) | PG_V0x0000000000000001UL;

/* special 1:1 mappings in the first 2MB must not be global */
if (va >= (vaddr_t)NBPD_L2(1ULL << 21))
npte |= pg_g_kern;

if (!(prot & PROT_EXEC0x04))
npte |= pg_nx;
opte = pmap_pte_set(pte, npte)_atomic_swap_64((pte), (npte));
510#ifdef LARGEPAGES
/* XXX For now... */
if (opte & PG_PS0x0000000000000080UL)
panic("%s: PG_PS", __func__);
514#endif
if (pmap_valid_entry(opte)((opte) & 0x0000000000000001UL)) {
if (pa & PMAP_NOCACHE0x1 && (opte & PG_N0x0000000000000010UL) == 0)
	wbinvd_on_all_cpus();
/* This shouldn't happen */
pmap_tlb_shootpage(pmap_kernel()(&kernel_pmap_store), va, 1);
pmap_tlb_shootwait();
}
522}

524/*
* pmap_kremove: remove a kernel mapping(s) without R/M (pv_entry) tracking
*
* => no need to lock anything
* => caller must dispose of any vm_page mapped in the va range
* => note: not an inline function
* => we assume the va is page aligned and the len is a multiple of PAGE_SIZE
* => we assume kernel only unmaps valid addresses and thus don't bother
*    checking the valid bit before doing TLB flushing
*/

535void
536pmap_kremove(vaddr_t sva, vsize_t len)
537{
pt_entry_t *pte, opte;
vaddr_t va, eva;

eva = sva + len;

for (va = sva; va != eva; va += PAGE_SIZE(1 << 12)) {
pte = kvtopte(va);

opte = pmap_pte_set(pte, 0)_atomic_swap_64((pte), (0));
547#ifdef LARGEPAGES
KASSERT((opte & PG_PS) == 0)(((opte & 0x0000000000000080UL) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c", 548, "(opte & PG_PS) == 0"
));
549#endif
KASSERT((opte & PG_PVLIST) == 0)(((opte & 0x0000000000000400UL) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c", 550, "(opte & PG_PVLIST) == 0"
));
}

pmap_tlb_shootrange(pmap_kernel()(&kernel_pmap_store), sva, eva, 1);
pmap_tlb_shootwait();
555}

557/*
* pmap_set_pml4_early
*
* Utility function to map 2GB of 2MB pages to 'pa'. The VA that is assigned
* is the pml4 entry for 'early mappings' (see pmap.h). This function is used
* by display drivers that need to map their framebuffers early, before the
* pmap is fully initialized (eg, to show panic messages).
*
* Users of this function must call pmap_clear_pml4_early to remove the
* mapping when finished.
*
* Parameters:
*  pa: phys addr to map
*
* Return value:
*  VA mapping to 'pa'. This mapping is 2GB in size and starts at the base
*   of the 2MB region containing 'va'.
*/
575vaddr_t
576pmap_set_pml4_early(paddr_t pa)
577{
extern paddr_t early_pte_pages;
pt_entry_t *pml4e, *pte;
int i, j, off;
paddr_t curpa;
vaddr_t va;

pml4e = (pt_entry_t *)(proc0.p_addr->u_pcb.pcb_cr3 + KERNBASE0xffffffff80000000);
pml4e[PDIR_SLOT_EARLY((511 - 4) - 1)] = (pd_entry_t)early_pte_pages | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL;

off = pa & PAGE_MASK_L2((1ULL << 21) - 1);
curpa = pa & L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
);

pte = (pt_entry_t *)PMAP_DIRECT_MAP(early_pte_pages)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (early_pte_pages));
memset(pte, 0, 3 * NBPG)__builtin_memset((pte), (0), (3 * (1 << 12)));

pte[0] = (early_pte_pages + NBPG(1 << 12)) | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL;
pte[1] = (early_pte_pages + 2 * NBPG(1 << 12)) | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL;

pte = (pt_entry_t *)PMAP_DIRECT_MAP(early_pte_pages + NBPG)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (early_pte_pages + (1 << 12)));
for (i = 0; i < 2; i++) {
/* 2 early pages of mappings */
for (j = 0; j < 512; j++) {
	/* j[0..511] : 2MB mappings per page */
	pte[(i * 512) + j] = curpa | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL | PG_PS0x0000000000000080UL;
	curpa += (2 * 1024 * 1024);
}
}

va = (vaddr_t)((PDIR_SLOT_EARLY((511 - 4) - 1) * 512ULL) << L3_SHIFT30) + off;
return VA_SIGN_NEG(va)((va) | 0xffff000000000000);
608}

610/*
* pmap_clear_pml4_early
*
* Clears the mapping previously established with pmap_set_pml4_early.
*/
615void
616pmap_clear_pml4_early(void)
617{
extern paddr_t early_pte_pages;
pt_entry_t *pml4e, *pte;

pte = (pt_entry_t *)PMAP_DIRECT_MAP(early_pte_pages)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (early_pte_pages));
memset(pte, 0, 3 * NBPG)__builtin_memset((pte), (0), (3 * (1 << 12)));

pml4e = (pd_entry_t *)pmap_kernel()(&kernel_pmap_store)->pm_pdir;
pml4e[PDIR_SLOT_EARLY((511 - 4) - 1)] = 0;
tlbflush();
627}

629/*
* p m a p   i n i t   f u n c t i o n s
*
* pmap_bootstrap and pmap_init are called during system startup
* to init the pmap module.   pmap_bootstrap() does a low level
* init just to get things rolling.   pmap_init() finishes the job.
*/

637/*
* pmap_bootstrap: get the system in a state where it can run with VM
*	properly enabled (called before main()).   the VM system is
*      fully init'd later...
*/

643paddr_t
644pmap_bootstrap(paddr_t first_avail, paddr_t max_pa)
645{
vaddr_t kva_start = VM_MIN_KERNEL_ADDRESS0xffff800000000000;
struct pmap *kpm;
int curslot, i, j, p;
long ndmpdp;
paddr_t dmpd, dmpdp, start_cur, cur_pa;
vaddr_t kva, kva_end;
pt_entry_t *pml3, *pml2;

/*
* define the boundaries of the managed kernel virtual address
* space.
*/

virtual_avail = kva_start;		/* first free KVA */

/*
* If PKU is available, initialize PROT_EXEC entry correctly,
* and enable the feature before it gets used
* XXX Some Hypervisors forget to save/restore PKU
*/
if (cpuid_level >= 0x7) {
uint32_t ecx, dummy;

CPUID_LEAF(0x7, 0, dummy, dummy, ecx, dummy)__asm volatile("cpuid" : "=a" (dummy), "=b" (dummy), "=c" (ecx
), "=d" (dummy) : "a" (0x7), "c" (0));
if (ecx & SEFF0ECX_PKU0x00000008) {
	lcr4(rcr4() | CR4_PKE0x00400000);
	pg_xo = PG_XO0x0800000000000000UL;
}
}

/*
* set up protection_codes: we need to be able to convert from
* a MI protection code (some combo of VM_PROT...) to something
* we can jam into a i386 PTE.
*/

protection_codes[PROT_NONE0x00] = pg_nx;			/* --- */
protection_codes[PROT_EXEC0x04] = pg_xo;		;	/* --x */
protection_codes[PROT_READ0x01] = PG_RO0x0000000000000000UL | pg_nx;		/* -r- */
protection_codes[PROT_READ0x01 | PROT_EXEC0x04] = PG_RO0x0000000000000000UL;	/* -rx */
protection_codes[PROT_WRITE0x02] = PG_RW0x0000000000000002UL | pg_nx;		/* w-- */
protection_codes[PROT_WRITE0x02 | PROT_EXEC0x04] = PG_RW0x0000000000000002UL;	/* w-x */
protection_codes[PROT_WRITE0x02 | PROT_READ0x01] = PG_RW0x0000000000000002UL | pg_nx; /* wr- */
protection_codes[PROT_READ0x01 | PROT_WRITE0x02 | PROT_EXEC0x04] = PG_RW0x0000000000000002UL;	/* wrx */

/*
* now we init the kernel's pmap
*
* the kernel pmap's pm_obj is not used for much.   however, in
* user pmaps the pm_obj contains the list of active PTPs.
* the pm_obj currently does not have a pager.
*/

kpm = pmap_kernel()(&kernel_pmap_store);
for (i = 0; i < PTP_LEVELS4 - 1; i++) {
uvm_obj_init(&kpm->pm_obj[i], &pmap_pager, 1);
kpm->pm_ptphint[i] = NULL((void *)0);
}
memset(&kpm->pm_list, 0, sizeof(kpm->pm_list))__builtin_memset((&kpm->pm_list), (0), (sizeof(kpm->
pm_list)));  /* pm_list not used */
kpm->pm_pdir = (pd_entry_t *)(proc0.p_addr->u_pcb.pcb_cr3 + KERNBASE0xffffffff80000000);
kpm->pm_pdirpa = proc0.p_addr->u_pcb.pcb_cr3;
kpm->pm_stats.wired_count = kpm->pm_stats.resident_count =
atop(kva_start - VM_MIN_KERNEL_ADDRESS)((kva_start - 0xffff800000000000) >> 12);
/*
* the above is just a rough estimate and not critical to the proper
* operation of the system.
*/

kpm->pm_type = PMAP_TYPE_NORMAL1;

curpcb({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curpcb->pcb_pmap = kpm;	/* proc0's pcb */

/*
* Configure and enable PCID use if supported.
* Currently we require INVPCID support.
*/
if ((cpu_ecxfeature & CPUIDECX_PCID0x00020000) && cpuid_level >= 0x07) {
uint32_t ebx, dummy;
CPUID_LEAF(0x7, 0, dummy, ebx, dummy, dummy)__asm volatile("cpuid" : "=a" (dummy), "=b" (ebx), "=c" (dummy
), "=d" (dummy) : "a" (0x7), "c" (0));
if (ebx & SEFF0EBX_INVPCID0x00000400) {
	pmap_use_pcid = 1;
	/*
	 * We cannot use global mappings because
	 * invpcid function 0 does not invalidate global
	 * mappings. The hardware can cache kernel
	 * mappings based on PCID_KERN, i.e. there is no
	 * need for global mappings.
	 */
	pg_g_kern = 0;
	lcr4( rcr4() | CR4_PCIDE0x00020000 );
	cr3_pcid_proc = PCID_PROC1;
	cr3_pcid_temp = PCID_TEMP3;
	cr3_reuse_pcid = CR3_REUSE_PCID(1ULL << 63);
	cr3_pcid_proc_intel = PCID_PROC_INTEL2;
}
}

/*
* Add PG_G attribute to already mapped kernel pages. pg_g_kern
* is calculated in locore0.S and may be set to:
*
* 0 if this CPU does not safely support global pages in the kernel
*  (Intel/Meltdown)
* PG_G if this CPU does safely support global pages in the kernel
*  (AMD)
*/
752#if KERNBASE0xffffffff80000000 == VM_MIN_KERNEL_ADDRESS0xffff800000000000
for (kva = VM_MIN_KERNEL_ADDRESS0xffff800000000000 ; kva < virtual_avail ;
754#else
kva_end = roundup((vaddr_t)&end, PAGE_SIZE)(((((vaddr_t)&end)+(((1 << 12))-1))/((1 << 12
)))*((1 << 12)));
for (kva = KERNBASE0xffffffff80000000; kva < kva_end ;
757#endif
    kva += PAGE_SIZE(1 << 12)) {
unsigned long p1i = pl1_i(kva)(((((kva) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12);
if (pmap_valid_entry(PTE_BASE[p1i])((((pt_entry_t *) (255 * (1ULL << 39)))[p1i]) & 0x0000000000000001UL
))
	PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[p1i] |= pg_g_kern;
}

/*
* Map the direct map. The first 4GB were mapped in locore, here
* we map the rest if it exists. We actually use the direct map
* here to set up the page tables, we're assuming that we're still
* operating in the lower 4GB of memory.
*
* Map (up to) the first 512GB of physical memory first. This part
* is handled differently than physical memory > 512GB since we have
* already mapped part of this range in locore0.
*/
ndmpdp = (max_pa + NBPD_L3(1ULL << 30) - 1) >> L3_SHIFT30;
if (ndmpdp < NDML2_ENTRIES4)
ndmpdp = NDML2_ENTRIES4;		/* At least 4GB */
if (ndmpdp > 512)
ndmpdp = 512;			/* At most 512GB */

dmpdp = kpm->pm_pdir[PDIR_SLOT_DIRECT(511 - 4)] & PG_FRAME0x000ffffffffff000UL;

dmpd = first_avail; first_avail += ndmpdp * PAGE_SIZE(1 << 12);

for (i = NDML2_ENTRIES4; i < NPDPG((1 << 12) / sizeof (pd_entry_t)) * ndmpdp; i++) {
paddr_t pdp;
vaddr_t va;

pdp = (paddr_t)&(((pd_entry_t *)dmpd)[i]);
va = PMAP_DIRECT_MAP(pdp)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pdp));

*((pd_entry_t *)va) = ((paddr_t)i << L2_SHIFT21);
*((pd_entry_t *)va) |= PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | PG_PS0x0000000000000080UL | pg_g_kern | PG_U0x0000000000000020UL |
    PG_M0x0000000000000040UL | pg_nx;
}

for (i = NDML2_ENTRIES4; i < ndmpdp; i++) {
paddr_t pdp;
vaddr_t va;

pdp = (paddr_t)&(((pd_entry_t *)dmpdp)[i]);
va = PMAP_DIRECT_MAP(pdp)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pdp));

*((pd_entry_t *)va) = dmpd + (i << PAGE_SHIFT12);
*((pd_entry_t *)va) |= PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | PG_U0x0000000000000020UL | PG_M0x0000000000000040UL | pg_nx;
}

kpm->pm_pdir[PDIR_SLOT_DIRECT(511 - 4)] = dmpdp | PG_V0x0000000000000001UL | PG_KW0x0000000000000002UL | PG_U0x0000000000000020UL |
   PG_M0x0000000000000040UL | pg_nx;

/* Map any remaining physical memory > 512GB */
for (curslot = 1 ; curslot < NUM_L4_SLOT_DIRECT4 ; curslot++) {
/*
 * Start of current range starts at PA (curslot) * 512GB
 */
start_cur = (paddr_t)(curslot * NBPD_L4(1ULL << 39));
if (max_pa > start_cur) {
	/* Next 512GB, new PML4e and L3(512GB) page */
	dmpd = first_avail; first_avail += PAGE_SIZE(1 << 12);
	pml3 = (pt_entry_t *)PMAP_DIRECT_MAP(dmpd)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (dmpd));
	kpm->pm_pdir[PDIR_SLOT_DIRECT(511 - 4) + curslot] = dmpd |
	    PG_KW0x0000000000000002UL | PG_V0x0000000000000001UL | PG_U0x0000000000000020UL | PG_M0x0000000000000040UL | pg_nx;

	/* Calculate full 1GB pages in this 512GB region */
	p = ((max_pa - start_cur) >> L3_SHIFT30);

	/* Check if a partial (<1GB) page remains */
	if (max_pa & L2_MASK0x000000003fe00000UL)
		p++;

	/*
	 * Handle the case where this range is full and there
	 * is still more memory after (p would be > 512).
	 */
	if (p > NPDPG((1 << 12) / sizeof (pd_entry_t)))
		p = NPDPG((1 << 12) / sizeof (pd_entry_t));

	/* Allocate 'p' L2(1GB) pages and populate */
	for (i = 0; i < p; i++) {
		dmpd = first_avail; first_avail += PAGE_SIZE(1 << 12);
		pml2 = (pt_entry_t *)PMAP_DIRECT_MAP(dmpd)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (dmpd));
		pml3[i] = dmpd |
		    PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | PG_U0x0000000000000020UL | PG_M0x0000000000000040UL | pg_nx;

		cur_pa = start_cur + (i << L3_SHIFT30);
		j = 0;

		while (cur_pa < max_pa && j < NPDPG((1 << 12) / sizeof (pd_entry_t))) {
			pml2[j] = curslot * NBPD_L4(1ULL << 39) +
			    (uint64_t)i * NBPD_L3(1ULL << 30) +
			    (uint64_t)j * NBPD_L2(1ULL << 21);
			pml2[j] |= PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | pg_g_kern |
			    PG_U0x0000000000000020UL | PG_M0x0000000000000040UL | pg_nx | PG_PS0x0000000000000080UL;
			cur_pa += NBPD_L2(1ULL << 21);
			j++;
		}
	}
}
}

tlbflush();

msgbuf_vaddr = virtual_avail;
virtual_avail += round_page(MSGBUFSIZE)((((32 * (1 << 12))) + ((1 << 12) - 1)) & ~((
<< 12) - 1));

idt_vaddr = virtual_avail;
virtual_avail += 2 * PAGE_SIZE(1 << 12);
idt_paddr = first_avail;			/* steal a page */
first_avail += 2 * PAGE_SIZE(1 << 12);

870#if defined(MULTIPROCESSOR1) || \
  (NACPI1 > 0 && !defined(SMALL_KERNEL))
/*
* Grab a page below 4G for things that need it (i.e.
* having an initial %cr3 for the MP trampoline).
*/
lo32_vaddr = virtual_avail;
virtual_avail += PAGE_SIZE(1 << 12);
lo32_paddr = first_avail;
first_avail += PAGE_SIZE(1 << 12);
880#endif

/*
* init the global lists.
*/
LIST_INIT(&pmaps)do { ((&pmaps)->lh_first) = ((void *)0); } while (0);

/*
* initialize the pmap pools.
*/

pool_init(&pmap_pmap_pool, sizeof(struct pmap), 0, IPL_VM0xa, 0,
   "pmappl", NULL((void *)0));
pool_init(&pmap_pv_pool, sizeof(struct pv_entry), 0, IPL_VM0xa, 0,
   "pvpl", &pool_allocator_single);
pool_sethiwat(&pmap_pv_pool, 32 * 1024);

/*
* initialize the PDE pool.
*/

pool_init(&pmap_pdp_pool, PAGE_SIZE(1 << 12), 0, IPL_VM0xa, 0,
   "pdppl", &pool_allocator_single);

kpm->pm_pdir_intel = NULL((void *)0);
kpm->pm_pdirpa_intel = 0;

/*
* ensure the TLB is sync'd with reality by flushing it...
*/

tlbflush();

return first_avail;
914}

916void
917pmap_init_percpu(void)
918{
pool_cache_init(&pmap_pv_pool);
920}

922/*
* pmap_randomize
*
* Randomizes the location of the kernel pmap
*/
927void
928pmap_randomize(void)
929{
pd_entry_t *pml4va, *oldpml4va;
paddr_t pml4pa;
int i;

pml4va = km_alloc(PAGE_SIZE(1 << 12), &kv_page, &kp_zero, &kd_nowait);
if (pml4va == NULL((void *)0))
1
Assuming 'pml4va' is not equal to NULL→
2
←
Taking false branch→
panic("%s: km_alloc failed", __func__);

/* Copy old PML4 page to new one */
oldpml4va = pmap_kernel()(&kernel_pmap_store)->pm_pdir;
memcpy(pml4va, oldpml4va, PAGE_SIZE)__builtin_memcpy((pml4va), (oldpml4va), ((1 << 12)));

/* Switch to new PML4 */
pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)pml4va, &pml4pa);
lcr3(pml4pa);

/* Fixup pmap_kernel and proc0's %cr3 */
pmap_kernel()(&kernel_pmap_store)->pm_pdirpa = pml4pa;
pmap_kernel()(&kernel_pmap_store)->pm_pdir = pml4va;
proc0.p_addr->u_pcb.pcb_cr3 = pml4pa;

/* Fixup recursive PTE PML4E slot. We are only changing the PA */
pml4va[PDIR_SLOT_PTE255] = pml4pa | (pml4va[PDIR_SLOT_PTE255] & ~PG_FRAME0x000ffffffffff000UL);

for (i = 0; i < NPDPG((1 << 12) / sizeof (pd_entry_t)); i++) {
3
←
Loop condition is true.  Entering loop body→
/* PTE slot already handled earlier */
if (i3.1
'i' is not equal to PDIR_SLOT_PTE
 == PDIR_SLOT_PTE255)
4
←
Taking false branch→
	continue;

if (pml4va[i] & PG_FRAME0x000ffffffffff000UL)
5
←
Assuming the condition is true→
6
←
Taking true branch→
	pmap_randomize_level(&pml4va[i], 3);
7
←
Calling 'pmap_randomize_level'→
}

/* Wipe out bootstrap PML4 */
memset(oldpml4va, 0, PAGE_SIZE)__builtin_memset((oldpml4va), (0), ((1 << 12)));
tlbflush();
966}

968void
969pmap_randomize_level(pd_entry_t *pde, int level)
970{
pd_entry_t *new_pd_va;
paddr_t old_pd_pa, new_pd_pa;
8
←
'new_pd_pa' declared without an initial value→
vaddr_t old_pd_va;
struct vm_page *pg;
int i;

if (level8.1
'level' is not equal to 0
 == 0)
return;

if (level < PTP_LEVELS4 - 1 && (*pde & PG_PS0x0000000000000080UL))
return;

new_pd_va = km_alloc(PAGE_SIZE(1 << 12), &kv_page, &kp_zero, &kd_nowait);
if (new_pd_va == NULL((void *)0))
9
←
Assuming 'new_pd_va' is not equal to NULL→
10
←
Taking false branch→
panic("%s: cannot allocate page for L%d page directory",
    __func__, level);

old_pd_pa = *pde & PG_FRAME0x000ffffffffff000UL;
old_pd_va = PMAP_DIRECT_MAP(old_pd_pa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (old_pd_pa));
pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)new_pd_va, &new_pd_pa);
11
←
Calling 'pmap_extract'→
18
←
Returning from 'pmap_extract'→
memcpy(new_pd_va, (void *)old_pd_va, PAGE_SIZE)__builtin_memcpy((new_pd_va), ((void *)old_pd_va), ((1 <<
 12)));
*pde = new_pd_pa | (*pde & ~PG_FRAME0x000ffffffffff000UL);
19
←
The left operand of '|' is a garbage value

tlbflush();
memset((void *)old_pd_va, 0, PAGE_SIZE)__builtin_memset(((void *)old_pd_va), (0), ((1 << 12)));

pg = PHYS_TO_VM_PAGE(old_pd_pa);
if (pg != NULL((void *)0)) {
pg->wire_count--;
pmap_kernel()(&kernel_pmap_store)->pm_stats.resident_count--;
if (pg->wire_count <= 1)
	uvm_pagefree(pg);
}

for (i = 0; i < NPDPG((1 << 12) / sizeof (pd_entry_t)); i++)
if (new_pd_va[i] & PG_FRAME0x000ffffffffff000UL)
	pmap_randomize_level(&new_pd_va[i], level - 1);
1008}

1010/*
* Pre-allocate PTPs for low memory, so that 1:1 mappings for various
* trampoline code can be entered.
*/
1014paddr_t
1015pmap_prealloc_lowmem_ptps(paddr_t first_avail)
1016{
pd_entry_t *pdes;
int level;
paddr_t newp;

pdes = pmap_kernel()(&kernel_pmap_store)->pm_pdir;
level = PTP_LEVELS4;
for (;;) {
newp = first_avail; first_avail += PAGE_SIZE(1 << 12);
memset((void *)PMAP_DIRECT_MAP(newp), 0, PAGE_SIZE)__builtin_memset(((void *)((vaddr_t)(((((511 - 4) * (1ULL <<
 39))) | 0xffff000000000000)) + (newp))), (0), ((1 << 12
)));
pdes[pl_i(0, level)(((((0) & ~0xffff000000000000)) & ptp_masks[(level)-1
]) >> ptp_shifts[(level)-1])] = (newp & PG_FRAME0x000ffffffffff000UL) | PG_V0x0000000000000001UL | PG_RW0x0000000000000002UL;
level--;
if (level <= 1)
	break;
pdes = normal_pdes[level - 2];
}

return first_avail;
1034}

1036/*
* pmap_init: no further initialization required on this platform
*/
1039void
1040pmap_init(void)
1041{
pmap_initialized = 1;
1043}

1045/*
* p v _ e n t r y   f u n c t i o n s
*/

1049/*
* main pv_entry manipulation functions:
*   pmap_enter_pv: enter a mapping onto a pv list
*   pmap_remove_pv: remove a mapping from a pv list
*/

1055/*
* pmap_enter_pv: enter a mapping onto a pv list
*
* => caller should adjust ptp's wire_count before calling
*
* pve: preallocated pve for us to use
* ptp: PTP in pmap that maps this VA
*/

1064void
1065pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, struct pmap *pmap,
  vaddr_t va, struct vm_page *ptp)
1067{
pve->pv_pmap = pmap;
pve->pv_va = va;
pve->pv_ptp = ptp;			/* NULL for kernel pmap */
mtx_enter(&pg->mdpage.pv_mtx);
pve->pv_next = pg->mdpage.pv_list;	/* add to ... */
pg->mdpage.pv_list = pve;		/* ... list */
mtx_leave(&pg->mdpage.pv_mtx);
1075}

1077/*
* pmap_remove_pv: try to remove a mapping from a pv_list
*
* => caller should adjust ptp's wire_count and free PTP if needed
* => we return the removed pve
*/

1084struct pv_entry *
1085pmap_remove_pv(struct vm_page *pg, struct pmap *pmap, vaddr_t va)
1086{
struct pv_entry *pve, **prevptr;

mtx_enter(&pg->mdpage.pv_mtx);
prevptr = &pg->mdpage.pv_list;
while ((pve = *prevptr) != NULL((void *)0)) {
if (pve->pv_pmap == pmap && pve->pv_va == va) {	/* match? */
	*prevptr = pve->pv_next;		/* remove it! */
	break;
}
prevptr = &pve->pv_next;		/* previous pointer */
}
mtx_leave(&pg->mdpage.pv_mtx);
return(pve);				/* return removed pve */
1100}

1102/*
* p t p   f u n c t i o n s
*/

1106struct vm_page *
1107pmap_find_ptp(struct pmap *pmap, vaddr_t va, paddr_t pa, int level)
1108{
int lidx = level - 1;
struct vm_page *pg;

if (pa != (paddr_t)-1 && pmap->pm_ptphint[lidx] &&
   pa == VM_PAGE_TO_PHYS(pmap->pm_ptphint[lidx])((pmap->pm_ptphint[lidx])->phys_addr))
return (pmap->pm_ptphint[lidx]);

pg = uvm_pagelookup(&pmap->pm_obj[lidx], ptp_va2o(va, level)((((((va) & ~0xffff000000000000)) & ptp_masks[((level
)+1)-1]) >> ptp_shifts[((level)+1)-1]) * (1 << 12
)));

return pg;
1119}

1121void
1122pmap_freepage(struct pmap *pmap, struct vm_page *ptp, int level,
  struct pg_to_free *pagelist)
1124{
int lidx;
struct uvm_object *obj;

lidx = level - 1;

obj = &pmap->pm_obj[lidx];
pmap->pm_stats.resident_count--;
if (pmap->pm_ptphint[lidx] == ptp)
pmap->pm_ptphint[lidx] = RBT_ROOT(uvm_objtree, &obj->memt)uvm_objtree_RBT_ROOT(&obj->memt);
ptp->wire_count = 0;
uvm_pagerealloc(ptp, NULL((void *)0), 0);
TAILQ_INSERT_TAIL(pagelist, ptp, pageq)do { (ptp)->pageq.tqe_next = ((void *)0); (ptp)->pageq.
tqe_prev = (pagelist)->tqh_last; *(pagelist)->tqh_last =
 (ptp); (pagelist)->tqh_last = &(ptp)->pageq.tqe_next
; } while (0);
1137}

1139void
1140pmap_free_ptp(struct pmap *pmap, struct vm_page *ptp, vaddr_t va,
  struct pg_to_free *pagelist)
1142{
unsigned long index;
int level;
vaddr_t invaladdr;

level = 1;
do {
pmap_freepage(pmap, ptp, level, pagelist);
index = pl_i(va, level + 1)(((((va) & ~0xffff000000000000)) & ptp_masks[(level +
 1)-1]) >> ptp_shifts[(level + 1)-1]);
pmap_pte_set(&normal_pdes[level - 1][index], 0)_atomic_swap_64((&normal_pdes[level - 1][index]), (0));
if (level == PTP_LEVELS4 - 1 && pmap->pm_pdir_intel != NULL((void *)0)) {
	/* Zap special meltdown PML4e */
	pmap_pte_set(&pmap->pm_pdir_intel[index], 0)_atomic_swap_64((&pmap->pm_pdir_intel[index]), (0));
	DPRINTF("%s: cleared meltdown PML4e @ index %lu "
	    "(va range start 0x%llx)\n", __func__, index,
	    (uint64_t)(index << L4_SHIFT));
}
invaladdr = level == 1 ? (vaddr_t)PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) :
    (vaddr_t)normal_pdes[level - 2];
pmap_tlb_shootpage(pmap, invaladdr + index * PAGE_SIZE(1 << 12),
    pmap_is_curpmap(curpcb({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curpcb->pcb_pmap));
if (level < PTP_LEVELS4 - 1) {
	ptp = pmap_find_ptp(pmap, va, (paddr_t)-1, level + 1);
	ptp->wire_count--;
	if (ptp->wire_count > 1)
		break;
}
} while (++level < PTP_LEVELS4);
1170}

1172/*
* pmap_get_ptp: get a PTP (if there isn't one, allocate a new one)
*
* => pmap should NOT be pmap_kernel()
*/

1178struct vm_page *
1179pmap_get_ptp(struct pmap *pmap, vaddr_t va)
1180{
struct vm_page *ptp, *pptp;
int i;
unsigned long index;
pd_entry_t *pva, *pva_intel;
paddr_t ppa, pa;
struct uvm_object *obj;

ptp = NULL((void *)0);
pa = (paddr_t)-1;

/*
* Loop through all page table levels seeing if we need to
* add a new page to that level.
*/
for (i = PTP_LEVELS4; i > 1; i--) {
/*
 * Save values from previous round.
 */
pptp = ptp;
ppa = pa;

index = pl_i(va, i)(((((va) & ~0xffff000000000000)) & ptp_masks[(i)-1]) >>
 ptp_shifts[(i)-1]);
pva = normal_pdes[i - 2];

if (pmap_valid_entry(pva[index])((pva[index]) & 0x0000000000000001UL)) {
	ppa = pva[index] & PG_FRAME0x000ffffffffff000UL;
	ptp = NULL((void *)0);
	continue;
}

obj = &pmap->pm_obj[i-2];
ptp = uvm_pagealloc(obj, ptp_va2o(va, i - 1)((((((va) & ~0xffff000000000000)) & ptp_masks[((i - 1
)+1)-1]) >> ptp_shifts[((i - 1)+1)-1]) * (1 << 12
)), NULL((void *)0),
    UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002);

if (ptp == NULL((void *)0))
	return NULL((void *)0);

atomic_clearbits_intx86_atomic_clearbits_u32(&ptp->pg_flags, PG_BUSY0x00000001);
ptp->wire_count = 1;
pmap->pm_ptphint[i - 2] = ptp;
pa = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr);
pva[index] = (pd_entry_t) (pa | PG_u0x0000000000000004UL | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL);

/*
 * Meltdown Special case - if we are adding a new PML4e for
 * usermode addresses, just copy the PML4e to the U-K page
 * table.
 */
if (pmap->pm_pdir_intel != NULL((void *)0) && i == PTP_LEVELS4 &&
    va < VM_MAXUSER_ADDRESS0x00007f7fffffc000) {
	pva_intel = pmap->pm_pdir_intel;
	pva_intel[index] = pva[index];
	DPRINTF("%s: copying usermode PML4e (content=0x%llx) "
	    "from 0x%llx -> 0x%llx\n", __func__, pva[index],
	    (uint64_t)&pva[index], (uint64_t)&pva_intel[index]);
}

pmap->pm_stats.resident_count++;
/*
 * If we're not in the top level, increase the
 * wire count of the parent page.
 */
if (i < PTP_LEVELS4) {
	if (pptp == NULL((void *)0))
		pptp = pmap_find_ptp(pmap, va, ppa, i);
1246#ifdef DIAGNOSTIC1
	if (pptp == NULL((void *)0))
		panic("%s: pde page disappeared", __func__);
1249#endif
	pptp->wire_count++;
}
}

/*
* ptp is not NULL if we just allocated a new ptp. If it's
* still NULL, we must look up the existing one.
*/
if (ptp == NULL((void *)0)) {
ptp = pmap_find_ptp(pmap, va, ppa, 1);
1260#ifdef DIAGNOSTIC1
if (ptp == NULL((void *)0)) {
	printf("va %lx ppa %lx\n", (unsigned long)va,
	    (unsigned long)ppa);
	panic("%s: unmanaged user PTP", __func__);
}
1266#endif
}

pmap->pm_ptphint[0] = ptp;
return(ptp);
1271}

1273/*
* p m a p  l i f e c y c l e   f u n c t i o n s
*/

1277/*
* pmap_pdp_ctor: constructor for the PDP cache.
*/

1281void
1282pmap_pdp_ctor(pd_entry_t *pdir)
1283{
paddr_t pdirpa;
int npde, i;
struct pmap *kpm = pmap_kernel()(&kernel_pmap_store);

/* fetch the physical address of the page directory. */
(void) pmap_extract(kpm, (vaddr_t) pdir, &pdirpa);

/* zero init area */
memset(pdir, 0, PDIR_SLOT_PTE * sizeof(pd_entry_t))__builtin_memset((pdir), (0), (255 * sizeof(pd_entry_t)));

/* put in recursive PDE to map the PTEs */
pdir[PDIR_SLOT_PTE255] = pdirpa | PG_V0x0000000000000001UL | PG_KW0x0000000000000002UL | pg_nx;

npde = nkptp[PTP_LEVELS4 - 1];

/* put in kernel VM PDEs */
memcpy(&pdir[PDIR_SLOT_KERN], &PDP_BASE[PDIR_SLOT_KERN],__builtin_memcpy((&pdir[256]), (&((pd_entry_t *)((char
 *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t
 *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) +
* (1ULL << 21))) + 255 * (1ULL << 12)))[256]
), (npde * sizeof(pd_entry_t)))
   npde * sizeof(pd_entry_t))__builtin_memcpy((&pdir[256]), (&((pd_entry_t *)((char
 *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t
 *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) +
* (1ULL << 21))) + 255 * (1ULL << 12)))[256]
), (npde * sizeof(pd_entry_t)));

/* zero the rest */
memset(&pdir[PDIR_SLOT_KERN + npde], 0,__builtin_memset((&pdir[256 + npde]), (0), ((((1 <<
 12) / (sizeof (pd_entry_t))) - (256 + npde)) * sizeof(pd_entry_t
)))
   (NTOPLEVEL_PDES - (PDIR_SLOT_KERN + npde)) * sizeof(pd_entry_t))__builtin_memset((&pdir[256 + npde]), (0), ((((1 <<
 12) / (sizeof (pd_entry_t))) - (256 + npde)) * sizeof(pd_entry_t
)));

for (i = 0; i < NUM_L4_SLOT_DIRECT4; i++)
pdir[PDIR_SLOT_DIRECT(511 - 4) + i] = kpm->pm_pdir[PDIR_SLOT_DIRECT(511 - 4) + i];

1310#if VM_MIN_KERNEL_ADDRESS0xffff800000000000 != KERNBASE0xffffffff80000000
pdir[pl4_pi(KERNBASE)(((((0xffffffff80000000) & ~0xffff000000000000)) & 0x0000ff8000000000UL
) >> 39)] = PDP_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
 (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
 << 12)))[pl4_pi(KERNBASE)(((((0xffffffff80000000) & ~0xffff000000000000)) & 0x0000ff8000000000UL
) >> 39)];
1312#endif
1313}

1315void
1316pmap_pdp_ctor_intel(pd_entry_t *pdir)
1317{
struct pmap *kpm = pmap_kernel()(&kernel_pmap_store);

/* Copy PML4es from pmap_kernel's U-K view */
memcpy(pdir, kpm->pm_pdir_intel, PAGE_SIZE)__builtin_memcpy((pdir), (kpm->pm_pdir_intel), ((1 <<
 12)));
1322}

1324/*
* pmap_create: create a pmap
*
* => note: old pmap interface took a "size" args which allowed for
*	the creation of "software only" pmaps (not in bsd).
*/

1331struct pmap *
1332pmap_create(void)
1333{
struct pmap *pmap;
int i;

pmap = pool_get(&pmap_pmap_pool, PR_WAITOK0x0001);

mtx_init(&pmap->pm_mtx, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&pmap->
pm_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9
 : ((0xa)))); } while (0);

/* init uvm_object */
for (i = 0; i < PTP_LEVELS4 - 1; i++) {
uvm_obj_init(&pmap->pm_obj[i], &pmap_pager, 1);
pmap->pm_ptphint[i] = NULL((void *)0);
}
pmap->pm_stats.wired_count = 0;
pmap->pm_stats.resident_count = 1;	/* count the PDP allocd below */
pmap->pm_type = PMAP_TYPE_NORMAL1;
pmap->eptp = 0;

/* allocate PDP */

/*
* note that there is no need to splvm to protect us from
* malloc since malloc allocates out of a submap and we should
* have already allocated kernel PTPs to cover the range...
*/

pmap->pm_pdir = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001);
pmap_pdp_ctor(pmap->pm_pdir);

pmap->pm_pdirpa = pmap->pm_pdir[PDIR_SLOT_PTE255] & PG_FRAME0x000ffffffffff000UL;

/*
* Intel CPUs need a special page table to be used during usermode
* execution, one that lacks all kernel mappings.
*/
if (cpu_meltdown) {
pmap->pm_pdir_intel = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001);
pmap_pdp_ctor_intel(pmap->pm_pdir_intel);
pmap->pm_stats.resident_count++;
if (!pmap_extract(pmap_kernel()(&kernel_pmap_store), (vaddr_t)pmap->pm_pdir_intel,
    &pmap->pm_pdirpa_intel))
	panic("%s: unknown PA mapping for meltdown PML4",
	    __func__);
} else {
pmap->pm_pdir_intel = NULL((void *)0);
pmap->pm_pdirpa_intel = 0;
}

mtx_enter(&pmaps_lock);
LIST_INSERT_HEAD(&pmaps, pmap, pm_list)do { if (((pmap)->pm_list.le_next = (&pmaps)->lh_first
) != ((void *)0)) (&pmaps)->lh_first->pm_list.le_prev
 = &(pmap)->pm_list.le_next; (&pmaps)->lh_first
 = (pmap); (pmap)->pm_list.le_prev = &(&pmaps)->
lh_first; } while (0);
mtx_leave(&pmaps_lock);
return (pmap);
1385}

1387/*
* pmap_destroy: drop reference count on pmap.   free pmap if
*	reference count goes to zero.
*/

1392void
1393pmap_destroy(struct pmap *pmap)
1394{
struct vm_page *pg;
int refs;
int i;

/*
* drop reference count
*/

refs = atomic_dec_int_nv(&pmap->pm_obj[0].uo_refs)_atomic_sub_int_nv((&pmap->pm_obj[0].uo_refs), 1);
if (refs > 0) {
return;
}

/*
* remove it from global list of pmaps
*/
mtx_enter(&pmaps_lock);
LIST_REMOVE(pmap, pm_list)do { if ((pmap)->pm_list.le_next != ((void *)0)) (pmap)->
pm_list.le_next->pm_list.le_prev = (pmap)->pm_list.le_prev
; *(pmap)->pm_list.le_prev = (pmap)->pm_list.le_next; (
(pmap)->pm_list.le_prev) = ((void *)-1); ((pmap)->pm_list
.le_next) = ((void *)-1); } while (0);
mtx_leave(&pmaps_lock);

/*
* free any remaining PTPs
*/

for (i = 0; i < PTP_LEVELS4 - 1; i++) {
while ((pg = RBT_ROOT(uvm_objtree,uvm_objtree_RBT_ROOT(&pmap->pm_obj[i].memt)
    &pmap->pm_obj[i].memt)uvm_objtree_RBT_ROOT(&pmap->pm_obj[i].memt)) != NULL((void *)0)) {
	KASSERT((pg->pg_flags & PG_BUSY) == 0)(((pg->pg_flags & 0x00000001) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c", 1422,
 "(pg->pg_flags & PG_BUSY) == 0"));

	pg->wire_count = 0;
	pmap->pm_stats.resident_count--;

	uvm_pagefree(pg);
}
}

pool_put(&pmap_pdp_pool, pmap->pm_pdir);

if (pmap->pm_pdir_intel != NULL((void *)0)) {
pmap->pm_stats.resident_count--;
pool_put(&pmap_pdp_pool, pmap->pm_pdir_intel);
}

pool_put(&pmap_pmap_pool, pmap);
1439}

1441/*
*	Add a reference to the specified pmap.
*/

1445void
1446pmap_reference(struct pmap *pmap)
1447{
atomic_inc_int(&pmap->pm_obj[0].uo_refs)_atomic_inc_int(&pmap->pm_obj[0].uo_refs);
1449}

1451/*
* pmap_activate: activate a process' pmap (fill in %cr3)
*
* => called from cpu_fork() and when switching pmaps during exec
* => if p is the curproc, then load it into the MMU
*/

1458void
1459pmap_activate(struct proc *p)
1460{
struct pcb *pcb = &p->p_addr->u_pcb;
struct pmap *pmap = p->p_vmspace->vm_map.pmap;

pcb->pcb_pmap = pmap;
pcb->pcb_cr3 = pmap->pm_pdirpa;
pcb->pcb_cr3 |= (pmap != pmap_kernel()(&kernel_pmap_store)) ? cr3_pcid_proc :
   (PCID_KERN0 | cr3_reuse_pcid);

if (p != curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc)
return;

if ((p->p_flag & P_SYSTEM0x00000200) == 0) {
struct cpu_info *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});

/* mark the pmap in use by this processor */
self->ci_proc_pmap = pmap;

/* in case we return to userspace without context switching */
if (cpu_meltdown) {
	self->ci_kern_cr3 = pcb->pcb_cr3 | cr3_reuse_pcid;
	self->ci_user_cr3 = pmap->pm_pdirpa_intel |
	    cr3_pcid_proc_intel;
}
}

lcr3(pcb->pcb_cr3);
1487}

1489/*
* pmap_deactivate: deactivate a process' pmap
*/

1493void
1494pmap_deactivate(struct proc *p)
1495{
if ((p->p_flag & P_SYSTEM0x00000200) == 0) {
struct cpu_info *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});

/*
 * mark the pmap no longer in use by this processor.
 */
KASSERT(self->ci_proc_pmap == p->p_vmspace->vm_map.pmap)((self->ci_proc_pmap == p->p_vmspace->vm_map.pmap) ?
 (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c"
, 1502, "self->ci_proc_pmap == p->p_vmspace->vm_map.pmap"
));
self->ci_proc_pmap = NULL((void *)0);
}
1505}

1507/*
* end of lifecycle functions
*/

1511/*
* some misc. functions
*/

1515int
1516pmap_pdes_valid(vaddr_t va, pd_entry_t *lastpde)
1517{
int i;
unsigned long index;
pd_entry_t pde;

for (i = PTP_LEVELS4; i > 1; i--) {
index = pl_i(va, i)(((((va) & ~0xffff000000000000)) & ptp_masks[(i)-1]) >>
 ptp_shifts[(i)-1]);
pde = normal_pdes[i - 2][index];
if (!pmap_valid_entry(pde)((pde) & 0x0000000000000001UL))
	return 0;
}
if (lastpde != NULL((void *)0))
*lastpde = pde;
return 1;
1531}

1533/*
* pmap_extract: extract a PA for the given VA
*/

1537int
1538pmap_extract(struct pmap *pmap, vaddr_t va, paddr_t *pap)
1539{
pt_entry_t *ptes, pte;
int level, offs;

if (pmap == pmap_kernel()(&kernel_pmap_store) && va >= PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) &&
12
←
Assuming the condition is false→
   va < PMAP_DIRECT_END((((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000
))) {
*pap = va - PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000));
return 1;
}

if (pmap != pmap_kernel()(&kernel_pmap_store))
13
←
Taking false branch→
mtx_enter(&pmap->pm_mtx);

level = pmap_find_pte_direct(pmap, va, &ptes, &offs);
pte = ptes[offs];

if (pmap != pmap_kernel()(&kernel_pmap_store))
14
←
Taking false branch→
mtx_leave(&pmap->pm_mtx);

if (__predict_true(level == 0 && pmap_valid_entry(pte))__builtin_expect(((level == 0 && ((pte) & 0x0000000000000001UL
)) != 0), 1)) {
15
←
Assuming 'level' is not equal to 0→
if (pap != NULL((void *)0))
	*pap = (pte & PG_FRAME0x000ffffffffff000UL) | (va & PAGE_MASK((1 << 12) - 1));
return 1;
}
if (level == 1 && (pte & (PG_PS0x0000000000000080UL|PG_V0x0000000000000001UL)) == (PG_PS0x0000000000000080UL|PG_V0x0000000000000001UL)) {
16
←
Assuming 'level' is not equal to 1→
if (pap != NULL((void *)0))
	*pap = (pte & PG_LGFRAME0x000fffffffe00000UL) | (va & PAGE_MASK_L2((1ULL << 21) - 1));
return 1;
}

return 0;
17
←
Returning without writing to '*'→
1570}

1572/*
* pmap_zero_page: zero a page
*/

1576void
1577pmap_zero_page(struct vm_page *pg)
1578{
pagezero(pmap_map_direct(pg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (((pg)->phys_addr))));
1580}

1582/*
* pmap_flush_cache: flush the cache for a virtual address.
*/
1585void
1586pmap_flush_cache(vaddr_t addr, vsize_t len)
1587{
vaddr_t	i;

if (curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_cflushsz == 0) {
wbinvd_on_all_cpus();
return;
}

/* all cpus that have clflush also have mfence. */
mfence();
for (i = addr; i < addr + len; i += curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_cflushsz)
clflush(i);
mfence();
1600}

1602/*
* pmap_copy_page: copy a page
*/

1606void
1607pmap_copy_page(struct vm_page *srcpg, struct vm_page *dstpg)
1608{
vaddr_t srcva = pmap_map_direct(srcpg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (((srcpg)->phys_addr)));
vaddr_t dstva = pmap_map_direct(dstpg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (((dstpg)->phys_addr)));

memcpy((void *)dstva, (void *)srcva, PAGE_SIZE)__builtin_memcpy(((void *)dstva), ((void *)srcva), ((1 <<
 12)));
1613}

1615/*
* p m a p   r e m o v e   f u n c t i o n s
*
* functions that remove mappings
*/

1621/*
* pmap_remove_ptes: remove PTEs from a PTP
*
* => PTP must be mapped into KVA
* => PTP should be null if pmap == pmap_kernel()
*/

1628void
1629pmap_remove_ptes(struct pmap *pmap, struct vm_page *ptp, vaddr_t ptpva,
  vaddr_t startva, vaddr_t endva, int flags, struct pv_entry **free_pvs)
1631{
struct pv_entry *pve;
pt_entry_t *pte = (pt_entry_t *) ptpva;
struct vm_page *pg;
pt_entry_t opte;

/*
* note that ptpva points to the PTE that maps startva.   this may
* or may not be the first PTE in the PTP.
*
* we loop through the PTP while there are still PTEs to look at
* and the wire_count is greater than 1 (because we use the wire_count
* to keep track of the number of real PTEs in the PTP).
*/

for (/*null*/; startva < endva && (ptp == NULL((void *)0) || ptp->wire_count > 1)
	     ; pte++, startva += PAGE_SIZE(1 << 12)) {
if (!pmap_valid_entry(*pte)((*pte) & 0x0000000000000001UL))
	continue;			/* VA not mapped */
if ((flags & PMAP_REMOVE_SKIPWIRED1) && (*pte & PG_W0x0000000000000200UL)) {
	continue;
}

/* atomically save the old PTE and zap! it */
opte = pmap_pte_set(pte, 0)_atomic_swap_64((pte), (0));

if (opte & PG_W0x0000000000000200UL)
	pmap->pm_stats.wired_count--;
pmap->pm_stats.resident_count--;

if (ptp != NULL((void *)0))
	ptp->wire_count--;		/* dropping a PTE */

pg = PHYS_TO_VM_PAGE(opte & PG_FRAME0x000ffffffffff000UL);

/*
 * if we are not on a pv list we are done.
 */

if ((opte & PG_PVLIST0x0000000000000400UL) == 0) {
1671#ifdef DIAGNOSTIC1
	if (pg != NULL((void *)0))
		panic("%s: managed page without PG_PVLIST: "
		    "va 0x%lx, opte 0x%llx", __func__,
		    startva, opte);
1676#endif
	continue;
}

1680#ifdef DIAGNOSTIC1
if (pg == NULL((void *)0))
	panic("%s: unmanaged page marked PG_PVLIST: "
	    "va 0x%lx, opte 0x%llx", __func__,
	    startva, opte);
1685#endif

/* sync R/M bits */
pmap_sync_flags_pte(pg, opte);
pve = pmap_remove_pv(pg, pmap, startva);
if (pve != NULL((void *)0)) {
	pve->pv_next = *free_pvs;
	*free_pvs = pve;
}

/* end of "for" loop: time for next pte */
}
1697}

1699/*
* pmap_remove_pte: remove a single PTE from a PTP
*
* => PTP must be mapped into KVA
* => PTP should be null if pmap == pmap_kernel()
* => returns true if we removed a mapping
*/

1707int
1708pmap_remove_pte(struct pmap *pmap, struct vm_page *ptp, pt_entry_t *pte,
  vaddr_t va, int flags, struct pv_entry **free_pvs)
1710{
struct pv_entry *pve;
struct vm_page *pg;
pt_entry_t opte;

if (!pmap_valid_entry(*pte)((*pte) & 0x0000000000000001UL))
return 0;		/* VA not mapped */
if ((flags & PMAP_REMOVE_SKIPWIRED1) && (*pte & PG_W0x0000000000000200UL)) {
return 0;
}

/* atomically save the old PTE and zap! it */
opte = pmap_pte_set(pte, 0)_atomic_swap_64((pte), (0));

if (opte & PG_W0x0000000000000200UL)
pmap->pm_stats.wired_count--;
pmap->pm_stats.resident_count--;

if (ptp != NULL((void *)0))
ptp->wire_count--;		/* dropping a PTE */

pg = PHYS_TO_VM_PAGE(opte & PG_FRAME0x000ffffffffff000UL);

/*
* if we are not on a pv list we are done.
*/
if ((opte & PG_PVLIST0x0000000000000400UL) == 0) {
1737#ifdef DIAGNOSTIC1
if (pg != NULL((void *)0))
	panic("%s: managed page without PG_PVLIST: "
	    "va 0x%lx, opte 0x%llx", __func__, va, opte);
1741#endif
return 1;
}

1745#ifdef DIAGNOSTIC1
if (pg == NULL((void *)0))
panic("%s: unmanaged page marked PG_PVLIST: "
    "va 0x%lx, opte 0x%llx", __func__, va, opte);
1749#endif

/* sync R/M bits */
pmap_sync_flags_pte(pg, opte);
pve = pmap_remove_pv(pg, pmap, va);
if (pve != NULL((void *)0)) {
pve->pv_next = *free_pvs;
*free_pvs = pve;
}

return 1;
1760}

1762/*
* pmap_remove: top level mapping removal function
*
* => caller should not be holding any pmap locks
*/

1768void
1769pmap_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva)
1770{
if (pmap->pm_type == PMAP_TYPE_EPT2)
pmap_remove_ept(pmap, sva, eva);
else
pmap_do_remove(pmap, sva, eva, PMAP_REMOVE_ALL0);
1775}

1777/*
* pmap_do_remove: mapping removal guts
*
* => caller should not be holding any pmap locks
*/

1783void
1784pmap_do_remove(struct pmap *pmap, vaddr_t sva, vaddr_t eva, int flags)
1785{
pd_entry_t pde;
int result;
paddr_t ptppa;
vaddr_t blkendva;
struct vm_page *ptp;
struct pv_entry *pve;
struct pv_entry *free_pvs = NULL((void *)0);
vaddr_t va;
int shootall = 0, shootself;
struct pg_to_free empty_ptps;
paddr_t scr3;

TAILQ_INIT(&empty_ptps)do { (&empty_ptps)->tqh_first = ((void *)0); (&empty_ptps
)->tqh_last = &(&empty_ptps)->tqh_first; } while
 (0);

scr3 = pmap_map_ptes(pmap);
shootself = (scr3 == 0);

/*
* removing one page?  take shortcut function.
*/

if (sva + PAGE_SIZE(1 << 12) == eva) {
if (pmap_pdes_valid(sva, &pde)) {

	/* PA of the PTP */
	ptppa = pde & PG_FRAME0x000ffffffffff000UL;

	/* get PTP if non-kernel mapping */

	if (pmap == pmap_kernel()(&kernel_pmap_store)) {
		/* we never free kernel PTPs */
		ptp = NULL((void *)0);
	} else {
		ptp = pmap_find_ptp(pmap, sva, ptppa, 1);
1820#ifdef DIAGNOSTIC1
		if (ptp == NULL((void *)0))
			panic("%s: unmanaged PTP detected "
			    "in shortcut path", __func__);
1824#endif
	}

	/* do it! */
	result = pmap_remove_pte(pmap, ptp,
	    &PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(sva)(((((sva) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)], sva, flags, &free_pvs);

	/*
	 * if mapping removed and the PTP is no longer
	 * being used, free it!
	 */

	if (result && ptp && ptp->wire_count <= 1)
		pmap_free_ptp(pmap, ptp, sva, &empty_ptps);
	pmap_tlb_shootpage(pmap, sva, shootself);
	pmap_unmap_ptes(pmap, scr3);
	pmap_tlb_shootwait();
} else {
	pmap_unmap_ptes(pmap, scr3);
}

goto cleanup;
}

if ((eva - sva > 32 * PAGE_SIZE(1 << 12)) && sva < VM_MIN_KERNEL_ADDRESS0xffff800000000000)
shootall = 1;

for (va = sva; va < eva; va = blkendva) {
/* determine range of block */
blkendva = x86_round_pdr(va + 1)((((unsigned long)(va + 1)) + ((1ULL << 21) - 1)) &
 ~((1ULL << 21) - 1));
if (blkendva > eva)
	blkendva = eva;

/*
 * XXXCDC: our PTE mappings should never be removed
 * with pmap_remove!  if we allow this (and why would
 * we?) then we end up freeing the pmap's page
 * directory page (PDP) before we are finished using
 * it when we hit it in the recursive mapping.  this
 * is BAD.
 *
 * long term solution is to move the PTEs out of user
 * address space.  and into kernel address space (up
 * with APTE).  then we can set VM_MAXUSER_ADDRESS to
 * be VM_MAX_ADDRESS.
 */

if (pl_i(va, PTP_LEVELS)(((((va) & ~0xffff000000000000)) & ptp_masks[(4)-1]) >>
 ptp_shifts[(4)-1]) == PDIR_SLOT_PTE255)
	/* XXXCDC: ugly hack to avoid freeing PDP here */
	continue;

if (!pmap_pdes_valid(va, &pde))
	continue;

/* PA of the PTP */
ptppa = pde & PG_FRAME0x000ffffffffff000UL;

/* get PTP if non-kernel mapping */
if (pmap == pmap_kernel()(&kernel_pmap_store)) {
	/* we never free kernel PTPs */
	ptp = NULL((void *)0);
} else {
	ptp = pmap_find_ptp(pmap, va, ptppa, 1);
1887#ifdef DIAGNOSTIC1
	if (ptp == NULL((void *)0))
		panic("%s: unmanaged PTP detected", __func__);
1890#endif
}
pmap_remove_ptes(pmap, ptp, (vaddr_t)&PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)],
    va, blkendva, flags, &free_pvs);

/* if PTP is no longer being used, free it! */
if (ptp && ptp->wire_count <= 1) {
	pmap_free_ptp(pmap, ptp, va, &empty_ptps);
}
}

if (shootall)
pmap_tlb_shoottlb(pmap, shootself);
else
pmap_tlb_shootrange(pmap, sva, eva, shootself);

pmap_unmap_ptes(pmap, scr3);
pmap_tlb_shootwait();

1909cleanup:
while ((pve = free_pvs) != NULL((void *)0)) {
free_pvs = pve->pv_next;
pool_put(&pmap_pv_pool, pve);
}

while ((ptp = TAILQ_FIRST(&empty_ptps)((&empty_ptps)->tqh_first)) != NULL((void *)0)) {
TAILQ_REMOVE(&empty_ptps, ptp, pageq)do { if (((ptp)->pageq.tqe_next) != ((void *)0)) (ptp)->
pageq.tqe_next->pageq.tqe_prev = (ptp)->pageq.tqe_prev;
 else (&empty_ptps)->tqh_last = (ptp)->pageq.tqe_prev
; *(ptp)->pageq.tqe_prev = (ptp)->pageq.tqe_next; ((ptp
)->pageq.tqe_prev) = ((void *)-1); ((ptp)->pageq.tqe_next
) = ((void *)-1); } while (0);
uvm_pagefree(ptp);
}
1919}

1921/*
* pmap_page_remove: remove a managed vm_page from all pmaps that map it
*
* => R/M bits are sync'd back to attrs
*/

1927void
1928pmap_page_remove(struct vm_page *pg)
1929{
struct pv_entry *pve;
struct pmap *pm;
pt_entry_t opte;
1933#ifdef DIAGNOSTIC1
pd_entry_t pde;
1935#endif
struct pg_to_free empty_ptps;
struct vm_page *ptp;
paddr_t scr3;
int shootself;

TAILQ_INIT(&empty_ptps)do { (&empty_ptps)->tqh_first = ((void *)0); (&empty_ptps
)->tqh_last = &(&empty_ptps)->tqh_first; } while
 (0);

mtx_enter(&pg->mdpage.pv_mtx);
while ((pve = pg->mdpage.pv_list) != NULL((void *)0)) {
pmap_reference(pve->pv_pmap);
pm = pve->pv_pmap;
mtx_leave(&pg->mdpage.pv_mtx);

/* XXX use direct map? */
scr3 = pmap_map_ptes(pm);	/* locks pmap */
shootself = (scr3 == 0);

/*
 * We dropped the pvlist lock before grabbing the pmap
 * lock to avoid lock ordering problems.  This means
 * we have to check the pvlist again since somebody
 * else might have modified it.  All we care about is
 * that the pvlist entry matches the pmap we just
 * locked.  If it doesn't, unlock the pmap and try
 * again.
 */
mtx_enter(&pg->mdpage.pv_mtx);
if ((pve = pg->mdpage.pv_list) == NULL((void *)0) ||
    pve->pv_pmap != pm) {
	mtx_leave(&pg->mdpage.pv_mtx);
	pmap_unmap_ptes(pm, scr3);	/* unlocks pmap */
	pmap_destroy(pm);
	mtx_enter(&pg->mdpage.pv_mtx);
	continue;
}

pg->mdpage.pv_list = pve->pv_next;
mtx_leave(&pg->mdpage.pv_mtx);

1975#ifdef DIAGNOSTIC1
if (pve->pv_ptp != NULL((void *)0) && pmap_pdes_valid(pve->pv_va, &pde) &&
   (pde & PG_FRAME0x000ffffffffff000UL) != VM_PAGE_TO_PHYS(pve->pv_ptp)((pve->pv_ptp)->phys_addr)) {
	printf("%s: pg=%p: va=%lx, pv_ptp=%p\n", __func__,
	       pg, pve->pv_va, pve->pv_ptp);
	printf("%s: PTP's phys addr: "
	       "actual=%lx, recorded=%lx\n", __func__,
	       (unsigned long)(pde & PG_FRAME0x000ffffffffff000UL),
		VM_PAGE_TO_PHYS(pve->pv_ptp)((pve->pv_ptp)->phys_addr));
	panic("%s: mapped managed page has "
	      "invalid pv_ptp field", __func__);
}
1987#endif

/* atomically save the old PTE and zap it */
opte = pmap_pte_set(&PTE_BASE[pl1_i(pve->pv_va)], 0)_atomic_swap_64((&((pt_entry_t *) (255 * (1ULL << 39
)))[(((((pve->pv_va) & ~0xffff000000000000)) & (((
0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)|0x00000000001ff000UL)) >> 12)]), (0));

if (opte & PG_W0x0000000000000200UL)
	pve->pv_pmap->pm_stats.wired_count--;
pve->pv_pmap->pm_stats.resident_count--;

pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va, shootself);

pmap_sync_flags_pte(pg, opte);

/* update the PTP reference count.  free if last reference. */
if (pve->pv_ptp != NULL((void *)0)) {
	pve->pv_ptp->wire_count--;
	if (pve->pv_ptp->wire_count <= 1) {
		pmap_free_ptp(pve->pv_pmap, pve->pv_ptp,
		    pve->pv_va, &empty_ptps);
	}
}
pmap_unmap_ptes(pve->pv_pmap, scr3);	/* unlocks pmap */
pmap_destroy(pve->pv_pmap);
pool_put(&pmap_pv_pool, pve);
mtx_enter(&pg->mdpage.pv_mtx);
}
mtx_leave(&pg->mdpage.pv_mtx);

pmap_tlb_shootwait();

while ((ptp = TAILQ_FIRST(&empty_ptps)((&empty_ptps)->tqh_first)) != NULL((void *)0)) {
TAILQ_REMOVE(&empty_ptps, ptp, pageq)do { if (((ptp)->pageq.tqe_next) != ((void *)0)) (ptp)->
pageq.tqe_next->pageq.tqe_prev = (ptp)->pageq.tqe_prev;
 else (&empty_ptps)->tqh_last = (ptp)->pageq.tqe_prev
; *(ptp)->pageq.tqe_prev = (ptp)->pageq.tqe_next; ((ptp
)->pageq.tqe_prev) = ((void *)-1); ((ptp)->pageq.tqe_next
) = ((void *)-1); } while (0);
uvm_pagefree(ptp);
}
2021}

2023/*
* p m a p   a t t r i b u t e  f u n c t i o n s
* functions that test/change managed page's attributes
* since a page can be mapped multiple times we must check each PTE that
* maps it by going down the pv lists.
*/

2030/*
* pmap_test_attrs: test a page's attributes
*/

2034int
2035pmap_test_attrs(struct vm_page *pg, unsigned int testbits)
2036{
struct pv_entry *pve;
pt_entry_t *ptes;
int level, offs;
u_long mybits, testflags;

testflags = pmap_pte2flags(testbits);

if (pg->pg_flags & testflags)
return 1;

mybits = 0;
mtx_enter(&pg->mdpage.pv_mtx);
for (pve = pg->mdpage.pv_list; pve != NULL((void *)0) && mybits == 0;
   pve = pve->pv_next) {
level = pmap_find_pte_direct(pve->pv_pmap, pve->pv_va, &ptes,
    &offs);
mybits |= (ptes[offs] & testbits);
}
mtx_leave(&pg->mdpage.pv_mtx);

if (mybits == 0)
return 0;

atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, pmap_pte2flags(mybits));

return 1;
2063}

2065/*
* pmap_clear_attrs: change a page's attributes
*
* => we return 1 if we cleared one of the bits we were asked to
*/

2071int
2072pmap_clear_attrs(struct vm_page *pg, unsigned long clearbits)
2073{
struct pv_entry *pve;
pt_entry_t *ptes, opte;
u_long clearflags;
int result, level, offs;

clearflags = pmap_pte2flags(clearbits);

result = pg->pg_flags & clearflags;
if (result)
atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, clearflags);

mtx_enter(&pg->mdpage.pv_mtx);
for (pve = pg->mdpage.pv_list; pve != NULL((void *)0); pve = pve->pv_next) {
level = pmap_find_pte_direct(pve->pv_pmap, pve->pv_va, &ptes,
    &offs);
opte = ptes[offs];
if (opte & clearbits) {
	result = 1;
	pmap_pte_clearbits(&ptes[offs], (opte & clearbits))x86_atomic_clearbits_u64(&ptes[offs], (opte & clearbits
));
	pmap_tlb_shootpage(pve->pv_pmap, pve->pv_va,
		pmap_is_curpmap(pve->pv_pmap));
}
}
mtx_leave(&pg->mdpage.pv_mtx);

pmap_tlb_shootwait();

return (result != 0);
2102}

2104/*
* p m a p   p r o t e c t i o n   f u n c t i o n s
*/

2108/*
* pmap_page_protect: change the protection of all recorded mappings
*	of a managed page
*
* => NOTE: this is an inline function in pmap.h
*/

2115/* see pmap.h */

2117/*
* pmap_protect: set the protection in of the pages in a pmap
*
* => NOTE: this is an inline function in pmap.h
*/

2123/* see pmap.h */

2125/*
* pmap_write_protect: write-protect pages in a pmap
*/

2129void
2130pmap_write_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
2131{
pt_entry_t *spte, *epte;
pt_entry_t clear = 0, set = 0;
vaddr_t blockend;
int shootall = 0, shootself;
vaddr_t va;
paddr_t scr3;

scr3 = pmap_map_ptes(pmap);
shootself = (scr3 == 0);

/* should be ok, but just in case ... */
sva &= PG_FRAME0x000ffffffffff000UL;
eva &= PG_FRAME0x000ffffffffff000UL;

if (!(prot & PROT_READ0x01))
set |= pg_xo;
if (!(prot & PROT_WRITE0x02))
clear = PG_RW0x0000000000000002UL;
if (!(prot & PROT_EXEC0x04))
set |= pg_nx;

if ((eva - sva > 32 * PAGE_SIZE(1 << 12)) && sva < VM_MIN_KERNEL_ADDRESS0xffff800000000000)
shootall = 1;

for (va = sva; va < eva ; va = blockend) {
blockend = (va & L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)) + NBPD_L2(1ULL << 21);
if (blockend > eva)
	blockend = eva;

/*
 * XXXCDC: our PTE mappings should never be write-protected!
 *
 * long term solution is to move the PTEs out of user
 * address space.  and into kernel address space (up
 * with APTE).  then we can set VM_MAXUSER_ADDRESS to
 * be VM_MAX_ADDRESS.
 */

/* XXXCDC: ugly hack to avoid freeing PDP here */
if (pl_i(va, PTP_LEVELS)(((((va) & ~0xffff000000000000)) & ptp_masks[(4)-1]) >>
 ptp_shifts[(4)-1]) == PDIR_SLOT_PTE255)
	continue;

/* empty block? */
if (!pmap_pdes_valid(va, NULL((void *)0)))
	continue;

2178#ifdef DIAGNOSTIC1
if (va >= VM_MAXUSER_ADDRESS0x00007f7fffffc000 && va < VM_MAX_ADDRESS0x00007fbfdfeff000)
	panic("%s: PTE space", __func__);
2181#endif

spte = &PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)];
epte = &PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(blockend)(((((blockend) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)];

for (/*null */; spte < epte ; spte++) {
	if (!pmap_valid_entry(*spte)((*spte) & 0x0000000000000001UL))
		continue;
	pmap_pte_clearbits(spte, clear)x86_atomic_clearbits_u64(spte, clear);
	pmap_pte_setbits(spte, set)x86_atomic_setbits_u64(spte, set);
}
}

if (shootall)
pmap_tlb_shoottlb(pmap, shootself);
else
pmap_tlb_shootrange(pmap, sva, eva, shootself);

pmap_unmap_ptes(pmap, scr3);
pmap_tlb_shootwait();
2201}

2203/*
* end of protection functions
*/

2207/*
* pmap_unwire: clear the wired bit in the PTE
*
* => mapping should already be in map
*/

2213void
2214pmap_unwire(struct pmap *pmap, vaddr_t va)
2215{
pt_entry_t *ptes;
int level, offs;

level = pmap_find_pte_direct(pmap, va, &ptes, &offs);

if (level == 0) {

2223#ifdef DIAGNOSTIC1
if (!pmap_valid_entry(ptes[offs])((ptes[offs]) & 0x0000000000000001UL))
	panic("%s: invalid (unmapped) va 0x%lx", __func__, va);
2226#endif
if (__predict_true((ptes[offs] & PG_W) != 0)__builtin_expect((((ptes[offs] & 0x0000000000000200UL) !=
 0) != 0), 1)) {
	pmap_pte_clearbits(&ptes[offs], PG_W)x86_atomic_clearbits_u64(&ptes[offs], 0x0000000000000200UL
);
	pmap->pm_stats.wired_count--;
}
2231#ifdef DIAGNOSTIC1
else {
	printf("%s: wiring for pmap %p va 0x%lx "
	       "didn't change!\n", __func__, pmap, va);
}
2236#endif
}
2238#ifdef DIAGNOSTIC1
else {
panic("%s: invalid PDE", __func__);
}
2242#endif
2243}

2245#if 0
2246/*
* pmap_collect: free resources held by a pmap
*
* => optional function.
* => called when a process is swapped out to free memory.
*/

2253void
2254pmap_collect(struct pmap *pmap)
2255{
/*
* free all of the pt pages by removing the physical mappings
* for its entire address space.
*/

pmap_do_remove(pmap, VM_MIN_ADDRESS(1 << 12), VM_MAX_ADDRESS0x00007fbfdfeff000,
   PMAP_REMOVE_SKIPWIRED1);
2263}
2264#endif

2266void
2267pmap_enter_special(vaddr_t va, paddr_t pa, vm_prot_t prot)
2268{
uint64_t l4idx, l3idx, l2idx, l1idx;
pd_entry_t *pd, *ptp;
paddr_t npa;
struct pmap *pmap = pmap_kernel()(&kernel_pmap_store);
pt_entry_t *ptes;
int level, offs;

/* If CPU is secure, no need to do anything */
if (!cpu_meltdown)
return;

/* Must be kernel VA */
if (va < VM_MIN_KERNEL_ADDRESS0xffff800000000000)
panic("%s: invalid special mapping va 0x%lx requested",
    __func__, va);

if (pmap->pm_pdir_intel == NULL((void *)0))
pmap->pm_pdir_intel = pool_get(&pmap_pdp_pool,
    PR_WAITOK0x0001 | PR_ZERO0x0008);

l4idx = (va & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39; /* PML4E idx */
l3idx = (va & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30; /* PDPTE idx */
l2idx = (va & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21; /* PDE idx */
l1idx = (va & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12; /* PTE idx */

DPRINTF("%s: va=0x%llx pa=0x%llx l4idx=%lld l3idx=%lld "
   "l2idx=%lld l1idx=%lld\n", __func__, (uint64_t)va,
   (uint64_t)pa, l4idx, l3idx, l2idx, l1idx);

/* Start at PML4 / top level */
pd = pmap->pm_pdir_intel;

if (pd == NULL((void *)0))
panic("%s: PML4 not initialized for pmap @ %p", __func__,
    pmap);

/* npa = physaddr of PDPT */
npa = pd[l4idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));

/* Valid PML4e for the 512GB region containing va? */
if (!npa) {
/* No valid PML4E - allocate PDPT page and set PML4E */

ptp = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001 | PR_ZERO0x0008);

if (!pmap_extract(pmap, (vaddr_t)ptp, &npa))
	panic("%s: can't locate PDPT page", __func__);

pd[l4idx] = (npa | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL);

DPRINTF("%s: allocated new PDPT page at phys 0x%llx, "
    "setting PML4e[%lld] = 0x%llx\n", __func__,
    (uint64_t)npa, l4idx, pd[l4idx]);
}

pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa));
if (pd == NULL((void *)0))
panic("%s: can't locate PDPT @ pa=0x%llx", __func__,
    (uint64_t)npa);

/* npa = physaddr of PD page */
npa = pd[l3idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));

/* Valid PDPTe for the 1GB region containing va? */
if (!npa) {
/* No valid PDPTe - allocate PD page and set PDPTe */

ptp = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001 | PR_ZERO0x0008);

if (!pmap_extract(pmap, (vaddr_t)ptp, &npa))
	panic("%s: can't locate PD page", __func__);

pd[l3idx] = (npa | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL);

DPRINTF("%s: allocated new PD page at phys 0x%llx, "
    "setting PDPTe[%lld] = 0x%llx\n", __func__,
    (uint64_t)npa, l3idx, pd[l3idx]);
}

pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa));
if (pd == NULL((void *)0))
panic("%s: can't locate PD page @ pa=0x%llx", __func__,
    (uint64_t)npa);

/* npa = physaddr of PT page */
npa = pd[l2idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));

/* Valid PDE for the 2MB region containing va? */
if (!npa) {
/* No valid PDE - allocate PT page and set PDE */

ptp = pool_get(&pmap_pdp_pool, PR_WAITOK0x0001 | PR_ZERO0x0008);

if (!pmap_extract(pmap, (vaddr_t)ptp, &npa))
	panic("%s: can't locate PT page", __func__);

pd[l2idx] = (npa | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL);

DPRINTF("%s: allocated new PT page at phys 0x%llx, "
    "setting PDE[%lld] = 0x%llx\n", __func__,
    (uint64_t)npa, l2idx, pd[l2idx]);
}

pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa));
if (pd == NULL((void *)0))
panic("%s: can't locate PT page @ pa=0x%llx", __func__,
    (uint64_t)npa);

DPRINTF("%s: setting PTE, PT page @ phys 0x%llx virt 0x%llx prot "
   "0x%llx was 0x%llx\n", __func__, (uint64_t)npa, (uint64_t)pd,
   (uint64_t)prot, (uint64_t)pd[l1idx]);

pd[l1idx] = pa | protection_codes[prot] | PG_V0x0000000000000001UL | PG_W0x0000000000000200UL;

/*
* Look up the corresponding U+K entry.  If we're installing the
* same PA into the U-K map then set the PG_G bit on both and copy
* the cache-control bits from the U+K entry to the U-K entry.
*/
level = pmap_find_pte_direct(pmap, va, &ptes, &offs);
if (__predict_true(level == 0 && pmap_valid_entry(ptes[offs]))__builtin_expect(((level == 0 && ((ptes[offs]) & 0x0000000000000001UL
)) != 0), 1)) {
if (((pd[l1idx] ^ ptes[offs]) & PG_FRAME0x000ffffffffff000UL) == 0) {
	pd[l1idx] |= PG_G0x0000000000000100UL | (ptes[offs] & (PG_N0x0000000000000010UL | PG_WT0x0000000000000008UL));
	ptes[offs] |= PG_G0x0000000000000100UL;
} else {
	DPRINTF("%s: special diffing mapping at %llx\n",
	    __func__, (long long)va);
}
} else
DPRINTF("%s: no U+K mapping for special mapping?\n", __func__);

DPRINTF("%s: setting PTE[%lld] = 0x%llx\n", __func__, l1idx, pd[l1idx]);
2401}

2403void
2404pmap_remove_ept(struct pmap *pmap, vaddr_t sgpa, vaddr_t egpa)
2405{
vaddr_t v;
2407#if NVMM1 > 0
struct vmx_invept_descriptor vid;
2409#endif /* NVMM > 0 */

DPRINTF("%s: sgpa=0x%llx egpa=0x%llx\n", __func__, (uint64_t)sgpa,
   (uint64_t)egpa);
for (v = sgpa; v < egpa + PAGE_SIZE(1 << 12); v += PAGE_SIZE(1 << 12))
pmap_do_remove_ept(pmap, v);

2416#if NVMM1 > 0
if (pmap->eptp != 0) {
memset(&vid, 0, sizeof(vid))__builtin_memset((&vid), (0), (sizeof(vid)));
vid.vid_eptp = pmap->eptp;
DPRINTF("%s: flushing EPT TLB for EPTP 0x%llx\n", __func__,
    vid.vid_eptp);
invept(IA32_VMX_INVEPT_SINGLE_CTX0x1, &vid);
}
2424#endif /* NVMM > 0 */
2425}

2427void
2428pmap_do_remove_ept(struct pmap *pmap, paddr_t gpa)
2429{
uint64_t l4idx, l3idx, l2idx, l1idx;
struct vm_page *pg3, *pg2, *pg1;
paddr_t npa3, npa2, npa1;
pd_entry_t *pd4, *pd3, *pd2, *pd1;
pd_entry_t *pptes;

l4idx = (gpa & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39; /* PML4E idx */
l3idx = (gpa & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30; /* PDPTE idx */
l2idx = (gpa & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21; /* PDE idx */
l1idx = (gpa & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12; /* PTE idx */

/* Start at PML4 / top level */
pd4 = (pd_entry_t *)pmap->pm_pdir;

if (pd4 == NULL((void *)0))
return;

/* npa3 = physaddr of PDPT */
npa3 = pd4[l4idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));
if (!npa3)
return;
pd3 = (pd_entry_t *)PMAP_DIRECT_MAP(npa3)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa3));
pg3 = PHYS_TO_VM_PAGE(npa3);

/* npa2 = physaddr of PD page */
npa2 = pd3[l3idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));
if (!npa2)
return;
pd2 = (pd_entry_t *)PMAP_DIRECT_MAP(npa2)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa2));
pg2 = PHYS_TO_VM_PAGE(npa2);

/* npa1 = physaddr of PT page */
npa1 = pd2[l2idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));
if (!npa1)
return;
pd1 = (pd_entry_t *)PMAP_DIRECT_MAP(npa1)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa1));
pg1 = PHYS_TO_VM_PAGE(npa1);

if (pd1[l1idx] == 0)
return;

pd1[l1idx] = 0;
pg1->wire_count--;
pmap->pm_stats.resident_count--;

if (pg1->wire_count > 1)
return;

pg1->wire_count = 0;
pptes = (pd_entry_t *)PMAP_DIRECT_MAP(npa2)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa2));
pptes[l2idx] = 0;
uvm_pagefree(pg1);
pmap->pm_stats.resident_count--;

pg2->wire_count--;
if (pg2->wire_count > 1)
return;

pg2->wire_count = 0;
pptes = (pd_entry_t *)PMAP_DIRECT_MAP(npa3)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa3));
pptes[l3idx] = 0;
uvm_pagefree(pg2);
pmap->pm_stats.resident_count--;

pg3->wire_count--;
if (pg3->wire_count > 1)
return;

pg3->wire_count = 0;
pptes = pd4;
pptes[l4idx] = 0;
uvm_pagefree(pg3);
pmap->pm_stats.resident_count--;
2503}

2505int
2506pmap_enter_ept(struct pmap *pmap, paddr_t gpa, paddr_t hpa, vm_prot_t prot)
2507{
uint64_t l4idx, l3idx, l2idx, l1idx;
pd_entry_t *pd, npte;
struct vm_page *ptp, *pptp;
paddr_t npa;
struct uvm_object *obj;

if (gpa > MAXDSIZ((paddr_t)128*1024*1024*1024))
return ENOMEM12;

l4idx = (gpa & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39; /* PML4E idx */
l3idx = (gpa & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30; /* PDPTE idx */
l2idx = (gpa & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21; /* PDE idx */
l1idx = (gpa & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12; /* PTE idx */

/* Start at PML4 / top level */
pd = (pd_entry_t *)pmap->pm_pdir;

if (pd == NULL((void *)0))
return ENOMEM12;

/* npa = physaddr of PDPT */
npa = pd[l4idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));

/* Valid PML4e for the 512GB region containing gpa? */
if (!npa) {
/* No valid PML4e - allocate PDPT page and set PML4e */
obj = &pmap->pm_obj[2];	/* PML4 UVM object */
ptp = uvm_pagealloc(obj, ptp_va2o(gpa, 3)((((((gpa) & ~0xffff000000000000)) & ptp_masks[((3)+1
)-1]) >> ptp_shifts[((3)+1)-1]) * (1 << 12)), NULL((void *)0),
    UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002);

if (ptp == NULL((void *)0))
	return ENOMEM12;

/*
 * New PDPT page - we are setting the first entry, so set
 * the wired count to 1
 */
ptp->wire_count = 1;

/* Calculate phys address of this new PDPT page */
npa = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr);

/*
 * Higher levels get full perms; specific permissions are
 * entered at the lowest level.
 */
pd[l4idx] = (npa | EPT_R(1ULL << 0) | EPT_W(1ULL << 1) | EPT_X(1ULL << 2));

pmap->pm_stats.resident_count++;

pptp = ptp;
} else {
/* Already allocated PML4e */
pptp = PHYS_TO_VM_PAGE(npa);
}

pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa));
if (pd == NULL((void *)0))
panic("%s: can't locate PDPT @ pa=0x%llx", __func__,
    (uint64_t)npa);

/* npa = physaddr of PD page */
npa = pd[l3idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));

/* Valid PDPTe for the 1GB region containing gpa? */
if (!npa) {
/* No valid PDPTe - allocate PD page and set PDPTe */
obj = &pmap->pm_obj[1];	/* PDPT UVM object */
ptp = uvm_pagealloc(obj, ptp_va2o(gpa, 2)((((((gpa) & ~0xffff000000000000)) & ptp_masks[((2)+1
)-1]) >> ptp_shifts[((2)+1)-1]) * (1 << 12)), NULL((void *)0),
    UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002);

if (ptp == NULL((void *)0))
	return ENOMEM12;

/*
 * New PD page - we are setting the first entry, so set
 * the wired count to 1
 */
ptp->wire_count = 1;
pptp->wire_count++;

npa = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr);

/*
 * Higher levels get full perms; specific permissions are
 * entered at the lowest level.
 */
pd[l3idx] = (npa | EPT_R(1ULL << 0) | EPT_W(1ULL << 1) | EPT_X(1ULL << 2));

pmap->pm_stats.resident_count++;

pptp = ptp;
} else {
/* Already allocated PDPTe */
pptp = PHYS_TO_VM_PAGE(npa);
}

pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa));
if (pd == NULL((void *)0))
panic("%s: can't locate PD page @ pa=0x%llx", __func__,
    (uint64_t)npa);

/* npa = physaddr of PT page */
npa = pd[l2idx] & PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));

/* Valid PDE for the 2MB region containing gpa? */
if (!npa) {
/* No valid PDE - allocate PT page and set PDE */
obj = &pmap->pm_obj[0];	/* PDE UVM object */
ptp = uvm_pagealloc(obj, ptp_va2o(gpa, 1)((((((gpa) & ~0xffff000000000000)) & ptp_masks[((1)+1
)-1]) >> ptp_shifts[((1)+1)-1]) * (1 << 12)), NULL((void *)0),
    UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002);

if (ptp == NULL((void *)0))
	return ENOMEM12;

pptp->wire_count++;

npa = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr);

/*
 * Higher level get full perms; specific permissions are
 * entered at the lowest level.
 */
pd[l2idx] = (npa | EPT_R(1ULL << 0) | EPT_W(1ULL << 1) | EPT_X(1ULL << 2));

pmap->pm_stats.resident_count++;

} else {
/* Find final ptp */
ptp = PHYS_TO_VM_PAGE(npa);
if (ptp == NULL((void *)0))
	panic("%s: ptp page vanished?", __func__);
}

pd = (pd_entry_t *)PMAP_DIRECT_MAP(npa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (npa));
if (pd == NULL((void *)0))
panic("%s: can't locate PT page @ pa=0x%llx", __func__,
    (uint64_t)npa);

npte = hpa | EPT_WB(6ULL << 3);
if (prot & PROT_READ0x01)
npte |= EPT_R(1ULL << 0);
if (prot & PROT_WRITE0x02)
npte |= EPT_W(1ULL << 1);
if (prot & PROT_EXEC0x04)
npte |= EPT_X(1ULL << 2);

if (pd[l1idx] == 0) {
ptp->wire_count++;
pmap->pm_stats.resident_count++;
} else {
/* XXX flush ept */
}

pd[l1idx] = npte;

return 0;
2665}

2667/*
* pmap_enter: enter a mapping into a pmap
*
* => must be done "now" ... no lazy-evaluation
*/

2673int
2674pmap_enter(struct pmap *pmap, vaddr_t va, paddr_t pa, vm_prot_t prot, int flags)
2675{
pt_entry_t opte, npte;
struct vm_page *ptp, *pg = NULL((void *)0);
struct pv_entry *pve, *opve = NULL((void *)0);
int ptpdelta, wireddelta, resdelta;
int wired = (flags & PMAP_WIRED0x00000010) != 0;
int nocache = (pa & PMAP_NOCACHE0x1) != 0;
int wc = (pa & PMAP_WC0x2) != 0;
int error, shootself;
paddr_t scr3;

if (pmap->pm_type == PMAP_TYPE_EPT2)
return pmap_enter_ept(pmap, va, pa, prot);

KASSERT(!(wc && nocache))((!(wc && nocache)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/arch/amd64/amd64/pmap.c", 2689, "!(wc && nocache)"
));
pa &= PMAP_PA_MASK~((paddr_t)((1 << 12) - 1));

2692#ifdef DIAGNOSTIC1
if (va == (vaddr_t) PDP_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
 (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
 << 12))))
panic("%s: trying to map over PDP!", __func__);

/* sanity check: kernel PTPs should already have been pre-allocated */
if (va >= VM_MIN_KERNEL_ADDRESS0xffff800000000000 &&
   !pmap_valid_entry(pmap->pm_pdir[pl_i(va, PTP_LEVELS)])((pmap->pm_pdir[(((((va) & ~0xffff000000000000)) &
 ptp_masks[(4)-1]) >> ptp_shifts[(4)-1])]) & 0x0000000000000001UL
))
panic("%s: missing kernel PTP for va %lx!", __func__, va);

2701#endif

pve = pool_get(&pmap_pv_pool, PR_NOWAIT0x0002);
if (pve == NULL((void *)0)) {
if (flags & PMAP_CANFAIL0x00000020) {
	error = ENOMEM12;
	goto out;
}
panic("%s: no pv entries available", __func__);
}

/*
* map in ptes and get a pointer to our PTP (unless we are the kernel)
*/

scr3 = pmap_map_ptes(pmap);
shootself = (scr3 == 0);
if (pmap == pmap_kernel()(&kernel_pmap_store)) {
ptp = NULL((void *)0);
} else {
ptp = pmap_get_ptp(pmap, va);
if (ptp == NULL((void *)0)) {
	if (flags & PMAP_CANFAIL0x00000020) {
		pmap_unmap_ptes(pmap, scr3);
		error = ENOMEM12;
		goto out;
	}
	panic("%s: get ptp failed", __func__);
}
}
opte = PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)];		/* old PTE */

/*
* is there currently a valid mapping at our VA?
*/

if (pmap_valid_entry(opte)((opte) & 0x0000000000000001UL)) {
/*
 * first, calculate pm_stats updates.  resident count will not
 * change since we are replacing/changing a valid mapping.
 * wired count might change...
 */

resdelta = 0;
if (wired && (opte & PG_W0x0000000000000200UL) == 0)
	wireddelta = 1;
else if (!wired && (opte & PG_W0x0000000000000200UL) != 0)
	wireddelta = -1;
else
	wireddelta = 0;
ptpdelta = 0;

/*
 * is the currently mapped PA the same as the one we
 * want to map?
 */

if ((opte & PG_FRAME0x000ffffffffff000UL) == pa) {

	/* if this is on the PVLIST, sync R/M bit */
	if (opte & PG_PVLIST0x0000000000000400UL) {
		pg = PHYS_TO_VM_PAGE(pa);
2763#ifdef DIAGNOSTIC1
		if (pg == NULL((void *)0))
			panic("%s: same pa, PG_PVLIST "
			    "mapping with unmanaged page: "
			    "va 0x%lx, opte 0x%llx, pa 0x%lx",
			    __func__, va, opte, pa);
2769#endif
		pmap_sync_flags_pte(pg, opte);
	} else {
2772#ifdef DIAGNOSTIC1
		if (PHYS_TO_VM_PAGE(pa) != NULL((void *)0))
			panic("%s: same pa, no PG_PVLIST "
			    "mapping with managed page: "
			    "va 0x%lx, opte 0x%llx, pa 0x%lx",
			    __func__, va, opte, pa);
2778#endif
	}
	goto enter_now;
}

/*
 * changing PAs: we must remove the old one first
 */

/*
 * if current mapping is on a pvlist,
 * remove it (sync R/M bits)
 */

if (opte & PG_PVLIST0x0000000000000400UL) {
	pg = PHYS_TO_VM_PAGE(opte & PG_FRAME0x000ffffffffff000UL);
2794#ifdef DIAGNOSTIC1
	if (pg == NULL((void *)0))
		panic("%s: PG_PVLIST mapping with unmanaged "
		    "page: va 0x%lx, opte 0x%llx, pa 0x%lx",
		    __func__, va, opte, pa);
2799#endif
	pmap_sync_flags_pte(pg, opte);
	opve = pmap_remove_pv(pg, pmap, va);
	pg = NULL((void *)0); /* This is not the page we are looking for */
}
} else {	/* opte not valid */
resdelta = 1;
if (wired)
	wireddelta = 1;
else
	wireddelta = 0;
if (ptp != NULL((void *)0))
	ptpdelta = 1;
else
	ptpdelta = 0;
}

/*
* pve is either NULL or points to a now-free pv_entry structure
* (the latter case is if we called pmap_remove_pv above).
*
* if this entry is to be on a pvlist, enter it now.
*/

if (pmap_initialized)
pg = PHYS_TO_VM_PAGE(pa);

if (pg != NULL((void *)0)) {
pmap_enter_pv(pg, pve, pmap, va, ptp);
pve = NULL((void *)0);
}

2831enter_now:
/*
* at this point pg is !NULL if we want the PG_PVLIST bit set
*/

pmap->pm_stats.resident_count += resdelta;
pmap->pm_stats.wired_count += wireddelta;
if (ptp != NULL((void *)0))
ptp->wire_count += ptpdelta;

KASSERT(pg == PHYS_TO_VM_PAGE(pa))((pg == PHYS_TO_VM_PAGE(pa)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/arch/amd64/amd64/pmap.c", 2841, "pg == PHYS_TO_VM_PAGE(pa)"
));

npte = pa | protection_codes[prot] | PG_V0x0000000000000001UL;
if (pg != NULL((void *)0)) {
npte |= PG_PVLIST0x0000000000000400UL;
/*
 * make sure that if the page is write combined all
 * instances of pmap_enter make it so.
 */
if (pg->pg_flags & PG_PMAP_WC0x04000000) {
	KASSERT(nocache == 0)((nocache == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/amd64/pmap.c"
, 2851, "nocache == 0"));
	wc = 1;
}
}
if (wc)
npte |= pmap_pg_wc;
if (wired)
npte |= PG_W0x0000000000000200UL;
if (nocache)
npte |= PG_N0x0000000000000010UL;
if (va < VM_MAXUSER_ADDRESS0x00007f7fffffc000)
npte |= ((flags & PMAP_EFI0x00000040) ? 0 : PG_u0x0000000000000004UL);
else if (va < VM_MAX_ADDRESS0x00007fbfdfeff000)
npte |= (PG_u0x0000000000000004UL | PG_RW0x0000000000000002UL);	/* XXXCDC: no longer needed? */
if (pmap == pmap_kernel()(&kernel_pmap_store))
npte |= pg_g_kern;

/*
* If the old entry wasn't valid, we can just update it and
* go.  If it was valid, and this isn't a read->write
* transition, then we can safely just update it and flush
* any old TLB entries.
*
* If it _was_ valid and this _is_ a read->write transition,
* then this could be a CoW resolution and we need to make
* sure no CPU can see the new writable mapping while another
* still has the old mapping in its TLB, so insert a correct
* but unwritable mapping, flush any old TLB entries, then
* make it writable.
*/
if (! pmap_valid_entry(opte)((opte) & 0x0000000000000001UL)) {
PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)] = npte;
} else if ((opte | (npte ^ PG_RW0x0000000000000002UL)) & PG_RW0x0000000000000002UL) {
/* previously writable or not making writable */
PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)] = npte;
if (nocache && (opte & PG_N0x0000000000000010UL) == 0)
	wbinvd_on_all_cpus();
pmap_tlb_shootpage(pmap, va, shootself);
} else {
PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)] = npte ^ PG_RW0x0000000000000002UL;
if (nocache && (opte & PG_N0x0000000000000010UL) == 0) /* XXX impossible? */
	wbinvd_on_all_cpus();
pmap_tlb_shootpage(pmap, va, shootself);
pmap_tlb_shootwait();
PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))[pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)] = npte;
}

pmap_unmap_ptes(pmap, scr3);
pmap_tlb_shootwait();

error = 0;

2903out:
if (pve != NULL((void *)0))
pool_put(&pmap_pv_pool, pve);
if (opve != NULL((void *)0))
pool_put(&pmap_pv_pool, opve);

return error;
2910}

2912int
2913pmap_get_physpage(vaddr_t va, int level, paddr_t *paddrp)
2914{
struct vm_page *ptp;
struct pmap *kpm = pmap_kernel()(&kernel_pmap_store);

if (uvm.page_init_done == 0) {
vaddr_t va;

/*
 * we're growing the kernel pmap early (from
 * uvm_pageboot_alloc()).  this case must be
 * handled a little differently.
 */

va = pmap_steal_memory(PAGE_SIZE(1 << 12), NULL((void *)0), NULL((void *)0));
*paddrp = PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)));
} else {
ptp = uvm_pagealloc(&kpm->pm_obj[level - 1],
		    ptp_va2o(va, level)((((((va) & ~0xffff000000000000)) & ptp_masks[((level
)+1)-1]) >> ptp_shifts[((level)+1)-1]) * (1 << 12
)), NULL((void *)0),
		    UVM_PGA_USERESERVE0x0001|UVM_PGA_ZERO0x0002);
if (ptp == NULL((void *)0))
	panic("%s: out of memory", __func__);
atomic_clearbits_intx86_atomic_clearbits_u32(&ptp->pg_flags, PG_BUSY0x00000001);
ptp->wire_count = 1;
*paddrp = VM_PAGE_TO_PHYS(ptp)((ptp)->phys_addr);
}
kpm->pm_stats.resident_count++;
return 1;
2941}

2943/*
* Allocate the amount of specified ptps for a ptp level, and populate
* all levels below accordingly, mapping virtual addresses starting at
* kva.
*
* Used by pmap_growkernel.
*/
2950void
2951pmap_alloc_level(vaddr_t kva, int lvl, long *needed_ptps)
2952{
unsigned long i;
vaddr_t va;
paddr_t pa;
unsigned long index, endindex;
int level;
pd_entry_t *pdep;

for (level = lvl; level > 1; level--) {
if (level == PTP_LEVELS4)
	pdep = pmap_kernel()(&kernel_pmap_store)->pm_pdir;
else
	pdep = normal_pdes[level - 2];
va = kva;
index = pl_i(kva, level)(((((kva) & ~0xffff000000000000)) & ptp_masks[(level)
-1]) >> ptp_shifts[(level)-1]);
endindex = index + needed_ptps[level - 1];
/*
 * XXX special case for first time call.
 */
if (nkptp[level - 1] != 0)
	index++;
else
	endindex--;

for (i = index; i <= endindex; i++) {
	pmap_get_physpage(va, level - 1, &pa);
	pdep[i] = pa | PG_RW0x0000000000000002UL | PG_V0x0000000000000001UL | pg_nx;
	nkptp[level - 1]++;
	va += nbpd[level - 1];
}
}
2983}

2985/*
* pmap_growkernel: increase usage of KVM space
*
* => we allocate new PTPs for the kernel and install them in all
*	the pmaps on the system.
*/

2992static vaddr_t pmap_maxkvaddr = VM_MIN_KERNEL_ADDRESS0xffff800000000000;

2994vaddr_t
2995pmap_growkernel(vaddr_t maxkvaddr)
2996{
struct pmap *kpm = pmap_kernel()(&kernel_pmap_store), *pm;
int s, i;
unsigned newpdes;
long needed_kptp[PTP_LEVELS4], target_nptp, old;

if (maxkvaddr <= pmap_maxkvaddr)
return pmap_maxkvaddr;

maxkvaddr = x86_round_pdr(maxkvaddr)((((unsigned long)(maxkvaddr)) + ((1ULL << 21) - 1)) &
 ~((1ULL << 21) - 1));
old = nkptp[PTP_LEVELS4 - 1];
/*
* This loop could be optimized more, but pmap_growkernel()
* is called infrequently.
*/
for (i = PTP_LEVELS4 - 1; i >= 1; i--) {
target_nptp = pl_i(maxkvaddr, i + 1)(((((maxkvaddr) & ~0xffff000000000000)) & ptp_masks[(
i + 1)-1]) >> ptp_shifts[(i + 1)-1]) -
    pl_i(VM_MIN_KERNEL_ADDRESS, i + 1)(((((0xffff800000000000) & ~0xffff000000000000)) & ptp_masks
[(i + 1)-1]) >> ptp_shifts[(i + 1)-1]);
/*
 * XXX only need to check toplevel.
 */
if (target_nptp > nkptpmax[i])
	panic("%s: out of KVA space", __func__);
needed_kptp[i] = target_nptp - nkptp[i] + 1;
}


s = splhigh()splraise(0xd);	/* to be safe */
pmap_alloc_level(pmap_maxkvaddr, PTP_LEVELS4, needed_kptp);

/*
* If the number of top level entries changed, update all
* pmaps.
*/
if (needed_kptp[PTP_LEVELS4 - 1] != 0) {
newpdes = nkptp[PTP_LEVELS4 - 1] - old;
mtx_enter(&pmaps_lock);
LIST_FOREACH(pm, &pmaps, pm_list)for((pm) = ((&pmaps)->lh_first); (pm)!= ((void *)0); (
pm) = ((pm)->pm_list.le_next)) {
	memcpy(&pm->pm_pdir[PDIR_SLOT_KERN + old],__builtin_memcpy((&pm->pm_pdir[256 + old]), (&kpm->
pm_pdir[256 + old]), (newpdes * sizeof (pd_entry_t)))
	       &kpm->pm_pdir[PDIR_SLOT_KERN + old],__builtin_memcpy((&pm->pm_pdir[256 + old]), (&kpm->
pm_pdir[256 + old]), (newpdes * sizeof (pd_entry_t)))
	       newpdes * sizeof (pd_entry_t))__builtin_memcpy((&pm->pm_pdir[256 + old]), (&kpm->
pm_pdir[256 + old]), (newpdes * sizeof (pd_entry_t)));
}
mtx_leave(&pmaps_lock);
}
pmap_maxkvaddr = maxkvaddr;
splx(s)spllower(s);

return maxkvaddr;
3044}

3046vaddr_t
3047pmap_steal_memory(vsize_t size, vaddr_t *start, vaddr_t *end)
3048{
int segno;
u_int npg;
vaddr_t va;
paddr_t pa;
struct vm_physseg *seg;

size = round_page(size)(((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1
));
npg = atop(size)((size) >> 12);

for (segno = 0, seg = vm_physmem; segno < vm_nphysseg; segno++, seg++) {
if (seg->avail_end - seg->avail_start < npg)
	continue;
/*
 * We can only steal at an ``unused'' segment boundary,
 * i.e. either at the start or at the end.
 */
if (seg->avail_start == seg->start ||
    seg->avail_end == seg->end)
	break;
}
if (segno == vm_nphysseg) {
panic("%s: out of memory", __func__);
} else {
if (seg->avail_start == seg->start) {
	pa = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12);
	seg->avail_start += npg;
	seg->start += npg;
} else {
	pa = ptoa(seg->avail_end)((paddr_t)(seg->avail_end) << 12) - size;
	seg->avail_end -= npg;
	seg->end -= npg;
}
/*
 * If all the segment has been consumed now, remove it.
 * Note that the crash dump code still knows about it
 * and will dump it correctly.
 */
if (seg->start == seg->end) {
	if (vm_nphysseg-- == 1)
		panic("%s: out of memory", __func__);
	while (segno < vm_nphysseg) {
		seg[0] = seg[1]; /* struct copy */
		seg++;
		segno++;
	}
}

va = PMAP_DIRECT_MAP(pa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pa));
memset((void *)va, 0, size)__builtin_memset(((void *)va), (0), (size));
}

if (start != NULL((void *)0))
*start = virtual_avail;
if (end != NULL((void *)0))
*end = VM_MAX_KERNEL_ADDRESS0xffff800100000000;

return (va);
3106}

3108void
3109pmap_virtual_space(vaddr_t *vstartp, vaddr_t *vendp)
3110{
*vstartp = virtual_avail;
*vendp = VM_MAX_KERNEL_ADDRESS0xffff800100000000;
3113}

3115/*
* pmap_convert
*
* Converts 'pmap' to the new 'mode'.
*
* Parameters:
*  pmap: the pmap to convert
*  mode: the new mode (see pmap.h, PMAP_TYPE_xxx)
*/
3124void
3125pmap_convert(struct pmap *pmap, int mode)
3126{
pt_entry_t *pte;

pmap->pm_type = mode;

if (mode == PMAP_TYPE_EPT2) {
/* Clear PML4 */
pte = (pt_entry_t *)pmap->pm_pdir;
memset(pte, 0, PAGE_SIZE)__builtin_memset((pte), (0), ((1 << 12)));

/* Give back the meltdown pdir */
if (pmap->pm_pdir_intel != NULL((void *)0)) {
	pool_put(&pmap_pdp_pool, pmap->pm_pdir_intel);
	pmap->pm_pdir_intel = NULL((void *)0);
}
}
3142}

3144#ifdef MULTIPROCESSOR1
3145/*
* Locking for tlb shootdown.
*
* We lock by setting tlb_shoot_wait to the number of cpus that will
* receive our tlb shootdown. After sending the IPIs, we don't need to
* worry about locking order or interrupts spinning for the lock because
* the call that grabs the "lock" isn't the one that releases it. And
* there is nothing that can block the IPI that releases the lock.
*
* The functions are organized so that we first count the number of
* cpus we need to send the IPI to, then we grab the counter, then
* we send the IPIs, then we finally do our own shootdown.
*
* Our shootdown is last to make it parallel with the other cpus
* to shorten the spin time.
*
* Notice that we depend on failures to send IPIs only being able to
* happen during boot. If they happen later, the above assumption
* doesn't hold since we can end up in situations where noone will
* release the lock if we get an interrupt in a bad moment.
*/
3166#ifdef MP_LOCKDEBUG
3167#include <ddb/db_output.h>
3168extern int __mp_lock_spinout;
3169#endif

3171volatile long tlb_shoot_wait __attribute__((section(".kudata")));

3173volatile vaddr_t tlb_shoot_addr1 __attribute__((section(".kudata")));
3174volatile vaddr_t tlb_shoot_addr2 __attribute__((section(".kudata")));
3175volatile int tlb_shoot_first_pcid __attribute__((section(".kudata")));


3178/* Obtain the "lock" for TLB shooting */
3179static inline int
3180pmap_start_tlb_shoot(long wait, const char *func)
3181{
int s = splvm()splraise(0xa);

while (atomic_cas_ulong(&tlb_shoot_wait, 0, wait)_atomic_cas_ulong((&tlb_shoot_wait), (0), (wait)) != 0) {
3185#ifdef MP_LOCKDEBUG
int nticks = __mp_lock_spinout;
3187#endif
while (tlb_shoot_wait != 0) {
	CPU_BUSY_CYCLE()__asm volatile("pause": : : "memory");
3190#ifdef MP_LOCKDEBUG
	if (--nticks <= 0) {
		db_printf("%s: spun out", func);
		db_enter();
		nticks = __mp_lock_spinout;
	}
3196#endif
}
}

return s;
3201}

3203void
3204pmap_tlb_shootpage(struct pmap *pm, vaddr_t va, int shootself)
3205{
struct cpu_info *ci, *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
CPU_INFO_ITERATORint cii;
long wait = 0;
u_int64_t mask = 0;
int is_kva = va >= VM_MIN_KERNEL_ADDRESS0xffff800000000000;

CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
if (ci == self || !(ci->ci_flags & CPUF_RUNNING0x2000))
	continue;
if (!is_kva && !pmap_is_active(pm, ci))
	continue;
mask |= (1ULL << ci->ci_cpuid);
wait++;
}

if (wait > 0) {
int s = pmap_start_tlb_shoot(wait, __func__);

tlb_shoot_first_pcid = is_kva ? PCID_KERN0 : PCID_PROC1;
tlb_shoot_addr1 = va;
CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
	if ((mask & (1ULL << ci->ci_cpuid)) == 0)
		continue;
	if (x86_fast_ipi(ci, LAPIC_IPI_INVLPG(0xf0 + 1)) != 0)
		panic("%s: ipi failed", __func__);
}
splx(s)spllower(s);
}

if (!pmap_use_pcid) {
if (shootself)
	pmap_update_pg(va);
} else if (is_kva) {
invpcid(INVPCID_ADDR0, PCID_PROC1, va);
invpcid(INVPCID_ADDR0, PCID_KERN0, va);
} else if (shootself) {
invpcid(INVPCID_ADDR0, PCID_PROC1, va);
if (cpu_meltdown)
	invpcid(INVPCID_ADDR0, PCID_PROC_INTEL2, va);
}
3246}

3248void
3249pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva, int shootself)
3250{
struct cpu_info *ci, *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
CPU_INFO_ITERATORint cii;
long wait = 0;
u_int64_t mask = 0;
int is_kva = sva >= VM_MIN_KERNEL_ADDRESS0xffff800000000000;
vaddr_t va;

CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
if (ci == self || !(ci->ci_flags & CPUF_RUNNING0x2000))
	continue;
if (!is_kva && !pmap_is_active(pm, ci))
	continue;
mask |= (1ULL << ci->ci_cpuid);
wait++;
}

if (wait > 0) {
int s = pmap_start_tlb_shoot(wait, __func__);

tlb_shoot_first_pcid = is_kva ? PCID_KERN0 : PCID_PROC1;
tlb_shoot_addr1 = sva;
tlb_shoot_addr2 = eva;
CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
	if ((mask & (1ULL << ci->ci_cpuid)) == 0)
		continue;
	if (x86_fast_ipi(ci, LAPIC_IPI_INVLRANGE(0xf0 + 2)) != 0)
		panic("%s: ipi failed", __func__);
}
splx(s)spllower(s);
}

if (!pmap_use_pcid) {
if (shootself) {
	for (va = sva; va < eva; va += PAGE_SIZE(1 << 12))
		pmap_update_pg(va);
}
} else if (is_kva) {
for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) {
	invpcid(INVPCID_ADDR0, PCID_PROC1, va);
	invpcid(INVPCID_ADDR0, PCID_KERN0, va);
}
} else if (shootself) {
if (cpu_meltdown) {
	for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) {
		invpcid(INVPCID_ADDR0, PCID_PROC1, va);
		invpcid(INVPCID_ADDR0, PCID_PROC_INTEL2, va);
	}
} else {
	for (va = sva; va < eva; va += PAGE_SIZE(1 << 12))
		invpcid(INVPCID_ADDR0, PCID_PROC1, va);
}
}
3303}

3305void
3306pmap_tlb_shoottlb(struct pmap *pm, int shootself)
3307{
struct cpu_info *ci, *self = curcpu()({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;});
CPU_INFO_ITERATORint cii;
long wait = 0;
u_int64_t mask = 0;

KASSERT(pm != pmap_kernel())((pm != (&kernel_pmap_store)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/arch/amd64/amd64/pmap.c", 3313, "pm != pmap_kernel()"
));

CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
if (ci == self || !pmap_is_active(pm, ci) ||
    !(ci->ci_flags & CPUF_RUNNING0x2000))
	continue;
mask |= (1ULL << ci->ci_cpuid);
wait++;
}

if (wait) {
int s = pmap_start_tlb_shoot(wait, __func__);

CPU_INFO_FOREACH(cii, ci)for (cii = 0, ci = cpu_info_list; ci != ((void *)0); ci = ci->
ci_next) {
	if ((mask & (1ULL << ci->ci_cpuid)) == 0)
		continue;
	if (x86_fast_ipi(ci, LAPIC_IPI_INVLTLB(0xf0 + 0)) != 0)
		panic("%s: ipi failed", __func__);
}
splx(s)spllower(s);
}

if (shootself) {
if (!pmap_use_pcid)
	tlbflush();
else {
	invpcid(INVPCID_PCID1, PCID_PROC1, 0);
	if (cpu_meltdown)
		invpcid(INVPCID_PCID1, PCID_PROC_INTEL2, 0);
}
}
3344}

3346void
3347pmap_tlb_shootwait(void)
3348{
3349#ifdef MP_LOCKDEBUG
int nticks = __mp_lock_spinout;
3351#endif
while (tlb_shoot_wait != 0) {
CPU_BUSY_CYCLE()__asm volatile("pause": : : "memory");
3354#ifdef MP_LOCKDEBUG
if (--nticks <= 0) {
	db_printf("%s: spun out", __func__);
	db_enter();
	nticks = __mp_lock_spinout;
}
3360#endif
}
3362}

3364#else /* MULTIPROCESSOR */

3366void
3367pmap_tlb_shootpage(struct pmap *pm, vaddr_t va, int shootself)
3368{
if (!pmap_use_pcid) {
if (shootself)
	pmap_update_pg(va);
} else if (va >= VM_MIN_KERNEL_ADDRESS0xffff800000000000) {
invpcid(INVPCID_ADDR0, PCID_PROC1, va);
invpcid(INVPCID_ADDR0, PCID_KERN0, va);
} else if (shootself) {
invpcid(INVPCID_ADDR0, PCID_PROC1, va);
if (cpu_meltdown)
	invpcid(INVPCID_ADDR0, PCID_PROC_INTEL2, va);
}
3380}

3382void
3383pmap_tlb_shootrange(struct pmap *pm, vaddr_t sva, vaddr_t eva, int shootself)
3384{
vaddr_t va;

if (!pmap_use_pcid) {
if (shootself) {
	for (va = sva; va < eva; va += PAGE_SIZE(1 << 12))
		pmap_update_pg(va);
}
} else if (sva >= VM_MIN_KERNEL_ADDRESS0xffff800000000000) {
for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) {
	invpcid(INVPCID_ADDR0, PCID_PROC1, va);
	invpcid(INVPCID_ADDR0, PCID_KERN0, va);
}
} else if (shootself) {
if (cpu_meltdown) {
	for (va = sva; va < eva; va += PAGE_SIZE(1 << 12)) {
		invpcid(INVPCID_ADDR0, PCID_PROC1, va);
		invpcid(INVPCID_ADDR0, PCID_PROC_INTEL2, va);
	}
} else {
	for (va = sva; va < eva; va += PAGE_SIZE(1 << 12))
		invpcid(INVPCID_ADDR0, PCID_PROC1, va);
}
}
3408}

3410void
3411pmap_tlb_shoottlb(struct pmap *pm, int shootself)
3412{
if (shootself) {
if (!pmap_use_pcid)
	tlbflush();
else {
	invpcid(INVPCID_PCID1, PCID_PROC1, 0);
	if (cpu_meltdown)
		invpcid(INVPCID_PCID1, PCID_PROC_INTEL2, 0);
}
}
3422}
3423#endif /* MULTIPROCESSOR */