/usr/src/sys/uvm/uvm

Bug Summary

File:	uvm/uvm_page.c
Warning:	line 157, column 2 Access to field 'pgops' results in a dereference of a null pointer (loaded from field 'uobject')

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name uvm_page.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/uvm/uvm_page.c

/usr/src/sys/uvm/uvm_page.c

→

1/*	$OpenBSD: uvm_page.c,v 1.160 2021/12/15 12:53:53 mpi Exp $	*/
2/*	$NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $	*/
3 
4/*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * Copyright (c) 1991, 1993, The Regents of the University of California.
7 *
8 * All rights reserved.
9 *
10 * This code is derived from software contributed to Berkeley by
11 * The Mach Operating System project at Carnegie-Mellon University.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 *    notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 *    notice, this list of conditions and the following disclaimer in the
20 *    documentation and/or other materials provided with the distribution.
21 * 3. Neither the name of the University nor the names of its contributors
22 *    may be used to endorse or promote products derived from this software
23 *    without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 *
37 *	@(#)vm_page.c   8.3 (Berkeley) 3/21/94
38 * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp
39 *
40 *
41 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
42 * All rights reserved.
43 *
44 * Permission to use, copy, modify and distribute this software and
45 * its documentation is hereby granted, provided that both the copyright
46 * notice and this permission notice appear in all copies of the
47 * software, derivative works or modified versions, and any portions
48 * thereof, and that both notices appear in supporting documentation.
49 *
50 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
51 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
52 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
53 *
54 * Carnegie Mellon requests users of this software to return to
55 *
56 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
57 *  School of Computer Science
58 *  Carnegie Mellon University
59 *  Pittsburgh PA 15213-3890
60 *
61 * any improvements or extensions that they make and grant Carnegie the
62 * rights to redistribute these changes.
63 */
64 
65/*
66 * uvm_page.c: page ops.
67 */
68 
69#include <sys/param.h>
70#include <sys/systm.h>
71#include <sys/sched.h>
72#include <sys/vnode.h>
73#include <sys/mount.h>
74#include <sys/proc.h>
75#include <sys/smr.h>
76 
77#include <uvm/uvm.h>
78 
79/*
80 * for object trees
81 */
82RBT_GENERATE(uvm_objtree, vm_page, objt, uvm_pagecmp)static int uvm_objtree_RBT_COMPARE(const void *lptr, const void
 *rptr) { const struct vm_page *l = lptr, *r = rptr; return uvm_pagecmp
(l, r); } static const struct rb_type uvm_objtree_RBT_INFO = {
 uvm_objtree_RBT_COMPARE, ((void *)0), __builtin_offsetof(struct
 vm_page, objt), }; const struct rb_type *const uvm_objtree_RBT_TYPE
 = &uvm_objtree_RBT_INFO;
83 
84int
85uvm_pagecmp(const struct vm_page *a, const struct vm_page *b)
86{
87	return a->offset < b->offset ? -1 : a->offset > b->offset;
88}
89 
90/*
91 * global vars... XXXCDC: move to uvm. structure.
92 */
93/*
94 * physical memory config is stored in vm_physmem.
95 */
96struct vm_physseg vm_physmem[VM_PHYSSEG_MAX16];	/* XXXCDC: uvm.physmem */
97int vm_nphysseg = 0;				/* XXXCDC: uvm.nphysseg */
98 
99/*
100 * Some supported CPUs in a given architecture don't support all
101 * of the things necessary to do idle page zero'ing efficiently.
102 * We therefore provide a way to disable it from machdep code here.
103 */
104 
105/*
106 * local variables
107 */
108/*
109 * these variables record the values returned by vm_page_bootstrap,
110 * for debugging purposes.  The implementation of uvm_pageboot_alloc
111 * and pmap_startup here also uses them internally.
112 */
113static vaddr_t      virtual_space_start;
114static vaddr_t      virtual_space_end;
115 
116/*
117 * local prototypes
118 */
119static void uvm_pageinsert(struct vm_page *);
120static void uvm_pageremove(struct vm_page *);
121int uvm_page_owner_locked_p(struct vm_page *);
122 
123/*
124 * inline functions
125 */
126/*
127 * uvm_pageinsert: insert a page in the object
128 *
129 * => caller must lock object
130 * => call should have already set pg's object and offset pointers
131 *    and bumped the version counter
132 */
133inline static void
134uvm_pageinsert(struct vm_page *pg)
135{
136	struct vm_page	*dupe;
137 
138	KASSERT(UVM_OBJ_IS_DUMMY(pg->uobject) ||(((((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)) || rw_write_held(
pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 139, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
139	    rw_write_held(pg->uobject->vmobjlock))(((((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)) || rw_write_held(
pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 139, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
));
140	KASSERT((pg->pg_flags & PG_TABLED) == 0)(((pg->pg_flags & 0x00000004) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 140, "(pg->pg_flags & PG_TABLED) == 0"
));
141 
142	dupe = RBT_INSERT(uvm_objtree, &pg->uobject->memt, pg)uvm_objtree_RBT_INSERT(&pg->uobject->memt, pg);
143	/* not allowed to insert over another page */
144	KASSERT(dupe == NULL)((dupe == ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 144, "dupe == NULL"));
145	atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_TABLED0x00000004);
146	pg->uobject->uo_npages++;
147}
148 
149/*
150 * uvm_page_remove: remove page from object
151 *
152 * => caller must lock object
153 */
154static inline void
155uvm_pageremove(struct vm_page *pg)
156{
157	KASSERT(UVM_OBJ_IS_DUMMY(pg->uobject) ||(((((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)) || rw_write_held(
pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 158, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
43
←
Access to field 'pgops' results in a dereference of a null pointer (loaded from field 'uobject')
158	    rw_write_held(pg->uobject->vmobjlock))(((((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)) || rw_write_held(
pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 158, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
));
159	KASSERT(pg->pg_flags & PG_TABLED)((pg->pg_flags & 0x00000004) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 159, "pg->pg_flags & PG_TABLED"
));
160 
161	RBT_REMOVE(uvm_objtree, &pg->uobject->memt, pg)uvm_objtree_RBT_REMOVE(&pg->uobject->memt, pg);
162 
163	atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_TABLED0x00000004);
164	pg->uobject->uo_npages--;
165	pg->uobject = NULL((void *)0);
166	pg->pg_version++;
167}
168 
169/*
170 * uvm_page_init: init the page system.   called from uvm_init().
171 *
172 * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
173 */
174void
175uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
176{
177	vsize_t freepages, pagecount, n;
178	vm_page_t pagearray, curpg;
179	int lcv, i;
180	paddr_t paddr, pgno;
181	struct vm_physseg *seg;
182 
183	/*
184	 * init the page queues and page queue locks
185	 */
186 
187	TAILQ_INIT(&uvm.page_active)do { (&uvm.page_active)->tqh_first = ((void *)0); (&
uvm.page_active)->tqh_last = &(&uvm.page_active)->
tqh_first; } while (0);
188	TAILQ_INIT(&uvm.page_inactive_swp)do { (&uvm.page_inactive_swp)->tqh_first = ((void *)0)
; (&uvm.page_inactive_swp)->tqh_last = &(&uvm.
page_inactive_swp)->tqh_first; } while (0);
189	TAILQ_INIT(&uvm.page_inactive_obj)do { (&uvm.page_inactive_obj)->tqh_first = ((void *)0)
; (&uvm.page_inactive_obj)->tqh_last = &(&uvm.
page_inactive_obj)->tqh_first; } while (0);
190	mtx_init(&uvm.pageqlock, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.pageqlock
), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((
0xa)))); } while (0);
191	mtx_init(&uvm.fpageqlock, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.fpageqlock
), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((
0xa)))); } while (0);
192	uvm_pmr_init();
193 
194	/*
195	 * allocate vm_page structures.
196	 */
197 
198	/*
199	 * sanity check:
200	 * before calling this function the MD code is expected to register
201	 * some free RAM with the uvm_page_physload() function.   our job
202	 * now is to allocate vm_page structures for this memory.
203	 */
204 
205	if (vm_nphysseg == 0)
206		panic("uvm_page_bootstrap: no memory pre-allocated");
207 
208	/*
209	 * first calculate the number of free pages...
210	 *
211	 * note that we use start/end rather than avail_start/avail_end.
212	 * this allows us to allocate extra vm_page structures in case we
213	 * want to return some memory to the pool after booting.
214	 */
215 
216	freepages = 0;
217	for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++)
218		freepages += (seg->end - seg->start);
219 
220	/*
221	 * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
222	 * use.   for each page of memory we use we need a vm_page structure.
223	 * thus, the total number of pages we can use is the total size of
224	 * the memory divided by the PAGE_SIZE plus the size of the vm_page
225	 * structure.   we add one to freepages as a fudge factor to avoid
226	 * truncation errors (since we can only allocate in terms of whole
227	 * pages).
228	 */
229 
230	pagecount = (((paddr_t)freepages + 1) << PAGE_SHIFT12) /
231	    (PAGE_SIZE(1 << 12) + sizeof(struct vm_page));
232	pagearray = (vm_page_t)uvm_pageboot_alloc(pagecount *
233	    sizeof(struct vm_page));
234	memset(pagearray, 0, pagecount * sizeof(struct vm_page))__builtin_memset((pagearray), (0), (pagecount * sizeof(struct
 vm_page)));
235 
236	/* init the vm_page structures and put them in the correct place. */
237	for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) {
238		n = seg->end - seg->start;
239		if (n > pagecount) {
240			panic("uvm_page_init: lost %ld page(s) in init",
241			    (long)(n - pagecount));
242			    /* XXXCDC: shouldn't happen? */
243			/* n = pagecount; */
244		}
245 
246		/* set up page array pointers */
247		seg->pgs = pagearray;
248		pagearray += n;
249		pagecount -= n;
250		seg->lastpg = seg->pgs + (n - 1);
251 
252		/* init and free vm_pages (we've already zeroed them) */
253		pgno = seg->start;
254		paddr = ptoa(pgno)((paddr_t)(pgno) << 12);
255		for (i = 0, curpg = seg->pgs; i < n;
256		    i++, curpg++, pgno++, paddr += PAGE_SIZE(1 << 12)) {
257			curpg->phys_addr = paddr;
258			VM_MDPAGE_INIT(curpg)do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(curpg
)->mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) <
 0x9) ? 0x9 : ((0xa)))); } while (0); (curpg)->mdpage.pv_list
 = ((void *)0); } while (0);
259			if (pgno >= seg->avail_start &&
260			    pgno < seg->avail_end) {
261				uvmexp.npages++;
262			}
263		}
264 
265		/* Add pages to free pool. */
266		uvm_pmr_freepages(&seg->pgs[seg->avail_start - seg->start],
267		    seg->avail_end - seg->avail_start);
268	}
269 
270	/*
271	 * pass up the values of virtual_space_start and
272	 * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
273	 * layers of the VM.
274	 */
275 
276	*kvm_startp = round_page(virtual_space_start)(((virtual_space_start) + ((1 << 12) - 1)) & ~((1 <<
 12) - 1));
277	*kvm_endp = trunc_page(virtual_space_end)((virtual_space_end) & ~((1 << 12) - 1));
278 
279	/* init locks for kernel threads */
280	mtx_init(&uvm.aiodoned_lock, IPL_BIO)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.aiodoned_lock
), ((((0x6)) > 0x0 && ((0x6)) < 0x9) ? 0x9 : ((
0x6)))); } while (0);
281 
282	/*
283	 * init reserve thresholds
284	 * XXXCDC - values may need adjusting
285	 */
286	uvmexp.reserve_pagedaemon = 4;
287	uvmexp.reserve_kernel = 8;
288	uvmexp.anonminpct = 10;
289	uvmexp.vnodeminpct = 10;
290	uvmexp.vtextminpct = 5;
291	uvmexp.anonmin = uvmexp.anonminpct * 256 / 100;
292	uvmexp.vnodemin = uvmexp.vnodeminpct * 256 / 100;
293	uvmexp.vtextmin = uvmexp.vtextminpct * 256 / 100;
294 
295	uvm.page_init_done = TRUE1;
296}
297 
298/*
299 * uvm_setpagesize: set the page size
300 *
301 * => sets page_shift and page_mask from uvmexp.pagesize.
302 */
303void
304uvm_setpagesize(void)
305{
306	if (uvmexp.pagesize == 0)
307		uvmexp.pagesize = DEFAULT_PAGE_SIZE4096;
308	uvmexp.pagemask = uvmexp.pagesize - 1;
309	if ((uvmexp.pagemask & uvmexp.pagesize) != 0)
310		panic("uvm_setpagesize: page size not a power of two");
311	for (uvmexp.pageshift = 0; ; uvmexp.pageshift++)
312		if ((1 << uvmexp.pageshift) == uvmexp.pagesize)
313			break;
314}
315 
316/*
317 * uvm_pageboot_alloc: steal memory from physmem for bootstrapping
318 */
319vaddr_t
320uvm_pageboot_alloc(vsize_t size)
321{
322#if defined(PMAP_STEAL_MEMORY)
323	vaddr_t addr;
324 
325	/*
326	 * defer bootstrap allocation to MD code (it may want to allocate
327	 * from a direct-mapped segment).  pmap_steal_memory should round
328	 * off virtual_space_start/virtual_space_end.
329	 */
330 
331	addr = pmap_steal_memory(size, &virtual_space_start,
332	    &virtual_space_end);
333 
334	return addr;
335 
336#else /* !PMAP_STEAL_MEMORY */
337 
338	static boolean_t initialized = FALSE0;
339	vaddr_t addr, vaddr;
340	paddr_t paddr;
341 
342	/* round to page size */
343	size = round_page(size)(((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1
));
344 
345	/* on first call to this function, initialize ourselves. */
346	if (initialized == FALSE0) {
347		pmap_virtual_space(&virtual_space_start, &virtual_space_end);
348 
349		/* round it the way we like it */
350		virtual_space_start = round_page(virtual_space_start)(((virtual_space_start) + ((1 << 12) - 1)) & ~((1 <<
 12) - 1));
351		virtual_space_end = trunc_page(virtual_space_end)((virtual_space_end) & ~((1 << 12) - 1));
352 
353		initialized = TRUE1;
354	}
355 
356	/* allocate virtual memory for this request */
357	if (virtual_space_start == virtual_space_end ||
358	    (virtual_space_end - virtual_space_start) < size)
359		panic("uvm_pageboot_alloc: out of virtual space");
360 
361	addr = virtual_space_start;
362 
363#ifdef PMAP_GROWKERNEL
364	/*
365	 * If the kernel pmap can't map the requested space,
366	 * then allocate more resources for it.
367	 */
368	if (uvm_maxkaddr < (addr + size)) {
369		uvm_maxkaddr = pmap_growkernel(addr + size);
370		if (uvm_maxkaddr < (addr + size))
371			panic("uvm_pageboot_alloc: pmap_growkernel() failed");
372	}
373#endif
374 
375	virtual_space_start += size;
376 
377	/* allocate and mapin physical pages to back new virtual pages */
378	for (vaddr = round_page(addr)(((addr) + ((1 << 12) - 1)) & ~((1 << 12) - 1
)) ; vaddr < addr + size ;
379	    vaddr += PAGE_SIZE(1 << 12)) {
380		if (!uvm_page_physget(&paddr))
381			panic("uvm_pageboot_alloc: out of memory");
382 
383		/*
384		 * Note this memory is no longer managed, so using
385		 * pmap_kenter is safe.
386		 */
387		pmap_kenter_pa(vaddr, paddr, PROT_READ0x01 | PROT_WRITE0x02);
388	}
389	pmap_update(pmap_kernel());
390	return addr;
391#endif	/* PMAP_STEAL_MEMORY */
392}
393 
394#if !defined(PMAP_STEAL_MEMORY)
395/*
396 * uvm_page_physget: "steal" one page from the vm_physmem structure.
397 *
398 * => attempt to allocate it off the end of a segment in which the "avail"
399 *    values match the start/end values.   if we can't do that, then we
400 *    will advance both values (making them equal, and removing some
401 *    vm_page structures from the non-avail area).
402 * => return false if out of memory.
403 */
404 
405boolean_t
406uvm_page_physget(paddr_t *paddrp)
407{
408	int lcv;
409	struct vm_physseg *seg;
410 
411	/* pass 1: try allocating from a matching end */
412#if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) || \
413	(VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2)
414	for (lcv = vm_nphysseg - 1, seg = vm_physmem + lcv; lcv >= 0;
415	    lcv--, seg--)
416#else
417	for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++)
418#endif
419	{
420		if (uvm.page_init_done == TRUE1)
421			panic("uvm_page_physget: called _after_ bootstrap");
422 
423		/* try from front */
424		if (seg->avail_start == seg->start &&
425		    seg->avail_start < seg->avail_end) {
426			*paddrp = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12);
427			seg->avail_start++;
428			seg->start++;
429			/* nothing left?   nuke it */
430			if (seg->avail_start == seg->end) {
431				if (vm_nphysseg == 1)
432				    panic("uvm_page_physget: out of memory!");
433				vm_nphysseg--;
434				for (; lcv < vm_nphysseg; lcv++, seg++)
435					/* structure copy */
436					seg[0] = seg[1];
437			}
438			return TRUE1;
439		}
440 
441		/* try from rear */
442		if (seg->avail_end == seg->end &&
443		    seg->avail_start < seg->avail_end) {
444			*paddrp = ptoa(seg->avail_end - 1)((paddr_t)(seg->avail_end - 1) << 12);
445			seg->avail_end--;
446			seg->end--;
447			/* nothing left?   nuke it */
448			if (seg->avail_end == seg->start) {
449				if (vm_nphysseg == 1)
450				    panic("uvm_page_physget: out of memory!");
451				vm_nphysseg--;
452				for (; lcv < vm_nphysseg ; lcv++, seg++)
453					/* structure copy */
454					seg[0] = seg[1];
455			}
456			return TRUE1;
457		}
458	}
459 
460	/* pass2: forget about matching ends, just allocate something */
461#if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) || \
462	(VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2)
463	for (lcv = vm_nphysseg - 1, seg = vm_physmem + lcv; lcv >= 0;
464	    lcv--, seg--)
465#else
466	for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++)
467#endif
468	{
469 
470		/* any room in this bank? */
471		if (seg->avail_start >= seg->avail_end)
472			continue;  /* nope */
473 
474		*paddrp = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12);
475		seg->avail_start++;
476		/* truncate! */
477		seg->start = seg->avail_start;
478 
479		/* nothing left?   nuke it */
480		if (seg->avail_start == seg->end) {
481			if (vm_nphysseg == 1)
482				panic("uvm_page_physget: out of memory!");
483			vm_nphysseg--;
484			for (; lcv < vm_nphysseg ; lcv++, seg++)
485				/* structure copy */
486				seg[0] = seg[1];
487		}
488		return TRUE1;
489	}
490 
491	return FALSE0;        /* whoops! */
492}
493 
494#endif /* PMAP_STEAL_MEMORY */
495 
496/*
497 * uvm_page_physload: load physical memory into VM system
498 *
499 * => all args are PFs
500 * => all pages in start/end get vm_page structures
501 * => areas marked by avail_start/avail_end get added to the free page pool
502 * => we are limited to VM_PHYSSEG_MAX physical memory segments
503 */
504 
505void
506uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start,
507    paddr_t avail_end, int flags)
508{
509	int preload, lcv;
510	psize_t npages;
511	struct vm_page *pgs;
512	struct vm_physseg *ps, *seg;
513 
514#ifdef DIAGNOSTIC1
515	if (uvmexp.pagesize == 0)
516		panic("uvm_page_physload: page size not set!");
517 
518	if (start >= end)
519		panic("uvm_page_physload: start >= end");
520#endif
521 
522	/* do we have room? */
523	if (vm_nphysseg == VM_PHYSSEG_MAX16) {
524		printf("uvm_page_physload: unable to load physical memory "
525		    "segment\n");
526		printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
527		    VM_PHYSSEG_MAX16, (long long)start, (long long)end);
528		printf("\tincrease VM_PHYSSEG_MAX\n");
529		return;
530	}
531 
532	/*
533	 * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been
534	 * called yet, so malloc is not available).
535	 */
536	for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++) {
537		if (seg->pgs)
538			break;
539	}
540	preload = (lcv == vm_nphysseg);
541 
542	/* if VM is already running, attempt to malloc() vm_page structures */
543	if (!preload) {
544		/*
545		 * XXXCDC: need some sort of lockout for this case
546		 * right now it is only used by devices so it should be alright.
547		 */
548 		paddr_t paddr;
549 
550 		npages = end - start;  /* # of pages */
551 
552		pgs = km_alloc(round_page(npages * sizeof(*pgs))(((npages * sizeof(*pgs)) + ((1 << 12) - 1)) & ~((1
 << 12) - 1)),
553		    &kv_any, &kp_zero, &kd_waitok);
554		if (pgs == NULL((void *)0)) {
555			printf("uvm_page_physload: can not malloc vm_page "
556			    "structs for segment\n");
557			printf("\tignoring 0x%lx -> 0x%lx\n", start, end);
558			return;
559		}
560		/* init phys_addr and free pages, XXX uvmexp.npages */
561		for (lcv = 0, paddr = ptoa(start)((paddr_t)(start) << 12); lcv < npages;
562		    lcv++, paddr += PAGE_SIZE(1 << 12)) {
563			pgs[lcv].phys_addr = paddr;
564			VM_MDPAGE_INIT(&pgs[lcv])do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(&
pgs[lcv])->mdpage.pv_mtx), ((((0xa)) > 0x0 && (
(0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); (&pgs[lcv
])->mdpage.pv_list = ((void *)0); } while (0);
565			if (atop(paddr)((paddr) >> 12) >= avail_start &&
566			    atop(paddr)((paddr) >> 12) < avail_end) {
567				if (flags & PHYSLOAD_DEVICE0x01) {
568					atomic_setbits_intx86_atomic_setbits_u32(&pgs[lcv].pg_flags,
569					    PG_DEV0x00000200);
570					pgs[lcv].wire_count = 1;
571				} else {
572#if defined(VM_PHYSSEG_NOADD)
573		panic("uvm_page_physload: tried to add RAM after vm_mem_init");
574#endif
575				}
576			}
577		}
578 
579		/* Add pages to free pool. */
580		if ((flags & PHYSLOAD_DEVICE0x01) == 0) {
581			uvm_pmr_freepages(&pgs[avail_start - start],
582			    avail_end - avail_start);
583		}
584 
585		/* XXXCDC: need hook to tell pmap to rebuild pv_list, etc... */
586	} else {
587		/* gcc complains if these don't get init'd */
588		pgs = NULL((void *)0);
589		npages = 0;
590 
591	}
592 
593	/* now insert us in the proper place in vm_physmem[] */
594#if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_RANDOM1)
595	/* random: put it at the end (easy!) */
596	ps = &vm_physmem[vm_nphysseg];
597#elif (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2)
598	{
599		int x;
600		/* sort by address for binary search */
601		for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++)
602			if (start < seg->start)
603				break;
604		ps = seg;
605		/* move back other entries, if necessary ... */
606		for (x = vm_nphysseg, seg = vm_physmem + x - 1; x > lcv;
607		    x--, seg--)
608			/* structure copy */
609			seg[1] = seg[0];
610	}
611#elif (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3)
612	{
613		int x;
614		/* sort by largest segment first */
615		for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++)
616			if ((end - start) >
617			    (seg->end - seg->start))
618				break;
619		ps = &vm_physmem[lcv];
620		/* move back other entries, if necessary ... */
621		for (x = vm_nphysseg, seg = vm_physmem + x - 1; x > lcv;
622		    x--, seg--)
623			/* structure copy */
624			seg[1] = seg[0];
625	}
626#else
627	panic("uvm_page_physload: unknown physseg strategy selected!");
628#endif
629 
630	ps->start = start;
631	ps->end = end;
632	ps->avail_start = avail_start;
633	ps->avail_end = avail_end;
634	if (preload) {
635		ps->pgs = NULL((void *)0);
636	} else {
637		ps->pgs = pgs;
638		ps->lastpg = pgs + npages - 1;
639	}
640	vm_nphysseg++;
641 
642	return;
643}
644 
645#ifdef DDB1 /* XXXCDC: TMP TMP TMP DEBUG DEBUG DEBUG */
646 
647void uvm_page_physdump(void); /* SHUT UP GCC */
648 
649/* call from DDB */
650void
651uvm_page_physdump(void)
652{
653	int lcv;
654	struct vm_physseg *seg;
655 
656	printf("uvm_page_physdump: physical memory config [segs=%d of %d]:\n",
657	    vm_nphysseg, VM_PHYSSEG_MAX16);
658	for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++)
659		printf("0x%llx->0x%llx [0x%llx->0x%llx]\n",
660		    (long long)seg->start,
661		    (long long)seg->end,
662		    (long long)seg->avail_start,
663		    (long long)seg->avail_end);
664	printf("STRATEGY = ");
665	switch (VM_PHYSSEG_STRAT3) {
666	case VM_PSTRAT_RANDOM1: printf("RANDOM\n"); break;
667	case VM_PSTRAT_BSEARCH2: printf("BSEARCH\n"); break;
668	case VM_PSTRAT_BIGFIRST3: printf("BIGFIRST\n"); break;
669	default: printf("<<UNKNOWN>>!!!!\n");
670	}
671}
672#endif
673 
674void
675uvm_shutdown(void)
676{
677#ifdef UVM_SWAP_ENCRYPT1
678	uvm_swap_finicrypt_all();
679#endif
680	smr_flush()smr_barrier_impl(1);
681}
682 
683/*
684 * Perform insert of a given page in the specified anon of obj.
685 * This is basically, uvm_pagealloc, but with the page already given.
686 */
687void
688uvm_pagealloc_pg(struct vm_page *pg, struct uvm_object *obj, voff_t off,
689    struct vm_anon *anon)
690{
691	int	flags;
692 
693	KASSERT(obj == NULL || anon == NULL)((obj == ((void *)0) || anon == ((void *)0)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 693, "obj == NULL || anon == NULL"
));
694	KASSERT(anon == NULL || off == 0)((anon == ((void *)0) || off == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 694, "anon == NULL || off == 0"
));
695	KASSERT(off == trunc_page(off))((off == ((off) & ~((1 << 12) - 1))) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 695, "off == trunc_page(off)"
));
696	KASSERT(obj == NULL || UVM_OBJ_IS_DUMMY(obj) ||((obj == ((void *)0) || (((obj)->pgops == &pmap_pager)
 || ((obj)->pgops == &bufcache_pager)) || rw_write_held
(obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 697, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)"
))
697	    rw_write_held(obj->vmobjlock))((obj == ((void *)0) || (((obj)->pgops == &pmap_pager)
 || ((obj)->pgops == &bufcache_pager)) || rw_write_held
(obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 697, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)"
));
698	KASSERT(anon == NULL || anon->an_lock == NULL ||((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held
(anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 699, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)"
))
699	    rw_write_held(anon->an_lock))((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held
(anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 699, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)"
));
700 
701	flags = PG_BUSY0x00000001 | PG_FAKE0x00000040;
702	pg->offset = off;
703	pg->uobject = obj;
704	pg->uanon = anon;
705	KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 705, "uvm_page_owner_locked_p(pg)"
));
706	if (anon) {
707		anon->an_page = pg;
708		flags |= PQ_ANON0x00100000;
709	} else if (obj)
710		uvm_pageinsert(pg);
711	atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, flags);
712#if defined(UVM_PAGE_TRKOWN)
713	pg->owner_tag = NULL((void *)0);
714#endif
715	UVM_PAGE_OWN(pg, "new alloc");
716}
717 
718/*
719 * uvm_pglistalloc: allocate a list of pages
720 *
721 * => allocated pages are placed at the tail of rlist.  rlist is
722 *    assumed to be properly initialized by caller.
723 * => returns 0 on success or errno on failure
724 * => doesn't take into account clean non-busy pages on inactive list
725 *	that could be used(?)
726 * => params:
727 *	size		the size of the allocation, rounded to page size.
728 *	low		the low address of the allowed allocation range.
729 *	high		the high address of the allowed allocation range.
730 *	alignment	memory must be aligned to this power-of-two boundary.
731 *	boundary	no segment in the allocation may cross this 
732 *			power-of-two boundary (relative to zero).
733 * => flags:
734 *	UVM_PLA_NOWAIT	fail if allocation fails
735 *	UVM_PLA_WAITOK	wait for memory to become avail
736 *	UVM_PLA_ZERO	return zeroed memory
737 */
738int
739uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment,
740    paddr_t boundary, struct pglist *rlist, int nsegs, int flags)
741{
742	KASSERT((alignment & (alignment - 1)) == 0)(((alignment & (alignment - 1)) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 742, "(alignment & (alignment - 1)) == 0"
));
743	KASSERT((boundary & (boundary - 1)) == 0)(((boundary & (boundary - 1)) == 0) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 743, "(boundary & (boundary - 1)) == 0"
));
744	KASSERT(!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT))((!(flags & 0x0001) ^ !(flags & 0x0002)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 744, "!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT)"
));
745 
746	if (size == 0)
747		return EINVAL22;
748	size = atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) -
 1))) >> 12);
749 
750	/*
751	 * XXX uvm_pglistalloc is currently only used for kernel
752	 * objects. Unlike the checks in uvm_pagealloc, below, here
753	 * we are always allowed to use the kernel reserve.
754	 */
755	flags |= UVM_PLA_USERESERVE0x0040;
756 
757	if ((high & PAGE_MASK((1 << 12) - 1)) != PAGE_MASK((1 << 12) - 1)) {
758		printf("uvm_pglistalloc: Upper boundary 0x%lx "
759		    "not on pagemask.\n", (unsigned long)high);
760	}
761 
762	/*
763	 * Our allocations are always page granularity, so our alignment
764	 * must be, too.
765	 */
766	if (alignment < PAGE_SIZE(1 << 12))
767		alignment = PAGE_SIZE(1 << 12);
768 
769	low = atop(roundup(low, alignment))((((((low)+((alignment)-1))/(alignment))*(alignment))) >>
 12);
770	/*
771	 * high + 1 may result in overflow, in which case high becomes 0x0,
772	 * which is the 'don't care' value.
773	 * The only requirement in that case is that low is also 0x0, or the
774	 * low<high assert will fail.
775	 */
776	high = atop(high + 1)((high + 1) >> 12);
777	alignment = atop(alignment)((alignment) >> 12);
778	if (boundary < PAGE_SIZE(1 << 12) && boundary != 0)
779		boundary = PAGE_SIZE(1 << 12);
780	boundary = atop(boundary)((boundary) >> 12);
781 
782	return uvm_pmr_getpages(size, low, high, alignment, boundary, nsegs,
783	    flags, rlist);
784}
785 
786/*
787 * uvm_pglistfree: free a list of pages
788 *
789 * => pages should already be unmapped
790 */
791void
792uvm_pglistfree(struct pglist *list)
793{
794	uvm_pmr_freepageq(list);
795}
796 
797/*
798 * interface used by the buffer cache to allocate a buffer at a time.
799 * The pages are allocated wired in DMA accessible memory
800 */
801int
802uvm_pagealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size,
803    int flags)
804{
805	struct pglist    plist;
806	struct vm_page  *pg;
807	int              i, r;
808 
809	KASSERT(UVM_OBJ_IS_BUFCACHE(obj))((((obj)->pgops == &bufcache_pager)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 809, "UVM_OBJ_IS_BUFCACHE(obj)"
));
810	KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 810, "_kernel_lock_held()"));
811 
812	TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)->
tqh_last = &(&plist)->tqh_first; } while (0);
813	r = uvm_pglistalloc(size, dma_constraint.ucr_low,
814	    dma_constraint.ucr_high, 0, 0, &plist, atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) -
 1))) >> 12),
815	    flags);
816	if (r == 0) {
817		i = 0;
818		while ((pg = TAILQ_FIRST(&plist)((&plist)->tqh_first)) != NULL((void *)0)) {
819			pg->wire_count = 1;
820			atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008 | PG_FAKE0x00000040);
821			KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 821, "(pg->pg_flags & PG_DEV) == 0"
));
822			TAILQ_REMOVE(&plist, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&plist)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->
pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->pageq.tqe_prev
) = ((void *)-1); ((pg)->pageq.tqe_next) = ((void *)-1); }
 while (0);
823			uvm_pagealloc_pg(pg, obj, off + ptoa(i++)((paddr_t)(i++) << 12), NULL((void *)0));
824		}
825	}
826	return r;
827}
828 
829/*
830 * interface used by the buffer cache to reallocate a buffer at a time.
831 * The pages are reallocated wired outside the DMA accessible region.
832 *
833 */
834int
835uvm_pagerealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size,
836    int flags, struct uvm_constraint_range *where)
837{
838	struct pglist    plist;
839	struct vm_page  *pg, *tpg;
840	int              i, r;
841	voff_t		offset;
842 
843	KASSERT(UVM_OBJ_IS_BUFCACHE(obj))((((obj)->pgops == &bufcache_pager)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 843, "UVM_OBJ_IS_BUFCACHE(obj)"
));
844	KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 844, "_kernel_lock_held()"));
845 
846	TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)->
tqh_last = &(&plist)->tqh_first; } while (0);
847	if (size == 0)
848		panic("size 0 uvm_pagerealloc");
849	r = uvm_pglistalloc(size, where->ucr_low, where->ucr_high, 0,
850	    0, &plist, atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) -
 1))) >> 12), flags);
851	if (r == 0) {
852		i = 0;
853		while((pg = TAILQ_FIRST(&plist)((&plist)->tqh_first)) != NULL((void *)0)) {
854			offset = off + ptoa(i++)((paddr_t)(i++) << 12);
855			tpg = uvm_pagelookup(obj, offset);
856			KASSERT(tpg != NULL)((tpg != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 856, "tpg != NULL"));
857			pg->wire_count = 1;
858			atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008 | PG_FAKE0x00000040);
859			KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 859, "(pg->pg_flags & PG_DEV) == 0"
));
860			TAILQ_REMOVE(&plist, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&plist)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->
pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->pageq.tqe_prev
) = ((void *)-1); ((pg)->pageq.tqe_next) = ((void *)-1); }
 while (0);
861			uvm_pagecopy(tpg, pg);
862			KASSERT(tpg->wire_count == 1)((tpg->wire_count == 1) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 862, "tpg->wire_count == 1"
));
863			tpg->wire_count = 0;
864			uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
865			uvm_pagefree(tpg);
866			uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
867			uvm_pagealloc_pg(pg, obj, offset, NULL((void *)0));
868		}
869	}
870	return r;
871}
872 
873/*
874 * uvm_pagealloc: allocate vm_page from a particular free list.
875 *
876 * => return null if no pages free
877 * => wake up pagedaemon if number of free pages drops below low water mark
878 * => only one of obj or anon can be non-null
879 * => caller must activate/deactivate page if it is not wired.
880 */
881 
882struct vm_page *
883uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
884    int flags)
885{
886	struct vm_page *pg;
887	struct pglist pgl;
888	int pmr_flags;
889 
890	KASSERT(obj == NULL || anon == NULL)((obj == ((void *)0) || anon == ((void *)0)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 890, "obj == NULL || anon == NULL"
));
891	KASSERT(anon == NULL || off == 0)((anon == ((void *)0) || off == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 891, "anon == NULL || off == 0"
));
892	KASSERT(off == trunc_page(off))((off == ((off) & ~((1 << 12) - 1))) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 892, "off == trunc_page(off)"
));
893	KASSERT(obj == NULL || UVM_OBJ_IS_DUMMY(obj) ||((obj == ((void *)0) || (((obj)->pgops == &pmap_pager)
 || ((obj)->pgops == &bufcache_pager)) || rw_write_held
(obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 894, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)"
))
894	    rw_write_held(obj->vmobjlock))((obj == ((void *)0) || (((obj)->pgops == &pmap_pager)
 || ((obj)->pgops == &bufcache_pager)) || rw_write_held
(obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 894, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)"
));
895	KASSERT(anon == NULL || anon->an_lock == NULL ||((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held
(anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 896, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)"
))
896	    rw_write_held(anon->an_lock))((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held
(anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 896, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)"
));
897 
898	pmr_flags = UVM_PLA_NOWAIT0x0002;
899 
900	/*
901	 * We're allowed to use the kernel reserve if the page is
902	 * being allocated to a kernel object.
903	 */
904	if ((flags & UVM_PGA_USERESERVE0x0001) ||
905	    (obj != NULL((void *)0) && UVM_OBJ_IS_KERN_OBJECT(obj)((obj)->uo_refs == (-2))))
906	    	pmr_flags |= UVM_PLA_USERESERVE0x0040;
907 
908	if (flags & UVM_PGA_ZERO0x0002)
909		pmr_flags |= UVM_PLA_ZERO0x0004;
910	TAILQ_INIT(&pgl)do { (&pgl)->tqh_first = ((void *)0); (&pgl)->tqh_last
 = &(&pgl)->tqh_first; } while (0);
911	if (uvm_pmr_getpages(1, 0, 0, 1, 0, 1, pmr_flags, &pgl) != 0)
912		goto fail;
913 
914	pg = TAILQ_FIRST(&pgl)((&pgl)->tqh_first);
915	KASSERT(pg != NULL && TAILQ_NEXT(pg, pageq) == NULL)((pg != ((void *)0) && ((pg)->pageq.tqe_next) == (
(void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 915, "pg != NULL && TAILQ_NEXT(pg, pageq) == NULL")
);
916 
917	uvm_pagealloc_pg(pg, obj, off, anon);
918	KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 918, "(pg->pg_flags & PG_DEV) == 0"
));
919	if (flags & UVM_PGA_ZERO0x0002)
920		atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
921	else
922		atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
923 
924	return pg;
925 
926fail:
927	return NULL((void *)0);
928}
929 
930/*
931 * uvm_pagerealloc: reallocate a page from one object to another
932 */
933 
934void
935uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff)
936{
937 
938	/* remove it from the old object */
939	if (pg->uobject) {
940		uvm_pageremove(pg);
941	}
942 
943	/* put it in the new object */
944	if (newobj) {
945		pg->uobject = newobj;
946		pg->offset = newoff;
947		pg->pg_version++;
948		uvm_pageinsert(pg);
949	}
950}
951 
952/*
953 * uvm_pageclean: clean page
954 *
955 * => erase page's identity (i.e. remove from object)
956 * => caller must lock page queues if `pg' is managed
957 * => assumes all valid mappings of pg are gone
958 */
959void
960uvm_pageclean(struct vm_page *pg)
961{
962	u_int flags_to_clear = 0;
963 
964	if ((pg->pg_flags & (PG_TABLED0x00000004|PQ_ACTIVE0x00040000|PQ_INACTIVE0x00020000)) &&
965	    (pg->uobject34.1
Field 'uobject' is equal to NULL
34.1
Field 'uobject' is equal to NULL
 == NULL((void *)0) || !UVM_OBJ_IS_PMAP(pg->uobject)((pg->uobject)->pgops == &pmap_pager)))
966		MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
 *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
 db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0);
35
←
Assuming the condition is false→
36
←
Loop condition is false.  Exiting loop→
967 
968#ifdef DEBUG
969	if (pg->uobject == (void *)0xdeadbeef &&
970	    pg->uanon == (void *)0xdeadbeef) {
971		panic("uvm_pagefree: freeing free page %p", pg);
972	}
973#endif
974 
975	KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 975, "(pg->pg_flags & PG_DEV) == 0"
));
37
←
Assuming the condition is true→
38
←
'?' condition is true→
976	KASSERT(pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) ||((pg->uobject == ((void *)0) || (((pg->uobject)->pgops
 == &pmap_pager) || ((pg->uobject)->pgops == &bufcache_pager
)) || rw_write_held(pg->uobject->vmobjlock)) ? (void)0 :
 __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 977, "pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
977	    rw_write_held(pg->uobject->vmobjlock))((pg->uobject == ((void *)0) || (((pg->uobject)->pgops
 == &pmap_pager) || ((pg->uobject)->pgops == &bufcache_pager
)) || rw_write_held(pg->uobject->vmobjlock)) ? (void)0 :
 __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 977, "pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
));
978	KASSERT(pg->uobject != NULL || pg->uanon == NULL ||((pg->uobject != ((void *)0) || pg->uanon == ((void *)0
) || rw_write_held(pg->uanon->an_lock)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 979, "pg->uobject != NULL || pg->uanon == NULL || rw_write_held(pg->uanon->an_lock)"
))
39
←
Assuming field 'uanon' is equal to null→
979	    rw_write_held(pg->uanon->an_lock))((pg->uobject != ((void *)0) || pg->uanon == ((void *)0
) || rw_write_held(pg->uanon->an_lock)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 979, "pg->uobject != NULL || pg->uanon == NULL || rw_write_held(pg->uanon->an_lock)"
));
980 
981	/*
982	 * if the page was an object page (and thus "TABLED"), remove it
983	 * from the object.
984	 */
985	if (pg->pg_flags & PG_TABLED0x00000004)
40
←
Assuming the condition is true→
41
←
Taking true branch→
986		uvm_pageremove(pg);
42
←
Calling 'uvm_pageremove'→
987 
988	/* now remove the page from the queues */
989	if (pg->pg_flags & PQ_ACTIVE0x00040000) {
990		TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev;
 *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
991		flags_to_clear |= PQ_ACTIVE0x00040000;
992		uvmexp.active--;
993	}
994	if (pg->pg_flags & PQ_INACTIVE0x00020000) {
995		if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000))
996			TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
997		else
998			TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
999		flags_to_clear |= PQ_INACTIVE0x00020000;
1000		uvmexp.inactive--;
1001	}
1002 
1003	/* if the page was wired, unwire it now. */
1004	if (pg->wire_count) {
1005		pg->wire_count = 0;
1006		uvmexp.wired--;
1007	}
1008	if (pg->uanon) {
1009		pg->uanon->an_page = NULL((void *)0);
1010		pg->uanon = NULL((void *)0);
1011	}
1012 
1013	/* Clean page state bits. */
1014	flags_to_clear |= PQ_ANON0x00100000|PQ_AOBJ0x00200000|PQ_ENCRYPT0x00400000|PG_ZERO0x00000100|PG_FAKE0x00000040|PG_BUSY0x00000001|
1015	    PG_RELEASED0x00000020|PG_CLEAN0x00000008|PG_CLEANCHK0x00000010;
1016	atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, flags_to_clear);
1017 
1018#ifdef DEBUG
1019	pg->uobject = (void *)0xdeadbeef;
1020	pg->offset = 0xdeadbeef;
1021	pg->uanon = (void *)0xdeadbeef;
1022#endif
1023}
1024 
1025/*
1026 * uvm_pagefree: free page
1027 *
1028 * => erase page's identity (i.e. remove from object)
1029 * => put page on free list
1030 * => caller must lock page queues if `pg' is managed
1031 * => assumes all valid mappings of pg are gone
1032 */
1033void
1034uvm_pagefree(struct vm_page *pg)
1035{
1036	if ((pg->pg_flags & (PG_TABLED0x00000004|PQ_ACTIVE0x00040000|PQ_INACTIVE0x00020000)) &&
30
←
Assuming the condition is true→
1037	    (pg->uobject == NULL((void *)0) || !UVM_OBJ_IS_PMAP(pg->uobject)((pg->uobject)->pgops == &pmap_pager)))
31
←
Assuming field 'uobject' is equal to NULL→
1038		MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
 *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
 db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0);
32
←
Assuming the condition is false→
33
←
Loop condition is false.  Exiting loop→
1039 
1040	uvm_pageclean(pg);
34
←
Calling 'uvm_pageclean'→
1041	uvm_pmr_freepages(pg, 1);
1042}
1043 
1044/*
1045 * uvm_page_unbusy: unbusy an array of pages.
1046 *
1047 * => pages must either all belong to the same object, or all belong to anons.
1048 * => if pages are anon-owned, anons must have 0 refcount.
1049 */
1050void
1051uvm_page_unbusy(struct vm_page **pgs, int npgs)
1052{
1053	struct vm_page *pg;
1054	struct uvm_object *uobj;
1055	int i;
1056 
1057	for (i = 0; i < npgs; i++) {
1
Assuming 'i' is < 'npgs'→
2
←
Loop condition is true.  Entering loop body→
1058		pg = pgs[i];
1059 
1060		if (pg == NULL((void *)0) || pg == PGO_DONTCARE((struct vm_page *) -1L)) {
3
←
Assuming 'pg' is not equal to NULL→
4
←
Assuming the condition is false→
5
←
Taking false branch→
1061			continue;
1062		}
1063 
1064		KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1064, "uvm_page_owner_locked_p(pg)"
));
6
←
Calling 'uvm_page_owner_locked_p'→
13
←
Returning from 'uvm_page_owner_locked_p'→
14
←
'?' condition is true→
1065		KASSERT(pg->pg_flags & PG_BUSY)((pg->pg_flags & 0x00000001) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1065, "pg->pg_flags & PG_BUSY"
));
15
←
Assuming the condition is true→
16
←
'?' condition is true→
1066 
1067		if (pg->pg_flags & PG_WANTED0x00000002) {
17
←
Assuming the condition is false→
18
←
Taking false branch→
1068			wakeup(pg);
1069		}
1070		if (pg->pg_flags & PG_RELEASED0x00000020) {
19
←
Assuming the condition is true→
20
←
Taking true branch→
1071			uobj = pg->uobject;
1072			if (uobj20.1
'uobj' is not equal to NULL
20.1
'uobj' is not equal to NULL
 != NULL((void *)0)) {
21
←
Taking true branch→
1073				uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1074				pmap_page_protect(pg, PROT_NONE0x00);
22
←
Calling 'pmap_page_protect'→
26
←
Returning from 'pmap_page_protect'→
1075				/* XXX won't happen right now */
1076				if (pg->pg_flags & PQ_AOBJ0x00200000)
27
←
Assuming the condition is false→
28
←
Taking false branch→
1077					uao_dropswap(uobj,
1078					    pg->offset >> PAGE_SHIFT12);
1079				uvm_pagefree(pg);
29
←
Calling 'uvm_pagefree'→
1080				uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1081			} else {
1082				atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001);
1083				UVM_PAGE_OWN(pg, NULL);
1084				rw_enter(pg->uanon->an_lock, RW_WRITE0x0001UL);
1085				uvm_anon_release(pg->uanon);
1086			}
1087		} else {
1088			atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_WANTED0x00000002|PG_BUSY0x00000001);
1089			UVM_PAGE_OWN(pg, NULL);
1090		}
1091	}
1092}
1093 
1094#if defined(UVM_PAGE_TRKOWN)
1095/*
1096 * uvm_page_own: set or release page ownership
1097 *
1098 * => this is a debugging function that keeps track of who sets PG_BUSY
1099 *	and where they do it.   it can be used to track down problems
1100 *	such a thread setting "PG_BUSY" and never releasing it.
1101 * => if "tag" is NULL then we are releasing page ownership
1102 */
1103void
1104uvm_page_own(struct vm_page *pg, char *tag)
1105{
1106	/* gain ownership? */
1107	if (tag) {
1108		if (pg->owner_tag) {
1109			printf("uvm_page_own: page %p already owned "
1110			    "by thread %d [%s]\n", pg,
1111			     pg->owner, pg->owner_tag);
1112			panic("uvm_page_own");
1113		}
1114		pg->owner = (curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc) ? curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
 (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
 __ci;})->ci_curproc->p_tid :  (pid_t) -1;
1115		pg->owner_tag = tag;
1116		return;
1117	}
1118 
1119	/* drop ownership */
1120	if (pg->owner_tag == NULL((void *)0)) {
1121		printf("uvm_page_own: dropping ownership of an non-owned "
1122		    "page (%p)\n", pg);
1123		panic("uvm_page_own");
1124	}
1125	pg->owner_tag = NULL((void *)0);
1126	return;
1127}
1128#endif
1129 
1130/*
1131 * when VM_PHYSSEG_MAX is 1, we can simplify these functions
1132 */
1133 
1134#if VM_PHYSSEG_MAX16 > 1
1135/*
1136 * vm_physseg_find: find vm_physseg structure that belongs to a PA
1137 */
1138int
1139vm_physseg_find(paddr_t pframe, int *offp)
1140{
1141	struct vm_physseg *seg;
1142 
1143#if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2)
1144	/* binary search for it */
1145	int	start, len, try;
1146 
1147	/*
1148	 * if try is too large (thus target is less than than try) we reduce
1149	 * the length to trunc(len/2) [i.e. everything smaller than "try"]
1150	 *
1151	 * if the try is too small (thus target is greater than try) then
1152	 * we set the new start to be (try + 1).   this means we need to
1153	 * reduce the length to (round(len/2) - 1).
1154	 *
1155	 * note "adjust" below which takes advantage of the fact that
1156	 *  (round(len/2) - 1) == trunc((len - 1) / 2)
1157	 * for any value of len we may have
1158	 */
1159 
1160	for (start = 0, len = vm_nphysseg ; len != 0 ; len = len / 2) {
1161		try = start + (len / 2);	/* try in the middle */
1162		seg = vm_physmem + try;
1163 
1164		/* start past our try? */
1165		if (pframe >= seg->start) {
1166			/* was try correct? */
1167			if (pframe < seg->end) {
1168				if (offp)
1169					*offp = pframe - seg->start;
1170				return try;            /* got it */
1171			}
1172			start = try + 1;	/* next time, start here */
1173			len--;			/* "adjust" */
1174		} else {
1175			/*
1176			 * pframe before try, just reduce length of
1177			 * region, done in "for" loop
1178			 */
1179		}
1180	}
1181	return -1;
1182 
1183#else
1184	/* linear search for it */
1185	int	lcv;
1186 
1187	for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) {
1188		if (pframe >= seg->start && pframe < seg->end) {
1189			if (offp)
1190				*offp = pframe - seg->start;
1191			return lcv;		   /* got it */
1192		}
1193	}
1194	return -1;
1195 
1196#endif
1197}
1198 
1199/*
1200 * PHYS_TO_VM_PAGE: find vm_page for a PA.   used by MI code to get vm_pages
1201 * back from an I/O mapping (ugh!).   used in some MD code as well.
1202 */
1203struct vm_page *
1204PHYS_TO_VM_PAGE(paddr_t pa)
1205{
1206	paddr_t pf = atop(pa)((pa) >> 12);
1207	int	off;
1208	int	psi;
1209 
1210	psi = vm_physseg_find(pf, &off);
1211 
1212	return (psi == -1) ? NULL((void *)0) : &vm_physmem[psi].pgs[off];
1213}
1214#endif /* VM_PHYSSEG_MAX > 1 */
1215 
1216/*
1217 * uvm_pagelookup: look up a page
1218 */
1219struct vm_page *
1220uvm_pagelookup(struct uvm_object *obj, voff_t off)
1221{
1222	/* XXX if stack is too much, handroll */
1223	struct vm_page pg;
1224 
1225	pg.offset = off;
1226	return RBT_FIND(uvm_objtree, &obj->memt, &pg)uvm_objtree_RBT_FIND(&obj->memt, &pg);
1227}
1228 
1229/*
1230 * uvm_pagewire: wire the page, thus removing it from the daemon's grasp
1231 *
1232 * => caller must lock page queues
1233 */
1234void
1235uvm_pagewire(struct vm_page *pg)
1236{
1237	KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1237, "uvm_page_owner_locked_p(pg)"
));
1238	MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
 *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
 db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0);
1239 
1240	if (pg->wire_count == 0) {
1241		if (pg->pg_flags & PQ_ACTIVE0x00040000) {
1242			TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev;
 *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
1243			atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000);
1244			uvmexp.active--;
1245		}
1246		if (pg->pg_flags & PQ_INACTIVE0x00020000) {
1247			if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000))
1248				TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
1249			else
1250				TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
1251			atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000);
1252			uvmexp.inactive--;
1253		}
1254		uvmexp.wired++;
1255	}
1256	pg->wire_count++;
1257}
1258 
1259/*
1260 * uvm_pageunwire: unwire the page.
1261 *
1262 * => activate if wire count goes to zero.
1263 * => caller must lock page queues
1264 */
1265void
1266uvm_pageunwire(struct vm_page *pg)
1267{
1268	KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1268, "uvm_page_owner_locked_p(pg)"
));
1269	MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
 *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
 db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0);
1270 
1271	pg->wire_count--;
1272	if (pg->wire_count == 0) {
1273		TAILQ_INSERT_TAIL(&uvm.page_active, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev
 = (&uvm.page_active)->tqh_last; *(&uvm.page_active
)->tqh_last = (pg); (&uvm.page_active)->tqh_last = &
(pg)->pageq.tqe_next; } while (0);
1274		uvmexp.active++;
1275		atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000);
1276		uvmexp.wired--;
1277	}
1278}
1279 
1280/*
1281 * uvm_pagedeactivate: deactivate page -- no pmaps have access to page
1282 *
1283 * => caller must lock page queues
1284 * => caller must check to make sure page is not wired
1285 * => object that page belongs to must be locked (so we can adjust pg->flags)
1286 */
1287void
1288uvm_pagedeactivate(struct vm_page *pg)
1289{
1290	KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1290, "uvm_page_owner_locked_p(pg)"
));
1291	MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
 *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
 db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0);
1292 
1293	if (pg->pg_flags & PQ_ACTIVE0x00040000) {
1294		TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev;
 *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
1295		atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000);
1296		uvmexp.active--;
1297	}
1298	if ((pg->pg_flags & PQ_INACTIVE0x00020000) == 0) {
1299		KASSERT(pg->wire_count == 0)((pg->wire_count == 0) ? (void)0 : __assert("diagnostic ",
 "/usr/src/sys/uvm/uvm_page.c", 1299, "pg->wire_count == 0"
));
1300		if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000))
1301			TAILQ_INSERT_TAIL(&uvm.page_inactive_swp, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev
 = (&uvm.page_inactive_swp)->tqh_last; *(&uvm.page_inactive_swp
)->tqh_last = (pg); (&uvm.page_inactive_swp)->tqh_last
 = &(pg)->pageq.tqe_next; } while (0);
1302		else
1303			TAILQ_INSERT_TAIL(&uvm.page_inactive_obj, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev
 = (&uvm.page_inactive_obj)->tqh_last; *(&uvm.page_inactive_obj
)->tqh_last = (pg); (&uvm.page_inactive_obj)->tqh_last
 = &(pg)->pageq.tqe_next; } while (0);
1304		atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000);
1305		uvmexp.inactive++;
1306		pmap_clear_reference(pg)pmap_clear_attrs(pg, 0x0000000000000020UL);
1307		/*
1308		 * update the "clean" bit.  this isn't 100%
1309		 * accurate, and doesn't have to be.  we'll
1310		 * re-sync it after we zap all mappings when
1311		 * scanning the inactive list.
1312		 */
1313		if ((pg->pg_flags & PG_CLEAN0x00000008) != 0 &&
1314		    pmap_is_modified(pg)pmap_test_attrs(pg, 0x0000000000000040UL))
1315			atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
1316	}
1317}
1318 
1319/*
1320 * uvm_pageactivate: activate page
1321 *
1322 * => caller must lock page queues
1323 */
1324void
1325uvm_pageactivate(struct vm_page *pg)
1326{
1327	KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1327, "uvm_page_owner_locked_p(pg)"
));
1328	MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
 *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
 db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0);
1329 
1330	if (pg->pg_flags & PQ_INACTIVE0x00020000) {
1331		if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000))
1332			TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
1333		else
1334			TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
1335		atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000);
1336		uvmexp.inactive--;
1337	}
1338	if (pg->wire_count == 0) {
1339		/*
1340		 * if page is already active, remove it from list so we
1341		 * can put it at tail.  if it wasn't active, then mark
1342		 * it active and bump active count
1343		 */
1344		if (pg->pg_flags & PQ_ACTIVE0x00040000)
1345			TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev;
 *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0);
1346		else {
1347			atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000);
1348			uvmexp.active++;
1349		}
1350 
1351		TAILQ_INSERT_TAIL(&uvm.page_active, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev
 = (&uvm.page_active)->tqh_last; *(&uvm.page_active
)->tqh_last = (pg); (&uvm.page_active)->tqh_last = &
(pg)->pageq.tqe_next; } while (0);
1352	}
1353}
1354 
1355/*
1356 * uvm_pagezero: zero fill a page
1357 */
1358void
1359uvm_pagezero(struct vm_page *pg)
1360{
1361	atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
1362	pmap_zero_page(pg);
1363}
1364 
1365/*
1366 * uvm_pagecopy: copy a page
1367 */
1368void
1369uvm_pagecopy(struct vm_page *src, struct vm_page *dst)
1370{
1371	atomic_clearbits_intx86_atomic_clearbits_u32(&dst->pg_flags, PG_CLEAN0x00000008);
1372	pmap_copy_page(src, dst);
1373}
1374 
1375/*
1376 * uvm_page_owner_locked_p: return true if object associated with page is
1377 * locked.  this is a weak check for runtime assertions only.
1378 */
1379int
1380uvm_page_owner_locked_p(struct vm_page *pg)
1381{
1382	if (pg->uobject != NULL((void *)0)) {
7
←
Assuming field 'uobject' is not equal to NULL→
8
←
Taking true branch→
1383		if (UVM_OBJ_IS_DUMMY(pg->uobject)(((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)))
9
←
Assuming the condition is false→
10
←
Assuming the condition is false→
11
←
Taking false branch→
1384			return 1;
1385		return rw_write_held(pg->uobject->vmobjlock);
12
←
Returning without writing to 'pg->uobject'→
1386	}
1387	if (pg->uanon != NULL((void *)0)) {
1388		return rw_write_held(pg->uanon->an_lock);
1389	}
1390	return 1;
1391}
1392 
1393/*
1394 * uvm_pagecount: count the number of physical pages in the address range.
1395 */
1396psize_t
1397uvm_pagecount(struct uvm_constraint_range* constraint)
1398{
1399	int lcv;
1400	psize_t sz;
1401	paddr_t low, high;
1402	paddr_t ps_low, ps_high;
1403 
1404	/* Algorithm uses page numbers. */
1405	low = atop(constraint->ucr_low)((constraint->ucr_low) >> 12);
1406	high = atop(constraint->ucr_high)((constraint->ucr_high) >> 12);
1407 
1408	sz = 0;
1409	for (lcv = 0; lcv < vm_nphysseg; lcv++) {
1410		ps_low = MAX(low, vm_physmem[lcv].avail_start)(((low)>(vm_physmem[lcv].avail_start))?(low):(vm_physmem[lcv
].avail_start));
1411		ps_high = MIN(high, vm_physmem[lcv].avail_end)(((high)<(vm_physmem[lcv].avail_end))?(high):(vm_physmem[lcv
].avail_end));
1412		if (ps_low < ps_high)
1413			sz += ps_high - ps_low;
1414	}
1415	return sz;
1416}

←

/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj/machine/pmap.h

1/*	$OpenBSD: pmap.h,v 1.78 2021/06/18 06:17:28 guenther Exp $	*/
2/*	$NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $	*/
3 
4/*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28 
29/*
30 * Copyright (c) 2001 Wasabi Systems, Inc.
31 * All rights reserved.
32 *
33 * Written by Frank van der Linden for Wasabi Systems, Inc.
34 *
35 * Redistribution and use in source and binary forms, with or without
36 * modification, are permitted provided that the following conditions
37 * are met:
38 * 1. Redistributions of source code must retain the above copyright
39 *    notice, this list of conditions and the following disclaimer.
40 * 2. Redistributions in binary form must reproduce the above copyright
41 *    notice, this list of conditions and the following disclaimer in the
42 *    documentation and/or other materials provided with the distribution.
43 * 3. All advertising materials mentioning features or use of this software
44 *    must display the following acknowledgement:
45 *      This product includes software developed for the NetBSD Project by
46 *      Wasabi Systems, Inc.
47 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
48 *    or promote products derived from this software without specific prior
49 *    written permission.
50 *
51 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
52 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
53 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
54 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
55 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
56 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
57 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
58 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
59 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
60 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
61 * POSSIBILITY OF SUCH DAMAGE.
62 */
63 
64/*
65 * pmap.h: see pmap.c for the history of this pmap module.
66 */
67 
68#ifndef	_MACHINE_PMAP_H_
69#define	_MACHINE_PMAP_H_
70 
71#ifndef _LOCORE
72#ifdef _KERNEL1
73#include <machine/cpufunc.h>
74#endif /* _KERNEL */
75#include <sys/mutex.h>
76#include <uvm/uvm_object.h>
77#include <machine/pte.h>
78#endif
79 
80/*
81 * The x86_64 pmap module closely resembles the i386 one. It uses
82 * the same recursive entry scheme. See the i386 pmap.h for a
83 * description. The alternate area trick for accessing non-current
84 * pmaps has been removed, though, because it performs badly on SMP
85 * systems.
86 * The most obvious difference to i386 is that 2 extra levels of page
87 * table need to be dealt with. The level 1 page table pages are at:
88 *
89 * l1: 0x00007f8000000000 - 0x00007fffffffffff     (39 bits, needs PML4 entry)
90 *
91 * The other levels are kept as physical pages in 3 UVM objects and are
92 * temporarily mapped for virtual access when needed.
93 *
94 * The other obvious difference from i386 is that it has a direct map of all
95 * physical memory in the VA range:
96 *
97 *     0xfffffd8000000000 - 0xffffff7fffffffff
98 *
99 * The direct map is used in some cases to access PTEs of non-current pmaps.
100 *
101 * Note that address space is signed, so the layout for 48 bits is:
102 *
103 *  +---------------------------------+ 0xffffffffffffffff
104 *  |         Kernel Image            |
105 *  +---------------------------------+ 0xffffff8000000000
106 *  |         Direct Map              |
107 *  +---------------------------------+ 0xfffffd8000000000
108 *  ~                                 ~
109 *  |                                 |
110 *  |         Kernel Space            |
111 *  |                                 |
112 *  |                                 |
113 *  +---------------------------------+ 0xffff800000000000 = 0x0000800000000000
114 *  |    L1 table (PTE pages)         |
115 *  +---------------------------------+ 0x00007f8000000000
116 *  ~                                 ~
117 *  |                                 |
118 *  |         User Space              |
119 *  |                                 |
120 *  |                                 |
121 *  +---------------------------------+ 0x0000000000000000
122 *
123 * In other words, there is a 'VA hole' at 0x0000800000000000 -
124 * 0xffff800000000000 which will trap, just as on, for example,
125 * sparcv9.
126 *
127 * The unused space can be used if needed, but it adds a little more
128 * complexity to the calculations.
129 */
130 
131/*
132 * Mask to get rid of the sign-extended part of addresses.
133 */
134#define VA_SIGN_MASK0xffff000000000000		0xffff000000000000
135#define VA_SIGN_NEG(va)((va) | 0xffff000000000000)		((va) | VA_SIGN_MASK0xffff000000000000)
136/*
137 * XXXfvdl this one's not right.
138 */
139#define VA_SIGN_POS(va)((va) & ~0xffff000000000000)		((va) & ~VA_SIGN_MASK0xffff000000000000)
140 
141#define L4_SLOT_PTE255		255
142#define L4_SLOT_KERN256		256
143#define L4_SLOT_KERNBASE511	511
144#define NUM_L4_SLOT_DIRECT4	4
145#define L4_SLOT_DIRECT(511 - 4)		(L4_SLOT_KERNBASE511 - NUM_L4_SLOT_DIRECT4)
146#define L4_SLOT_EARLY((511 - 4) - 1)		(L4_SLOT_DIRECT(511 - 4) - 1)
147 
148#define PDIR_SLOT_KERN256		L4_SLOT_KERN256
149#define PDIR_SLOT_PTE255		L4_SLOT_PTE255
150#define PDIR_SLOT_DIRECT(511 - 4)	L4_SLOT_DIRECT(511 - 4)
151#define PDIR_SLOT_EARLY((511 - 4) - 1)		L4_SLOT_EARLY((511 - 4) - 1)
152 
153/*
154 * the following defines give the virtual addresses of various MMU
155 * data structures:
156 * PTE_BASE: the base VA of the linear PTE mappings
157 * PDP_PDE: the VA of the PDE that points back to the PDP
158 *
159 */
160 
161#define PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))  ((pt_entry_t *) (L4_SLOT_PTE255 * NBPD_L4(1ULL << 39)))
162#define PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000))	(VA_SIGN_NEG((L4_SLOT_DIRECT * NBPD_L4))((((511 - 4) * (1ULL << 39))) | 0xffff000000000000))
163#define PMAP_DIRECT_END((((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000
))		(VA_SIGN_NEG(((L4_SLOT_DIRECT + \(((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000
)
164    NUM_L4_SLOT_DIRECT) * NBPD_L4))(((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000
))
165 
166#define L1_BASE((pt_entry_t *) (255 * (1ULL << 39)))		PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))
167 
168#define L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
 39))) + 255 * (1ULL << 30))) ((pd_entry_t *)((char *)L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) + L4_SLOT_PTE255 * NBPD_L3(1ULL << 30)))
169#define L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *
) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255
 * (1ULL << 21))) ((pd_entry_t *)((char *)L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
 39))) + 255 * (1ULL << 30))) + L4_SLOT_PTE255 * NBPD_L2(1ULL << 21)))
170#define L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
 (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
 << 12))) ((pd_entry_t *)((char *)L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *
) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255
 * (1ULL << 21))) + L4_SLOT_PTE255 * NBPD_L1(1ULL << 12)))
171 
172#define PDP_PDE(((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t
 *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255
 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (
1ULL << 12))) + 255)		(L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
 (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
 << 12))) + PDIR_SLOT_PTE255)
173 
174#define PDP_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
 (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
 << 12)))	L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
 (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
 << 12)))
175 
176#define NKL4_MAX_ENTRIES(unsigned long)1	(unsigned long)1
177#define NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512)	(unsigned long)(NKL4_MAX_ENTRIES(unsigned long)1 * 512)
178#define NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512
)	(unsigned long)(NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512) * 512)
179#define NKL1_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)((unsigned long
)1 * 512) * 512) * 512)	(unsigned long)(NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512
) * 512)
180 
181#define NKL4_KIMG_ENTRIES1	1
182#define NKL3_KIMG_ENTRIES1	1
183#define NKL2_KIMG_ENTRIES64	64
184 
185/* number of pages of direct map entries set up by locore0.S */
186#define NDML4_ENTRIES1		1
187#define NDML3_ENTRIES1		1
188#define NDML2_ENTRIES4		4	/* 4GB */
189 
190/*
191 * Since kva space is below the kernel in its entirety, we start off
192 * with zero entries on each level.
193 */
194#define NKL4_START_ENTRIES0	0
195#define NKL3_START_ENTRIES0	0
196#define NKL2_START_ENTRIES0	0
197#define NKL1_START_ENTRIES0	0	/* XXX */
198 
199#define NTOPLEVEL_PDES((1 << 12) / (sizeof (pd_entry_t)))		(PAGE_SIZE(1 << 12) / (sizeof (pd_entry_t)))
200 
201#define NPDPG((1 << 12) / sizeof (pd_entry_t))			(PAGE_SIZE(1 << 12) / sizeof (pd_entry_t))
202 
203/*
204 * pl*_pi: index in the ptp page for a pde mapping a VA.
205 * (pl*_i below is the index in the virtual array of all pdes per level)
206 */
207#define pl1_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x00000000001ff000UL
) >> 12)	(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12)
208#define pl2_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x000000003fe00000UL
) >> 21)	(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21)
209#define pl3_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x0000007fc0000000UL
) >> 30)	(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30)
210#define pl4_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x0000ff8000000000UL
) >> 39)	(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39)
211 
212/*
213 * pl*_i: generate index into pde/pte arrays in virtual space
214 */
215#define pl1_i(VA)(((((VA) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)	(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L1_FRAME(((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)|0x00000000001ff000UL)) >> L1_SHIFT12)
216#define pl2_i(VA)(((((VA) & ~0xffff000000000000)) & ((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)) >> 21)	(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)) >> L2_SHIFT21)
217#define pl3_i(VA)(((((VA) & ~0xffff000000000000)) & (0x0000ff8000000000UL
|0x0000007fc0000000UL)) >> 30)	(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L3_FRAME(0x0000ff8000000000UL|0x0000007fc0000000UL)) >> L3_SHIFT30)
218#define pl4_i(VA)(((((VA) & ~0xffff000000000000)) & 0x0000ff8000000000UL
) >> 39)	(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L4_FRAME0x0000ff8000000000UL) >> L4_SHIFT39)
219#define pl_i(va, lvl)(((((va) & ~0xffff000000000000)) & ptp_masks[(lvl)-1]
) >> ptp_shifts[(lvl)-1]) \
220        (((VA_SIGN_POS(va)((va) & ~0xffff000000000000)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1])
221 
222#define PTP_MASK_INITIALIZER{ (((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)|0x00000000001ff000UL), ((0x0000ff8000000000UL|0x0000007fc0000000UL
)|0x000000003fe00000UL), (0x0000ff8000000000UL|0x0000007fc0000000UL
), 0x0000ff8000000000UL }	{ L1_FRAME(((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)|0x00000000001ff000UL), L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
), L3_FRAME(0x0000ff8000000000UL|0x0000007fc0000000UL), L4_FRAME0x0000ff8000000000UL }
223#define PTP_SHIFT_INITIALIZER{ 12, 21, 30, 39 }	{ L1_SHIFT12, L2_SHIFT21, L3_SHIFT30, L4_SHIFT39 }
224#define NKPTP_INITIALIZER{ 0, 0, 0, 0 }	{ NKL1_START_ENTRIES0, NKL2_START_ENTRIES0, \
225				  NKL3_START_ENTRIES0, NKL4_START_ENTRIES0 }
226#define NKPTPMAX_INITIALIZER{ (unsigned long)((unsigned long)((unsigned long)((unsigned long
)1 * 512) * 512) * 512), (unsigned long)((unsigned long)((unsigned
 long)1 * 512) * 512), (unsigned long)((unsigned long)1 * 512
), (unsigned long)1 }	{ NKL1_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)((unsigned long
)1 * 512) * 512) * 512), NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512
), \
227				  NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512), NKL4_MAX_ENTRIES(unsigned long)1 }
228#define NBPD_INITIALIZER{ (1ULL << 12), (1ULL << 21), (1ULL << 30),
 (1ULL << 39) }	{ NBPD_L1(1ULL << 12), NBPD_L2(1ULL << 21), NBPD_L3(1ULL << 30), NBPD_L4(1ULL << 39) }
229#define PDES_INITIALIZER{ ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
 39))) + 255 * (1ULL << 30))), ((pd_entry_t *)((char *)
((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21
))), ((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t
 *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255
 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (
1ULL << 12))) }	{ L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
 39))) + 255 * (1ULL << 30))), L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *
) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255
 * (1ULL << 21))), L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
 (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
 << 12))) }
230 
231/*
232 * PTP macros:
233 *   a PTP's index is the PD index of the PDE that points to it
234 *   a PTP's offset is the byte-offset in the PTE space that this PTP is at
235 *   a PTP's VA is the first VA mapped by that PTP
236 */
237 
238#define ptp_va2o(va, lvl)((((((va) & ~0xffff000000000000)) & ptp_masks[((lvl)+
1)-1]) >> ptp_shifts[((lvl)+1)-1]) * (1 << 12))	(pl_i(va, (lvl)+1)(((((va) & ~0xffff000000000000)) & ptp_masks[((lvl)+1
)-1]) >> ptp_shifts[((lvl)+1)-1]) * PAGE_SIZE(1 << 12))
239 
240#define PTP_LEVELS4	4
241 
242/*
243 * PG_AVAIL usage: we make use of the ignored bits of the PTE
244 */
245 
246#define PG_W0x0000000000000200UL		PG_AVAIL10x0000000000000200UL	/* "wired" mapping */
247#define PG_PVLIST0x0000000000000400UL	PG_AVAIL20x0000000000000400UL	/* mapping has entry on pvlist */
248/* PG_AVAIL3 not used */
249 
250/*
251 * PCID assignments.
252 * The shootdown code assumes KERN, PROC, and PROC_INTEL are both
253 * consecutive and in that order.
254 */
255#define PCID_KERN0	0	/* for pmap_kernel() */
256#define PCID_PROC1	1	/* non-pmap_kernel(), U+K */
257#define PCID_PROC_INTEL2	2	/* non-pmap_kernel(), U-K (meltdown) */
258#define PCID_TEMP3	3	/* temp mapping of another non-pmap_kernel() */
259 
260extern int pmap_use_pcid;	/* non-zero if PCID support is enabled */
261 
262/*
263 * Number of PTEs per cache line.  8 byte pte, 64-byte cache line
264 * Used to avoid false sharing of cache lines.
265 */
266#define NPTECL8		8
267 
268 
269#if defined(_KERNEL1) && !defined(_LOCORE)
270/*
271 * pmap data structures: see pmap.c for details of locking.
272 */
273 
274struct pmap;
275typedef struct pmap *pmap_t;
276 
277/*
278 * we maintain a list of all non-kernel pmaps
279 */
280 
281LIST_HEAD(pmap_head, pmap)struct pmap_head { struct pmap *lh_first; }; /* struct pmap_head: head of a pmap list */
282 
283/*
284 * the pmap structure
285 *
286 * note that the pm_obj contains the reference count,
287 * page list, and number of PTPs within the pmap.
288 */
289 
290#define PMAP_TYPE_NORMAL1	1
291#define PMAP_TYPE_EPT2		2
292#define PMAP_TYPE_RVI3		3
293#define pmap_nested(pm)((pm)->pm_type != 1) ((pm)->pm_type != PMAP_TYPE_NORMAL1)
294 
295struct pmap {
296	struct mutex pm_mtx;
297	struct uvm_object pm_obj[PTP_LEVELS4-1]; /* objects for lvl >= 1) */
298	LIST_ENTRY(pmap)struct { struct pmap *le_next; struct pmap **le_prev; } pm_list;	/* list (lck by pm_list lock) */
299	/*
300	 * pm_pdir         : VA of page table to be used when executing in
301	 *                   privileged mode
302	 * pm_pdirpa       : PA of page table to be used when executing in
303	 *                   privileged mode
304	 * pm_pdir_intel   : VA of special page table to be used when executing
305	 *                   on an Intel CPU in usermode (no kernel mappings)
306	 * pm_pdirpa_intel : PA of special page table to be used when executing
307	 *                   on an Intel CPU in usermode (no kernel mappings)
308	 */
309	pd_entry_t *pm_pdir, *pm_pdir_intel;
310	paddr_t pm_pdirpa, pm_pdirpa_intel;
311 
312	struct vm_page *pm_ptphint[PTP_LEVELS4-1];
313					/* pointer to a PTP in our pmap */
314	struct pmap_statistics pm_stats;  /* pmap stats (lck by object lock) */
315 
316	int pm_type;			/* Type of pmap this is (PMAP_TYPE_x) */
317	uint64_t eptp;			/* cached EPTP (used by vmm) */
318};
319 
320/*
321 * MD flags that we use for pmap_enter (in the pa):
322 */
323#define PMAP_PA_MASK~((paddr_t)((1 << 12) - 1))	~((paddr_t)PAGE_MASK((1 << 12) - 1)) /* to remove the flags */
324#define	PMAP_NOCACHE0x1	0x1 /* set the non-cacheable bit. */
325#define	PMAP_WC0x2		0x2 /* set page write combining. */
326 
327/*
328 * We keep mod/ref flags in struct vm_page->pg_flags.
329 */
330#define	PG_PMAP_MOD0x01000000	PG_PMAP00x01000000
331#define	PG_PMAP_REF0x02000000	PG_PMAP10x02000000
332#define	PG_PMAP_WC0x04000000      PG_PMAP20x04000000
333 
334/*
335 * for each managed physical page we maintain a list of <PMAP,VA>'s
336 * which it is mapped at.
337 */
338struct pv_entry {			/* locked by its list's pvh_lock */
339	struct pv_entry *pv_next;	/* next entry */
340	struct pmap *pv_pmap;		/* the pmap */
341	vaddr_t pv_va;			/* the virtual address */
342	struct vm_page *pv_ptp;		/* the vm_page of the PTP */
343};
344 
345/*
346 * global kernel variables
347 */
348 
349extern struct pmap kernel_pmap_store;	/* kernel pmap */
350 
351extern long nkptp[];
352 
353extern const paddr_t ptp_masks[];
354extern const int ptp_shifts[];
355extern const long nbpd[], nkptpmax[];
356 
357/*
358 * macros
359 */
360 
361#define	pmap_kernel()(&kernel_pmap_store)			(&kernel_pmap_store)
362#define	pmap_resident_count(pmap)((pmap)->pm_stats.resident_count)	((pmap)->pm_stats.resident_count)
363#define	pmap_wired_count(pmap)((pmap)->pm_stats.wired_count)		((pmap)->pm_stats.wired_count)
364#define	pmap_update(pmap)		/* nothing (yet) */
365 
366#define pmap_clear_modify(pg)pmap_clear_attrs(pg, 0x0000000000000040UL)		pmap_clear_attrs(pg, PG_M0x0000000000000040UL)
367#define pmap_clear_reference(pg)pmap_clear_attrs(pg, 0x0000000000000020UL)	pmap_clear_attrs(pg, PG_U0x0000000000000020UL)
368#define pmap_copy(DP,SP,D,L,S)
369#define pmap_is_modified(pg)pmap_test_attrs(pg, 0x0000000000000040UL)		pmap_test_attrs(pg, PG_M0x0000000000000040UL)
370#define pmap_is_referenced(pg)pmap_test_attrs(pg, 0x0000000000000020UL)		pmap_test_attrs(pg, PG_U0x0000000000000020UL)
371#define pmap_move(DP,SP,D,L,S)
372#define pmap_valid_entry(E)((E) & 0x0000000000000001UL) 		((E) & PG_V0x0000000000000001UL) /* is PDE or PTE valid? */
373 
374#define pmap_proc_iflush(p,va,len)	/* nothing */
375#define pmap_unuse_final(p)		/* nothing */
376#define	pmap_remove_holes(vm)do { } while (0)		do { /* nothing */ } while (0)
377 
378 
379/*
380 * prototypes
381 */
382 
383void		map_tramps(void);	/* machdep.c */
384paddr_t		pmap_bootstrap(paddr_t, paddr_t);
385void		pmap_randomize(void);
386void		pmap_randomize_level(pd_entry_t *, int);
387int		pmap_clear_attrs(struct vm_page *, unsigned long);
388static void	pmap_page_protect(struct vm_page *, vm_prot_t);
389void		pmap_page_remove (struct vm_page *);
390static void	pmap_protect(struct pmap *, vaddr_t,
391				vaddr_t, vm_prot_t);
392void		pmap_remove(struct pmap *, vaddr_t, vaddr_t);
393int		pmap_test_attrs(struct vm_page *, unsigned);
394static void	pmap_update_pg(vaddr_t);
395void		pmap_write_protect(struct pmap *, vaddr_t,
396				vaddr_t, vm_prot_t);
397void		pmap_fix_ept(struct pmap *, vaddr_t);
398 
399paddr_t	pmap_prealloc_lowmem_ptps(paddr_t);
400 
401void	pagezero(vaddr_t);
402 
403int	pmap_convert(struct pmap *, int);
404void	pmap_enter_special(vaddr_t, paddr_t, vm_prot_t);
405vaddr_t	pmap_set_pml4_early(paddr_t pa);
406void	pmap_clear_pml4_early(void);
407 
408/*
409 * functions for flushing the cache for vaddrs and pages.
410 * these functions are not part of the MI pmap interface and thus
411 * should not be used as such.
412 */
413void	pmap_flush_cache(vaddr_t, vsize_t);
414#define pmap_flush_page(paddr)do { ((void)0); pmap_flush_cache(((vaddr_t)(((((511 - 4) * (1ULL
 << 39))) | 0xffff000000000000)) + (paddr)), (1 <<
 12)); } while ( 0) do {					\
415	KDASSERT(PHYS_TO_VM_PAGE(paddr) != NULL)((void)0);			\
416	pmap_flush_cache(PMAP_DIRECT_MAP(paddr)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (paddr)), PAGE_SIZE(1 << 12));		\
417} while (/* CONSTCOND */ 0)
418 
419#define	PMAP_STEAL_MEMORY	/* enable pmap_steal_memory() */
420#define PMAP_GROWKERNEL		/* turn on pmap_growkernel interface */
421 
422/*
423 * inline functions
424 */
425 
426static inline void
427pmap_remove_all(struct pmap *pmap)
428{
429	/* Nothing. */
430}
431 
432/*
433 * pmap_update_pg: flush one page from the TLB (or flush the whole thing
434 *	if hardware doesn't support one-page flushing)
435 */
436 
437inline static void
438pmap_update_pg(vaddr_t va)
439{
440	invlpg(va);
441}
442 
443/*
444 * pmap_page_protect: change the protection of all recorded mappings
445 *	of a managed page
446 *
447 * => this function is a frontend for pmap_page_remove/pmap_clear_attrs
448 * => we only have to worry about making the page more protected.
449 *	unprotecting a page is done on-demand at fault time.
450 */
451 
452inline static void
453pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
454{
455	if ((prot & PROT_WRITE0x02) == 0) {
23
←
Taking true branch→
456		if (prot & (PROT_READ0x01 | PROT_EXEC0x04)) {
24
←
Taking false branch→
457			(void) pmap_clear_attrs(pg, PG_RW0x0000000000000002UL);
458		} else {
459			pmap_page_remove(pg);
25
←
Value assigned to field 'uobject'→
460		}
461	}
462}
463 
464/*
465 * pmap_protect: change the protection of pages in a pmap
466 *
467 * => this function is a frontend for pmap_remove/pmap_write_protect
468 * => we only have to worry about making the page more protected.
469 *	unprotecting a page is done on-demand at fault time.
470 */
471 
472inline static void
473pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
474{
475	if ((prot & PROT_WRITE0x02) == 0) {
476		if (prot & (PROT_READ0x01| PROT_EXEC0x04)) {
477			pmap_write_protect(pmap, sva, eva, prot);
478		} else {
479			pmap_remove(pmap, sva, eva);
480		}
481	}
482}
483 
484/*
485 * various address inlines
486 *
487 *  vtopte: return a pointer to the PTE mapping a VA, works only for
488 *  user and PT addresses
489 *
490 *  kvtopte: return a pointer to the PTE mapping a kernel VA
491 */
492 
493static inline pt_entry_t *
494vtopte(vaddr_t va)
495{
496	return (PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12));
497}
498 
499static inline pt_entry_t *
500kvtopte(vaddr_t va)
501{
502#ifdef LARGEPAGES
503	{
504		pd_entry_t *pde;
505 
506		pde = L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl2_i(va)(((((va) & ~0xffff000000000000)) & ((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)) >> 21);
507		if (*pde & PG_PS0x0000000000000080UL)
508			return ((pt_entry_t *)pde);
509	}
510#endif
511 
512	return (PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12));
513}
514 
515#define PMAP_DIRECT_MAP(pa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pa))	((vaddr_t)PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) + (pa))
516#define PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)))	((paddr_t)(va) - PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)))
517#define pmap_map_direct(pg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (((pg)->phys_addr)))	PMAP_DIRECT_MAP(VM_PAGE_TO_PHYS(pg))((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (((pg)->phys_addr)))
518#define pmap_unmap_direct(va)PHYS_TO_VM_PAGE(((paddr_t)(va) - (((((511 - 4) * (1ULL <<
 39))) | 0xffff000000000000))))	PHYS_TO_VM_PAGE(PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
))))
519 
520#define __HAVE_PMAP_DIRECT
521 
522#endif /* _KERNEL && !_LOCORE */
523 
524#ifndef _LOCORE
525struct pv_entry;
526struct vm_page_md {
527	struct mutex pv_mtx;
528	struct pv_entry *pv_list;
529};
530 
531#define VM_MDPAGE_INIT(pg)do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(pg
)->mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) <
 0x9) ? 0x9 : ((0xa)))); } while (0); (pg)->mdpage.pv_list
 = ((void *)0); } while (0) do {		\
532	mtx_init(&(pg)->mdpage.pv_mtx, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&(pg)->
mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9
) ? 0x9 : ((0xa)))); } while (0); \
533	(pg)->mdpage.pv_list = NULL((void *)0);	\
534} while (0)
535#endif	/* !_LOCORE */
536 
537#endif	/* _MACHINE_PMAP_H_ */