Bug Summary

File:uvm/uvm_page.c
Warning:line 157, column 2
Access to field 'pgops' results in a dereference of a null pointer (loaded from field 'uobject')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple amd64-unknown-openbsd7.0 -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name uvm_page.c -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model static -mframe-pointer=all -relaxed-aliasing -fno-rounding-math -mconstructor-aliases -ffreestanding -mcmodel=kernel -target-cpu x86-64 -target-feature +retpoline-indirect-calls -target-feature +retpoline-indirect-branches -target-feature -sse2 -target-feature -sse -target-feature -3dnow -target-feature -mmx -target-feature +save-args -disable-red-zone -no-implicit-float -tune-cpu generic -debugger-tuning=gdb -fcoverage-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -nostdsysteminc -nobuiltininc -resource-dir /usr/local/lib/clang/13.0.0 -I /usr/src/sys -I /usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -I /usr/src/sys/arch -I /usr/src/sys/dev/pci/drm/include -I /usr/src/sys/dev/pci/drm/include/uapi -I /usr/src/sys/dev/pci/drm/amd/include/asic_reg -I /usr/src/sys/dev/pci/drm/amd/include -I /usr/src/sys/dev/pci/drm/amd/amdgpu -I /usr/src/sys/dev/pci/drm/amd/display -I /usr/src/sys/dev/pci/drm/amd/display/include -I /usr/src/sys/dev/pci/drm/amd/display/dc -I /usr/src/sys/dev/pci/drm/amd/display/amdgpu_dm -I /usr/src/sys/dev/pci/drm/amd/pm/inc -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu11 -I /usr/src/sys/dev/pci/drm/amd/pm/swsmu/smu12 -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/hwmgr -I /usr/src/sys/dev/pci/drm/amd/pm/powerplay/smumgr -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc -I /usr/src/sys/dev/pci/drm/amd/display/dc/inc/hw -I /usr/src/sys/dev/pci/drm/amd/display/dc/clk_mgr -I /usr/src/sys/dev/pci/drm/amd/display/modules/inc -I /usr/src/sys/dev/pci/drm/amd/display/modules/hdcp -I /usr/src/sys/dev/pci/drm/amd/display/dmub/inc -I /usr/src/sys/dev/pci/drm/i915 -D DDB -D DIAGNOSTIC -D KTRACE -D ACCOUNTING -D KMEMSTATS -D PTRACE -D POOL_DEBUG -D CRYPTO -D SYSVMSG -D SYSVSEM -D SYSVSHM -D UVM_SWAP_ENCRYPT -D FFS -D FFS2 -D FFS_SOFTUPDATES -D UFS_DIRHASH -D QUOTA -D EXT2FS -D MFS -D NFSCLIENT -D NFSSERVER -D CD9660 -D UDF -D MSDOSFS -D FIFO -D FUSE -D SOCKET_SPLICE -D TCP_ECN -D TCP_SIGNATURE -D INET6 -D IPSEC -D PPP_BSDCOMP -D PPP_DEFLATE -D PIPEX -D MROUTING -D MPLS -D BOOT_CONFIG -D USER_PCICONF -D APERTURE -D MTRR -D NTFS -D HIBERNATE -D PCIVERBOSE -D USBVERBOSE -D WSDISPLAY_COMPAT_USL -D WSDISPLAY_COMPAT_RAWKBD -D WSDISPLAY_DEFAULTSCREENS=6 -D X86EMU -D ONEWIREVERBOSE -D MULTIPROCESSOR -D MAXUSERS=80 -D _KERNEL -D CONFIG_DRM_AMD_DC_DCN3_0 -O2 -Wno-pointer-sign -Wno-address-of-packed-member -Wno-constant-conversion -Wno-unused-but-set-variable -Wno-gnu-folding-constant -fdebug-compilation-dir=/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj -ferror-limit 19 -fwrapv -D_RET_PROTECTOR -ret-protector -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-valloc -fno-builtin-free -fno-builtin-strdup -fno-builtin-strndup -analyzer-output=html -faddrsig -o /usr/obj/sys/arch/amd64/compile/GENERIC.MP/scan-build/2022-01-12-131800-47421-1 -x c /usr/src/sys/uvm/uvm_page.c

/usr/src/sys/uvm/uvm_page.c

1/* $OpenBSD: uvm_page.c,v 1.160 2021/12/15 12:53:53 mpi Exp $ */
2/* $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $ */
3
4/*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * Copyright (c) 1991, 1993, The Regents of the University of California.
7 *
8 * All rights reserved.
9 *
10 * This code is derived from software contributed to Berkeley by
11 * The Mach Operating System project at Carnegie-Mellon University.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 * 3. Neither the name of the University nor the names of its contributors
22 * may be used to endorse or promote products derived from this software
23 * without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 *
37 * @(#)vm_page.c 8.3 (Berkeley) 3/21/94
38 * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp
39 *
40 *
41 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
42 * All rights reserved.
43 *
44 * Permission to use, copy, modify and distribute this software and
45 * its documentation is hereby granted, provided that both the copyright
46 * notice and this permission notice appear in all copies of the
47 * software, derivative works or modified versions, and any portions
48 * thereof, and that both notices appear in supporting documentation.
49 *
50 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
51 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
52 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
53 *
54 * Carnegie Mellon requests users of this software to return to
55 *
56 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
57 * School of Computer Science
58 * Carnegie Mellon University
59 * Pittsburgh PA 15213-3890
60 *
61 * any improvements or extensions that they make and grant Carnegie the
62 * rights to redistribute these changes.
63 */
64
65/*
66 * uvm_page.c: page ops.
67 */
68
69#include <sys/param.h>
70#include <sys/systm.h>
71#include <sys/sched.h>
72#include <sys/vnode.h>
73#include <sys/mount.h>
74#include <sys/proc.h>
75#include <sys/smr.h>
76
77#include <uvm/uvm.h>
78
79/*
80 * for object trees
81 */
82RBT_GENERATE(uvm_objtree, vm_page, objt, uvm_pagecmp)static int uvm_objtree_RBT_COMPARE(const void *lptr, const void
*rptr) { const struct vm_page *l = lptr, *r = rptr; return uvm_pagecmp
(l, r); } static const struct rb_type uvm_objtree_RBT_INFO = {
uvm_objtree_RBT_COMPARE, ((void *)0), __builtin_offsetof(struct
vm_page, objt), }; const struct rb_type *const uvm_objtree_RBT_TYPE
= &uvm_objtree_RBT_INFO
;
83
84int
85uvm_pagecmp(const struct vm_page *a, const struct vm_page *b)
86{
87 return a->offset < b->offset ? -1 : a->offset > b->offset;
88}
89
90/*
91 * global vars... XXXCDC: move to uvm. structure.
92 */
93/*
94 * physical memory config is stored in vm_physmem.
95 */
96struct vm_physseg vm_physmem[VM_PHYSSEG_MAX16]; /* XXXCDC: uvm.physmem */
97int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */
98
99/*
100 * Some supported CPUs in a given architecture don't support all
101 * of the things necessary to do idle page zero'ing efficiently.
102 * We therefore provide a way to disable it from machdep code here.
103 */
104
105/*
106 * local variables
107 */
108/*
109 * these variables record the values returned by vm_page_bootstrap,
110 * for debugging purposes. The implementation of uvm_pageboot_alloc
111 * and pmap_startup here also uses them internally.
112 */
113static vaddr_t virtual_space_start;
114static vaddr_t virtual_space_end;
115
116/*
117 * local prototypes
118 */
119static void uvm_pageinsert(struct vm_page *);
120static void uvm_pageremove(struct vm_page *);
121int uvm_page_owner_locked_p(struct vm_page *);
122
123/*
124 * inline functions
125 */
126/*
127 * uvm_pageinsert: insert a page in the object
128 *
129 * => caller must lock object
130 * => call should have already set pg's object and offset pointers
131 * and bumped the version counter
132 */
133inline static void
134uvm_pageinsert(struct vm_page *pg)
135{
136 struct vm_page *dupe;
137
138 KASSERT(UVM_OBJ_IS_DUMMY(pg->uobject) ||(((((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)) || rw_write_held(
pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 139, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
139 rw_write_held(pg->uobject->vmobjlock))(((((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)) || rw_write_held(
pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 139, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
;
140 KASSERT((pg->pg_flags & PG_TABLED) == 0)(((pg->pg_flags & 0x00000004) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 140, "(pg->pg_flags & PG_TABLED) == 0"
))
;
141
142 dupe = RBT_INSERT(uvm_objtree, &pg->uobject->memt, pg)uvm_objtree_RBT_INSERT(&pg->uobject->memt, pg);
143 /* not allowed to insert over another page */
144 KASSERT(dupe == NULL)((dupe == ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 144, "dupe == NULL"))
;
145 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_TABLED0x00000004);
146 pg->uobject->uo_npages++;
147}
148
149/*
150 * uvm_page_remove: remove page from object
151 *
152 * => caller must lock object
153 */
154static inline void
155uvm_pageremove(struct vm_page *pg)
156{
157 KASSERT(UVM_OBJ_IS_DUMMY(pg->uobject) ||(((((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)) || rw_write_held(
pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 158, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
43
Access to field 'pgops' results in a dereference of a null pointer (loaded from field 'uobject')
158 rw_write_held(pg->uobject->vmobjlock))(((((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager)) || rw_write_held(
pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 158, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
;
159 KASSERT(pg->pg_flags & PG_TABLED)((pg->pg_flags & 0x00000004) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 159, "pg->pg_flags & PG_TABLED"
))
;
160
161 RBT_REMOVE(uvm_objtree, &pg->uobject->memt, pg)uvm_objtree_RBT_REMOVE(&pg->uobject->memt, pg);
162
163 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_TABLED0x00000004);
164 pg->uobject->uo_npages--;
165 pg->uobject = NULL((void *)0);
166 pg->pg_version++;
167}
168
169/*
170 * uvm_page_init: init the page system. called from uvm_init().
171 *
172 * => we return the range of kernel virtual memory in kvm_startp/kvm_endp
173 */
174void
175uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp)
176{
177 vsize_t freepages, pagecount, n;
178 vm_page_t pagearray, curpg;
179 int lcv, i;
180 paddr_t paddr, pgno;
181 struct vm_physseg *seg;
182
183 /*
184 * init the page queues and page queue locks
185 */
186
187 TAILQ_INIT(&uvm.page_active)do { (&uvm.page_active)->tqh_first = ((void *)0); (&
uvm.page_active)->tqh_last = &(&uvm.page_active)->
tqh_first; } while (0)
;
188 TAILQ_INIT(&uvm.page_inactive_swp)do { (&uvm.page_inactive_swp)->tqh_first = ((void *)0)
; (&uvm.page_inactive_swp)->tqh_last = &(&uvm.
page_inactive_swp)->tqh_first; } while (0)
;
189 TAILQ_INIT(&uvm.page_inactive_obj)do { (&uvm.page_inactive_obj)->tqh_first = ((void *)0)
; (&uvm.page_inactive_obj)->tqh_last = &(&uvm.
page_inactive_obj)->tqh_first; } while (0)
;
190 mtx_init(&uvm.pageqlock, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.pageqlock
), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((
0xa)))); } while (0)
;
191 mtx_init(&uvm.fpageqlock, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.fpageqlock
), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((
0xa)))); } while (0)
;
192 uvm_pmr_init();
193
194 /*
195 * allocate vm_page structures.
196 */
197
198 /*
199 * sanity check:
200 * before calling this function the MD code is expected to register
201 * some free RAM with the uvm_page_physload() function. our job
202 * now is to allocate vm_page structures for this memory.
203 */
204
205 if (vm_nphysseg == 0)
206 panic("uvm_page_bootstrap: no memory pre-allocated");
207
208 /*
209 * first calculate the number of free pages...
210 *
211 * note that we use start/end rather than avail_start/avail_end.
212 * this allows us to allocate extra vm_page structures in case we
213 * want to return some memory to the pool after booting.
214 */
215
216 freepages = 0;
217 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++)
218 freepages += (seg->end - seg->start);
219
220 /*
221 * we now know we have (PAGE_SIZE * freepages) bytes of memory we can
222 * use. for each page of memory we use we need a vm_page structure.
223 * thus, the total number of pages we can use is the total size of
224 * the memory divided by the PAGE_SIZE plus the size of the vm_page
225 * structure. we add one to freepages as a fudge factor to avoid
226 * truncation errors (since we can only allocate in terms of whole
227 * pages).
228 */
229
230 pagecount = (((paddr_t)freepages + 1) << PAGE_SHIFT12) /
231 (PAGE_SIZE(1 << 12) + sizeof(struct vm_page));
232 pagearray = (vm_page_t)uvm_pageboot_alloc(pagecount *
233 sizeof(struct vm_page));
234 memset(pagearray, 0, pagecount * sizeof(struct vm_page))__builtin_memset((pagearray), (0), (pagecount * sizeof(struct
vm_page)))
;
235
236 /* init the vm_page structures and put them in the correct place. */
237 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) {
238 n = seg->end - seg->start;
239 if (n > pagecount) {
240 panic("uvm_page_init: lost %ld page(s) in init",
241 (long)(n - pagecount));
242 /* XXXCDC: shouldn't happen? */
243 /* n = pagecount; */
244 }
245
246 /* set up page array pointers */
247 seg->pgs = pagearray;
248 pagearray += n;
249 pagecount -= n;
250 seg->lastpg = seg->pgs + (n - 1);
251
252 /* init and free vm_pages (we've already zeroed them) */
253 pgno = seg->start;
254 paddr = ptoa(pgno)((paddr_t)(pgno) << 12);
255 for (i = 0, curpg = seg->pgs; i < n;
256 i++, curpg++, pgno++, paddr += PAGE_SIZE(1 << 12)) {
257 curpg->phys_addr = paddr;
258 VM_MDPAGE_INIT(curpg)do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(curpg
)->mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) <
0x9) ? 0x9 : ((0xa)))); } while (0); (curpg)->mdpage.pv_list
= ((void *)0); } while (0)
;
259 if (pgno >= seg->avail_start &&
260 pgno < seg->avail_end) {
261 uvmexp.npages++;
262 }
263 }
264
265 /* Add pages to free pool. */
266 uvm_pmr_freepages(&seg->pgs[seg->avail_start - seg->start],
267 seg->avail_end - seg->avail_start);
268 }
269
270 /*
271 * pass up the values of virtual_space_start and
272 * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper
273 * layers of the VM.
274 */
275
276 *kvm_startp = round_page(virtual_space_start)(((virtual_space_start) + ((1 << 12) - 1)) & ~((1 <<
12) - 1))
;
277 *kvm_endp = trunc_page(virtual_space_end)((virtual_space_end) & ~((1 << 12) - 1));
278
279 /* init locks for kernel threads */
280 mtx_init(&uvm.aiodoned_lock, IPL_BIO)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.aiodoned_lock
), ((((0x6)) > 0x0 && ((0x6)) < 0x9) ? 0x9 : ((
0x6)))); } while (0)
;
281
282 /*
283 * init reserve thresholds
284 * XXXCDC - values may need adjusting
285 */
286 uvmexp.reserve_pagedaemon = 4;
287 uvmexp.reserve_kernel = 8;
288 uvmexp.anonminpct = 10;
289 uvmexp.vnodeminpct = 10;
290 uvmexp.vtextminpct = 5;
291 uvmexp.anonmin = uvmexp.anonminpct * 256 / 100;
292 uvmexp.vnodemin = uvmexp.vnodeminpct * 256 / 100;
293 uvmexp.vtextmin = uvmexp.vtextminpct * 256 / 100;
294
295 uvm.page_init_done = TRUE1;
296}
297
298/*
299 * uvm_setpagesize: set the page size
300 *
301 * => sets page_shift and page_mask from uvmexp.pagesize.
302 */
303void
304uvm_setpagesize(void)
305{
306 if (uvmexp.pagesize == 0)
307 uvmexp.pagesize = DEFAULT_PAGE_SIZE4096;
308 uvmexp.pagemask = uvmexp.pagesize - 1;
309 if ((uvmexp.pagemask & uvmexp.pagesize) != 0)
310 panic("uvm_setpagesize: page size not a power of two");
311 for (uvmexp.pageshift = 0; ; uvmexp.pageshift++)
312 if ((1 << uvmexp.pageshift) == uvmexp.pagesize)
313 break;
314}
315
316/*
317 * uvm_pageboot_alloc: steal memory from physmem for bootstrapping
318 */
319vaddr_t
320uvm_pageboot_alloc(vsize_t size)
321{
322#if defined(PMAP_STEAL_MEMORY)
323 vaddr_t addr;
324
325 /*
326 * defer bootstrap allocation to MD code (it may want to allocate
327 * from a direct-mapped segment). pmap_steal_memory should round
328 * off virtual_space_start/virtual_space_end.
329 */
330
331 addr = pmap_steal_memory(size, &virtual_space_start,
332 &virtual_space_end);
333
334 return addr;
335
336#else /* !PMAP_STEAL_MEMORY */
337
338 static boolean_t initialized = FALSE0;
339 vaddr_t addr, vaddr;
340 paddr_t paddr;
341
342 /* round to page size */
343 size = round_page(size)(((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1
))
;
344
345 /* on first call to this function, initialize ourselves. */
346 if (initialized == FALSE0) {
347 pmap_virtual_space(&virtual_space_start, &virtual_space_end);
348
349 /* round it the way we like it */
350 virtual_space_start = round_page(virtual_space_start)(((virtual_space_start) + ((1 << 12) - 1)) & ~((1 <<
12) - 1))
;
351 virtual_space_end = trunc_page(virtual_space_end)((virtual_space_end) & ~((1 << 12) - 1));
352
353 initialized = TRUE1;
354 }
355
356 /* allocate virtual memory for this request */
357 if (virtual_space_start == virtual_space_end ||
358 (virtual_space_end - virtual_space_start) < size)
359 panic("uvm_pageboot_alloc: out of virtual space");
360
361 addr = virtual_space_start;
362
363#ifdef PMAP_GROWKERNEL
364 /*
365 * If the kernel pmap can't map the requested space,
366 * then allocate more resources for it.
367 */
368 if (uvm_maxkaddr < (addr + size)) {
369 uvm_maxkaddr = pmap_growkernel(addr + size);
370 if (uvm_maxkaddr < (addr + size))
371 panic("uvm_pageboot_alloc: pmap_growkernel() failed");
372 }
373#endif
374
375 virtual_space_start += size;
376
377 /* allocate and mapin physical pages to back new virtual pages */
378 for (vaddr = round_page(addr)(((addr) + ((1 << 12) - 1)) & ~((1 << 12) - 1
))
; vaddr < addr + size ;
379 vaddr += PAGE_SIZE(1 << 12)) {
380 if (!uvm_page_physget(&paddr))
381 panic("uvm_pageboot_alloc: out of memory");
382
383 /*
384 * Note this memory is no longer managed, so using
385 * pmap_kenter is safe.
386 */
387 pmap_kenter_pa(vaddr, paddr, PROT_READ0x01 | PROT_WRITE0x02);
388 }
389 pmap_update(pmap_kernel());
390 return addr;
391#endif /* PMAP_STEAL_MEMORY */
392}
393
394#if !defined(PMAP_STEAL_MEMORY)
395/*
396 * uvm_page_physget: "steal" one page from the vm_physmem structure.
397 *
398 * => attempt to allocate it off the end of a segment in which the "avail"
399 * values match the start/end values. if we can't do that, then we
400 * will advance both values (making them equal, and removing some
401 * vm_page structures from the non-avail area).
402 * => return false if out of memory.
403 */
404
405boolean_t
406uvm_page_physget(paddr_t *paddrp)
407{
408 int lcv;
409 struct vm_physseg *seg;
410
411 /* pass 1: try allocating from a matching end */
412#if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) || \
413 (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2)
414 for (lcv = vm_nphysseg - 1, seg = vm_physmem + lcv; lcv >= 0;
415 lcv--, seg--)
416#else
417 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++)
418#endif
419 {
420 if (uvm.page_init_done == TRUE1)
421 panic("uvm_page_physget: called _after_ bootstrap");
422
423 /* try from front */
424 if (seg->avail_start == seg->start &&
425 seg->avail_start < seg->avail_end) {
426 *paddrp = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12);
427 seg->avail_start++;
428 seg->start++;
429 /* nothing left? nuke it */
430 if (seg->avail_start == seg->end) {
431 if (vm_nphysseg == 1)
432 panic("uvm_page_physget: out of memory!");
433 vm_nphysseg--;
434 for (; lcv < vm_nphysseg; lcv++, seg++)
435 /* structure copy */
436 seg[0] = seg[1];
437 }
438 return TRUE1;
439 }
440
441 /* try from rear */
442 if (seg->avail_end == seg->end &&
443 seg->avail_start < seg->avail_end) {
444 *paddrp = ptoa(seg->avail_end - 1)((paddr_t)(seg->avail_end - 1) << 12);
445 seg->avail_end--;
446 seg->end--;
447 /* nothing left? nuke it */
448 if (seg->avail_end == seg->start) {
449 if (vm_nphysseg == 1)
450 panic("uvm_page_physget: out of memory!");
451 vm_nphysseg--;
452 for (; lcv < vm_nphysseg ; lcv++, seg++)
453 /* structure copy */
454 seg[0] = seg[1];
455 }
456 return TRUE1;
457 }
458 }
459
460 /* pass2: forget about matching ends, just allocate something */
461#if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) || \
462 (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2)
463 for (lcv = vm_nphysseg - 1, seg = vm_physmem + lcv; lcv >= 0;
464 lcv--, seg--)
465#else
466 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++)
467#endif
468 {
469
470 /* any room in this bank? */
471 if (seg->avail_start >= seg->avail_end)
472 continue; /* nope */
473
474 *paddrp = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12);
475 seg->avail_start++;
476 /* truncate! */
477 seg->start = seg->avail_start;
478
479 /* nothing left? nuke it */
480 if (seg->avail_start == seg->end) {
481 if (vm_nphysseg == 1)
482 panic("uvm_page_physget: out of memory!");
483 vm_nphysseg--;
484 for (; lcv < vm_nphysseg ; lcv++, seg++)
485 /* structure copy */
486 seg[0] = seg[1];
487 }
488 return TRUE1;
489 }
490
491 return FALSE0; /* whoops! */
492}
493
494#endif /* PMAP_STEAL_MEMORY */
495
496/*
497 * uvm_page_physload: load physical memory into VM system
498 *
499 * => all args are PFs
500 * => all pages in start/end get vm_page structures
501 * => areas marked by avail_start/avail_end get added to the free page pool
502 * => we are limited to VM_PHYSSEG_MAX physical memory segments
503 */
504
505void
506uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start,
507 paddr_t avail_end, int flags)
508{
509 int preload, lcv;
510 psize_t npages;
511 struct vm_page *pgs;
512 struct vm_physseg *ps, *seg;
513
514#ifdef DIAGNOSTIC1
515 if (uvmexp.pagesize == 0)
516 panic("uvm_page_physload: page size not set!");
517
518 if (start >= end)
519 panic("uvm_page_physload: start >= end");
520#endif
521
522 /* do we have room? */
523 if (vm_nphysseg == VM_PHYSSEG_MAX16) {
524 printf("uvm_page_physload: unable to load physical memory "
525 "segment\n");
526 printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n",
527 VM_PHYSSEG_MAX16, (long long)start, (long long)end);
528 printf("\tincrease VM_PHYSSEG_MAX\n");
529 return;
530 }
531
532 /*
533 * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been
534 * called yet, so malloc is not available).
535 */
536 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++) {
537 if (seg->pgs)
538 break;
539 }
540 preload = (lcv == vm_nphysseg);
541
542 /* if VM is already running, attempt to malloc() vm_page structures */
543 if (!preload) {
544 /*
545 * XXXCDC: need some sort of lockout for this case
546 * right now it is only used by devices so it should be alright.
547 */
548 paddr_t paddr;
549
550 npages = end - start; /* # of pages */
551
552 pgs = km_alloc(round_page(npages * sizeof(*pgs))(((npages * sizeof(*pgs)) + ((1 << 12) - 1)) & ~((1
<< 12) - 1))
,
553 &kv_any, &kp_zero, &kd_waitok);
554 if (pgs == NULL((void *)0)) {
555 printf("uvm_page_physload: can not malloc vm_page "
556 "structs for segment\n");
557 printf("\tignoring 0x%lx -> 0x%lx\n", start, end);
558 return;
559 }
560 /* init phys_addr and free pages, XXX uvmexp.npages */
561 for (lcv = 0, paddr = ptoa(start)((paddr_t)(start) << 12); lcv < npages;
562 lcv++, paddr += PAGE_SIZE(1 << 12)) {
563 pgs[lcv].phys_addr = paddr;
564 VM_MDPAGE_INIT(&pgs[lcv])do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(&
pgs[lcv])->mdpage.pv_mtx), ((((0xa)) > 0x0 && (
(0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); (&pgs[lcv
])->mdpage.pv_list = ((void *)0); } while (0)
;
565 if (atop(paddr)((paddr) >> 12) >= avail_start &&
566 atop(paddr)((paddr) >> 12) < avail_end) {
567 if (flags & PHYSLOAD_DEVICE0x01) {
568 atomic_setbits_intx86_atomic_setbits_u32(&pgs[lcv].pg_flags,
569 PG_DEV0x00000200);
570 pgs[lcv].wire_count = 1;
571 } else {
572#if defined(VM_PHYSSEG_NOADD)
573 panic("uvm_page_physload: tried to add RAM after vm_mem_init");
574#endif
575 }
576 }
577 }
578
579 /* Add pages to free pool. */
580 if ((flags & PHYSLOAD_DEVICE0x01) == 0) {
581 uvm_pmr_freepages(&pgs[avail_start - start],
582 avail_end - avail_start);
583 }
584
585 /* XXXCDC: need hook to tell pmap to rebuild pv_list, etc... */
586 } else {
587 /* gcc complains if these don't get init'd */
588 pgs = NULL((void *)0);
589 npages = 0;
590
591 }
592
593 /* now insert us in the proper place in vm_physmem[] */
594#if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_RANDOM1)
595 /* random: put it at the end (easy!) */
596 ps = &vm_physmem[vm_nphysseg];
597#elif (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2)
598 {
599 int x;
600 /* sort by address for binary search */
601 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++)
602 if (start < seg->start)
603 break;
604 ps = seg;
605 /* move back other entries, if necessary ... */
606 for (x = vm_nphysseg, seg = vm_physmem + x - 1; x > lcv;
607 x--, seg--)
608 /* structure copy */
609 seg[1] = seg[0];
610 }
611#elif (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3)
612 {
613 int x;
614 /* sort by largest segment first */
615 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++)
616 if ((end - start) >
617 (seg->end - seg->start))
618 break;
619 ps = &vm_physmem[lcv];
620 /* move back other entries, if necessary ... */
621 for (x = vm_nphysseg, seg = vm_physmem + x - 1; x > lcv;
622 x--, seg--)
623 /* structure copy */
624 seg[1] = seg[0];
625 }
626#else
627 panic("uvm_page_physload: unknown physseg strategy selected!");
628#endif
629
630 ps->start = start;
631 ps->end = end;
632 ps->avail_start = avail_start;
633 ps->avail_end = avail_end;
634 if (preload) {
635 ps->pgs = NULL((void *)0);
636 } else {
637 ps->pgs = pgs;
638 ps->lastpg = pgs + npages - 1;
639 }
640 vm_nphysseg++;
641
642 return;
643}
644
645#ifdef DDB1 /* XXXCDC: TMP TMP TMP DEBUG DEBUG DEBUG */
646
647void uvm_page_physdump(void); /* SHUT UP GCC */
648
649/* call from DDB */
650void
651uvm_page_physdump(void)
652{
653 int lcv;
654 struct vm_physseg *seg;
655
656 printf("uvm_page_physdump: physical memory config [segs=%d of %d]:\n",
657 vm_nphysseg, VM_PHYSSEG_MAX16);
658 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++)
659 printf("0x%llx->0x%llx [0x%llx->0x%llx]\n",
660 (long long)seg->start,
661 (long long)seg->end,
662 (long long)seg->avail_start,
663 (long long)seg->avail_end);
664 printf("STRATEGY = ");
665 switch (VM_PHYSSEG_STRAT3) {
666 case VM_PSTRAT_RANDOM1: printf("RANDOM\n"); break;
667 case VM_PSTRAT_BSEARCH2: printf("BSEARCH\n"); break;
668 case VM_PSTRAT_BIGFIRST3: printf("BIGFIRST\n"); break;
669 default: printf("<<UNKNOWN>>!!!!\n");
670 }
671}
672#endif
673
674void
675uvm_shutdown(void)
676{
677#ifdef UVM_SWAP_ENCRYPT1
678 uvm_swap_finicrypt_all();
679#endif
680 smr_flush()smr_barrier_impl(1);
681}
682
683/*
684 * Perform insert of a given page in the specified anon of obj.
685 * This is basically, uvm_pagealloc, but with the page already given.
686 */
687void
688uvm_pagealloc_pg(struct vm_page *pg, struct uvm_object *obj, voff_t off,
689 struct vm_anon *anon)
690{
691 int flags;
692
693 KASSERT(obj == NULL || anon == NULL)((obj == ((void *)0) || anon == ((void *)0)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 693, "obj == NULL || anon == NULL"
))
;
694 KASSERT(anon == NULL || off == 0)((anon == ((void *)0) || off == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 694, "anon == NULL || off == 0"
))
;
695 KASSERT(off == trunc_page(off))((off == ((off) & ~((1 << 12) - 1))) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 695, "off == trunc_page(off)"
))
;
696 KASSERT(obj == NULL || UVM_OBJ_IS_DUMMY(obj) ||((obj == ((void *)0) || (((obj)->pgops == &pmap_pager)
|| ((obj)->pgops == &bufcache_pager)) || rw_write_held
(obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 697, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)"
))
697 rw_write_held(obj->vmobjlock))((obj == ((void *)0) || (((obj)->pgops == &pmap_pager)
|| ((obj)->pgops == &bufcache_pager)) || rw_write_held
(obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 697, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)"
))
;
698 KASSERT(anon == NULL || anon->an_lock == NULL ||((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held
(anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 699, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)"
))
699 rw_write_held(anon->an_lock))((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held
(anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 699, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)"
))
;
700
701 flags = PG_BUSY0x00000001 | PG_FAKE0x00000040;
702 pg->offset = off;
703 pg->uobject = obj;
704 pg->uanon = anon;
705 KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 705, "uvm_page_owner_locked_p(pg)"
))
;
706 if (anon) {
707 anon->an_page = pg;
708 flags |= PQ_ANON0x00100000;
709 } else if (obj)
710 uvm_pageinsert(pg);
711 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, flags);
712#if defined(UVM_PAGE_TRKOWN)
713 pg->owner_tag = NULL((void *)0);
714#endif
715 UVM_PAGE_OWN(pg, "new alloc");
716}
717
718/*
719 * uvm_pglistalloc: allocate a list of pages
720 *
721 * => allocated pages are placed at the tail of rlist. rlist is
722 * assumed to be properly initialized by caller.
723 * => returns 0 on success or errno on failure
724 * => doesn't take into account clean non-busy pages on inactive list
725 * that could be used(?)
726 * => params:
727 * size the size of the allocation, rounded to page size.
728 * low the low address of the allowed allocation range.
729 * high the high address of the allowed allocation range.
730 * alignment memory must be aligned to this power-of-two boundary.
731 * boundary no segment in the allocation may cross this
732 * power-of-two boundary (relative to zero).
733 * => flags:
734 * UVM_PLA_NOWAIT fail if allocation fails
735 * UVM_PLA_WAITOK wait for memory to become avail
736 * UVM_PLA_ZERO return zeroed memory
737 */
738int
739uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment,
740 paddr_t boundary, struct pglist *rlist, int nsegs, int flags)
741{
742 KASSERT((alignment & (alignment - 1)) == 0)(((alignment & (alignment - 1)) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 742, "(alignment & (alignment - 1)) == 0"
))
;
743 KASSERT((boundary & (boundary - 1)) == 0)(((boundary & (boundary - 1)) == 0) ? (void)0 : __assert(
"diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 743, "(boundary & (boundary - 1)) == 0"
))
;
744 KASSERT(!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT))((!(flags & 0x0001) ^ !(flags & 0x0002)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 744, "!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT)"
))
;
745
746 if (size == 0)
747 return EINVAL22;
748 size = atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) -
1))) >> 12)
;
749
750 /*
751 * XXX uvm_pglistalloc is currently only used for kernel
752 * objects. Unlike the checks in uvm_pagealloc, below, here
753 * we are always allowed to use the kernel reserve.
754 */
755 flags |= UVM_PLA_USERESERVE0x0040;
756
757 if ((high & PAGE_MASK((1 << 12) - 1)) != PAGE_MASK((1 << 12) - 1)) {
758 printf("uvm_pglistalloc: Upper boundary 0x%lx "
759 "not on pagemask.\n", (unsigned long)high);
760 }
761
762 /*
763 * Our allocations are always page granularity, so our alignment
764 * must be, too.
765 */
766 if (alignment < PAGE_SIZE(1 << 12))
767 alignment = PAGE_SIZE(1 << 12);
768
769 low = atop(roundup(low, alignment))((((((low)+((alignment)-1))/(alignment))*(alignment))) >>
12)
;
770 /*
771 * high + 1 may result in overflow, in which case high becomes 0x0,
772 * which is the 'don't care' value.
773 * The only requirement in that case is that low is also 0x0, or the
774 * low<high assert will fail.
775 */
776 high = atop(high + 1)((high + 1) >> 12);
777 alignment = atop(alignment)((alignment) >> 12);
778 if (boundary < PAGE_SIZE(1 << 12) && boundary != 0)
779 boundary = PAGE_SIZE(1 << 12);
780 boundary = atop(boundary)((boundary) >> 12);
781
782 return uvm_pmr_getpages(size, low, high, alignment, boundary, nsegs,
783 flags, rlist);
784}
785
786/*
787 * uvm_pglistfree: free a list of pages
788 *
789 * => pages should already be unmapped
790 */
791void
792uvm_pglistfree(struct pglist *list)
793{
794 uvm_pmr_freepageq(list);
795}
796
797/*
798 * interface used by the buffer cache to allocate a buffer at a time.
799 * The pages are allocated wired in DMA accessible memory
800 */
801int
802uvm_pagealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size,
803 int flags)
804{
805 struct pglist plist;
806 struct vm_page *pg;
807 int i, r;
808
809 KASSERT(UVM_OBJ_IS_BUFCACHE(obj))((((obj)->pgops == &bufcache_pager)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 809, "UVM_OBJ_IS_BUFCACHE(obj)"
))
;
810 KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 810, "_kernel_lock_held()"))
;
811
812 TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)->
tqh_last = &(&plist)->tqh_first; } while (0)
;
813 r = uvm_pglistalloc(size, dma_constraint.ucr_low,
814 dma_constraint.ucr_high, 0, 0, &plist, atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) -
1))) >> 12)
,
815 flags);
816 if (r == 0) {
817 i = 0;
818 while ((pg = TAILQ_FIRST(&plist)((&plist)->tqh_first)) != NULL((void *)0)) {
819 pg->wire_count = 1;
820 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008 | PG_FAKE0x00000040);
821 KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 821, "(pg->pg_flags & PG_DEV) == 0"
))
;
822 TAILQ_REMOVE(&plist, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&plist)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->
pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->pageq.tqe_prev
) = ((void *)-1); ((pg)->pageq.tqe_next) = ((void *)-1); }
while (0)
;
823 uvm_pagealloc_pg(pg, obj, off + ptoa(i++)((paddr_t)(i++) << 12), NULL((void *)0));
824 }
825 }
826 return r;
827}
828
829/*
830 * interface used by the buffer cache to reallocate a buffer at a time.
831 * The pages are reallocated wired outside the DMA accessible region.
832 *
833 */
834int
835uvm_pagerealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size,
836 int flags, struct uvm_constraint_range *where)
837{
838 struct pglist plist;
839 struct vm_page *pg, *tpg;
840 int i, r;
841 voff_t offset;
842
843 KASSERT(UVM_OBJ_IS_BUFCACHE(obj))((((obj)->pgops == &bufcache_pager)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 843, "UVM_OBJ_IS_BUFCACHE(obj)"
))
;
844 KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 844, "_kernel_lock_held()"))
;
845
846 TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)->
tqh_last = &(&plist)->tqh_first; } while (0)
;
847 if (size == 0)
848 panic("size 0 uvm_pagerealloc");
849 r = uvm_pglistalloc(size, where->ucr_low, where->ucr_high, 0,
850 0, &plist, atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) -
1))) >> 12)
, flags);
851 if (r == 0) {
852 i = 0;
853 while((pg = TAILQ_FIRST(&plist)((&plist)->tqh_first)) != NULL((void *)0)) {
854 offset = off + ptoa(i++)((paddr_t)(i++) << 12);
855 tpg = uvm_pagelookup(obj, offset);
856 KASSERT(tpg != NULL)((tpg != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 856, "tpg != NULL"))
;
857 pg->wire_count = 1;
858 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008 | PG_FAKE0x00000040);
859 KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 859, "(pg->pg_flags & PG_DEV) == 0"
))
;
860 TAILQ_REMOVE(&plist, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&plist)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->
pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->pageq.tqe_prev
) = ((void *)-1); ((pg)->pageq.tqe_next) = ((void *)-1); }
while (0)
;
861 uvm_pagecopy(tpg, pg);
862 KASSERT(tpg->wire_count == 1)((tpg->wire_count == 1) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 862, "tpg->wire_count == 1"
))
;
863 tpg->wire_count = 0;
864 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
865 uvm_pagefree(tpg);
866 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
867 uvm_pagealloc_pg(pg, obj, offset, NULL((void *)0));
868 }
869 }
870 return r;
871}
872
873/*
874 * uvm_pagealloc: allocate vm_page from a particular free list.
875 *
876 * => return null if no pages free
877 * => wake up pagedaemon if number of free pages drops below low water mark
878 * => only one of obj or anon can be non-null
879 * => caller must activate/deactivate page if it is not wired.
880 */
881
882struct vm_page *
883uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon,
884 int flags)
885{
886 struct vm_page *pg;
887 struct pglist pgl;
888 int pmr_flags;
889
890 KASSERT(obj == NULL || anon == NULL)((obj == ((void *)0) || anon == ((void *)0)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 890, "obj == NULL || anon == NULL"
))
;
891 KASSERT(anon == NULL || off == 0)((anon == ((void *)0) || off == 0) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 891, "anon == NULL || off == 0"
))
;
892 KASSERT(off == trunc_page(off))((off == ((off) & ~((1 << 12) - 1))) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 892, "off == trunc_page(off)"
))
;
893 KASSERT(obj == NULL || UVM_OBJ_IS_DUMMY(obj) ||((obj == ((void *)0) || (((obj)->pgops == &pmap_pager)
|| ((obj)->pgops == &bufcache_pager)) || rw_write_held
(obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 894, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)"
))
894 rw_write_held(obj->vmobjlock))((obj == ((void *)0) || (((obj)->pgops == &pmap_pager)
|| ((obj)->pgops == &bufcache_pager)) || rw_write_held
(obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 894, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)"
))
;
895 KASSERT(anon == NULL || anon->an_lock == NULL ||((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held
(anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 896, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)"
))
896 rw_write_held(anon->an_lock))((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held
(anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 896, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)"
))
;
897
898 pmr_flags = UVM_PLA_NOWAIT0x0002;
899
900 /*
901 * We're allowed to use the kernel reserve if the page is
902 * being allocated to a kernel object.
903 */
904 if ((flags & UVM_PGA_USERESERVE0x0001) ||
905 (obj != NULL((void *)0) && UVM_OBJ_IS_KERN_OBJECT(obj)((obj)->uo_refs == (-2))))
906 pmr_flags |= UVM_PLA_USERESERVE0x0040;
907
908 if (flags & UVM_PGA_ZERO0x0002)
909 pmr_flags |= UVM_PLA_ZERO0x0004;
910 TAILQ_INIT(&pgl)do { (&pgl)->tqh_first = ((void *)0); (&pgl)->tqh_last
= &(&pgl)->tqh_first; } while (0)
;
911 if (uvm_pmr_getpages(1, 0, 0, 1, 0, 1, pmr_flags, &pgl) != 0)
912 goto fail;
913
914 pg = TAILQ_FIRST(&pgl)((&pgl)->tqh_first);
915 KASSERT(pg != NULL && TAILQ_NEXT(pg, pageq) == NULL)((pg != ((void *)0) && ((pg)->pageq.tqe_next) == (
(void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c"
, 915, "pg != NULL && TAILQ_NEXT(pg, pageq) == NULL")
)
;
916
917 uvm_pagealloc_pg(pg, obj, off, anon);
918 KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 918, "(pg->pg_flags & PG_DEV) == 0"
))
;
919 if (flags & UVM_PGA_ZERO0x0002)
920 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
921 else
922 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
923
924 return pg;
925
926fail:
927 return NULL((void *)0);
928}
929
930/*
931 * uvm_pagerealloc: reallocate a page from one object to another
932 */
933
934void
935uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff)
936{
937
938 /* remove it from the old object */
939 if (pg->uobject) {
940 uvm_pageremove(pg);
941 }
942
943 /* put it in the new object */
944 if (newobj) {
945 pg->uobject = newobj;
946 pg->offset = newoff;
947 pg->pg_version++;
948 uvm_pageinsert(pg);
949 }
950}
951
952/*
953 * uvm_pageclean: clean page
954 *
955 * => erase page's identity (i.e. remove from object)
956 * => caller must lock page queues if `pg' is managed
957 * => assumes all valid mappings of pg are gone
958 */
959void
960uvm_pageclean(struct vm_page *pg)
961{
962 u_int flags_to_clear = 0;
963
964 if ((pg->pg_flags & (PG_TABLED0x00000004|PQ_ACTIVE0x00040000|PQ_INACTIVE0x00020000)) &&
965 (pg->uobject
34.1
Field 'uobject' is equal to NULL
34.1
Field 'uobject' is equal to NULL
== NULL((void *)0) || !UVM_OBJ_IS_PMAP(pg->uobject)((pg->uobject)->pgops == &pmap_pager)))
966 MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0)
;
35
Assuming the condition is false
36
Loop condition is false. Exiting loop
967
968#ifdef DEBUG
969 if (pg->uobject == (void *)0xdeadbeef &&
970 pg->uanon == (void *)0xdeadbeef) {
971 panic("uvm_pagefree: freeing free page %p", pg);
972 }
973#endif
974
975 KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 975, "(pg->pg_flags & PG_DEV) == 0"
))
;
37
Assuming the condition is true
38
'?' condition is true
976 KASSERT(pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) ||((pg->uobject == ((void *)0) || (((pg->uobject)->pgops
== &pmap_pager) || ((pg->uobject)->pgops == &bufcache_pager
)) || rw_write_held(pg->uobject->vmobjlock)) ? (void)0 :
__assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 977, "pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
977 rw_write_held(pg->uobject->vmobjlock))((pg->uobject == ((void *)0) || (((pg->uobject)->pgops
== &pmap_pager) || ((pg->uobject)->pgops == &bufcache_pager
)) || rw_write_held(pg->uobject->vmobjlock)) ? (void)0 :
__assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 977, "pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)"
))
;
978 KASSERT(pg->uobject != NULL || pg->uanon == NULL ||((pg->uobject != ((void *)0) || pg->uanon == ((void *)0
) || rw_write_held(pg->uanon->an_lock)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 979, "pg->uobject != NULL || pg->uanon == NULL || rw_write_held(pg->uanon->an_lock)"
))
39
Assuming field 'uanon' is equal to null
979 rw_write_held(pg->uanon->an_lock))((pg->uobject != ((void *)0) || pg->uanon == ((void *)0
) || rw_write_held(pg->uanon->an_lock)) ? (void)0 : __assert
("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 979, "pg->uobject != NULL || pg->uanon == NULL || rw_write_held(pg->uanon->an_lock)"
))
;
980
981 /*
982 * if the page was an object page (and thus "TABLED"), remove it
983 * from the object.
984 */
985 if (pg->pg_flags & PG_TABLED0x00000004)
40
Assuming the condition is true
41
Taking true branch
986 uvm_pageremove(pg);
42
Calling 'uvm_pageremove'
987
988 /* now remove the page from the queues */
989 if (pg->pg_flags & PQ_ACTIVE0x00040000) {
990 TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev;
*(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
991 flags_to_clear |= PQ_ACTIVE0x00040000;
992 uvmexp.active--;
993 }
994 if (pg->pg_flags & PQ_INACTIVE0x00020000) {
995 if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000))
996 TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
997 else
998 TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
999 flags_to_clear |= PQ_INACTIVE0x00020000;
1000 uvmexp.inactive--;
1001 }
1002
1003 /* if the page was wired, unwire it now. */
1004 if (pg->wire_count) {
1005 pg->wire_count = 0;
1006 uvmexp.wired--;
1007 }
1008 if (pg->uanon) {
1009 pg->uanon->an_page = NULL((void *)0);
1010 pg->uanon = NULL((void *)0);
1011 }
1012
1013 /* Clean page state bits. */
1014 flags_to_clear |= PQ_ANON0x00100000|PQ_AOBJ0x00200000|PQ_ENCRYPT0x00400000|PG_ZERO0x00000100|PG_FAKE0x00000040|PG_BUSY0x00000001|
1015 PG_RELEASED0x00000020|PG_CLEAN0x00000008|PG_CLEANCHK0x00000010;
1016 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, flags_to_clear);
1017
1018#ifdef DEBUG
1019 pg->uobject = (void *)0xdeadbeef;
1020 pg->offset = 0xdeadbeef;
1021 pg->uanon = (void *)0xdeadbeef;
1022#endif
1023}
1024
1025/*
1026 * uvm_pagefree: free page
1027 *
1028 * => erase page's identity (i.e. remove from object)
1029 * => put page on free list
1030 * => caller must lock page queues if `pg' is managed
1031 * => assumes all valid mappings of pg are gone
1032 */
1033void
1034uvm_pagefree(struct vm_page *pg)
1035{
1036 if ((pg->pg_flags & (PG_TABLED0x00000004|PQ_ACTIVE0x00040000|PQ_INACTIVE0x00020000)) &&
30
Assuming the condition is true
1037 (pg->uobject == NULL((void *)0) || !UVM_OBJ_IS_PMAP(pg->uobject)((pg->uobject)->pgops == &pmap_pager)))
31
Assuming field 'uobject' is equal to NULL
1038 MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0)
;
32
Assuming the condition is false
33
Loop condition is false. Exiting loop
1039
1040 uvm_pageclean(pg);
34
Calling 'uvm_pageclean'
1041 uvm_pmr_freepages(pg, 1);
1042}
1043
1044/*
1045 * uvm_page_unbusy: unbusy an array of pages.
1046 *
1047 * => pages must either all belong to the same object, or all belong to anons.
1048 * => if pages are anon-owned, anons must have 0 refcount.
1049 */
1050void
1051uvm_page_unbusy(struct vm_page **pgs, int npgs)
1052{
1053 struct vm_page *pg;
1054 struct uvm_object *uobj;
1055 int i;
1056
1057 for (i = 0; i < npgs; i++) {
1
Assuming 'i' is < 'npgs'
2
Loop condition is true. Entering loop body
1058 pg = pgs[i];
1059
1060 if (pg == NULL((void *)0) || pg == PGO_DONTCARE((struct vm_page *) -1L)) {
3
Assuming 'pg' is not equal to NULL
4
Assuming the condition is false
5
Taking false branch
1061 continue;
1062 }
1063
1064 KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1064, "uvm_page_owner_locked_p(pg)"
))
;
6
Calling 'uvm_page_owner_locked_p'
13
Returning from 'uvm_page_owner_locked_p'
14
'?' condition is true
1065 KASSERT(pg->pg_flags & PG_BUSY)((pg->pg_flags & 0x00000001) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1065, "pg->pg_flags & PG_BUSY"
))
;
15
Assuming the condition is true
16
'?' condition is true
1066
1067 if (pg->pg_flags & PG_WANTED0x00000002) {
17
Assuming the condition is false
18
Taking false branch
1068 wakeup(pg);
1069 }
1070 if (pg->pg_flags & PG_RELEASED0x00000020) {
19
Assuming the condition is true
20
Taking true branch
1071 uobj = pg->uobject;
1072 if (uobj
20.1
'uobj' is not equal to NULL
20.1
'uobj' is not equal to NULL
!= NULL((void *)0)) {
21
Taking true branch
1073 uvm_lock_pageq()mtx_enter(&uvm.pageqlock);
1074 pmap_page_protect(pg, PROT_NONE0x00);
22
Calling 'pmap_page_protect'
26
Returning from 'pmap_page_protect'
1075 /* XXX won't happen right now */
1076 if (pg->pg_flags & PQ_AOBJ0x00200000)
27
Assuming the condition is false
28
Taking false branch
1077 uao_dropswap(uobj,
1078 pg->offset >> PAGE_SHIFT12);
1079 uvm_pagefree(pg);
29
Calling 'uvm_pagefree'
1080 uvm_unlock_pageq()mtx_leave(&uvm.pageqlock);
1081 } else {
1082 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001);
1083 UVM_PAGE_OWN(pg, NULL);
1084 rw_enter(pg->uanon->an_lock, RW_WRITE0x0001UL);
1085 uvm_anon_release(pg->uanon);
1086 }
1087 } else {
1088 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_WANTED0x00000002|PG_BUSY0x00000001);
1089 UVM_PAGE_OWN(pg, NULL);
1090 }
1091 }
1092}
1093
1094#if defined(UVM_PAGE_TRKOWN)
1095/*
1096 * uvm_page_own: set or release page ownership
1097 *
1098 * => this is a debugging function that keeps track of who sets PG_BUSY
1099 * and where they do it. it can be used to track down problems
1100 * such a thread setting "PG_BUSY" and never releasing it.
1101 * => if "tag" is NULL then we are releasing page ownership
1102 */
1103void
1104uvm_page_own(struct vm_page *pg, char *tag)
1105{
1106 /* gain ownership? */
1107 if (tag) {
1108 if (pg->owner_tag) {
1109 printf("uvm_page_own: page %p already owned "
1110 "by thread %d [%s]\n", pg,
1111 pg->owner, pg->owner_tag);
1112 panic("uvm_page_own");
1113 }
1114 pg->owner = (curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc
) ? curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r"
(__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self)));
__ci;})->ci_curproc
->p_tid : (pid_t) -1;
1115 pg->owner_tag = tag;
1116 return;
1117 }
1118
1119 /* drop ownership */
1120 if (pg->owner_tag == NULL((void *)0)) {
1121 printf("uvm_page_own: dropping ownership of an non-owned "
1122 "page (%p)\n", pg);
1123 panic("uvm_page_own");
1124 }
1125 pg->owner_tag = NULL((void *)0);
1126 return;
1127}
1128#endif
1129
1130/*
1131 * when VM_PHYSSEG_MAX is 1, we can simplify these functions
1132 */
1133
1134#if VM_PHYSSEG_MAX16 > 1
1135/*
1136 * vm_physseg_find: find vm_physseg structure that belongs to a PA
1137 */
1138int
1139vm_physseg_find(paddr_t pframe, int *offp)
1140{
1141 struct vm_physseg *seg;
1142
1143#if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2)
1144 /* binary search for it */
1145 int start, len, try;
1146
1147 /*
1148 * if try is too large (thus target is less than than try) we reduce
1149 * the length to trunc(len/2) [i.e. everything smaller than "try"]
1150 *
1151 * if the try is too small (thus target is greater than try) then
1152 * we set the new start to be (try + 1). this means we need to
1153 * reduce the length to (round(len/2) - 1).
1154 *
1155 * note "adjust" below which takes advantage of the fact that
1156 * (round(len/2) - 1) == trunc((len - 1) / 2)
1157 * for any value of len we may have
1158 */
1159
1160 for (start = 0, len = vm_nphysseg ; len != 0 ; len = len / 2) {
1161 try = start + (len / 2); /* try in the middle */
1162 seg = vm_physmem + try;
1163
1164 /* start past our try? */
1165 if (pframe >= seg->start) {
1166 /* was try correct? */
1167 if (pframe < seg->end) {
1168 if (offp)
1169 *offp = pframe - seg->start;
1170 return try; /* got it */
1171 }
1172 start = try + 1; /* next time, start here */
1173 len--; /* "adjust" */
1174 } else {
1175 /*
1176 * pframe before try, just reduce length of
1177 * region, done in "for" loop
1178 */
1179 }
1180 }
1181 return -1;
1182
1183#else
1184 /* linear search for it */
1185 int lcv;
1186
1187 for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) {
1188 if (pframe >= seg->start && pframe < seg->end) {
1189 if (offp)
1190 *offp = pframe - seg->start;
1191 return lcv; /* got it */
1192 }
1193 }
1194 return -1;
1195
1196#endif
1197}
1198
1199/*
1200 * PHYS_TO_VM_PAGE: find vm_page for a PA. used by MI code to get vm_pages
1201 * back from an I/O mapping (ugh!). used in some MD code as well.
1202 */
1203struct vm_page *
1204PHYS_TO_VM_PAGE(paddr_t pa)
1205{
1206 paddr_t pf = atop(pa)((pa) >> 12);
1207 int off;
1208 int psi;
1209
1210 psi = vm_physseg_find(pf, &off);
1211
1212 return (psi == -1) ? NULL((void *)0) : &vm_physmem[psi].pgs[off];
1213}
1214#endif /* VM_PHYSSEG_MAX > 1 */
1215
1216/*
1217 * uvm_pagelookup: look up a page
1218 */
1219struct vm_page *
1220uvm_pagelookup(struct uvm_object *obj, voff_t off)
1221{
1222 /* XXX if stack is too much, handroll */
1223 struct vm_page pg;
1224
1225 pg.offset = off;
1226 return RBT_FIND(uvm_objtree, &obj->memt, &pg)uvm_objtree_RBT_FIND(&obj->memt, &pg);
1227}
1228
1229/*
1230 * uvm_pagewire: wire the page, thus removing it from the daemon's grasp
1231 *
1232 * => caller must lock page queues
1233 */
1234void
1235uvm_pagewire(struct vm_page *pg)
1236{
1237 KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1237, "uvm_page_owner_locked_p(pg)"
))
;
1238 MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0)
;
1239
1240 if (pg->wire_count == 0) {
1241 if (pg->pg_flags & PQ_ACTIVE0x00040000) {
1242 TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev;
*(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
1243 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000);
1244 uvmexp.active--;
1245 }
1246 if (pg->pg_flags & PQ_INACTIVE0x00020000) {
1247 if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000))
1248 TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
1249 else
1250 TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
1251 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000);
1252 uvmexp.inactive--;
1253 }
1254 uvmexp.wired++;
1255 }
1256 pg->wire_count++;
1257}
1258
1259/*
1260 * uvm_pageunwire: unwire the page.
1261 *
1262 * => activate if wire count goes to zero.
1263 * => caller must lock page queues
1264 */
1265void
1266uvm_pageunwire(struct vm_page *pg)
1267{
1268 KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1268, "uvm_page_owner_locked_p(pg)"
))
;
1269 MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0)
;
1270
1271 pg->wire_count--;
1272 if (pg->wire_count == 0) {
1273 TAILQ_INSERT_TAIL(&uvm.page_active, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev
= (&uvm.page_active)->tqh_last; *(&uvm.page_active
)->tqh_last = (pg); (&uvm.page_active)->tqh_last = &
(pg)->pageq.tqe_next; } while (0)
;
1274 uvmexp.active++;
1275 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000);
1276 uvmexp.wired--;
1277 }
1278}
1279
1280/*
1281 * uvm_pagedeactivate: deactivate page -- no pmaps have access to page
1282 *
1283 * => caller must lock page queues
1284 * => caller must check to make sure page is not wired
1285 * => object that page belongs to must be locked (so we can adjust pg->flags)
1286 */
1287void
1288uvm_pagedeactivate(struct vm_page *pg)
1289{
1290 KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1290, "uvm_page_owner_locked_p(pg)"
))
;
1291 MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0)
;
1292
1293 if (pg->pg_flags & PQ_ACTIVE0x00040000) {
1294 TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev;
*(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
1295 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000);
1296 uvmexp.active--;
1297 }
1298 if ((pg->pg_flags & PQ_INACTIVE0x00020000) == 0) {
1299 KASSERT(pg->wire_count == 0)((pg->wire_count == 0) ? (void)0 : __assert("diagnostic ",
"/usr/src/sys/uvm/uvm_page.c", 1299, "pg->wire_count == 0"
))
;
1300 if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000))
1301 TAILQ_INSERT_TAIL(&uvm.page_inactive_swp, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev
= (&uvm.page_inactive_swp)->tqh_last; *(&uvm.page_inactive_swp
)->tqh_last = (pg); (&uvm.page_inactive_swp)->tqh_last
= &(pg)->pageq.tqe_next; } while (0)
;
1302 else
1303 TAILQ_INSERT_TAIL(&uvm.page_inactive_obj, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev
= (&uvm.page_inactive_obj)->tqh_last; *(&uvm.page_inactive_obj
)->tqh_last = (pg); (&uvm.page_inactive_obj)->tqh_last
= &(pg)->pageq.tqe_next; } while (0)
;
1304 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000);
1305 uvmexp.inactive++;
1306 pmap_clear_reference(pg)pmap_clear_attrs(pg, 0x0000000000000020UL);
1307 /*
1308 * update the "clean" bit. this isn't 100%
1309 * accurate, and doesn't have to be. we'll
1310 * re-sync it after we zap all mappings when
1311 * scanning the inactive list.
1312 */
1313 if ((pg->pg_flags & PG_CLEAN0x00000008) != 0 &&
1314 pmap_is_modified(pg)pmap_test_attrs(pg, 0x0000000000000040UL))
1315 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
1316 }
1317}
1318
1319/*
1320 * uvm_pageactivate: activate page
1321 *
1322 * => caller must lock page queues
1323 */
1324void
1325uvm_pageactivate(struct vm_page *pg)
1326{
1327 KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic "
, "/usr/src/sys/uvm/uvm_page.c", 1327, "uvm_page_owner_locked_p(pg)"
))
;
1328 MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info
*__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof
(struct cpu_info, ci_self))); __ci;})) && !(panicstr ||
db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock
), __func__); } while (0)
;
1329
1330 if (pg->pg_flags & PQ_INACTIVE0x00020000) {
1331 if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000))
1332 TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
1333 else
1334 TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev
; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
1335 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000);
1336 uvmexp.inactive--;
1337 }
1338 if (pg->wire_count == 0) {
1339 /*
1340 * if page is already active, remove it from list so we
1341 * can put it at tail. if it wasn't active, then mark
1342 * it active and bump active count
1343 */
1344 if (pg->pg_flags & PQ_ACTIVE0x00040000)
1345 TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq
.tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else (
&uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev;
*(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->
pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = (
(void *)-1); } while (0)
;
1346 else {
1347 atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000);
1348 uvmexp.active++;
1349 }
1350
1351 TAILQ_INSERT_TAIL(&uvm.page_active, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev
= (&uvm.page_active)->tqh_last; *(&uvm.page_active
)->tqh_last = (pg); (&uvm.page_active)->tqh_last = &
(pg)->pageq.tqe_next; } while (0)
;
1352 }
1353}
1354
1355/*
1356 * uvm_pagezero: zero fill a page
1357 */
1358void
1359uvm_pagezero(struct vm_page *pg)
1360{
1361 atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008);
1362 pmap_zero_page(pg);
1363}
1364
1365/*
1366 * uvm_pagecopy: copy a page
1367 */
1368void
1369uvm_pagecopy(struct vm_page *src, struct vm_page *dst)
1370{
1371 atomic_clearbits_intx86_atomic_clearbits_u32(&dst->pg_flags, PG_CLEAN0x00000008);
1372 pmap_copy_page(src, dst);
1373}
1374
1375/*
1376 * uvm_page_owner_locked_p: return true if object associated with page is
1377 * locked. this is a weak check for runtime assertions only.
1378 */
1379int
1380uvm_page_owner_locked_p(struct vm_page *pg)
1381{
1382 if (pg->uobject != NULL((void *)0)) {
7
Assuming field 'uobject' is not equal to NULL
8
Taking true branch
1383 if (UVM_OBJ_IS_DUMMY(pg->uobject)(((pg->uobject)->pgops == &pmap_pager) || ((pg->
uobject)->pgops == &bufcache_pager))
)
9
Assuming the condition is false
10
Assuming the condition is false
11
Taking false branch
1384 return 1;
1385 return rw_write_held(pg->uobject->vmobjlock);
12
Returning without writing to 'pg->uobject'
1386 }
1387 if (pg->uanon != NULL((void *)0)) {
1388 return rw_write_held(pg->uanon->an_lock);
1389 }
1390 return 1;
1391}
1392
1393/*
1394 * uvm_pagecount: count the number of physical pages in the address range.
1395 */
1396psize_t
1397uvm_pagecount(struct uvm_constraint_range* constraint)
1398{
1399 int lcv;
1400 psize_t sz;
1401 paddr_t low, high;
1402 paddr_t ps_low, ps_high;
1403
1404 /* Algorithm uses page numbers. */
1405 low = atop(constraint->ucr_low)((constraint->ucr_low) >> 12);
1406 high = atop(constraint->ucr_high)((constraint->ucr_high) >> 12);
1407
1408 sz = 0;
1409 for (lcv = 0; lcv < vm_nphysseg; lcv++) {
1410 ps_low = MAX(low, vm_physmem[lcv].avail_start)(((low)>(vm_physmem[lcv].avail_start))?(low):(vm_physmem[lcv
].avail_start))
;
1411 ps_high = MIN(high, vm_physmem[lcv].avail_end)(((high)<(vm_physmem[lcv].avail_end))?(high):(vm_physmem[lcv
].avail_end))
;
1412 if (ps_low < ps_high)
1413 sz += ps_high - ps_low;
1414 }
1415 return sz;
1416}

/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj/machine/pmap.h

1/* $OpenBSD: pmap.h,v 1.78 2021/06/18 06:17:28 guenther Exp $ */
2/* $NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $ */
3
4/*
5 * Copyright (c) 1997 Charles D. Cranor and Washington University.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/*
30 * Copyright (c) 2001 Wasabi Systems, Inc.
31 * All rights reserved.
32 *
33 * Written by Frank van der Linden for Wasabi Systems, Inc.
34 *
35 * Redistribution and use in source and binary forms, with or without
36 * modification, are permitted provided that the following conditions
37 * are met:
38 * 1. Redistributions of source code must retain the above copyright
39 * notice, this list of conditions and the following disclaimer.
40 * 2. Redistributions in binary form must reproduce the above copyright
41 * notice, this list of conditions and the following disclaimer in the
42 * documentation and/or other materials provided with the distribution.
43 * 3. All advertising materials mentioning features or use of this software
44 * must display the following acknowledgement:
45 * This product includes software developed for the NetBSD Project by
46 * Wasabi Systems, Inc.
47 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
48 * or promote products derived from this software without specific prior
49 * written permission.
50 *
51 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
52 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
53 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
54 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
55 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
56 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
57 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
58 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
59 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
60 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
61 * POSSIBILITY OF SUCH DAMAGE.
62 */
63
64/*
65 * pmap.h: see pmap.c for the history of this pmap module.
66 */
67
68#ifndef _MACHINE_PMAP_H_
69#define _MACHINE_PMAP_H_
70
71#ifndef _LOCORE
72#ifdef _KERNEL1
73#include <machine/cpufunc.h>
74#endif /* _KERNEL */
75#include <sys/mutex.h>
76#include <uvm/uvm_object.h>
77#include <machine/pte.h>
78#endif
79
80/*
81 * The x86_64 pmap module closely resembles the i386 one. It uses
82 * the same recursive entry scheme. See the i386 pmap.h for a
83 * description. The alternate area trick for accessing non-current
84 * pmaps has been removed, though, because it performs badly on SMP
85 * systems.
86 * The most obvious difference to i386 is that 2 extra levels of page
87 * table need to be dealt with. The level 1 page table pages are at:
88 *
89 * l1: 0x00007f8000000000 - 0x00007fffffffffff (39 bits, needs PML4 entry)
90 *
91 * The other levels are kept as physical pages in 3 UVM objects and are
92 * temporarily mapped for virtual access when needed.
93 *
94 * The other obvious difference from i386 is that it has a direct map of all
95 * physical memory in the VA range:
96 *
97 * 0xfffffd8000000000 - 0xffffff7fffffffff
98 *
99 * The direct map is used in some cases to access PTEs of non-current pmaps.
100 *
101 * Note that address space is signed, so the layout for 48 bits is:
102 *
103 * +---------------------------------+ 0xffffffffffffffff
104 * | Kernel Image |
105 * +---------------------------------+ 0xffffff8000000000
106 * | Direct Map |
107 * +---------------------------------+ 0xfffffd8000000000
108 * ~ ~
109 * | |
110 * | Kernel Space |
111 * | |
112 * | |
113 * +---------------------------------+ 0xffff800000000000 = 0x0000800000000000
114 * | L1 table (PTE pages) |
115 * +---------------------------------+ 0x00007f8000000000
116 * ~ ~
117 * | |
118 * | User Space |
119 * | |
120 * | |
121 * +---------------------------------+ 0x0000000000000000
122 *
123 * In other words, there is a 'VA hole' at 0x0000800000000000 -
124 * 0xffff800000000000 which will trap, just as on, for example,
125 * sparcv9.
126 *
127 * The unused space can be used if needed, but it adds a little more
128 * complexity to the calculations.
129 */
130
131/*
132 * Mask to get rid of the sign-extended part of addresses.
133 */
134#define VA_SIGN_MASK0xffff000000000000 0xffff000000000000
135#define VA_SIGN_NEG(va)((va) | 0xffff000000000000) ((va) | VA_SIGN_MASK0xffff000000000000)
136/*
137 * XXXfvdl this one's not right.
138 */
139#define VA_SIGN_POS(va)((va) & ~0xffff000000000000) ((va) & ~VA_SIGN_MASK0xffff000000000000)
140
141#define L4_SLOT_PTE255 255
142#define L4_SLOT_KERN256 256
143#define L4_SLOT_KERNBASE511 511
144#define NUM_L4_SLOT_DIRECT4 4
145#define L4_SLOT_DIRECT(511 - 4) (L4_SLOT_KERNBASE511 - NUM_L4_SLOT_DIRECT4)
146#define L4_SLOT_EARLY((511 - 4) - 1) (L4_SLOT_DIRECT(511 - 4) - 1)
147
148#define PDIR_SLOT_KERN256 L4_SLOT_KERN256
149#define PDIR_SLOT_PTE255 L4_SLOT_PTE255
150#define PDIR_SLOT_DIRECT(511 - 4) L4_SLOT_DIRECT(511 - 4)
151#define PDIR_SLOT_EARLY((511 - 4) - 1) L4_SLOT_EARLY((511 - 4) - 1)
152
153/*
154 * the following defines give the virtual addresses of various MMU
155 * data structures:
156 * PTE_BASE: the base VA of the linear PTE mappings
157 * PDP_PDE: the VA of the PDE that points back to the PDP
158 *
159 */
160
161#define PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) ((pt_entry_t *) (L4_SLOT_PTE255 * NBPD_L4(1ULL << 39)))
162#define PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) (VA_SIGN_NEG((L4_SLOT_DIRECT * NBPD_L4))((((511 - 4) * (1ULL << 39))) | 0xffff000000000000))
163#define PMAP_DIRECT_END((((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000
))
(VA_SIGN_NEG(((L4_SLOT_DIRECT + \(((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000
)
164 NUM_L4_SLOT_DIRECT) * NBPD_L4))(((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000
)
)
165
166#define L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) PTE_BASE((pt_entry_t *) (255 * (1ULL << 39)))
167
168#define L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
39))) + 255 * (1ULL << 30)))
((pd_entry_t *)((char *)L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) + L4_SLOT_PTE255 * NBPD_L3(1ULL << 30)))
169#define L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *
) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255
* (1ULL << 21)))
((pd_entry_t *)((char *)L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
39))) + 255 * (1ULL << 30)))
+ L4_SLOT_PTE255 * NBPD_L2(1ULL << 21)))
170#define L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
(1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
<< 12)))
((pd_entry_t *)((char *)L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *
) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255
* (1ULL << 21)))
+ L4_SLOT_PTE255 * NBPD_L1(1ULL << 12)))
171
172#define PDP_PDE(((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t
*)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255
* (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (
1ULL << 12))) + 255)
(L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
(1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
<< 12)))
+ PDIR_SLOT_PTE255)
173
174#define PDP_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
(1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
<< 12)))
L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
(1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
<< 12)))
175
176#define NKL4_MAX_ENTRIES(unsigned long)1 (unsigned long)1
177#define NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512) (unsigned long)(NKL4_MAX_ENTRIES(unsigned long)1 * 512)
178#define NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512
)
(unsigned long)(NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512) * 512)
179#define NKL1_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)((unsigned long
)1 * 512) * 512) * 512)
(unsigned long)(NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512
)
* 512)
180
181#define NKL4_KIMG_ENTRIES1 1
182#define NKL3_KIMG_ENTRIES1 1
183#define NKL2_KIMG_ENTRIES64 64
184
185/* number of pages of direct map entries set up by locore0.S */
186#define NDML4_ENTRIES1 1
187#define NDML3_ENTRIES1 1
188#define NDML2_ENTRIES4 4 /* 4GB */
189
190/*
191 * Since kva space is below the kernel in its entirety, we start off
192 * with zero entries on each level.
193 */
194#define NKL4_START_ENTRIES0 0
195#define NKL3_START_ENTRIES0 0
196#define NKL2_START_ENTRIES0 0
197#define NKL1_START_ENTRIES0 0 /* XXX */
198
199#define NTOPLEVEL_PDES((1 << 12) / (sizeof (pd_entry_t))) (PAGE_SIZE(1 << 12) / (sizeof (pd_entry_t)))
200
201#define NPDPG((1 << 12) / sizeof (pd_entry_t)) (PAGE_SIZE(1 << 12) / sizeof (pd_entry_t))
202
203/*
204 * pl*_pi: index in the ptp page for a pde mapping a VA.
205 * (pl*_i below is the index in the virtual array of all pdes per level)
206 */
207#define pl1_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x00000000001ff000UL
) >> 12)
(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12)
208#define pl2_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x000000003fe00000UL
) >> 21)
(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21)
209#define pl3_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x0000007fc0000000UL
) >> 30)
(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30)
210#define pl4_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x0000ff8000000000UL
) >> 39)
(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39)
211
212/*
213 * pl*_i: generate index into pde/pte arrays in virtual space
214 */
215#define pl1_i(VA)(((((VA) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)
(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L1_FRAME(((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)|0x00000000001ff000UL)
) >> L1_SHIFT12)
216#define pl2_i(VA)(((((VA) & ~0xffff000000000000)) & ((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)) >> 21)
(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)
) >> L2_SHIFT21)
217#define pl3_i(VA)(((((VA) & ~0xffff000000000000)) & (0x0000ff8000000000UL
|0x0000007fc0000000UL)) >> 30)
(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L3_FRAME(0x0000ff8000000000UL|0x0000007fc0000000UL)) >> L3_SHIFT30)
218#define pl4_i(VA)(((((VA) & ~0xffff000000000000)) & 0x0000ff8000000000UL
) >> 39)
(((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L4_FRAME0x0000ff8000000000UL) >> L4_SHIFT39)
219#define pl_i(va, lvl)(((((va) & ~0xffff000000000000)) & ptp_masks[(lvl)-1]
) >> ptp_shifts[(lvl)-1])
\
220 (((VA_SIGN_POS(va)((va) & ~0xffff000000000000)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1])
221
222#define PTP_MASK_INITIALIZER{ (((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)|0x00000000001ff000UL), ((0x0000ff8000000000UL|0x0000007fc0000000UL
)|0x000000003fe00000UL), (0x0000ff8000000000UL|0x0000007fc0000000UL
), 0x0000ff8000000000UL }
{ L1_FRAME(((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)|0x00000000001ff000UL)
, L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL
)
, L3_FRAME(0x0000ff8000000000UL|0x0000007fc0000000UL), L4_FRAME0x0000ff8000000000UL }
223#define PTP_SHIFT_INITIALIZER{ 12, 21, 30, 39 } { L1_SHIFT12, L2_SHIFT21, L3_SHIFT30, L4_SHIFT39 }
224#define NKPTP_INITIALIZER{ 0, 0, 0, 0 } { NKL1_START_ENTRIES0, NKL2_START_ENTRIES0, \
225 NKL3_START_ENTRIES0, NKL4_START_ENTRIES0 }
226#define NKPTPMAX_INITIALIZER{ (unsigned long)((unsigned long)((unsigned long)((unsigned long
)1 * 512) * 512) * 512), (unsigned long)((unsigned long)((unsigned
long)1 * 512) * 512), (unsigned long)((unsigned long)1 * 512
), (unsigned long)1 }
{ NKL1_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)((unsigned long
)1 * 512) * 512) * 512)
, NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512
)
, \
227 NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512), NKL4_MAX_ENTRIES(unsigned long)1 }
228#define NBPD_INITIALIZER{ (1ULL << 12), (1ULL << 21), (1ULL << 30),
(1ULL << 39) }
{ NBPD_L1(1ULL << 12), NBPD_L2(1ULL << 21), NBPD_L3(1ULL << 30), NBPD_L4(1ULL << 39) }
229#define PDES_INITIALIZER{ ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
39))) + 255 * (1ULL << 30))), ((pd_entry_t *)((char *)
((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21
))), ((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t
*)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255
* (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (
1ULL << 12))) }
{ L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL <<
39))) + 255 * (1ULL << 30)))
, L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *
) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255
* (1ULL << 21)))
, L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *
)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 *
(1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL
<< 12)))
}
230
231/*
232 * PTP macros:
233 * a PTP's index is the PD index of the PDE that points to it
234 * a PTP's offset is the byte-offset in the PTE space that this PTP is at
235 * a PTP's VA is the first VA mapped by that PTP
236 */
237
238#define ptp_va2o(va, lvl)((((((va) & ~0xffff000000000000)) & ptp_masks[((lvl)+
1)-1]) >> ptp_shifts[((lvl)+1)-1]) * (1 << 12))
(pl_i(va, (lvl)+1)(((((va) & ~0xffff000000000000)) & ptp_masks[((lvl)+1
)-1]) >> ptp_shifts[((lvl)+1)-1])
* PAGE_SIZE(1 << 12))
239
240#define PTP_LEVELS4 4
241
242/*
243 * PG_AVAIL usage: we make use of the ignored bits of the PTE
244 */
245
246#define PG_W0x0000000000000200UL PG_AVAIL10x0000000000000200UL /* "wired" mapping */
247#define PG_PVLIST0x0000000000000400UL PG_AVAIL20x0000000000000400UL /* mapping has entry on pvlist */
248/* PG_AVAIL3 not used */
249
250/*
251 * PCID assignments.
252 * The shootdown code assumes KERN, PROC, and PROC_INTEL are both
253 * consecutive and in that order.
254 */
255#define PCID_KERN0 0 /* for pmap_kernel() */
256#define PCID_PROC1 1 /* non-pmap_kernel(), U+K */
257#define PCID_PROC_INTEL2 2 /* non-pmap_kernel(), U-K (meltdown) */
258#define PCID_TEMP3 3 /* temp mapping of another non-pmap_kernel() */
259
260extern int pmap_use_pcid; /* non-zero if PCID support is enabled */
261
262/*
263 * Number of PTEs per cache line. 8 byte pte, 64-byte cache line
264 * Used to avoid false sharing of cache lines.
265 */
266#define NPTECL8 8
267
268
269#if defined(_KERNEL1) && !defined(_LOCORE)
270/*
271 * pmap data structures: see pmap.c for details of locking.
272 */
273
274struct pmap;
275typedef struct pmap *pmap_t;
276
277/*
278 * we maintain a list of all non-kernel pmaps
279 */
280
281LIST_HEAD(pmap_head, pmap)struct pmap_head { struct pmap *lh_first; }; /* struct pmap_head: head of a pmap list */
282
283/*
284 * the pmap structure
285 *
286 * note that the pm_obj contains the reference count,
287 * page list, and number of PTPs within the pmap.
288 */
289
290#define PMAP_TYPE_NORMAL1 1
291#define PMAP_TYPE_EPT2 2
292#define PMAP_TYPE_RVI3 3
293#define pmap_nested(pm)((pm)->pm_type != 1) ((pm)->pm_type != PMAP_TYPE_NORMAL1)
294
295struct pmap {
296 struct mutex pm_mtx;
297 struct uvm_object pm_obj[PTP_LEVELS4-1]; /* objects for lvl >= 1) */
298 LIST_ENTRY(pmap)struct { struct pmap *le_next; struct pmap **le_prev; } pm_list; /* list (lck by pm_list lock) */
299 /*
300 * pm_pdir : VA of page table to be used when executing in
301 * privileged mode
302 * pm_pdirpa : PA of page table to be used when executing in
303 * privileged mode
304 * pm_pdir_intel : VA of special page table to be used when executing
305 * on an Intel CPU in usermode (no kernel mappings)
306 * pm_pdirpa_intel : PA of special page table to be used when executing
307 * on an Intel CPU in usermode (no kernel mappings)
308 */
309 pd_entry_t *pm_pdir, *pm_pdir_intel;
310 paddr_t pm_pdirpa, pm_pdirpa_intel;
311
312 struct vm_page *pm_ptphint[PTP_LEVELS4-1];
313 /* pointer to a PTP in our pmap */
314 struct pmap_statistics pm_stats; /* pmap stats (lck by object lock) */
315
316 int pm_type; /* Type of pmap this is (PMAP_TYPE_x) */
317 uint64_t eptp; /* cached EPTP (used by vmm) */
318};
319
320/*
321 * MD flags that we use for pmap_enter (in the pa):
322 */
323#define PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)) ~((paddr_t)PAGE_MASK((1 << 12) - 1)) /* to remove the flags */
324#define PMAP_NOCACHE0x1 0x1 /* set the non-cacheable bit. */
325#define PMAP_WC0x2 0x2 /* set page write combining. */
326
327/*
328 * We keep mod/ref flags in struct vm_page->pg_flags.
329 */
330#define PG_PMAP_MOD0x01000000 PG_PMAP00x01000000
331#define PG_PMAP_REF0x02000000 PG_PMAP10x02000000
332#define PG_PMAP_WC0x04000000 PG_PMAP20x04000000
333
334/*
335 * for each managed physical page we maintain a list of <PMAP,VA>'s
336 * which it is mapped at.
337 */
338struct pv_entry { /* locked by its list's pvh_lock */
339 struct pv_entry *pv_next; /* next entry */
340 struct pmap *pv_pmap; /* the pmap */
341 vaddr_t pv_va; /* the virtual address */
342 struct vm_page *pv_ptp; /* the vm_page of the PTP */
343};
344
345/*
346 * global kernel variables
347 */
348
349extern struct pmap kernel_pmap_store; /* kernel pmap */
350
351extern long nkptp[];
352
353extern const paddr_t ptp_masks[];
354extern const int ptp_shifts[];
355extern const long nbpd[], nkptpmax[];
356
357/*
358 * macros
359 */
360
361#define pmap_kernel()(&kernel_pmap_store) (&kernel_pmap_store)
362#define pmap_resident_count(pmap)((pmap)->pm_stats.resident_count) ((pmap)->pm_stats.resident_count)
363#define pmap_wired_count(pmap)((pmap)->pm_stats.wired_count) ((pmap)->pm_stats.wired_count)
364#define pmap_update(pmap) /* nothing (yet) */
365
366#define pmap_clear_modify(pg)pmap_clear_attrs(pg, 0x0000000000000040UL) pmap_clear_attrs(pg, PG_M0x0000000000000040UL)
367#define pmap_clear_reference(pg)pmap_clear_attrs(pg, 0x0000000000000020UL) pmap_clear_attrs(pg, PG_U0x0000000000000020UL)
368#define pmap_copy(DP,SP,D,L,S)
369#define pmap_is_modified(pg)pmap_test_attrs(pg, 0x0000000000000040UL) pmap_test_attrs(pg, PG_M0x0000000000000040UL)
370#define pmap_is_referenced(pg)pmap_test_attrs(pg, 0x0000000000000020UL) pmap_test_attrs(pg, PG_U0x0000000000000020UL)
371#define pmap_move(DP,SP,D,L,S)
372#define pmap_valid_entry(E)((E) & 0x0000000000000001UL) ((E) & PG_V0x0000000000000001UL) /* is PDE or PTE valid? */
373
374#define pmap_proc_iflush(p,va,len) /* nothing */
375#define pmap_unuse_final(p) /* nothing */
376#define pmap_remove_holes(vm)do { } while (0) do { /* nothing */ } while (0)
377
378
379/*
380 * prototypes
381 */
382
383void map_tramps(void); /* machdep.c */
384paddr_t pmap_bootstrap(paddr_t, paddr_t);
385void pmap_randomize(void);
386void pmap_randomize_level(pd_entry_t *, int);
387int pmap_clear_attrs(struct vm_page *, unsigned long);
388static void pmap_page_protect(struct vm_page *, vm_prot_t);
389void pmap_page_remove (struct vm_page *);
390static void pmap_protect(struct pmap *, vaddr_t,
391 vaddr_t, vm_prot_t);
392void pmap_remove(struct pmap *, vaddr_t, vaddr_t);
393int pmap_test_attrs(struct vm_page *, unsigned);
394static void pmap_update_pg(vaddr_t);
395void pmap_write_protect(struct pmap *, vaddr_t,
396 vaddr_t, vm_prot_t);
397void pmap_fix_ept(struct pmap *, vaddr_t);
398
399paddr_t pmap_prealloc_lowmem_ptps(paddr_t);
400
401void pagezero(vaddr_t);
402
403int pmap_convert(struct pmap *, int);
404void pmap_enter_special(vaddr_t, paddr_t, vm_prot_t);
405vaddr_t pmap_set_pml4_early(paddr_t pa);
406void pmap_clear_pml4_early(void);
407
408/*
409 * functions for flushing the cache for vaddrs and pages.
410 * these functions are not part of the MI pmap interface and thus
411 * should not be used as such.
412 */
413void pmap_flush_cache(vaddr_t, vsize_t);
414#define pmap_flush_page(paddr)do { ((void)0); pmap_flush_cache(((vaddr_t)(((((511 - 4) * (1ULL
<< 39))) | 0xffff000000000000)) + (paddr)), (1 <<
12)); } while ( 0)
do { \
415 KDASSERT(PHYS_TO_VM_PAGE(paddr) != NULL)((void)0); \
416 pmap_flush_cache(PMAP_DIRECT_MAP(paddr)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (paddr))
, PAGE_SIZE(1 << 12)); \
417} while (/* CONSTCOND */ 0)
418
419#define PMAP_STEAL_MEMORY /* enable pmap_steal_memory() */
420#define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */
421
422/*
423 * inline functions
424 */
425
426static inline void
427pmap_remove_all(struct pmap *pmap)
428{
429 /* Nothing. */
430}
431
432/*
433 * pmap_update_pg: flush one page from the TLB (or flush the whole thing
434 * if hardware doesn't support one-page flushing)
435 */
436
437inline static void
438pmap_update_pg(vaddr_t va)
439{
440 invlpg(va);
441}
442
443/*
444 * pmap_page_protect: change the protection of all recorded mappings
445 * of a managed page
446 *
447 * => this function is a frontend for pmap_page_remove/pmap_clear_attrs
448 * => we only have to worry about making the page more protected.
449 * unprotecting a page is done on-demand at fault time.
450 */
451
452inline static void
453pmap_page_protect(struct vm_page *pg, vm_prot_t prot)
454{
455 if ((prot & PROT_WRITE0x02) == 0) {
23
Taking true branch
456 if (prot & (PROT_READ0x01 | PROT_EXEC0x04)) {
24
Taking false branch
457 (void) pmap_clear_attrs(pg, PG_RW0x0000000000000002UL);
458 } else {
459 pmap_page_remove(pg);
25
Value assigned to field 'uobject'
460 }
461 }
462}
463
464/*
465 * pmap_protect: change the protection of pages in a pmap
466 *
467 * => this function is a frontend for pmap_remove/pmap_write_protect
468 * => we only have to worry about making the page more protected.
469 * unprotecting a page is done on-demand at fault time.
470 */
471
472inline static void
473pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot)
474{
475 if ((prot & PROT_WRITE0x02) == 0) {
476 if (prot & (PROT_READ0x01| PROT_EXEC0x04)) {
477 pmap_write_protect(pmap, sva, eva, prot);
478 } else {
479 pmap_remove(pmap, sva, eva);
480 }
481 }
482}
483
484/*
485 * various address inlines
486 *
487 * vtopte: return a pointer to the PTE mapping a VA, works only for
488 * user and PT addresses
489 *
490 * kvtopte: return a pointer to the PTE mapping a kernel VA
491 */
492
493static inline pt_entry_t *
494vtopte(vaddr_t va)
495{
496 return (PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)
);
497}
498
499static inline pt_entry_t *
500kvtopte(vaddr_t va)
501{
502#ifdef LARGEPAGES
503 {
504 pd_entry_t *pde;
505
506 pde = L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl2_i(va)(((((va) & ~0xffff000000000000)) & ((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)) >> 21)
;
507 if (*pde & PG_PS0x0000000000000080UL)
508 return ((pt_entry_t *)pde);
509 }
510#endif
511
512 return (PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL
|0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL
)) >> 12)
);
513}
514
515#define PMAP_DIRECT_MAP(pa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (pa))
((vaddr_t)PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) + (pa))
516#define PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)))
((paddr_t)(va) - PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)))
517#define pmap_map_direct(pg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (((pg)->phys_addr)))
PMAP_DIRECT_MAP(VM_PAGE_TO_PHYS(pg))((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)) + (((pg)->phys_addr)))
518#define pmap_unmap_direct(va)PHYS_TO_VM_PAGE(((paddr_t)(va) - (((((511 - 4) * (1ULL <<
39))) | 0xffff000000000000))))
PHYS_TO_VM_PAGE(PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000
)))
)
519
520#define __HAVE_PMAP_DIRECT
521
522#endif /* _KERNEL && !_LOCORE */
523
524#ifndef _LOCORE
525struct pv_entry;
526struct vm_page_md {
527 struct mutex pv_mtx;
528 struct pv_entry *pv_list;
529};
530
531#define VM_MDPAGE_INIT(pg)do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(pg
)->mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) <
0x9) ? 0x9 : ((0xa)))); } while (0); (pg)->mdpage.pv_list
= ((void *)0); } while (0)
do { \
532 mtx_init(&(pg)->mdpage.pv_mtx, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&(pg)->
mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9
) ? 0x9 : ((0xa)))); } while (0)
; \
533 (pg)->mdpage.pv_list = NULL((void *)0); \
534} while (0)
535#endif /* !_LOCORE */
536
537#endif /* _MACHINE_PMAP_H_ */