| File: | uvm/uvm_page.c |
| Warning: | line 157, column 2 Access to field 'pgops' results in a dereference of a null pointer (loaded from field 'uobject') |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* $OpenBSD: uvm_page.c,v 1.173 2023/08/12 07:22:56 mpi Exp $ */ | ||||
| 2 | /* $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $ */ | ||||
| 3 | |||||
| 4 | /* | ||||
| 5 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | ||||
| 6 | * Copyright (c) 1991, 1993, The Regents of the University of California. | ||||
| 7 | * | ||||
| 8 | * All rights reserved. | ||||
| 9 | * | ||||
| 10 | * This code is derived from software contributed to Berkeley by | ||||
| 11 | * The Mach Operating System project at Carnegie-Mellon University. | ||||
| 12 | * | ||||
| 13 | * Redistribution and use in source and binary forms, with or without | ||||
| 14 | * modification, are permitted provided that the following conditions | ||||
| 15 | * are met: | ||||
| 16 | * 1. Redistributions of source code must retain the above copyright | ||||
| 17 | * notice, this list of conditions and the following disclaimer. | ||||
| 18 | * 2. Redistributions in binary form must reproduce the above copyright | ||||
| 19 | * notice, this list of conditions and the following disclaimer in the | ||||
| 20 | * documentation and/or other materials provided with the distribution. | ||||
| 21 | * 3. Neither the name of the University nor the names of its contributors | ||||
| 22 | * may be used to endorse or promote products derived from this software | ||||
| 23 | * without specific prior written permission. | ||||
| 24 | * | ||||
| 25 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | ||||
| 26 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||||
| 27 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||||
| 28 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | ||||
| 29 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | ||||
| 30 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | ||||
| 31 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | ||||
| 32 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | ||||
| 33 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | ||||
| 34 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | ||||
| 35 | * SUCH DAMAGE. | ||||
| 36 | * | ||||
| 37 | * @(#)vm_page.c 8.3 (Berkeley) 3/21/94 | ||||
| 38 | * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp | ||||
| 39 | * | ||||
| 40 | * | ||||
| 41 | * Copyright (c) 1987, 1990 Carnegie-Mellon University. | ||||
| 42 | * All rights reserved. | ||||
| 43 | * | ||||
| 44 | * Permission to use, copy, modify and distribute this software and | ||||
| 45 | * its documentation is hereby granted, provided that both the copyright | ||||
| 46 | * notice and this permission notice appear in all copies of the | ||||
| 47 | * software, derivative works or modified versions, and any portions | ||||
| 48 | * thereof, and that both notices appear in supporting documentation. | ||||
| 49 | * | ||||
| 50 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" | ||||
| 51 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND | ||||
| 52 | * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. | ||||
| 53 | * | ||||
| 54 | * Carnegie Mellon requests users of this software to return to | ||||
| 55 | * | ||||
| 56 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU | ||||
| 57 | * School of Computer Science | ||||
| 58 | * Carnegie Mellon University | ||||
| 59 | * Pittsburgh PA 15213-3890 | ||||
| 60 | * | ||||
| 61 | * any improvements or extensions that they make and grant Carnegie the | ||||
| 62 | * rights to redistribute these changes. | ||||
| 63 | */ | ||||
| 64 | |||||
| 65 | /* | ||||
| 66 | * uvm_page.c: page ops. | ||||
| 67 | */ | ||||
| 68 | |||||
| 69 | #include <sys/param.h> | ||||
| 70 | #include <sys/systm.h> | ||||
| 71 | #include <sys/sched.h> | ||||
| 72 | #include <sys/vnode.h> | ||||
| 73 | #include <sys/mount.h> | ||||
| 74 | #include <sys/proc.h> | ||||
| 75 | #include <sys/smr.h> | ||||
| 76 | |||||
| 77 | #include <uvm/uvm.h> | ||||
| 78 | |||||
| 79 | /* | ||||
| 80 | * for object trees | ||||
| 81 | */ | ||||
| 82 | RBT_GENERATE(uvm_objtree, vm_page, objt, uvm_pagecmp)static int uvm_objtree_RBT_COMPARE(const void *lptr, const void *rptr) { const struct vm_page *l = lptr, *r = rptr; return uvm_pagecmp (l, r); } static const struct rb_type uvm_objtree_RBT_INFO = { uvm_objtree_RBT_COMPARE, ((void *)0), __builtin_offsetof(struct vm_page, objt), }; const struct rb_type *const uvm_objtree_RBT_TYPE = &uvm_objtree_RBT_INFO; | ||||
| 83 | |||||
| 84 | int | ||||
| 85 | uvm_pagecmp(const struct vm_page *a, const struct vm_page *b) | ||||
| 86 | { | ||||
| 87 | return a->offset < b->offset ? -1 : a->offset > b->offset; | ||||
| 88 | } | ||||
| 89 | |||||
| 90 | /* | ||||
| 91 | * global vars... XXXCDC: move to uvm. structure. | ||||
| 92 | */ | ||||
| 93 | /* | ||||
| 94 | * physical memory config is stored in vm_physmem. | ||||
| 95 | */ | ||||
| 96 | struct vm_physseg vm_physmem[VM_PHYSSEG_MAX16]; /* XXXCDC: uvm.physmem */ | ||||
| 97 | int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */ | ||||
| 98 | |||||
| 99 | /* | ||||
| 100 | * Some supported CPUs in a given architecture don't support all | ||||
| 101 | * of the things necessary to do idle page zero'ing efficiently. | ||||
| 102 | * We therefore provide a way to disable it from machdep code here. | ||||
| 103 | */ | ||||
| 104 | |||||
| 105 | /* | ||||
| 106 | * local variables | ||||
| 107 | */ | ||||
| 108 | /* | ||||
| 109 | * these variables record the values returned by vm_page_bootstrap, | ||||
| 110 | * for debugging purposes. The implementation of uvm_pageboot_alloc | ||||
| 111 | * and pmap_startup here also uses them internally. | ||||
| 112 | */ | ||||
| 113 | static vaddr_t virtual_space_start; | ||||
| 114 | static vaddr_t virtual_space_end; | ||||
| 115 | |||||
| 116 | /* | ||||
| 117 | * local prototypes | ||||
| 118 | */ | ||||
| 119 | static void uvm_pageinsert(struct vm_page *); | ||||
| 120 | static void uvm_pageremove(struct vm_page *); | ||||
| 121 | int uvm_page_owner_locked_p(struct vm_page *); | ||||
| 122 | |||||
| 123 | /* | ||||
| 124 | * inline functions | ||||
| 125 | */ | ||||
| 126 | /* | ||||
| 127 | * uvm_pageinsert: insert a page in the object | ||||
| 128 | * | ||||
| 129 | * => caller must lock object | ||||
| 130 | * => call should have already set pg's object and offset pointers | ||||
| 131 | * and bumped the version counter | ||||
| 132 | */ | ||||
| 133 | static inline void | ||||
| 134 | uvm_pageinsert(struct vm_page *pg) | ||||
| 135 | { | ||||
| 136 | struct vm_page *dupe; | ||||
| 137 | |||||
| 138 | KASSERT(UVM_OBJ_IS_DUMMY(pg->uobject) ||(((((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager)) || rw_write_held( pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 139, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )) | ||||
| 139 | rw_write_held(pg->uobject->vmobjlock))(((((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager)) || rw_write_held( pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 139, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )); | ||||
| 140 | KASSERT((pg->pg_flags & PG_TABLED) == 0)(((pg->pg_flags & 0x00000004) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 140, "(pg->pg_flags & PG_TABLED) == 0" )); | ||||
| 141 | |||||
| 142 | dupe = RBT_INSERT(uvm_objtree, &pg->uobject->memt, pg)uvm_objtree_RBT_INSERT(&pg->uobject->memt, pg); | ||||
| 143 | /* not allowed to insert over another page */ | ||||
| 144 | KASSERT(dupe == NULL)((dupe == ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 144, "dupe == NULL")); | ||||
| 145 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_TABLED0x00000004); | ||||
| 146 | pg->uobject->uo_npages++; | ||||
| 147 | } | ||||
| 148 | |||||
| 149 | /* | ||||
| 150 | * uvm_page_remove: remove page from object | ||||
| 151 | * | ||||
| 152 | * => caller must lock object | ||||
| 153 | */ | ||||
| 154 | static inline void | ||||
| 155 | uvm_pageremove(struct vm_page *pg) | ||||
| 156 | { | ||||
| 157 | KASSERT(UVM_OBJ_IS_DUMMY(pg->uobject) ||(((((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager)) || rw_write_held( pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 158, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )) | ||||
| |||||
| 158 | rw_write_held(pg->uobject->vmobjlock))(((((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager)) || rw_write_held( pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 158, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )); | ||||
| 159 | KASSERT(pg->pg_flags & PG_TABLED)((pg->pg_flags & 0x00000004) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 159, "pg->pg_flags & PG_TABLED" )); | ||||
| 160 | |||||
| 161 | RBT_REMOVE(uvm_objtree, &pg->uobject->memt, pg)uvm_objtree_RBT_REMOVE(&pg->uobject->memt, pg); | ||||
| 162 | |||||
| 163 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_TABLED0x00000004); | ||||
| 164 | pg->uobject->uo_npages--; | ||||
| 165 | pg->uobject = NULL((void *)0); | ||||
| 166 | pg->pg_version++; | ||||
| 167 | } | ||||
| 168 | |||||
| 169 | /* | ||||
| 170 | * uvm_page_init: init the page system. called from uvm_init(). | ||||
| 171 | * | ||||
| 172 | * => we return the range of kernel virtual memory in kvm_startp/kvm_endp | ||||
| 173 | */ | ||||
| 174 | void | ||||
| 175 | uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp) | ||||
| 176 | { | ||||
| 177 | vsize_t freepages, pagecount, n; | ||||
| 178 | vm_page_t pagearray, curpg; | ||||
| 179 | int lcv, i; | ||||
| 180 | paddr_t paddr, pgno; | ||||
| 181 | struct vm_physseg *seg; | ||||
| 182 | |||||
| 183 | /* | ||||
| 184 | * init the page queues and page queue locks | ||||
| 185 | */ | ||||
| 186 | |||||
| 187 | TAILQ_INIT(&uvm.page_active)do { (&uvm.page_active)->tqh_first = ((void *)0); (& uvm.page_active)->tqh_last = &(&uvm.page_active)-> tqh_first; } while (0); | ||||
| 188 | TAILQ_INIT(&uvm.page_inactive)do { (&uvm.page_inactive)->tqh_first = ((void *)0); (& uvm.page_inactive)->tqh_last = &(&uvm.page_inactive )->tqh_first; } while (0); | ||||
| 189 | mtx_init(&uvm.pageqlock, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.pageqlock ), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : (( 0xa)))); } while (0); | ||||
| 190 | mtx_init(&uvm.fpageqlock, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.fpageqlock ), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : (( 0xa)))); } while (0); | ||||
| 191 | uvm_pmr_init(); | ||||
| 192 | |||||
| 193 | /* | ||||
| 194 | * allocate vm_page structures. | ||||
| 195 | */ | ||||
| 196 | |||||
| 197 | /* | ||||
| 198 | * sanity check: | ||||
| 199 | * before calling this function the MD code is expected to register | ||||
| 200 | * some free RAM with the uvm_page_physload() function. our job | ||||
| 201 | * now is to allocate vm_page structures for this memory. | ||||
| 202 | */ | ||||
| 203 | |||||
| 204 | if (vm_nphysseg == 0) | ||||
| 205 | panic("uvm_page_bootstrap: no memory pre-allocated"); | ||||
| 206 | |||||
| 207 | /* | ||||
| 208 | * first calculate the number of free pages... | ||||
| 209 | * | ||||
| 210 | * note that we use start/end rather than avail_start/avail_end. | ||||
| 211 | * this allows us to allocate extra vm_page structures in case we | ||||
| 212 | * want to return some memory to the pool after booting. | ||||
| 213 | */ | ||||
| 214 | |||||
| 215 | freepages = 0; | ||||
| 216 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) | ||||
| 217 | freepages += (seg->end - seg->start); | ||||
| 218 | |||||
| 219 | /* | ||||
| 220 | * we now know we have (PAGE_SIZE * freepages) bytes of memory we can | ||||
| 221 | * use. for each page of memory we use we need a vm_page structure. | ||||
| 222 | * thus, the total number of pages we can use is the total size of | ||||
| 223 | * the memory divided by the PAGE_SIZE plus the size of the vm_page | ||||
| 224 | * structure. we add one to freepages as a fudge factor to avoid | ||||
| 225 | * truncation errors (since we can only allocate in terms of whole | ||||
| 226 | * pages). | ||||
| 227 | */ | ||||
| 228 | |||||
| 229 | pagecount = (((paddr_t)freepages + 1) << PAGE_SHIFT12) / | ||||
| 230 | (PAGE_SIZE(1 << 12) + sizeof(struct vm_page)); | ||||
| 231 | pagearray = (vm_page_t)uvm_pageboot_alloc(pagecount * | ||||
| 232 | sizeof(struct vm_page)); | ||||
| 233 | memset(pagearray, 0, pagecount * sizeof(struct vm_page))__builtin_memset((pagearray), (0), (pagecount * sizeof(struct vm_page))); | ||||
| 234 | |||||
| 235 | /* init the vm_page structures and put them in the correct place. */ | ||||
| 236 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) { | ||||
| 237 | n = seg->end - seg->start; | ||||
| 238 | if (n > pagecount) { | ||||
| 239 | panic("uvm_page_init: lost %ld page(s) in init", | ||||
| 240 | (long)(n - pagecount)); | ||||
| 241 | /* XXXCDC: shouldn't happen? */ | ||||
| 242 | /* n = pagecount; */ | ||||
| 243 | } | ||||
| 244 | |||||
| 245 | /* set up page array pointers */ | ||||
| 246 | seg->pgs = pagearray; | ||||
| 247 | pagearray += n; | ||||
| 248 | pagecount -= n; | ||||
| 249 | seg->lastpg = seg->pgs + (n - 1); | ||||
| 250 | |||||
| 251 | /* init and free vm_pages (we've already zeroed them) */ | ||||
| 252 | pgno = seg->start; | ||||
| 253 | paddr = ptoa(pgno)((paddr_t)(pgno) << 12); | ||||
| 254 | for (i = 0, curpg = seg->pgs; i < n; | ||||
| 255 | i++, curpg++, pgno++, paddr += PAGE_SIZE(1 << 12)) { | ||||
| 256 | curpg->phys_addr = paddr; | ||||
| 257 | VM_MDPAGE_INIT(curpg)do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(curpg )->mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); (curpg)->mdpage.pv_list = ((void *)0); } while (0); | ||||
| 258 | if (pgno >= seg->avail_start && | ||||
| 259 | pgno < seg->avail_end) { | ||||
| 260 | uvmexp.npages++; | ||||
| 261 | } | ||||
| 262 | } | ||||
| 263 | |||||
| 264 | /* Add pages to free pool. */ | ||||
| 265 | uvm_pmr_freepages(&seg->pgs[seg->avail_start - seg->start], | ||||
| 266 | seg->avail_end - seg->avail_start); | ||||
| 267 | } | ||||
| 268 | |||||
| 269 | /* | ||||
| 270 | * pass up the values of virtual_space_start and | ||||
| 271 | * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper | ||||
| 272 | * layers of the VM. | ||||
| 273 | */ | ||||
| 274 | |||||
| 275 | *kvm_startp = round_page(virtual_space_start)(((virtual_space_start) + ((1 << 12) - 1)) & ~((1 << 12) - 1)); | ||||
| 276 | *kvm_endp = trunc_page(virtual_space_end)((virtual_space_end) & ~((1 << 12) - 1)); | ||||
| 277 | |||||
| 278 | /* init locks for kernel threads */ | ||||
| 279 | mtx_init(&uvm.aiodoned_lock, IPL_BIO)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.aiodoned_lock ), ((((0x3)) > 0x0 && ((0x3)) < 0x9) ? 0x9 : (( 0x3)))); } while (0); | ||||
| 280 | |||||
| 281 | /* | ||||
| 282 | * init reserve thresholds | ||||
| 283 | * XXXCDC - values may need adjusting | ||||
| 284 | */ | ||||
| 285 | uvmexp.reserve_pagedaemon = 4; | ||||
| 286 | uvmexp.reserve_kernel = 8; | ||||
| 287 | uvmexp.anonminpct = 10; | ||||
| 288 | uvmexp.vnodeminpct = 10; | ||||
| 289 | uvmexp.vtextminpct = 5; | ||||
| 290 | uvmexp.anonmin = uvmexp.anonminpct * 256 / 100; | ||||
| 291 | uvmexp.vnodemin = uvmexp.vnodeminpct * 256 / 100; | ||||
| 292 | uvmexp.vtextmin = uvmexp.vtextminpct * 256 / 100; | ||||
| 293 | |||||
| 294 | uvm.page_init_done = TRUE1; | ||||
| 295 | } | ||||
| 296 | |||||
| 297 | /* | ||||
| 298 | * uvm_setpagesize: set the page size | ||||
| 299 | * | ||||
| 300 | * => sets page_shift and page_mask from uvmexp.pagesize. | ||||
| 301 | */ | ||||
| 302 | void | ||||
| 303 | uvm_setpagesize(void) | ||||
| 304 | { | ||||
| 305 | if (uvmexp.pagesize == 0) | ||||
| 306 | uvmexp.pagesize = DEFAULT_PAGE_SIZE4096; | ||||
| 307 | uvmexp.pagemask = uvmexp.pagesize - 1; | ||||
| 308 | if ((uvmexp.pagemask & uvmexp.pagesize) != 0) | ||||
| 309 | panic("uvm_setpagesize: page size not a power of two"); | ||||
| 310 | for (uvmexp.pageshift = 0; ; uvmexp.pageshift++) | ||||
| 311 | if ((1 << uvmexp.pageshift) == uvmexp.pagesize) | ||||
| 312 | break; | ||||
| 313 | } | ||||
| 314 | |||||
| 315 | /* | ||||
| 316 | * uvm_pageboot_alloc: steal memory from physmem for bootstrapping | ||||
| 317 | */ | ||||
| 318 | vaddr_t | ||||
| 319 | uvm_pageboot_alloc(vsize_t size) | ||||
| 320 | { | ||||
| 321 | #if defined(PMAP_STEAL_MEMORY) | ||||
| 322 | vaddr_t addr; | ||||
| 323 | |||||
| 324 | /* | ||||
| 325 | * defer bootstrap allocation to MD code (it may want to allocate | ||||
| 326 | * from a direct-mapped segment). pmap_steal_memory should round | ||||
| 327 | * off virtual_space_start/virtual_space_end. | ||||
| 328 | */ | ||||
| 329 | |||||
| 330 | addr = pmap_steal_memory(size, &virtual_space_start, | ||||
| 331 | &virtual_space_end); | ||||
| 332 | |||||
| 333 | return addr; | ||||
| 334 | |||||
| 335 | #else /* !PMAP_STEAL_MEMORY */ | ||||
| 336 | |||||
| 337 | static boolean_t initialized = FALSE0; | ||||
| 338 | vaddr_t addr, vaddr; | ||||
| 339 | paddr_t paddr; | ||||
| 340 | |||||
| 341 | /* round to page size */ | ||||
| 342 | size = round_page(size)(((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1 )); | ||||
| 343 | |||||
| 344 | /* on first call to this function, initialize ourselves. */ | ||||
| 345 | if (initialized == FALSE0) { | ||||
| 346 | pmap_virtual_space(&virtual_space_start, &virtual_space_end); | ||||
| 347 | |||||
| 348 | /* round it the way we like it */ | ||||
| 349 | virtual_space_start = round_page(virtual_space_start)(((virtual_space_start) + ((1 << 12) - 1)) & ~((1 << 12) - 1)); | ||||
| 350 | virtual_space_end = trunc_page(virtual_space_end)((virtual_space_end) & ~((1 << 12) - 1)); | ||||
| 351 | |||||
| 352 | initialized = TRUE1; | ||||
| 353 | } | ||||
| 354 | |||||
| 355 | /* allocate virtual memory for this request */ | ||||
| 356 | if (virtual_space_start == virtual_space_end || | ||||
| 357 | (virtual_space_end - virtual_space_start) < size) | ||||
| 358 | panic("uvm_pageboot_alloc: out of virtual space"); | ||||
| 359 | |||||
| 360 | addr = virtual_space_start; | ||||
| 361 | |||||
| 362 | #ifdef PMAP_GROWKERNEL | ||||
| 363 | /* | ||||
| 364 | * If the kernel pmap can't map the requested space, | ||||
| 365 | * then allocate more resources for it. | ||||
| 366 | */ | ||||
| 367 | if (uvm_maxkaddr < (addr + size)) { | ||||
| 368 | uvm_maxkaddr = pmap_growkernel(addr + size); | ||||
| 369 | if (uvm_maxkaddr < (addr + size)) | ||||
| 370 | panic("uvm_pageboot_alloc: pmap_growkernel() failed"); | ||||
| 371 | } | ||||
| 372 | #endif | ||||
| 373 | |||||
| 374 | virtual_space_start += size; | ||||
| 375 | |||||
| 376 | /* allocate and mapin physical pages to back new virtual pages */ | ||||
| 377 | for (vaddr = round_page(addr)(((addr) + ((1 << 12) - 1)) & ~((1 << 12) - 1 )) ; vaddr < addr + size ; | ||||
| 378 | vaddr += PAGE_SIZE(1 << 12)) { | ||||
| 379 | if (!uvm_page_physget(&paddr)) | ||||
| 380 | panic("uvm_pageboot_alloc: out of memory"); | ||||
| 381 | |||||
| 382 | /* | ||||
| 383 | * Note this memory is no longer managed, so using | ||||
| 384 | * pmap_kenter is safe. | ||||
| 385 | */ | ||||
| 386 | pmap_kenter_pa(vaddr, paddr, PROT_READ0x01 | PROT_WRITE0x02); | ||||
| 387 | } | ||||
| 388 | pmap_update(pmap_kernel()); | ||||
| 389 | return addr; | ||||
| 390 | #endif /* PMAP_STEAL_MEMORY */ | ||||
| 391 | } | ||||
| 392 | |||||
| 393 | #if !defined(PMAP_STEAL_MEMORY) | ||||
| 394 | /* | ||||
| 395 | * uvm_page_physget: "steal" one page from the vm_physmem structure. | ||||
| 396 | * | ||||
| 397 | * => attempt to allocate it off the end of a segment in which the "avail" | ||||
| 398 | * values match the start/end values. if we can't do that, then we | ||||
| 399 | * will advance both values (making them equal, and removing some | ||||
| 400 | * vm_page structures from the non-avail area). | ||||
| 401 | * => return false if out of memory. | ||||
| 402 | */ | ||||
| 403 | |||||
| 404 | boolean_t | ||||
| 405 | uvm_page_physget(paddr_t *paddrp) | ||||
| 406 | { | ||||
| 407 | int lcv; | ||||
| 408 | struct vm_physseg *seg; | ||||
| 409 | |||||
| 410 | /* pass 1: try allocating from a matching end */ | ||||
| 411 | #if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) || \ | ||||
| 412 | (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2) | ||||
| 413 | for (lcv = vm_nphysseg - 1, seg = vm_physmem + lcv; lcv >= 0; | ||||
| 414 | lcv--, seg--) | ||||
| 415 | #else | ||||
| 416 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) | ||||
| 417 | #endif | ||||
| 418 | { | ||||
| 419 | if (uvm.page_init_done == TRUE1) | ||||
| 420 | panic("uvm_page_physget: called _after_ bootstrap"); | ||||
| 421 | |||||
| 422 | /* try from front */ | ||||
| 423 | if (seg->avail_start == seg->start && | ||||
| 424 | seg->avail_start < seg->avail_end) { | ||||
| 425 | *paddrp = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12); | ||||
| 426 | seg->avail_start++; | ||||
| 427 | seg->start++; | ||||
| 428 | /* nothing left? nuke it */ | ||||
| 429 | if (seg->avail_start == seg->end) { | ||||
| 430 | if (vm_nphysseg == 1) | ||||
| 431 | panic("uvm_page_physget: out of memory!"); | ||||
| 432 | vm_nphysseg--; | ||||
| 433 | for (; lcv < vm_nphysseg; lcv++, seg++) | ||||
| 434 | /* structure copy */ | ||||
| 435 | seg[0] = seg[1]; | ||||
| 436 | } | ||||
| 437 | return TRUE1; | ||||
| 438 | } | ||||
| 439 | |||||
| 440 | /* try from rear */ | ||||
| 441 | if (seg->avail_end == seg->end && | ||||
| 442 | seg->avail_start < seg->avail_end) { | ||||
| 443 | *paddrp = ptoa(seg->avail_end - 1)((paddr_t)(seg->avail_end - 1) << 12); | ||||
| 444 | seg->avail_end--; | ||||
| 445 | seg->end--; | ||||
| 446 | /* nothing left? nuke it */ | ||||
| 447 | if (seg->avail_end == seg->start) { | ||||
| 448 | if (vm_nphysseg == 1) | ||||
| 449 | panic("uvm_page_physget: out of memory!"); | ||||
| 450 | vm_nphysseg--; | ||||
| 451 | for (; lcv < vm_nphysseg ; lcv++, seg++) | ||||
| 452 | /* structure copy */ | ||||
| 453 | seg[0] = seg[1]; | ||||
| 454 | } | ||||
| 455 | return TRUE1; | ||||
| 456 | } | ||||
| 457 | } | ||||
| 458 | |||||
| 459 | /* pass2: forget about matching ends, just allocate something */ | ||||
| 460 | #if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) || \ | ||||
| 461 | (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2) | ||||
| 462 | for (lcv = vm_nphysseg - 1, seg = vm_physmem + lcv; lcv >= 0; | ||||
| 463 | lcv--, seg--) | ||||
| 464 | #else | ||||
| 465 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) | ||||
| 466 | #endif | ||||
| 467 | { | ||||
| 468 | |||||
| 469 | /* any room in this bank? */ | ||||
| 470 | if (seg->avail_start >= seg->avail_end) | ||||
| 471 | continue; /* nope */ | ||||
| 472 | |||||
| 473 | *paddrp = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12); | ||||
| 474 | seg->avail_start++; | ||||
| 475 | /* truncate! */ | ||||
| 476 | seg->start = seg->avail_start; | ||||
| 477 | |||||
| 478 | /* nothing left? nuke it */ | ||||
| 479 | if (seg->avail_start == seg->end) { | ||||
| 480 | if (vm_nphysseg == 1) | ||||
| 481 | panic("uvm_page_physget: out of memory!"); | ||||
| 482 | vm_nphysseg--; | ||||
| 483 | for (; lcv < vm_nphysseg ; lcv++, seg++) | ||||
| 484 | /* structure copy */ | ||||
| 485 | seg[0] = seg[1]; | ||||
| 486 | } | ||||
| 487 | return TRUE1; | ||||
| 488 | } | ||||
| 489 | |||||
| 490 | return FALSE0; /* whoops! */ | ||||
| 491 | } | ||||
| 492 | |||||
| 493 | #endif /* PMAP_STEAL_MEMORY */ | ||||
| 494 | |||||
| 495 | /* | ||||
| 496 | * uvm_page_physload: load physical memory into VM system | ||||
| 497 | * | ||||
| 498 | * => all args are PFs | ||||
| 499 | * => all pages in start/end get vm_page structures | ||||
| 500 | * => areas marked by avail_start/avail_end get added to the free page pool | ||||
| 501 | * => we are limited to VM_PHYSSEG_MAX physical memory segments | ||||
| 502 | */ | ||||
| 503 | |||||
| 504 | void | ||||
| 505 | uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start, | ||||
| 506 | paddr_t avail_end, int flags) | ||||
| 507 | { | ||||
| 508 | int preload, lcv; | ||||
| 509 | psize_t npages; | ||||
| 510 | struct vm_page *pgs; | ||||
| 511 | struct vm_physseg *ps, *seg; | ||||
| 512 | |||||
| 513 | #ifdef DIAGNOSTIC1 | ||||
| 514 | if (uvmexp.pagesize == 0) | ||||
| 515 | panic("uvm_page_physload: page size not set!"); | ||||
| 516 | |||||
| 517 | if (start >= end) | ||||
| 518 | panic("uvm_page_physload: start >= end"); | ||||
| 519 | #endif | ||||
| 520 | |||||
| 521 | /* do we have room? */ | ||||
| 522 | if (vm_nphysseg == VM_PHYSSEG_MAX16) { | ||||
| 523 | printf("uvm_page_physload: unable to load physical memory " | ||||
| 524 | "segment\n"); | ||||
| 525 | printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n", | ||||
| 526 | VM_PHYSSEG_MAX16, (long long)start, (long long)end); | ||||
| 527 | printf("\tincrease VM_PHYSSEG_MAX\n"); | ||||
| 528 | return; | ||||
| 529 | } | ||||
| 530 | |||||
| 531 | /* | ||||
| 532 | * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been | ||||
| 533 | * called yet, so malloc is not available). | ||||
| 534 | */ | ||||
| 535 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++) { | ||||
| 536 | if (seg->pgs) | ||||
| 537 | break; | ||||
| 538 | } | ||||
| 539 | preload = (lcv == vm_nphysseg); | ||||
| 540 | |||||
| 541 | /* if VM is already running, attempt to malloc() vm_page structures */ | ||||
| 542 | if (!preload) { | ||||
| 543 | /* | ||||
| 544 | * XXXCDC: need some sort of lockout for this case | ||||
| 545 | * right now it is only used by devices so it should be alright. | ||||
| 546 | */ | ||||
| 547 | paddr_t paddr; | ||||
| 548 | |||||
| 549 | npages = end - start; /* # of pages */ | ||||
| 550 | |||||
| 551 | pgs = km_alloc(round_page(npages * sizeof(*pgs))(((npages * sizeof(*pgs)) + ((1 << 12) - 1)) & ~((1 << 12) - 1)), | ||||
| 552 | &kv_any, &kp_zero, &kd_waitok); | ||||
| 553 | if (pgs == NULL((void *)0)) { | ||||
| 554 | printf("uvm_page_physload: can not malloc vm_page " | ||||
| 555 | "structs for segment\n"); | ||||
| 556 | printf("\tignoring 0x%lx -> 0x%lx\n", start, end); | ||||
| 557 | return; | ||||
| 558 | } | ||||
| 559 | /* init phys_addr and free pages, XXX uvmexp.npages */ | ||||
| 560 | for (lcv = 0, paddr = ptoa(start)((paddr_t)(start) << 12); lcv < npages; | ||||
| 561 | lcv++, paddr += PAGE_SIZE(1 << 12)) { | ||||
| 562 | pgs[lcv].phys_addr = paddr; | ||||
| 563 | VM_MDPAGE_INIT(&pgs[lcv])do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(& pgs[lcv])->mdpage.pv_mtx), ((((0xa)) > 0x0 && ( (0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); (&pgs[lcv ])->mdpage.pv_list = ((void *)0); } while (0); | ||||
| 564 | if (atop(paddr)((paddr) >> 12) >= avail_start && | ||||
| 565 | atop(paddr)((paddr) >> 12) < avail_end) { | ||||
| 566 | if (flags & PHYSLOAD_DEVICE0x01) { | ||||
| 567 | atomic_setbits_intx86_atomic_setbits_u32(&pgs[lcv].pg_flags, | ||||
| 568 | PG_DEV0x00000200); | ||||
| 569 | pgs[lcv].wire_count = 1; | ||||
| 570 | } else { | ||||
| 571 | #if defined(VM_PHYSSEG_NOADD) | ||||
| 572 | panic("uvm_page_physload: tried to add RAM after vm_mem_init"); | ||||
| 573 | #endif | ||||
| 574 | } | ||||
| 575 | } | ||||
| 576 | } | ||||
| 577 | |||||
| 578 | /* Add pages to free pool. */ | ||||
| 579 | if ((flags & PHYSLOAD_DEVICE0x01) == 0) { | ||||
| 580 | uvm_pmr_freepages(&pgs[avail_start - start], | ||||
| 581 | avail_end - avail_start); | ||||
| 582 | } | ||||
| 583 | |||||
| 584 | /* XXXCDC: need hook to tell pmap to rebuild pv_list, etc... */ | ||||
| 585 | } else { | ||||
| 586 | /* gcc complains if these don't get init'd */ | ||||
| 587 | pgs = NULL((void *)0); | ||||
| 588 | npages = 0; | ||||
| 589 | |||||
| 590 | } | ||||
| 591 | |||||
| 592 | /* now insert us in the proper place in vm_physmem[] */ | ||||
| 593 | #if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_RANDOM1) | ||||
| 594 | /* random: put it at the end (easy!) */ | ||||
| 595 | ps = &vm_physmem[vm_nphysseg]; | ||||
| 596 | #elif (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2) | ||||
| 597 | { | ||||
| 598 | int x; | ||||
| 599 | /* sort by address for binary search */ | ||||
| 600 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++) | ||||
| 601 | if (start < seg->start) | ||||
| 602 | break; | ||||
| 603 | ps = seg; | ||||
| 604 | /* move back other entries, if necessary ... */ | ||||
| 605 | for (x = vm_nphysseg, seg = vm_physmem + x - 1; x > lcv; | ||||
| 606 | x--, seg--) | ||||
| 607 | /* structure copy */ | ||||
| 608 | seg[1] = seg[0]; | ||||
| 609 | } | ||||
| 610 | #elif (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) | ||||
| 611 | { | ||||
| 612 | int x; | ||||
| 613 | /* sort by largest segment first */ | ||||
| 614 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++) | ||||
| 615 | if ((end - start) > | ||||
| 616 | (seg->end - seg->start)) | ||||
| 617 | break; | ||||
| 618 | ps = &vm_physmem[lcv]; | ||||
| 619 | /* move back other entries, if necessary ... */ | ||||
| 620 | for (x = vm_nphysseg, seg = vm_physmem + x - 1; x > lcv; | ||||
| 621 | x--, seg--) | ||||
| 622 | /* structure copy */ | ||||
| 623 | seg[1] = seg[0]; | ||||
| 624 | } | ||||
| 625 | #else | ||||
| 626 | panic("uvm_page_physload: unknown physseg strategy selected!"); | ||||
| 627 | #endif | ||||
| 628 | |||||
| 629 | ps->start = start; | ||||
| 630 | ps->end = end; | ||||
| 631 | ps->avail_start = avail_start; | ||||
| 632 | ps->avail_end = avail_end; | ||||
| 633 | if (preload) { | ||||
| 634 | ps->pgs = NULL((void *)0); | ||||
| 635 | } else { | ||||
| 636 | ps->pgs = pgs; | ||||
| 637 | ps->lastpg = pgs + npages - 1; | ||||
| 638 | } | ||||
| 639 | vm_nphysseg++; | ||||
| 640 | |||||
| 641 | return; | ||||
| 642 | } | ||||
| 643 | |||||
| 644 | #ifdef DDB1 /* XXXCDC: TMP TMP TMP DEBUG DEBUG DEBUG */ | ||||
| 645 | |||||
| 646 | void uvm_page_physdump(void); /* SHUT UP GCC */ | ||||
| 647 | |||||
| 648 | /* call from DDB */ | ||||
| 649 | void | ||||
| 650 | uvm_page_physdump(void) | ||||
| 651 | { | ||||
| 652 | int lcv; | ||||
| 653 | struct vm_physseg *seg; | ||||
| 654 | |||||
| 655 | printf("uvm_page_physdump: physical memory config [segs=%d of %d]:\n", | ||||
| 656 | vm_nphysseg, VM_PHYSSEG_MAX16); | ||||
| 657 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) | ||||
| 658 | printf("0x%llx->0x%llx [0x%llx->0x%llx]\n", | ||||
| 659 | (long long)seg->start, | ||||
| 660 | (long long)seg->end, | ||||
| 661 | (long long)seg->avail_start, | ||||
| 662 | (long long)seg->avail_end); | ||||
| 663 | printf("STRATEGY = "); | ||||
| 664 | switch (VM_PHYSSEG_STRAT3) { | ||||
| 665 | case VM_PSTRAT_RANDOM1: printf("RANDOM\n"); break; | ||||
| 666 | case VM_PSTRAT_BSEARCH2: printf("BSEARCH\n"); break; | ||||
| 667 | case VM_PSTRAT_BIGFIRST3: printf("BIGFIRST\n"); break; | ||||
| 668 | default: printf("<<UNKNOWN>>!!!!\n"); | ||||
| 669 | } | ||||
| 670 | } | ||||
| 671 | #endif | ||||
| 672 | |||||
| 673 | void | ||||
| 674 | uvm_shutdown(void) | ||||
| 675 | { | ||||
| 676 | #ifdef UVM_SWAP_ENCRYPT1 | ||||
| 677 | uvm_swap_finicrypt_all(); | ||||
| 678 | #endif | ||||
| 679 | smr_flush()smr_barrier_impl(1); | ||||
| 680 | } | ||||
| 681 | |||||
| 682 | /* | ||||
| 683 | * Perform insert of a given page in the specified anon of obj. | ||||
| 684 | * This is basically, uvm_pagealloc, but with the page already given. | ||||
| 685 | */ | ||||
| 686 | void | ||||
| 687 | uvm_pagealloc_pg(struct vm_page *pg, struct uvm_object *obj, voff_t off, | ||||
| 688 | struct vm_anon *anon) | ||||
| 689 | { | ||||
| 690 | int flags; | ||||
| 691 | |||||
| 692 | KASSERT(obj == NULL || anon == NULL)((obj == ((void *)0) || anon == ((void *)0)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 692, "obj == NULL || anon == NULL" )); | ||||
| 693 | KASSERT(anon == NULL || off == 0)((anon == ((void *)0) || off == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 693, "anon == NULL || off == 0" )); | ||||
| 694 | KASSERT(off == trunc_page(off))((off == ((off) & ~((1 << 12) - 1))) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 694, "off == trunc_page(off)" )); | ||||
| 695 | KASSERT(obj == NULL || UVM_OBJ_IS_DUMMY(obj) ||((obj == ((void *)0) || (((obj)->pgops == &pmap_pager) || ((obj)->pgops == &bufcache_pager)) || rw_write_held (obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 696, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)" )) | ||||
| 696 | rw_write_held(obj->vmobjlock))((obj == ((void *)0) || (((obj)->pgops == &pmap_pager) || ((obj)->pgops == &bufcache_pager)) || rw_write_held (obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 696, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)" )); | ||||
| 697 | KASSERT(anon == NULL || anon->an_lock == NULL ||((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held (anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 698, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)" )) | ||||
| 698 | rw_write_held(anon->an_lock))((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held (anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 698, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)" )); | ||||
| 699 | |||||
| 700 | flags = PG_BUSY0x00000001 | PG_FAKE0x00000040; | ||||
| 701 | pg->offset = off; | ||||
| 702 | pg->uobject = obj; | ||||
| 703 | pg->uanon = anon; | ||||
| 704 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 704, "uvm_page_owner_locked_p(pg)" )); | ||||
| 705 | if (anon) { | ||||
| 706 | anon->an_page = pg; | ||||
| 707 | flags |= PQ_ANON0x00100000; | ||||
| 708 | } else if (obj) | ||||
| 709 | uvm_pageinsert(pg); | ||||
| 710 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, flags); | ||||
| 711 | #if defined(UVM_PAGE_TRKOWN) | ||||
| 712 | pg->owner_tag = NULL((void *)0); | ||||
| 713 | #endif | ||||
| 714 | UVM_PAGE_OWN(pg, "new alloc"); | ||||
| 715 | } | ||||
| 716 | |||||
| 717 | /* | ||||
| 718 | * uvm_pglistalloc: allocate a list of pages | ||||
| 719 | * | ||||
| 720 | * => allocated pages are placed at the tail of rlist. rlist is | ||||
| 721 | * assumed to be properly initialized by caller. | ||||
| 722 | * => returns 0 on success or errno on failure | ||||
| 723 | * => doesn't take into account clean non-busy pages on inactive list | ||||
| 724 | * that could be used(?) | ||||
| 725 | * => params: | ||||
| 726 | * size the size of the allocation, rounded to page size. | ||||
| 727 | * low the low address of the allowed allocation range. | ||||
| 728 | * high the high address of the allowed allocation range. | ||||
| 729 | * alignment memory must be aligned to this power-of-two boundary. | ||||
| 730 | * boundary no segment in the allocation may cross this | ||||
| 731 | * power-of-two boundary (relative to zero). | ||||
| 732 | * => flags: | ||||
| 733 | * UVM_PLA_NOWAIT fail if allocation fails | ||||
| 734 | * UVM_PLA_WAITOK wait for memory to become avail | ||||
| 735 | * UVM_PLA_ZERO return zeroed memory | ||||
| 736 | */ | ||||
| 737 | int | ||||
| 738 | uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment, | ||||
| 739 | paddr_t boundary, struct pglist *rlist, int nsegs, int flags) | ||||
| 740 | { | ||||
| 741 | KASSERT((alignment & (alignment - 1)) == 0)(((alignment & (alignment - 1)) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 741, "(alignment & (alignment - 1)) == 0" )); | ||||
| 742 | KASSERT((boundary & (boundary - 1)) == 0)(((boundary & (boundary - 1)) == 0) ? (void)0 : __assert( "diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 742, "(boundary & (boundary - 1)) == 0" )); | ||||
| 743 | KASSERT(!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT))((!(flags & 0x0001) ^ !(flags & 0x0002)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 743, "!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT)" )); | ||||
| 744 | |||||
| 745 | if (size == 0) | ||||
| 746 | return EINVAL22; | ||||
| 747 | size = atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1))) >> 12); | ||||
| 748 | |||||
| 749 | /* | ||||
| 750 | * XXX uvm_pglistalloc is currently only used for kernel | ||||
| 751 | * objects. Unlike the checks in uvm_pagealloc, below, here | ||||
| 752 | * we are always allowed to use the kernel reserve. | ||||
| 753 | */ | ||||
| 754 | flags |= UVM_PLA_USERESERVE0x0040; | ||||
| 755 | |||||
| 756 | if ((high & PAGE_MASK((1 << 12) - 1)) != PAGE_MASK((1 << 12) - 1)) { | ||||
| 757 | printf("uvm_pglistalloc: Upper boundary 0x%lx " | ||||
| 758 | "not on pagemask.\n", (unsigned long)high); | ||||
| 759 | } | ||||
| 760 | |||||
| 761 | /* | ||||
| 762 | * Our allocations are always page granularity, so our alignment | ||||
| 763 | * must be, too. | ||||
| 764 | */ | ||||
| 765 | if (alignment < PAGE_SIZE(1 << 12)) | ||||
| 766 | alignment = PAGE_SIZE(1 << 12); | ||||
| 767 | |||||
| 768 | low = atop(roundup(low, alignment))((((((low)+((alignment)-1))/(alignment))*(alignment))) >> 12); | ||||
| 769 | /* | ||||
| 770 | * high + 1 may result in overflow, in which case high becomes 0x0, | ||||
| 771 | * which is the 'don't care' value. | ||||
| 772 | * The only requirement in that case is that low is also 0x0, or the | ||||
| 773 | * low<high assert will fail. | ||||
| 774 | */ | ||||
| 775 | high = atop(high + 1)((high + 1) >> 12); | ||||
| 776 | alignment = atop(alignment)((alignment) >> 12); | ||||
| 777 | if (boundary < PAGE_SIZE(1 << 12) && boundary != 0) | ||||
| 778 | boundary = PAGE_SIZE(1 << 12); | ||||
| 779 | boundary = atop(boundary)((boundary) >> 12); | ||||
| 780 | |||||
| 781 | return uvm_pmr_getpages(size, low, high, alignment, boundary, nsegs, | ||||
| 782 | flags, rlist); | ||||
| 783 | } | ||||
| 784 | |||||
| 785 | /* | ||||
| 786 | * uvm_pglistfree: free a list of pages | ||||
| 787 | * | ||||
| 788 | * => pages should already be unmapped | ||||
| 789 | */ | ||||
| 790 | void | ||||
| 791 | uvm_pglistfree(struct pglist *list) | ||||
| 792 | { | ||||
| 793 | uvm_pmr_freepageq(list); | ||||
| 794 | } | ||||
| 795 | |||||
| 796 | /* | ||||
| 797 | * interface used by the buffer cache to allocate a buffer at a time. | ||||
| 798 | * The pages are allocated wired in DMA accessible memory | ||||
| 799 | */ | ||||
| 800 | int | ||||
| 801 | uvm_pagealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size, | ||||
| 802 | int flags) | ||||
| 803 | { | ||||
| 804 | struct pglist plist; | ||||
| 805 | struct vm_page *pg; | ||||
| 806 | int i, r; | ||||
| 807 | |||||
| 808 | KASSERT(UVM_OBJ_IS_BUFCACHE(obj))((((obj)->pgops == &bufcache_pager)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 808, "UVM_OBJ_IS_BUFCACHE(obj)" )); | ||||
| 809 | KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 809, "_kernel_lock_held()")); | ||||
| 810 | |||||
| 811 | TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)-> tqh_last = &(&plist)->tqh_first; } while (0); | ||||
| 812 | r = uvm_pglistalloc(size, dma_constraint.ucr_low, | ||||
| 813 | dma_constraint.ucr_high, 0, 0, &plist, atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1))) >> 12), | ||||
| 814 | flags); | ||||
| 815 | if (r == 0) { | ||||
| 816 | i = 0; | ||||
| 817 | while ((pg = TAILQ_FIRST(&plist)((&plist)->tqh_first)) != NULL((void *)0)) { | ||||
| 818 | pg->wire_count = 1; | ||||
| 819 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008 | PG_FAKE0x00000040); | ||||
| 820 | KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 820, "(pg->pg_flags & PG_DEV) == 0" )); | ||||
| 821 | TAILQ_REMOVE(&plist, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &plist)->tqh_last = (pg)->pageq.tqe_prev; *(pg)-> pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->pageq.tqe_prev ) = ((void *)-1); ((pg)->pageq.tqe_next) = ((void *)-1); } while (0); | ||||
| 822 | uvm_pagealloc_pg(pg, obj, off + ptoa(i++)((paddr_t)(i++) << 12), NULL((void *)0)); | ||||
| 823 | } | ||||
| 824 | } | ||||
| 825 | return r; | ||||
| 826 | } | ||||
| 827 | |||||
| 828 | /* | ||||
| 829 | * interface used by the buffer cache to reallocate a buffer at a time. | ||||
| 830 | * The pages are reallocated wired outside the DMA accessible region. | ||||
| 831 | * | ||||
| 832 | */ | ||||
| 833 | int | ||||
| 834 | uvm_pagerealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size, | ||||
| 835 | int flags, struct uvm_constraint_range *where) | ||||
| 836 | { | ||||
| 837 | struct pglist plist; | ||||
| 838 | struct vm_page *pg, *tpg; | ||||
| 839 | int i, r; | ||||
| 840 | voff_t offset; | ||||
| 841 | |||||
| 842 | KASSERT(UVM_OBJ_IS_BUFCACHE(obj))((((obj)->pgops == &bufcache_pager)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 842, "UVM_OBJ_IS_BUFCACHE(obj)" )); | ||||
| 843 | KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 843, "_kernel_lock_held()")); | ||||
| 844 | |||||
| 845 | TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)-> tqh_last = &(&plist)->tqh_first; } while (0); | ||||
| 846 | if (size == 0) | ||||
| 847 | panic("size 0 uvm_pagerealloc"); | ||||
| 848 | r = uvm_pglistalloc(size, where->ucr_low, where->ucr_high, 0, | ||||
| 849 | 0, &plist, atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1))) >> 12), flags); | ||||
| 850 | if (r == 0) { | ||||
| 851 | i = 0; | ||||
| 852 | while((pg = TAILQ_FIRST(&plist)((&plist)->tqh_first)) != NULL((void *)0)) { | ||||
| 853 | offset = off + ptoa(i++)((paddr_t)(i++) << 12); | ||||
| 854 | tpg = uvm_pagelookup(obj, offset); | ||||
| 855 | KASSERT(tpg != NULL)((tpg != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 855, "tpg != NULL")); | ||||
| 856 | pg->wire_count = 1; | ||||
| 857 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008 | PG_FAKE0x00000040); | ||||
| 858 | KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 858, "(pg->pg_flags & PG_DEV) == 0" )); | ||||
| 859 | TAILQ_REMOVE(&plist, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &plist)->tqh_last = (pg)->pageq.tqe_prev; *(pg)-> pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->pageq.tqe_prev ) = ((void *)-1); ((pg)->pageq.tqe_next) = ((void *)-1); } while (0); | ||||
| 860 | uvm_pagecopy(tpg, pg); | ||||
| 861 | KASSERT(tpg->wire_count == 1)((tpg->wire_count == 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 861, "tpg->wire_count == 1" )); | ||||
| 862 | tpg->wire_count = 0; | ||||
| 863 | uvm_lock_pageq()mtx_enter(&uvm.pageqlock); | ||||
| 864 | uvm_pagefree(tpg); | ||||
| 865 | uvm_unlock_pageq()mtx_leave(&uvm.pageqlock); | ||||
| 866 | uvm_pagealloc_pg(pg, obj, offset, NULL((void *)0)); | ||||
| 867 | } | ||||
| 868 | } | ||||
| 869 | return r; | ||||
| 870 | } | ||||
| 871 | |||||
| 872 | /* | ||||
| 873 | * uvm_pagealloc: allocate vm_page from a particular free list. | ||||
| 874 | * | ||||
| 875 | * => return null if no pages free | ||||
| 876 | * => wake up pagedaemon if number of free pages drops below low water mark | ||||
| 877 | * => only one of obj or anon can be non-null | ||||
| 878 | * => caller must activate/deactivate page if it is not wired. | ||||
| 879 | */ | ||||
| 880 | |||||
| 881 | struct vm_page * | ||||
| 882 | uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon, | ||||
| 883 | int flags) | ||||
| 884 | { | ||||
| 885 | struct vm_page *pg; | ||||
| 886 | struct pglist pgl; | ||||
| 887 | int pmr_flags; | ||||
| 888 | |||||
| 889 | KASSERT(obj == NULL || anon == NULL)((obj == ((void *)0) || anon == ((void *)0)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 889, "obj == NULL || anon == NULL" )); | ||||
| 890 | KASSERT(anon == NULL || off == 0)((anon == ((void *)0) || off == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 890, "anon == NULL || off == 0" )); | ||||
| 891 | KASSERT(off == trunc_page(off))((off == ((off) & ~((1 << 12) - 1))) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 891, "off == trunc_page(off)" )); | ||||
| 892 | KASSERT(obj == NULL || UVM_OBJ_IS_DUMMY(obj) ||((obj == ((void *)0) || (((obj)->pgops == &pmap_pager) || ((obj)->pgops == &bufcache_pager)) || rw_write_held (obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 893, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)" )) | ||||
| 893 | rw_write_held(obj->vmobjlock))((obj == ((void *)0) || (((obj)->pgops == &pmap_pager) || ((obj)->pgops == &bufcache_pager)) || rw_write_held (obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 893, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)" )); | ||||
| 894 | KASSERT(anon == NULL || anon->an_lock == NULL ||((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held (anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 895, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)" )) | ||||
| 895 | rw_write_held(anon->an_lock))((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held (anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 895, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)" )); | ||||
| 896 | |||||
| 897 | pmr_flags = UVM_PLA_NOWAIT0x0002; | ||||
| 898 | |||||
| 899 | /* | ||||
| 900 | * We're allowed to use the kernel reserve if the page is | ||||
| 901 | * being allocated to a kernel object. | ||||
| 902 | */ | ||||
| 903 | if ((flags & UVM_PGA_USERESERVE0x0001) || | ||||
| 904 | (obj != NULL((void *)0) && UVM_OBJ_IS_KERN_OBJECT(obj)((obj)->uo_refs == (-2)))) | ||||
| 905 | pmr_flags |= UVM_PLA_USERESERVE0x0040; | ||||
| 906 | |||||
| 907 | if (flags & UVM_PGA_ZERO0x0002) | ||||
| 908 | pmr_flags |= UVM_PLA_ZERO0x0004; | ||||
| 909 | TAILQ_INIT(&pgl)do { (&pgl)->tqh_first = ((void *)0); (&pgl)->tqh_last = &(&pgl)->tqh_first; } while (0); | ||||
| 910 | if (uvm_pmr_getpages(1, 0, 0, 1, 0, 1, pmr_flags, &pgl) != 0) | ||||
| 911 | goto fail; | ||||
| 912 | |||||
| 913 | pg = TAILQ_FIRST(&pgl)((&pgl)->tqh_first); | ||||
| 914 | KASSERT(pg != NULL && TAILQ_NEXT(pg, pageq) == NULL)((pg != ((void *)0) && ((pg)->pageq.tqe_next) == ( (void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 914, "pg != NULL && TAILQ_NEXT(pg, pageq) == NULL") ); | ||||
| 915 | |||||
| 916 | uvm_pagealloc_pg(pg, obj, off, anon); | ||||
| 917 | KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 917, "(pg->pg_flags & PG_DEV) == 0" )); | ||||
| 918 | if (flags & UVM_PGA_ZERO0x0002) | ||||
| 919 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008); | ||||
| 920 | else | ||||
| 921 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008); | ||||
| 922 | |||||
| 923 | return pg; | ||||
| 924 | |||||
| 925 | fail: | ||||
| 926 | return NULL((void *)0); | ||||
| 927 | } | ||||
| 928 | |||||
| 929 | /* | ||||
| 930 | * uvm_pagerealloc: reallocate a page from one object to another | ||||
| 931 | */ | ||||
| 932 | |||||
| 933 | void | ||||
| 934 | uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff) | ||||
| 935 | { | ||||
| 936 | |||||
| 937 | /* remove it from the old object */ | ||||
| 938 | if (pg->uobject) { | ||||
| 939 | uvm_pageremove(pg); | ||||
| 940 | } | ||||
| 941 | |||||
| 942 | /* put it in the new object */ | ||||
| 943 | if (newobj) { | ||||
| 944 | pg->uobject = newobj; | ||||
| 945 | pg->offset = newoff; | ||||
| 946 | pg->pg_version++; | ||||
| 947 | uvm_pageinsert(pg); | ||||
| 948 | } | ||||
| 949 | } | ||||
| 950 | |||||
| 951 | /* | ||||
| 952 | * uvm_pageclean: clean page | ||||
| 953 | * | ||||
| 954 | * => erase page's identity (i.e. remove from object) | ||||
| 955 | * => caller must lock page queues if `pg' is managed | ||||
| 956 | * => assumes all valid mappings of pg are gone | ||||
| 957 | */ | ||||
| 958 | void | ||||
| 959 | uvm_pageclean(struct vm_page *pg) | ||||
| 960 | { | ||||
| 961 | u_int flags_to_clear = 0; | ||||
| 962 | |||||
| 963 | if ((pg->pg_flags & (PG_TABLED0x00000004|PQ_ACTIVE0x00040000|PQ_INACTIVE0x00020000)) && | ||||
| 964 | (pg->uobject
| ||||
| 965 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
| 966 | |||||
| 967 | #ifdef DEBUG | ||||
| 968 | if (pg->uobject == (void *)0xdeadbeef && | ||||
| 969 | pg->uanon == (void *)0xdeadbeef) { | ||||
| 970 | panic("uvm_pagefree: freeing free page %p", pg); | ||||
| 971 | } | ||||
| 972 | #endif | ||||
| 973 | |||||
| 974 | KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 974, "(pg->pg_flags & PG_DEV) == 0" )); | ||||
| 975 | KASSERT(pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) ||((pg->uobject == ((void *)0) || (((pg->uobject)->pgops == &pmap_pager) || ((pg->uobject)->pgops == &bufcache_pager )) || rw_write_held(pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 976, "pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )) | ||||
| 976 | rw_write_held(pg->uobject->vmobjlock))((pg->uobject == ((void *)0) || (((pg->uobject)->pgops == &pmap_pager) || ((pg->uobject)->pgops == &bufcache_pager )) || rw_write_held(pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 976, "pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )); | ||||
| 977 | KASSERT(pg->uobject != NULL || pg->uanon == NULL ||((pg->uobject != ((void *)0) || pg->uanon == ((void *)0 ) || rw_write_held(pg->uanon->an_lock)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 978, "pg->uobject != NULL || pg->uanon == NULL || rw_write_held(pg->uanon->an_lock)" )) | ||||
| 978 | rw_write_held(pg->uanon->an_lock))((pg->uobject != ((void *)0) || pg->uanon == ((void *)0 ) || rw_write_held(pg->uanon->an_lock)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 978, "pg->uobject != NULL || pg->uanon == NULL || rw_write_held(pg->uanon->an_lock)" )); | ||||
| 979 | |||||
| 980 | /* | ||||
| 981 | * if the page was an object page (and thus "TABLED"), remove it | ||||
| 982 | * from the object. | ||||
| 983 | */ | ||||
| 984 | if (pg->pg_flags & PG_TABLED0x00000004) | ||||
| 985 | uvm_pageremove(pg); | ||||
| 986 | |||||
| 987 | /* | ||||
| 988 | * now remove the page from the queues | ||||
| 989 | */ | ||||
| 990 | uvm_pagedequeue(pg); | ||||
| 991 | |||||
| 992 | /* | ||||
| 993 | * if the page was wired, unwire it now. | ||||
| 994 | */ | ||||
| 995 | if (pg->wire_count) { | ||||
| 996 | pg->wire_count = 0; | ||||
| 997 | uvmexp.wired--; | ||||
| 998 | } | ||||
| 999 | if (pg->uanon) { | ||||
| 1000 | pg->uanon->an_page = NULL((void *)0); | ||||
| 1001 | pg->uanon = NULL((void *)0); | ||||
| 1002 | } | ||||
| 1003 | |||||
| 1004 | /* Clean page state bits. */ | ||||
| 1005 | flags_to_clear |= PQ_ANON0x00100000|PQ_AOBJ0x00200000|PQ_ENCRYPT0x00400000|PG_ZERO0x00000100|PG_FAKE0x00000040|PG_BUSY0x00000001| | ||||
| 1006 | PG_RELEASED0x00000020|PG_CLEAN0x00000008|PG_CLEANCHK0x00000010; | ||||
| 1007 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, flags_to_clear); | ||||
| 1008 | |||||
| 1009 | #ifdef DEBUG | ||||
| 1010 | pg->uobject = (void *)0xdeadbeef; | ||||
| 1011 | pg->offset = 0xdeadbeef; | ||||
| 1012 | pg->uanon = (void *)0xdeadbeef; | ||||
| 1013 | #endif | ||||
| 1014 | } | ||||
| 1015 | |||||
| 1016 | /* | ||||
| 1017 | * uvm_pagefree: free page | ||||
| 1018 | * | ||||
| 1019 | * => erase page's identity (i.e. remove from object) | ||||
| 1020 | * => put page on free list | ||||
| 1021 | * => caller must lock page queues if `pg' is managed | ||||
| 1022 | * => assumes all valid mappings of pg are gone | ||||
| 1023 | */ | ||||
| 1024 | void | ||||
| 1025 | uvm_pagefree(struct vm_page *pg) | ||||
| 1026 | { | ||||
| 1027 | if ((pg->pg_flags & (PG_TABLED0x00000004|PQ_ACTIVE0x00040000|PQ_INACTIVE0x00020000)) && | ||||
| 1028 | (pg->uobject == NULL((void *)0) || !UVM_OBJ_IS_PMAP(pg->uobject)((pg->uobject)->pgops == &pmap_pager))) | ||||
| 1029 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
| 1030 | |||||
| 1031 | uvm_pageclean(pg); | ||||
| 1032 | uvm_pmr_freepages(pg, 1); | ||||
| 1033 | } | ||||
| 1034 | |||||
| 1035 | /* | ||||
| 1036 | * uvm_page_unbusy: unbusy an array of pages. | ||||
| 1037 | * | ||||
| 1038 | * => pages must either all belong to the same object, or all belong to anons. | ||||
| 1039 | * => if pages are object-owned, object must be locked. | ||||
| 1040 | * => if pages are anon-owned, anons must have 0 refcount. | ||||
| 1041 | * => caller must make sure that anon-owned pages are not PG_RELEASED. | ||||
| 1042 | */ | ||||
| 1043 | void | ||||
| 1044 | uvm_page_unbusy(struct vm_page **pgs, int npgs) | ||||
| 1045 | { | ||||
| 1046 | struct vm_page *pg; | ||||
| 1047 | int i; | ||||
| 1048 | |||||
| 1049 | for (i = 0; i < npgs; i++) { | ||||
| |||||
| 1050 | pg = pgs[i]; | ||||
| 1051 | |||||
| 1052 | if (pg == NULL((void *)0) || pg == PGO_DONTCARE((struct vm_page *) -1L)) { | ||||
| 1053 | continue; | ||||
| 1054 | } | ||||
| 1055 | |||||
| 1056 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1056, "uvm_page_owner_locked_p(pg)" )); | ||||
| 1057 | KASSERT(pg->pg_flags & PG_BUSY)((pg->pg_flags & 0x00000001) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1057, "pg->pg_flags & PG_BUSY" )); | ||||
| 1058 | |||||
| 1059 | if (pg->pg_flags & PG_WANTED0x00000002) { | ||||
| 1060 | wakeup(pg); | ||||
| 1061 | } | ||||
| 1062 | if (pg->pg_flags & PG_RELEASED0x00000020) { | ||||
| 1063 | KASSERT(pg->uobject != NULL ||((pg->uobject != ((void *)0) || (pg->uanon != ((void *) 0) && pg->uanon->an_ref > 0)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1064, "pg->uobject != NULL || (pg->uanon != NULL && pg->uanon->an_ref > 0)" )) | ||||
| 1064 | (pg->uanon != NULL && pg->uanon->an_ref > 0))((pg->uobject != ((void *)0) || (pg->uanon != ((void *) 0) && pg->uanon->an_ref > 0)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1064, "pg->uobject != NULL || (pg->uanon != NULL && pg->uanon->an_ref > 0)" )); | ||||
| 1065 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_RELEASED0x00000020); | ||||
| 1066 | pmap_page_protect(pg, PROT_NONE0x00); | ||||
| 1067 | uvm_pagefree(pg); | ||||
| 1068 | } else { | ||||
| 1069 | KASSERT((pg->pg_flags & PG_FAKE) == 0)(((pg->pg_flags & 0x00000040) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1069, "(pg->pg_flags & PG_FAKE) == 0" )); | ||||
| 1070 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_WANTED0x00000002|PG_BUSY0x00000001); | ||||
| 1071 | UVM_PAGE_OWN(pg, NULL); | ||||
| 1072 | } | ||||
| 1073 | } | ||||
| 1074 | } | ||||
| 1075 | |||||
| 1076 | /* | ||||
| 1077 | * uvm_pagewait: wait for a busy page | ||||
| 1078 | * | ||||
| 1079 | * => page must be known PG_BUSY | ||||
| 1080 | * => object must be locked | ||||
| 1081 | * => object will be unlocked on return | ||||
| 1082 | */ | ||||
| 1083 | void | ||||
| 1084 | uvm_pagewait(struct vm_page *pg, struct rwlock *lock, const char *wmesg) | ||||
| 1085 | { | ||||
| 1086 | KASSERT(rw_lock_held(lock))((rw_lock_held(lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 1086, "rw_lock_held(lock)")); | ||||
| 1087 | KASSERT((pg->pg_flags & PG_BUSY) != 0)(((pg->pg_flags & 0x00000001) != 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1087, "(pg->pg_flags & PG_BUSY) != 0" )); | ||||
| 1088 | |||||
| 1089 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_WANTED0x00000002); | ||||
| 1090 | rwsleep_nsec(pg, lock, PVM4 | PNORELOCK0x200, wmesg, INFSLP0xffffffffffffffffULL); | ||||
| 1091 | } | ||||
| 1092 | |||||
| 1093 | #if defined(UVM_PAGE_TRKOWN) | ||||
| 1094 | /* | ||||
| 1095 | * uvm_page_own: set or release page ownership | ||||
| 1096 | * | ||||
| 1097 | * => this is a debugging function that keeps track of who sets PG_BUSY | ||||
| 1098 | * and where they do it. it can be used to track down problems | ||||
| 1099 | * such a thread setting "PG_BUSY" and never releasing it. | ||||
| 1100 | * => if "tag" is NULL then we are releasing page ownership | ||||
| 1101 | */ | ||||
| 1102 | void | ||||
| 1103 | uvm_page_own(struct vm_page *pg, char *tag) | ||||
| 1104 | { | ||||
| 1105 | /* gain ownership? */ | ||||
| 1106 | if (tag) { | ||||
| 1107 | if (pg->owner_tag) { | ||||
| 1108 | printf("uvm_page_own: page %p already owned " | ||||
| 1109 | "by thread %d [%s]\n", pg, | ||||
| 1110 | pg->owner, pg->owner_tag); | ||||
| 1111 | panic("uvm_page_own"); | ||||
| 1112 | } | ||||
| 1113 | pg->owner = (curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc) ? curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_tid : (pid_t) -1; | ||||
| 1114 | pg->owner_tag = tag; | ||||
| 1115 | return; | ||||
| 1116 | } | ||||
| 1117 | |||||
| 1118 | /* drop ownership */ | ||||
| 1119 | if (pg->owner_tag == NULL((void *)0)) { | ||||
| 1120 | printf("uvm_page_own: dropping ownership of an non-owned " | ||||
| 1121 | "page (%p)\n", pg); | ||||
| 1122 | panic("uvm_page_own"); | ||||
| 1123 | } | ||||
| 1124 | pg->owner_tag = NULL((void *)0); | ||||
| 1125 | return; | ||||
| 1126 | } | ||||
| 1127 | #endif | ||||
| 1128 | |||||
| 1129 | /* | ||||
| 1130 | * when VM_PHYSSEG_MAX is 1, we can simplify these functions | ||||
| 1131 | */ | ||||
| 1132 | |||||
| 1133 | #if VM_PHYSSEG_MAX16 > 1 | ||||
| 1134 | /* | ||||
| 1135 | * vm_physseg_find: find vm_physseg structure that belongs to a PA | ||||
| 1136 | */ | ||||
| 1137 | int | ||||
| 1138 | vm_physseg_find(paddr_t pframe, int *offp) | ||||
| 1139 | { | ||||
| 1140 | struct vm_physseg *seg; | ||||
| 1141 | |||||
| 1142 | #if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2) | ||||
| 1143 | /* binary search for it */ | ||||
| 1144 | int start, len, try; | ||||
| 1145 | |||||
| 1146 | /* | ||||
| 1147 | * if try is too large (thus target is less than try) we reduce | ||||
| 1148 | * the length to trunc(len/2) [i.e. everything smaller than "try"] | ||||
| 1149 | * | ||||
| 1150 | * if the try is too small (thus target is greater than try) then | ||||
| 1151 | * we set the new start to be (try + 1). this means we need to | ||||
| 1152 | * reduce the length to (round(len/2) - 1). | ||||
| 1153 | * | ||||
| 1154 | * note "adjust" below which takes advantage of the fact that | ||||
| 1155 | * (round(len/2) - 1) == trunc((len - 1) / 2) | ||||
| 1156 | * for any value of len we may have | ||||
| 1157 | */ | ||||
| 1158 | |||||
| 1159 | for (start = 0, len = vm_nphysseg ; len != 0 ; len = len / 2) { | ||||
| 1160 | try = start + (len / 2); /* try in the middle */ | ||||
| 1161 | seg = vm_physmem + try; | ||||
| 1162 | |||||
| 1163 | /* start past our try? */ | ||||
| 1164 | if (pframe >= seg->start) { | ||||
| 1165 | /* was try correct? */ | ||||
| 1166 | if (pframe < seg->end) { | ||||
| 1167 | if (offp) | ||||
| 1168 | *offp = pframe - seg->start; | ||||
| 1169 | return try; /* got it */ | ||||
| 1170 | } | ||||
| 1171 | start = try + 1; /* next time, start here */ | ||||
| 1172 | len--; /* "adjust" */ | ||||
| 1173 | } else { | ||||
| 1174 | /* | ||||
| 1175 | * pframe before try, just reduce length of | ||||
| 1176 | * region, done in "for" loop | ||||
| 1177 | */ | ||||
| 1178 | } | ||||
| 1179 | } | ||||
| 1180 | return -1; | ||||
| 1181 | |||||
| 1182 | #else | ||||
| 1183 | /* linear search for it */ | ||||
| 1184 | int lcv; | ||||
| 1185 | |||||
| 1186 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) { | ||||
| 1187 | if (pframe >= seg->start && pframe < seg->end) { | ||||
| 1188 | if (offp) | ||||
| 1189 | *offp = pframe - seg->start; | ||||
| 1190 | return lcv; /* got it */ | ||||
| 1191 | } | ||||
| 1192 | } | ||||
| 1193 | return -1; | ||||
| 1194 | |||||
| 1195 | #endif | ||||
| 1196 | } | ||||
| 1197 | |||||
| 1198 | /* | ||||
| 1199 | * PHYS_TO_VM_PAGE: find vm_page for a PA. used by MI code to get vm_pages | ||||
| 1200 | * back from an I/O mapping (ugh!). used in some MD code as well. | ||||
| 1201 | */ | ||||
| 1202 | struct vm_page * | ||||
| 1203 | PHYS_TO_VM_PAGE(paddr_t pa) | ||||
| 1204 | { | ||||
| 1205 | paddr_t pf = atop(pa)((pa) >> 12); | ||||
| 1206 | int off; | ||||
| 1207 | int psi; | ||||
| 1208 | |||||
| 1209 | psi = vm_physseg_find(pf, &off); | ||||
| 1210 | |||||
| 1211 | return (psi == -1) ? NULL((void *)0) : &vm_physmem[psi].pgs[off]; | ||||
| 1212 | } | ||||
| 1213 | #endif /* VM_PHYSSEG_MAX > 1 */ | ||||
| 1214 | |||||
| 1215 | /* | ||||
| 1216 | * uvm_pagelookup: look up a page | ||||
| 1217 | */ | ||||
| 1218 | struct vm_page * | ||||
| 1219 | uvm_pagelookup(struct uvm_object *obj, voff_t off) | ||||
| 1220 | { | ||||
| 1221 | /* XXX if stack is too much, handroll */ | ||||
| 1222 | struct vm_page p, *pg; | ||||
| 1223 | |||||
| 1224 | p.offset = off; | ||||
| 1225 | pg = RBT_FIND(uvm_objtree, &obj->memt, &p)uvm_objtree_RBT_FIND(&obj->memt, &p); | ||||
| 1226 | |||||
| 1227 | KASSERT(pg == NULL || obj->uo_npages != 0)((pg == ((void *)0) || obj->uo_npages != 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1227, "pg == NULL || obj->uo_npages != 0" )); | ||||
| 1228 | KASSERT(pg == NULL || (pg->pg_flags & PG_RELEASED) == 0 ||((pg == ((void *)0) || (pg->pg_flags & 0x00000020) == 0 || (pg->pg_flags & 0x00000001) != 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1229, "pg == NULL || (pg->pg_flags & PG_RELEASED) == 0 || (pg->pg_flags & PG_BUSY) != 0" )) | ||||
| 1229 | (pg->pg_flags & PG_BUSY) != 0)((pg == ((void *)0) || (pg->pg_flags & 0x00000020) == 0 || (pg->pg_flags & 0x00000001) != 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1229, "pg == NULL || (pg->pg_flags & PG_RELEASED) == 0 || (pg->pg_flags & PG_BUSY) != 0" )); | ||||
| 1230 | return (pg); | ||||
| 1231 | } | ||||
| 1232 | |||||
| 1233 | /* | ||||
| 1234 | * uvm_pagewire: wire the page, thus removing it from the daemon's grasp | ||||
| 1235 | * | ||||
| 1236 | * => caller must lock page queues | ||||
| 1237 | */ | ||||
| 1238 | void | ||||
| 1239 | uvm_pagewire(struct vm_page *pg) | ||||
| 1240 | { | ||||
| 1241 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1241, "uvm_page_owner_locked_p(pg)" )); | ||||
| 1242 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
| 1243 | |||||
| 1244 | if (pg->wire_count == 0) { | ||||
| 1245 | uvm_pagedequeue(pg); | ||||
| 1246 | uvmexp.wired++; | ||||
| 1247 | } | ||||
| 1248 | pg->wire_count++; | ||||
| 1249 | } | ||||
| 1250 | |||||
| 1251 | /* | ||||
| 1252 | * uvm_pageunwire: unwire the page. | ||||
| 1253 | * | ||||
| 1254 | * => activate if wire count goes to zero. | ||||
| 1255 | * => caller must lock page queues | ||||
| 1256 | */ | ||||
| 1257 | void | ||||
| 1258 | uvm_pageunwire(struct vm_page *pg) | ||||
| 1259 | { | ||||
| 1260 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1260, "uvm_page_owner_locked_p(pg)" )); | ||||
| 1261 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
| 1262 | |||||
| 1263 | pg->wire_count--; | ||||
| 1264 | if (pg->wire_count == 0) { | ||||
| 1265 | uvm_pageactivate(pg); | ||||
| 1266 | uvmexp.wired--; | ||||
| 1267 | } | ||||
| 1268 | } | ||||
| 1269 | |||||
| 1270 | /* | ||||
| 1271 | * uvm_pagedeactivate: deactivate page -- no pmaps have access to page | ||||
| 1272 | * | ||||
| 1273 | * => caller must lock page queues | ||||
| 1274 | * => caller must check to make sure page is not wired | ||||
| 1275 | * => object that page belongs to must be locked (so we can adjust pg->flags) | ||||
| 1276 | */ | ||||
| 1277 | void | ||||
| 1278 | uvm_pagedeactivate(struct vm_page *pg) | ||||
| 1279 | { | ||||
| 1280 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1280, "uvm_page_owner_locked_p(pg)" )); | ||||
| 1281 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
| 1282 | |||||
| 1283 | if (pg->pg_flags & PQ_ACTIVE0x00040000) { | ||||
| 1284 | TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
| 1285 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000); | ||||
| 1286 | uvmexp.active--; | ||||
| 1287 | } | ||||
| 1288 | if ((pg->pg_flags & PQ_INACTIVE0x00020000) == 0) { | ||||
| 1289 | KASSERT(pg->wire_count == 0)((pg->wire_count == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1289, "pg->wire_count == 0" )); | ||||
| 1290 | TAILQ_INSERT_TAIL(&uvm.page_inactive, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev = (&uvm.page_inactive)->tqh_last; *(&uvm.page_inactive )->tqh_last = (pg); (&uvm.page_inactive)->tqh_last = &(pg)->pageq.tqe_next; } while (0); | ||||
| 1291 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000); | ||||
| 1292 | uvmexp.inactive++; | ||||
| 1293 | pmap_clear_reference(pg)pmap_clear_attrs(pg, 0x0000000000000020UL); | ||||
| 1294 | /* | ||||
| 1295 | * update the "clean" bit. this isn't 100% | ||||
| 1296 | * accurate, and doesn't have to be. we'll | ||||
| 1297 | * re-sync it after we zap all mappings when | ||||
| 1298 | * scanning the inactive list. | ||||
| 1299 | */ | ||||
| 1300 | if ((pg->pg_flags & PG_CLEAN0x00000008) != 0 && | ||||
| 1301 | pmap_is_modified(pg)pmap_test_attrs(pg, 0x0000000000000040UL)) | ||||
| 1302 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008); | ||||
| 1303 | } | ||||
| 1304 | } | ||||
| 1305 | |||||
| 1306 | /* | ||||
| 1307 | * uvm_pageactivate: activate page | ||||
| 1308 | * | ||||
| 1309 | * => caller must lock page queues | ||||
| 1310 | */ | ||||
| 1311 | void | ||||
| 1312 | uvm_pageactivate(struct vm_page *pg) | ||||
| 1313 | { | ||||
| 1314 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1314, "uvm_page_owner_locked_p(pg)" )); | ||||
| 1315 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
| 1316 | |||||
| 1317 | uvm_pagedequeue(pg); | ||||
| 1318 | if (pg->wire_count == 0) { | ||||
| 1319 | TAILQ_INSERT_TAIL(&uvm.page_active, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev = (&uvm.page_active)->tqh_last; *(&uvm.page_active )->tqh_last = (pg); (&uvm.page_active)->tqh_last = & (pg)->pageq.tqe_next; } while (0); | ||||
| 1320 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000); | ||||
| 1321 | uvmexp.active++; | ||||
| 1322 | |||||
| 1323 | } | ||||
| 1324 | } | ||||
| 1325 | |||||
| 1326 | /* | ||||
| 1327 | * uvm_pagedequeue: remove a page from any paging queue | ||||
| 1328 | */ | ||||
| 1329 | void | ||||
| 1330 | uvm_pagedequeue(struct vm_page *pg) | ||||
| 1331 | { | ||||
| 1332 | if (pg->pg_flags & PQ_ACTIVE0x00040000) { | ||||
| 1333 | TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
| 1334 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000); | ||||
| 1335 | uvmexp.active--; | ||||
| 1336 | } | ||||
| 1337 | if (pg->pg_flags & PQ_INACTIVE0x00020000) { | ||||
| 1338 | TAILQ_REMOVE(&uvm.page_inactive, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_inactive)->tqh_last = (pg)->pageq.tqe_prev ; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
| 1339 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000); | ||||
| 1340 | uvmexp.inactive--; | ||||
| 1341 | } | ||||
| 1342 | } | ||||
| 1343 | /* | ||||
| 1344 | * uvm_pagezero: zero fill a page | ||||
| 1345 | */ | ||||
| 1346 | void | ||||
| 1347 | uvm_pagezero(struct vm_page *pg) | ||||
| 1348 | { | ||||
| 1349 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008); | ||||
| 1350 | pmap_zero_page(pg); | ||||
| 1351 | } | ||||
| 1352 | |||||
| 1353 | /* | ||||
| 1354 | * uvm_pagecopy: copy a page | ||||
| 1355 | */ | ||||
| 1356 | void | ||||
| 1357 | uvm_pagecopy(struct vm_page *src, struct vm_page *dst) | ||||
| 1358 | { | ||||
| 1359 | atomic_clearbits_intx86_atomic_clearbits_u32(&dst->pg_flags, PG_CLEAN0x00000008); | ||||
| 1360 | pmap_copy_page(src, dst); | ||||
| 1361 | } | ||||
| 1362 | |||||
| 1363 | /* | ||||
| 1364 | * uvm_page_owner_locked_p: return true if object associated with page is | ||||
| 1365 | * locked. this is a weak check for runtime assertions only. | ||||
| 1366 | */ | ||||
| 1367 | int | ||||
| 1368 | uvm_page_owner_locked_p(struct vm_page *pg) | ||||
| 1369 | { | ||||
| 1370 | if (pg->uobject != NULL((void *)0)) { | ||||
| 1371 | if (UVM_OBJ_IS_DUMMY(pg->uobject)(((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager))) | ||||
| 1372 | return 1; | ||||
| 1373 | return rw_write_held(pg->uobject->vmobjlock); | ||||
| 1374 | } | ||||
| 1375 | if (pg->uanon != NULL((void *)0)) { | ||||
| 1376 | return rw_write_held(pg->uanon->an_lock); | ||||
| 1377 | } | ||||
| 1378 | return 1; | ||||
| 1379 | } | ||||
| 1380 | |||||
| 1381 | /* | ||||
| 1382 | * uvm_pagecount: count the number of physical pages in the address range. | ||||
| 1383 | */ | ||||
| 1384 | psize_t | ||||
| 1385 | uvm_pagecount(struct uvm_constraint_range* constraint) | ||||
| 1386 | { | ||||
| 1387 | int lcv; | ||||
| 1388 | psize_t sz; | ||||
| 1389 | paddr_t low, high; | ||||
| 1390 | paddr_t ps_low, ps_high; | ||||
| 1391 | |||||
| 1392 | /* Algorithm uses page numbers. */ | ||||
| 1393 | low = atop(constraint->ucr_low)((constraint->ucr_low) >> 12); | ||||
| 1394 | high = atop(constraint->ucr_high)((constraint->ucr_high) >> 12); | ||||
| 1395 | |||||
| 1396 | sz = 0; | ||||
| 1397 | for (lcv = 0; lcv < vm_nphysseg; lcv++) { | ||||
| 1398 | ps_low = MAX(low, vm_physmem[lcv].avail_start)(((low)>(vm_physmem[lcv].avail_start))?(low):(vm_physmem[lcv ].avail_start)); | ||||
| 1399 | ps_high = MIN(high, vm_physmem[lcv].avail_end)(((high)<(vm_physmem[lcv].avail_end))?(high):(vm_physmem[lcv ].avail_end)); | ||||
| 1400 | if (ps_low < ps_high) | ||||
| 1401 | sz += ps_high - ps_low; | ||||
| 1402 | } | ||||
| 1403 | return sz; | ||||
| 1404 | } |
| 1 | /* $OpenBSD: pmap.h,v 1.88 2023/12/29 13:23:28 jca Exp $ */ | ||||
| 2 | /* $NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $ */ | ||||
| 3 | |||||
| 4 | /* | ||||
| 5 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | ||||
| 6 | * All rights reserved. | ||||
| 7 | * | ||||
| 8 | * Redistribution and use in source and binary forms, with or without | ||||
| 9 | * modification, are permitted provided that the following conditions | ||||
| 10 | * are met: | ||||
| 11 | * 1. Redistributions of source code must retain the above copyright | ||||
| 12 | * notice, this list of conditions and the following disclaimer. | ||||
| 13 | * 2. Redistributions in binary form must reproduce the above copyright | ||||
| 14 | * notice, this list of conditions and the following disclaimer in the | ||||
| 15 | * documentation and/or other materials provided with the distribution. | ||||
| 16 | * | ||||
| 17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | ||||
| 18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | ||||
| 19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | ||||
| 20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | ||||
| 21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | ||||
| 22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | ||||
| 23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | ||||
| 24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | ||||
| 25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | ||||
| 26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | ||||
| 27 | */ | ||||
| 28 | |||||
| 29 | /* | ||||
| 30 | * Copyright (c) 2001 Wasabi Systems, Inc. | ||||
| 31 | * All rights reserved. | ||||
| 32 | * | ||||
| 33 | * Written by Frank van der Linden for Wasabi Systems, Inc. | ||||
| 34 | * | ||||
| 35 | * Redistribution and use in source and binary forms, with or without | ||||
| 36 | * modification, are permitted provided that the following conditions | ||||
| 37 | * are met: | ||||
| 38 | * 1. Redistributions of source code must retain the above copyright | ||||
| 39 | * notice, this list of conditions and the following disclaimer. | ||||
| 40 | * 2. Redistributions in binary form must reproduce the above copyright | ||||
| 41 | * notice, this list of conditions and the following disclaimer in the | ||||
| 42 | * documentation and/or other materials provided with the distribution. | ||||
| 43 | * 3. All advertising materials mentioning features or use of this software | ||||
| 44 | * must display the following acknowledgement: | ||||
| 45 | * This product includes software developed for the NetBSD Project by | ||||
| 46 | * Wasabi Systems, Inc. | ||||
| 47 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse | ||||
| 48 | * or promote products derived from this software without specific prior | ||||
| 49 | * written permission. | ||||
| 50 | * | ||||
| 51 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND | ||||
| 52 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED | ||||
| 53 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR | ||||
| 54 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC | ||||
| 55 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||||
| 56 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||||
| 57 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||||
| 58 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||||
| 59 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||||
| 60 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||||
| 61 | * POSSIBILITY OF SUCH DAMAGE. | ||||
| 62 | */ | ||||
| 63 | |||||
| 64 | /* | ||||
| 65 | * pmap.h: see pmap.c for the history of this pmap module. | ||||
| 66 | */ | ||||
| 67 | |||||
| 68 | #ifndef _MACHINE_PMAP_H_ | ||||
| 69 | #define _MACHINE_PMAP_H_ | ||||
| 70 | |||||
| 71 | #ifndef _LOCORE | ||||
| 72 | #ifdef _KERNEL1 | ||||
| 73 | #include <lib/libkern/libkern.h> /* for KASSERT() */ | ||||
| 74 | #include <machine/cpufunc.h> | ||||
| 75 | #endif /* _KERNEL */ | ||||
| 76 | #include <sys/mutex.h> | ||||
| 77 | #include <uvm/uvm_object.h> | ||||
| 78 | #include <machine/pte.h> | ||||
| 79 | #endif | ||||
| 80 | |||||
| 81 | /* | ||||
| 82 | * The x86_64 pmap module closely resembles the i386 one. It uses | ||||
| 83 | * the same recursive entry scheme. See the i386 pmap.h for a | ||||
| 84 | * description. The alternate area trick for accessing non-current | ||||
| 85 | * pmaps has been removed, though, because it performs badly on SMP | ||||
| 86 | * systems. | ||||
| 87 | * The most obvious difference to i386 is that 2 extra levels of page | ||||
| 88 | * table need to be dealt with. The level 1 page table pages are at: | ||||
| 89 | * | ||||
| 90 | * l1: 0x00007f8000000000 - 0x00007fffffffffff (39 bits, needs PML4 entry) | ||||
| 91 | * | ||||
| 92 | * The other levels are kept as physical pages in 3 UVM objects and are | ||||
| 93 | * temporarily mapped for virtual access when needed. | ||||
| 94 | * | ||||
| 95 | * The other obvious difference from i386 is that it has a direct map of all | ||||
| 96 | * physical memory in the VA range: | ||||
| 97 | * | ||||
| 98 | * 0xfffffd8000000000 - 0xffffff7fffffffff | ||||
| 99 | * | ||||
| 100 | * The direct map is used in some cases to access PTEs of non-current pmaps. | ||||
| 101 | * | ||||
| 102 | * Note that address space is signed, so the layout for 48 bits is: | ||||
| 103 | * | ||||
| 104 | * +---------------------------------+ 0xffffffffffffffff | ||||
| 105 | * | Kernel Image | | ||||
| 106 | * +---------------------------------+ 0xffffff8000000000 | ||||
| 107 | * | Direct Map | | ||||
| 108 | * +---------------------------------+ 0xfffffd8000000000 | ||||
| 109 | * ~ ~ | ||||
| 110 | * | | | ||||
| 111 | * | Kernel Space | | ||||
| 112 | * | | | ||||
| 113 | * | | | ||||
| 114 | * +---------------------------------+ 0xffff800000000000 = 0x0000800000000000 | ||||
| 115 | * | L1 table (PTE pages) | | ||||
| 116 | * +---------------------------------+ 0x00007f8000000000 | ||||
| 117 | * ~ ~ | ||||
| 118 | * | | | ||||
| 119 | * | User Space | | ||||
| 120 | * | | | ||||
| 121 | * | | | ||||
| 122 | * +---------------------------------+ 0x0000000000000000 | ||||
| 123 | * | ||||
| 124 | * In other words, there is a 'VA hole' at 0x0000800000000000 - | ||||
| 125 | * 0xffff800000000000 which will trap, just as on, for example, | ||||
| 126 | * sparcv9. | ||||
| 127 | * | ||||
| 128 | * The unused space can be used if needed, but it adds a little more | ||||
| 129 | * complexity to the calculations. | ||||
| 130 | */ | ||||
| 131 | |||||
| 132 | /* | ||||
| 133 | * Mask to get rid of the sign-extended part of addresses. | ||||
| 134 | */ | ||||
| 135 | #define VA_SIGN_MASK0xffff000000000000 0xffff000000000000 | ||||
| 136 | #define VA_SIGN_NEG(va)((va) | 0xffff000000000000) ((va) | VA_SIGN_MASK0xffff000000000000) | ||||
| 137 | /* | ||||
| 138 | * XXXfvdl this one's not right. | ||||
| 139 | */ | ||||
| 140 | #define VA_SIGN_POS(va)((va) & ~0xffff000000000000) ((va) & ~VA_SIGN_MASK0xffff000000000000) | ||||
| 141 | |||||
| 142 | #define L4_SLOT_PTE255 255 | ||||
| 143 | #define L4_SLOT_KERN256 256 | ||||
| 144 | #define L4_SLOT_KERNBASE511 511 | ||||
| 145 | #define NUM_L4_SLOT_DIRECT4 4 | ||||
| 146 | #define L4_SLOT_DIRECT(511 - 4) (L4_SLOT_KERNBASE511 - NUM_L4_SLOT_DIRECT4) | ||||
| 147 | #define L4_SLOT_EARLY((511 - 4) - 1) (L4_SLOT_DIRECT(511 - 4) - 1) | ||||
| 148 | |||||
| 149 | #define PDIR_SLOT_KERN256 L4_SLOT_KERN256 | ||||
| 150 | #define PDIR_SLOT_PTE255 L4_SLOT_PTE255 | ||||
| 151 | #define PDIR_SLOT_DIRECT(511 - 4) L4_SLOT_DIRECT(511 - 4) | ||||
| 152 | #define PDIR_SLOT_EARLY((511 - 4) - 1) L4_SLOT_EARLY((511 - 4) - 1) | ||||
| 153 | |||||
| 154 | /* | ||||
| 155 | * the following defines give the virtual addresses of various MMU | ||||
| 156 | * data structures: | ||||
| 157 | * PTE_BASE: the base VA of the linear PTE mappings | ||||
| 158 | * PDP_PDE: the VA of the PDE that points back to the PDP | ||||
| 159 | * | ||||
| 160 | */ | ||||
| 161 | |||||
| 162 | #define PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) ((pt_entry_t *) (L4_SLOT_PTE255 * NBPD_L4(1ULL << 39))) | ||||
| 163 | #define PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) (VA_SIGN_NEG((L4_SLOT_DIRECT * NBPD_L4))((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) | ||||
| 164 | #define PMAP_DIRECT_END((((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000 )) (VA_SIGN_NEG(((L4_SLOT_DIRECT + \(((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000 ) | ||||
| 165 | NUM_L4_SLOT_DIRECT) * NBPD_L4))(((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000 )) | ||||
| 166 | |||||
| 167 | #define L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) | ||||
| 168 | |||||
| 169 | #define L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) ((pd_entry_t *)((char *)L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) + L4_SLOT_PTE255 * NBPD_L3(1ULL << 30))) | ||||
| 170 | #define L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t * ) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) ((pd_entry_t *)((char *)L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + L4_SLOT_PTE255 * NBPD_L2(1ULL << 21))) | ||||
| 171 | #define L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) ((pd_entry_t *)((char *)L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t * ) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + L4_SLOT_PTE255 * NBPD_L1(1ULL << 12))) | ||||
| 172 | |||||
| 173 | #define PDP_PDE(((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * ( 1ULL << 12))) + 255) (L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) + PDIR_SLOT_PTE255) | ||||
| 174 | |||||
| 175 | #define PDP_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) | ||||
| 176 | |||||
| 177 | #define NKL4_MAX_ENTRIES(unsigned long)1 (unsigned long)1 | ||||
| 178 | #define NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512) (unsigned long)(NKL4_MAX_ENTRIES(unsigned long)1 * 512) | ||||
| 179 | #define NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512 ) (unsigned long)(NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512) * 512) | ||||
| 180 | #define NKL1_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)((unsigned long )1 * 512) * 512) * 512) (unsigned long)(NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512 ) * 512) | ||||
| 181 | |||||
| 182 | #define NKL4_KIMG_ENTRIES1 1 | ||||
| 183 | #define NKL3_KIMG_ENTRIES1 1 | ||||
| 184 | #define NKL2_KIMG_ENTRIES64 64 | ||||
| 185 | |||||
| 186 | /* number of pages of direct map entries set up by locore0.S */ | ||||
| 187 | #define NDML4_ENTRIES1 1 | ||||
| 188 | #define NDML3_ENTRIES1 1 | ||||
| 189 | #define NDML2_ENTRIES4 4 /* 4GB */ | ||||
| 190 | |||||
| 191 | /* | ||||
| 192 | * Since kva space is below the kernel in its entirety, we start off | ||||
| 193 | * with zero entries on each level. | ||||
| 194 | */ | ||||
| 195 | #define NKL4_START_ENTRIES0 0 | ||||
| 196 | #define NKL3_START_ENTRIES0 0 | ||||
| 197 | #define NKL2_START_ENTRIES0 0 | ||||
| 198 | #define NKL1_START_ENTRIES0 0 /* XXX */ | ||||
| 199 | |||||
| 200 | #define NTOPLEVEL_PDES((1 << 12) / (sizeof (pd_entry_t))) (PAGE_SIZE(1 << 12) / (sizeof (pd_entry_t))) | ||||
| 201 | |||||
| 202 | #define NPDPG((1 << 12) / sizeof (pd_entry_t)) (PAGE_SIZE(1 << 12) / sizeof (pd_entry_t)) | ||||
| 203 | |||||
| 204 | /* | ||||
| 205 | * pl*_pi: index in the ptp page for a pde mapping a VA. | ||||
| 206 | * (pl*_i below is the index in the virtual array of all pdes per level) | ||||
| 207 | */ | ||||
| 208 | #define pl1_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x00000000001ff000UL ) >> 12) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12) | ||||
| 209 | #define pl2_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x000000003fe00000UL ) >> 21) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21) | ||||
| 210 | #define pl3_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x0000007fc0000000UL ) >> 30) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30) | ||||
| 211 | #define pl4_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x0000ff8000000000UL ) >> 39) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39) | ||||
| 212 | |||||
| 213 | /* | ||||
| 214 | * pl*_i: generate index into pde/pte arrays in virtual space | ||||
| 215 | */ | ||||
| 216 | #define pl1_i(VA)(((((VA) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L1_FRAME(((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )|0x00000000001ff000UL)) >> L1_SHIFT12) | ||||
| 217 | #define pl2_i(VA)(((((VA) & ~0xffff000000000000)) & ((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)) >> 21) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )) >> L2_SHIFT21) | ||||
| 218 | #define pl3_i(VA)(((((VA) & ~0xffff000000000000)) & (0x0000ff8000000000UL |0x0000007fc0000000UL)) >> 30) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L3_FRAME(0x0000ff8000000000UL|0x0000007fc0000000UL)) >> L3_SHIFT30) | ||||
| 219 | #define pl4_i(VA)(((((VA) & ~0xffff000000000000)) & 0x0000ff8000000000UL ) >> 39) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L4_FRAME0x0000ff8000000000UL) >> L4_SHIFT39) | ||||
| 220 | #define pl_i(va, lvl)(((((va) & ~0xffff000000000000)) & ptp_masks[(lvl)-1] ) >> ptp_shifts[(lvl)-1]) \ | ||||
| 221 | (((VA_SIGN_POS(va)((va) & ~0xffff000000000000)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1]) | ||||
| 222 | |||||
| 223 | #define PTP_MASK_INITIALIZER{ (((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )|0x00000000001ff000UL), ((0x0000ff8000000000UL|0x0000007fc0000000UL )|0x000000003fe00000UL), (0x0000ff8000000000UL|0x0000007fc0000000UL ), 0x0000ff8000000000UL } { L1_FRAME(((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )|0x00000000001ff000UL), L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL ), L3_FRAME(0x0000ff8000000000UL|0x0000007fc0000000UL), L4_FRAME0x0000ff8000000000UL } | ||||
| 224 | #define PTP_SHIFT_INITIALIZER{ 12, 21, 30, 39 } { L1_SHIFT12, L2_SHIFT21, L3_SHIFT30, L4_SHIFT39 } | ||||
| 225 | #define NKPTP_INITIALIZER{ 0, 0, 0, 0 } { NKL1_START_ENTRIES0, NKL2_START_ENTRIES0, \ | ||||
| 226 | NKL3_START_ENTRIES0, NKL4_START_ENTRIES0 } | ||||
| 227 | #define NKPTPMAX_INITIALIZER{ (unsigned long)((unsigned long)((unsigned long)((unsigned long )1 * 512) * 512) * 512), (unsigned long)((unsigned long)((unsigned long)1 * 512) * 512), (unsigned long)((unsigned long)1 * 512 ), (unsigned long)1 } { NKL1_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)((unsigned long )1 * 512) * 512) * 512), NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512 ), \ | ||||
| 228 | NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512), NKL4_MAX_ENTRIES(unsigned long)1 } | ||||
| 229 | #define NBPD_INITIALIZER{ (1ULL << 12), (1ULL << 21), (1ULL << 30), (1ULL << 39) } { NBPD_L1(1ULL << 12), NBPD_L2(1ULL << 21), NBPD_L3(1ULL << 30), NBPD_L4(1ULL << 39) } | ||||
| 230 | #define PDES_INITIALIZER{ ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))), ((pd_entry_t *)((char *) ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21 ))), ((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * ( 1ULL << 12))) } { L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))), L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t * ) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))), L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) } | ||||
| 231 | |||||
| 232 | /* | ||||
| 233 | * PTP macros: | ||||
| 234 | * a PTP's index is the PD index of the PDE that points to it | ||||
| 235 | * a PTP's offset is the byte-offset in the PTE space that this PTP is at | ||||
| 236 | * a PTP's VA is the first VA mapped by that PTP | ||||
| 237 | */ | ||||
| 238 | |||||
| 239 | #define ptp_va2o(va, lvl)((((((va) & ~0xffff000000000000)) & ptp_masks[((lvl)+ 1)-1]) >> ptp_shifts[((lvl)+1)-1]) * (1 << 12)) (pl_i(va, (lvl)+1)(((((va) & ~0xffff000000000000)) & ptp_masks[((lvl)+1 )-1]) >> ptp_shifts[((lvl)+1)-1]) * PAGE_SIZE(1 << 12)) | ||||
| 240 | |||||
| 241 | #define PTP_LEVELS4 4 | ||||
| 242 | |||||
| 243 | /* | ||||
| 244 | * PG_AVAIL usage: we make use of the ignored bits of the PTE | ||||
| 245 | */ | ||||
| 246 | |||||
| 247 | #define PG_W0x0000000000000200UL PG_AVAIL10x0000000000000200UL /* "wired" mapping */ | ||||
| 248 | #define PG_PVLIST0x0000000000000400UL PG_AVAIL20x0000000000000400UL /* mapping has entry on pvlist */ | ||||
| 249 | /* PG_AVAIL3 not used */ | ||||
| 250 | |||||
| 251 | /* | ||||
| 252 | * PCID assignments. | ||||
| 253 | * The shootdown code assumes KERN, PROC, and PROC_INTEL are both | ||||
| 254 | * consecutive and in that order. | ||||
| 255 | */ | ||||
| 256 | #define PCID_KERN0 0 /* for pmap_kernel() */ | ||||
| 257 | #define PCID_PROC1 1 /* non-pmap_kernel(), U+K */ | ||||
| 258 | #define PCID_PROC_INTEL2 2 /* non-pmap_kernel(), U-K (meltdown) */ | ||||
| 259 | #define PCID_TEMP3 3 /* temp mapping of another non-pmap_kernel() */ | ||||
| 260 | #define PCID_EFI4 4 /* EFI runtime services */ | ||||
| 261 | |||||
| 262 | extern int pmap_use_pcid; /* non-zero if PCID support is enabled */ | ||||
| 263 | |||||
| 264 | /* | ||||
| 265 | * Number of PTEs per cache line. 8 byte pte, 64-byte cache line | ||||
| 266 | * Used to avoid false sharing of cache lines. | ||||
| 267 | */ | ||||
| 268 | #define NPTECL8 8 | ||||
| 269 | |||||
| 270 | |||||
| 271 | #if defined(_KERNEL1) && !defined(_LOCORE) | ||||
| 272 | /* | ||||
| 273 | * pmap data structures: see pmap.c for details of locking. | ||||
| 274 | */ | ||||
| 275 | |||||
| 276 | struct pmap; | ||||
| 277 | typedef struct pmap *pmap_t; | ||||
| 278 | |||||
| 279 | /* | ||||
| 280 | * we maintain a list of all non-kernel pmaps | ||||
| 281 | */ | ||||
| 282 | |||||
| 283 | LIST_HEAD(pmap_head, pmap)struct pmap_head { struct pmap *lh_first; }; /* struct pmap_head: head of a pmap list */ | ||||
| 284 | |||||
| 285 | /* | ||||
| 286 | * the pmap structure | ||||
| 287 | * | ||||
| 288 | * note that the pm_obj contains the reference count, | ||||
| 289 | * page list, and number of PTPs within the pmap. | ||||
| 290 | */ | ||||
| 291 | |||||
| 292 | #define PMAP_TYPE_NORMAL1 1 | ||||
| 293 | #define PMAP_TYPE_EPT2 2 | ||||
| 294 | #define PMAP_TYPE_RVI3 3 | ||||
| 295 | #define pmap_nested(pm)((pm)->pm_type != 1) ((pm)->pm_type != PMAP_TYPE_NORMAL1) | ||||
| 296 | |||||
| 297 | struct pmap { | ||||
| 298 | struct mutex pm_mtx; | ||||
| 299 | struct uvm_object pm_obj[PTP_LEVELS4-1]; /* objects for lvl >= 1) */ | ||||
| 300 | LIST_ENTRY(pmap)struct { struct pmap *le_next; struct pmap **le_prev; } pm_list; /* list (lck by pm_list lock) */ | ||||
| 301 | /* | ||||
| 302 | * pm_pdir : VA of page table to be used when executing in | ||||
| 303 | * privileged mode | ||||
| 304 | * pm_pdirpa : PA of page table to be used when executing in | ||||
| 305 | * privileged mode | ||||
| 306 | * pm_pdir_intel : VA of special page table to be used when executing | ||||
| 307 | * on an Intel CPU in usermode (no kernel mappings) | ||||
| 308 | * pm_pdirpa_intel : PA of special page table to be used when executing | ||||
| 309 | * on an Intel CPU in usermode (no kernel mappings) | ||||
| 310 | */ | ||||
| 311 | pd_entry_t *pm_pdir, *pm_pdir_intel; | ||||
| 312 | paddr_t pm_pdirpa, pm_pdirpa_intel; | ||||
| 313 | |||||
| 314 | struct vm_page *pm_ptphint[PTP_LEVELS4-1]; | ||||
| 315 | /* pointer to a PTP in our pmap */ | ||||
| 316 | struct pmap_statistics pm_stats; /* pmap stats (lck by object lock) */ | ||||
| 317 | |||||
| 318 | int pm_type; /* Type of pmap this is (PMAP_TYPE_x) */ | ||||
| 319 | uint64_t eptp; /* cached EPTP (used by vmm) */ | ||||
| 320 | }; | ||||
| 321 | |||||
| 322 | #define PMAP_EFI0x00000040 PMAP_MD00x00000040 | ||||
| 323 | |||||
| 324 | /* | ||||
| 325 | * MD flags that we use for pmap_enter (in the pa): | ||||
| 326 | */ | ||||
| 327 | #define PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)) ~((paddr_t)PAGE_MASK((1 << 12) - 1)) /* to remove the flags */ | ||||
| 328 | #define PMAP_NOCACHE0x1 0x1 /* set the non-cacheable bit. */ | ||||
| 329 | #define PMAP_WC0x2 0x2 /* set page write combining. */ | ||||
| 330 | |||||
| 331 | /* | ||||
| 332 | * We keep mod/ref flags in struct vm_page->pg_flags. | ||||
| 333 | */ | ||||
| 334 | #define PG_PMAP_MOD0x01000000 PG_PMAP00x01000000 | ||||
| 335 | #define PG_PMAP_REF0x02000000 PG_PMAP10x02000000 | ||||
| 336 | #define PG_PMAP_WC0x04000000 PG_PMAP20x04000000 | ||||
| 337 | |||||
| 338 | /* | ||||
| 339 | * for each managed physical page we maintain a list of <PMAP,VA>'s | ||||
| 340 | * which it is mapped at. | ||||
| 341 | */ | ||||
| 342 | struct pv_entry { /* locked by its list's pvh_lock */ | ||||
| 343 | struct pv_entry *pv_next; /* next entry */ | ||||
| 344 | struct pmap *pv_pmap; /* the pmap */ | ||||
| 345 | vaddr_t pv_va; /* the virtual address */ | ||||
| 346 | struct vm_page *pv_ptp; /* the vm_page of the PTP */ | ||||
| 347 | }; | ||||
| 348 | |||||
| 349 | /* | ||||
| 350 | * global kernel variables | ||||
| 351 | */ | ||||
| 352 | |||||
| 353 | extern struct pmap kernel_pmap_store; /* kernel pmap */ | ||||
| 354 | |||||
| 355 | extern long nkptp[]; | ||||
| 356 | |||||
| 357 | extern const paddr_t ptp_masks[]; | ||||
| 358 | extern const int ptp_shifts[]; | ||||
| 359 | extern const long nbpd[], nkptpmax[]; | ||||
| 360 | |||||
| 361 | /* | ||||
| 362 | * macros | ||||
| 363 | */ | ||||
| 364 | |||||
| 365 | #define pmap_kernel()(&kernel_pmap_store) (&kernel_pmap_store) | ||||
| 366 | #define pmap_resident_count(pmap)((pmap)->pm_stats.resident_count) ((pmap)->pm_stats.resident_count) | ||||
| 367 | #define pmap_wired_count(pmap)((pmap)->pm_stats.wired_count) ((pmap)->pm_stats.wired_count) | ||||
| 368 | #define pmap_update(pmap) /* nothing (yet) */ | ||||
| 369 | |||||
| 370 | #define pmap_clear_modify(pg)pmap_clear_attrs(pg, 0x0000000000000040UL) pmap_clear_attrs(pg, PG_M0x0000000000000040UL) | ||||
| 371 | #define pmap_clear_reference(pg)pmap_clear_attrs(pg, 0x0000000000000020UL) pmap_clear_attrs(pg, PG_U0x0000000000000020UL) | ||||
| 372 | #define pmap_is_modified(pg)pmap_test_attrs(pg, 0x0000000000000040UL) pmap_test_attrs(pg, PG_M0x0000000000000040UL) | ||||
| 373 | #define pmap_is_referenced(pg)pmap_test_attrs(pg, 0x0000000000000020UL) pmap_test_attrs(pg, PG_U0x0000000000000020UL) | ||||
| 374 | #define pmap_move(DP,SP,D,L,S) | ||||
| 375 | #define pmap_valid_entry(E)((E) & 0x0000000000000001UL) ((E) & PG_V0x0000000000000001UL) /* is PDE or PTE valid? */ | ||||
| 376 | |||||
| 377 | #define pmap_proc_iflush(p,va,len) /* nothing */ | ||||
| 378 | #define pmap_unuse_final(p) /* nothing */ | ||||
| 379 | #define pmap_remove_holes(vm)do { } while (0) do { /* nothing */ } while (0) | ||||
| 380 | |||||
| 381 | |||||
| 382 | /* | ||||
| 383 | * prototypes | ||||
| 384 | */ | ||||
| 385 | |||||
| 386 | void map_tramps(void); /* machdep.c */ | ||||
| 387 | paddr_t pmap_bootstrap(paddr_t, paddr_t); | ||||
| 388 | void pmap_init_percpu(void); | ||||
| 389 | void pmap_randomize(void); | ||||
| 390 | void pmap_randomize_level(pd_entry_t *, int); | ||||
| 391 | int pmap_clear_attrs(struct vm_page *, unsigned long); | ||||
| 392 | static void pmap_page_protect(struct vm_page *, vm_prot_t); | ||||
| 393 | void pmap_page_remove (struct vm_page *); | ||||
| 394 | static void pmap_protect(struct pmap *, vaddr_t, | ||||
| 395 | vaddr_t, vm_prot_t); | ||||
| 396 | void pmap_remove(struct pmap *, vaddr_t, vaddr_t); | ||||
| 397 | int pmap_test_attrs(struct vm_page *, unsigned); | ||||
| 398 | static void pmap_update_pg(vaddr_t); | ||||
| 399 | void pmap_write_protect(struct pmap *, vaddr_t, | ||||
| 400 | vaddr_t, vm_prot_t); | ||||
| 401 | void pmap_fix_ept(struct pmap *, vaddr_t); | ||||
| 402 | |||||
| 403 | paddr_t pmap_prealloc_lowmem_ptps(paddr_t); | ||||
| 404 | |||||
| 405 | void pagezero(vaddr_t); | ||||
| 406 | |||||
| 407 | void pmap_convert(struct pmap *, int); | ||||
| 408 | void pmap_enter_special(vaddr_t, paddr_t, vm_prot_t); | ||||
| 409 | vaddr_t pmap_set_pml4_early(paddr_t pa); | ||||
| 410 | void pmap_clear_pml4_early(void); | ||||
| 411 | |||||
| 412 | /* | ||||
| 413 | * functions for flushing the cache for vaddrs and pages. | ||||
| 414 | * these functions are not part of the MI pmap interface and thus | ||||
| 415 | * should not be used as such. | ||||
| 416 | */ | ||||
| 417 | void pmap_flush_cache(vaddr_t, vsize_t); | ||||
| 418 | #define pmap_flush_page(paddr)do { ((void)0); pmap_flush_cache(((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) + (paddr)), (1 << 12)); } while ( 0) do { \ | ||||
| 419 | KDASSERT(PHYS_TO_VM_PAGE(paddr) != NULL)((void)0); \ | ||||
| 420 | pmap_flush_cache(PMAP_DIRECT_MAP(paddr)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (paddr)), PAGE_SIZE(1 << 12)); \ | ||||
| 421 | } while (/* CONSTCOND */ 0) | ||||
| 422 | |||||
| 423 | #define PMAP_CHECK_COPYIN(pg_xo == 0) (pg_xo == 0) | ||||
| 424 | |||||
| 425 | #define PMAP_STEAL_MEMORY /* enable pmap_steal_memory() */ | ||||
| 426 | #define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */ | ||||
| 427 | |||||
| 428 | /* | ||||
| 429 | * inline functions | ||||
| 430 | */ | ||||
| 431 | |||||
| 432 | static inline void | ||||
| 433 | pmap_remove_all(struct pmap *pmap) | ||||
| 434 | { | ||||
| 435 | /* Nothing. */ | ||||
| 436 | } | ||||
| 437 | |||||
| 438 | /* | ||||
| 439 | * pmap_update_pg: flush one page from the TLB (or flush the whole thing | ||||
| 440 | * if hardware doesn't support one-page flushing) | ||||
| 441 | */ | ||||
| 442 | |||||
| 443 | static inline void | ||||
| 444 | pmap_update_pg(vaddr_t va) | ||||
| 445 | { | ||||
| 446 | invlpg(va); | ||||
| 447 | } | ||||
| 448 | |||||
| 449 | /* | ||||
| 450 | * pmap_page_protect: change the protection of all recorded mappings | ||||
| 451 | * of a managed page | ||||
| 452 | * | ||||
| 453 | * => this function is a frontend for pmap_page_remove/pmap_clear_attrs | ||||
| 454 | * => we only have to worry about making the page more protected. | ||||
| 455 | * unprotecting a page is done on-demand at fault time. | ||||
| 456 | */ | ||||
| 457 | |||||
| 458 | static inline void | ||||
| 459 | pmap_page_protect(struct vm_page *pg, vm_prot_t prot) | ||||
| 460 | { | ||||
| 461 | if (prot
| ||||
| 462 | (void) pmap_clear_attrs(pg, PG_RW0x0000000000000002UL); | ||||
| 463 | } else { | ||||
| 464 | KASSERT(prot == PROT_NONE)((prot == 0x00) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/arch/amd64/compile/GENERIC.MP/obj/machine/pmap.h" , 464, "prot == PROT_NONE")); | ||||
| 465 | pmap_page_remove(pg); | ||||
| 466 | } | ||||
| 467 | } | ||||
| 468 | |||||
| 469 | /* | ||||
| 470 | * pmap_protect: change the protection of pages in a pmap | ||||
| 471 | * | ||||
| 472 | * => this function is a frontend for pmap_remove/pmap_write_protect | ||||
| 473 | * => we only have to worry about making the page more protected. | ||||
| 474 | * unprotecting a page is done on-demand at fault time. | ||||
| 475 | */ | ||||
| 476 | |||||
| 477 | static inline void | ||||
| 478 | pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) | ||||
| 479 | { | ||||
| 480 | if (prot != PROT_NONE0x00) { | ||||
| 481 | pmap_write_protect(pmap, sva, eva, prot); | ||||
| 482 | } else { | ||||
| 483 | pmap_remove(pmap, sva, eva); | ||||
| 484 | } | ||||
| 485 | } | ||||
| 486 | |||||
| 487 | /* | ||||
| 488 | * various address inlines | ||||
| 489 | * | ||||
| 490 | * vtopte: return a pointer to the PTE mapping a VA, works only for | ||||
| 491 | * user and PT addresses | ||||
| 492 | * | ||||
| 493 | * kvtopte: return a pointer to the PTE mapping a kernel VA | ||||
| 494 | */ | ||||
| 495 | |||||
| 496 | static inline pt_entry_t * | ||||
| 497 | vtopte(vaddr_t va) | ||||
| 498 | { | ||||
| 499 | return (PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)); | ||||
| 500 | } | ||||
| 501 | |||||
| 502 | static inline pt_entry_t * | ||||
| 503 | kvtopte(vaddr_t va) | ||||
| 504 | { | ||||
| 505 | #ifdef LARGEPAGES | ||||
| 506 | { | ||||
| 507 | pd_entry_t *pde; | ||||
| 508 | |||||
| 509 | pde = L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl2_i(va)(((((va) & ~0xffff000000000000)) & ((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)) >> 21); | ||||
| 510 | if (*pde & PG_PS0x0000000000000080UL) | ||||
| 511 | return ((pt_entry_t *)pde); | ||||
| 512 | } | ||||
| 513 | #endif | ||||
| 514 | |||||
| 515 | return (PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)); | ||||
| 516 | } | ||||
| 517 | |||||
| 518 | #define PMAP_DIRECT_MAP(pa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (pa)) ((vaddr_t)PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) + (pa)) | ||||
| 519 | #define PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 ))) ((paddr_t)(va) - PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000))) | ||||
| 520 | #define pmap_map_direct(pg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (((pg)->phys_addr))) PMAP_DIRECT_MAP(VM_PAGE_TO_PHYS(pg))((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (((pg)->phys_addr))) | ||||
| 521 | #define pmap_unmap_direct(va)PHYS_TO_VM_PAGE(((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)))) PHYS_TO_VM_PAGE(PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )))) | ||||
| 522 | |||||
| 523 | #define __HAVE_PMAP_DIRECT | ||||
| 524 | #define __HAVE_PMAP_MPSAFE_ENTER_COW | ||||
| 525 | |||||
| 526 | #endif /* _KERNEL && !_LOCORE */ | ||||
| 527 | |||||
| 528 | #ifndef _LOCORE | ||||
| 529 | struct pv_entry; | ||||
| 530 | struct vm_page_md { | ||||
| 531 | struct mutex pv_mtx; | ||||
| 532 | struct pv_entry *pv_list; | ||||
| 533 | }; | ||||
| 534 | |||||
| 535 | #define VM_MDPAGE_INIT(pg)do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(pg )->mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); (pg)->mdpage.pv_list = ((void *)0); } while (0) do { \ | ||||
| 536 | mtx_init(&(pg)->mdpage.pv_mtx, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&(pg)-> mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9 ) ? 0x9 : ((0xa)))); } while (0); \ | ||||
| 537 | (pg)->mdpage.pv_list = NULL((void *)0); \ | ||||
| 538 | } while (0) | ||||
| 539 | #endif /* !_LOCORE */ | ||||
| 540 | |||||
| 541 | #endif /* _MACHINE_PMAP_H_ */ |