File: | uvm/uvm_page.c |
Warning: | line 157, column 2 Access to field 'pgops' results in a dereference of a null pointer (loaded from field 'uobject') |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | /* $OpenBSD: uvm_page.c,v 1.160 2021/12/15 12:53:53 mpi Exp $ */ | ||||
2 | /* $NetBSD: uvm_page.c,v 1.44 2000/11/27 08:40:04 chs Exp $ */ | ||||
3 | |||||
4 | /* | ||||
5 | * Copyright (c) 1997 Charles D. Cranor and Washington University. | ||||
6 | * Copyright (c) 1991, 1993, The Regents of the University of California. | ||||
7 | * | ||||
8 | * All rights reserved. | ||||
9 | * | ||||
10 | * This code is derived from software contributed to Berkeley by | ||||
11 | * The Mach Operating System project at Carnegie-Mellon University. | ||||
12 | * | ||||
13 | * Redistribution and use in source and binary forms, with or without | ||||
14 | * modification, are permitted provided that the following conditions | ||||
15 | * are met: | ||||
16 | * 1. Redistributions of source code must retain the above copyright | ||||
17 | * notice, this list of conditions and the following disclaimer. | ||||
18 | * 2. Redistributions in binary form must reproduce the above copyright | ||||
19 | * notice, this list of conditions and the following disclaimer in the | ||||
20 | * documentation and/or other materials provided with the distribution. | ||||
21 | * 3. Neither the name of the University nor the names of its contributors | ||||
22 | * may be used to endorse or promote products derived from this software | ||||
23 | * without specific prior written permission. | ||||
24 | * | ||||
25 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND | ||||
26 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||||
27 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||||
28 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE | ||||
29 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL | ||||
30 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS | ||||
31 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) | ||||
32 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT | ||||
33 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY | ||||
34 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF | ||||
35 | * SUCH DAMAGE. | ||||
36 | * | ||||
37 | * @(#)vm_page.c 8.3 (Berkeley) 3/21/94 | ||||
38 | * from: Id: uvm_page.c,v 1.1.2.18 1998/02/06 05:24:42 chs Exp | ||||
39 | * | ||||
40 | * | ||||
41 | * Copyright (c) 1987, 1990 Carnegie-Mellon University. | ||||
42 | * All rights reserved. | ||||
43 | * | ||||
44 | * Permission to use, copy, modify and distribute this software and | ||||
45 | * its documentation is hereby granted, provided that both the copyright | ||||
46 | * notice and this permission notice appear in all copies of the | ||||
47 | * software, derivative works or modified versions, and any portions | ||||
48 | * thereof, and that both notices appear in supporting documentation. | ||||
49 | * | ||||
50 | * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" | ||||
51 | * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND | ||||
52 | * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. | ||||
53 | * | ||||
54 | * Carnegie Mellon requests users of this software to return to | ||||
55 | * | ||||
56 | * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU | ||||
57 | * School of Computer Science | ||||
58 | * Carnegie Mellon University | ||||
59 | * Pittsburgh PA 15213-3890 | ||||
60 | * | ||||
61 | * any improvements or extensions that they make and grant Carnegie the | ||||
62 | * rights to redistribute these changes. | ||||
63 | */ | ||||
64 | |||||
65 | /* | ||||
66 | * uvm_page.c: page ops. | ||||
67 | */ | ||||
68 | |||||
69 | #include <sys/param.h> | ||||
70 | #include <sys/systm.h> | ||||
71 | #include <sys/sched.h> | ||||
72 | #include <sys/vnode.h> | ||||
73 | #include <sys/mount.h> | ||||
74 | #include <sys/proc.h> | ||||
75 | #include <sys/smr.h> | ||||
76 | |||||
77 | #include <uvm/uvm.h> | ||||
78 | |||||
79 | /* | ||||
80 | * for object trees | ||||
81 | */ | ||||
82 | RBT_GENERATE(uvm_objtree, vm_page, objt, uvm_pagecmp)static int uvm_objtree_RBT_COMPARE(const void *lptr, const void *rptr) { const struct vm_page *l = lptr, *r = rptr; return uvm_pagecmp (l, r); } static const struct rb_type uvm_objtree_RBT_INFO = { uvm_objtree_RBT_COMPARE, ((void *)0), __builtin_offsetof(struct vm_page, objt), }; const struct rb_type *const uvm_objtree_RBT_TYPE = &uvm_objtree_RBT_INFO; | ||||
83 | |||||
84 | int | ||||
85 | uvm_pagecmp(const struct vm_page *a, const struct vm_page *b) | ||||
86 | { | ||||
87 | return a->offset < b->offset ? -1 : a->offset > b->offset; | ||||
88 | } | ||||
89 | |||||
90 | /* | ||||
91 | * global vars... XXXCDC: move to uvm. structure. | ||||
92 | */ | ||||
93 | /* | ||||
94 | * physical memory config is stored in vm_physmem. | ||||
95 | */ | ||||
96 | struct vm_physseg vm_physmem[VM_PHYSSEG_MAX16]; /* XXXCDC: uvm.physmem */ | ||||
97 | int vm_nphysseg = 0; /* XXXCDC: uvm.nphysseg */ | ||||
98 | |||||
99 | /* | ||||
100 | * Some supported CPUs in a given architecture don't support all | ||||
101 | * of the things necessary to do idle page zero'ing efficiently. | ||||
102 | * We therefore provide a way to disable it from machdep code here. | ||||
103 | */ | ||||
104 | |||||
105 | /* | ||||
106 | * local variables | ||||
107 | */ | ||||
108 | /* | ||||
109 | * these variables record the values returned by vm_page_bootstrap, | ||||
110 | * for debugging purposes. The implementation of uvm_pageboot_alloc | ||||
111 | * and pmap_startup here also uses them internally. | ||||
112 | */ | ||||
113 | static vaddr_t virtual_space_start; | ||||
114 | static vaddr_t virtual_space_end; | ||||
115 | |||||
116 | /* | ||||
117 | * local prototypes | ||||
118 | */ | ||||
119 | static void uvm_pageinsert(struct vm_page *); | ||||
120 | static void uvm_pageremove(struct vm_page *); | ||||
121 | int uvm_page_owner_locked_p(struct vm_page *); | ||||
122 | |||||
123 | /* | ||||
124 | * inline functions | ||||
125 | */ | ||||
126 | /* | ||||
127 | * uvm_pageinsert: insert a page in the object | ||||
128 | * | ||||
129 | * => caller must lock object | ||||
130 | * => call should have already set pg's object and offset pointers | ||||
131 | * and bumped the version counter | ||||
132 | */ | ||||
133 | inline static void | ||||
134 | uvm_pageinsert(struct vm_page *pg) | ||||
135 | { | ||||
136 | struct vm_page *dupe; | ||||
137 | |||||
138 | KASSERT(UVM_OBJ_IS_DUMMY(pg->uobject) ||(((((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager)) || rw_write_held( pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 139, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )) | ||||
139 | rw_write_held(pg->uobject->vmobjlock))(((((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager)) || rw_write_held( pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 139, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )); | ||||
140 | KASSERT((pg->pg_flags & PG_TABLED) == 0)(((pg->pg_flags & 0x00000004) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 140, "(pg->pg_flags & PG_TABLED) == 0" )); | ||||
141 | |||||
142 | dupe = RBT_INSERT(uvm_objtree, &pg->uobject->memt, pg)uvm_objtree_RBT_INSERT(&pg->uobject->memt, pg); | ||||
143 | /* not allowed to insert over another page */ | ||||
144 | KASSERT(dupe == NULL)((dupe == ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 144, "dupe == NULL")); | ||||
145 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_TABLED0x00000004); | ||||
146 | pg->uobject->uo_npages++; | ||||
147 | } | ||||
148 | |||||
149 | /* | ||||
150 | * uvm_page_remove: remove page from object | ||||
151 | * | ||||
152 | * => caller must lock object | ||||
153 | */ | ||||
154 | static inline void | ||||
155 | uvm_pageremove(struct vm_page *pg) | ||||
156 | { | ||||
157 | KASSERT(UVM_OBJ_IS_DUMMY(pg->uobject) ||(((((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager)) || rw_write_held( pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 158, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )) | ||||
| |||||
158 | rw_write_held(pg->uobject->vmobjlock))(((((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager)) || rw_write_held( pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 158, "UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )); | ||||
159 | KASSERT(pg->pg_flags & PG_TABLED)((pg->pg_flags & 0x00000004) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 159, "pg->pg_flags & PG_TABLED" )); | ||||
160 | |||||
161 | RBT_REMOVE(uvm_objtree, &pg->uobject->memt, pg)uvm_objtree_RBT_REMOVE(&pg->uobject->memt, pg); | ||||
162 | |||||
163 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_TABLED0x00000004); | ||||
164 | pg->uobject->uo_npages--; | ||||
165 | pg->uobject = NULL((void *)0); | ||||
166 | pg->pg_version++; | ||||
167 | } | ||||
168 | |||||
169 | /* | ||||
170 | * uvm_page_init: init the page system. called from uvm_init(). | ||||
171 | * | ||||
172 | * => we return the range of kernel virtual memory in kvm_startp/kvm_endp | ||||
173 | */ | ||||
174 | void | ||||
175 | uvm_page_init(vaddr_t *kvm_startp, vaddr_t *kvm_endp) | ||||
176 | { | ||||
177 | vsize_t freepages, pagecount, n; | ||||
178 | vm_page_t pagearray, curpg; | ||||
179 | int lcv, i; | ||||
180 | paddr_t paddr, pgno; | ||||
181 | struct vm_physseg *seg; | ||||
182 | |||||
183 | /* | ||||
184 | * init the page queues and page queue locks | ||||
185 | */ | ||||
186 | |||||
187 | TAILQ_INIT(&uvm.page_active)do { (&uvm.page_active)->tqh_first = ((void *)0); (& uvm.page_active)->tqh_last = &(&uvm.page_active)-> tqh_first; } while (0); | ||||
188 | TAILQ_INIT(&uvm.page_inactive_swp)do { (&uvm.page_inactive_swp)->tqh_first = ((void *)0) ; (&uvm.page_inactive_swp)->tqh_last = &(&uvm. page_inactive_swp)->tqh_first; } while (0); | ||||
189 | TAILQ_INIT(&uvm.page_inactive_obj)do { (&uvm.page_inactive_obj)->tqh_first = ((void *)0) ; (&uvm.page_inactive_obj)->tqh_last = &(&uvm. page_inactive_obj)->tqh_first; } while (0); | ||||
190 | mtx_init(&uvm.pageqlock, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.pageqlock ), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : (( 0xa)))); } while (0); | ||||
191 | mtx_init(&uvm.fpageqlock, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.fpageqlock ), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : (( 0xa)))); } while (0); | ||||
192 | uvm_pmr_init(); | ||||
193 | |||||
194 | /* | ||||
195 | * allocate vm_page structures. | ||||
196 | */ | ||||
197 | |||||
198 | /* | ||||
199 | * sanity check: | ||||
200 | * before calling this function the MD code is expected to register | ||||
201 | * some free RAM with the uvm_page_physload() function. our job | ||||
202 | * now is to allocate vm_page structures for this memory. | ||||
203 | */ | ||||
204 | |||||
205 | if (vm_nphysseg == 0) | ||||
206 | panic("uvm_page_bootstrap: no memory pre-allocated"); | ||||
207 | |||||
208 | /* | ||||
209 | * first calculate the number of free pages... | ||||
210 | * | ||||
211 | * note that we use start/end rather than avail_start/avail_end. | ||||
212 | * this allows us to allocate extra vm_page structures in case we | ||||
213 | * want to return some memory to the pool after booting. | ||||
214 | */ | ||||
215 | |||||
216 | freepages = 0; | ||||
217 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) | ||||
218 | freepages += (seg->end - seg->start); | ||||
219 | |||||
220 | /* | ||||
221 | * we now know we have (PAGE_SIZE * freepages) bytes of memory we can | ||||
222 | * use. for each page of memory we use we need a vm_page structure. | ||||
223 | * thus, the total number of pages we can use is the total size of | ||||
224 | * the memory divided by the PAGE_SIZE plus the size of the vm_page | ||||
225 | * structure. we add one to freepages as a fudge factor to avoid | ||||
226 | * truncation errors (since we can only allocate in terms of whole | ||||
227 | * pages). | ||||
228 | */ | ||||
229 | |||||
230 | pagecount = (((paddr_t)freepages + 1) << PAGE_SHIFT12) / | ||||
231 | (PAGE_SIZE(1 << 12) + sizeof(struct vm_page)); | ||||
232 | pagearray = (vm_page_t)uvm_pageboot_alloc(pagecount * | ||||
233 | sizeof(struct vm_page)); | ||||
234 | memset(pagearray, 0, pagecount * sizeof(struct vm_page))__builtin_memset((pagearray), (0), (pagecount * sizeof(struct vm_page))); | ||||
235 | |||||
236 | /* init the vm_page structures and put them in the correct place. */ | ||||
237 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) { | ||||
238 | n = seg->end - seg->start; | ||||
239 | if (n > pagecount) { | ||||
240 | panic("uvm_page_init: lost %ld page(s) in init", | ||||
241 | (long)(n - pagecount)); | ||||
242 | /* XXXCDC: shouldn't happen? */ | ||||
243 | /* n = pagecount; */ | ||||
244 | } | ||||
245 | |||||
246 | /* set up page array pointers */ | ||||
247 | seg->pgs = pagearray; | ||||
248 | pagearray += n; | ||||
249 | pagecount -= n; | ||||
250 | seg->lastpg = seg->pgs + (n - 1); | ||||
251 | |||||
252 | /* init and free vm_pages (we've already zeroed them) */ | ||||
253 | pgno = seg->start; | ||||
254 | paddr = ptoa(pgno)((paddr_t)(pgno) << 12); | ||||
255 | for (i = 0, curpg = seg->pgs; i < n; | ||||
256 | i++, curpg++, pgno++, paddr += PAGE_SIZE(1 << 12)) { | ||||
257 | curpg->phys_addr = paddr; | ||||
258 | VM_MDPAGE_INIT(curpg)do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(curpg )->mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); (curpg)->mdpage.pv_list = ((void *)0); } while (0); | ||||
259 | if (pgno >= seg->avail_start && | ||||
260 | pgno < seg->avail_end) { | ||||
261 | uvmexp.npages++; | ||||
262 | } | ||||
263 | } | ||||
264 | |||||
265 | /* Add pages to free pool. */ | ||||
266 | uvm_pmr_freepages(&seg->pgs[seg->avail_start - seg->start], | ||||
267 | seg->avail_end - seg->avail_start); | ||||
268 | } | ||||
269 | |||||
270 | /* | ||||
271 | * pass up the values of virtual_space_start and | ||||
272 | * virtual_space_end (obtained by uvm_pageboot_alloc) to the upper | ||||
273 | * layers of the VM. | ||||
274 | */ | ||||
275 | |||||
276 | *kvm_startp = round_page(virtual_space_start)(((virtual_space_start) + ((1 << 12) - 1)) & ~((1 << 12) - 1)); | ||||
277 | *kvm_endp = trunc_page(virtual_space_end)((virtual_space_end) & ~((1 << 12) - 1)); | ||||
278 | |||||
279 | /* init locks for kernel threads */ | ||||
280 | mtx_init(&uvm.aiodoned_lock, IPL_BIO)do { (void)(((void *)0)); (void)(0); __mtx_init((&uvm.aiodoned_lock ), ((((0x6)) > 0x0 && ((0x6)) < 0x9) ? 0x9 : (( 0x6)))); } while (0); | ||||
281 | |||||
282 | /* | ||||
283 | * init reserve thresholds | ||||
284 | * XXXCDC - values may need adjusting | ||||
285 | */ | ||||
286 | uvmexp.reserve_pagedaemon = 4; | ||||
287 | uvmexp.reserve_kernel = 8; | ||||
288 | uvmexp.anonminpct = 10; | ||||
289 | uvmexp.vnodeminpct = 10; | ||||
290 | uvmexp.vtextminpct = 5; | ||||
291 | uvmexp.anonmin = uvmexp.anonminpct * 256 / 100; | ||||
292 | uvmexp.vnodemin = uvmexp.vnodeminpct * 256 / 100; | ||||
293 | uvmexp.vtextmin = uvmexp.vtextminpct * 256 / 100; | ||||
294 | |||||
295 | uvm.page_init_done = TRUE1; | ||||
296 | } | ||||
297 | |||||
298 | /* | ||||
299 | * uvm_setpagesize: set the page size | ||||
300 | * | ||||
301 | * => sets page_shift and page_mask from uvmexp.pagesize. | ||||
302 | */ | ||||
303 | void | ||||
304 | uvm_setpagesize(void) | ||||
305 | { | ||||
306 | if (uvmexp.pagesize == 0) | ||||
307 | uvmexp.pagesize = DEFAULT_PAGE_SIZE4096; | ||||
308 | uvmexp.pagemask = uvmexp.pagesize - 1; | ||||
309 | if ((uvmexp.pagemask & uvmexp.pagesize) != 0) | ||||
310 | panic("uvm_setpagesize: page size not a power of two"); | ||||
311 | for (uvmexp.pageshift = 0; ; uvmexp.pageshift++) | ||||
312 | if ((1 << uvmexp.pageshift) == uvmexp.pagesize) | ||||
313 | break; | ||||
314 | } | ||||
315 | |||||
316 | /* | ||||
317 | * uvm_pageboot_alloc: steal memory from physmem for bootstrapping | ||||
318 | */ | ||||
319 | vaddr_t | ||||
320 | uvm_pageboot_alloc(vsize_t size) | ||||
321 | { | ||||
322 | #if defined(PMAP_STEAL_MEMORY) | ||||
323 | vaddr_t addr; | ||||
324 | |||||
325 | /* | ||||
326 | * defer bootstrap allocation to MD code (it may want to allocate | ||||
327 | * from a direct-mapped segment). pmap_steal_memory should round | ||||
328 | * off virtual_space_start/virtual_space_end. | ||||
329 | */ | ||||
330 | |||||
331 | addr = pmap_steal_memory(size, &virtual_space_start, | ||||
332 | &virtual_space_end); | ||||
333 | |||||
334 | return addr; | ||||
335 | |||||
336 | #else /* !PMAP_STEAL_MEMORY */ | ||||
337 | |||||
338 | static boolean_t initialized = FALSE0; | ||||
339 | vaddr_t addr, vaddr; | ||||
340 | paddr_t paddr; | ||||
341 | |||||
342 | /* round to page size */ | ||||
343 | size = round_page(size)(((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1 )); | ||||
344 | |||||
345 | /* on first call to this function, initialize ourselves. */ | ||||
346 | if (initialized == FALSE0) { | ||||
347 | pmap_virtual_space(&virtual_space_start, &virtual_space_end); | ||||
348 | |||||
349 | /* round it the way we like it */ | ||||
350 | virtual_space_start = round_page(virtual_space_start)(((virtual_space_start) + ((1 << 12) - 1)) & ~((1 << 12) - 1)); | ||||
351 | virtual_space_end = trunc_page(virtual_space_end)((virtual_space_end) & ~((1 << 12) - 1)); | ||||
352 | |||||
353 | initialized = TRUE1; | ||||
354 | } | ||||
355 | |||||
356 | /* allocate virtual memory for this request */ | ||||
357 | if (virtual_space_start == virtual_space_end || | ||||
358 | (virtual_space_end - virtual_space_start) < size) | ||||
359 | panic("uvm_pageboot_alloc: out of virtual space"); | ||||
360 | |||||
361 | addr = virtual_space_start; | ||||
362 | |||||
363 | #ifdef PMAP_GROWKERNEL | ||||
364 | /* | ||||
365 | * If the kernel pmap can't map the requested space, | ||||
366 | * then allocate more resources for it. | ||||
367 | */ | ||||
368 | if (uvm_maxkaddr < (addr + size)) { | ||||
369 | uvm_maxkaddr = pmap_growkernel(addr + size); | ||||
370 | if (uvm_maxkaddr < (addr + size)) | ||||
371 | panic("uvm_pageboot_alloc: pmap_growkernel() failed"); | ||||
372 | } | ||||
373 | #endif | ||||
374 | |||||
375 | virtual_space_start += size; | ||||
376 | |||||
377 | /* allocate and mapin physical pages to back new virtual pages */ | ||||
378 | for (vaddr = round_page(addr)(((addr) + ((1 << 12) - 1)) & ~((1 << 12) - 1 )) ; vaddr < addr + size ; | ||||
379 | vaddr += PAGE_SIZE(1 << 12)) { | ||||
380 | if (!uvm_page_physget(&paddr)) | ||||
381 | panic("uvm_pageboot_alloc: out of memory"); | ||||
382 | |||||
383 | /* | ||||
384 | * Note this memory is no longer managed, so using | ||||
385 | * pmap_kenter is safe. | ||||
386 | */ | ||||
387 | pmap_kenter_pa(vaddr, paddr, PROT_READ0x01 | PROT_WRITE0x02); | ||||
388 | } | ||||
389 | pmap_update(pmap_kernel()); | ||||
390 | return addr; | ||||
391 | #endif /* PMAP_STEAL_MEMORY */ | ||||
392 | } | ||||
393 | |||||
394 | #if !defined(PMAP_STEAL_MEMORY) | ||||
395 | /* | ||||
396 | * uvm_page_physget: "steal" one page from the vm_physmem structure. | ||||
397 | * | ||||
398 | * => attempt to allocate it off the end of a segment in which the "avail" | ||||
399 | * values match the start/end values. if we can't do that, then we | ||||
400 | * will advance both values (making them equal, and removing some | ||||
401 | * vm_page structures from the non-avail area). | ||||
402 | * => return false if out of memory. | ||||
403 | */ | ||||
404 | |||||
405 | boolean_t | ||||
406 | uvm_page_physget(paddr_t *paddrp) | ||||
407 | { | ||||
408 | int lcv; | ||||
409 | struct vm_physseg *seg; | ||||
410 | |||||
411 | /* pass 1: try allocating from a matching end */ | ||||
412 | #if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) || \ | ||||
413 | (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2) | ||||
414 | for (lcv = vm_nphysseg - 1, seg = vm_physmem + lcv; lcv >= 0; | ||||
415 | lcv--, seg--) | ||||
416 | #else | ||||
417 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) | ||||
418 | #endif | ||||
419 | { | ||||
420 | if (uvm.page_init_done == TRUE1) | ||||
421 | panic("uvm_page_physget: called _after_ bootstrap"); | ||||
422 | |||||
423 | /* try from front */ | ||||
424 | if (seg->avail_start == seg->start && | ||||
425 | seg->avail_start < seg->avail_end) { | ||||
426 | *paddrp = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12); | ||||
427 | seg->avail_start++; | ||||
428 | seg->start++; | ||||
429 | /* nothing left? nuke it */ | ||||
430 | if (seg->avail_start == seg->end) { | ||||
431 | if (vm_nphysseg == 1) | ||||
432 | panic("uvm_page_physget: out of memory!"); | ||||
433 | vm_nphysseg--; | ||||
434 | for (; lcv < vm_nphysseg; lcv++, seg++) | ||||
435 | /* structure copy */ | ||||
436 | seg[0] = seg[1]; | ||||
437 | } | ||||
438 | return TRUE1; | ||||
439 | } | ||||
440 | |||||
441 | /* try from rear */ | ||||
442 | if (seg->avail_end == seg->end && | ||||
443 | seg->avail_start < seg->avail_end) { | ||||
444 | *paddrp = ptoa(seg->avail_end - 1)((paddr_t)(seg->avail_end - 1) << 12); | ||||
445 | seg->avail_end--; | ||||
446 | seg->end--; | ||||
447 | /* nothing left? nuke it */ | ||||
448 | if (seg->avail_end == seg->start) { | ||||
449 | if (vm_nphysseg == 1) | ||||
450 | panic("uvm_page_physget: out of memory!"); | ||||
451 | vm_nphysseg--; | ||||
452 | for (; lcv < vm_nphysseg ; lcv++, seg++) | ||||
453 | /* structure copy */ | ||||
454 | seg[0] = seg[1]; | ||||
455 | } | ||||
456 | return TRUE1; | ||||
457 | } | ||||
458 | } | ||||
459 | |||||
460 | /* pass2: forget about matching ends, just allocate something */ | ||||
461 | #if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) || \ | ||||
462 | (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2) | ||||
463 | for (lcv = vm_nphysseg - 1, seg = vm_physmem + lcv; lcv >= 0; | ||||
464 | lcv--, seg--) | ||||
465 | #else | ||||
466 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) | ||||
467 | #endif | ||||
468 | { | ||||
469 | |||||
470 | /* any room in this bank? */ | ||||
471 | if (seg->avail_start >= seg->avail_end) | ||||
472 | continue; /* nope */ | ||||
473 | |||||
474 | *paddrp = ptoa(seg->avail_start)((paddr_t)(seg->avail_start) << 12); | ||||
475 | seg->avail_start++; | ||||
476 | /* truncate! */ | ||||
477 | seg->start = seg->avail_start; | ||||
478 | |||||
479 | /* nothing left? nuke it */ | ||||
480 | if (seg->avail_start == seg->end) { | ||||
481 | if (vm_nphysseg == 1) | ||||
482 | panic("uvm_page_physget: out of memory!"); | ||||
483 | vm_nphysseg--; | ||||
484 | for (; lcv < vm_nphysseg ; lcv++, seg++) | ||||
485 | /* structure copy */ | ||||
486 | seg[0] = seg[1]; | ||||
487 | } | ||||
488 | return TRUE1; | ||||
489 | } | ||||
490 | |||||
491 | return FALSE0; /* whoops! */ | ||||
492 | } | ||||
493 | |||||
494 | #endif /* PMAP_STEAL_MEMORY */ | ||||
495 | |||||
496 | /* | ||||
497 | * uvm_page_physload: load physical memory into VM system | ||||
498 | * | ||||
499 | * => all args are PFs | ||||
500 | * => all pages in start/end get vm_page structures | ||||
501 | * => areas marked by avail_start/avail_end get added to the free page pool | ||||
502 | * => we are limited to VM_PHYSSEG_MAX physical memory segments | ||||
503 | */ | ||||
504 | |||||
505 | void | ||||
506 | uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start, | ||||
507 | paddr_t avail_end, int flags) | ||||
508 | { | ||||
509 | int preload, lcv; | ||||
510 | psize_t npages; | ||||
511 | struct vm_page *pgs; | ||||
512 | struct vm_physseg *ps, *seg; | ||||
513 | |||||
514 | #ifdef DIAGNOSTIC1 | ||||
515 | if (uvmexp.pagesize == 0) | ||||
516 | panic("uvm_page_physload: page size not set!"); | ||||
517 | |||||
518 | if (start >= end) | ||||
519 | panic("uvm_page_physload: start >= end"); | ||||
520 | #endif | ||||
521 | |||||
522 | /* do we have room? */ | ||||
523 | if (vm_nphysseg == VM_PHYSSEG_MAX16) { | ||||
524 | printf("uvm_page_physload: unable to load physical memory " | ||||
525 | "segment\n"); | ||||
526 | printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n", | ||||
527 | VM_PHYSSEG_MAX16, (long long)start, (long long)end); | ||||
528 | printf("\tincrease VM_PHYSSEG_MAX\n"); | ||||
529 | return; | ||||
530 | } | ||||
531 | |||||
532 | /* | ||||
533 | * check to see if this is a "preload" (i.e. uvm_mem_init hasn't been | ||||
534 | * called yet, so malloc is not available). | ||||
535 | */ | ||||
536 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++) { | ||||
537 | if (seg->pgs) | ||||
538 | break; | ||||
539 | } | ||||
540 | preload = (lcv == vm_nphysseg); | ||||
541 | |||||
542 | /* if VM is already running, attempt to malloc() vm_page structures */ | ||||
543 | if (!preload) { | ||||
544 | /* | ||||
545 | * XXXCDC: need some sort of lockout for this case | ||||
546 | * right now it is only used by devices so it should be alright. | ||||
547 | */ | ||||
548 | paddr_t paddr; | ||||
549 | |||||
550 | npages = end - start; /* # of pages */ | ||||
551 | |||||
552 | pgs = km_alloc(round_page(npages * sizeof(*pgs))(((npages * sizeof(*pgs)) + ((1 << 12) - 1)) & ~((1 << 12) - 1)), | ||||
553 | &kv_any, &kp_zero, &kd_waitok); | ||||
554 | if (pgs == NULL((void *)0)) { | ||||
555 | printf("uvm_page_physload: can not malloc vm_page " | ||||
556 | "structs for segment\n"); | ||||
557 | printf("\tignoring 0x%lx -> 0x%lx\n", start, end); | ||||
558 | return; | ||||
559 | } | ||||
560 | /* init phys_addr and free pages, XXX uvmexp.npages */ | ||||
561 | for (lcv = 0, paddr = ptoa(start)((paddr_t)(start) << 12); lcv < npages; | ||||
562 | lcv++, paddr += PAGE_SIZE(1 << 12)) { | ||||
563 | pgs[lcv].phys_addr = paddr; | ||||
564 | VM_MDPAGE_INIT(&pgs[lcv])do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(& pgs[lcv])->mdpage.pv_mtx), ((((0xa)) > 0x0 && ( (0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); (&pgs[lcv ])->mdpage.pv_list = ((void *)0); } while (0); | ||||
565 | if (atop(paddr)((paddr) >> 12) >= avail_start && | ||||
566 | atop(paddr)((paddr) >> 12) < avail_end) { | ||||
567 | if (flags & PHYSLOAD_DEVICE0x01) { | ||||
568 | atomic_setbits_intx86_atomic_setbits_u32(&pgs[lcv].pg_flags, | ||||
569 | PG_DEV0x00000200); | ||||
570 | pgs[lcv].wire_count = 1; | ||||
571 | } else { | ||||
572 | #if defined(VM_PHYSSEG_NOADD) | ||||
573 | panic("uvm_page_physload: tried to add RAM after vm_mem_init"); | ||||
574 | #endif | ||||
575 | } | ||||
576 | } | ||||
577 | } | ||||
578 | |||||
579 | /* Add pages to free pool. */ | ||||
580 | if ((flags & PHYSLOAD_DEVICE0x01) == 0) { | ||||
581 | uvm_pmr_freepages(&pgs[avail_start - start], | ||||
582 | avail_end - avail_start); | ||||
583 | } | ||||
584 | |||||
585 | /* XXXCDC: need hook to tell pmap to rebuild pv_list, etc... */ | ||||
586 | } else { | ||||
587 | /* gcc complains if these don't get init'd */ | ||||
588 | pgs = NULL((void *)0); | ||||
589 | npages = 0; | ||||
590 | |||||
591 | } | ||||
592 | |||||
593 | /* now insert us in the proper place in vm_physmem[] */ | ||||
594 | #if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_RANDOM1) | ||||
595 | /* random: put it at the end (easy!) */ | ||||
596 | ps = &vm_physmem[vm_nphysseg]; | ||||
597 | #elif (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2) | ||||
598 | { | ||||
599 | int x; | ||||
600 | /* sort by address for binary search */ | ||||
601 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++) | ||||
602 | if (start < seg->start) | ||||
603 | break; | ||||
604 | ps = seg; | ||||
605 | /* move back other entries, if necessary ... */ | ||||
606 | for (x = vm_nphysseg, seg = vm_physmem + x - 1; x > lcv; | ||||
607 | x--, seg--) | ||||
608 | /* structure copy */ | ||||
609 | seg[1] = seg[0]; | ||||
610 | } | ||||
611 | #elif (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BIGFIRST3) | ||||
612 | { | ||||
613 | int x; | ||||
614 | /* sort by largest segment first */ | ||||
615 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg; lcv++, seg++) | ||||
616 | if ((end - start) > | ||||
617 | (seg->end - seg->start)) | ||||
618 | break; | ||||
619 | ps = &vm_physmem[lcv]; | ||||
620 | /* move back other entries, if necessary ... */ | ||||
621 | for (x = vm_nphysseg, seg = vm_physmem + x - 1; x > lcv; | ||||
622 | x--, seg--) | ||||
623 | /* structure copy */ | ||||
624 | seg[1] = seg[0]; | ||||
625 | } | ||||
626 | #else | ||||
627 | panic("uvm_page_physload: unknown physseg strategy selected!"); | ||||
628 | #endif | ||||
629 | |||||
630 | ps->start = start; | ||||
631 | ps->end = end; | ||||
632 | ps->avail_start = avail_start; | ||||
633 | ps->avail_end = avail_end; | ||||
634 | if (preload) { | ||||
635 | ps->pgs = NULL((void *)0); | ||||
636 | } else { | ||||
637 | ps->pgs = pgs; | ||||
638 | ps->lastpg = pgs + npages - 1; | ||||
639 | } | ||||
640 | vm_nphysseg++; | ||||
641 | |||||
642 | return; | ||||
643 | } | ||||
644 | |||||
645 | #ifdef DDB1 /* XXXCDC: TMP TMP TMP DEBUG DEBUG DEBUG */ | ||||
646 | |||||
647 | void uvm_page_physdump(void); /* SHUT UP GCC */ | ||||
648 | |||||
649 | /* call from DDB */ | ||||
650 | void | ||||
651 | uvm_page_physdump(void) | ||||
652 | { | ||||
653 | int lcv; | ||||
654 | struct vm_physseg *seg; | ||||
655 | |||||
656 | printf("uvm_page_physdump: physical memory config [segs=%d of %d]:\n", | ||||
657 | vm_nphysseg, VM_PHYSSEG_MAX16); | ||||
658 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) | ||||
659 | printf("0x%llx->0x%llx [0x%llx->0x%llx]\n", | ||||
660 | (long long)seg->start, | ||||
661 | (long long)seg->end, | ||||
662 | (long long)seg->avail_start, | ||||
663 | (long long)seg->avail_end); | ||||
664 | printf("STRATEGY = "); | ||||
665 | switch (VM_PHYSSEG_STRAT3) { | ||||
666 | case VM_PSTRAT_RANDOM1: printf("RANDOM\n"); break; | ||||
667 | case VM_PSTRAT_BSEARCH2: printf("BSEARCH\n"); break; | ||||
668 | case VM_PSTRAT_BIGFIRST3: printf("BIGFIRST\n"); break; | ||||
669 | default: printf("<<UNKNOWN>>!!!!\n"); | ||||
670 | } | ||||
671 | } | ||||
672 | #endif | ||||
673 | |||||
674 | void | ||||
675 | uvm_shutdown(void) | ||||
676 | { | ||||
677 | #ifdef UVM_SWAP_ENCRYPT1 | ||||
678 | uvm_swap_finicrypt_all(); | ||||
679 | #endif | ||||
680 | smr_flush()smr_barrier_impl(1); | ||||
681 | } | ||||
682 | |||||
683 | /* | ||||
684 | * Perform insert of a given page in the specified anon of obj. | ||||
685 | * This is basically, uvm_pagealloc, but with the page already given. | ||||
686 | */ | ||||
687 | void | ||||
688 | uvm_pagealloc_pg(struct vm_page *pg, struct uvm_object *obj, voff_t off, | ||||
689 | struct vm_anon *anon) | ||||
690 | { | ||||
691 | int flags; | ||||
692 | |||||
693 | KASSERT(obj == NULL || anon == NULL)((obj == ((void *)0) || anon == ((void *)0)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 693, "obj == NULL || anon == NULL" )); | ||||
694 | KASSERT(anon == NULL || off == 0)((anon == ((void *)0) || off == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 694, "anon == NULL || off == 0" )); | ||||
695 | KASSERT(off == trunc_page(off))((off == ((off) & ~((1 << 12) - 1))) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 695, "off == trunc_page(off)" )); | ||||
696 | KASSERT(obj == NULL || UVM_OBJ_IS_DUMMY(obj) ||((obj == ((void *)0) || (((obj)->pgops == &pmap_pager) || ((obj)->pgops == &bufcache_pager)) || rw_write_held (obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 697, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)" )) | ||||
697 | rw_write_held(obj->vmobjlock))((obj == ((void *)0) || (((obj)->pgops == &pmap_pager) || ((obj)->pgops == &bufcache_pager)) || rw_write_held (obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 697, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)" )); | ||||
698 | KASSERT(anon == NULL || anon->an_lock == NULL ||((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held (anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 699, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)" )) | ||||
699 | rw_write_held(anon->an_lock))((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held (anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 699, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)" )); | ||||
700 | |||||
701 | flags = PG_BUSY0x00000001 | PG_FAKE0x00000040; | ||||
702 | pg->offset = off; | ||||
703 | pg->uobject = obj; | ||||
704 | pg->uanon = anon; | ||||
705 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 705, "uvm_page_owner_locked_p(pg)" )); | ||||
706 | if (anon) { | ||||
707 | anon->an_page = pg; | ||||
708 | flags |= PQ_ANON0x00100000; | ||||
709 | } else if (obj) | ||||
710 | uvm_pageinsert(pg); | ||||
711 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, flags); | ||||
712 | #if defined(UVM_PAGE_TRKOWN) | ||||
713 | pg->owner_tag = NULL((void *)0); | ||||
714 | #endif | ||||
715 | UVM_PAGE_OWN(pg, "new alloc"); | ||||
716 | } | ||||
717 | |||||
718 | /* | ||||
719 | * uvm_pglistalloc: allocate a list of pages | ||||
720 | * | ||||
721 | * => allocated pages are placed at the tail of rlist. rlist is | ||||
722 | * assumed to be properly initialized by caller. | ||||
723 | * => returns 0 on success or errno on failure | ||||
724 | * => doesn't take into account clean non-busy pages on inactive list | ||||
725 | * that could be used(?) | ||||
726 | * => params: | ||||
727 | * size the size of the allocation, rounded to page size. | ||||
728 | * low the low address of the allowed allocation range. | ||||
729 | * high the high address of the allowed allocation range. | ||||
730 | * alignment memory must be aligned to this power-of-two boundary. | ||||
731 | * boundary no segment in the allocation may cross this | ||||
732 | * power-of-two boundary (relative to zero). | ||||
733 | * => flags: | ||||
734 | * UVM_PLA_NOWAIT fail if allocation fails | ||||
735 | * UVM_PLA_WAITOK wait for memory to become avail | ||||
736 | * UVM_PLA_ZERO return zeroed memory | ||||
737 | */ | ||||
738 | int | ||||
739 | uvm_pglistalloc(psize_t size, paddr_t low, paddr_t high, paddr_t alignment, | ||||
740 | paddr_t boundary, struct pglist *rlist, int nsegs, int flags) | ||||
741 | { | ||||
742 | KASSERT((alignment & (alignment - 1)) == 0)(((alignment & (alignment - 1)) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 742, "(alignment & (alignment - 1)) == 0" )); | ||||
743 | KASSERT((boundary & (boundary - 1)) == 0)(((boundary & (boundary - 1)) == 0) ? (void)0 : __assert( "diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 743, "(boundary & (boundary - 1)) == 0" )); | ||||
744 | KASSERT(!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT))((!(flags & 0x0001) ^ !(flags & 0x0002)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 744, "!(flags & UVM_PLA_WAITOK) ^ !(flags & UVM_PLA_NOWAIT)" )); | ||||
745 | |||||
746 | if (size == 0) | ||||
747 | return EINVAL22; | ||||
748 | size = atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1))) >> 12); | ||||
749 | |||||
750 | /* | ||||
751 | * XXX uvm_pglistalloc is currently only used for kernel | ||||
752 | * objects. Unlike the checks in uvm_pagealloc, below, here | ||||
753 | * we are always allowed to use the kernel reserve. | ||||
754 | */ | ||||
755 | flags |= UVM_PLA_USERESERVE0x0040; | ||||
756 | |||||
757 | if ((high & PAGE_MASK((1 << 12) - 1)) != PAGE_MASK((1 << 12) - 1)) { | ||||
758 | printf("uvm_pglistalloc: Upper boundary 0x%lx " | ||||
759 | "not on pagemask.\n", (unsigned long)high); | ||||
760 | } | ||||
761 | |||||
762 | /* | ||||
763 | * Our allocations are always page granularity, so our alignment | ||||
764 | * must be, too. | ||||
765 | */ | ||||
766 | if (alignment < PAGE_SIZE(1 << 12)) | ||||
767 | alignment = PAGE_SIZE(1 << 12); | ||||
768 | |||||
769 | low = atop(roundup(low, alignment))((((((low)+((alignment)-1))/(alignment))*(alignment))) >> 12); | ||||
770 | /* | ||||
771 | * high + 1 may result in overflow, in which case high becomes 0x0, | ||||
772 | * which is the 'don't care' value. | ||||
773 | * The only requirement in that case is that low is also 0x0, or the | ||||
774 | * low<high assert will fail. | ||||
775 | */ | ||||
776 | high = atop(high + 1)((high + 1) >> 12); | ||||
777 | alignment = atop(alignment)((alignment) >> 12); | ||||
778 | if (boundary < PAGE_SIZE(1 << 12) && boundary != 0) | ||||
779 | boundary = PAGE_SIZE(1 << 12); | ||||
780 | boundary = atop(boundary)((boundary) >> 12); | ||||
781 | |||||
782 | return uvm_pmr_getpages(size, low, high, alignment, boundary, nsegs, | ||||
783 | flags, rlist); | ||||
784 | } | ||||
785 | |||||
786 | /* | ||||
787 | * uvm_pglistfree: free a list of pages | ||||
788 | * | ||||
789 | * => pages should already be unmapped | ||||
790 | */ | ||||
791 | void | ||||
792 | uvm_pglistfree(struct pglist *list) | ||||
793 | { | ||||
794 | uvm_pmr_freepageq(list); | ||||
795 | } | ||||
796 | |||||
797 | /* | ||||
798 | * interface used by the buffer cache to allocate a buffer at a time. | ||||
799 | * The pages are allocated wired in DMA accessible memory | ||||
800 | */ | ||||
801 | int | ||||
802 | uvm_pagealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size, | ||||
803 | int flags) | ||||
804 | { | ||||
805 | struct pglist plist; | ||||
806 | struct vm_page *pg; | ||||
807 | int i, r; | ||||
808 | |||||
809 | KASSERT(UVM_OBJ_IS_BUFCACHE(obj))((((obj)->pgops == &bufcache_pager)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 809, "UVM_OBJ_IS_BUFCACHE(obj)" )); | ||||
810 | KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 810, "_kernel_lock_held()")); | ||||
811 | |||||
812 | TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)-> tqh_last = &(&plist)->tqh_first; } while (0); | ||||
813 | r = uvm_pglistalloc(size, dma_constraint.ucr_low, | ||||
814 | dma_constraint.ucr_high, 0, 0, &plist, atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1))) >> 12), | ||||
815 | flags); | ||||
816 | if (r == 0) { | ||||
817 | i = 0; | ||||
818 | while ((pg = TAILQ_FIRST(&plist)((&plist)->tqh_first)) != NULL((void *)0)) { | ||||
819 | pg->wire_count = 1; | ||||
820 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008 | PG_FAKE0x00000040); | ||||
821 | KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 821, "(pg->pg_flags & PG_DEV) == 0" )); | ||||
822 | TAILQ_REMOVE(&plist, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &plist)->tqh_last = (pg)->pageq.tqe_prev; *(pg)-> pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->pageq.tqe_prev ) = ((void *)-1); ((pg)->pageq.tqe_next) = ((void *)-1); } while (0); | ||||
823 | uvm_pagealloc_pg(pg, obj, off + ptoa(i++)((paddr_t)(i++) << 12), NULL((void *)0)); | ||||
824 | } | ||||
825 | } | ||||
826 | return r; | ||||
827 | } | ||||
828 | |||||
829 | /* | ||||
830 | * interface used by the buffer cache to reallocate a buffer at a time. | ||||
831 | * The pages are reallocated wired outside the DMA accessible region. | ||||
832 | * | ||||
833 | */ | ||||
834 | int | ||||
835 | uvm_pagerealloc_multi(struct uvm_object *obj, voff_t off, vsize_t size, | ||||
836 | int flags, struct uvm_constraint_range *where) | ||||
837 | { | ||||
838 | struct pglist plist; | ||||
839 | struct vm_page *pg, *tpg; | ||||
840 | int i, r; | ||||
841 | voff_t offset; | ||||
842 | |||||
843 | KASSERT(UVM_OBJ_IS_BUFCACHE(obj))((((obj)->pgops == &bufcache_pager)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 843, "UVM_OBJ_IS_BUFCACHE(obj)" )); | ||||
844 | KERNEL_ASSERT_LOCKED()((_kernel_lock_held()) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 844, "_kernel_lock_held()")); | ||||
845 | |||||
846 | TAILQ_INIT(&plist)do { (&plist)->tqh_first = ((void *)0); (&plist)-> tqh_last = &(&plist)->tqh_first; } while (0); | ||||
847 | if (size == 0) | ||||
848 | panic("size 0 uvm_pagerealloc"); | ||||
849 | r = uvm_pglistalloc(size, where->ucr_low, where->ucr_high, 0, | ||||
850 | 0, &plist, atop(round_page(size))(((((size) + ((1 << 12) - 1)) & ~((1 << 12) - 1))) >> 12), flags); | ||||
851 | if (r == 0) { | ||||
852 | i = 0; | ||||
853 | while((pg = TAILQ_FIRST(&plist)((&plist)->tqh_first)) != NULL((void *)0)) { | ||||
854 | offset = off + ptoa(i++)((paddr_t)(i++) << 12); | ||||
855 | tpg = uvm_pagelookup(obj, offset); | ||||
856 | KASSERT(tpg != NULL)((tpg != ((void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 856, "tpg != NULL")); | ||||
857 | pg->wire_count = 1; | ||||
858 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008 | PG_FAKE0x00000040); | ||||
859 | KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 859, "(pg->pg_flags & PG_DEV) == 0" )); | ||||
860 | TAILQ_REMOVE(&plist, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &plist)->tqh_last = (pg)->pageq.tqe_prev; *(pg)-> pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)->pageq.tqe_prev ) = ((void *)-1); ((pg)->pageq.tqe_next) = ((void *)-1); } while (0); | ||||
861 | uvm_pagecopy(tpg, pg); | ||||
862 | KASSERT(tpg->wire_count == 1)((tpg->wire_count == 1) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 862, "tpg->wire_count == 1" )); | ||||
863 | tpg->wire_count = 0; | ||||
864 | uvm_lock_pageq()mtx_enter(&uvm.pageqlock); | ||||
865 | uvm_pagefree(tpg); | ||||
866 | uvm_unlock_pageq()mtx_leave(&uvm.pageqlock); | ||||
867 | uvm_pagealloc_pg(pg, obj, offset, NULL((void *)0)); | ||||
868 | } | ||||
869 | } | ||||
870 | return r; | ||||
871 | } | ||||
872 | |||||
873 | /* | ||||
874 | * uvm_pagealloc: allocate vm_page from a particular free list. | ||||
875 | * | ||||
876 | * => return null if no pages free | ||||
877 | * => wake up pagedaemon if number of free pages drops below low water mark | ||||
878 | * => only one of obj or anon can be non-null | ||||
879 | * => caller must activate/deactivate page if it is not wired. | ||||
880 | */ | ||||
881 | |||||
882 | struct vm_page * | ||||
883 | uvm_pagealloc(struct uvm_object *obj, voff_t off, struct vm_anon *anon, | ||||
884 | int flags) | ||||
885 | { | ||||
886 | struct vm_page *pg; | ||||
887 | struct pglist pgl; | ||||
888 | int pmr_flags; | ||||
889 | |||||
890 | KASSERT(obj == NULL || anon == NULL)((obj == ((void *)0) || anon == ((void *)0)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 890, "obj == NULL || anon == NULL" )); | ||||
891 | KASSERT(anon == NULL || off == 0)((anon == ((void *)0) || off == 0) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 891, "anon == NULL || off == 0" )); | ||||
892 | KASSERT(off == trunc_page(off))((off == ((off) & ~((1 << 12) - 1))) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 892, "off == trunc_page(off)" )); | ||||
893 | KASSERT(obj == NULL || UVM_OBJ_IS_DUMMY(obj) ||((obj == ((void *)0) || (((obj)->pgops == &pmap_pager) || ((obj)->pgops == &bufcache_pager)) || rw_write_held (obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 894, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)" )) | ||||
894 | rw_write_held(obj->vmobjlock))((obj == ((void *)0) || (((obj)->pgops == &pmap_pager) || ((obj)->pgops == &bufcache_pager)) || rw_write_held (obj->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 894, "obj == NULL || UVM_OBJ_IS_DUMMY(obj) || rw_write_held(obj->vmobjlock)" )); | ||||
895 | KASSERT(anon == NULL || anon->an_lock == NULL ||((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held (anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 896, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)" )) | ||||
896 | rw_write_held(anon->an_lock))((anon == ((void *)0) || anon->an_lock == ((void *)0) || rw_write_held (anon->an_lock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 896, "anon == NULL || anon->an_lock == NULL || rw_write_held(anon->an_lock)" )); | ||||
897 | |||||
898 | pmr_flags = UVM_PLA_NOWAIT0x0002; | ||||
899 | |||||
900 | /* | ||||
901 | * We're allowed to use the kernel reserve if the page is | ||||
902 | * being allocated to a kernel object. | ||||
903 | */ | ||||
904 | if ((flags & UVM_PGA_USERESERVE0x0001) || | ||||
905 | (obj != NULL((void *)0) && UVM_OBJ_IS_KERN_OBJECT(obj)((obj)->uo_refs == (-2)))) | ||||
906 | pmr_flags |= UVM_PLA_USERESERVE0x0040; | ||||
907 | |||||
908 | if (flags & UVM_PGA_ZERO0x0002) | ||||
909 | pmr_flags |= UVM_PLA_ZERO0x0004; | ||||
910 | TAILQ_INIT(&pgl)do { (&pgl)->tqh_first = ((void *)0); (&pgl)->tqh_last = &(&pgl)->tqh_first; } while (0); | ||||
911 | if (uvm_pmr_getpages(1, 0, 0, 1, 0, 1, pmr_flags, &pgl) != 0) | ||||
912 | goto fail; | ||||
913 | |||||
914 | pg = TAILQ_FIRST(&pgl)((&pgl)->tqh_first); | ||||
915 | KASSERT(pg != NULL && TAILQ_NEXT(pg, pageq) == NULL)((pg != ((void *)0) && ((pg)->pageq.tqe_next) == ( (void *)0)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c" , 915, "pg != NULL && TAILQ_NEXT(pg, pageq) == NULL") ); | ||||
916 | |||||
917 | uvm_pagealloc_pg(pg, obj, off, anon); | ||||
918 | KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 918, "(pg->pg_flags & PG_DEV) == 0" )); | ||||
919 | if (flags & UVM_PGA_ZERO0x0002) | ||||
920 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008); | ||||
921 | else | ||||
922 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PG_CLEAN0x00000008); | ||||
923 | |||||
924 | return pg; | ||||
925 | |||||
926 | fail: | ||||
927 | return NULL((void *)0); | ||||
928 | } | ||||
929 | |||||
930 | /* | ||||
931 | * uvm_pagerealloc: reallocate a page from one object to another | ||||
932 | */ | ||||
933 | |||||
934 | void | ||||
935 | uvm_pagerealloc(struct vm_page *pg, struct uvm_object *newobj, voff_t newoff) | ||||
936 | { | ||||
937 | |||||
938 | /* remove it from the old object */ | ||||
939 | if (pg->uobject) { | ||||
940 | uvm_pageremove(pg); | ||||
941 | } | ||||
942 | |||||
943 | /* put it in the new object */ | ||||
944 | if (newobj) { | ||||
945 | pg->uobject = newobj; | ||||
946 | pg->offset = newoff; | ||||
947 | pg->pg_version++; | ||||
948 | uvm_pageinsert(pg); | ||||
949 | } | ||||
950 | } | ||||
951 | |||||
952 | /* | ||||
953 | * uvm_pageclean: clean page | ||||
954 | * | ||||
955 | * => erase page's identity (i.e. remove from object) | ||||
956 | * => caller must lock page queues if `pg' is managed | ||||
957 | * => assumes all valid mappings of pg are gone | ||||
958 | */ | ||||
959 | void | ||||
960 | uvm_pageclean(struct vm_page *pg) | ||||
961 | { | ||||
962 | u_int flags_to_clear = 0; | ||||
963 | |||||
964 | if ((pg->pg_flags & (PG_TABLED0x00000004|PQ_ACTIVE0x00040000|PQ_INACTIVE0x00020000)) && | ||||
965 | (pg->uobject
| ||||
966 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
967 | |||||
968 | #ifdef DEBUG | ||||
969 | if (pg->uobject == (void *)0xdeadbeef && | ||||
970 | pg->uanon == (void *)0xdeadbeef) { | ||||
971 | panic("uvm_pagefree: freeing free page %p", pg); | ||||
972 | } | ||||
973 | #endif | ||||
974 | |||||
975 | KASSERT((pg->pg_flags & PG_DEV) == 0)(((pg->pg_flags & 0x00000200) == 0) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 975, "(pg->pg_flags & PG_DEV) == 0" )); | ||||
976 | KASSERT(pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) ||((pg->uobject == ((void *)0) || (((pg->uobject)->pgops == &pmap_pager) || ((pg->uobject)->pgops == &bufcache_pager )) || rw_write_held(pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 977, "pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )) | ||||
977 | rw_write_held(pg->uobject->vmobjlock))((pg->uobject == ((void *)0) || (((pg->uobject)->pgops == &pmap_pager) || ((pg->uobject)->pgops == &bufcache_pager )) || rw_write_held(pg->uobject->vmobjlock)) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 977, "pg->uobject == NULL || UVM_OBJ_IS_DUMMY(pg->uobject) || rw_write_held(pg->uobject->vmobjlock)" )); | ||||
978 | KASSERT(pg->uobject != NULL || pg->uanon == NULL ||((pg->uobject != ((void *)0) || pg->uanon == ((void *)0 ) || rw_write_held(pg->uanon->an_lock)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 979, "pg->uobject != NULL || pg->uanon == NULL || rw_write_held(pg->uanon->an_lock)" )) | ||||
979 | rw_write_held(pg->uanon->an_lock))((pg->uobject != ((void *)0) || pg->uanon == ((void *)0 ) || rw_write_held(pg->uanon->an_lock)) ? (void)0 : __assert ("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 979, "pg->uobject != NULL || pg->uanon == NULL || rw_write_held(pg->uanon->an_lock)" )); | ||||
980 | |||||
981 | /* | ||||
982 | * if the page was an object page (and thus "TABLED"), remove it | ||||
983 | * from the object. | ||||
984 | */ | ||||
985 | if (pg->pg_flags & PG_TABLED0x00000004) | ||||
986 | uvm_pageremove(pg); | ||||
987 | |||||
988 | /* now remove the page from the queues */ | ||||
989 | if (pg->pg_flags & PQ_ACTIVE0x00040000) { | ||||
990 | TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
991 | flags_to_clear |= PQ_ACTIVE0x00040000; | ||||
992 | uvmexp.active--; | ||||
993 | } | ||||
994 | if (pg->pg_flags & PQ_INACTIVE0x00020000) { | ||||
995 | if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000)) | ||||
996 | TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev ; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
997 | else | ||||
998 | TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev ; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
999 | flags_to_clear |= PQ_INACTIVE0x00020000; | ||||
1000 | uvmexp.inactive--; | ||||
1001 | } | ||||
1002 | |||||
1003 | /* if the page was wired, unwire it now. */ | ||||
1004 | if (pg->wire_count) { | ||||
1005 | pg->wire_count = 0; | ||||
1006 | uvmexp.wired--; | ||||
1007 | } | ||||
1008 | if (pg->uanon) { | ||||
1009 | pg->uanon->an_page = NULL((void *)0); | ||||
1010 | pg->uanon = NULL((void *)0); | ||||
1011 | } | ||||
1012 | |||||
1013 | /* Clean page state bits. */ | ||||
1014 | flags_to_clear |= PQ_ANON0x00100000|PQ_AOBJ0x00200000|PQ_ENCRYPT0x00400000|PG_ZERO0x00000100|PG_FAKE0x00000040|PG_BUSY0x00000001| | ||||
1015 | PG_RELEASED0x00000020|PG_CLEAN0x00000008|PG_CLEANCHK0x00000010; | ||||
1016 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, flags_to_clear); | ||||
1017 | |||||
1018 | #ifdef DEBUG | ||||
1019 | pg->uobject = (void *)0xdeadbeef; | ||||
1020 | pg->offset = 0xdeadbeef; | ||||
1021 | pg->uanon = (void *)0xdeadbeef; | ||||
1022 | #endif | ||||
1023 | } | ||||
1024 | |||||
1025 | /* | ||||
1026 | * uvm_pagefree: free page | ||||
1027 | * | ||||
1028 | * => erase page's identity (i.e. remove from object) | ||||
1029 | * => put page on free list | ||||
1030 | * => caller must lock page queues if `pg' is managed | ||||
1031 | * => assumes all valid mappings of pg are gone | ||||
1032 | */ | ||||
1033 | void | ||||
1034 | uvm_pagefree(struct vm_page *pg) | ||||
1035 | { | ||||
1036 | if ((pg->pg_flags & (PG_TABLED0x00000004|PQ_ACTIVE0x00040000|PQ_INACTIVE0x00020000)) && | ||||
1037 | (pg->uobject == NULL((void *)0) || !UVM_OBJ_IS_PMAP(pg->uobject)((pg->uobject)->pgops == &pmap_pager))) | ||||
1038 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
1039 | |||||
1040 | uvm_pageclean(pg); | ||||
1041 | uvm_pmr_freepages(pg, 1); | ||||
1042 | } | ||||
1043 | |||||
1044 | /* | ||||
1045 | * uvm_page_unbusy: unbusy an array of pages. | ||||
1046 | * | ||||
1047 | * => pages must either all belong to the same object, or all belong to anons. | ||||
1048 | * => if pages are anon-owned, anons must have 0 refcount. | ||||
1049 | */ | ||||
1050 | void | ||||
1051 | uvm_page_unbusy(struct vm_page **pgs, int npgs) | ||||
1052 | { | ||||
1053 | struct vm_page *pg; | ||||
1054 | struct uvm_object *uobj; | ||||
1055 | int i; | ||||
1056 | |||||
1057 | for (i = 0; i < npgs; i++) { | ||||
| |||||
1058 | pg = pgs[i]; | ||||
1059 | |||||
1060 | if (pg == NULL((void *)0) || pg == PGO_DONTCARE((struct vm_page *) -1L)) { | ||||
1061 | continue; | ||||
1062 | } | ||||
1063 | |||||
1064 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1064, "uvm_page_owner_locked_p(pg)" )); | ||||
1065 | KASSERT(pg->pg_flags & PG_BUSY)((pg->pg_flags & 0x00000001) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1065, "pg->pg_flags & PG_BUSY" )); | ||||
1066 | |||||
1067 | if (pg->pg_flags & PG_WANTED0x00000002) { | ||||
1068 | wakeup(pg); | ||||
1069 | } | ||||
1070 | if (pg->pg_flags & PG_RELEASED0x00000020) { | ||||
1071 | uobj = pg->uobject; | ||||
1072 | if (uobj
| ||||
1073 | uvm_lock_pageq()mtx_enter(&uvm.pageqlock); | ||||
1074 | pmap_page_protect(pg, PROT_NONE0x00); | ||||
1075 | /* XXX won't happen right now */ | ||||
1076 | if (pg->pg_flags & PQ_AOBJ0x00200000) | ||||
1077 | uao_dropswap(uobj, | ||||
1078 | pg->offset >> PAGE_SHIFT12); | ||||
1079 | uvm_pagefree(pg); | ||||
1080 | uvm_unlock_pageq()mtx_leave(&uvm.pageqlock); | ||||
1081 | } else { | ||||
1082 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_BUSY0x00000001); | ||||
1083 | UVM_PAGE_OWN(pg, NULL); | ||||
1084 | rw_enter(pg->uanon->an_lock, RW_WRITE0x0001UL); | ||||
1085 | uvm_anon_release(pg->uanon); | ||||
1086 | } | ||||
1087 | } else { | ||||
1088 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_WANTED0x00000002|PG_BUSY0x00000001); | ||||
1089 | UVM_PAGE_OWN(pg, NULL); | ||||
1090 | } | ||||
1091 | } | ||||
1092 | } | ||||
1093 | |||||
1094 | #if defined(UVM_PAGE_TRKOWN) | ||||
1095 | /* | ||||
1096 | * uvm_page_own: set or release page ownership | ||||
1097 | * | ||||
1098 | * => this is a debugging function that keeps track of who sets PG_BUSY | ||||
1099 | * and where they do it. it can be used to track down problems | ||||
1100 | * such a thread setting "PG_BUSY" and never releasing it. | ||||
1101 | * => if "tag" is NULL then we are releasing page ownership | ||||
1102 | */ | ||||
1103 | void | ||||
1104 | uvm_page_own(struct vm_page *pg, char *tag) | ||||
1105 | { | ||||
1106 | /* gain ownership? */ | ||||
1107 | if (tag) { | ||||
1108 | if (pg->owner_tag) { | ||||
1109 | printf("uvm_page_own: page %p already owned " | ||||
1110 | "by thread %d [%s]\n", pg, | ||||
1111 | pg->owner, pg->owner_tag); | ||||
1112 | panic("uvm_page_own"); | ||||
1113 | } | ||||
1114 | pg->owner = (curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc) ? curproc({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof(struct cpu_info, ci_self))); __ci;})->ci_curproc->p_tid : (pid_t) -1; | ||||
1115 | pg->owner_tag = tag; | ||||
1116 | return; | ||||
1117 | } | ||||
1118 | |||||
1119 | /* drop ownership */ | ||||
1120 | if (pg->owner_tag == NULL((void *)0)) { | ||||
1121 | printf("uvm_page_own: dropping ownership of an non-owned " | ||||
1122 | "page (%p)\n", pg); | ||||
1123 | panic("uvm_page_own"); | ||||
1124 | } | ||||
1125 | pg->owner_tag = NULL((void *)0); | ||||
1126 | return; | ||||
1127 | } | ||||
1128 | #endif | ||||
1129 | |||||
1130 | /* | ||||
1131 | * when VM_PHYSSEG_MAX is 1, we can simplify these functions | ||||
1132 | */ | ||||
1133 | |||||
1134 | #if VM_PHYSSEG_MAX16 > 1 | ||||
1135 | /* | ||||
1136 | * vm_physseg_find: find vm_physseg structure that belongs to a PA | ||||
1137 | */ | ||||
1138 | int | ||||
1139 | vm_physseg_find(paddr_t pframe, int *offp) | ||||
1140 | { | ||||
1141 | struct vm_physseg *seg; | ||||
1142 | |||||
1143 | #if (VM_PHYSSEG_STRAT3 == VM_PSTRAT_BSEARCH2) | ||||
1144 | /* binary search for it */ | ||||
1145 | int start, len, try; | ||||
1146 | |||||
1147 | /* | ||||
1148 | * if try is too large (thus target is less than than try) we reduce | ||||
1149 | * the length to trunc(len/2) [i.e. everything smaller than "try"] | ||||
1150 | * | ||||
1151 | * if the try is too small (thus target is greater than try) then | ||||
1152 | * we set the new start to be (try + 1). this means we need to | ||||
1153 | * reduce the length to (round(len/2) - 1). | ||||
1154 | * | ||||
1155 | * note "adjust" below which takes advantage of the fact that | ||||
1156 | * (round(len/2) - 1) == trunc((len - 1) / 2) | ||||
1157 | * for any value of len we may have | ||||
1158 | */ | ||||
1159 | |||||
1160 | for (start = 0, len = vm_nphysseg ; len != 0 ; len = len / 2) { | ||||
1161 | try = start + (len / 2); /* try in the middle */ | ||||
1162 | seg = vm_physmem + try; | ||||
1163 | |||||
1164 | /* start past our try? */ | ||||
1165 | if (pframe >= seg->start) { | ||||
1166 | /* was try correct? */ | ||||
1167 | if (pframe < seg->end) { | ||||
1168 | if (offp) | ||||
1169 | *offp = pframe - seg->start; | ||||
1170 | return try; /* got it */ | ||||
1171 | } | ||||
1172 | start = try + 1; /* next time, start here */ | ||||
1173 | len--; /* "adjust" */ | ||||
1174 | } else { | ||||
1175 | /* | ||||
1176 | * pframe before try, just reduce length of | ||||
1177 | * region, done in "for" loop | ||||
1178 | */ | ||||
1179 | } | ||||
1180 | } | ||||
1181 | return -1; | ||||
1182 | |||||
1183 | #else | ||||
1184 | /* linear search for it */ | ||||
1185 | int lcv; | ||||
1186 | |||||
1187 | for (lcv = 0, seg = vm_physmem; lcv < vm_nphysseg ; lcv++, seg++) { | ||||
1188 | if (pframe >= seg->start && pframe < seg->end) { | ||||
1189 | if (offp) | ||||
1190 | *offp = pframe - seg->start; | ||||
1191 | return lcv; /* got it */ | ||||
1192 | } | ||||
1193 | } | ||||
1194 | return -1; | ||||
1195 | |||||
1196 | #endif | ||||
1197 | } | ||||
1198 | |||||
1199 | /* | ||||
1200 | * PHYS_TO_VM_PAGE: find vm_page for a PA. used by MI code to get vm_pages | ||||
1201 | * back from an I/O mapping (ugh!). used in some MD code as well. | ||||
1202 | */ | ||||
1203 | struct vm_page * | ||||
1204 | PHYS_TO_VM_PAGE(paddr_t pa) | ||||
1205 | { | ||||
1206 | paddr_t pf = atop(pa)((pa) >> 12); | ||||
1207 | int off; | ||||
1208 | int psi; | ||||
1209 | |||||
1210 | psi = vm_physseg_find(pf, &off); | ||||
1211 | |||||
1212 | return (psi == -1) ? NULL((void *)0) : &vm_physmem[psi].pgs[off]; | ||||
1213 | } | ||||
1214 | #endif /* VM_PHYSSEG_MAX > 1 */ | ||||
1215 | |||||
1216 | /* | ||||
1217 | * uvm_pagelookup: look up a page | ||||
1218 | */ | ||||
1219 | struct vm_page * | ||||
1220 | uvm_pagelookup(struct uvm_object *obj, voff_t off) | ||||
1221 | { | ||||
1222 | /* XXX if stack is too much, handroll */ | ||||
1223 | struct vm_page pg; | ||||
1224 | |||||
1225 | pg.offset = off; | ||||
1226 | return RBT_FIND(uvm_objtree, &obj->memt, &pg)uvm_objtree_RBT_FIND(&obj->memt, &pg); | ||||
1227 | } | ||||
1228 | |||||
1229 | /* | ||||
1230 | * uvm_pagewire: wire the page, thus removing it from the daemon's grasp | ||||
1231 | * | ||||
1232 | * => caller must lock page queues | ||||
1233 | */ | ||||
1234 | void | ||||
1235 | uvm_pagewire(struct vm_page *pg) | ||||
1236 | { | ||||
1237 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1237, "uvm_page_owner_locked_p(pg)" )); | ||||
1238 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
1239 | |||||
1240 | if (pg->wire_count == 0) { | ||||
1241 | if (pg->pg_flags & PQ_ACTIVE0x00040000) { | ||||
1242 | TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
1243 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000); | ||||
1244 | uvmexp.active--; | ||||
1245 | } | ||||
1246 | if (pg->pg_flags & PQ_INACTIVE0x00020000) { | ||||
1247 | if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000)) | ||||
1248 | TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev ; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
1249 | else | ||||
1250 | TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev ; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
1251 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000); | ||||
1252 | uvmexp.inactive--; | ||||
1253 | } | ||||
1254 | uvmexp.wired++; | ||||
1255 | } | ||||
1256 | pg->wire_count++; | ||||
1257 | } | ||||
1258 | |||||
1259 | /* | ||||
1260 | * uvm_pageunwire: unwire the page. | ||||
1261 | * | ||||
1262 | * => activate if wire count goes to zero. | ||||
1263 | * => caller must lock page queues | ||||
1264 | */ | ||||
1265 | void | ||||
1266 | uvm_pageunwire(struct vm_page *pg) | ||||
1267 | { | ||||
1268 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1268, "uvm_page_owner_locked_p(pg)" )); | ||||
1269 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
1270 | |||||
1271 | pg->wire_count--; | ||||
1272 | if (pg->wire_count == 0) { | ||||
1273 | TAILQ_INSERT_TAIL(&uvm.page_active, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev = (&uvm.page_active)->tqh_last; *(&uvm.page_active )->tqh_last = (pg); (&uvm.page_active)->tqh_last = & (pg)->pageq.tqe_next; } while (0); | ||||
1274 | uvmexp.active++; | ||||
1275 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000); | ||||
1276 | uvmexp.wired--; | ||||
1277 | } | ||||
1278 | } | ||||
1279 | |||||
1280 | /* | ||||
1281 | * uvm_pagedeactivate: deactivate page -- no pmaps have access to page | ||||
1282 | * | ||||
1283 | * => caller must lock page queues | ||||
1284 | * => caller must check to make sure page is not wired | ||||
1285 | * => object that page belongs to must be locked (so we can adjust pg->flags) | ||||
1286 | */ | ||||
1287 | void | ||||
1288 | uvm_pagedeactivate(struct vm_page *pg) | ||||
1289 | { | ||||
1290 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1290, "uvm_page_owner_locked_p(pg)" )); | ||||
1291 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
1292 | |||||
1293 | if (pg->pg_flags & PQ_ACTIVE0x00040000) { | ||||
1294 | TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
1295 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000); | ||||
1296 | uvmexp.active--; | ||||
1297 | } | ||||
1298 | if ((pg->pg_flags & PQ_INACTIVE0x00020000) == 0) { | ||||
1299 | KASSERT(pg->wire_count == 0)((pg->wire_count == 0) ? (void)0 : __assert("diagnostic ", "/usr/src/sys/uvm/uvm_page.c", 1299, "pg->wire_count == 0" )); | ||||
1300 | if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000)) | ||||
1301 | TAILQ_INSERT_TAIL(&uvm.page_inactive_swp, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev = (&uvm.page_inactive_swp)->tqh_last; *(&uvm.page_inactive_swp )->tqh_last = (pg); (&uvm.page_inactive_swp)->tqh_last = &(pg)->pageq.tqe_next; } while (0); | ||||
1302 | else | ||||
1303 | TAILQ_INSERT_TAIL(&uvm.page_inactive_obj, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev = (&uvm.page_inactive_obj)->tqh_last; *(&uvm.page_inactive_obj )->tqh_last = (pg); (&uvm.page_inactive_obj)->tqh_last = &(pg)->pageq.tqe_next; } while (0); | ||||
1304 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000); | ||||
1305 | uvmexp.inactive++; | ||||
1306 | pmap_clear_reference(pg)pmap_clear_attrs(pg, 0x0000000000000020UL); | ||||
1307 | /* | ||||
1308 | * update the "clean" bit. this isn't 100% | ||||
1309 | * accurate, and doesn't have to be. we'll | ||||
1310 | * re-sync it after we zap all mappings when | ||||
1311 | * scanning the inactive list. | ||||
1312 | */ | ||||
1313 | if ((pg->pg_flags & PG_CLEAN0x00000008) != 0 && | ||||
1314 | pmap_is_modified(pg)pmap_test_attrs(pg, 0x0000000000000040UL)) | ||||
1315 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008); | ||||
1316 | } | ||||
1317 | } | ||||
1318 | |||||
1319 | /* | ||||
1320 | * uvm_pageactivate: activate page | ||||
1321 | * | ||||
1322 | * => caller must lock page queues | ||||
1323 | */ | ||||
1324 | void | ||||
1325 | uvm_pageactivate(struct vm_page *pg) | ||||
1326 | { | ||||
1327 | KASSERT(uvm_page_owner_locked_p(pg))((uvm_page_owner_locked_p(pg)) ? (void)0 : __assert("diagnostic " , "/usr/src/sys/uvm/uvm_page.c", 1327, "uvm_page_owner_locked_p(pg)" )); | ||||
1328 | MUTEX_ASSERT_LOCKED(&uvm.pageqlock)do { if (((&uvm.pageqlock)->mtx_owner != ({struct cpu_info *__ci; asm volatile("movq %%gs:%P1,%0" : "=r" (__ci) :"n" (__builtin_offsetof (struct cpu_info, ci_self))); __ci;})) && !(panicstr || db_active)) panic("mutex %p not held in %s", (&uvm.pageqlock ), __func__); } while (0); | ||||
1329 | |||||
1330 | if (pg->pg_flags & PQ_INACTIVE0x00020000) { | ||||
1331 | if (pg->pg_flags & PQ_SWAPBACKED(0x00100000|0x00200000)) | ||||
1332 | TAILQ_REMOVE(&uvm.page_inactive_swp, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_inactive_swp)->tqh_last = (pg)->pageq.tqe_prev ; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
1333 | else | ||||
1334 | TAILQ_REMOVE(&uvm.page_inactive_obj, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_inactive_obj)->tqh_last = (pg)->pageq.tqe_prev ; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
1335 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PQ_INACTIVE0x00020000); | ||||
1336 | uvmexp.inactive--; | ||||
1337 | } | ||||
1338 | if (pg->wire_count == 0) { | ||||
1339 | /* | ||||
1340 | * if page is already active, remove it from list so we | ||||
1341 | * can put it at tail. if it wasn't active, then mark | ||||
1342 | * it active and bump active count | ||||
1343 | */ | ||||
1344 | if (pg->pg_flags & PQ_ACTIVE0x00040000) | ||||
1345 | TAILQ_REMOVE(&uvm.page_active, pg, pageq)do { if (((pg)->pageq.tqe_next) != ((void *)0)) (pg)->pageq .tqe_next->pageq.tqe_prev = (pg)->pageq.tqe_prev; else ( &uvm.page_active)->tqh_last = (pg)->pageq.tqe_prev; *(pg)->pageq.tqe_prev = (pg)->pageq.tqe_next; ((pg)-> pageq.tqe_prev) = ((void *)-1); ((pg)->pageq.tqe_next) = ( (void *)-1); } while (0); | ||||
1346 | else { | ||||
1347 | atomic_setbits_intx86_atomic_setbits_u32(&pg->pg_flags, PQ_ACTIVE0x00040000); | ||||
1348 | uvmexp.active++; | ||||
1349 | } | ||||
1350 | |||||
1351 | TAILQ_INSERT_TAIL(&uvm.page_active, pg, pageq)do { (pg)->pageq.tqe_next = ((void *)0); (pg)->pageq.tqe_prev = (&uvm.page_active)->tqh_last; *(&uvm.page_active )->tqh_last = (pg); (&uvm.page_active)->tqh_last = & (pg)->pageq.tqe_next; } while (0); | ||||
1352 | } | ||||
1353 | } | ||||
1354 | |||||
1355 | /* | ||||
1356 | * uvm_pagezero: zero fill a page | ||||
1357 | */ | ||||
1358 | void | ||||
1359 | uvm_pagezero(struct vm_page *pg) | ||||
1360 | { | ||||
1361 | atomic_clearbits_intx86_atomic_clearbits_u32(&pg->pg_flags, PG_CLEAN0x00000008); | ||||
1362 | pmap_zero_page(pg); | ||||
1363 | } | ||||
1364 | |||||
1365 | /* | ||||
1366 | * uvm_pagecopy: copy a page | ||||
1367 | */ | ||||
1368 | void | ||||
1369 | uvm_pagecopy(struct vm_page *src, struct vm_page *dst) | ||||
1370 | { | ||||
1371 | atomic_clearbits_intx86_atomic_clearbits_u32(&dst->pg_flags, PG_CLEAN0x00000008); | ||||
1372 | pmap_copy_page(src, dst); | ||||
1373 | } | ||||
1374 | |||||
1375 | /* | ||||
1376 | * uvm_page_owner_locked_p: return true if object associated with page is | ||||
1377 | * locked. this is a weak check for runtime assertions only. | ||||
1378 | */ | ||||
1379 | int | ||||
1380 | uvm_page_owner_locked_p(struct vm_page *pg) | ||||
1381 | { | ||||
1382 | if (pg->uobject != NULL((void *)0)) { | ||||
1383 | if (UVM_OBJ_IS_DUMMY(pg->uobject)(((pg->uobject)->pgops == &pmap_pager) || ((pg-> uobject)->pgops == &bufcache_pager))) | ||||
1384 | return 1; | ||||
1385 | return rw_write_held(pg->uobject->vmobjlock); | ||||
1386 | } | ||||
1387 | if (pg->uanon != NULL((void *)0)) { | ||||
1388 | return rw_write_held(pg->uanon->an_lock); | ||||
1389 | } | ||||
1390 | return 1; | ||||
1391 | } | ||||
1392 | |||||
1393 | /* | ||||
1394 | * uvm_pagecount: count the number of physical pages in the address range. | ||||
1395 | */ | ||||
1396 | psize_t | ||||
1397 | uvm_pagecount(struct uvm_constraint_range* constraint) | ||||
1398 | { | ||||
1399 | int lcv; | ||||
1400 | psize_t sz; | ||||
1401 | paddr_t low, high; | ||||
1402 | paddr_t ps_low, ps_high; | ||||
1403 | |||||
1404 | /* Algorithm uses page numbers. */ | ||||
1405 | low = atop(constraint->ucr_low)((constraint->ucr_low) >> 12); | ||||
1406 | high = atop(constraint->ucr_high)((constraint->ucr_high) >> 12); | ||||
1407 | |||||
1408 | sz = 0; | ||||
1409 | for (lcv = 0; lcv < vm_nphysseg; lcv++) { | ||||
1410 | ps_low = MAX(low, vm_physmem[lcv].avail_start)(((low)>(vm_physmem[lcv].avail_start))?(low):(vm_physmem[lcv ].avail_start)); | ||||
1411 | ps_high = MIN(high, vm_physmem[lcv].avail_end)(((high)<(vm_physmem[lcv].avail_end))?(high):(vm_physmem[lcv ].avail_end)); | ||||
1412 | if (ps_low < ps_high) | ||||
1413 | sz += ps_high - ps_low; | ||||
1414 | } | ||||
1415 | return sz; | ||||
1416 | } |
1 | /* $OpenBSD: pmap.h,v 1.78 2021/06/18 06:17:28 guenther Exp $ */ |
2 | /* $NetBSD: pmap.h,v 1.1 2003/04/26 18:39:46 fvdl Exp $ */ |
3 | |
4 | /* |
5 | * Copyright (c) 1997 Charles D. Cranor and Washington University. |
6 | * All rights reserved. |
7 | * |
8 | * Redistribution and use in source and binary forms, with or without |
9 | * modification, are permitted provided that the following conditions |
10 | * are met: |
11 | * 1. Redistributions of source code must retain the above copyright |
12 | * notice, this list of conditions and the following disclaimer. |
13 | * 2. Redistributions in binary form must reproduce the above copyright |
14 | * notice, this list of conditions and the following disclaimer in the |
15 | * documentation and/or other materials provided with the distribution. |
16 | * |
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
27 | */ |
28 | |
29 | /* |
30 | * Copyright (c) 2001 Wasabi Systems, Inc. |
31 | * All rights reserved. |
32 | * |
33 | * Written by Frank van der Linden for Wasabi Systems, Inc. |
34 | * |
35 | * Redistribution and use in source and binary forms, with or without |
36 | * modification, are permitted provided that the following conditions |
37 | * are met: |
38 | * 1. Redistributions of source code must retain the above copyright |
39 | * notice, this list of conditions and the following disclaimer. |
40 | * 2. Redistributions in binary form must reproduce the above copyright |
41 | * notice, this list of conditions and the following disclaimer in the |
42 | * documentation and/or other materials provided with the distribution. |
43 | * 3. All advertising materials mentioning features or use of this software |
44 | * must display the following acknowledgement: |
45 | * This product includes software developed for the NetBSD Project by |
46 | * Wasabi Systems, Inc. |
47 | * 4. The name of Wasabi Systems, Inc. may not be used to endorse |
48 | * or promote products derived from this software without specific prior |
49 | * written permission. |
50 | * |
51 | * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND |
52 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED |
53 | * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
54 | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC |
55 | * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR |
56 | * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
57 | * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS |
58 | * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN |
59 | * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
60 | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
61 | * POSSIBILITY OF SUCH DAMAGE. |
62 | */ |
63 | |
64 | /* |
65 | * pmap.h: see pmap.c for the history of this pmap module. |
66 | */ |
67 | |
68 | #ifndef _MACHINE_PMAP_H_ |
69 | #define _MACHINE_PMAP_H_ |
70 | |
71 | #ifndef _LOCORE |
72 | #ifdef _KERNEL1 |
73 | #include <machine/cpufunc.h> |
74 | #endif /* _KERNEL */ |
75 | #include <sys/mutex.h> |
76 | #include <uvm/uvm_object.h> |
77 | #include <machine/pte.h> |
78 | #endif |
79 | |
80 | /* |
81 | * The x86_64 pmap module closely resembles the i386 one. It uses |
82 | * the same recursive entry scheme. See the i386 pmap.h for a |
83 | * description. The alternate area trick for accessing non-current |
84 | * pmaps has been removed, though, because it performs badly on SMP |
85 | * systems. |
86 | * The most obvious difference to i386 is that 2 extra levels of page |
87 | * table need to be dealt with. The level 1 page table pages are at: |
88 | * |
89 | * l1: 0x00007f8000000000 - 0x00007fffffffffff (39 bits, needs PML4 entry) |
90 | * |
91 | * The other levels are kept as physical pages in 3 UVM objects and are |
92 | * temporarily mapped for virtual access when needed. |
93 | * |
94 | * The other obvious difference from i386 is that it has a direct map of all |
95 | * physical memory in the VA range: |
96 | * |
97 | * 0xfffffd8000000000 - 0xffffff7fffffffff |
98 | * |
99 | * The direct map is used in some cases to access PTEs of non-current pmaps. |
100 | * |
101 | * Note that address space is signed, so the layout for 48 bits is: |
102 | * |
103 | * +---------------------------------+ 0xffffffffffffffff |
104 | * | Kernel Image | |
105 | * +---------------------------------+ 0xffffff8000000000 |
106 | * | Direct Map | |
107 | * +---------------------------------+ 0xfffffd8000000000 |
108 | * ~ ~ |
109 | * | | |
110 | * | Kernel Space | |
111 | * | | |
112 | * | | |
113 | * +---------------------------------+ 0xffff800000000000 = 0x0000800000000000 |
114 | * | L1 table (PTE pages) | |
115 | * +---------------------------------+ 0x00007f8000000000 |
116 | * ~ ~ |
117 | * | | |
118 | * | User Space | |
119 | * | | |
120 | * | | |
121 | * +---------------------------------+ 0x0000000000000000 |
122 | * |
123 | * In other words, there is a 'VA hole' at 0x0000800000000000 - |
124 | * 0xffff800000000000 which will trap, just as on, for example, |
125 | * sparcv9. |
126 | * |
127 | * The unused space can be used if needed, but it adds a little more |
128 | * complexity to the calculations. |
129 | */ |
130 | |
131 | /* |
132 | * Mask to get rid of the sign-extended part of addresses. |
133 | */ |
134 | #define VA_SIGN_MASK0xffff000000000000 0xffff000000000000 |
135 | #define VA_SIGN_NEG(va)((va) | 0xffff000000000000) ((va) | VA_SIGN_MASK0xffff000000000000) |
136 | /* |
137 | * XXXfvdl this one's not right. |
138 | */ |
139 | #define VA_SIGN_POS(va)((va) & ~0xffff000000000000) ((va) & ~VA_SIGN_MASK0xffff000000000000) |
140 | |
141 | #define L4_SLOT_PTE255 255 |
142 | #define L4_SLOT_KERN256 256 |
143 | #define L4_SLOT_KERNBASE511 511 |
144 | #define NUM_L4_SLOT_DIRECT4 4 |
145 | #define L4_SLOT_DIRECT(511 - 4) (L4_SLOT_KERNBASE511 - NUM_L4_SLOT_DIRECT4) |
146 | #define L4_SLOT_EARLY((511 - 4) - 1) (L4_SLOT_DIRECT(511 - 4) - 1) |
147 | |
148 | #define PDIR_SLOT_KERN256 L4_SLOT_KERN256 |
149 | #define PDIR_SLOT_PTE255 L4_SLOT_PTE255 |
150 | #define PDIR_SLOT_DIRECT(511 - 4) L4_SLOT_DIRECT(511 - 4) |
151 | #define PDIR_SLOT_EARLY((511 - 4) - 1) L4_SLOT_EARLY((511 - 4) - 1) |
152 | |
153 | /* |
154 | * the following defines give the virtual addresses of various MMU |
155 | * data structures: |
156 | * PTE_BASE: the base VA of the linear PTE mappings |
157 | * PDP_PDE: the VA of the PDE that points back to the PDP |
158 | * |
159 | */ |
160 | |
161 | #define PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) ((pt_entry_t *) (L4_SLOT_PTE255 * NBPD_L4(1ULL << 39))) |
162 | #define PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) (VA_SIGN_NEG((L4_SLOT_DIRECT * NBPD_L4))((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) |
163 | #define PMAP_DIRECT_END((((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000 )) (VA_SIGN_NEG(((L4_SLOT_DIRECT + \(((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000 ) |
164 | NUM_L4_SLOT_DIRECT) * NBPD_L4))(((((511 - 4) + 4) * (1ULL << 39))) | 0xffff000000000000 )) |
165 | |
166 | #define L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) |
167 | |
168 | #define L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) ((pd_entry_t *)((char *)L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) + L4_SLOT_PTE255 * NBPD_L3(1ULL << 30))) |
169 | #define L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t * ) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) ((pd_entry_t *)((char *)L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + L4_SLOT_PTE255 * NBPD_L2(1ULL << 21))) |
170 | #define L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) ((pd_entry_t *)((char *)L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t * ) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + L4_SLOT_PTE255 * NBPD_L1(1ULL << 12))) |
171 | |
172 | #define PDP_PDE(((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * ( 1ULL << 12))) + 255) (L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) + PDIR_SLOT_PTE255) |
173 | |
174 | #define PDP_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) |
175 | |
176 | #define NKL4_MAX_ENTRIES(unsigned long)1 (unsigned long)1 |
177 | #define NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512) (unsigned long)(NKL4_MAX_ENTRIES(unsigned long)1 * 512) |
178 | #define NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512 ) (unsigned long)(NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512) * 512) |
179 | #define NKL1_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)((unsigned long )1 * 512) * 512) * 512) (unsigned long)(NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512 ) * 512) |
180 | |
181 | #define NKL4_KIMG_ENTRIES1 1 |
182 | #define NKL3_KIMG_ENTRIES1 1 |
183 | #define NKL2_KIMG_ENTRIES64 64 |
184 | |
185 | /* number of pages of direct map entries set up by locore0.S */ |
186 | #define NDML4_ENTRIES1 1 |
187 | #define NDML3_ENTRIES1 1 |
188 | #define NDML2_ENTRIES4 4 /* 4GB */ |
189 | |
190 | /* |
191 | * Since kva space is below the kernel in its entirety, we start off |
192 | * with zero entries on each level. |
193 | */ |
194 | #define NKL4_START_ENTRIES0 0 |
195 | #define NKL3_START_ENTRIES0 0 |
196 | #define NKL2_START_ENTRIES0 0 |
197 | #define NKL1_START_ENTRIES0 0 /* XXX */ |
198 | |
199 | #define NTOPLEVEL_PDES((1 << 12) / (sizeof (pd_entry_t))) (PAGE_SIZE(1 << 12) / (sizeof (pd_entry_t))) |
200 | |
201 | #define NPDPG((1 << 12) / sizeof (pd_entry_t)) (PAGE_SIZE(1 << 12) / sizeof (pd_entry_t)) |
202 | |
203 | /* |
204 | * pl*_pi: index in the ptp page for a pde mapping a VA. |
205 | * (pl*_i below is the index in the virtual array of all pdes per level) |
206 | */ |
207 | #define pl1_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x00000000001ff000UL ) >> 12) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L1_MASK0x00000000001ff000UL) >> L1_SHIFT12) |
208 | #define pl2_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x000000003fe00000UL ) >> 21) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L2_MASK0x000000003fe00000UL) >> L2_SHIFT21) |
209 | #define pl3_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x0000007fc0000000UL ) >> 30) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L3_MASK0x0000007fc0000000UL) >> L3_SHIFT30) |
210 | #define pl4_pi(VA)(((((VA) & ~0xffff000000000000)) & 0x0000ff8000000000UL ) >> 39) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L4_MASK0x0000ff8000000000UL) >> L4_SHIFT39) |
211 | |
212 | /* |
213 | * pl*_i: generate index into pde/pte arrays in virtual space |
214 | */ |
215 | #define pl1_i(VA)(((((VA) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L1_FRAME(((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )|0x00000000001ff000UL)) >> L1_SHIFT12) |
216 | #define pl2_i(VA)(((((VA) & ~0xffff000000000000)) & ((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)) >> 21) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )) >> L2_SHIFT21) |
217 | #define pl3_i(VA)(((((VA) & ~0xffff000000000000)) & (0x0000ff8000000000UL |0x0000007fc0000000UL)) >> 30) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L3_FRAME(0x0000ff8000000000UL|0x0000007fc0000000UL)) >> L3_SHIFT30) |
218 | #define pl4_i(VA)(((((VA) & ~0xffff000000000000)) & 0x0000ff8000000000UL ) >> 39) (((VA_SIGN_POS(VA)((VA) & ~0xffff000000000000)) & L4_FRAME0x0000ff8000000000UL) >> L4_SHIFT39) |
219 | #define pl_i(va, lvl)(((((va) & ~0xffff000000000000)) & ptp_masks[(lvl)-1] ) >> ptp_shifts[(lvl)-1]) \ |
220 | (((VA_SIGN_POS(va)((va) & ~0xffff000000000000)) & ptp_masks[(lvl)-1]) >> ptp_shifts[(lvl)-1]) |
221 | |
222 | #define PTP_MASK_INITIALIZER{ (((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )|0x00000000001ff000UL), ((0x0000ff8000000000UL|0x0000007fc0000000UL )|0x000000003fe00000UL), (0x0000ff8000000000UL|0x0000007fc0000000UL ), 0x0000ff8000000000UL } { L1_FRAME(((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL )|0x00000000001ff000UL), L2_FRAME((0x0000ff8000000000UL|0x0000007fc0000000UL)|0x000000003fe00000UL ), L3_FRAME(0x0000ff8000000000UL|0x0000007fc0000000UL), L4_FRAME0x0000ff8000000000UL } |
223 | #define PTP_SHIFT_INITIALIZER{ 12, 21, 30, 39 } { L1_SHIFT12, L2_SHIFT21, L3_SHIFT30, L4_SHIFT39 } |
224 | #define NKPTP_INITIALIZER{ 0, 0, 0, 0 } { NKL1_START_ENTRIES0, NKL2_START_ENTRIES0, \ |
225 | NKL3_START_ENTRIES0, NKL4_START_ENTRIES0 } |
226 | #define NKPTPMAX_INITIALIZER{ (unsigned long)((unsigned long)((unsigned long)((unsigned long )1 * 512) * 512) * 512), (unsigned long)((unsigned long)((unsigned long)1 * 512) * 512), (unsigned long)((unsigned long)1 * 512 ), (unsigned long)1 } { NKL1_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)((unsigned long )1 * 512) * 512) * 512), NKL2_MAX_ENTRIES(unsigned long)((unsigned long)((unsigned long)1 * 512) * 512 ), \ |
227 | NKL3_MAX_ENTRIES(unsigned long)((unsigned long)1 * 512), NKL4_MAX_ENTRIES(unsigned long)1 } |
228 | #define NBPD_INITIALIZER{ (1ULL << 12), (1ULL << 21), (1ULL << 30), (1ULL << 39) } { NBPD_L1(1ULL << 12), NBPD_L2(1ULL << 21), NBPD_L3(1ULL << 30), NBPD_L4(1ULL << 39) } |
229 | #define PDES_INITIALIZER{ ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))), ((pd_entry_t *)((char *) ((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21 ))), ((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * ( 1ULL << 12))) } { L2_BASE((pd_entry_t *)((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))), L3_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pt_entry_t * ) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))), L4_BASE((pd_entry_t *)((char *)((pd_entry_t *)((char *)((pd_entry_t * )((char *)((pt_entry_t *) (255 * (1ULL << 39))) + 255 * (1ULL << 30))) + 255 * (1ULL << 21))) + 255 * (1ULL << 12))) } |
230 | |
231 | /* |
232 | * PTP macros: |
233 | * a PTP's index is the PD index of the PDE that points to it |
234 | * a PTP's offset is the byte-offset in the PTE space that this PTP is at |
235 | * a PTP's VA is the first VA mapped by that PTP |
236 | */ |
237 | |
238 | #define ptp_va2o(va, lvl)((((((va) & ~0xffff000000000000)) & ptp_masks[((lvl)+ 1)-1]) >> ptp_shifts[((lvl)+1)-1]) * (1 << 12)) (pl_i(va, (lvl)+1)(((((va) & ~0xffff000000000000)) & ptp_masks[((lvl)+1 )-1]) >> ptp_shifts[((lvl)+1)-1]) * PAGE_SIZE(1 << 12)) |
239 | |
240 | #define PTP_LEVELS4 4 |
241 | |
242 | /* |
243 | * PG_AVAIL usage: we make use of the ignored bits of the PTE |
244 | */ |
245 | |
246 | #define PG_W0x0000000000000200UL PG_AVAIL10x0000000000000200UL /* "wired" mapping */ |
247 | #define PG_PVLIST0x0000000000000400UL PG_AVAIL20x0000000000000400UL /* mapping has entry on pvlist */ |
248 | /* PG_AVAIL3 not used */ |
249 | |
250 | /* |
251 | * PCID assignments. |
252 | * The shootdown code assumes KERN, PROC, and PROC_INTEL are both |
253 | * consecutive and in that order. |
254 | */ |
255 | #define PCID_KERN0 0 /* for pmap_kernel() */ |
256 | #define PCID_PROC1 1 /* non-pmap_kernel(), U+K */ |
257 | #define PCID_PROC_INTEL2 2 /* non-pmap_kernel(), U-K (meltdown) */ |
258 | #define PCID_TEMP3 3 /* temp mapping of another non-pmap_kernel() */ |
259 | |
260 | extern int pmap_use_pcid; /* non-zero if PCID support is enabled */ |
261 | |
262 | /* |
263 | * Number of PTEs per cache line. 8 byte pte, 64-byte cache line |
264 | * Used to avoid false sharing of cache lines. |
265 | */ |
266 | #define NPTECL8 8 |
267 | |
268 | |
269 | #if defined(_KERNEL1) && !defined(_LOCORE) |
270 | /* |
271 | * pmap data structures: see pmap.c for details of locking. |
272 | */ |
273 | |
274 | struct pmap; |
275 | typedef struct pmap *pmap_t; |
276 | |
277 | /* |
278 | * we maintain a list of all non-kernel pmaps |
279 | */ |
280 | |
281 | LIST_HEAD(pmap_head, pmap)struct pmap_head { struct pmap *lh_first; }; /* struct pmap_head: head of a pmap list */ |
282 | |
283 | /* |
284 | * the pmap structure |
285 | * |
286 | * note that the pm_obj contains the reference count, |
287 | * page list, and number of PTPs within the pmap. |
288 | */ |
289 | |
290 | #define PMAP_TYPE_NORMAL1 1 |
291 | #define PMAP_TYPE_EPT2 2 |
292 | #define PMAP_TYPE_RVI3 3 |
293 | #define pmap_nested(pm)((pm)->pm_type != 1) ((pm)->pm_type != PMAP_TYPE_NORMAL1) |
294 | |
295 | struct pmap { |
296 | struct mutex pm_mtx; |
297 | struct uvm_object pm_obj[PTP_LEVELS4-1]; /* objects for lvl >= 1) */ |
298 | LIST_ENTRY(pmap)struct { struct pmap *le_next; struct pmap **le_prev; } pm_list; /* list (lck by pm_list lock) */ |
299 | /* |
300 | * pm_pdir : VA of page table to be used when executing in |
301 | * privileged mode |
302 | * pm_pdirpa : PA of page table to be used when executing in |
303 | * privileged mode |
304 | * pm_pdir_intel : VA of special page table to be used when executing |
305 | * on an Intel CPU in usermode (no kernel mappings) |
306 | * pm_pdirpa_intel : PA of special page table to be used when executing |
307 | * on an Intel CPU in usermode (no kernel mappings) |
308 | */ |
309 | pd_entry_t *pm_pdir, *pm_pdir_intel; |
310 | paddr_t pm_pdirpa, pm_pdirpa_intel; |
311 | |
312 | struct vm_page *pm_ptphint[PTP_LEVELS4-1]; |
313 | /* pointer to a PTP in our pmap */ |
314 | struct pmap_statistics pm_stats; /* pmap stats (lck by object lock) */ |
315 | |
316 | int pm_type; /* Type of pmap this is (PMAP_TYPE_x) */ |
317 | uint64_t eptp; /* cached EPTP (used by vmm) */ |
318 | }; |
319 | |
320 | /* |
321 | * MD flags that we use for pmap_enter (in the pa): |
322 | */ |
323 | #define PMAP_PA_MASK~((paddr_t)((1 << 12) - 1)) ~((paddr_t)PAGE_MASK((1 << 12) - 1)) /* to remove the flags */ |
324 | #define PMAP_NOCACHE0x1 0x1 /* set the non-cacheable bit. */ |
325 | #define PMAP_WC0x2 0x2 /* set page write combining. */ |
326 | |
327 | /* |
328 | * We keep mod/ref flags in struct vm_page->pg_flags. |
329 | */ |
330 | #define PG_PMAP_MOD0x01000000 PG_PMAP00x01000000 |
331 | #define PG_PMAP_REF0x02000000 PG_PMAP10x02000000 |
332 | #define PG_PMAP_WC0x04000000 PG_PMAP20x04000000 |
333 | |
334 | /* |
335 | * for each managed physical page we maintain a list of <PMAP,VA>'s |
336 | * which it is mapped at. |
337 | */ |
338 | struct pv_entry { /* locked by its list's pvh_lock */ |
339 | struct pv_entry *pv_next; /* next entry */ |
340 | struct pmap *pv_pmap; /* the pmap */ |
341 | vaddr_t pv_va; /* the virtual address */ |
342 | struct vm_page *pv_ptp; /* the vm_page of the PTP */ |
343 | }; |
344 | |
345 | /* |
346 | * global kernel variables |
347 | */ |
348 | |
349 | extern struct pmap kernel_pmap_store; /* kernel pmap */ |
350 | |
351 | extern long nkptp[]; |
352 | |
353 | extern const paddr_t ptp_masks[]; |
354 | extern const int ptp_shifts[]; |
355 | extern const long nbpd[], nkptpmax[]; |
356 | |
357 | /* |
358 | * macros |
359 | */ |
360 | |
361 | #define pmap_kernel()(&kernel_pmap_store) (&kernel_pmap_store) |
362 | #define pmap_resident_count(pmap)((pmap)->pm_stats.resident_count) ((pmap)->pm_stats.resident_count) |
363 | #define pmap_wired_count(pmap)((pmap)->pm_stats.wired_count) ((pmap)->pm_stats.wired_count) |
364 | #define pmap_update(pmap) /* nothing (yet) */ |
365 | |
366 | #define pmap_clear_modify(pg)pmap_clear_attrs(pg, 0x0000000000000040UL) pmap_clear_attrs(pg, PG_M0x0000000000000040UL) |
367 | #define pmap_clear_reference(pg)pmap_clear_attrs(pg, 0x0000000000000020UL) pmap_clear_attrs(pg, PG_U0x0000000000000020UL) |
368 | #define pmap_copy(DP,SP,D,L,S) |
369 | #define pmap_is_modified(pg)pmap_test_attrs(pg, 0x0000000000000040UL) pmap_test_attrs(pg, PG_M0x0000000000000040UL) |
370 | #define pmap_is_referenced(pg)pmap_test_attrs(pg, 0x0000000000000020UL) pmap_test_attrs(pg, PG_U0x0000000000000020UL) |
371 | #define pmap_move(DP,SP,D,L,S) |
372 | #define pmap_valid_entry(E)((E) & 0x0000000000000001UL) ((E) & PG_V0x0000000000000001UL) /* is PDE or PTE valid? */ |
373 | |
374 | #define pmap_proc_iflush(p,va,len) /* nothing */ |
375 | #define pmap_unuse_final(p) /* nothing */ |
376 | #define pmap_remove_holes(vm)do { } while (0) do { /* nothing */ } while (0) |
377 | |
378 | |
379 | /* |
380 | * prototypes |
381 | */ |
382 | |
383 | void map_tramps(void); /* machdep.c */ |
384 | paddr_t pmap_bootstrap(paddr_t, paddr_t); |
385 | void pmap_randomize(void); |
386 | void pmap_randomize_level(pd_entry_t *, int); |
387 | int pmap_clear_attrs(struct vm_page *, unsigned long); |
388 | static void pmap_page_protect(struct vm_page *, vm_prot_t); |
389 | void pmap_page_remove (struct vm_page *); |
390 | static void pmap_protect(struct pmap *, vaddr_t, |
391 | vaddr_t, vm_prot_t); |
392 | void pmap_remove(struct pmap *, vaddr_t, vaddr_t); |
393 | int pmap_test_attrs(struct vm_page *, unsigned); |
394 | static void pmap_update_pg(vaddr_t); |
395 | void pmap_write_protect(struct pmap *, vaddr_t, |
396 | vaddr_t, vm_prot_t); |
397 | void pmap_fix_ept(struct pmap *, vaddr_t); |
398 | |
399 | paddr_t pmap_prealloc_lowmem_ptps(paddr_t); |
400 | |
401 | void pagezero(vaddr_t); |
402 | |
403 | int pmap_convert(struct pmap *, int); |
404 | void pmap_enter_special(vaddr_t, paddr_t, vm_prot_t); |
405 | vaddr_t pmap_set_pml4_early(paddr_t pa); |
406 | void pmap_clear_pml4_early(void); |
407 | |
408 | /* |
409 | * functions for flushing the cache for vaddrs and pages. |
410 | * these functions are not part of the MI pmap interface and thus |
411 | * should not be used as such. |
412 | */ |
413 | void pmap_flush_cache(vaddr_t, vsize_t); |
414 | #define pmap_flush_page(paddr)do { ((void)0); pmap_flush_cache(((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) + (paddr)), (1 << 12)); } while ( 0) do { \ |
415 | KDASSERT(PHYS_TO_VM_PAGE(paddr) != NULL)((void)0); \ |
416 | pmap_flush_cache(PMAP_DIRECT_MAP(paddr)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (paddr)), PAGE_SIZE(1 << 12)); \ |
417 | } while (/* CONSTCOND */ 0) |
418 | |
419 | #define PMAP_STEAL_MEMORY /* enable pmap_steal_memory() */ |
420 | #define PMAP_GROWKERNEL /* turn on pmap_growkernel interface */ |
421 | |
422 | /* |
423 | * inline functions |
424 | */ |
425 | |
426 | static inline void |
427 | pmap_remove_all(struct pmap *pmap) |
428 | { |
429 | /* Nothing. */ |
430 | } |
431 | |
432 | /* |
433 | * pmap_update_pg: flush one page from the TLB (or flush the whole thing |
434 | * if hardware doesn't support one-page flushing) |
435 | */ |
436 | |
437 | inline static void |
438 | pmap_update_pg(vaddr_t va) |
439 | { |
440 | invlpg(va); |
441 | } |
442 | |
443 | /* |
444 | * pmap_page_protect: change the protection of all recorded mappings |
445 | * of a managed page |
446 | * |
447 | * => this function is a frontend for pmap_page_remove/pmap_clear_attrs |
448 | * => we only have to worry about making the page more protected. |
449 | * unprotecting a page is done on-demand at fault time. |
450 | */ |
451 | |
452 | inline static void |
453 | pmap_page_protect(struct vm_page *pg, vm_prot_t prot) |
454 | { |
455 | if ((prot & PROT_WRITE0x02) == 0) { |
456 | if (prot & (PROT_READ0x01 | PROT_EXEC0x04)) { |
457 | (void) pmap_clear_attrs(pg, PG_RW0x0000000000000002UL); |
458 | } else { |
459 | pmap_page_remove(pg); |
460 | } |
461 | } |
462 | } |
463 | |
464 | /* |
465 | * pmap_protect: change the protection of pages in a pmap |
466 | * |
467 | * => this function is a frontend for pmap_remove/pmap_write_protect |
468 | * => we only have to worry about making the page more protected. |
469 | * unprotecting a page is done on-demand at fault time. |
470 | */ |
471 | |
472 | inline static void |
473 | pmap_protect(struct pmap *pmap, vaddr_t sva, vaddr_t eva, vm_prot_t prot) |
474 | { |
475 | if ((prot & PROT_WRITE0x02) == 0) { |
476 | if (prot & (PROT_READ0x01| PROT_EXEC0x04)) { |
477 | pmap_write_protect(pmap, sva, eva, prot); |
478 | } else { |
479 | pmap_remove(pmap, sva, eva); |
480 | } |
481 | } |
482 | } |
483 | |
484 | /* |
485 | * various address inlines |
486 | * |
487 | * vtopte: return a pointer to the PTE mapping a VA, works only for |
488 | * user and PT addresses |
489 | * |
490 | * kvtopte: return a pointer to the PTE mapping a kernel VA |
491 | */ |
492 | |
493 | static inline pt_entry_t * |
494 | vtopte(vaddr_t va) |
495 | { |
496 | return (PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)); |
497 | } |
498 | |
499 | static inline pt_entry_t * |
500 | kvtopte(vaddr_t va) |
501 | { |
502 | #ifdef LARGEPAGES |
503 | { |
504 | pd_entry_t *pde; |
505 | |
506 | pde = L1_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl2_i(va)(((((va) & ~0xffff000000000000)) & ((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)) >> 21); |
507 | if (*pde & PG_PS0x0000000000000080UL) |
508 | return ((pt_entry_t *)pde); |
509 | } |
510 | #endif |
511 | |
512 | return (PTE_BASE((pt_entry_t *) (255 * (1ULL << 39))) + pl1_i(va)(((((va) & ~0xffff000000000000)) & (((0x0000ff8000000000UL |0x0000007fc0000000UL)|0x000000003fe00000UL)|0x00000000001ff000UL )) >> 12)); |
513 | } |
514 | |
515 | #define PMAP_DIRECT_MAP(pa)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (pa)) ((vaddr_t)PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)) + (pa)) |
516 | #define PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 ))) ((paddr_t)(va) - PMAP_DIRECT_BASE(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000))) |
517 | #define pmap_map_direct(pg)((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (((pg)->phys_addr))) PMAP_DIRECT_MAP(VM_PAGE_TO_PHYS(pg))((vaddr_t)(((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )) + (((pg)->phys_addr))) |
518 | #define pmap_unmap_direct(va)PHYS_TO_VM_PAGE(((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000)))) PHYS_TO_VM_PAGE(PMAP_DIRECT_UNMAP(va)((paddr_t)(va) - (((((511 - 4) * (1ULL << 39))) | 0xffff000000000000 )))) |
519 | |
520 | #define __HAVE_PMAP_DIRECT |
521 | |
522 | #endif /* _KERNEL && !_LOCORE */ |
523 | |
524 | #ifndef _LOCORE |
525 | struct pv_entry; |
526 | struct vm_page_md { |
527 | struct mutex pv_mtx; |
528 | struct pv_entry *pv_list; |
529 | }; |
530 | |
531 | #define VM_MDPAGE_INIT(pg)do { do { (void)(((void *)0)); (void)(0); __mtx_init((&(pg )->mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9) ? 0x9 : ((0xa)))); } while (0); (pg)->mdpage.pv_list = ((void *)0); } while (0) do { \ |
532 | mtx_init(&(pg)->mdpage.pv_mtx, IPL_VM)do { (void)(((void *)0)); (void)(0); __mtx_init((&(pg)-> mdpage.pv_mtx), ((((0xa)) > 0x0 && ((0xa)) < 0x9 ) ? 0x9 : ((0xa)))); } while (0); \ |
533 | (pg)->mdpage.pv_list = NULL((void *)0); \ |
534 | } while (0) |
535 | #endif /* !_LOCORE */ |
536 | |
537 | #endif /* _MACHINE_PMAP_H_ */ |